SelectionDAG.cpp revision 9cf37e8b48732fccd4c301ed51aafed7074bd84e
1//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This implements the SelectionDAG class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "SDNodeOrdering.h" 16#include "SDNodeDbgValue.h" 17#include "llvm/Constants.h" 18#include "llvm/Analysis/DebugInfo.h" 19#include "llvm/Analysis/ValueTracking.h" 20#include "llvm/Function.h" 21#include "llvm/GlobalAlias.h" 22#include "llvm/GlobalVariable.h" 23#include "llvm/Intrinsics.h" 24#include "llvm/DerivedTypes.h" 25#include "llvm/Assembly/Writer.h" 26#include "llvm/CallingConv.h" 27#include "llvm/CodeGen/MachineBasicBlock.h" 28#include "llvm/CodeGen/MachineConstantPool.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineModuleInfo.h" 31#include "llvm/Target/TargetRegisterInfo.h" 32#include "llvm/Target/TargetData.h" 33#include "llvm/Target/TargetLowering.h" 34#include "llvm/Target/TargetSelectionDAGInfo.h" 35#include "llvm/Target/TargetOptions.h" 36#include "llvm/Target/TargetInstrInfo.h" 37#include "llvm/Target/TargetIntrinsicInfo.h" 38#include "llvm/Target/TargetMachine.h" 39#include "llvm/Support/CommandLine.h" 40#include "llvm/Support/Debug.h" 41#include "llvm/Support/ErrorHandling.h" 42#include "llvm/Support/ManagedStatic.h" 43#include "llvm/Support/MathExtras.h" 44#include "llvm/Support/raw_ostream.h" 45#include "llvm/Support/Mutex.h" 46#include "llvm/ADT/SetVector.h" 47#include "llvm/ADT/SmallPtrSet.h" 48#include "llvm/ADT/SmallSet.h" 49#include "llvm/ADT/SmallVector.h" 50#include "llvm/ADT/StringExtras.h" 51#include <algorithm> 52#include <cmath> 53using namespace llvm; 54 55/// makeVTList - Return an instance of the SDVTList struct initialized with the 56/// specified members. 57static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 58 SDVTList Res = {VTs, NumVTs}; 59 return Res; 60} 61 62static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { 63 switch (VT.getSimpleVT().SimpleTy) { 64 default: llvm_unreachable("Unknown FP format"); 65 case MVT::f32: return &APFloat::IEEEsingle; 66 case MVT::f64: return &APFloat::IEEEdouble; 67 case MVT::f80: return &APFloat::x87DoubleExtended; 68 case MVT::f128: return &APFloat::IEEEquad; 69 case MVT::ppcf128: return &APFloat::PPCDoubleDouble; 70 } 71} 72 73SelectionDAG::DAGUpdateListener::~DAGUpdateListener() {} 74 75//===----------------------------------------------------------------------===// 76// ConstantFPSDNode Class 77//===----------------------------------------------------------------------===// 78 79/// isExactlyValue - We don't rely on operator== working on double values, as 80/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 81/// As such, this method can be used to do an exact bit-for-bit comparison of 82/// two floating point values. 83bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 84 return getValueAPF().bitwiseIsEqual(V); 85} 86 87bool ConstantFPSDNode::isValueValidForType(EVT VT, 88 const APFloat& Val) { 89 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 90 91 // PPC long double cannot be converted to any other type. 92 if (VT == MVT::ppcf128 || 93 &Val.getSemantics() == &APFloat::PPCDoubleDouble) 94 return false; 95 96 // convert modifies in place, so make a copy. 97 APFloat Val2 = APFloat(Val); 98 bool losesInfo; 99 (void) Val2.convert(*EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, 100 &losesInfo); 101 return !losesInfo; 102} 103 104//===----------------------------------------------------------------------===// 105// ISD Namespace 106//===----------------------------------------------------------------------===// 107 108/// isBuildVectorAllOnes - Return true if the specified node is a 109/// BUILD_VECTOR where all of the elements are ~0 or undef. 110bool ISD::isBuildVectorAllOnes(const SDNode *N) { 111 // Look through a bit convert. 112 if (N->getOpcode() == ISD::BITCAST) 113 N = N->getOperand(0).getNode(); 114 115 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 116 117 unsigned i = 0, e = N->getNumOperands(); 118 119 // Skip over all of the undef values. 120 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 121 ++i; 122 123 // Do not accept an all-undef vector. 124 if (i == e) return false; 125 126 // Do not accept build_vectors that aren't all constants or which have non-~0 127 // elements. 128 SDValue NotZero = N->getOperand(i); 129 if (isa<ConstantSDNode>(NotZero)) { 130 if (!cast<ConstantSDNode>(NotZero)->isAllOnesValue()) 131 return false; 132 } else if (isa<ConstantFPSDNode>(NotZero)) { 133 if (!cast<ConstantFPSDNode>(NotZero)->getValueAPF(). 134 bitcastToAPInt().isAllOnesValue()) 135 return false; 136 } else 137 return false; 138 139 // Okay, we have at least one ~0 value, check to see if the rest match or are 140 // undefs. 141 for (++i; i != e; ++i) 142 if (N->getOperand(i) != NotZero && 143 N->getOperand(i).getOpcode() != ISD::UNDEF) 144 return false; 145 return true; 146} 147 148 149/// isBuildVectorAllZeros - Return true if the specified node is a 150/// BUILD_VECTOR where all of the elements are 0 or undef. 151bool ISD::isBuildVectorAllZeros(const SDNode *N) { 152 // Look through a bit convert. 153 if (N->getOpcode() == ISD::BITCAST) 154 N = N->getOperand(0).getNode(); 155 156 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 157 158 unsigned i = 0, e = N->getNumOperands(); 159 160 // Skip over all of the undef values. 161 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 162 ++i; 163 164 // Do not accept an all-undef vector. 165 if (i == e) return false; 166 167 // Do not accept build_vectors that aren't all constants or which have non-0 168 // elements. 169 SDValue Zero = N->getOperand(i); 170 if (isa<ConstantSDNode>(Zero)) { 171 if (!cast<ConstantSDNode>(Zero)->isNullValue()) 172 return false; 173 } else if (isa<ConstantFPSDNode>(Zero)) { 174 if (!cast<ConstantFPSDNode>(Zero)->getValueAPF().isPosZero()) 175 return false; 176 } else 177 return false; 178 179 // Okay, we have at least one 0 value, check to see if the rest match or are 180 // undefs. 181 for (++i; i != e; ++i) 182 if (N->getOperand(i) != Zero && 183 N->getOperand(i).getOpcode() != ISD::UNDEF) 184 return false; 185 return true; 186} 187 188/// isScalarToVector - Return true if the specified node is a 189/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 190/// element is not an undef. 191bool ISD::isScalarToVector(const SDNode *N) { 192 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 193 return true; 194 195 if (N->getOpcode() != ISD::BUILD_VECTOR) 196 return false; 197 if (N->getOperand(0).getOpcode() == ISD::UNDEF) 198 return false; 199 unsigned NumElems = N->getNumOperands(); 200 if (NumElems == 1) 201 return false; 202 for (unsigned i = 1; i < NumElems; ++i) { 203 SDValue V = N->getOperand(i); 204 if (V.getOpcode() != ISD::UNDEF) 205 return false; 206 } 207 return true; 208} 209 210/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 211/// when given the operation for (X op Y). 212ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 213 // To perform this operation, we just need to swap the L and G bits of the 214 // operation. 215 unsigned OldL = (Operation >> 2) & 1; 216 unsigned OldG = (Operation >> 1) & 1; 217 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 218 (OldL << 1) | // New G bit 219 (OldG << 2)); // New L bit. 220} 221 222/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 223/// 'op' is a valid SetCC operation. 224ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 225 unsigned Operation = Op; 226 if (isInteger) 227 Operation ^= 7; // Flip L, G, E bits, but not U. 228 else 229 Operation ^= 15; // Flip all of the condition bits. 230 231 if (Operation > ISD::SETTRUE2) 232 Operation &= ~8; // Don't let N and U bits get set. 233 234 return ISD::CondCode(Operation); 235} 236 237 238/// isSignedOp - For an integer comparison, return 1 if the comparison is a 239/// signed operation and 2 if the result is an unsigned comparison. Return zero 240/// if the operation does not depend on the sign of the input (setne and seteq). 241static int isSignedOp(ISD::CondCode Opcode) { 242 switch (Opcode) { 243 default: llvm_unreachable("Illegal integer setcc operation!"); 244 case ISD::SETEQ: 245 case ISD::SETNE: return 0; 246 case ISD::SETLT: 247 case ISD::SETLE: 248 case ISD::SETGT: 249 case ISD::SETGE: return 1; 250 case ISD::SETULT: 251 case ISD::SETULE: 252 case ISD::SETUGT: 253 case ISD::SETUGE: return 2; 254 } 255} 256 257/// getSetCCOrOperation - Return the result of a logical OR between different 258/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 259/// returns SETCC_INVALID if it is not possible to represent the resultant 260/// comparison. 261ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 262 bool isInteger) { 263 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 264 // Cannot fold a signed integer setcc with an unsigned integer setcc. 265 return ISD::SETCC_INVALID; 266 267 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 268 269 // If the N and U bits get set then the resultant comparison DOES suddenly 270 // care about orderedness, and is true when ordered. 271 if (Op > ISD::SETTRUE2) 272 Op &= ~16; // Clear the U bit if the N bit is set. 273 274 // Canonicalize illegal integer setcc's. 275 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 276 Op = ISD::SETNE; 277 278 return ISD::CondCode(Op); 279} 280 281/// getSetCCAndOperation - Return the result of a logical AND between different 282/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 283/// function returns zero if it is not possible to represent the resultant 284/// comparison. 285ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 286 bool isInteger) { 287 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 288 // Cannot fold a signed setcc with an unsigned setcc. 289 return ISD::SETCC_INVALID; 290 291 // Combine all of the condition bits. 292 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 293 294 // Canonicalize illegal integer setcc's. 295 if (isInteger) { 296 switch (Result) { 297 default: break; 298 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 299 case ISD::SETOEQ: // SETEQ & SETU[LG]E 300 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 301 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 302 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 303 } 304 } 305 306 return Result; 307} 308 309//===----------------------------------------------------------------------===// 310// SDNode Profile Support 311//===----------------------------------------------------------------------===// 312 313/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 314/// 315static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 316 ID.AddInteger(OpC); 317} 318 319/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 320/// solely with their pointer. 321static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 322 ID.AddPointer(VTList.VTs); 323} 324 325/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 326/// 327static void AddNodeIDOperands(FoldingSetNodeID &ID, 328 const SDValue *Ops, unsigned NumOps) { 329 for (; NumOps; --NumOps, ++Ops) { 330 ID.AddPointer(Ops->getNode()); 331 ID.AddInteger(Ops->getResNo()); 332 } 333} 334 335/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 336/// 337static void AddNodeIDOperands(FoldingSetNodeID &ID, 338 const SDUse *Ops, unsigned NumOps) { 339 for (; NumOps; --NumOps, ++Ops) { 340 ID.AddPointer(Ops->getNode()); 341 ID.AddInteger(Ops->getResNo()); 342 } 343} 344 345static void AddNodeIDNode(FoldingSetNodeID &ID, 346 unsigned short OpC, SDVTList VTList, 347 const SDValue *OpList, unsigned N) { 348 AddNodeIDOpcode(ID, OpC); 349 AddNodeIDValueTypes(ID, VTList); 350 AddNodeIDOperands(ID, OpList, N); 351} 352 353/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 354/// the NodeID data. 355static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 356 switch (N->getOpcode()) { 357 case ISD::TargetExternalSymbol: 358 case ISD::ExternalSymbol: 359 llvm_unreachable("Should only be used on nodes with operands"); 360 default: break; // Normal nodes don't need extra info. 361 case ISD::TargetConstant: 362 case ISD::Constant: 363 ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 364 break; 365 case ISD::TargetConstantFP: 366 case ISD::ConstantFP: { 367 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 368 break; 369 } 370 case ISD::TargetGlobalAddress: 371 case ISD::GlobalAddress: 372 case ISD::TargetGlobalTLSAddress: 373 case ISD::GlobalTLSAddress: { 374 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 375 ID.AddPointer(GA->getGlobal()); 376 ID.AddInteger(GA->getOffset()); 377 ID.AddInteger(GA->getTargetFlags()); 378 break; 379 } 380 case ISD::BasicBlock: 381 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 382 break; 383 case ISD::Register: 384 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 385 break; 386 case ISD::RegisterMask: 387 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 388 break; 389 case ISD::SRCVALUE: 390 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 391 break; 392 case ISD::FrameIndex: 393 case ISD::TargetFrameIndex: 394 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 395 break; 396 case ISD::JumpTable: 397 case ISD::TargetJumpTable: 398 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 399 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 400 break; 401 case ISD::ConstantPool: 402 case ISD::TargetConstantPool: { 403 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 404 ID.AddInteger(CP->getAlignment()); 405 ID.AddInteger(CP->getOffset()); 406 if (CP->isMachineConstantPoolEntry()) 407 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 408 else 409 ID.AddPointer(CP->getConstVal()); 410 ID.AddInteger(CP->getTargetFlags()); 411 break; 412 } 413 case ISD::LOAD: { 414 const LoadSDNode *LD = cast<LoadSDNode>(N); 415 ID.AddInteger(LD->getMemoryVT().getRawBits()); 416 ID.AddInteger(LD->getRawSubclassData()); 417 break; 418 } 419 case ISD::STORE: { 420 const StoreSDNode *ST = cast<StoreSDNode>(N); 421 ID.AddInteger(ST->getMemoryVT().getRawBits()); 422 ID.AddInteger(ST->getRawSubclassData()); 423 break; 424 } 425 case ISD::ATOMIC_CMP_SWAP: 426 case ISD::ATOMIC_SWAP: 427 case ISD::ATOMIC_LOAD_ADD: 428 case ISD::ATOMIC_LOAD_SUB: 429 case ISD::ATOMIC_LOAD_AND: 430 case ISD::ATOMIC_LOAD_OR: 431 case ISD::ATOMIC_LOAD_XOR: 432 case ISD::ATOMIC_LOAD_NAND: 433 case ISD::ATOMIC_LOAD_MIN: 434 case ISD::ATOMIC_LOAD_MAX: 435 case ISD::ATOMIC_LOAD_UMIN: 436 case ISD::ATOMIC_LOAD_UMAX: 437 case ISD::ATOMIC_LOAD: 438 case ISD::ATOMIC_STORE: { 439 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 440 ID.AddInteger(AT->getMemoryVT().getRawBits()); 441 ID.AddInteger(AT->getRawSubclassData()); 442 break; 443 } 444 case ISD::VECTOR_SHUFFLE: { 445 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 446 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 447 i != e; ++i) 448 ID.AddInteger(SVN->getMaskElt(i)); 449 break; 450 } 451 case ISD::TargetBlockAddress: 452 case ISD::BlockAddress: { 453 ID.AddPointer(cast<BlockAddressSDNode>(N)->getBlockAddress()); 454 ID.AddInteger(cast<BlockAddressSDNode>(N)->getTargetFlags()); 455 break; 456 } 457 } // end switch (N->getOpcode()) 458} 459 460/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 461/// data. 462static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 463 AddNodeIDOpcode(ID, N->getOpcode()); 464 // Add the return value info. 465 AddNodeIDValueTypes(ID, N->getVTList()); 466 // Add the operand info. 467 AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 468 469 // Handle SDNode leafs with special info. 470 AddNodeIDCustom(ID, N); 471} 472 473/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 474/// the CSE map that carries volatility, temporalness, indexing mode, and 475/// extension/truncation information. 476/// 477static inline unsigned 478encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 479 bool isNonTemporal, bool isInvariant) { 480 assert((ConvType & 3) == ConvType && 481 "ConvType may not require more than 2 bits!"); 482 assert((AM & 7) == AM && 483 "AM may not require more than 3 bits!"); 484 return ConvType | 485 (AM << 2) | 486 (isVolatile << 5) | 487 (isNonTemporal << 6) | 488 (isInvariant << 7); 489} 490 491//===----------------------------------------------------------------------===// 492// SelectionDAG Class 493//===----------------------------------------------------------------------===// 494 495/// doNotCSE - Return true if CSE should not be performed for this node. 496static bool doNotCSE(SDNode *N) { 497 if (N->getValueType(0) == MVT::Glue) 498 return true; // Never CSE anything that produces a flag. 499 500 switch (N->getOpcode()) { 501 default: break; 502 case ISD::HANDLENODE: 503 case ISD::EH_LABEL: 504 return true; // Never CSE these nodes. 505 } 506 507 // Check that remaining values produced are not flags. 508 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 509 if (N->getValueType(i) == MVT::Glue) 510 return true; // Never CSE anything that produces a flag. 511 512 return false; 513} 514 515/// RemoveDeadNodes - This method deletes all unreachable nodes in the 516/// SelectionDAG. 517void SelectionDAG::RemoveDeadNodes() { 518 // Create a dummy node (which is not added to allnodes), that adds a reference 519 // to the root node, preventing it from being deleted. 520 HandleSDNode Dummy(getRoot()); 521 522 SmallVector<SDNode*, 128> DeadNodes; 523 524 // Add all obviously-dead nodes to the DeadNodes worklist. 525 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 526 if (I->use_empty()) 527 DeadNodes.push_back(I); 528 529 RemoveDeadNodes(DeadNodes); 530 531 // If the root changed (e.g. it was a dead load, update the root). 532 setRoot(Dummy.getValue()); 533} 534 535/// RemoveDeadNodes - This method deletes the unreachable nodes in the 536/// given list, and any nodes that become unreachable as a result. 537void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes, 538 DAGUpdateListener *UpdateListener) { 539 540 // Process the worklist, deleting the nodes and adding their uses to the 541 // worklist. 542 while (!DeadNodes.empty()) { 543 SDNode *N = DeadNodes.pop_back_val(); 544 545 if (UpdateListener) 546 UpdateListener->NodeDeleted(N, 0); 547 548 // Take the node out of the appropriate CSE map. 549 RemoveNodeFromCSEMaps(N); 550 551 // Next, brutally remove the operand list. This is safe to do, as there are 552 // no cycles in the graph. 553 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 554 SDUse &Use = *I++; 555 SDNode *Operand = Use.getNode(); 556 Use.set(SDValue()); 557 558 // Now that we removed this operand, see if there are no uses of it left. 559 if (Operand->use_empty()) 560 DeadNodes.push_back(Operand); 561 } 562 563 DeallocateNode(N); 564 } 565} 566 567void SelectionDAG::RemoveDeadNode(SDNode *N, DAGUpdateListener *UpdateListener){ 568 SmallVector<SDNode*, 16> DeadNodes(1, N); 569 570 // Create a dummy node that adds a reference to the root node, preventing 571 // it from being deleted. (This matters if the root is an operand of the 572 // dead node.) 573 HandleSDNode Dummy(getRoot()); 574 575 RemoveDeadNodes(DeadNodes, UpdateListener); 576} 577 578void SelectionDAG::DeleteNode(SDNode *N) { 579 // First take this out of the appropriate CSE map. 580 RemoveNodeFromCSEMaps(N); 581 582 // Finally, remove uses due to operands of this node, remove from the 583 // AllNodes list, and delete the node. 584 DeleteNodeNotInCSEMaps(N); 585} 586 587void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 588 assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 589 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 590 591 // Drop all of the operands and decrement used node's use counts. 592 N->DropOperands(); 593 594 DeallocateNode(N); 595} 596 597void SelectionDAG::DeallocateNode(SDNode *N) { 598 if (N->OperandsNeedDelete) 599 delete[] N->OperandList; 600 601 // Set the opcode to DELETED_NODE to help catch bugs when node 602 // memory is reallocated. 603 N->NodeType = ISD::DELETED_NODE; 604 605 NodeAllocator.Deallocate(AllNodes.remove(N)); 606 607 // Remove the ordering of this node. 608 Ordering->remove(N); 609 610 // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. 611 ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); 612 for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) 613 DbgVals[i]->setIsInvalidated(); 614} 615 616/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 617/// correspond to it. This is useful when we're about to delete or repurpose 618/// the node. We don't want future request for structurally identical nodes 619/// to return N anymore. 620bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 621 bool Erased = false; 622 switch (N->getOpcode()) { 623 case ISD::HANDLENODE: return false; // noop. 624 case ISD::CONDCODE: 625 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 626 "Cond code doesn't exist!"); 627 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 628 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 629 break; 630 case ISD::ExternalSymbol: 631 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 632 break; 633 case ISD::TargetExternalSymbol: { 634 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 635 Erased = TargetExternalSymbols.erase( 636 std::pair<std::string,unsigned char>(ESN->getSymbol(), 637 ESN->getTargetFlags())); 638 break; 639 } 640 case ISD::VALUETYPE: { 641 EVT VT = cast<VTSDNode>(N)->getVT(); 642 if (VT.isExtended()) { 643 Erased = ExtendedValueTypeNodes.erase(VT); 644 } else { 645 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 646 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 647 } 648 break; 649 } 650 default: 651 // Remove it from the CSE Map. 652 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 653 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 654 Erased = CSEMap.RemoveNode(N); 655 break; 656 } 657#ifndef NDEBUG 658 // Verify that the node was actually in one of the CSE maps, unless it has a 659 // flag result (which cannot be CSE'd) or is one of the special cases that are 660 // not subject to CSE. 661 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 662 !N->isMachineOpcode() && !doNotCSE(N)) { 663 N->dump(this); 664 dbgs() << "\n"; 665 llvm_unreachable("Node is not in map!"); 666 } 667#endif 668 return Erased; 669} 670 671/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 672/// maps and modified in place. Add it back to the CSE maps, unless an identical 673/// node already exists, in which case transfer all its users to the existing 674/// node. This transfer can potentially trigger recursive merging. 675/// 676void 677SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N, 678 DAGUpdateListener *UpdateListener) { 679 // For node types that aren't CSE'd, just act as if no identical node 680 // already exists. 681 if (!doNotCSE(N)) { 682 SDNode *Existing = CSEMap.GetOrInsertNode(N); 683 if (Existing != N) { 684 // If there was already an existing matching node, use ReplaceAllUsesWith 685 // to replace the dead one with the existing one. This can cause 686 // recursive merging of other unrelated nodes down the line. 687 ReplaceAllUsesWith(N, Existing, UpdateListener); 688 689 // N is now dead. Inform the listener if it exists and delete it. 690 if (UpdateListener) 691 UpdateListener->NodeDeleted(N, Existing); 692 DeleteNodeNotInCSEMaps(N); 693 return; 694 } 695 } 696 697 // If the node doesn't already exist, we updated it. Inform a listener if 698 // it exists. 699 if (UpdateListener) 700 UpdateListener->NodeUpdated(N); 701} 702 703/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 704/// were replaced with those specified. If this node is never memoized, 705/// return null, otherwise return a pointer to the slot it would take. If a 706/// node already exists with these operands, the slot will be non-null. 707SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 708 void *&InsertPos) { 709 if (doNotCSE(N)) 710 return 0; 711 712 SDValue Ops[] = { Op }; 713 FoldingSetNodeID ID; 714 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 715 AddNodeIDCustom(ID, N); 716 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 717 return Node; 718} 719 720/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 721/// were replaced with those specified. If this node is never memoized, 722/// return null, otherwise return a pointer to the slot it would take. If a 723/// node already exists with these operands, the slot will be non-null. 724SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 725 SDValue Op1, SDValue Op2, 726 void *&InsertPos) { 727 if (doNotCSE(N)) 728 return 0; 729 730 SDValue Ops[] = { Op1, Op2 }; 731 FoldingSetNodeID ID; 732 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 733 AddNodeIDCustom(ID, N); 734 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 735 return Node; 736} 737 738 739/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 740/// were replaced with those specified. If this node is never memoized, 741/// return null, otherwise return a pointer to the slot it would take. If a 742/// node already exists with these operands, the slot will be non-null. 743SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 744 const SDValue *Ops,unsigned NumOps, 745 void *&InsertPos) { 746 if (doNotCSE(N)) 747 return 0; 748 749 FoldingSetNodeID ID; 750 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 751 AddNodeIDCustom(ID, N); 752 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 753 return Node; 754} 755 756#ifndef NDEBUG 757/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 758static void VerifyNodeCommon(SDNode *N) { 759 switch (N->getOpcode()) { 760 default: 761 break; 762 case ISD::BUILD_PAIR: { 763 EVT VT = N->getValueType(0); 764 assert(N->getNumValues() == 1 && "Too many results!"); 765 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 766 "Wrong return type!"); 767 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 768 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 769 "Mismatched operand types!"); 770 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 771 "Wrong operand type!"); 772 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 773 "Wrong return type size"); 774 break; 775 } 776 case ISD::BUILD_VECTOR: { 777 assert(N->getNumValues() == 1 && "Too many results!"); 778 assert(N->getValueType(0).isVector() && "Wrong return type!"); 779 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 780 "Wrong number of operands!"); 781 EVT EltVT = N->getValueType(0).getVectorElementType(); 782 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 783 assert((I->getValueType() == EltVT || 784 (EltVT.isInteger() && I->getValueType().isInteger() && 785 EltVT.bitsLE(I->getValueType()))) && 786 "Wrong operand type!"); 787 assert(I->getValueType() == N->getOperand(0).getValueType() && 788 "Operands must all have the same type"); 789 } 790 break; 791 } 792 } 793} 794 795/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 796static void VerifySDNode(SDNode *N) { 797 // The SDNode allocators cannot be used to allocate nodes with fields that are 798 // not present in an SDNode! 799 assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 800 assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 801 assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 802 assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 803 assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 804 assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 805 assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 806 assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 807 assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 808 assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 809 assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 810 assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 811 assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 812 assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 813 assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 814 assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 815 assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 816 assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 817 assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 818 819 VerifyNodeCommon(N); 820} 821 822/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 823/// invalid. 824static void VerifyMachineNode(SDNode *N) { 825 // The MachineNode allocators cannot be used to allocate nodes with fields 826 // that are not present in a MachineNode! 827 // Currently there are no such nodes. 828 829 VerifyNodeCommon(N); 830} 831#endif // NDEBUG 832 833/// getEVTAlignment - Compute the default alignment value for the 834/// given type. 835/// 836unsigned SelectionDAG::getEVTAlignment(EVT VT) const { 837 Type *Ty = VT == MVT::iPTR ? 838 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 839 VT.getTypeForEVT(*getContext()); 840 841 return TLI.getTargetData()->getABITypeAlignment(Ty); 842} 843 844// EntryNode could meaningfully have debug info if we can find it... 845SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 846 : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), 847 OptLevel(OL), EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), 848 Root(getEntryNode()), Ordering(0) { 849 AllNodes.push_back(&EntryNode); 850 Ordering = new SDNodeOrdering(); 851 DbgInfo = new SDDbgInfo(); 852} 853 854void SelectionDAG::init(MachineFunction &mf) { 855 MF = &mf; 856 Context = &mf.getFunction()->getContext(); 857} 858 859SelectionDAG::~SelectionDAG() { 860 allnodes_clear(); 861 delete Ordering; 862 delete DbgInfo; 863} 864 865void SelectionDAG::allnodes_clear() { 866 assert(&*AllNodes.begin() == &EntryNode); 867 AllNodes.remove(AllNodes.begin()); 868 while (!AllNodes.empty()) 869 DeallocateNode(AllNodes.begin()); 870} 871 872void SelectionDAG::clear() { 873 allnodes_clear(); 874 OperandAllocator.Reset(); 875 CSEMap.clear(); 876 877 ExtendedValueTypeNodes.clear(); 878 ExternalSymbols.clear(); 879 TargetExternalSymbols.clear(); 880 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 881 static_cast<CondCodeSDNode*>(0)); 882 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 883 static_cast<SDNode*>(0)); 884 885 EntryNode.UseList = 0; 886 AllNodes.push_back(&EntryNode); 887 Root = getEntryNode(); 888 Ordering->clear(); 889 DbgInfo->clear(); 890} 891 892SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 893 return VT.bitsGT(Op.getValueType()) ? 894 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 895 getNode(ISD::TRUNCATE, DL, VT, Op); 896} 897 898SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 899 return VT.bitsGT(Op.getValueType()) ? 900 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 901 getNode(ISD::TRUNCATE, DL, VT, Op); 902} 903 904SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 905 return VT.bitsGT(Op.getValueType()) ? 906 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 907 getNode(ISD::TRUNCATE, DL, VT, Op); 908} 909 910SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, DebugLoc DL, EVT VT) { 911 assert(!VT.isVector() && 912 "getZeroExtendInReg should use the vector element type instead of " 913 "the vector type!"); 914 if (Op.getValueType() == VT) return Op; 915 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 916 APInt Imm = APInt::getLowBitsSet(BitWidth, 917 VT.getSizeInBits()); 918 return getNode(ISD::AND, DL, Op.getValueType(), Op, 919 getConstant(Imm, Op.getValueType())); 920} 921 922/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 923/// 924SDValue SelectionDAG::getNOT(DebugLoc DL, SDValue Val, EVT VT) { 925 EVT EltVT = VT.getScalarType(); 926 SDValue NegOne = 927 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 928 return getNode(ISD::XOR, DL, VT, Val, NegOne); 929} 930 931SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 932 EVT EltVT = VT.getScalarType(); 933 assert((EltVT.getSizeInBits() >= 64 || 934 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 935 "getConstant with a uint64_t value that doesn't fit in the type!"); 936 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 937} 938 939SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 940 return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 941} 942 943SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 944 assert(VT.isInteger() && "Cannot create FP integer constant!"); 945 946 EVT EltVT = VT.getScalarType(); 947 const ConstantInt *Elt = &Val; 948 949 // In some cases the vector type is legal but the element type is illegal and 950 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 951 // inserted value (the type does not need to match the vector element type). 952 // Any extra bits introduced will be truncated away. 953 if (VT.isVector() && TLI.getTypeAction(*getContext(), EltVT) == 954 TargetLowering::TypePromoteInteger) { 955 EltVT = TLI.getTypeToTransformTo(*getContext(), EltVT); 956 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 957 Elt = ConstantInt::get(*getContext(), NewVal); 958 } 959 960 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 961 "APInt size does not match type size!"); 962 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 963 FoldingSetNodeID ID; 964 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 965 ID.AddPointer(Elt); 966 void *IP = 0; 967 SDNode *N = NULL; 968 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 969 if (!VT.isVector()) 970 return SDValue(N, 0); 971 972 if (!N) { 973 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 974 CSEMap.InsertNode(N, IP); 975 AllNodes.push_back(N); 976 } 977 978 SDValue Result(N, 0); 979 if (VT.isVector()) { 980 SmallVector<SDValue, 8> Ops; 981 Ops.assign(VT.getVectorNumElements(), Result); 982 Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); 983 } 984 return Result; 985} 986 987SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 988 return getConstant(Val, TLI.getPointerTy(), isTarget); 989} 990 991 992SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 993 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 994} 995 996SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 997 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 998 999 EVT EltVT = VT.getScalarType(); 1000 1001 // Do the map lookup using the actual bit pattern for the floating point 1002 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1003 // we don't have issues with SNANs. 1004 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1005 FoldingSetNodeID ID; 1006 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1007 ID.AddPointer(&V); 1008 void *IP = 0; 1009 SDNode *N = NULL; 1010 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1011 if (!VT.isVector()) 1012 return SDValue(N, 0); 1013 1014 if (!N) { 1015 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1016 CSEMap.InsertNode(N, IP); 1017 AllNodes.push_back(N); 1018 } 1019 1020 SDValue Result(N, 0); 1021 if (VT.isVector()) { 1022 SmallVector<SDValue, 8> Ops; 1023 Ops.assign(VT.getVectorNumElements(), Result); 1024 // FIXME DebugLoc info might be appropriate here 1025 Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); 1026 } 1027 return Result; 1028} 1029 1030SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1031 EVT EltVT = VT.getScalarType(); 1032 if (EltVT==MVT::f32) 1033 return getConstantFP(APFloat((float)Val), VT, isTarget); 1034 else if (EltVT==MVT::f64) 1035 return getConstantFP(APFloat(Val), VT, isTarget); 1036 else if (EltVT==MVT::f80 || EltVT==MVT::f128) { 1037 bool ignored; 1038 APFloat apf = APFloat(Val); 1039 apf.convert(*EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1040 &ignored); 1041 return getConstantFP(apf, VT, isTarget); 1042 } else { 1043 assert(0 && "Unsupported type in getConstantFP"); 1044 return SDValue(); 1045 } 1046} 1047 1048SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, DebugLoc DL, 1049 EVT VT, int64_t Offset, 1050 bool isTargetGA, 1051 unsigned char TargetFlags) { 1052 assert((TargetFlags == 0 || isTargetGA) && 1053 "Cannot set target flags on target-independent globals"); 1054 1055 // Truncate (with sign-extension) the offset value to the pointer size. 1056 EVT PTy = TLI.getPointerTy(); 1057 unsigned BitWidth = PTy.getSizeInBits(); 1058 if (BitWidth < 64) 1059 Offset = (Offset << (64 - BitWidth) >> (64 - BitWidth)); 1060 1061 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1062 if (!GVar) { 1063 // If GV is an alias then use the aliasee for determining thread-localness. 1064 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1065 GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1066 } 1067 1068 unsigned Opc; 1069 if (GVar && GVar->isThreadLocal()) 1070 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1071 else 1072 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1073 1074 FoldingSetNodeID ID; 1075 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1076 ID.AddPointer(GV); 1077 ID.AddInteger(Offset); 1078 ID.AddInteger(TargetFlags); 1079 void *IP = 0; 1080 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1081 return SDValue(E, 0); 1082 1083 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL, GV, VT, 1084 Offset, TargetFlags); 1085 CSEMap.InsertNode(N, IP); 1086 AllNodes.push_back(N); 1087 return SDValue(N, 0); 1088} 1089 1090SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1091 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1092 FoldingSetNodeID ID; 1093 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1094 ID.AddInteger(FI); 1095 void *IP = 0; 1096 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1097 return SDValue(E, 0); 1098 1099 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1100 CSEMap.InsertNode(N, IP); 1101 AllNodes.push_back(N); 1102 return SDValue(N, 0); 1103} 1104 1105SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1106 unsigned char TargetFlags) { 1107 assert((TargetFlags == 0 || isTarget) && 1108 "Cannot set target flags on target-independent jump tables"); 1109 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1110 FoldingSetNodeID ID; 1111 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1112 ID.AddInteger(JTI); 1113 ID.AddInteger(TargetFlags); 1114 void *IP = 0; 1115 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1116 return SDValue(E, 0); 1117 1118 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1119 TargetFlags); 1120 CSEMap.InsertNode(N, IP); 1121 AllNodes.push_back(N); 1122 return SDValue(N, 0); 1123} 1124 1125SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1126 unsigned Alignment, int Offset, 1127 bool isTarget, 1128 unsigned char TargetFlags) { 1129 assert((TargetFlags == 0 || isTarget) && 1130 "Cannot set target flags on target-independent globals"); 1131 if (Alignment == 0) 1132 Alignment = TLI.getTargetData()->getPrefTypeAlignment(C->getType()); 1133 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1134 FoldingSetNodeID ID; 1135 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1136 ID.AddInteger(Alignment); 1137 ID.AddInteger(Offset); 1138 ID.AddPointer(C); 1139 ID.AddInteger(TargetFlags); 1140 void *IP = 0; 1141 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1142 return SDValue(E, 0); 1143 1144 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1145 Alignment, TargetFlags); 1146 CSEMap.InsertNode(N, IP); 1147 AllNodes.push_back(N); 1148 return SDValue(N, 0); 1149} 1150 1151 1152SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1153 unsigned Alignment, int Offset, 1154 bool isTarget, 1155 unsigned char TargetFlags) { 1156 assert((TargetFlags == 0 || isTarget) && 1157 "Cannot set target flags on target-independent globals"); 1158 if (Alignment == 0) 1159 Alignment = TLI.getTargetData()->getPrefTypeAlignment(C->getType()); 1160 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1161 FoldingSetNodeID ID; 1162 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1163 ID.AddInteger(Alignment); 1164 ID.AddInteger(Offset); 1165 C->addSelectionDAGCSEId(ID); 1166 ID.AddInteger(TargetFlags); 1167 void *IP = 0; 1168 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1169 return SDValue(E, 0); 1170 1171 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1172 Alignment, TargetFlags); 1173 CSEMap.InsertNode(N, IP); 1174 AllNodes.push_back(N); 1175 return SDValue(N, 0); 1176} 1177 1178SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1179 FoldingSetNodeID ID; 1180 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1181 ID.AddPointer(MBB); 1182 void *IP = 0; 1183 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1184 return SDValue(E, 0); 1185 1186 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1187 CSEMap.InsertNode(N, IP); 1188 AllNodes.push_back(N); 1189 return SDValue(N, 0); 1190} 1191 1192SDValue SelectionDAG::getValueType(EVT VT) { 1193 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1194 ValueTypeNodes.size()) 1195 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1196 1197 SDNode *&N = VT.isExtended() ? 1198 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1199 1200 if (N) return SDValue(N, 0); 1201 N = new (NodeAllocator) VTSDNode(VT); 1202 AllNodes.push_back(N); 1203 return SDValue(N, 0); 1204} 1205 1206SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1207 SDNode *&N = ExternalSymbols[Sym]; 1208 if (N) return SDValue(N, 0); 1209 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1210 AllNodes.push_back(N); 1211 return SDValue(N, 0); 1212} 1213 1214SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1215 unsigned char TargetFlags) { 1216 SDNode *&N = 1217 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1218 TargetFlags)]; 1219 if (N) return SDValue(N, 0); 1220 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1221 AllNodes.push_back(N); 1222 return SDValue(N, 0); 1223} 1224 1225SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1226 if ((unsigned)Cond >= CondCodeNodes.size()) 1227 CondCodeNodes.resize(Cond+1); 1228 1229 if (CondCodeNodes[Cond] == 0) { 1230 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1231 CondCodeNodes[Cond] = N; 1232 AllNodes.push_back(N); 1233 } 1234 1235 return SDValue(CondCodeNodes[Cond], 0); 1236} 1237 1238// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1239// the shuffle mask M that point at N1 to point at N2, and indices that point 1240// N2 to point at N1. 1241static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1242 std::swap(N1, N2); 1243 int NElts = M.size(); 1244 for (int i = 0; i != NElts; ++i) { 1245 if (M[i] >= NElts) 1246 M[i] -= NElts; 1247 else if (M[i] >= 0) 1248 M[i] += NElts; 1249 } 1250} 1251 1252SDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1, 1253 SDValue N2, const int *Mask) { 1254 assert(N1.getValueType() == N2.getValueType() && "Invalid VECTOR_SHUFFLE"); 1255 assert(VT.isVector() && N1.getValueType().isVector() && 1256 "Vector Shuffle VTs must be a vectors"); 1257 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() 1258 && "Vector Shuffle VTs must have same element type"); 1259 1260 // Canonicalize shuffle undef, undef -> undef 1261 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1262 return getUNDEF(VT); 1263 1264 // Validate that all indices in Mask are within the range of the elements 1265 // input to the shuffle. 1266 unsigned NElts = VT.getVectorNumElements(); 1267 SmallVector<int, 8> MaskVec; 1268 for (unsigned i = 0; i != NElts; ++i) { 1269 assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1270 MaskVec.push_back(Mask[i]); 1271 } 1272 1273 // Canonicalize shuffle v, v -> v, undef 1274 if (N1 == N2) { 1275 N2 = getUNDEF(VT); 1276 for (unsigned i = 0; i != NElts; ++i) 1277 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1278 } 1279 1280 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1281 if (N1.getOpcode() == ISD::UNDEF) 1282 commuteShuffle(N1, N2, MaskVec); 1283 1284 // Canonicalize all index into lhs, -> shuffle lhs, undef 1285 // Canonicalize all index into rhs, -> shuffle rhs, undef 1286 bool AllLHS = true, AllRHS = true; 1287 bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1288 for (unsigned i = 0; i != NElts; ++i) { 1289 if (MaskVec[i] >= (int)NElts) { 1290 if (N2Undef) 1291 MaskVec[i] = -1; 1292 else 1293 AllLHS = false; 1294 } else if (MaskVec[i] >= 0) { 1295 AllRHS = false; 1296 } 1297 } 1298 if (AllLHS && AllRHS) 1299 return getUNDEF(VT); 1300 if (AllLHS && !N2Undef) 1301 N2 = getUNDEF(VT); 1302 if (AllRHS) { 1303 N1 = getUNDEF(VT); 1304 commuteShuffle(N1, N2, MaskVec); 1305 } 1306 1307 // If Identity shuffle, or all shuffle in to undef, return that node. 1308 bool AllUndef = true; 1309 bool Identity = true; 1310 for (unsigned i = 0; i != NElts; ++i) { 1311 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1312 if (MaskVec[i] >= 0) AllUndef = false; 1313 } 1314 if (Identity && NElts == N1.getValueType().getVectorNumElements()) 1315 return N1; 1316 if (AllUndef) 1317 return getUNDEF(VT); 1318 1319 FoldingSetNodeID ID; 1320 SDValue Ops[2] = { N1, N2 }; 1321 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1322 for (unsigned i = 0; i != NElts; ++i) 1323 ID.AddInteger(MaskVec[i]); 1324 1325 void* IP = 0; 1326 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1327 return SDValue(E, 0); 1328 1329 // Allocate the mask array for the node out of the BumpPtrAllocator, since 1330 // SDNode doesn't have access to it. This memory will be "leaked" when 1331 // the node is deallocated, but recovered when the NodeAllocator is released. 1332 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1333 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1334 1335 ShuffleVectorSDNode *N = 1336 new (NodeAllocator) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc); 1337 CSEMap.InsertNode(N, IP); 1338 AllNodes.push_back(N); 1339 return SDValue(N, 0); 1340} 1341 1342SDValue SelectionDAG::getConvertRndSat(EVT VT, DebugLoc dl, 1343 SDValue Val, SDValue DTy, 1344 SDValue STy, SDValue Rnd, SDValue Sat, 1345 ISD::CvtCode Code) { 1346 // If the src and dest types are the same and the conversion is between 1347 // integer types of the same sign or two floats, no conversion is necessary. 1348 if (DTy == STy && 1349 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1350 return Val; 1351 1352 FoldingSetNodeID ID; 1353 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1354 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1355 void* IP = 0; 1356 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1357 return SDValue(E, 0); 1358 1359 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl, Ops, 5, 1360 Code); 1361 CSEMap.InsertNode(N, IP); 1362 AllNodes.push_back(N); 1363 return SDValue(N, 0); 1364} 1365 1366SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1367 FoldingSetNodeID ID; 1368 AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1369 ID.AddInteger(RegNo); 1370 void *IP = 0; 1371 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1372 return SDValue(E, 0); 1373 1374 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1375 CSEMap.InsertNode(N, IP); 1376 AllNodes.push_back(N); 1377 return SDValue(N, 0); 1378} 1379 1380SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1381 FoldingSetNodeID ID; 1382 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0); 1383 ID.AddPointer(RegMask); 1384 void *IP = 0; 1385 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1386 return SDValue(E, 0); 1387 1388 SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask); 1389 CSEMap.InsertNode(N, IP); 1390 AllNodes.push_back(N); 1391 return SDValue(N, 0); 1392} 1393 1394SDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) { 1395 FoldingSetNodeID ID; 1396 SDValue Ops[] = { Root }; 1397 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1398 ID.AddPointer(Label); 1399 void *IP = 0; 1400 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1401 return SDValue(E, 0); 1402 1403 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl, Root, Label); 1404 CSEMap.InsertNode(N, IP); 1405 AllNodes.push_back(N); 1406 return SDValue(N, 0); 1407} 1408 1409 1410SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1411 bool isTarget, 1412 unsigned char TargetFlags) { 1413 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1414 1415 FoldingSetNodeID ID; 1416 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1417 ID.AddPointer(BA); 1418 ID.AddInteger(TargetFlags); 1419 void *IP = 0; 1420 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1421 return SDValue(E, 0); 1422 1423 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, TargetFlags); 1424 CSEMap.InsertNode(N, IP); 1425 AllNodes.push_back(N); 1426 return SDValue(N, 0); 1427} 1428 1429SDValue SelectionDAG::getSrcValue(const Value *V) { 1430 assert((!V || V->getType()->isPointerTy()) && 1431 "SrcValue is not a pointer?"); 1432 1433 FoldingSetNodeID ID; 1434 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1435 ID.AddPointer(V); 1436 1437 void *IP = 0; 1438 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1439 return SDValue(E, 0); 1440 1441 SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1442 CSEMap.InsertNode(N, IP); 1443 AllNodes.push_back(N); 1444 return SDValue(N, 0); 1445} 1446 1447/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1448SDValue SelectionDAG::getMDNode(const MDNode *MD) { 1449 FoldingSetNodeID ID; 1450 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1451 ID.AddPointer(MD); 1452 1453 void *IP = 0; 1454 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1455 return SDValue(E, 0); 1456 1457 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1458 CSEMap.InsertNode(N, IP); 1459 AllNodes.push_back(N); 1460 return SDValue(N, 0); 1461} 1462 1463 1464/// getShiftAmountOperand - Return the specified value casted to 1465/// the target's desired shift amount type. 1466SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1467 EVT OpTy = Op.getValueType(); 1468 MVT ShTy = TLI.getShiftAmountTy(LHSTy); 1469 if (OpTy == ShTy || OpTy.isVector()) return Op; 1470 1471 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1472 return getNode(Opcode, Op.getDebugLoc(), ShTy, Op); 1473} 1474 1475/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1476/// specified value type. 1477SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1478 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1479 unsigned ByteSize = VT.getStoreSize(); 1480 Type *Ty = VT.getTypeForEVT(*getContext()); 1481 unsigned StackAlign = 1482 std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign); 1483 1484 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1485 return getFrameIndex(FrameIdx, TLI.getPointerTy()); 1486} 1487 1488/// CreateStackTemporary - Create a stack temporary suitable for holding 1489/// either of the specified value types. 1490SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1491 unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1492 VT2.getStoreSizeInBits())/8; 1493 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1494 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1495 const TargetData *TD = TLI.getTargetData(); 1496 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1497 TD->getPrefTypeAlignment(Ty2)); 1498 1499 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1500 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1501 return getFrameIndex(FrameIdx, TLI.getPointerTy()); 1502} 1503 1504SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1505 SDValue N2, ISD::CondCode Cond, DebugLoc dl) { 1506 // These setcc operations always fold. 1507 switch (Cond) { 1508 default: break; 1509 case ISD::SETFALSE: 1510 case ISD::SETFALSE2: return getConstant(0, VT); 1511 case ISD::SETTRUE: 1512 case ISD::SETTRUE2: return getConstant(1, VT); 1513 1514 case ISD::SETOEQ: 1515 case ISD::SETOGT: 1516 case ISD::SETOGE: 1517 case ISD::SETOLT: 1518 case ISD::SETOLE: 1519 case ISD::SETONE: 1520 case ISD::SETO: 1521 case ISD::SETUO: 1522 case ISD::SETUEQ: 1523 case ISD::SETUNE: 1524 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1525 break; 1526 } 1527 1528 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1529 const APInt &C2 = N2C->getAPIntValue(); 1530 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1531 const APInt &C1 = N1C->getAPIntValue(); 1532 1533 switch (Cond) { 1534 default: llvm_unreachable("Unknown integer setcc!"); 1535 case ISD::SETEQ: return getConstant(C1 == C2, VT); 1536 case ISD::SETNE: return getConstant(C1 != C2, VT); 1537 case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1538 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1539 case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1540 case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1541 case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1542 case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1543 case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1544 case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1545 } 1546 } 1547 } 1548 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1549 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1550 // No compile time operations on this type yet. 1551 if (N1C->getValueType(0) == MVT::ppcf128) 1552 return SDValue(); 1553 1554 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1555 switch (Cond) { 1556 default: break; 1557 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1558 return getUNDEF(VT); 1559 // fall through 1560 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1561 case ISD::SETNE: if (R==APFloat::cmpUnordered) 1562 return getUNDEF(VT); 1563 // fall through 1564 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1565 R==APFloat::cmpLessThan, VT); 1566 case ISD::SETLT: if (R==APFloat::cmpUnordered) 1567 return getUNDEF(VT); 1568 // fall through 1569 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1570 case ISD::SETGT: if (R==APFloat::cmpUnordered) 1571 return getUNDEF(VT); 1572 // fall through 1573 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1574 case ISD::SETLE: if (R==APFloat::cmpUnordered) 1575 return getUNDEF(VT); 1576 // fall through 1577 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1578 R==APFloat::cmpEqual, VT); 1579 case ISD::SETGE: if (R==APFloat::cmpUnordered) 1580 return getUNDEF(VT); 1581 // fall through 1582 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1583 R==APFloat::cmpEqual, VT); 1584 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1585 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1586 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1587 R==APFloat::cmpEqual, VT); 1588 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1589 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1590 R==APFloat::cmpLessThan, VT); 1591 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1592 R==APFloat::cmpUnordered, VT); 1593 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1594 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1595 } 1596 } else { 1597 // Ensure that the constant occurs on the RHS. 1598 return getSetCC(dl, VT, N2, N1, ISD::getSetCCSwappedOperands(Cond)); 1599 } 1600 } 1601 1602 // Could not fold it. 1603 return SDValue(); 1604} 1605 1606/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1607/// use this predicate to simplify operations downstream. 1608bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1609 // This predicate is not safe for vector operations. 1610 if (Op.getValueType().isVector()) 1611 return false; 1612 1613 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1614 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1615} 1616 1617/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1618/// this predicate to simplify operations downstream. Mask is known to be zero 1619/// for bits that V cannot have. 1620bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1621 unsigned Depth) const { 1622 APInt KnownZero, KnownOne; 1623 ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth); 1624 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1625 return (KnownZero & Mask) == Mask; 1626} 1627 1628/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1629/// known to be either zero or one and return them in the KnownZero/KnownOne 1630/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1631/// processing. 1632void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, 1633 APInt &KnownZero, APInt &KnownOne, 1634 unsigned Depth) const { 1635 unsigned BitWidth = Mask.getBitWidth(); 1636 assert(BitWidth == Op.getValueType().getScalarType().getSizeInBits() && 1637 "Mask size mismatches value type size!"); 1638 1639 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1640 if (Depth == 6 || Mask == 0) 1641 return; // Limit search depth. 1642 1643 APInt KnownZero2, KnownOne2; 1644 1645 switch (Op.getOpcode()) { 1646 case ISD::Constant: 1647 // We know all of the bits for a constant! 1648 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue() & Mask; 1649 KnownZero = ~KnownOne & Mask; 1650 return; 1651 case ISD::AND: 1652 // If either the LHS or the RHS are Zero, the result is zero. 1653 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1654 ComputeMaskedBits(Op.getOperand(0), Mask & ~KnownZero, 1655 KnownZero2, KnownOne2, Depth+1); 1656 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1657 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1658 1659 // Output known-1 bits are only known if set in both the LHS & RHS. 1660 KnownOne &= KnownOne2; 1661 // Output known-0 are known to be clear if zero in either the LHS | RHS. 1662 KnownZero |= KnownZero2; 1663 return; 1664 case ISD::OR: 1665 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1666 ComputeMaskedBits(Op.getOperand(0), Mask & ~KnownOne, 1667 KnownZero2, KnownOne2, Depth+1); 1668 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1669 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1670 1671 // Output known-0 bits are only known if clear in both the LHS & RHS. 1672 KnownZero &= KnownZero2; 1673 // Output known-1 are known to be set if set in either the LHS | RHS. 1674 KnownOne |= KnownOne2; 1675 return; 1676 case ISD::XOR: { 1677 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1678 ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1); 1679 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1680 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1681 1682 // Output known-0 bits are known if clear or set in both the LHS & RHS. 1683 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1684 // Output known-1 are known to be set if set in only one of the LHS, RHS. 1685 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1686 KnownZero = KnownZeroOut; 1687 return; 1688 } 1689 case ISD::MUL: { 1690 APInt Mask2 = APInt::getAllOnesValue(BitWidth); 1691 ComputeMaskedBits(Op.getOperand(1), Mask2, KnownZero, KnownOne, Depth+1); 1692 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1); 1693 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1694 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1695 1696 // If low bits are zero in either operand, output low known-0 bits. 1697 // Also compute a conserative estimate for high known-0 bits. 1698 // More trickiness is possible, but this is sufficient for the 1699 // interesting case of alignment computation. 1700 KnownOne.clearAllBits(); 1701 unsigned TrailZ = KnownZero.countTrailingOnes() + 1702 KnownZero2.countTrailingOnes(); 1703 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1704 KnownZero2.countLeadingOnes(), 1705 BitWidth) - BitWidth; 1706 1707 TrailZ = std::min(TrailZ, BitWidth); 1708 LeadZ = std::min(LeadZ, BitWidth); 1709 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1710 APInt::getHighBitsSet(BitWidth, LeadZ); 1711 KnownZero &= Mask; 1712 return; 1713 } 1714 case ISD::UDIV: { 1715 // For the purposes of computing leading zeros we can conservatively 1716 // treat a udiv as a logical right shift by the power of 2 known to 1717 // be less than the denominator. 1718 APInt AllOnes = APInt::getAllOnesValue(BitWidth); 1719 ComputeMaskedBits(Op.getOperand(0), 1720 AllOnes, KnownZero2, KnownOne2, Depth+1); 1721 unsigned LeadZ = KnownZero2.countLeadingOnes(); 1722 1723 KnownOne2.clearAllBits(); 1724 KnownZero2.clearAllBits(); 1725 ComputeMaskedBits(Op.getOperand(1), 1726 AllOnes, KnownZero2, KnownOne2, Depth+1); 1727 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1728 if (RHSUnknownLeadingOnes != BitWidth) 1729 LeadZ = std::min(BitWidth, 1730 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1731 1732 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ) & Mask; 1733 return; 1734 } 1735 case ISD::SELECT: 1736 ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero, KnownOne, Depth+1); 1737 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero2, KnownOne2, Depth+1); 1738 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1739 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1740 1741 // Only known if known in both the LHS and RHS. 1742 KnownOne &= KnownOne2; 1743 KnownZero &= KnownZero2; 1744 return; 1745 case ISD::SELECT_CC: 1746 ComputeMaskedBits(Op.getOperand(3), Mask, KnownZero, KnownOne, Depth+1); 1747 ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero2, KnownOne2, Depth+1); 1748 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1749 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1750 1751 // Only known if known in both the LHS and RHS. 1752 KnownOne &= KnownOne2; 1753 KnownZero &= KnownZero2; 1754 return; 1755 case ISD::SADDO: 1756 case ISD::UADDO: 1757 case ISD::SSUBO: 1758 case ISD::USUBO: 1759 case ISD::SMULO: 1760 case ISD::UMULO: 1761 if (Op.getResNo() != 1) 1762 return; 1763 // The boolean result conforms to getBooleanContents. Fall through. 1764 case ISD::SETCC: 1765 // If we know the result of a setcc has the top bits zero, use this info. 1766 if (TLI.getBooleanContents(Op.getValueType().isVector()) == 1767 TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1768 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1769 return; 1770 case ISD::SHL: 1771 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1772 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1773 unsigned ShAmt = SA->getZExtValue(); 1774 1775 // If the shift count is an invalid immediate, don't do anything. 1776 if (ShAmt >= BitWidth) 1777 return; 1778 1779 ComputeMaskedBits(Op.getOperand(0), Mask.lshr(ShAmt), 1780 KnownZero, KnownOne, Depth+1); 1781 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1782 KnownZero <<= ShAmt; 1783 KnownOne <<= ShAmt; 1784 // low bits known zero. 1785 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1786 } 1787 return; 1788 case ISD::SRL: 1789 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1790 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1791 unsigned ShAmt = SA->getZExtValue(); 1792 1793 // If the shift count is an invalid immediate, don't do anything. 1794 if (ShAmt >= BitWidth) 1795 return; 1796 1797 ComputeMaskedBits(Op.getOperand(0), (Mask << ShAmt), 1798 KnownZero, KnownOne, Depth+1); 1799 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1800 KnownZero = KnownZero.lshr(ShAmt); 1801 KnownOne = KnownOne.lshr(ShAmt); 1802 1803 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt) & Mask; 1804 KnownZero |= HighBits; // High bits known zero. 1805 } 1806 return; 1807 case ISD::SRA: 1808 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1809 unsigned ShAmt = SA->getZExtValue(); 1810 1811 // If the shift count is an invalid immediate, don't do anything. 1812 if (ShAmt >= BitWidth) 1813 return; 1814 1815 APInt InDemandedMask = (Mask << ShAmt); 1816 // If any of the demanded bits are produced by the sign extension, we also 1817 // demand the input sign bit. 1818 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt) & Mask; 1819 if (HighBits.getBoolValue()) 1820 InDemandedMask |= APInt::getSignBit(BitWidth); 1821 1822 ComputeMaskedBits(Op.getOperand(0), InDemandedMask, KnownZero, KnownOne, 1823 Depth+1); 1824 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1825 KnownZero = KnownZero.lshr(ShAmt); 1826 KnownOne = KnownOne.lshr(ShAmt); 1827 1828 // Handle the sign bits. 1829 APInt SignBit = APInt::getSignBit(BitWidth); 1830 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1831 1832 if (KnownZero.intersects(SignBit)) { 1833 KnownZero |= HighBits; // New bits are known zero. 1834 } else if (KnownOne.intersects(SignBit)) { 1835 KnownOne |= HighBits; // New bits are known one. 1836 } 1837 } 1838 return; 1839 case ISD::SIGN_EXTEND_INREG: { 1840 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1841 unsigned EBits = EVT.getScalarType().getSizeInBits(); 1842 1843 // Sign extension. Compute the demanded bits in the result that are not 1844 // present in the input. 1845 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits) & Mask; 1846 1847 APInt InSignBit = APInt::getSignBit(EBits); 1848 APInt InputDemandedBits = Mask & APInt::getLowBitsSet(BitWidth, EBits); 1849 1850 // If the sign extended bits are demanded, we know that the sign 1851 // bit is demanded. 1852 InSignBit = InSignBit.zext(BitWidth); 1853 if (NewBits.getBoolValue()) 1854 InputDemandedBits |= InSignBit; 1855 1856 ComputeMaskedBits(Op.getOperand(0), InputDemandedBits, 1857 KnownZero, KnownOne, Depth+1); 1858 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1859 1860 // If the sign bit of the input is known set or clear, then we know the 1861 // top bits of the result. 1862 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1863 KnownZero |= NewBits; 1864 KnownOne &= ~NewBits; 1865 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1866 KnownOne |= NewBits; 1867 KnownZero &= ~NewBits; 1868 } else { // Input sign bit unknown 1869 KnownZero &= ~NewBits; 1870 KnownOne &= ~NewBits; 1871 } 1872 return; 1873 } 1874 case ISD::CTTZ: 1875 case ISD::CTTZ_ZERO_UNDEF: 1876 case ISD::CTLZ: 1877 case ISD::CTLZ_ZERO_UNDEF: 1878 case ISD::CTPOP: { 1879 unsigned LowBits = Log2_32(BitWidth)+1; 1880 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 1881 KnownOne.clearAllBits(); 1882 return; 1883 } 1884 case ISD::LOAD: { 1885 if (ISD::isZEXTLoad(Op.getNode())) { 1886 LoadSDNode *LD = cast<LoadSDNode>(Op); 1887 EVT VT = LD->getMemoryVT(); 1888 unsigned MemBits = VT.getScalarType().getSizeInBits(); 1889 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits) & Mask; 1890 } 1891 return; 1892 } 1893 case ISD::ZERO_EXTEND: { 1894 EVT InVT = Op.getOperand(0).getValueType(); 1895 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1896 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; 1897 APInt InMask = Mask.trunc(InBits); 1898 KnownZero = KnownZero.trunc(InBits); 1899 KnownOne = KnownOne.trunc(InBits); 1900 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1901 KnownZero = KnownZero.zext(BitWidth); 1902 KnownOne = KnownOne.zext(BitWidth); 1903 KnownZero |= NewBits; 1904 return; 1905 } 1906 case ISD::SIGN_EXTEND: { 1907 EVT InVT = Op.getOperand(0).getValueType(); 1908 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1909 APInt InSignBit = APInt::getSignBit(InBits); 1910 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; 1911 APInt InMask = Mask.trunc(InBits); 1912 1913 // If any of the sign extended bits are demanded, we know that the sign 1914 // bit is demanded. Temporarily set this bit in the mask for our callee. 1915 if (NewBits.getBoolValue()) 1916 InMask |= InSignBit; 1917 1918 KnownZero = KnownZero.trunc(InBits); 1919 KnownOne = KnownOne.trunc(InBits); 1920 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1921 1922 // Note if the sign bit is known to be zero or one. 1923 bool SignBitKnownZero = KnownZero.isNegative(); 1924 bool SignBitKnownOne = KnownOne.isNegative(); 1925 assert(!(SignBitKnownZero && SignBitKnownOne) && 1926 "Sign bit can't be known to be both zero and one!"); 1927 1928 // If the sign bit wasn't actually demanded by our caller, we don't 1929 // want it set in the KnownZero and KnownOne result values. Reset the 1930 // mask and reapply it to the result values. 1931 InMask = Mask.trunc(InBits); 1932 KnownZero &= InMask; 1933 KnownOne &= InMask; 1934 1935 KnownZero = KnownZero.zext(BitWidth); 1936 KnownOne = KnownOne.zext(BitWidth); 1937 1938 // If the sign bit is known zero or one, the top bits match. 1939 if (SignBitKnownZero) 1940 KnownZero |= NewBits; 1941 else if (SignBitKnownOne) 1942 KnownOne |= NewBits; 1943 return; 1944 } 1945 case ISD::ANY_EXTEND: { 1946 EVT InVT = Op.getOperand(0).getValueType(); 1947 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1948 APInt InMask = Mask.trunc(InBits); 1949 KnownZero = KnownZero.trunc(InBits); 1950 KnownOne = KnownOne.trunc(InBits); 1951 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1952 KnownZero = KnownZero.zext(BitWidth); 1953 KnownOne = KnownOne.zext(BitWidth); 1954 return; 1955 } 1956 case ISD::TRUNCATE: { 1957 EVT InVT = Op.getOperand(0).getValueType(); 1958 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1959 APInt InMask = Mask.zext(InBits); 1960 KnownZero = KnownZero.zext(InBits); 1961 KnownOne = KnownOne.zext(InBits); 1962 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1963 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1964 KnownZero = KnownZero.trunc(BitWidth); 1965 KnownOne = KnownOne.trunc(BitWidth); 1966 break; 1967 } 1968 case ISD::AssertZext: { 1969 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1970 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 1971 ComputeMaskedBits(Op.getOperand(0), Mask & InMask, KnownZero, 1972 KnownOne, Depth+1); 1973 KnownZero |= (~InMask) & Mask; 1974 return; 1975 } 1976 case ISD::FGETSIGN: 1977 // All bits are zero except the low bit. 1978 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1979 return; 1980 1981 case ISD::SUB: { 1982 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 1983 // We know that the top bits of C-X are clear if X contains less bits 1984 // than C (i.e. no wrap-around can happen). For example, 20-X is 1985 // positive if we can prove that X is >= 0 and < 16. 1986 if (CLHS->getAPIntValue().isNonNegative()) { 1987 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 1988 // NLZ can't be BitWidth with no sign bit 1989 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 1990 ComputeMaskedBits(Op.getOperand(1), MaskV, KnownZero2, KnownOne2, 1991 Depth+1); 1992 1993 // If all of the MaskV bits are known to be zero, then we know the 1994 // output top bits are zero, because we now know that the output is 1995 // from [0-C]. 1996 if ((KnownZero2 & MaskV) == MaskV) { 1997 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 1998 // Top bits known zero. 1999 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2) & Mask; 2000 } 2001 } 2002 } 2003 } 2004 // fall through 2005 case ISD::ADD: 2006 case ISD::ADDE: { 2007 // Output known-0 bits are known if clear or set in both the low clear bits 2008 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 2009 // low 3 bits clear. 2010 APInt Mask2 = APInt::getLowBitsSet(BitWidth, 2011 BitWidth - Mask.countLeadingZeros()); 2012 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1); 2013 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2014 unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 2015 2016 ComputeMaskedBits(Op.getOperand(1), Mask2, KnownZero2, KnownOne2, Depth+1); 2017 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2018 KnownZeroOut = std::min(KnownZeroOut, 2019 KnownZero2.countTrailingOnes()); 2020 2021 if (Op.getOpcode() == ISD::ADD) { 2022 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2023 return; 2024 } 2025 2026 // With ADDE, a carry bit may be added in, so we can only use this 2027 // information if we know (at least) that the low two bits are clear. We 2028 // then return to the caller that the low bit is unknown but that other bits 2029 // are known zero. 2030 if (KnownZeroOut >= 2) // ADDE 2031 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2032 return; 2033 } 2034 case ISD::SREM: 2035 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2036 const APInt &RA = Rem->getAPIntValue().abs(); 2037 if (RA.isPowerOf2()) { 2038 APInt LowBits = RA - 1; 2039 APInt Mask2 = LowBits | APInt::getSignBit(BitWidth); 2040 ComputeMaskedBits(Op.getOperand(0), Mask2,KnownZero2,KnownOne2,Depth+1); 2041 2042 // The low bits of the first operand are unchanged by the srem. 2043 KnownZero = KnownZero2 & LowBits; 2044 KnownOne = KnownOne2 & LowBits; 2045 2046 // If the first operand is non-negative or has all low bits zero, then 2047 // the upper bits are all zero. 2048 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2049 KnownZero |= ~LowBits; 2050 2051 // If the first operand is negative and not all low bits are zero, then 2052 // the upper bits are all one. 2053 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2054 KnownOne |= ~LowBits; 2055 2056 KnownZero &= Mask; 2057 KnownOne &= Mask; 2058 2059 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2060 } 2061 } 2062 return; 2063 case ISD::UREM: { 2064 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2065 const APInt &RA = Rem->getAPIntValue(); 2066 if (RA.isPowerOf2()) { 2067 APInt LowBits = (RA - 1); 2068 APInt Mask2 = LowBits & Mask; 2069 KnownZero |= ~LowBits & Mask; 2070 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero, KnownOne,Depth+1); 2071 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2072 break; 2073 } 2074 } 2075 2076 // Since the result is less than or equal to either operand, any leading 2077 // zero bits in either operand must also exist in the result. 2078 APInt AllOnes = APInt::getAllOnesValue(BitWidth); 2079 ComputeMaskedBits(Op.getOperand(0), AllOnes, KnownZero, KnownOne, 2080 Depth+1); 2081 ComputeMaskedBits(Op.getOperand(1), AllOnes, KnownZero2, KnownOne2, 2082 Depth+1); 2083 2084 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2085 KnownZero2.countLeadingOnes()); 2086 KnownOne.clearAllBits(); 2087 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders) & Mask; 2088 return; 2089 } 2090 case ISD::FrameIndex: 2091 case ISD::TargetFrameIndex: 2092 if (unsigned Align = InferPtrAlignment(Op)) { 2093 // The low bits are known zero if the pointer is aligned. 2094 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2095 return; 2096 } 2097 break; 2098 2099 default: 2100 if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2101 break; 2102 // Fallthrough 2103 case ISD::INTRINSIC_WO_CHAIN: 2104 case ISD::INTRINSIC_W_CHAIN: 2105 case ISD::INTRINSIC_VOID: 2106 // Allow the target to implement this method for its nodes. 2107 TLI.computeMaskedBitsForTargetNode(Op, Mask, KnownZero, KnownOne, *this, 2108 Depth); 2109 return; 2110 } 2111} 2112 2113/// ComputeNumSignBits - Return the number of times the sign bit of the 2114/// register is replicated into the other bits. We know that at least 1 bit 2115/// is always equal to the sign bit (itself), but other cases can give us 2116/// information. For example, immediately after an "SRA X, 2", we know that 2117/// the top 3 bits are all equal to each other, so we return 3. 2118unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2119 EVT VT = Op.getValueType(); 2120 assert(VT.isInteger() && "Invalid VT!"); 2121 unsigned VTBits = VT.getScalarType().getSizeInBits(); 2122 unsigned Tmp, Tmp2; 2123 unsigned FirstAnswer = 1; 2124 2125 if (Depth == 6) 2126 return 1; // Limit search depth. 2127 2128 switch (Op.getOpcode()) { 2129 default: break; 2130 case ISD::AssertSext: 2131 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2132 return VTBits-Tmp+1; 2133 case ISD::AssertZext: 2134 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2135 return VTBits-Tmp; 2136 2137 case ISD::Constant: { 2138 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2139 return Val.getNumSignBits(); 2140 } 2141 2142 case ISD::SIGN_EXTEND: 2143 Tmp = VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2144 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2145 2146 case ISD::SIGN_EXTEND_INREG: 2147 // Max of the input and what this extends. 2148 Tmp = 2149 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2150 Tmp = VTBits-Tmp+1; 2151 2152 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2153 return std::max(Tmp, Tmp2); 2154 2155 case ISD::SRA: 2156 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2157 // SRA X, C -> adds C sign bits. 2158 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2159 Tmp += C->getZExtValue(); 2160 if (Tmp > VTBits) Tmp = VTBits; 2161 } 2162 return Tmp; 2163 case ISD::SHL: 2164 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2165 // shl destroys sign bits. 2166 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2167 if (C->getZExtValue() >= VTBits || // Bad shift. 2168 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2169 return Tmp - C->getZExtValue(); 2170 } 2171 break; 2172 case ISD::AND: 2173 case ISD::OR: 2174 case ISD::XOR: // NOT is handled here. 2175 // Logical binary ops preserve the number of sign bits at the worst. 2176 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2177 if (Tmp != 1) { 2178 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2179 FirstAnswer = std::min(Tmp, Tmp2); 2180 // We computed what we know about the sign bits as our first 2181 // answer. Now proceed to the generic code that uses 2182 // ComputeMaskedBits, and pick whichever answer is better. 2183 } 2184 break; 2185 2186 case ISD::SELECT: 2187 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2188 if (Tmp == 1) return 1; // Early out. 2189 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2190 return std::min(Tmp, Tmp2); 2191 2192 case ISD::SADDO: 2193 case ISD::UADDO: 2194 case ISD::SSUBO: 2195 case ISD::USUBO: 2196 case ISD::SMULO: 2197 case ISD::UMULO: 2198 if (Op.getResNo() != 1) 2199 break; 2200 // The boolean result conforms to getBooleanContents. Fall through. 2201 case ISD::SETCC: 2202 // If setcc returns 0/-1, all bits are sign bits. 2203 if (TLI.getBooleanContents(Op.getValueType().isVector()) == 2204 TargetLowering::ZeroOrNegativeOneBooleanContent) 2205 return VTBits; 2206 break; 2207 case ISD::ROTL: 2208 case ISD::ROTR: 2209 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2210 unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2211 2212 // Handle rotate right by N like a rotate left by 32-N. 2213 if (Op.getOpcode() == ISD::ROTR) 2214 RotAmt = (VTBits-RotAmt) & (VTBits-1); 2215 2216 // If we aren't rotating out all of the known-in sign bits, return the 2217 // number that are left. This handles rotl(sext(x), 1) for example. 2218 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2219 if (Tmp > RotAmt+1) return Tmp-RotAmt; 2220 } 2221 break; 2222 case ISD::ADD: 2223 // Add can have at most one carry bit. Thus we know that the output 2224 // is, at worst, one more bit than the inputs. 2225 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2226 if (Tmp == 1) return 1; // Early out. 2227 2228 // Special case decrementing a value (ADD X, -1): 2229 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2230 if (CRHS->isAllOnesValue()) { 2231 APInt KnownZero, KnownOne; 2232 APInt Mask = APInt::getAllOnesValue(VTBits); 2233 ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1); 2234 2235 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2236 // sign bits set. 2237 if ((KnownZero | APInt(VTBits, 1)) == Mask) 2238 return VTBits; 2239 2240 // If we are subtracting one from a positive number, there is no carry 2241 // out of the result. 2242 if (KnownZero.isNegative()) 2243 return Tmp; 2244 } 2245 2246 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2247 if (Tmp2 == 1) return 1; 2248 return std::min(Tmp, Tmp2)-1; 2249 break; 2250 2251 case ISD::SUB: 2252 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2253 if (Tmp2 == 1) return 1; 2254 2255 // Handle NEG. 2256 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2257 if (CLHS->isNullValue()) { 2258 APInt KnownZero, KnownOne; 2259 APInt Mask = APInt::getAllOnesValue(VTBits); 2260 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 2261 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2262 // sign bits set. 2263 if ((KnownZero | APInt(VTBits, 1)) == Mask) 2264 return VTBits; 2265 2266 // If the input is known to be positive (the sign bit is known clear), 2267 // the output of the NEG has the same number of sign bits as the input. 2268 if (KnownZero.isNegative()) 2269 return Tmp2; 2270 2271 // Otherwise, we treat this like a SUB. 2272 } 2273 2274 // Sub can have at most one carry bit. Thus we know that the output 2275 // is, at worst, one more bit than the inputs. 2276 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2277 if (Tmp == 1) return 1; // Early out. 2278 return std::min(Tmp, Tmp2)-1; 2279 break; 2280 case ISD::TRUNCATE: 2281 // FIXME: it's tricky to do anything useful for this, but it is an important 2282 // case for targets like X86. 2283 break; 2284 } 2285 2286 // Handle LOADX separately here. EXTLOAD case will fallthrough. 2287 if (Op.getOpcode() == ISD::LOAD) { 2288 LoadSDNode *LD = cast<LoadSDNode>(Op); 2289 unsigned ExtType = LD->getExtensionType(); 2290 switch (ExtType) { 2291 default: break; 2292 case ISD::SEXTLOAD: // '17' bits known 2293 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2294 return VTBits-Tmp+1; 2295 case ISD::ZEXTLOAD: // '16' bits known 2296 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2297 return VTBits-Tmp; 2298 } 2299 } 2300 2301 // Allow the target to implement this method for its nodes. 2302 if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2303 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2304 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2305 Op.getOpcode() == ISD::INTRINSIC_VOID) { 2306 unsigned NumBits = TLI.ComputeNumSignBitsForTargetNode(Op, Depth); 2307 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2308 } 2309 2310 // Finally, if we can prove that the top bits of the result are 0's or 1's, 2311 // use this information. 2312 APInt KnownZero, KnownOne; 2313 APInt Mask = APInt::getAllOnesValue(VTBits); 2314 ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth); 2315 2316 if (KnownZero.isNegative()) { // sign bit is 0 2317 Mask = KnownZero; 2318 } else if (KnownOne.isNegative()) { // sign bit is 1; 2319 Mask = KnownOne; 2320 } else { 2321 // Nothing known. 2322 return FirstAnswer; 2323 } 2324 2325 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2326 // the number of identical bits in the top of the input value. 2327 Mask = ~Mask; 2328 Mask <<= Mask.getBitWidth()-VTBits; 2329 // Return # leading zeros. We use 'min' here in case Val was zero before 2330 // shifting. We don't want to return '64' as for an i32 "0". 2331 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2332} 2333 2334/// isBaseWithConstantOffset - Return true if the specified operand is an 2335/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2336/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2337/// semantics as an ADD. This handles the equivalence: 2338/// X|Cst == X+Cst iff X&Cst = 0. 2339bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2340 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2341 !isa<ConstantSDNode>(Op.getOperand(1))) 2342 return false; 2343 2344 if (Op.getOpcode() == ISD::OR && 2345 !MaskedValueIsZero(Op.getOperand(0), 2346 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2347 return false; 2348 2349 return true; 2350} 2351 2352 2353bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2354 // If we're told that NaNs won't happen, assume they won't. 2355 if (getTarget().Options.NoNaNsFPMath) 2356 return true; 2357 2358 // If the value is a constant, we can obviously see if it is a NaN or not. 2359 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2360 return !C->getValueAPF().isNaN(); 2361 2362 // TODO: Recognize more cases here. 2363 2364 return false; 2365} 2366 2367bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2368 // If the value is a constant, we can obviously see if it is a zero or not. 2369 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2370 return !C->isZero(); 2371 2372 // TODO: Recognize more cases here. 2373 switch (Op.getOpcode()) { 2374 default: break; 2375 case ISD::OR: 2376 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2377 return !C->isNullValue(); 2378 break; 2379 } 2380 2381 return false; 2382} 2383 2384bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2385 // Check the obvious case. 2386 if (A == B) return true; 2387 2388 // For for negative and positive zero. 2389 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2390 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2391 if (CA->isZero() && CB->isZero()) return true; 2392 2393 // Otherwise they may not be equal. 2394 return false; 2395} 2396 2397/// getNode - Gets or creates the specified node. 2398/// 2399SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) { 2400 FoldingSetNodeID ID; 2401 AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2402 void *IP = 0; 2403 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2404 return SDValue(E, 0); 2405 2406 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL, getVTList(VT)); 2407 CSEMap.InsertNode(N, IP); 2408 2409 AllNodes.push_back(N); 2410#ifndef NDEBUG 2411 VerifySDNode(N); 2412#endif 2413 return SDValue(N, 0); 2414} 2415 2416SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 2417 EVT VT, SDValue Operand) { 2418 // Constant fold unary operations with an integer constant operand. 2419 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2420 const APInt &Val = C->getAPIntValue(); 2421 switch (Opcode) { 2422 default: break; 2423 case ISD::SIGN_EXTEND: 2424 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2425 case ISD::ANY_EXTEND: 2426 case ISD::ZERO_EXTEND: 2427 case ISD::TRUNCATE: 2428 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2429 case ISD::UINT_TO_FP: 2430 case ISD::SINT_TO_FP: { 2431 // No compile time operations on ppcf128. 2432 if (VT == MVT::ppcf128) break; 2433 APFloat apf(APInt::getNullValue(VT.getSizeInBits())); 2434 (void)apf.convertFromAPInt(Val, 2435 Opcode==ISD::SINT_TO_FP, 2436 APFloat::rmNearestTiesToEven); 2437 return getConstantFP(apf, VT); 2438 } 2439 case ISD::BITCAST: 2440 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2441 return getConstantFP(Val.bitsToFloat(), VT); 2442 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2443 return getConstantFP(Val.bitsToDouble(), VT); 2444 break; 2445 case ISD::BSWAP: 2446 return getConstant(Val.byteSwap(), VT); 2447 case ISD::CTPOP: 2448 return getConstant(Val.countPopulation(), VT); 2449 case ISD::CTLZ: 2450 case ISD::CTLZ_ZERO_UNDEF: 2451 return getConstant(Val.countLeadingZeros(), VT); 2452 case ISD::CTTZ: 2453 case ISD::CTTZ_ZERO_UNDEF: 2454 return getConstant(Val.countTrailingZeros(), VT); 2455 } 2456 } 2457 2458 // Constant fold unary operations with a floating point constant operand. 2459 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2460 APFloat V = C->getValueAPF(); // make copy 2461 if (VT != MVT::ppcf128 && Operand.getValueType() != MVT::ppcf128) { 2462 switch (Opcode) { 2463 case ISD::FNEG: 2464 V.changeSign(); 2465 return getConstantFP(V, VT); 2466 case ISD::FABS: 2467 V.clearSign(); 2468 return getConstantFP(V, VT); 2469 case ISD::FP_ROUND: 2470 case ISD::FP_EXTEND: { 2471 bool ignored; 2472 // This can return overflow, underflow, or inexact; we don't care. 2473 // FIXME need to be more flexible about rounding mode. 2474 (void)V.convert(*EVTToAPFloatSemantics(VT), 2475 APFloat::rmNearestTiesToEven, &ignored); 2476 return getConstantFP(V, VT); 2477 } 2478 case ISD::FP_TO_SINT: 2479 case ISD::FP_TO_UINT: { 2480 integerPart x[2]; 2481 bool ignored; 2482 assert(integerPartWidth >= 64); 2483 // FIXME need to be more flexible about rounding mode. 2484 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2485 Opcode==ISD::FP_TO_SINT, 2486 APFloat::rmTowardZero, &ignored); 2487 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2488 break; 2489 APInt api(VT.getSizeInBits(), x); 2490 return getConstant(api, VT); 2491 } 2492 case ISD::BITCAST: 2493 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2494 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2495 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2496 return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2497 break; 2498 } 2499 } 2500 } 2501 2502 unsigned OpOpcode = Operand.getNode()->getOpcode(); 2503 switch (Opcode) { 2504 case ISD::TokenFactor: 2505 case ISD::MERGE_VALUES: 2506 case ISD::CONCAT_VECTORS: 2507 return Operand; // Factor, merge or concat of one node? No need. 2508 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2509 case ISD::FP_EXTEND: 2510 assert(VT.isFloatingPoint() && 2511 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2512 if (Operand.getValueType() == VT) return Operand; // noop conversion. 2513 assert((!VT.isVector() || 2514 VT.getVectorNumElements() == 2515 Operand.getValueType().getVectorNumElements()) && 2516 "Vector element count mismatch!"); 2517 if (Operand.getOpcode() == ISD::UNDEF) 2518 return getUNDEF(VT); 2519 break; 2520 case ISD::SIGN_EXTEND: 2521 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2522 "Invalid SIGN_EXTEND!"); 2523 if (Operand.getValueType() == VT) return Operand; // noop extension 2524 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2525 "Invalid sext node, dst < src!"); 2526 assert((!VT.isVector() || 2527 VT.getVectorNumElements() == 2528 Operand.getValueType().getVectorNumElements()) && 2529 "Vector element count mismatch!"); 2530 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2531 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2532 else if (OpOpcode == ISD::UNDEF) 2533 // sext(undef) = 0, because the top bits will all be the same. 2534 return getConstant(0, VT); 2535 break; 2536 case ISD::ZERO_EXTEND: 2537 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2538 "Invalid ZERO_EXTEND!"); 2539 if (Operand.getValueType() == VT) return Operand; // noop extension 2540 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2541 "Invalid zext node, dst < src!"); 2542 assert((!VT.isVector() || 2543 VT.getVectorNumElements() == 2544 Operand.getValueType().getVectorNumElements()) && 2545 "Vector element count mismatch!"); 2546 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2547 return getNode(ISD::ZERO_EXTEND, DL, VT, 2548 Operand.getNode()->getOperand(0)); 2549 else if (OpOpcode == ISD::UNDEF) 2550 // zext(undef) = 0, because the top bits will be zero. 2551 return getConstant(0, VT); 2552 break; 2553 case ISD::ANY_EXTEND: 2554 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2555 "Invalid ANY_EXTEND!"); 2556 if (Operand.getValueType() == VT) return Operand; // noop extension 2557 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2558 "Invalid anyext node, dst < src!"); 2559 assert((!VT.isVector() || 2560 VT.getVectorNumElements() == 2561 Operand.getValueType().getVectorNumElements()) && 2562 "Vector element count mismatch!"); 2563 2564 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2565 OpOpcode == ISD::ANY_EXTEND) 2566 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2567 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2568 else if (OpOpcode == ISD::UNDEF) 2569 return getUNDEF(VT); 2570 2571 // (ext (trunx x)) -> x 2572 if (OpOpcode == ISD::TRUNCATE) { 2573 SDValue OpOp = Operand.getNode()->getOperand(0); 2574 if (OpOp.getValueType() == VT) 2575 return OpOp; 2576 } 2577 break; 2578 case ISD::TRUNCATE: 2579 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2580 "Invalid TRUNCATE!"); 2581 if (Operand.getValueType() == VT) return Operand; // noop truncate 2582 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2583 "Invalid truncate node, src < dst!"); 2584 assert((!VT.isVector() || 2585 VT.getVectorNumElements() == 2586 Operand.getValueType().getVectorNumElements()) && 2587 "Vector element count mismatch!"); 2588 if (OpOpcode == ISD::TRUNCATE) 2589 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2590 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2591 OpOpcode == ISD::ANY_EXTEND) { 2592 // If the source is smaller than the dest, we still need an extend. 2593 if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2594 .bitsLT(VT.getScalarType())) 2595 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2596 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2597 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2598 return Operand.getNode()->getOperand(0); 2599 } 2600 if (OpOpcode == ISD::UNDEF) 2601 return getUNDEF(VT); 2602 break; 2603 case ISD::BITCAST: 2604 // Basic sanity checking. 2605 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2606 && "Cannot BITCAST between types of different sizes!"); 2607 if (VT == Operand.getValueType()) return Operand; // noop conversion. 2608 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2609 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2610 if (OpOpcode == ISD::UNDEF) 2611 return getUNDEF(VT); 2612 break; 2613 case ISD::SCALAR_TO_VECTOR: 2614 assert(VT.isVector() && !Operand.getValueType().isVector() && 2615 (VT.getVectorElementType() == Operand.getValueType() || 2616 (VT.getVectorElementType().isInteger() && 2617 Operand.getValueType().isInteger() && 2618 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2619 "Illegal SCALAR_TO_VECTOR node!"); 2620 if (OpOpcode == ISD::UNDEF) 2621 return getUNDEF(VT); 2622 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2623 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2624 isa<ConstantSDNode>(Operand.getOperand(1)) && 2625 Operand.getConstantOperandVal(1) == 0 && 2626 Operand.getOperand(0).getValueType() == VT) 2627 return Operand.getOperand(0); 2628 break; 2629 case ISD::FNEG: 2630 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2631 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 2632 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2633 Operand.getNode()->getOperand(0)); 2634 if (OpOpcode == ISD::FNEG) // --X -> X 2635 return Operand.getNode()->getOperand(0); 2636 break; 2637 case ISD::FABS: 2638 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2639 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2640 break; 2641 } 2642 2643 SDNode *N; 2644 SDVTList VTs = getVTList(VT); 2645 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2646 FoldingSetNodeID ID; 2647 SDValue Ops[1] = { Operand }; 2648 AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2649 void *IP = 0; 2650 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2651 return SDValue(E, 0); 2652 2653 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); 2654 CSEMap.InsertNode(N, IP); 2655 } else { 2656 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); 2657 } 2658 2659 AllNodes.push_back(N); 2660#ifndef NDEBUG 2661 VerifySDNode(N); 2662#endif 2663 return SDValue(N, 0); 2664} 2665 2666SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, 2667 EVT VT, 2668 ConstantSDNode *Cst1, 2669 ConstantSDNode *Cst2) { 2670 const APInt &C1 = Cst1->getAPIntValue(), &C2 = Cst2->getAPIntValue(); 2671 2672 switch (Opcode) { 2673 case ISD::ADD: return getConstant(C1 + C2, VT); 2674 case ISD::SUB: return getConstant(C1 - C2, VT); 2675 case ISD::MUL: return getConstant(C1 * C2, VT); 2676 case ISD::UDIV: 2677 if (C2.getBoolValue()) return getConstant(C1.udiv(C2), VT); 2678 break; 2679 case ISD::UREM: 2680 if (C2.getBoolValue()) return getConstant(C1.urem(C2), VT); 2681 break; 2682 case ISD::SDIV: 2683 if (C2.getBoolValue()) return getConstant(C1.sdiv(C2), VT); 2684 break; 2685 case ISD::SREM: 2686 if (C2.getBoolValue()) return getConstant(C1.srem(C2), VT); 2687 break; 2688 case ISD::AND: return getConstant(C1 & C2, VT); 2689 case ISD::OR: return getConstant(C1 | C2, VT); 2690 case ISD::XOR: return getConstant(C1 ^ C2, VT); 2691 case ISD::SHL: return getConstant(C1 << C2, VT); 2692 case ISD::SRL: return getConstant(C1.lshr(C2), VT); 2693 case ISD::SRA: return getConstant(C1.ashr(C2), VT); 2694 case ISD::ROTL: return getConstant(C1.rotl(C2), VT); 2695 case ISD::ROTR: return getConstant(C1.rotr(C2), VT); 2696 default: break; 2697 } 2698 2699 return SDValue(); 2700} 2701 2702SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 2703 SDValue N1, SDValue N2) { 2704 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2705 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2706 switch (Opcode) { 2707 default: break; 2708 case ISD::TokenFactor: 2709 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2710 N2.getValueType() == MVT::Other && "Invalid token factor!"); 2711 // Fold trivial token factors. 2712 if (N1.getOpcode() == ISD::EntryToken) return N2; 2713 if (N2.getOpcode() == ISD::EntryToken) return N1; 2714 if (N1 == N2) return N1; 2715 break; 2716 case ISD::CONCAT_VECTORS: 2717 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2718 // one big BUILD_VECTOR. 2719 if (N1.getOpcode() == ISD::BUILD_VECTOR && 2720 N2.getOpcode() == ISD::BUILD_VECTOR) { 2721 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2722 N1.getNode()->op_end()); 2723 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2724 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2725 } 2726 break; 2727 case ISD::AND: 2728 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2729 assert(N1.getValueType() == N2.getValueType() && 2730 N1.getValueType() == VT && "Binary operator types must match!"); 2731 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2732 // worth handling here. 2733 if (N2C && N2C->isNullValue()) 2734 return N2; 2735 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2736 return N1; 2737 break; 2738 case ISD::OR: 2739 case ISD::XOR: 2740 case ISD::ADD: 2741 case ISD::SUB: 2742 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2743 assert(N1.getValueType() == N2.getValueType() && 2744 N1.getValueType() == VT && "Binary operator types must match!"); 2745 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2746 // it's worth handling here. 2747 if (N2C && N2C->isNullValue()) 2748 return N1; 2749 break; 2750 case ISD::UDIV: 2751 case ISD::UREM: 2752 case ISD::MULHU: 2753 case ISD::MULHS: 2754 case ISD::MUL: 2755 case ISD::SDIV: 2756 case ISD::SREM: 2757 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2758 assert(N1.getValueType() == N2.getValueType() && 2759 N1.getValueType() == VT && "Binary operator types must match!"); 2760 break; 2761 case ISD::FADD: 2762 case ISD::FSUB: 2763 case ISD::FMUL: 2764 case ISD::FDIV: 2765 case ISD::FREM: 2766 if (getTarget().Options.UnsafeFPMath) { 2767 if (Opcode == ISD::FADD) { 2768 // 0+x --> x 2769 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2770 if (CFP->getValueAPF().isZero()) 2771 return N2; 2772 // x+0 --> x 2773 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2774 if (CFP->getValueAPF().isZero()) 2775 return N1; 2776 } else if (Opcode == ISD::FSUB) { 2777 // x-0 --> x 2778 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2779 if (CFP->getValueAPF().isZero()) 2780 return N1; 2781 } 2782 } 2783 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2784 assert(N1.getValueType() == N2.getValueType() && 2785 N1.getValueType() == VT && "Binary operator types must match!"); 2786 break; 2787 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2788 assert(N1.getValueType() == VT && 2789 N1.getValueType().isFloatingPoint() && 2790 N2.getValueType().isFloatingPoint() && 2791 "Invalid FCOPYSIGN!"); 2792 break; 2793 case ISD::SHL: 2794 case ISD::SRA: 2795 case ISD::SRL: 2796 case ISD::ROTL: 2797 case ISD::ROTR: 2798 assert(VT == N1.getValueType() && 2799 "Shift operators return type must be the same as their first arg"); 2800 assert(VT.isInteger() && N2.getValueType().isInteger() && 2801 "Shifts only work on integers"); 2802 // Verify that the shift amount VT is bit enough to hold valid shift 2803 // amounts. This catches things like trying to shift an i1024 value by an 2804 // i8, which is easy to fall into in generic code that uses 2805 // TLI.getShiftAmount(). 2806 assert(N2.getValueType().getSizeInBits() >= 2807 Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 2808 "Invalid use of small shift amount with oversized value!"); 2809 2810 // Always fold shifts of i1 values so the code generator doesn't need to 2811 // handle them. Since we know the size of the shift has to be less than the 2812 // size of the value, the shift/rotate count is guaranteed to be zero. 2813 if (VT == MVT::i1) 2814 return N1; 2815 if (N2C && N2C->isNullValue()) 2816 return N1; 2817 break; 2818 case ISD::FP_ROUND_INREG: { 2819 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2820 assert(VT == N1.getValueType() && "Not an inreg round!"); 2821 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 2822 "Cannot FP_ROUND_INREG integer types"); 2823 assert(EVT.isVector() == VT.isVector() && 2824 "FP_ROUND_INREG type should be vector iff the operand " 2825 "type is vector!"); 2826 assert((!EVT.isVector() || 2827 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2828 "Vector element counts must match in FP_ROUND_INREG"); 2829 assert(EVT.bitsLE(VT) && "Not rounding down!"); 2830 (void)EVT; 2831 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 2832 break; 2833 } 2834 case ISD::FP_ROUND: 2835 assert(VT.isFloatingPoint() && 2836 N1.getValueType().isFloatingPoint() && 2837 VT.bitsLE(N1.getValueType()) && 2838 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 2839 if (N1.getValueType() == VT) return N1; // noop conversion. 2840 break; 2841 case ISD::AssertSext: 2842 case ISD::AssertZext: { 2843 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2844 assert(VT == N1.getValueType() && "Not an inreg extend!"); 2845 assert(VT.isInteger() && EVT.isInteger() && 2846 "Cannot *_EXTEND_INREG FP types"); 2847 assert(!EVT.isVector() && 2848 "AssertSExt/AssertZExt type should be the vector element type " 2849 "rather than the vector type!"); 2850 assert(EVT.bitsLE(VT) && "Not extending!"); 2851 if (VT == EVT) return N1; // noop assertion. 2852 break; 2853 } 2854 case ISD::SIGN_EXTEND_INREG: { 2855 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2856 assert(VT == N1.getValueType() && "Not an inreg extend!"); 2857 assert(VT.isInteger() && EVT.isInteger() && 2858 "Cannot *_EXTEND_INREG FP types"); 2859 assert(EVT.isVector() == VT.isVector() && 2860 "SIGN_EXTEND_INREG type should be vector iff the operand " 2861 "type is vector!"); 2862 assert((!EVT.isVector() || 2863 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2864 "Vector element counts must match in SIGN_EXTEND_INREG"); 2865 assert(EVT.bitsLE(VT) && "Not extending!"); 2866 if (EVT == VT) return N1; // Not actually extending 2867 2868 if (N1C) { 2869 APInt Val = N1C->getAPIntValue(); 2870 unsigned FromBits = EVT.getScalarType().getSizeInBits(); 2871 Val <<= Val.getBitWidth()-FromBits; 2872 Val = Val.ashr(Val.getBitWidth()-FromBits); 2873 return getConstant(Val, VT); 2874 } 2875 break; 2876 } 2877 case ISD::EXTRACT_VECTOR_ELT: 2878 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 2879 if (N1.getOpcode() == ISD::UNDEF) 2880 return getUNDEF(VT); 2881 2882 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 2883 // expanding copies of large vectors from registers. 2884 if (N2C && 2885 N1.getOpcode() == ISD::CONCAT_VECTORS && 2886 N1.getNumOperands() > 0) { 2887 unsigned Factor = 2888 N1.getOperand(0).getValueType().getVectorNumElements(); 2889 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 2890 N1.getOperand(N2C->getZExtValue() / Factor), 2891 getConstant(N2C->getZExtValue() % Factor, 2892 N2.getValueType())); 2893 } 2894 2895 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 2896 // expanding large vector constants. 2897 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 2898 SDValue Elt = N1.getOperand(N2C->getZExtValue()); 2899 EVT VEltTy = N1.getValueType().getVectorElementType(); 2900 if (Elt.getValueType() != VEltTy) { 2901 // If the vector element type is not legal, the BUILD_VECTOR operands 2902 // are promoted and implicitly truncated. Make that explicit here. 2903 Elt = getNode(ISD::TRUNCATE, DL, VEltTy, Elt); 2904 } 2905 if (VT != VEltTy) { 2906 // If the vector element type is not legal, the EXTRACT_VECTOR_ELT 2907 // result is implicitly extended. 2908 Elt = getNode(ISD::ANY_EXTEND, DL, VT, Elt); 2909 } 2910 return Elt; 2911 } 2912 2913 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 2914 // operations are lowered to scalars. 2915 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 2916 // If the indices are the same, return the inserted element else 2917 // if the indices are known different, extract the element from 2918 // the original vector. 2919 SDValue N1Op2 = N1.getOperand(2); 2920 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 2921 2922 if (N1Op2C && N2C) { 2923 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 2924 if (VT == N1.getOperand(1).getValueType()) 2925 return N1.getOperand(1); 2926 else 2927 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 2928 } 2929 2930 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 2931 } 2932 } 2933 break; 2934 case ISD::EXTRACT_ELEMENT: 2935 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 2936 assert(!N1.getValueType().isVector() && !VT.isVector() && 2937 (N1.getValueType().isInteger() == VT.isInteger()) && 2938 N1.getValueType() != VT && 2939 "Wrong types for EXTRACT_ELEMENT!"); 2940 2941 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 2942 // 64-bit integers into 32-bit parts. Instead of building the extract of 2943 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 2944 if (N1.getOpcode() == ISD::BUILD_PAIR) 2945 return N1.getOperand(N2C->getZExtValue()); 2946 2947 // EXTRACT_ELEMENT of a constant int is also very common. 2948 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 2949 unsigned ElementSize = VT.getSizeInBits(); 2950 unsigned Shift = ElementSize * N2C->getZExtValue(); 2951 APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 2952 return getConstant(ShiftedVal.trunc(ElementSize), VT); 2953 } 2954 break; 2955 case ISD::EXTRACT_SUBVECTOR: { 2956 SDValue Index = N2; 2957 if (VT.isSimple() && N1.getValueType().isSimple()) { 2958 assert(VT.isVector() && N1.getValueType().isVector() && 2959 "Extract subvector VTs must be a vectors!"); 2960 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() && 2961 "Extract subvector VTs must have the same element type!"); 2962 assert(VT.getSimpleVT() <= N1.getValueType().getSimpleVT() && 2963 "Extract subvector must be from larger vector to smaller vector!"); 2964 2965 if (isa<ConstantSDNode>(Index.getNode())) { 2966 assert((VT.getVectorNumElements() + 2967 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 2968 <= N1.getValueType().getVectorNumElements()) 2969 && "Extract subvector overflow!"); 2970 } 2971 2972 // Trivial extraction. 2973 if (VT.getSimpleVT() == N1.getValueType().getSimpleVT()) 2974 return N1; 2975 } 2976 break; 2977 } 2978 } 2979 2980 if (N1C) { 2981 if (N2C) { 2982 SDValue SV = FoldConstantArithmetic(Opcode, VT, N1C, N2C); 2983 if (SV.getNode()) return SV; 2984 } else { // Cannonicalize constant to RHS if commutative 2985 if (isCommutativeBinOp(Opcode)) { 2986 std::swap(N1C, N2C); 2987 std::swap(N1, N2); 2988 } 2989 } 2990 } 2991 2992 // Constant fold FP operations. 2993 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 2994 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 2995 if (N1CFP) { 2996 if (!N2CFP && isCommutativeBinOp(Opcode)) { 2997 // Cannonicalize constant to RHS if commutative 2998 std::swap(N1CFP, N2CFP); 2999 std::swap(N1, N2); 3000 } else if (N2CFP && VT != MVT::ppcf128) { 3001 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 3002 APFloat::opStatus s; 3003 switch (Opcode) { 3004 case ISD::FADD: 3005 s = V1.add(V2, APFloat::rmNearestTiesToEven); 3006 if (s != APFloat::opInvalidOp) 3007 return getConstantFP(V1, VT); 3008 break; 3009 case ISD::FSUB: 3010 s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 3011 if (s!=APFloat::opInvalidOp) 3012 return getConstantFP(V1, VT); 3013 break; 3014 case ISD::FMUL: 3015 s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 3016 if (s!=APFloat::opInvalidOp) 3017 return getConstantFP(V1, VT); 3018 break; 3019 case ISD::FDIV: 3020 s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3021 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3022 return getConstantFP(V1, VT); 3023 break; 3024 case ISD::FREM : 3025 s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3026 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3027 return getConstantFP(V1, VT); 3028 break; 3029 case ISD::FCOPYSIGN: 3030 V1.copySign(V2); 3031 return getConstantFP(V1, VT); 3032 default: break; 3033 } 3034 } 3035 } 3036 3037 // Canonicalize an UNDEF to the RHS, even over a constant. 3038 if (N1.getOpcode() == ISD::UNDEF) { 3039 if (isCommutativeBinOp(Opcode)) { 3040 std::swap(N1, N2); 3041 } else { 3042 switch (Opcode) { 3043 case ISD::FP_ROUND_INREG: 3044 case ISD::SIGN_EXTEND_INREG: 3045 case ISD::SUB: 3046 case ISD::FSUB: 3047 case ISD::FDIV: 3048 case ISD::FREM: 3049 case ISD::SRA: 3050 return N1; // fold op(undef, arg2) -> undef 3051 case ISD::UDIV: 3052 case ISD::SDIV: 3053 case ISD::UREM: 3054 case ISD::SREM: 3055 case ISD::SRL: 3056 case ISD::SHL: 3057 if (!VT.isVector()) 3058 return getConstant(0, VT); // fold op(undef, arg2) -> 0 3059 // For vectors, we can't easily build an all zero vector, just return 3060 // the LHS. 3061 return N2; 3062 } 3063 } 3064 } 3065 3066 // Fold a bunch of operators when the RHS is undef. 3067 if (N2.getOpcode() == ISD::UNDEF) { 3068 switch (Opcode) { 3069 case ISD::XOR: 3070 if (N1.getOpcode() == ISD::UNDEF) 3071 // Handle undef ^ undef -> 0 special case. This is a common 3072 // idiom (misuse). 3073 return getConstant(0, VT); 3074 // fallthrough 3075 case ISD::ADD: 3076 case ISD::ADDC: 3077 case ISD::ADDE: 3078 case ISD::SUB: 3079 case ISD::UDIV: 3080 case ISD::SDIV: 3081 case ISD::UREM: 3082 case ISD::SREM: 3083 return N2; // fold op(arg1, undef) -> undef 3084 case ISD::FADD: 3085 case ISD::FSUB: 3086 case ISD::FMUL: 3087 case ISD::FDIV: 3088 case ISD::FREM: 3089 if (getTarget().Options.UnsafeFPMath) 3090 return N2; 3091 break; 3092 case ISD::MUL: 3093 case ISD::AND: 3094 case ISD::SRL: 3095 case ISD::SHL: 3096 if (!VT.isVector()) 3097 return getConstant(0, VT); // fold op(arg1, undef) -> 0 3098 // For vectors, we can't easily build an all zero vector, just return 3099 // the LHS. 3100 return N1; 3101 case ISD::OR: 3102 if (!VT.isVector()) 3103 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3104 // For vectors, we can't easily build an all one vector, just return 3105 // the LHS. 3106 return N1; 3107 case ISD::SRA: 3108 return N1; 3109 } 3110 } 3111 3112 // Memoize this node if possible. 3113 SDNode *N; 3114 SDVTList VTs = getVTList(VT); 3115 if (VT != MVT::Glue) { 3116 SDValue Ops[] = { N1, N2 }; 3117 FoldingSetNodeID ID; 3118 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3119 void *IP = 0; 3120 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3121 return SDValue(E, 0); 3122 3123 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); 3124 CSEMap.InsertNode(N, IP); 3125 } else { 3126 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); 3127 } 3128 3129 AllNodes.push_back(N); 3130#ifndef NDEBUG 3131 VerifySDNode(N); 3132#endif 3133 return SDValue(N, 0); 3134} 3135 3136SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3137 SDValue N1, SDValue N2, SDValue N3) { 3138 // Perform various simplifications. 3139 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3140 switch (Opcode) { 3141 case ISD::CONCAT_VECTORS: 3142 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3143 // one big BUILD_VECTOR. 3144 if (N1.getOpcode() == ISD::BUILD_VECTOR && 3145 N2.getOpcode() == ISD::BUILD_VECTOR && 3146 N3.getOpcode() == ISD::BUILD_VECTOR) { 3147 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3148 N1.getNode()->op_end()); 3149 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3150 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3151 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3152 } 3153 break; 3154 case ISD::SETCC: { 3155 // Use FoldSetCC to simplify SETCC's. 3156 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3157 if (Simp.getNode()) return Simp; 3158 break; 3159 } 3160 case ISD::SELECT: 3161 if (N1C) { 3162 if (N1C->getZExtValue()) 3163 return N2; // select true, X, Y -> X 3164 else 3165 return N3; // select false, X, Y -> Y 3166 } 3167 3168 if (N2 == N3) return N2; // select C, X, X -> X 3169 break; 3170 case ISD::VECTOR_SHUFFLE: 3171 llvm_unreachable("should use getVectorShuffle constructor!"); 3172 break; 3173 case ISD::INSERT_SUBVECTOR: { 3174 SDValue Index = N3; 3175 if (VT.isSimple() && N1.getValueType().isSimple() 3176 && N2.getValueType().isSimple()) { 3177 assert(VT.isVector() && N1.getValueType().isVector() && 3178 N2.getValueType().isVector() && 3179 "Insert subvector VTs must be a vectors"); 3180 assert(VT == N1.getValueType() && 3181 "Dest and insert subvector source types must match!"); 3182 assert(N2.getValueType().getSimpleVT() <= N1.getValueType().getSimpleVT() && 3183 "Insert subvector must be from smaller vector to larger vector!"); 3184 if (isa<ConstantSDNode>(Index.getNode())) { 3185 assert((N2.getValueType().getVectorNumElements() + 3186 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3187 <= VT.getVectorNumElements()) 3188 && "Insert subvector overflow!"); 3189 } 3190 3191 // Trivial insertion. 3192 if (VT.getSimpleVT() == N2.getValueType().getSimpleVT()) 3193 return N2; 3194 } 3195 break; 3196 } 3197 case ISD::BITCAST: 3198 // Fold bit_convert nodes from a type to themselves. 3199 if (N1.getValueType() == VT) 3200 return N1; 3201 break; 3202 } 3203 3204 // Memoize node if it doesn't produce a flag. 3205 SDNode *N; 3206 SDVTList VTs = getVTList(VT); 3207 if (VT != MVT::Glue) { 3208 SDValue Ops[] = { N1, N2, N3 }; 3209 FoldingSetNodeID ID; 3210 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3211 void *IP = 0; 3212 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3213 return SDValue(E, 0); 3214 3215 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); 3216 CSEMap.InsertNode(N, IP); 3217 } else { 3218 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); 3219 } 3220 3221 AllNodes.push_back(N); 3222#ifndef NDEBUG 3223 VerifySDNode(N); 3224#endif 3225 return SDValue(N, 0); 3226} 3227 3228SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3229 SDValue N1, SDValue N2, SDValue N3, 3230 SDValue N4) { 3231 SDValue Ops[] = { N1, N2, N3, N4 }; 3232 return getNode(Opcode, DL, VT, Ops, 4); 3233} 3234 3235SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3236 SDValue N1, SDValue N2, SDValue N3, 3237 SDValue N4, SDValue N5) { 3238 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3239 return getNode(Opcode, DL, VT, Ops, 5); 3240} 3241 3242/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3243/// the incoming stack arguments to be loaded from the stack. 3244SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3245 SmallVector<SDValue, 8> ArgChains; 3246 3247 // Include the original chain at the beginning of the list. When this is 3248 // used by target LowerCall hooks, this helps legalize find the 3249 // CALLSEQ_BEGIN node. 3250 ArgChains.push_back(Chain); 3251 3252 // Add a chain value for each stack argument. 3253 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3254 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3255 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3256 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3257 if (FI->getIndex() < 0) 3258 ArgChains.push_back(SDValue(L, 1)); 3259 3260 // Build a tokenfactor for all the chains. 3261 return getNode(ISD::TokenFactor, Chain.getDebugLoc(), MVT::Other, 3262 &ArgChains[0], ArgChains.size()); 3263} 3264 3265/// SplatByte - Distribute ByteVal over NumBits bits. 3266static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) { 3267 APInt Val = APInt(NumBits, ByteVal); 3268 unsigned Shift = 8; 3269 for (unsigned i = NumBits; i > 8; i >>= 1) { 3270 Val = (Val << Shift) | Val; 3271 Shift <<= 1; 3272 } 3273 return Val; 3274} 3275 3276/// getMemsetValue - Vectorized representation of the memset value 3277/// operand. 3278static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3279 DebugLoc dl) { 3280 assert(Value.getOpcode() != ISD::UNDEF); 3281 3282 unsigned NumBits = VT.getScalarType().getSizeInBits(); 3283 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3284 APInt Val = SplatByte(NumBits, C->getZExtValue() & 255); 3285 if (VT.isInteger()) 3286 return DAG.getConstant(Val, VT); 3287 return DAG.getConstantFP(APFloat(Val), VT); 3288 } 3289 3290 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3291 if (NumBits > 8) { 3292 // Use a multiplication with 0x010101... to extend the input to the 3293 // required length. 3294 APInt Magic = SplatByte(NumBits, 0x01); 3295 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3296 } 3297 3298 return Value; 3299} 3300 3301/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3302/// used when a memcpy is turned into a memset when the source is a constant 3303/// string ptr. 3304static SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG, 3305 const TargetLowering &TLI, 3306 std::string &Str, unsigned Offset) { 3307 // Handle vector with all elements zero. 3308 if (Str.empty()) { 3309 if (VT.isInteger()) 3310 return DAG.getConstant(0, VT); 3311 else if (VT == MVT::f32 || VT == MVT::f64) 3312 return DAG.getConstantFP(0.0, VT); 3313 else if (VT.isVector()) { 3314 unsigned NumElts = VT.getVectorNumElements(); 3315 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3316 return DAG.getNode(ISD::BITCAST, dl, VT, 3317 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3318 EltVT, NumElts))); 3319 } else 3320 llvm_unreachable("Expected type!"); 3321 } 3322 3323 assert(!VT.isVector() && "Can't handle vector type here!"); 3324 unsigned NumBits = VT.getSizeInBits(); 3325 unsigned MSB = NumBits / 8; 3326 uint64_t Val = 0; 3327 if (TLI.isLittleEndian()) 3328 Offset = Offset + MSB - 1; 3329 for (unsigned i = 0; i != MSB; ++i) { 3330 Val = (Val << 8) | (unsigned char)Str[Offset]; 3331 Offset += TLI.isLittleEndian() ? -1 : 1; 3332 } 3333 return DAG.getConstant(Val, VT); 3334} 3335 3336/// getMemBasePlusOffset - Returns base and offset node for the 3337/// 3338static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, 3339 SelectionDAG &DAG) { 3340 EVT VT = Base.getValueType(); 3341 return DAG.getNode(ISD::ADD, Base.getDebugLoc(), 3342 VT, Base, DAG.getConstant(Offset, VT)); 3343} 3344 3345/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3346/// 3347static bool isMemSrcFromString(SDValue Src, std::string &Str) { 3348 unsigned SrcDelta = 0; 3349 GlobalAddressSDNode *G = NULL; 3350 if (Src.getOpcode() == ISD::GlobalAddress) 3351 G = cast<GlobalAddressSDNode>(Src); 3352 else if (Src.getOpcode() == ISD::ADD && 3353 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3354 Src.getOperand(1).getOpcode() == ISD::Constant) { 3355 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3356 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3357 } 3358 if (!G) 3359 return false; 3360 3361 const GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal()); 3362 if (GV && GetConstantStringInfo(GV, Str, SrcDelta, false)) 3363 return true; 3364 3365 return false; 3366} 3367 3368/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3369/// to replace the memset / memcpy. Return true if the number of memory ops 3370/// is below the threshold. It returns the types of the sequence of 3371/// memory ops to perform memset / memcpy by reference. 3372static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3373 unsigned Limit, uint64_t Size, 3374 unsigned DstAlign, unsigned SrcAlign, 3375 bool IsZeroVal, 3376 bool MemcpyStrSrc, 3377 SelectionDAG &DAG, 3378 const TargetLowering &TLI) { 3379 assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3380 "Expecting memcpy / memset source to meet alignment requirement!"); 3381 // If 'SrcAlign' is zero, that means the memory operation does not need to 3382 // load the value, i.e. memset or memcpy from constant string. Otherwise, 3383 // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3384 // is the specified alignment of the memory operation. If it is zero, that 3385 // means it's possible to change the alignment of the destination. 3386 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3387 // not need to be loaded. 3388 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3389 IsZeroVal, MemcpyStrSrc, 3390 DAG.getMachineFunction()); 3391 3392 if (VT == MVT::Other) { 3393 if (DstAlign >= TLI.getTargetData()->getPointerPrefAlignment() || 3394 TLI.allowsUnalignedMemoryAccesses(VT)) { 3395 VT = TLI.getPointerTy(); 3396 } else { 3397 switch (DstAlign & 7) { 3398 case 0: VT = MVT::i64; break; 3399 case 4: VT = MVT::i32; break; 3400 case 2: VT = MVT::i16; break; 3401 default: VT = MVT::i8; break; 3402 } 3403 } 3404 3405 MVT LVT = MVT::i64; 3406 while (!TLI.isTypeLegal(LVT)) 3407 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3408 assert(LVT.isInteger()); 3409 3410 if (VT.bitsGT(LVT)) 3411 VT = LVT; 3412 } 3413 3414 unsigned NumMemOps = 0; 3415 while (Size != 0) { 3416 unsigned VTSize = VT.getSizeInBits() / 8; 3417 while (VTSize > Size) { 3418 // For now, only use non-vector load / store's for the left-over pieces. 3419 if (VT.isVector() || VT.isFloatingPoint()) { 3420 VT = MVT::i64; 3421 while (!TLI.isTypeLegal(VT)) 3422 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); 3423 VTSize = VT.getSizeInBits() / 8; 3424 } else { 3425 // This can result in a type that is not legal on the target, e.g. 3426 // 1 or 2 bytes on PPC. 3427 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); 3428 VTSize >>= 1; 3429 } 3430 } 3431 3432 if (++NumMemOps > Limit) 3433 return false; 3434 MemOps.push_back(VT); 3435 Size -= VTSize; 3436 } 3437 3438 return true; 3439} 3440 3441static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, 3442 SDValue Chain, SDValue Dst, 3443 SDValue Src, uint64_t Size, 3444 unsigned Align, bool isVol, 3445 bool AlwaysInline, 3446 MachinePointerInfo DstPtrInfo, 3447 MachinePointerInfo SrcPtrInfo) { 3448 // Turn a memcpy of undef to nop. 3449 if (Src.getOpcode() == ISD::UNDEF) 3450 return Chain; 3451 3452 // Expand memcpy to a series of load and store ops if the size operand falls 3453 // below a certain threshold. 3454 // TODO: In the AlwaysInline case, if the size is big then generate a loop 3455 // rather than maybe a humongous number of loads and stores. 3456 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3457 std::vector<EVT> MemOps; 3458 bool DstAlignCanChange = false; 3459 MachineFunction &MF = DAG.getMachineFunction(); 3460 MachineFrameInfo *MFI = MF.getFrameInfo(); 3461 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3462 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3463 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3464 DstAlignCanChange = true; 3465 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3466 if (Align > SrcAlign) 3467 SrcAlign = Align; 3468 std::string Str; 3469 bool CopyFromStr = isMemSrcFromString(Src, Str); 3470 bool isZeroStr = CopyFromStr && Str.empty(); 3471 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3472 3473 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3474 (DstAlignCanChange ? 0 : Align), 3475 (isZeroStr ? 0 : SrcAlign), 3476 true, CopyFromStr, DAG, TLI)) 3477 return SDValue(); 3478 3479 if (DstAlignCanChange) { 3480 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3481 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3482 if (NewAlign > Align) { 3483 // Give the stack frame object a larger alignment if needed. 3484 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3485 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3486 Align = NewAlign; 3487 } 3488 } 3489 3490 SmallVector<SDValue, 8> OutChains; 3491 unsigned NumMemOps = MemOps.size(); 3492 uint64_t SrcOff = 0, DstOff = 0; 3493 for (unsigned i = 0; i != NumMemOps; ++i) { 3494 EVT VT = MemOps[i]; 3495 unsigned VTSize = VT.getSizeInBits() / 8; 3496 SDValue Value, Store; 3497 3498 if (CopyFromStr && 3499 (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3500 // It's unlikely a store of a vector immediate can be done in a single 3501 // instruction. It would require a load from a constantpool first. 3502 // We only handle zero vectors here. 3503 // FIXME: Handle other cases where store of vector immediate is done in 3504 // a single instruction. 3505 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str, SrcOff); 3506 Store = DAG.getStore(Chain, dl, Value, 3507 getMemBasePlusOffset(Dst, DstOff, DAG), 3508 DstPtrInfo.getWithOffset(DstOff), isVol, 3509 false, Align); 3510 } else { 3511 // The type might not be legal for the target. This should only happen 3512 // if the type is smaller than a legal type, as on PPC, so the right 3513 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3514 // to Load/Store if NVT==VT. 3515 // FIXME does the case above also need this? 3516 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3517 assert(NVT.bitsGE(VT)); 3518 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3519 getMemBasePlusOffset(Src, SrcOff, DAG), 3520 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3521 MinAlign(SrcAlign, SrcOff)); 3522 Store = DAG.getTruncStore(Chain, dl, Value, 3523 getMemBasePlusOffset(Dst, DstOff, DAG), 3524 DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3525 false, Align); 3526 } 3527 OutChains.push_back(Store); 3528 SrcOff += VTSize; 3529 DstOff += VTSize; 3530 } 3531 3532 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3533 &OutChains[0], OutChains.size()); 3534} 3535 3536static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, 3537 SDValue Chain, SDValue Dst, 3538 SDValue Src, uint64_t Size, 3539 unsigned Align, bool isVol, 3540 bool AlwaysInline, 3541 MachinePointerInfo DstPtrInfo, 3542 MachinePointerInfo SrcPtrInfo) { 3543 // Turn a memmove of undef to nop. 3544 if (Src.getOpcode() == ISD::UNDEF) 3545 return Chain; 3546 3547 // Expand memmove to a series of load and store ops if the size operand falls 3548 // below a certain threshold. 3549 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3550 std::vector<EVT> MemOps; 3551 bool DstAlignCanChange = false; 3552 MachineFunction &MF = DAG.getMachineFunction(); 3553 MachineFrameInfo *MFI = MF.getFrameInfo(); 3554 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3555 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3556 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3557 DstAlignCanChange = true; 3558 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3559 if (Align > SrcAlign) 3560 SrcAlign = Align; 3561 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3562 3563 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3564 (DstAlignCanChange ? 0 : Align), 3565 SrcAlign, true, false, DAG, TLI)) 3566 return SDValue(); 3567 3568 if (DstAlignCanChange) { 3569 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3570 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3571 if (NewAlign > Align) { 3572 // Give the stack frame object a larger alignment if needed. 3573 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3574 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3575 Align = NewAlign; 3576 } 3577 } 3578 3579 uint64_t SrcOff = 0, DstOff = 0; 3580 SmallVector<SDValue, 8> LoadValues; 3581 SmallVector<SDValue, 8> LoadChains; 3582 SmallVector<SDValue, 8> OutChains; 3583 unsigned NumMemOps = MemOps.size(); 3584 for (unsigned i = 0; i < NumMemOps; i++) { 3585 EVT VT = MemOps[i]; 3586 unsigned VTSize = VT.getSizeInBits() / 8; 3587 SDValue Value, Store; 3588 3589 Value = DAG.getLoad(VT, dl, Chain, 3590 getMemBasePlusOffset(Src, SrcOff, DAG), 3591 SrcPtrInfo.getWithOffset(SrcOff), isVol, 3592 false, false, SrcAlign); 3593 LoadValues.push_back(Value); 3594 LoadChains.push_back(Value.getValue(1)); 3595 SrcOff += VTSize; 3596 } 3597 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3598 &LoadChains[0], LoadChains.size()); 3599 OutChains.clear(); 3600 for (unsigned i = 0; i < NumMemOps; i++) { 3601 EVT VT = MemOps[i]; 3602 unsigned VTSize = VT.getSizeInBits() / 8; 3603 SDValue Value, Store; 3604 3605 Store = DAG.getStore(Chain, dl, LoadValues[i], 3606 getMemBasePlusOffset(Dst, DstOff, DAG), 3607 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3608 OutChains.push_back(Store); 3609 DstOff += VTSize; 3610 } 3611 3612 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3613 &OutChains[0], OutChains.size()); 3614} 3615 3616static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, 3617 SDValue Chain, SDValue Dst, 3618 SDValue Src, uint64_t Size, 3619 unsigned Align, bool isVol, 3620 MachinePointerInfo DstPtrInfo) { 3621 // Turn a memset of undef to nop. 3622 if (Src.getOpcode() == ISD::UNDEF) 3623 return Chain; 3624 3625 // Expand memset to a series of load/store ops if the size operand 3626 // falls below a certain threshold. 3627 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3628 std::vector<EVT> MemOps; 3629 bool DstAlignCanChange = false; 3630 MachineFunction &MF = DAG.getMachineFunction(); 3631 MachineFrameInfo *MFI = MF.getFrameInfo(); 3632 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3633 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3634 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3635 DstAlignCanChange = true; 3636 bool IsZeroVal = 3637 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3638 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3639 Size, (DstAlignCanChange ? 0 : Align), 0, 3640 IsZeroVal, false, DAG, TLI)) 3641 return SDValue(); 3642 3643 if (DstAlignCanChange) { 3644 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3645 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3646 if (NewAlign > Align) { 3647 // Give the stack frame object a larger alignment if needed. 3648 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3649 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3650 Align = NewAlign; 3651 } 3652 } 3653 3654 SmallVector<SDValue, 8> OutChains; 3655 uint64_t DstOff = 0; 3656 unsigned NumMemOps = MemOps.size(); 3657 3658 // Find the largest store and generate the bit pattern for it. 3659 EVT LargestVT = MemOps[0]; 3660 for (unsigned i = 1; i < NumMemOps; i++) 3661 if (MemOps[i].bitsGT(LargestVT)) 3662 LargestVT = MemOps[i]; 3663 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3664 3665 for (unsigned i = 0; i < NumMemOps; i++) { 3666 EVT VT = MemOps[i]; 3667 3668 // If this store is smaller than the largest store see whether we can get 3669 // the smaller value for free with a truncate. 3670 SDValue Value = MemSetValue; 3671 if (VT.bitsLT(LargestVT)) { 3672 if (!LargestVT.isVector() && !VT.isVector() && 3673 TLI.isTruncateFree(LargestVT, VT)) 3674 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3675 else 3676 Value = getMemsetValue(Src, VT, DAG, dl); 3677 } 3678 assert(Value.getValueType() == VT && "Value with wrong type."); 3679 SDValue Store = DAG.getStore(Chain, dl, Value, 3680 getMemBasePlusOffset(Dst, DstOff, DAG), 3681 DstPtrInfo.getWithOffset(DstOff), 3682 isVol, false, Align); 3683 OutChains.push_back(Store); 3684 DstOff += VT.getSizeInBits() / 8; 3685 } 3686 3687 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3688 &OutChains[0], OutChains.size()); 3689} 3690 3691SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, 3692 SDValue Src, SDValue Size, 3693 unsigned Align, bool isVol, bool AlwaysInline, 3694 MachinePointerInfo DstPtrInfo, 3695 MachinePointerInfo SrcPtrInfo) { 3696 3697 // Check to see if we should lower the memcpy to loads and stores first. 3698 // For cases within the target-specified limits, this is the best choice. 3699 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3700 if (ConstantSize) { 3701 // Memcpy with size zero? Just return the original chain. 3702 if (ConstantSize->isNullValue()) 3703 return Chain; 3704 3705 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3706 ConstantSize->getZExtValue(),Align, 3707 isVol, false, DstPtrInfo, SrcPtrInfo); 3708 if (Result.getNode()) 3709 return Result; 3710 } 3711 3712 // Then check to see if we should lower the memcpy with target-specific 3713 // code. If the target chooses to do this, this is the next best. 3714 SDValue Result = 3715 TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 3716 isVol, AlwaysInline, 3717 DstPtrInfo, SrcPtrInfo); 3718 if (Result.getNode()) 3719 return Result; 3720 3721 // If we really need inline code and the target declined to provide it, 3722 // use a (potentially long) sequence of loads and stores. 3723 if (AlwaysInline) { 3724 assert(ConstantSize && "AlwaysInline requires a constant size!"); 3725 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3726 ConstantSize->getZExtValue(), Align, isVol, 3727 true, DstPtrInfo, SrcPtrInfo); 3728 } 3729 3730 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 3731 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 3732 // respect volatile, so they may do things like read or write memory 3733 // beyond the given memory regions. But fixing this isn't easy, and most 3734 // people don't care. 3735 3736 // Emit a library call. 3737 TargetLowering::ArgListTy Args; 3738 TargetLowering::ArgListEntry Entry; 3739 Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext()); 3740 Entry.Node = Dst; Args.push_back(Entry); 3741 Entry.Node = Src; Args.push_back(Entry); 3742 Entry.Node = Size; Args.push_back(Entry); 3743 // FIXME: pass in DebugLoc 3744 std::pair<SDValue,SDValue> CallResult = 3745 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3746 false, false, false, false, 0, 3747 TLI.getLibcallCallingConv(RTLIB::MEMCPY), false, 3748 /*isReturnValueUsed=*/false, 3749 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMCPY), 3750 TLI.getPointerTy()), 3751 Args, *this, dl); 3752 return CallResult.second; 3753} 3754 3755SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, 3756 SDValue Src, SDValue Size, 3757 unsigned Align, bool isVol, 3758 MachinePointerInfo DstPtrInfo, 3759 MachinePointerInfo SrcPtrInfo) { 3760 3761 // Check to see if we should lower the memmove to loads and stores first. 3762 // For cases within the target-specified limits, this is the best choice. 3763 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3764 if (ConstantSize) { 3765 // Memmove with size zero? Just return the original chain. 3766 if (ConstantSize->isNullValue()) 3767 return Chain; 3768 3769 SDValue Result = 3770 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 3771 ConstantSize->getZExtValue(), Align, isVol, 3772 false, DstPtrInfo, SrcPtrInfo); 3773 if (Result.getNode()) 3774 return Result; 3775 } 3776 3777 // Then check to see if we should lower the memmove with target-specific 3778 // code. If the target chooses to do this, this is the next best. 3779 SDValue Result = 3780 TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 3781 DstPtrInfo, SrcPtrInfo); 3782 if (Result.getNode()) 3783 return Result; 3784 3785 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 3786 // not be safe. See memcpy above for more details. 3787 3788 // Emit a library call. 3789 TargetLowering::ArgListTy Args; 3790 TargetLowering::ArgListEntry Entry; 3791 Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext()); 3792 Entry.Node = Dst; Args.push_back(Entry); 3793 Entry.Node = Src; Args.push_back(Entry); 3794 Entry.Node = Size; Args.push_back(Entry); 3795 // FIXME: pass in DebugLoc 3796 std::pair<SDValue,SDValue> CallResult = 3797 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3798 false, false, false, false, 0, 3799 TLI.getLibcallCallingConv(RTLIB::MEMMOVE), false, 3800 /*isReturnValueUsed=*/false, 3801 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMMOVE), 3802 TLI.getPointerTy()), 3803 Args, *this, dl); 3804 return CallResult.second; 3805} 3806 3807SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, 3808 SDValue Src, SDValue Size, 3809 unsigned Align, bool isVol, 3810 MachinePointerInfo DstPtrInfo) { 3811 3812 // Check to see if we should lower the memset to stores first. 3813 // For cases within the target-specified limits, this is the best choice. 3814 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3815 if (ConstantSize) { 3816 // Memset with size zero? Just return the original chain. 3817 if (ConstantSize->isNullValue()) 3818 return Chain; 3819 3820 SDValue Result = 3821 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 3822 Align, isVol, DstPtrInfo); 3823 3824 if (Result.getNode()) 3825 return Result; 3826 } 3827 3828 // Then check to see if we should lower the memset with target-specific 3829 // code. If the target chooses to do this, this is the next best. 3830 SDValue Result = 3831 TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 3832 DstPtrInfo); 3833 if (Result.getNode()) 3834 return Result; 3835 3836 // Emit a library call. 3837 Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext()); 3838 TargetLowering::ArgListTy Args; 3839 TargetLowering::ArgListEntry Entry; 3840 Entry.Node = Dst; Entry.Ty = IntPtrTy; 3841 Args.push_back(Entry); 3842 // Extend or truncate the argument to be an i32 value for the call. 3843 if (Src.getValueType().bitsGT(MVT::i32)) 3844 Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 3845 else 3846 Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 3847 Entry.Node = Src; 3848 Entry.Ty = Type::getInt32Ty(*getContext()); 3849 Entry.isSExt = true; 3850 Args.push_back(Entry); 3851 Entry.Node = Size; 3852 Entry.Ty = IntPtrTy; 3853 Entry.isSExt = false; 3854 Args.push_back(Entry); 3855 // FIXME: pass in DebugLoc 3856 std::pair<SDValue,SDValue> CallResult = 3857 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3858 false, false, false, false, 0, 3859 TLI.getLibcallCallingConv(RTLIB::MEMSET), false, 3860 /*isReturnValueUsed=*/false, 3861 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET), 3862 TLI.getPointerTy()), 3863 Args, *this, dl); 3864 return CallResult.second; 3865} 3866 3867SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3868 SDValue Chain, SDValue Ptr, SDValue Cmp, 3869 SDValue Swp, MachinePointerInfo PtrInfo, 3870 unsigned Alignment, 3871 AtomicOrdering Ordering, 3872 SynchronizationScope SynchScope) { 3873 if (Alignment == 0) // Ensure that codegen never sees alignment 0 3874 Alignment = getEVTAlignment(MemVT); 3875 3876 MachineFunction &MF = getMachineFunction(); 3877 unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 3878 3879 // For now, atomics are considered to be volatile always. 3880 // FIXME: Volatile isn't really correct; we should keep track of atomic 3881 // orderings in the memoperand. 3882 Flags |= MachineMemOperand::MOVolatile; 3883 3884 MachineMemOperand *MMO = 3885 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 3886 3887 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 3888 Ordering, SynchScope); 3889} 3890 3891SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3892 SDValue Chain, 3893 SDValue Ptr, SDValue Cmp, 3894 SDValue Swp, MachineMemOperand *MMO, 3895 AtomicOrdering Ordering, 3896 SynchronizationScope SynchScope) { 3897 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 3898 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 3899 3900 EVT VT = Cmp.getValueType(); 3901 3902 SDVTList VTs = getVTList(VT, MVT::Other); 3903 FoldingSetNodeID ID; 3904 ID.AddInteger(MemVT.getRawBits()); 3905 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 3906 AddNodeIDNode(ID, Opcode, VTs, Ops, 4); 3907 void* IP = 0; 3908 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 3909 cast<AtomicSDNode>(E)->refineAlignment(MMO); 3910 return SDValue(E, 0); 3911 } 3912 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 3913 Ptr, Cmp, Swp, MMO, Ordering, 3914 SynchScope); 3915 CSEMap.InsertNode(N, IP); 3916 AllNodes.push_back(N); 3917 return SDValue(N, 0); 3918} 3919 3920SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3921 SDValue Chain, 3922 SDValue Ptr, SDValue Val, 3923 const Value* PtrVal, 3924 unsigned Alignment, 3925 AtomicOrdering Ordering, 3926 SynchronizationScope SynchScope) { 3927 if (Alignment == 0) // Ensure that codegen never sees alignment 0 3928 Alignment = getEVTAlignment(MemVT); 3929 3930 MachineFunction &MF = getMachineFunction(); 3931 // A monotonic store does not load; a release store "loads" in the sense 3932 // that other stores cannot be sunk past it. 3933 // (An atomicrmw obviously both loads and stores.) 3934 unsigned Flags = MachineMemOperand::MOStore; 3935 if (Opcode != ISD::ATOMIC_STORE || Ordering > Monotonic) 3936 Flags |= MachineMemOperand::MOLoad; 3937 3938 // For now, atomics are considered to be volatile always. 3939 // FIXME: Volatile isn't really correct; we should keep track of atomic 3940 // orderings in the memoperand. 3941 Flags |= MachineMemOperand::MOVolatile; 3942 3943 MachineMemOperand *MMO = 3944 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 3945 MemVT.getStoreSize(), Alignment); 3946 3947 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 3948 Ordering, SynchScope); 3949} 3950 3951SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3952 SDValue Chain, 3953 SDValue Ptr, SDValue Val, 3954 MachineMemOperand *MMO, 3955 AtomicOrdering Ordering, 3956 SynchronizationScope SynchScope) { 3957 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 3958 Opcode == ISD::ATOMIC_LOAD_SUB || 3959 Opcode == ISD::ATOMIC_LOAD_AND || 3960 Opcode == ISD::ATOMIC_LOAD_OR || 3961 Opcode == ISD::ATOMIC_LOAD_XOR || 3962 Opcode == ISD::ATOMIC_LOAD_NAND || 3963 Opcode == ISD::ATOMIC_LOAD_MIN || 3964 Opcode == ISD::ATOMIC_LOAD_MAX || 3965 Opcode == ISD::ATOMIC_LOAD_UMIN || 3966 Opcode == ISD::ATOMIC_LOAD_UMAX || 3967 Opcode == ISD::ATOMIC_SWAP || 3968 Opcode == ISD::ATOMIC_STORE) && 3969 "Invalid Atomic Op"); 3970 3971 EVT VT = Val.getValueType(); 3972 3973 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 3974 getVTList(VT, MVT::Other); 3975 FoldingSetNodeID ID; 3976 ID.AddInteger(MemVT.getRawBits()); 3977 SDValue Ops[] = {Chain, Ptr, Val}; 3978 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3979 void* IP = 0; 3980 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 3981 cast<AtomicSDNode>(E)->refineAlignment(MMO); 3982 return SDValue(E, 0); 3983 } 3984 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 3985 Ptr, Val, MMO, 3986 Ordering, SynchScope); 3987 CSEMap.InsertNode(N, IP); 3988 AllNodes.push_back(N); 3989 return SDValue(N, 0); 3990} 3991 3992SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3993 EVT VT, SDValue Chain, 3994 SDValue Ptr, 3995 const Value* PtrVal, 3996 unsigned Alignment, 3997 AtomicOrdering Ordering, 3998 SynchronizationScope SynchScope) { 3999 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4000 Alignment = getEVTAlignment(MemVT); 4001 4002 MachineFunction &MF = getMachineFunction(); 4003 // A monotonic load does not store; an acquire load "stores" in the sense 4004 // that other loads cannot be hoisted past it. 4005 unsigned Flags = MachineMemOperand::MOLoad; 4006 if (Ordering > Monotonic) 4007 Flags |= MachineMemOperand::MOStore; 4008 4009 // For now, atomics are considered to be volatile always. 4010 // FIXME: Volatile isn't really correct; we should keep track of atomic 4011 // orderings in the memoperand. 4012 Flags |= MachineMemOperand::MOVolatile; 4013 4014 MachineMemOperand *MMO = 4015 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4016 MemVT.getStoreSize(), Alignment); 4017 4018 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 4019 Ordering, SynchScope); 4020} 4021 4022SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4023 EVT VT, SDValue Chain, 4024 SDValue Ptr, 4025 MachineMemOperand *MMO, 4026 AtomicOrdering Ordering, 4027 SynchronizationScope SynchScope) { 4028 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4029 4030 SDVTList VTs = getVTList(VT, MVT::Other); 4031 FoldingSetNodeID ID; 4032 ID.AddInteger(MemVT.getRawBits()); 4033 SDValue Ops[] = {Chain, Ptr}; 4034 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 4035 void* IP = 0; 4036 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4037 cast<AtomicSDNode>(E)->refineAlignment(MMO); 4038 return SDValue(E, 0); 4039 } 4040 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 4041 Ptr, MMO, Ordering, SynchScope); 4042 CSEMap.InsertNode(N, IP); 4043 AllNodes.push_back(N); 4044 return SDValue(N, 0); 4045} 4046 4047/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4048SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4049 DebugLoc dl) { 4050 if (NumOps == 1) 4051 return Ops[0]; 4052 4053 SmallVector<EVT, 4> VTs; 4054 VTs.reserve(NumOps); 4055 for (unsigned i = 0; i < NumOps; ++i) 4056 VTs.push_back(Ops[i].getValueType()); 4057 return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4058 Ops, NumOps); 4059} 4060 4061SDValue 4062SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, 4063 const EVT *VTs, unsigned NumVTs, 4064 const SDValue *Ops, unsigned NumOps, 4065 EVT MemVT, MachinePointerInfo PtrInfo, 4066 unsigned Align, bool Vol, 4067 bool ReadMem, bool WriteMem) { 4068 return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4069 MemVT, PtrInfo, Align, Vol, 4070 ReadMem, WriteMem); 4071} 4072 4073SDValue 4074SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, 4075 const SDValue *Ops, unsigned NumOps, 4076 EVT MemVT, MachinePointerInfo PtrInfo, 4077 unsigned Align, bool Vol, 4078 bool ReadMem, bool WriteMem) { 4079 if (Align == 0) // Ensure that codegen never sees alignment 0 4080 Align = getEVTAlignment(MemVT); 4081 4082 MachineFunction &MF = getMachineFunction(); 4083 unsigned Flags = 0; 4084 if (WriteMem) 4085 Flags |= MachineMemOperand::MOStore; 4086 if (ReadMem) 4087 Flags |= MachineMemOperand::MOLoad; 4088 if (Vol) 4089 Flags |= MachineMemOperand::MOVolatile; 4090 MachineMemOperand *MMO = 4091 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4092 4093 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4094} 4095 4096SDValue 4097SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, 4098 const SDValue *Ops, unsigned NumOps, 4099 EVT MemVT, MachineMemOperand *MMO) { 4100 assert((Opcode == ISD::INTRINSIC_VOID || 4101 Opcode == ISD::INTRINSIC_W_CHAIN || 4102 Opcode == ISD::PREFETCH || 4103 (Opcode <= INT_MAX && 4104 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4105 "Opcode is not a memory-accessing opcode!"); 4106 4107 // Memoize the node unless it returns a flag. 4108 MemIntrinsicSDNode *N; 4109 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4110 FoldingSetNodeID ID; 4111 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4112 void *IP = 0; 4113 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4114 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4115 return SDValue(E, 0); 4116 } 4117 4118 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, 4119 MemVT, MMO); 4120 CSEMap.InsertNode(N, IP); 4121 } else { 4122 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, 4123 MemVT, MMO); 4124 } 4125 AllNodes.push_back(N); 4126 return SDValue(N, 0); 4127} 4128 4129/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4130/// MachinePointerInfo record from it. This is particularly useful because the 4131/// code generator has many cases where it doesn't bother passing in a 4132/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4133static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4134 // If this is FI+Offset, we can model it. 4135 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4136 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4137 4138 // If this is (FI+Offset1)+Offset2, we can model it. 4139 if (Ptr.getOpcode() != ISD::ADD || 4140 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4141 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4142 return MachinePointerInfo(); 4143 4144 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4145 return MachinePointerInfo::getFixedStack(FI, Offset+ 4146 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4147} 4148 4149/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4150/// MachinePointerInfo record from it. This is particularly useful because the 4151/// code generator has many cases where it doesn't bother passing in a 4152/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4153static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4154 // If the 'Offset' value isn't a constant, we can't handle this. 4155 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4156 return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4157 if (OffsetOp.getOpcode() == ISD::UNDEF) 4158 return InferPointerInfo(Ptr); 4159 return MachinePointerInfo(); 4160} 4161 4162 4163SDValue 4164SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4165 EVT VT, DebugLoc dl, SDValue Chain, 4166 SDValue Ptr, SDValue Offset, 4167 MachinePointerInfo PtrInfo, EVT MemVT, 4168 bool isVolatile, bool isNonTemporal, bool isInvariant, 4169 unsigned Alignment, const MDNode *TBAAInfo) { 4170 assert(Chain.getValueType() == MVT::Other && 4171 "Invalid chain type"); 4172 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4173 Alignment = getEVTAlignment(VT); 4174 4175 unsigned Flags = MachineMemOperand::MOLoad; 4176 if (isVolatile) 4177 Flags |= MachineMemOperand::MOVolatile; 4178 if (isNonTemporal) 4179 Flags |= MachineMemOperand::MONonTemporal; 4180 if (isInvariant) 4181 Flags |= MachineMemOperand::MOInvariant; 4182 4183 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4184 // clients. 4185 if (PtrInfo.V == 0) 4186 PtrInfo = InferPointerInfo(Ptr, Offset); 4187 4188 MachineFunction &MF = getMachineFunction(); 4189 MachineMemOperand *MMO = 4190 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4191 TBAAInfo); 4192 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4193} 4194 4195SDValue 4196SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4197 EVT VT, DebugLoc dl, SDValue Chain, 4198 SDValue Ptr, SDValue Offset, EVT MemVT, 4199 MachineMemOperand *MMO) { 4200 if (VT == MemVT) { 4201 ExtType = ISD::NON_EXTLOAD; 4202 } else if (ExtType == ISD::NON_EXTLOAD) { 4203 assert(VT == MemVT && "Non-extending load from different memory type!"); 4204 } else { 4205 // Extending load. 4206 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4207 "Should only be an extending load, not truncating!"); 4208 assert(VT.isInteger() == MemVT.isInteger() && 4209 "Cannot convert from FP to Int or Int -> FP!"); 4210 assert(VT.isVector() == MemVT.isVector() && 4211 "Cannot use trunc store to convert to or from a vector!"); 4212 assert((!VT.isVector() || 4213 VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4214 "Cannot use trunc store to change the number of vector elements!"); 4215 } 4216 4217 bool Indexed = AM != ISD::UNINDEXED; 4218 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4219 "Unindexed load with an offset!"); 4220 4221 SDVTList VTs = Indexed ? 4222 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4223 SDValue Ops[] = { Chain, Ptr, Offset }; 4224 FoldingSetNodeID ID; 4225 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4226 ID.AddInteger(MemVT.getRawBits()); 4227 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4228 MMO->isNonTemporal(), 4229 MMO->isInvariant())); 4230 void *IP = 0; 4231 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4232 cast<LoadSDNode>(E)->refineAlignment(MMO); 4233 return SDValue(E, 0); 4234 } 4235 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl, VTs, AM, ExtType, 4236 MemVT, MMO); 4237 CSEMap.InsertNode(N, IP); 4238 AllNodes.push_back(N); 4239 return SDValue(N, 0); 4240} 4241 4242SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, 4243 SDValue Chain, SDValue Ptr, 4244 MachinePointerInfo PtrInfo, 4245 bool isVolatile, bool isNonTemporal, 4246 bool isInvariant, unsigned Alignment, 4247 const MDNode *TBAAInfo) { 4248 SDValue Undef = getUNDEF(Ptr.getValueType()); 4249 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4250 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4251 TBAAInfo); 4252} 4253 4254SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, 4255 SDValue Chain, SDValue Ptr, 4256 MachinePointerInfo PtrInfo, EVT MemVT, 4257 bool isVolatile, bool isNonTemporal, 4258 unsigned Alignment, const MDNode *TBAAInfo) { 4259 SDValue Undef = getUNDEF(Ptr.getValueType()); 4260 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4261 PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4262 TBAAInfo); 4263} 4264 4265 4266SDValue 4267SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, 4268 SDValue Offset, ISD::MemIndexedMode AM) { 4269 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4270 assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4271 "Load is already a indexed load!"); 4272 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4273 LD->getChain(), Base, Offset, LD->getPointerInfo(), 4274 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4275 false, LD->getAlignment()); 4276} 4277 4278SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, 4279 SDValue Ptr, MachinePointerInfo PtrInfo, 4280 bool isVolatile, bool isNonTemporal, 4281 unsigned Alignment, const MDNode *TBAAInfo) { 4282 assert(Chain.getValueType() == MVT::Other && 4283 "Invalid chain type"); 4284 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4285 Alignment = getEVTAlignment(Val.getValueType()); 4286 4287 unsigned Flags = MachineMemOperand::MOStore; 4288 if (isVolatile) 4289 Flags |= MachineMemOperand::MOVolatile; 4290 if (isNonTemporal) 4291 Flags |= MachineMemOperand::MONonTemporal; 4292 4293 if (PtrInfo.V == 0) 4294 PtrInfo = InferPointerInfo(Ptr); 4295 4296 MachineFunction &MF = getMachineFunction(); 4297 MachineMemOperand *MMO = 4298 MF.getMachineMemOperand(PtrInfo, Flags, 4299 Val.getValueType().getStoreSize(), Alignment, 4300 TBAAInfo); 4301 4302 return getStore(Chain, dl, Val, Ptr, MMO); 4303} 4304 4305SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, 4306 SDValue Ptr, MachineMemOperand *MMO) { 4307 assert(Chain.getValueType() == MVT::Other && 4308 "Invalid chain type"); 4309 EVT VT = Val.getValueType(); 4310 SDVTList VTs = getVTList(MVT::Other); 4311 SDValue Undef = getUNDEF(Ptr.getValueType()); 4312 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4313 FoldingSetNodeID ID; 4314 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4315 ID.AddInteger(VT.getRawBits()); 4316 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4317 MMO->isNonTemporal(), MMO->isInvariant())); 4318 void *IP = 0; 4319 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4320 cast<StoreSDNode>(E)->refineAlignment(MMO); 4321 return SDValue(E, 0); 4322 } 4323 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, 4324 false, VT, MMO); 4325 CSEMap.InsertNode(N, IP); 4326 AllNodes.push_back(N); 4327 return SDValue(N, 0); 4328} 4329 4330SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, 4331 SDValue Ptr, MachinePointerInfo PtrInfo, 4332 EVT SVT,bool isVolatile, bool isNonTemporal, 4333 unsigned Alignment, 4334 const MDNode *TBAAInfo) { 4335 assert(Chain.getValueType() == MVT::Other && 4336 "Invalid chain type"); 4337 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4338 Alignment = getEVTAlignment(SVT); 4339 4340 unsigned Flags = MachineMemOperand::MOStore; 4341 if (isVolatile) 4342 Flags |= MachineMemOperand::MOVolatile; 4343 if (isNonTemporal) 4344 Flags |= MachineMemOperand::MONonTemporal; 4345 4346 if (PtrInfo.V == 0) 4347 PtrInfo = InferPointerInfo(Ptr); 4348 4349 MachineFunction &MF = getMachineFunction(); 4350 MachineMemOperand *MMO = 4351 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4352 TBAAInfo); 4353 4354 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4355} 4356 4357SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, 4358 SDValue Ptr, EVT SVT, 4359 MachineMemOperand *MMO) { 4360 EVT VT = Val.getValueType(); 4361 4362 assert(Chain.getValueType() == MVT::Other && 4363 "Invalid chain type"); 4364 if (VT == SVT) 4365 return getStore(Chain, dl, Val, Ptr, MMO); 4366 4367 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4368 "Should only be a truncating store, not extending!"); 4369 assert(VT.isInteger() == SVT.isInteger() && 4370 "Can't do FP-INT conversion!"); 4371 assert(VT.isVector() == SVT.isVector() && 4372 "Cannot use trunc store to convert to or from a vector!"); 4373 assert((!VT.isVector() || 4374 VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4375 "Cannot use trunc store to change the number of vector elements!"); 4376 4377 SDVTList VTs = getVTList(MVT::Other); 4378 SDValue Undef = getUNDEF(Ptr.getValueType()); 4379 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4380 FoldingSetNodeID ID; 4381 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4382 ID.AddInteger(SVT.getRawBits()); 4383 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4384 MMO->isNonTemporal(), MMO->isInvariant())); 4385 void *IP = 0; 4386 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4387 cast<StoreSDNode>(E)->refineAlignment(MMO); 4388 return SDValue(E, 0); 4389 } 4390 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, 4391 true, SVT, MMO); 4392 CSEMap.InsertNode(N, IP); 4393 AllNodes.push_back(N); 4394 return SDValue(N, 0); 4395} 4396 4397SDValue 4398SelectionDAG::getIndexedStore(SDValue OrigStore, DebugLoc dl, SDValue Base, 4399 SDValue Offset, ISD::MemIndexedMode AM) { 4400 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4401 assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4402 "Store is already a indexed store!"); 4403 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4404 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4405 FoldingSetNodeID ID; 4406 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4407 ID.AddInteger(ST->getMemoryVT().getRawBits()); 4408 ID.AddInteger(ST->getRawSubclassData()); 4409 void *IP = 0; 4410 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4411 return SDValue(E, 0); 4412 4413 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, AM, 4414 ST->isTruncatingStore(), 4415 ST->getMemoryVT(), 4416 ST->getMemOperand()); 4417 CSEMap.InsertNode(N, IP); 4418 AllNodes.push_back(N); 4419 return SDValue(N, 0); 4420} 4421 4422SDValue SelectionDAG::getVAArg(EVT VT, DebugLoc dl, 4423 SDValue Chain, SDValue Ptr, 4424 SDValue SV, 4425 unsigned Align) { 4426 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4427 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4428} 4429 4430SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 4431 const SDUse *Ops, unsigned NumOps) { 4432 switch (NumOps) { 4433 case 0: return getNode(Opcode, DL, VT); 4434 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4435 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4436 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4437 default: break; 4438 } 4439 4440 // Copy from an SDUse array into an SDValue array for use with 4441 // the regular getNode logic. 4442 SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4443 return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4444} 4445 4446SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 4447 const SDValue *Ops, unsigned NumOps) { 4448 switch (NumOps) { 4449 case 0: return getNode(Opcode, DL, VT); 4450 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4451 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4452 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4453 default: break; 4454 } 4455 4456 switch (Opcode) { 4457 default: break; 4458 case ISD::SELECT_CC: { 4459 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4460 assert(Ops[0].getValueType() == Ops[1].getValueType() && 4461 "LHS and RHS of condition must have same type!"); 4462 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4463 "True and False arms of SelectCC must have same type!"); 4464 assert(Ops[2].getValueType() == VT && 4465 "select_cc node must be of same type as true and false value!"); 4466 break; 4467 } 4468 case ISD::BR_CC: { 4469 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4470 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4471 "LHS/RHS of comparison should match types!"); 4472 break; 4473 } 4474 } 4475 4476 // Memoize nodes. 4477 SDNode *N; 4478 SDVTList VTs = getVTList(VT); 4479 4480 if (VT != MVT::Glue) { 4481 FoldingSetNodeID ID; 4482 AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4483 void *IP = 0; 4484 4485 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4486 return SDValue(E, 0); 4487 4488 N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); 4489 CSEMap.InsertNode(N, IP); 4490 } else { 4491 N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); 4492 } 4493 4494 AllNodes.push_back(N); 4495#ifndef NDEBUG 4496 VerifySDNode(N); 4497#endif 4498 return SDValue(N, 0); 4499} 4500 4501SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 4502 const std::vector<EVT> &ResultTys, 4503 const SDValue *Ops, unsigned NumOps) { 4504 return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4505 Ops, NumOps); 4506} 4507 4508SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 4509 const EVT *VTs, unsigned NumVTs, 4510 const SDValue *Ops, unsigned NumOps) { 4511 if (NumVTs == 1) 4512 return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4513 return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4514} 4515 4516SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4517 const SDValue *Ops, unsigned NumOps) { 4518 if (VTList.NumVTs == 1) 4519 return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4520 4521#if 0 4522 switch (Opcode) { 4523 // FIXME: figure out how to safely handle things like 4524 // int foo(int x) { return 1 << (x & 255); } 4525 // int bar() { return foo(256); } 4526 case ISD::SRA_PARTS: 4527 case ISD::SRL_PARTS: 4528 case ISD::SHL_PARTS: 4529 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4530 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4531 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4532 else if (N3.getOpcode() == ISD::AND) 4533 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4534 // If the and is only masking out bits that cannot effect the shift, 4535 // eliminate the and. 4536 unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4537 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4538 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4539 } 4540 break; 4541 } 4542#endif 4543 4544 // Memoize the node unless it returns a flag. 4545 SDNode *N; 4546 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4547 FoldingSetNodeID ID; 4548 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4549 void *IP = 0; 4550 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4551 return SDValue(E, 0); 4552 4553 if (NumOps == 1) { 4554 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); 4555 } else if (NumOps == 2) { 4556 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); 4557 } else if (NumOps == 3) { 4558 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], 4559 Ops[2]); 4560 } else { 4561 N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); 4562 } 4563 CSEMap.InsertNode(N, IP); 4564 } else { 4565 if (NumOps == 1) { 4566 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); 4567 } else if (NumOps == 2) { 4568 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); 4569 } else if (NumOps == 3) { 4570 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], 4571 Ops[2]); 4572 } else { 4573 N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); 4574 } 4575 } 4576 AllNodes.push_back(N); 4577#ifndef NDEBUG 4578 VerifySDNode(N); 4579#endif 4580 return SDValue(N, 0); 4581} 4582 4583SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList) { 4584 return getNode(Opcode, DL, VTList, 0, 0); 4585} 4586 4587SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4588 SDValue N1) { 4589 SDValue Ops[] = { N1 }; 4590 return getNode(Opcode, DL, VTList, Ops, 1); 4591} 4592 4593SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4594 SDValue N1, SDValue N2) { 4595 SDValue Ops[] = { N1, N2 }; 4596 return getNode(Opcode, DL, VTList, Ops, 2); 4597} 4598 4599SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4600 SDValue N1, SDValue N2, SDValue N3) { 4601 SDValue Ops[] = { N1, N2, N3 }; 4602 return getNode(Opcode, DL, VTList, Ops, 3); 4603} 4604 4605SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4606 SDValue N1, SDValue N2, SDValue N3, 4607 SDValue N4) { 4608 SDValue Ops[] = { N1, N2, N3, N4 }; 4609 return getNode(Opcode, DL, VTList, Ops, 4); 4610} 4611 4612SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4613 SDValue N1, SDValue N2, SDValue N3, 4614 SDValue N4, SDValue N5) { 4615 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4616 return getNode(Opcode, DL, VTList, Ops, 5); 4617} 4618 4619SDVTList SelectionDAG::getVTList(EVT VT) { 4620 return makeVTList(SDNode::getValueTypeList(VT), 1); 4621} 4622 4623SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4624 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4625 E = VTList.rend(); I != E; ++I) 4626 if (I->NumVTs == 2 && I->VTs[0] == VT1 && I->VTs[1] == VT2) 4627 return *I; 4628 4629 EVT *Array = Allocator.Allocate<EVT>(2); 4630 Array[0] = VT1; 4631 Array[1] = VT2; 4632 SDVTList Result = makeVTList(Array, 2); 4633 VTList.push_back(Result); 4634 return Result; 4635} 4636 4637SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 4638 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4639 E = VTList.rend(); I != E; ++I) 4640 if (I->NumVTs == 3 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 4641 I->VTs[2] == VT3) 4642 return *I; 4643 4644 EVT *Array = Allocator.Allocate<EVT>(3); 4645 Array[0] = VT1; 4646 Array[1] = VT2; 4647 Array[2] = VT3; 4648 SDVTList Result = makeVTList(Array, 3); 4649 VTList.push_back(Result); 4650 return Result; 4651} 4652 4653SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 4654 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4655 E = VTList.rend(); I != E; ++I) 4656 if (I->NumVTs == 4 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 4657 I->VTs[2] == VT3 && I->VTs[3] == VT4) 4658 return *I; 4659 4660 EVT *Array = Allocator.Allocate<EVT>(4); 4661 Array[0] = VT1; 4662 Array[1] = VT2; 4663 Array[2] = VT3; 4664 Array[3] = VT4; 4665 SDVTList Result = makeVTList(Array, 4); 4666 VTList.push_back(Result); 4667 return Result; 4668} 4669 4670SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 4671 switch (NumVTs) { 4672 case 0: llvm_unreachable("Cannot have nodes without results!"); 4673 case 1: return getVTList(VTs[0]); 4674 case 2: return getVTList(VTs[0], VTs[1]); 4675 case 3: return getVTList(VTs[0], VTs[1], VTs[2]); 4676 case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]); 4677 default: break; 4678 } 4679 4680 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4681 E = VTList.rend(); I != E; ++I) { 4682 if (I->NumVTs != NumVTs || VTs[0] != I->VTs[0] || VTs[1] != I->VTs[1]) 4683 continue; 4684 4685 bool NoMatch = false; 4686 for (unsigned i = 2; i != NumVTs; ++i) 4687 if (VTs[i] != I->VTs[i]) { 4688 NoMatch = true; 4689 break; 4690 } 4691 if (!NoMatch) 4692 return *I; 4693 } 4694 4695 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 4696 std::copy(VTs, VTs+NumVTs, Array); 4697 SDVTList Result = makeVTList(Array, NumVTs); 4698 VTList.push_back(Result); 4699 return Result; 4700} 4701 4702 4703/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 4704/// specified operands. If the resultant node already exists in the DAG, 4705/// this does not modify the specified node, instead it returns the node that 4706/// already exists. If the resultant node does not exist in the DAG, the 4707/// input node is returned. As a degenerate case, if you specify the same 4708/// input operands as the node already has, the input node is returned. 4709SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 4710 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 4711 4712 // Check to see if there is no change. 4713 if (Op == N->getOperand(0)) return N; 4714 4715 // See if the modified node already exists. 4716 void *InsertPos = 0; 4717 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 4718 return Existing; 4719 4720 // Nope it doesn't. Remove the node from its current place in the maps. 4721 if (InsertPos) 4722 if (!RemoveNodeFromCSEMaps(N)) 4723 InsertPos = 0; 4724 4725 // Now we update the operands. 4726 N->OperandList[0].set(Op); 4727 4728 // If this gets put into a CSE map, add it. 4729 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4730 return N; 4731} 4732 4733SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 4734 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 4735 4736 // Check to see if there is no change. 4737 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 4738 return N; // No operands changed, just return the input node. 4739 4740 // See if the modified node already exists. 4741 void *InsertPos = 0; 4742 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 4743 return Existing; 4744 4745 // Nope it doesn't. Remove the node from its current place in the maps. 4746 if (InsertPos) 4747 if (!RemoveNodeFromCSEMaps(N)) 4748 InsertPos = 0; 4749 4750 // Now we update the operands. 4751 if (N->OperandList[0] != Op1) 4752 N->OperandList[0].set(Op1); 4753 if (N->OperandList[1] != Op2) 4754 N->OperandList[1].set(Op2); 4755 4756 // If this gets put into a CSE map, add it. 4757 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4758 return N; 4759} 4760 4761SDNode *SelectionDAG:: 4762UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 4763 SDValue Ops[] = { Op1, Op2, Op3 }; 4764 return UpdateNodeOperands(N, Ops, 3); 4765} 4766 4767SDNode *SelectionDAG:: 4768UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 4769 SDValue Op3, SDValue Op4) { 4770 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 4771 return UpdateNodeOperands(N, Ops, 4); 4772} 4773 4774SDNode *SelectionDAG:: 4775UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 4776 SDValue Op3, SDValue Op4, SDValue Op5) { 4777 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 4778 return UpdateNodeOperands(N, Ops, 5); 4779} 4780 4781SDNode *SelectionDAG:: 4782UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 4783 assert(N->getNumOperands() == NumOps && 4784 "Update with wrong number of operands"); 4785 4786 // Check to see if there is no change. 4787 bool AnyChange = false; 4788 for (unsigned i = 0; i != NumOps; ++i) { 4789 if (Ops[i] != N->getOperand(i)) { 4790 AnyChange = true; 4791 break; 4792 } 4793 } 4794 4795 // No operands changed, just return the input node. 4796 if (!AnyChange) return N; 4797 4798 // See if the modified node already exists. 4799 void *InsertPos = 0; 4800 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 4801 return Existing; 4802 4803 // Nope it doesn't. Remove the node from its current place in the maps. 4804 if (InsertPos) 4805 if (!RemoveNodeFromCSEMaps(N)) 4806 InsertPos = 0; 4807 4808 // Now we update the operands. 4809 for (unsigned i = 0; i != NumOps; ++i) 4810 if (N->OperandList[i] != Ops[i]) 4811 N->OperandList[i].set(Ops[i]); 4812 4813 // If this gets put into a CSE map, add it. 4814 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4815 return N; 4816} 4817 4818/// DropOperands - Release the operands and set this node to have 4819/// zero operands. 4820void SDNode::DropOperands() { 4821 // Unlike the code in MorphNodeTo that does this, we don't need to 4822 // watch for dead nodes here. 4823 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 4824 SDUse &Use = *I++; 4825 Use.set(SDValue()); 4826 } 4827} 4828 4829/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 4830/// machine opcode. 4831/// 4832SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4833 EVT VT) { 4834 SDVTList VTs = getVTList(VT); 4835 return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 4836} 4837 4838SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4839 EVT VT, SDValue Op1) { 4840 SDVTList VTs = getVTList(VT); 4841 SDValue Ops[] = { Op1 }; 4842 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 4843} 4844 4845SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4846 EVT VT, SDValue Op1, 4847 SDValue Op2) { 4848 SDVTList VTs = getVTList(VT); 4849 SDValue Ops[] = { Op1, Op2 }; 4850 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 4851} 4852 4853SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4854 EVT VT, SDValue Op1, 4855 SDValue Op2, SDValue Op3) { 4856 SDVTList VTs = getVTList(VT); 4857 SDValue Ops[] = { Op1, Op2, Op3 }; 4858 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4859} 4860 4861SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4862 EVT VT, const SDValue *Ops, 4863 unsigned NumOps) { 4864 SDVTList VTs = getVTList(VT); 4865 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4866} 4867 4868SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4869 EVT VT1, EVT VT2, const SDValue *Ops, 4870 unsigned NumOps) { 4871 SDVTList VTs = getVTList(VT1, VT2); 4872 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4873} 4874 4875SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4876 EVT VT1, EVT VT2) { 4877 SDVTList VTs = getVTList(VT1, VT2); 4878 return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 4879} 4880 4881SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4882 EVT VT1, EVT VT2, EVT VT3, 4883 const SDValue *Ops, unsigned NumOps) { 4884 SDVTList VTs = getVTList(VT1, VT2, VT3); 4885 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4886} 4887 4888SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4889 EVT VT1, EVT VT2, EVT VT3, EVT VT4, 4890 const SDValue *Ops, unsigned NumOps) { 4891 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 4892 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4893} 4894 4895SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4896 EVT VT1, EVT VT2, 4897 SDValue Op1) { 4898 SDVTList VTs = getVTList(VT1, VT2); 4899 SDValue Ops[] = { Op1 }; 4900 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 4901} 4902 4903SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4904 EVT VT1, EVT VT2, 4905 SDValue Op1, SDValue Op2) { 4906 SDVTList VTs = getVTList(VT1, VT2); 4907 SDValue Ops[] = { Op1, Op2 }; 4908 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 4909} 4910 4911SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4912 EVT VT1, EVT VT2, 4913 SDValue Op1, SDValue Op2, 4914 SDValue Op3) { 4915 SDVTList VTs = getVTList(VT1, VT2); 4916 SDValue Ops[] = { Op1, Op2, Op3 }; 4917 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4918} 4919 4920SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4921 EVT VT1, EVT VT2, EVT VT3, 4922 SDValue Op1, SDValue Op2, 4923 SDValue Op3) { 4924 SDVTList VTs = getVTList(VT1, VT2, VT3); 4925 SDValue Ops[] = { Op1, Op2, Op3 }; 4926 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4927} 4928 4929SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4930 SDVTList VTs, const SDValue *Ops, 4931 unsigned NumOps) { 4932 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 4933 // Reset the NodeID to -1. 4934 N->setNodeId(-1); 4935 return N; 4936} 4937 4938/// UpdadeDebugLocOnMergedSDNode - If the opt level is -O0 then it throws away 4939/// the line number information on the merged node since it is not possible to 4940/// preserve the information that operation is associated with multiple lines. 4941/// This will make the debugger working better at -O0, were there is a higher 4942/// probability having other instructions associated with that line. 4943/// 4944SDNode *SelectionDAG::UpdadeDebugLocOnMergedSDNode(SDNode *N, DebugLoc OLoc) { 4945 DebugLoc NLoc = N->getDebugLoc(); 4946 if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && (OLoc != NLoc)) { 4947 N->setDebugLoc(DebugLoc()); 4948 } 4949 return N; 4950} 4951 4952/// MorphNodeTo - This *mutates* the specified node to have the specified 4953/// return type, opcode, and operands. 4954/// 4955/// Note that MorphNodeTo returns the resultant node. If there is already a 4956/// node of the specified opcode and operands, it returns that node instead of 4957/// the current one. Note that the DebugLoc need not be the same. 4958/// 4959/// Using MorphNodeTo is faster than creating a new node and swapping it in 4960/// with ReplaceAllUsesWith both because it often avoids allocating a new 4961/// node, and because it doesn't require CSE recalculation for any of 4962/// the node's users. 4963/// 4964SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 4965 SDVTList VTs, const SDValue *Ops, 4966 unsigned NumOps) { 4967 // If an identical node already exists, use it. 4968 void *IP = 0; 4969 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 4970 FoldingSetNodeID ID; 4971 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 4972 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 4973 return UpdadeDebugLocOnMergedSDNode(ON, N->getDebugLoc()); 4974 } 4975 4976 if (!RemoveNodeFromCSEMaps(N)) 4977 IP = 0; 4978 4979 // Start the morphing. 4980 N->NodeType = Opc; 4981 N->ValueList = VTs.VTs; 4982 N->NumValues = VTs.NumVTs; 4983 4984 // Clear the operands list, updating used nodes to remove this from their 4985 // use list. Keep track of any operands that become dead as a result. 4986 SmallPtrSet<SDNode*, 16> DeadNodeSet; 4987 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 4988 SDUse &Use = *I++; 4989 SDNode *Used = Use.getNode(); 4990 Use.set(SDValue()); 4991 if (Used->use_empty()) 4992 DeadNodeSet.insert(Used); 4993 } 4994 4995 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 4996 // Initialize the memory references information. 4997 MN->setMemRefs(0, 0); 4998 // If NumOps is larger than the # of operands we can have in a 4999 // MachineSDNode, reallocate the operand list. 5000 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 5001 if (MN->OperandsNeedDelete) 5002 delete[] MN->OperandList; 5003 if (NumOps > array_lengthof(MN->LocalOperands)) 5004 // We're creating a final node that will live unmorphed for the 5005 // remainder of the current SelectionDAG iteration, so we can allocate 5006 // the operands directly out of a pool with no recycling metadata. 5007 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5008 Ops, NumOps); 5009 else 5010 MN->InitOperands(MN->LocalOperands, Ops, NumOps); 5011 MN->OperandsNeedDelete = false; 5012 } else 5013 MN->InitOperands(MN->OperandList, Ops, NumOps); 5014 } else { 5015 // If NumOps is larger than the # of operands we currently have, reallocate 5016 // the operand list. 5017 if (NumOps > N->NumOperands) { 5018 if (N->OperandsNeedDelete) 5019 delete[] N->OperandList; 5020 N->InitOperands(new SDUse[NumOps], Ops, NumOps); 5021 N->OperandsNeedDelete = true; 5022 } else 5023 N->InitOperands(N->OperandList, Ops, NumOps); 5024 } 5025 5026 // Delete any nodes that are still dead after adding the uses for the 5027 // new operands. 5028 if (!DeadNodeSet.empty()) { 5029 SmallVector<SDNode *, 16> DeadNodes; 5030 for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 5031 E = DeadNodeSet.end(); I != E; ++I) 5032 if ((*I)->use_empty()) 5033 DeadNodes.push_back(*I); 5034 RemoveDeadNodes(DeadNodes); 5035 } 5036 5037 if (IP) 5038 CSEMap.InsertNode(N, IP); // Memoize the new node. 5039 return N; 5040} 5041 5042 5043/// getMachineNode - These are used for target selectors to create a new node 5044/// with specified return type(s), MachineInstr opcode, and operands. 5045/// 5046/// Note that getMachineNode returns the resultant node. If there is already a 5047/// node of the specified opcode and operands, it returns that node instead of 5048/// the current one. 5049MachineSDNode * 5050SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT) { 5051 SDVTList VTs = getVTList(VT); 5052 return getMachineNode(Opcode, dl, VTs, 0, 0); 5053} 5054 5055MachineSDNode * 5056SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, SDValue Op1) { 5057 SDVTList VTs = getVTList(VT); 5058 SDValue Ops[] = { Op1 }; 5059 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5060} 5061 5062MachineSDNode * 5063SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5064 SDValue Op1, SDValue Op2) { 5065 SDVTList VTs = getVTList(VT); 5066 SDValue Ops[] = { Op1, Op2 }; 5067 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5068} 5069 5070MachineSDNode * 5071SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5072 SDValue Op1, SDValue Op2, SDValue Op3) { 5073 SDVTList VTs = getVTList(VT); 5074 SDValue Ops[] = { Op1, Op2, Op3 }; 5075 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5076} 5077 5078MachineSDNode * 5079SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5080 const SDValue *Ops, unsigned NumOps) { 5081 SDVTList VTs = getVTList(VT); 5082 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5083} 5084 5085MachineSDNode * 5086SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, EVT VT2) { 5087 SDVTList VTs = getVTList(VT1, VT2); 5088 return getMachineNode(Opcode, dl, VTs, 0, 0); 5089} 5090 5091MachineSDNode * 5092SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5093 EVT VT1, EVT VT2, SDValue Op1) { 5094 SDVTList VTs = getVTList(VT1, VT2); 5095 SDValue Ops[] = { Op1 }; 5096 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5097} 5098 5099MachineSDNode * 5100SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5101 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5102 SDVTList VTs = getVTList(VT1, VT2); 5103 SDValue Ops[] = { Op1, Op2 }; 5104 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5105} 5106 5107MachineSDNode * 5108SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5109 EVT VT1, EVT VT2, SDValue Op1, 5110 SDValue Op2, SDValue Op3) { 5111 SDVTList VTs = getVTList(VT1, VT2); 5112 SDValue Ops[] = { Op1, Op2, Op3 }; 5113 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5114} 5115 5116MachineSDNode * 5117SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5118 EVT VT1, EVT VT2, 5119 const SDValue *Ops, unsigned NumOps) { 5120 SDVTList VTs = getVTList(VT1, VT2); 5121 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5122} 5123 5124MachineSDNode * 5125SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5126 EVT VT1, EVT VT2, EVT VT3, 5127 SDValue Op1, SDValue Op2) { 5128 SDVTList VTs = getVTList(VT1, VT2, VT3); 5129 SDValue Ops[] = { Op1, Op2 }; 5130 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5131} 5132 5133MachineSDNode * 5134SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5135 EVT VT1, EVT VT2, EVT VT3, 5136 SDValue Op1, SDValue Op2, SDValue Op3) { 5137 SDVTList VTs = getVTList(VT1, VT2, VT3); 5138 SDValue Ops[] = { Op1, Op2, Op3 }; 5139 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5140} 5141 5142MachineSDNode * 5143SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5144 EVT VT1, EVT VT2, EVT VT3, 5145 const SDValue *Ops, unsigned NumOps) { 5146 SDVTList VTs = getVTList(VT1, VT2, VT3); 5147 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5148} 5149 5150MachineSDNode * 5151SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, 5152 EVT VT2, EVT VT3, EVT VT4, 5153 const SDValue *Ops, unsigned NumOps) { 5154 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5155 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5156} 5157 5158MachineSDNode * 5159SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5160 const std::vector<EVT> &ResultTys, 5161 const SDValue *Ops, unsigned NumOps) { 5162 SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5163 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5164} 5165 5166MachineSDNode * 5167SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, 5168 const SDValue *Ops, unsigned NumOps) { 5169 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5170 MachineSDNode *N; 5171 void *IP = 0; 5172 5173 if (DoCSE) { 5174 FoldingSetNodeID ID; 5175 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5176 IP = 0; 5177 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 5178 return cast<MachineSDNode>(UpdadeDebugLocOnMergedSDNode(E, DL)); 5179 } 5180 } 5181 5182 // Allocate a new MachineSDNode. 5183 N = new (NodeAllocator) MachineSDNode(~Opcode, DL, VTs); 5184 5185 // Initialize the operands list. 5186 if (NumOps > array_lengthof(N->LocalOperands)) 5187 // We're creating a final node that will live unmorphed for the 5188 // remainder of the current SelectionDAG iteration, so we can allocate 5189 // the operands directly out of a pool with no recycling metadata. 5190 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5191 Ops, NumOps); 5192 else 5193 N->InitOperands(N->LocalOperands, Ops, NumOps); 5194 N->OperandsNeedDelete = false; 5195 5196 if (DoCSE) 5197 CSEMap.InsertNode(N, IP); 5198 5199 AllNodes.push_back(N); 5200#ifndef NDEBUG 5201 VerifyMachineNode(N); 5202#endif 5203 return N; 5204} 5205 5206/// getTargetExtractSubreg - A convenience function for creating 5207/// TargetOpcode::EXTRACT_SUBREG nodes. 5208SDValue 5209SelectionDAG::getTargetExtractSubreg(int SRIdx, DebugLoc DL, EVT VT, 5210 SDValue Operand) { 5211 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5212 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5213 VT, Operand, SRIdxVal); 5214 return SDValue(Subreg, 0); 5215} 5216 5217/// getTargetInsertSubreg - A convenience function for creating 5218/// TargetOpcode::INSERT_SUBREG nodes. 5219SDValue 5220SelectionDAG::getTargetInsertSubreg(int SRIdx, DebugLoc DL, EVT VT, 5221 SDValue Operand, SDValue Subreg) { 5222 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5223 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5224 VT, Operand, Subreg, SRIdxVal); 5225 return SDValue(Result, 0); 5226} 5227 5228/// getNodeIfExists - Get the specified node if it's already available, or 5229/// else return NULL. 5230SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5231 const SDValue *Ops, unsigned NumOps) { 5232 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5233 FoldingSetNodeID ID; 5234 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5235 void *IP = 0; 5236 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5237 return E; 5238 } 5239 return NULL; 5240} 5241 5242/// getDbgValue - Creates a SDDbgValue node. 5243/// 5244SDDbgValue * 5245SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5246 DebugLoc DL, unsigned O) { 5247 return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5248} 5249 5250SDDbgValue * 5251SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5252 DebugLoc DL, unsigned O) { 5253 return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5254} 5255 5256SDDbgValue * 5257SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5258 DebugLoc DL, unsigned O) { 5259 return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5260} 5261 5262namespace { 5263 5264/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5265/// pointed to by a use iterator is deleted, increment the use iterator 5266/// so that it doesn't dangle. 5267/// 5268/// This class also manages a "downlink" DAGUpdateListener, to forward 5269/// messages to ReplaceAllUsesWith's callers. 5270/// 5271class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5272 SelectionDAG::DAGUpdateListener *DownLink; 5273 SDNode::use_iterator &UI; 5274 SDNode::use_iterator &UE; 5275 5276 virtual void NodeDeleted(SDNode *N, SDNode *E) { 5277 // Increment the iterator as needed. 5278 while (UI != UE && N == *UI) 5279 ++UI; 5280 5281 // Then forward the message. 5282 if (DownLink) DownLink->NodeDeleted(N, E); 5283 } 5284 5285 virtual void NodeUpdated(SDNode *N) { 5286 // Just forward the message. 5287 if (DownLink) DownLink->NodeUpdated(N); 5288 } 5289 5290public: 5291 RAUWUpdateListener(SelectionDAG::DAGUpdateListener *dl, 5292 SDNode::use_iterator &ui, 5293 SDNode::use_iterator &ue) 5294 : DownLink(dl), UI(ui), UE(ue) {} 5295}; 5296 5297} 5298 5299/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5300/// This can cause recursive merging of nodes in the DAG. 5301/// 5302/// This version assumes From has a single result value. 5303/// 5304void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To, 5305 DAGUpdateListener *UpdateListener) { 5306 SDNode *From = FromN.getNode(); 5307 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5308 "Cannot replace with this method!"); 5309 assert(From != To.getNode() && "Cannot replace uses of with self"); 5310 5311 // Iterate over all the existing uses of From. New uses will be added 5312 // to the beginning of the use list, which we avoid visiting. 5313 // This specifically avoids visiting uses of From that arise while the 5314 // replacement is happening, because any such uses would be the result 5315 // of CSE: If an existing node looks like From after one of its operands 5316 // is replaced by To, we don't want to replace of all its users with To 5317 // too. See PR3018 for more info. 5318 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5319 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5320 while (UI != UE) { 5321 SDNode *User = *UI; 5322 5323 // This node is about to morph, remove its old self from the CSE maps. 5324 RemoveNodeFromCSEMaps(User); 5325 5326 // A user can appear in a use list multiple times, and when this 5327 // happens the uses are usually next to each other in the list. 5328 // To help reduce the number of CSE recomputations, process all 5329 // the uses of this user that we can find this way. 5330 do { 5331 SDUse &Use = UI.getUse(); 5332 ++UI; 5333 Use.set(To); 5334 } while (UI != UE && *UI == User); 5335 5336 // Now that we have modified User, add it back to the CSE maps. If it 5337 // already exists there, recursively merge the results together. 5338 AddModifiedNodeToCSEMaps(User, &Listener); 5339 } 5340 5341 // If we just RAUW'd the root, take note. 5342 if (FromN == getRoot()) 5343 setRoot(To); 5344} 5345 5346/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5347/// This can cause recursive merging of nodes in the DAG. 5348/// 5349/// This version assumes that for each value of From, there is a 5350/// corresponding value in To in the same position with the same type. 5351/// 5352void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To, 5353 DAGUpdateListener *UpdateListener) { 5354#ifndef NDEBUG 5355 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5356 assert((!From->hasAnyUseOfValue(i) || 5357 From->getValueType(i) == To->getValueType(i)) && 5358 "Cannot use this version of ReplaceAllUsesWith!"); 5359#endif 5360 5361 // Handle the trivial case. 5362 if (From == To) 5363 return; 5364 5365 // Iterate over just the existing users of From. See the comments in 5366 // the ReplaceAllUsesWith above. 5367 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5368 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5369 while (UI != UE) { 5370 SDNode *User = *UI; 5371 5372 // This node is about to morph, remove its old self from the CSE maps. 5373 RemoveNodeFromCSEMaps(User); 5374 5375 // A user can appear in a use list multiple times, and when this 5376 // happens the uses are usually next to each other in the list. 5377 // To help reduce the number of CSE recomputations, process all 5378 // the uses of this user that we can find this way. 5379 do { 5380 SDUse &Use = UI.getUse(); 5381 ++UI; 5382 Use.setNode(To); 5383 } while (UI != UE && *UI == User); 5384 5385 // Now that we have modified User, add it back to the CSE maps. If it 5386 // already exists there, recursively merge the results together. 5387 AddModifiedNodeToCSEMaps(User, &Listener); 5388 } 5389 5390 // If we just RAUW'd the root, take note. 5391 if (From == getRoot().getNode()) 5392 setRoot(SDValue(To, getRoot().getResNo())); 5393} 5394 5395/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5396/// This can cause recursive merging of nodes in the DAG. 5397/// 5398/// This version can replace From with any result values. To must match the 5399/// number and types of values returned by From. 5400void SelectionDAG::ReplaceAllUsesWith(SDNode *From, 5401 const SDValue *To, 5402 DAGUpdateListener *UpdateListener) { 5403 if (From->getNumValues() == 1) // Handle the simple case efficiently. 5404 return ReplaceAllUsesWith(SDValue(From, 0), To[0], UpdateListener); 5405 5406 // Iterate over just the existing users of From. See the comments in 5407 // the ReplaceAllUsesWith above. 5408 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5409 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5410 while (UI != UE) { 5411 SDNode *User = *UI; 5412 5413 // This node is about to morph, remove its old self from the CSE maps. 5414 RemoveNodeFromCSEMaps(User); 5415 5416 // A user can appear in a use list multiple times, and when this 5417 // happens the uses are usually next to each other in the list. 5418 // To help reduce the number of CSE recomputations, process all 5419 // the uses of this user that we can find this way. 5420 do { 5421 SDUse &Use = UI.getUse(); 5422 const SDValue &ToOp = To[Use.getResNo()]; 5423 ++UI; 5424 Use.set(ToOp); 5425 } while (UI != UE && *UI == User); 5426 5427 // Now that we have modified User, add it back to the CSE maps. If it 5428 // already exists there, recursively merge the results together. 5429 AddModifiedNodeToCSEMaps(User, &Listener); 5430 } 5431 5432 // If we just RAUW'd the root, take note. 5433 if (From == getRoot().getNode()) 5434 setRoot(SDValue(To[getRoot().getResNo()])); 5435} 5436 5437/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5438/// uses of other values produced by From.getNode() alone. The Deleted 5439/// vector is handled the same way as for ReplaceAllUsesWith. 5440void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To, 5441 DAGUpdateListener *UpdateListener){ 5442 // Handle the really simple, really trivial case efficiently. 5443 if (From == To) return; 5444 5445 // Handle the simple, trivial, case efficiently. 5446 if (From.getNode()->getNumValues() == 1) { 5447 ReplaceAllUsesWith(From, To, UpdateListener); 5448 return; 5449 } 5450 5451 // Iterate over just the existing users of From. See the comments in 5452 // the ReplaceAllUsesWith above. 5453 SDNode::use_iterator UI = From.getNode()->use_begin(), 5454 UE = From.getNode()->use_end(); 5455 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5456 while (UI != UE) { 5457 SDNode *User = *UI; 5458 bool UserRemovedFromCSEMaps = false; 5459 5460 // A user can appear in a use list multiple times, and when this 5461 // happens the uses are usually next to each other in the list. 5462 // To help reduce the number of CSE recomputations, process all 5463 // the uses of this user that we can find this way. 5464 do { 5465 SDUse &Use = UI.getUse(); 5466 5467 // Skip uses of different values from the same node. 5468 if (Use.getResNo() != From.getResNo()) { 5469 ++UI; 5470 continue; 5471 } 5472 5473 // If this node hasn't been modified yet, it's still in the CSE maps, 5474 // so remove its old self from the CSE maps. 5475 if (!UserRemovedFromCSEMaps) { 5476 RemoveNodeFromCSEMaps(User); 5477 UserRemovedFromCSEMaps = true; 5478 } 5479 5480 ++UI; 5481 Use.set(To); 5482 } while (UI != UE && *UI == User); 5483 5484 // We are iterating over all uses of the From node, so if a use 5485 // doesn't use the specific value, no changes are made. 5486 if (!UserRemovedFromCSEMaps) 5487 continue; 5488 5489 // Now that we have modified User, add it back to the CSE maps. If it 5490 // already exists there, recursively merge the results together. 5491 AddModifiedNodeToCSEMaps(User, &Listener); 5492 } 5493 5494 // If we just RAUW'd the root, take note. 5495 if (From == getRoot()) 5496 setRoot(To); 5497} 5498 5499namespace { 5500 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5501 /// to record information about a use. 5502 struct UseMemo { 5503 SDNode *User; 5504 unsigned Index; 5505 SDUse *Use; 5506 }; 5507 5508 /// operator< - Sort Memos by User. 5509 bool operator<(const UseMemo &L, const UseMemo &R) { 5510 return (intptr_t)L.User < (intptr_t)R.User; 5511 } 5512} 5513 5514/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5515/// uses of other values produced by From.getNode() alone. The same value 5516/// may appear in both the From and To list. The Deleted vector is 5517/// handled the same way as for ReplaceAllUsesWith. 5518void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5519 const SDValue *To, 5520 unsigned Num, 5521 DAGUpdateListener *UpdateListener){ 5522 // Handle the simple, trivial case efficiently. 5523 if (Num == 1) 5524 return ReplaceAllUsesOfValueWith(*From, *To, UpdateListener); 5525 5526 // Read up all the uses and make records of them. This helps 5527 // processing new uses that are introduced during the 5528 // replacement process. 5529 SmallVector<UseMemo, 4> Uses; 5530 for (unsigned i = 0; i != Num; ++i) { 5531 unsigned FromResNo = From[i].getResNo(); 5532 SDNode *FromNode = From[i].getNode(); 5533 for (SDNode::use_iterator UI = FromNode->use_begin(), 5534 E = FromNode->use_end(); UI != E; ++UI) { 5535 SDUse &Use = UI.getUse(); 5536 if (Use.getResNo() == FromResNo) { 5537 UseMemo Memo = { *UI, i, &Use }; 5538 Uses.push_back(Memo); 5539 } 5540 } 5541 } 5542 5543 // Sort the uses, so that all the uses from a given User are together. 5544 std::sort(Uses.begin(), Uses.end()); 5545 5546 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5547 UseIndex != UseIndexEnd; ) { 5548 // We know that this user uses some value of From. If it is the right 5549 // value, update it. 5550 SDNode *User = Uses[UseIndex].User; 5551 5552 // This node is about to morph, remove its old self from the CSE maps. 5553 RemoveNodeFromCSEMaps(User); 5554 5555 // The Uses array is sorted, so all the uses for a given User 5556 // are next to each other in the list. 5557 // To help reduce the number of CSE recomputations, process all 5558 // the uses of this user that we can find this way. 5559 do { 5560 unsigned i = Uses[UseIndex].Index; 5561 SDUse &Use = *Uses[UseIndex].Use; 5562 ++UseIndex; 5563 5564 Use.set(To[i]); 5565 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5566 5567 // Now that we have modified User, add it back to the CSE maps. If it 5568 // already exists there, recursively merge the results together. 5569 AddModifiedNodeToCSEMaps(User, UpdateListener); 5570 } 5571} 5572 5573/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5574/// based on their topological order. It returns the maximum id and a vector 5575/// of the SDNodes* in assigned order by reference. 5576unsigned SelectionDAG::AssignTopologicalOrder() { 5577 5578 unsigned DAGSize = 0; 5579 5580 // SortedPos tracks the progress of the algorithm. Nodes before it are 5581 // sorted, nodes after it are unsorted. When the algorithm completes 5582 // it is at the end of the list. 5583 allnodes_iterator SortedPos = allnodes_begin(); 5584 5585 // Visit all the nodes. Move nodes with no operands to the front of 5586 // the list immediately. Annotate nodes that do have operands with their 5587 // operand count. Before we do this, the Node Id fields of the nodes 5588 // may contain arbitrary values. After, the Node Id fields for nodes 5589 // before SortedPos will contain the topological sort index, and the 5590 // Node Id fields for nodes At SortedPos and after will contain the 5591 // count of outstanding operands. 5592 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5593 SDNode *N = I++; 5594 checkForCycles(N); 5595 unsigned Degree = N->getNumOperands(); 5596 if (Degree == 0) { 5597 // A node with no uses, add it to the result array immediately. 5598 N->setNodeId(DAGSize++); 5599 allnodes_iterator Q = N; 5600 if (Q != SortedPos) 5601 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5602 assert(SortedPos != AllNodes.end() && "Overran node list"); 5603 ++SortedPos; 5604 } else { 5605 // Temporarily use the Node Id as scratch space for the degree count. 5606 N->setNodeId(Degree); 5607 } 5608 } 5609 5610 // Visit all the nodes. As we iterate, moves nodes into sorted order, 5611 // such that by the time the end is reached all nodes will be sorted. 5612 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5613 SDNode *N = I; 5614 checkForCycles(N); 5615 // N is in sorted position, so all its uses have one less operand 5616 // that needs to be sorted. 5617 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5618 UI != UE; ++UI) { 5619 SDNode *P = *UI; 5620 unsigned Degree = P->getNodeId(); 5621 assert(Degree != 0 && "Invalid node degree"); 5622 --Degree; 5623 if (Degree == 0) { 5624 // All of P's operands are sorted, so P may sorted now. 5625 P->setNodeId(DAGSize++); 5626 if (P != SortedPos) 5627 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5628 assert(SortedPos != AllNodes.end() && "Overran node list"); 5629 ++SortedPos; 5630 } else { 5631 // Update P's outstanding operand count. 5632 P->setNodeId(Degree); 5633 } 5634 } 5635 if (I == SortedPos) { 5636#ifndef NDEBUG 5637 SDNode *S = ++I; 5638 dbgs() << "Overran sorted position:\n"; 5639 S->dumprFull(); 5640#endif 5641 llvm_unreachable(0); 5642 } 5643 } 5644 5645 assert(SortedPos == AllNodes.end() && 5646 "Topological sort incomplete!"); 5647 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 5648 "First node in topological sort is not the entry token!"); 5649 assert(AllNodes.front().getNodeId() == 0 && 5650 "First node in topological sort has non-zero id!"); 5651 assert(AllNodes.front().getNumOperands() == 0 && 5652 "First node in topological sort has operands!"); 5653 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 5654 "Last node in topologic sort has unexpected id!"); 5655 assert(AllNodes.back().use_empty() && 5656 "Last node in topologic sort has users!"); 5657 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 5658 return DAGSize; 5659} 5660 5661/// AssignOrdering - Assign an order to the SDNode. 5662void SelectionDAG::AssignOrdering(const SDNode *SD, unsigned Order) { 5663 assert(SD && "Trying to assign an order to a null node!"); 5664 Ordering->add(SD, Order); 5665} 5666 5667/// GetOrdering - Get the order for the SDNode. 5668unsigned SelectionDAG::GetOrdering(const SDNode *SD) const { 5669 assert(SD && "Trying to get the order of a null node!"); 5670 return Ordering->getOrder(SD); 5671} 5672 5673/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 5674/// value is produced by SD. 5675void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 5676 DbgInfo->add(DB, SD, isParameter); 5677 if (SD) 5678 SD->setHasDebugValue(true); 5679} 5680 5681/// TransferDbgValues - Transfer SDDbgValues. 5682void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 5683 if (From == To || !From.getNode()->getHasDebugValue()) 5684 return; 5685 SDNode *FromNode = From.getNode(); 5686 SDNode *ToNode = To.getNode(); 5687 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 5688 SmallVector<SDDbgValue *, 2> ClonedDVs; 5689 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 5690 I != E; ++I) { 5691 SDDbgValue *Dbg = *I; 5692 if (Dbg->getKind() == SDDbgValue::SDNODE) { 5693 SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 5694 Dbg->getOffset(), Dbg->getDebugLoc(), 5695 Dbg->getOrder()); 5696 ClonedDVs.push_back(Clone); 5697 } 5698 } 5699 for (SmallVector<SDDbgValue *, 2>::iterator I = ClonedDVs.begin(), 5700 E = ClonedDVs.end(); I != E; ++I) 5701 AddDbgValue(*I, ToNode, false); 5702} 5703 5704//===----------------------------------------------------------------------===// 5705// SDNode Class 5706//===----------------------------------------------------------------------===// 5707 5708HandleSDNode::~HandleSDNode() { 5709 DropOperands(); 5710} 5711 5712GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, DebugLoc DL, 5713 const GlobalValue *GA, 5714 EVT VT, int64_t o, unsigned char TF) 5715 : SDNode(Opc, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 5716 TheGlobal = GA; 5717} 5718 5719MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, 5720 MachineMemOperand *mmo) 5721 : SDNode(Opc, dl, VTs), MemoryVT(memvt), MMO(mmo) { 5722 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 5723 MMO->isNonTemporal(), MMO->isInvariant()); 5724 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 5725 assert(isNonTemporal() == MMO->isNonTemporal() && 5726 "Non-temporal encoding error!"); 5727 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 5728} 5729 5730MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, 5731 const SDValue *Ops, unsigned NumOps, EVT memvt, 5732 MachineMemOperand *mmo) 5733 : SDNode(Opc, dl, VTs, Ops, NumOps), 5734 MemoryVT(memvt), MMO(mmo) { 5735 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 5736 MMO->isNonTemporal(), MMO->isInvariant()); 5737 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 5738 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 5739} 5740 5741/// Profile - Gather unique data for the node. 5742/// 5743void SDNode::Profile(FoldingSetNodeID &ID) const { 5744 AddNodeIDNode(ID, this); 5745} 5746 5747namespace { 5748 struct EVTArray { 5749 std::vector<EVT> VTs; 5750 5751 EVTArray() { 5752 VTs.reserve(MVT::LAST_VALUETYPE); 5753 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 5754 VTs.push_back(MVT((MVT::SimpleValueType)i)); 5755 } 5756 }; 5757} 5758 5759static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 5760static ManagedStatic<EVTArray> SimpleVTArray; 5761static ManagedStatic<sys::SmartMutex<true> > VTMutex; 5762 5763/// getValueTypeList - Return a pointer to the specified value type. 5764/// 5765const EVT *SDNode::getValueTypeList(EVT VT) { 5766 if (VT.isExtended()) { 5767 sys::SmartScopedLock<true> Lock(*VTMutex); 5768 return &(*EVTs->insert(VT).first); 5769 } else { 5770 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 5771 "Value type out of range!"); 5772 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 5773 } 5774} 5775 5776/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 5777/// indicated value. This method ignores uses of other values defined by this 5778/// operation. 5779bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 5780 assert(Value < getNumValues() && "Bad value!"); 5781 5782 // TODO: Only iterate over uses of a given value of the node 5783 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 5784 if (UI.getUse().getResNo() == Value) { 5785 if (NUses == 0) 5786 return false; 5787 --NUses; 5788 } 5789 } 5790 5791 // Found exactly the right number of uses? 5792 return NUses == 0; 5793} 5794 5795 5796/// hasAnyUseOfValue - Return true if there are any use of the indicated 5797/// value. This method ignores uses of other values defined by this operation. 5798bool SDNode::hasAnyUseOfValue(unsigned Value) const { 5799 assert(Value < getNumValues() && "Bad value!"); 5800 5801 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 5802 if (UI.getUse().getResNo() == Value) 5803 return true; 5804 5805 return false; 5806} 5807 5808 5809/// isOnlyUserOf - Return true if this node is the only use of N. 5810/// 5811bool SDNode::isOnlyUserOf(SDNode *N) const { 5812 bool Seen = false; 5813 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 5814 SDNode *User = *I; 5815 if (User == this) 5816 Seen = true; 5817 else 5818 return false; 5819 } 5820 5821 return Seen; 5822} 5823 5824/// isOperand - Return true if this node is an operand of N. 5825/// 5826bool SDValue::isOperandOf(SDNode *N) const { 5827 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 5828 if (*this == N->getOperand(i)) 5829 return true; 5830 return false; 5831} 5832 5833bool SDNode::isOperandOf(SDNode *N) const { 5834 for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 5835 if (this == N->OperandList[i].getNode()) 5836 return true; 5837 return false; 5838} 5839 5840/// reachesChainWithoutSideEffects - Return true if this operand (which must 5841/// be a chain) reaches the specified operand without crossing any 5842/// side-effecting instructions on any chain path. In practice, this looks 5843/// through token factors and non-volatile loads. In order to remain efficient, 5844/// this only looks a couple of nodes in, it does not do an exhaustive search. 5845bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 5846 unsigned Depth) const { 5847 if (*this == Dest) return true; 5848 5849 // Don't search too deeply, we just want to be able to see through 5850 // TokenFactor's etc. 5851 if (Depth == 0) return false; 5852 5853 // If this is a token factor, all inputs to the TF happen in parallel. If any 5854 // of the operands of the TF does not reach dest, then we cannot do the xform. 5855 if (getOpcode() == ISD::TokenFactor) { 5856 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 5857 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 5858 return false; 5859 return true; 5860 } 5861 5862 // Loads don't have side effects, look through them. 5863 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 5864 if (!Ld->isVolatile()) 5865 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 5866 } 5867 return false; 5868} 5869 5870/// hasPredecessor - Return true if N is a predecessor of this node. 5871/// N is either an operand of this node, or can be reached by recursively 5872/// traversing up the operands. 5873/// NOTE: This is an expensive method. Use it carefully. 5874bool SDNode::hasPredecessor(const SDNode *N) const { 5875 SmallPtrSet<const SDNode *, 32> Visited; 5876 SmallVector<const SDNode *, 16> Worklist; 5877 return hasPredecessorHelper(N, Visited, Worklist); 5878} 5879 5880bool SDNode::hasPredecessorHelper(const SDNode *N, 5881 SmallPtrSet<const SDNode *, 32> &Visited, 5882 SmallVector<const SDNode *, 16> &Worklist) const { 5883 if (Visited.empty()) { 5884 Worklist.push_back(this); 5885 } else { 5886 // Take a look in the visited set. If we've already encountered this node 5887 // we needn't search further. 5888 if (Visited.count(N)) 5889 return true; 5890 } 5891 5892 // Haven't visited N yet. Continue the search. 5893 while (!Worklist.empty()) { 5894 const SDNode *M = Worklist.pop_back_val(); 5895 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 5896 SDNode *Op = M->getOperand(i).getNode(); 5897 if (Visited.insert(Op)) 5898 Worklist.push_back(Op); 5899 if (Op == N) 5900 return true; 5901 } 5902 } 5903 5904 return false; 5905} 5906 5907uint64_t SDNode::getConstantOperandVal(unsigned Num) const { 5908 assert(Num < NumOperands && "Invalid child # of SDNode!"); 5909 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 5910} 5911 5912std::string SDNode::getOperationName(const SelectionDAG *G) const { 5913 switch (getOpcode()) { 5914 default: 5915 if (getOpcode() < ISD::BUILTIN_OP_END) 5916 return "<<Unknown DAG Node>>"; 5917 if (isMachineOpcode()) { 5918 if (G) 5919 if (const TargetInstrInfo *TII = G->getTarget().getInstrInfo()) 5920 if (getMachineOpcode() < TII->getNumOpcodes()) 5921 return TII->get(getMachineOpcode()).getName(); 5922 return "<<Unknown Machine Node #" + utostr(getOpcode()) + ">>"; 5923 } 5924 if (G) { 5925 const TargetLowering &TLI = G->getTargetLoweringInfo(); 5926 const char *Name = TLI.getTargetNodeName(getOpcode()); 5927 if (Name) return Name; 5928 return "<<Unknown Target Node #" + utostr(getOpcode()) + ">>"; 5929 } 5930 return "<<Unknown Node #" + utostr(getOpcode()) + ">>"; 5931 5932#ifndef NDEBUG 5933 case ISD::DELETED_NODE: 5934 return "<<Deleted Node!>>"; 5935#endif 5936 case ISD::PREFETCH: return "Prefetch"; 5937 case ISD::MEMBARRIER: return "MemBarrier"; 5938 case ISD::ATOMIC_FENCE: return "AtomicFence"; 5939 case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap"; 5940 case ISD::ATOMIC_SWAP: return "AtomicSwap"; 5941 case ISD::ATOMIC_LOAD_ADD: return "AtomicLoadAdd"; 5942 case ISD::ATOMIC_LOAD_SUB: return "AtomicLoadSub"; 5943 case ISD::ATOMIC_LOAD_AND: return "AtomicLoadAnd"; 5944 case ISD::ATOMIC_LOAD_OR: return "AtomicLoadOr"; 5945 case ISD::ATOMIC_LOAD_XOR: return "AtomicLoadXor"; 5946 case ISD::ATOMIC_LOAD_NAND: return "AtomicLoadNand"; 5947 case ISD::ATOMIC_LOAD_MIN: return "AtomicLoadMin"; 5948 case ISD::ATOMIC_LOAD_MAX: return "AtomicLoadMax"; 5949 case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin"; 5950 case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax"; 5951 case ISD::ATOMIC_LOAD: return "AtomicLoad"; 5952 case ISD::ATOMIC_STORE: return "AtomicStore"; 5953 case ISD::PCMARKER: return "PCMarker"; 5954 case ISD::READCYCLECOUNTER: return "ReadCycleCounter"; 5955 case ISD::SRCVALUE: return "SrcValue"; 5956 case ISD::MDNODE_SDNODE: return "MDNode"; 5957 case ISD::EntryToken: return "EntryToken"; 5958 case ISD::TokenFactor: return "TokenFactor"; 5959 case ISD::AssertSext: return "AssertSext"; 5960 case ISD::AssertZext: return "AssertZext"; 5961 5962 case ISD::BasicBlock: return "BasicBlock"; 5963 case ISD::VALUETYPE: return "ValueType"; 5964 case ISD::Register: return "Register"; 5965 case ISD::RegisterMask: return "RegisterMask"; 5966 case ISD::Constant: return "Constant"; 5967 case ISD::ConstantFP: return "ConstantFP"; 5968 case ISD::GlobalAddress: return "GlobalAddress"; 5969 case ISD::GlobalTLSAddress: return "GlobalTLSAddress"; 5970 case ISD::FrameIndex: return "FrameIndex"; 5971 case ISD::JumpTable: return "JumpTable"; 5972 case ISD::GLOBAL_OFFSET_TABLE: return "GLOBAL_OFFSET_TABLE"; 5973 case ISD::RETURNADDR: return "RETURNADDR"; 5974 case ISD::FRAMEADDR: return "FRAMEADDR"; 5975 case ISD::FRAME_TO_ARGS_OFFSET: return "FRAME_TO_ARGS_OFFSET"; 5976 case ISD::EXCEPTIONADDR: return "EXCEPTIONADDR"; 5977 case ISD::LSDAADDR: return "LSDAADDR"; 5978 case ISD::EHSELECTION: return "EHSELECTION"; 5979 case ISD::EH_RETURN: return "EH_RETURN"; 5980 case ISD::EH_SJLJ_SETJMP: return "EH_SJLJ_SETJMP"; 5981 case ISD::EH_SJLJ_LONGJMP: return "EH_SJLJ_LONGJMP"; 5982 case ISD::ConstantPool: return "ConstantPool"; 5983 case ISD::ExternalSymbol: return "ExternalSymbol"; 5984 case ISD::BlockAddress: return "BlockAddress"; 5985 case ISD::INTRINSIC_WO_CHAIN: 5986 case ISD::INTRINSIC_VOID: 5987 case ISD::INTRINSIC_W_CHAIN: { 5988 unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1; 5989 unsigned IID = cast<ConstantSDNode>(getOperand(OpNo))->getZExtValue(); 5990 if (IID < Intrinsic::num_intrinsics) 5991 return Intrinsic::getName((Intrinsic::ID)IID); 5992 else if (const TargetIntrinsicInfo *TII = G->getTarget().getIntrinsicInfo()) 5993 return TII->getName(IID); 5994 llvm_unreachable("Invalid intrinsic ID"); 5995 } 5996 5997 case ISD::BUILD_VECTOR: return "BUILD_VECTOR"; 5998 case ISD::TargetConstant: return "TargetConstant"; 5999 case ISD::TargetConstantFP:return "TargetConstantFP"; 6000 case ISD::TargetGlobalAddress: return "TargetGlobalAddress"; 6001 case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress"; 6002 case ISD::TargetFrameIndex: return "TargetFrameIndex"; 6003 case ISD::TargetJumpTable: return "TargetJumpTable"; 6004 case ISD::TargetConstantPool: return "TargetConstantPool"; 6005 case ISD::TargetExternalSymbol: return "TargetExternalSymbol"; 6006 case ISD::TargetBlockAddress: return "TargetBlockAddress"; 6007 6008 case ISD::CopyToReg: return "CopyToReg"; 6009 case ISD::CopyFromReg: return "CopyFromReg"; 6010 case ISD::UNDEF: return "undef"; 6011 case ISD::MERGE_VALUES: return "merge_values"; 6012 case ISD::INLINEASM: return "inlineasm"; 6013 case ISD::EH_LABEL: return "eh_label"; 6014 case ISD::HANDLENODE: return "handlenode"; 6015 6016 // Unary operators 6017 case ISD::FABS: return "fabs"; 6018 case ISD::FNEG: return "fneg"; 6019 case ISD::FSQRT: return "fsqrt"; 6020 case ISD::FSIN: return "fsin"; 6021 case ISD::FCOS: return "fcos"; 6022 case ISD::FTRUNC: return "ftrunc"; 6023 case ISD::FFLOOR: return "ffloor"; 6024 case ISD::FCEIL: return "fceil"; 6025 case ISD::FRINT: return "frint"; 6026 case ISD::FNEARBYINT: return "fnearbyint"; 6027 case ISD::FEXP: return "fexp"; 6028 case ISD::FEXP2: return "fexp2"; 6029 case ISD::FLOG: return "flog"; 6030 case ISD::FLOG2: return "flog2"; 6031 case ISD::FLOG10: return "flog10"; 6032 6033 // Binary operators 6034 case ISD::ADD: return "add"; 6035 case ISD::SUB: return "sub"; 6036 case ISD::MUL: return "mul"; 6037 case ISD::MULHU: return "mulhu"; 6038 case ISD::MULHS: return "mulhs"; 6039 case ISD::SDIV: return "sdiv"; 6040 case ISD::UDIV: return "udiv"; 6041 case ISD::SREM: return "srem"; 6042 case ISD::UREM: return "urem"; 6043 case ISD::SMUL_LOHI: return "smul_lohi"; 6044 case ISD::UMUL_LOHI: return "umul_lohi"; 6045 case ISD::SDIVREM: return "sdivrem"; 6046 case ISD::UDIVREM: return "udivrem"; 6047 case ISD::AND: return "and"; 6048 case ISD::OR: return "or"; 6049 case ISD::XOR: return "xor"; 6050 case ISD::SHL: return "shl"; 6051 case ISD::SRA: return "sra"; 6052 case ISD::SRL: return "srl"; 6053 case ISD::ROTL: return "rotl"; 6054 case ISD::ROTR: return "rotr"; 6055 case ISD::FADD: return "fadd"; 6056 case ISD::FSUB: return "fsub"; 6057 case ISD::FMUL: return "fmul"; 6058 case ISD::FDIV: return "fdiv"; 6059 case ISD::FMA: return "fma"; 6060 case ISD::FREM: return "frem"; 6061 case ISD::FCOPYSIGN: return "fcopysign"; 6062 case ISD::FGETSIGN: return "fgetsign"; 6063 case ISD::FPOW: return "fpow"; 6064 6065 case ISD::FPOWI: return "fpowi"; 6066 case ISD::SETCC: return "setcc"; 6067 case ISD::SELECT: return "select"; 6068 case ISD::VSELECT: return "vselect"; 6069 case ISD::SELECT_CC: return "select_cc"; 6070 case ISD::INSERT_VECTOR_ELT: return "insert_vector_elt"; 6071 case ISD::EXTRACT_VECTOR_ELT: return "extract_vector_elt"; 6072 case ISD::CONCAT_VECTORS: return "concat_vectors"; 6073 case ISD::INSERT_SUBVECTOR: return "insert_subvector"; 6074 case ISD::EXTRACT_SUBVECTOR: return "extract_subvector"; 6075 case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector"; 6076 case ISD::VECTOR_SHUFFLE: return "vector_shuffle"; 6077 case ISD::CARRY_FALSE: return "carry_false"; 6078 case ISD::ADDC: return "addc"; 6079 case ISD::ADDE: return "adde"; 6080 case ISD::SADDO: return "saddo"; 6081 case ISD::UADDO: return "uaddo"; 6082 case ISD::SSUBO: return "ssubo"; 6083 case ISD::USUBO: return "usubo"; 6084 case ISD::SMULO: return "smulo"; 6085 case ISD::UMULO: return "umulo"; 6086 case ISD::SUBC: return "subc"; 6087 case ISD::SUBE: return "sube"; 6088 case ISD::SHL_PARTS: return "shl_parts"; 6089 case ISD::SRA_PARTS: return "sra_parts"; 6090 case ISD::SRL_PARTS: return "srl_parts"; 6091 6092 // Conversion operators. 6093 case ISD::SIGN_EXTEND: return "sign_extend"; 6094 case ISD::ZERO_EXTEND: return "zero_extend"; 6095 case ISD::ANY_EXTEND: return "any_extend"; 6096 case ISD::SIGN_EXTEND_INREG: return "sign_extend_inreg"; 6097 case ISD::TRUNCATE: return "truncate"; 6098 case ISD::FP_ROUND: return "fp_round"; 6099 case ISD::FLT_ROUNDS_: return "flt_rounds"; 6100 case ISD::FP_ROUND_INREG: return "fp_round_inreg"; 6101 case ISD::FP_EXTEND: return "fp_extend"; 6102 6103 case ISD::SINT_TO_FP: return "sint_to_fp"; 6104 case ISD::UINT_TO_FP: return "uint_to_fp"; 6105 case ISD::FP_TO_SINT: return "fp_to_sint"; 6106 case ISD::FP_TO_UINT: return "fp_to_uint"; 6107 case ISD::BITCAST: return "bitcast"; 6108 case ISD::FP16_TO_FP32: return "fp16_to_fp32"; 6109 case ISD::FP32_TO_FP16: return "fp32_to_fp16"; 6110 6111 case ISD::CONVERT_RNDSAT: { 6112 switch (cast<CvtRndSatSDNode>(this)->getCvtCode()) { 6113 default: llvm_unreachable("Unknown cvt code!"); 6114 case ISD::CVT_FF: return "cvt_ff"; 6115 case ISD::CVT_FS: return "cvt_fs"; 6116 case ISD::CVT_FU: return "cvt_fu"; 6117 case ISD::CVT_SF: return "cvt_sf"; 6118 case ISD::CVT_UF: return "cvt_uf"; 6119 case ISD::CVT_SS: return "cvt_ss"; 6120 case ISD::CVT_SU: return "cvt_su"; 6121 case ISD::CVT_US: return "cvt_us"; 6122 case ISD::CVT_UU: return "cvt_uu"; 6123 } 6124 } 6125 6126 // Control flow instructions 6127 case ISD::BR: return "br"; 6128 case ISD::BRIND: return "brind"; 6129 case ISD::BR_JT: return "br_jt"; 6130 case ISD::BRCOND: return "brcond"; 6131 case ISD::BR_CC: return "br_cc"; 6132 case ISD::CALLSEQ_START: return "callseq_start"; 6133 case ISD::CALLSEQ_END: return "callseq_end"; 6134 6135 // Other operators 6136 case ISD::LOAD: return "load"; 6137 case ISD::STORE: return "store"; 6138 case ISD::VAARG: return "vaarg"; 6139 case ISD::VACOPY: return "vacopy"; 6140 case ISD::VAEND: return "vaend"; 6141 case ISD::VASTART: return "vastart"; 6142 case ISD::DYNAMIC_STACKALLOC: return "dynamic_stackalloc"; 6143 case ISD::EXTRACT_ELEMENT: return "extract_element"; 6144 case ISD::BUILD_PAIR: return "build_pair"; 6145 case ISD::STACKSAVE: return "stacksave"; 6146 case ISD::STACKRESTORE: return "stackrestore"; 6147 case ISD::TRAP: return "trap"; 6148 6149 // Bit manipulation 6150 case ISD::BSWAP: return "bswap"; 6151 case ISD::CTPOP: return "ctpop"; 6152 case ISD::CTTZ: return "cttz"; 6153 case ISD::CTTZ_ZERO_UNDEF: return "cttz_zero_undef"; 6154 case ISD::CTLZ: return "ctlz"; 6155 case ISD::CTLZ_ZERO_UNDEF: return "ctlz_zero_undef"; 6156 6157 // Trampolines 6158 case ISD::INIT_TRAMPOLINE: return "init_trampoline"; 6159 case ISD::ADJUST_TRAMPOLINE: return "adjust_trampoline"; 6160 6161 case ISD::CONDCODE: 6162 switch (cast<CondCodeSDNode>(this)->get()) { 6163 default: llvm_unreachable("Unknown setcc condition!"); 6164 case ISD::SETOEQ: return "setoeq"; 6165 case ISD::SETOGT: return "setogt"; 6166 case ISD::SETOGE: return "setoge"; 6167 case ISD::SETOLT: return "setolt"; 6168 case ISD::SETOLE: return "setole"; 6169 case ISD::SETONE: return "setone"; 6170 6171 case ISD::SETO: return "seto"; 6172 case ISD::SETUO: return "setuo"; 6173 case ISD::SETUEQ: return "setue"; 6174 case ISD::SETUGT: return "setugt"; 6175 case ISD::SETUGE: return "setuge"; 6176 case ISD::SETULT: return "setult"; 6177 case ISD::SETULE: return "setule"; 6178 case ISD::SETUNE: return "setune"; 6179 6180 case ISD::SETEQ: return "seteq"; 6181 case ISD::SETGT: return "setgt"; 6182 case ISD::SETGE: return "setge"; 6183 case ISD::SETLT: return "setlt"; 6184 case ISD::SETLE: return "setle"; 6185 case ISD::SETNE: return "setne"; 6186 6187 case ISD::SETTRUE: return "settrue"; 6188 case ISD::SETTRUE2: return "settrue2"; 6189 case ISD::SETFALSE: return "setfalse"; 6190 case ISD::SETFALSE2: return "setfalse2"; 6191 } 6192 } 6193} 6194 6195const char *SDNode::getIndexedModeName(ISD::MemIndexedMode AM) { 6196 switch (AM) { 6197 default: 6198 return ""; 6199 case ISD::PRE_INC: 6200 return "<pre-inc>"; 6201 case ISD::PRE_DEC: 6202 return "<pre-dec>"; 6203 case ISD::POST_INC: 6204 return "<post-inc>"; 6205 case ISD::POST_DEC: 6206 return "<post-dec>"; 6207 } 6208} 6209 6210std::string ISD::ArgFlagsTy::getArgFlagsString() { 6211 std::string S = "< "; 6212 6213 if (isZExt()) 6214 S += "zext "; 6215 if (isSExt()) 6216 S += "sext "; 6217 if (isInReg()) 6218 S += "inreg "; 6219 if (isSRet()) 6220 S += "sret "; 6221 if (isByVal()) 6222 S += "byval "; 6223 if (isNest()) 6224 S += "nest "; 6225 if (getByValAlign()) 6226 S += "byval-align:" + utostr(getByValAlign()) + " "; 6227 if (getOrigAlign()) 6228 S += "orig-align:" + utostr(getOrigAlign()) + " "; 6229 if (getByValSize()) 6230 S += "byval-size:" + utostr(getByValSize()) + " "; 6231 return S + ">"; 6232} 6233 6234void SDNode::dump() const { dump(0); } 6235void SDNode::dump(const SelectionDAG *G) const { 6236 print(dbgs(), G); 6237 dbgs() << '\n'; 6238} 6239 6240void SDNode::print_types(raw_ostream &OS, const SelectionDAG *G) const { 6241 OS << (void*)this << ": "; 6242 6243 for (unsigned i = 0, e = getNumValues(); i != e; ++i) { 6244 if (i) OS << ","; 6245 if (getValueType(i) == MVT::Other) 6246 OS << "ch"; 6247 else 6248 OS << getValueType(i).getEVTString(); 6249 } 6250 OS << " = " << getOperationName(G); 6251} 6252 6253void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { 6254 if (const MachineSDNode *MN = dyn_cast<MachineSDNode>(this)) { 6255 if (!MN->memoperands_empty()) { 6256 OS << "<"; 6257 OS << "Mem:"; 6258 for (MachineSDNode::mmo_iterator i = MN->memoperands_begin(), 6259 e = MN->memoperands_end(); i != e; ++i) { 6260 OS << **i; 6261 if (llvm::next(i) != e) 6262 OS << " "; 6263 } 6264 OS << ">"; 6265 } 6266 } else if (const ShuffleVectorSDNode *SVN = 6267 dyn_cast<ShuffleVectorSDNode>(this)) { 6268 OS << "<"; 6269 for (unsigned i = 0, e = ValueList[0].getVectorNumElements(); i != e; ++i) { 6270 int Idx = SVN->getMaskElt(i); 6271 if (i) OS << ","; 6272 if (Idx < 0) 6273 OS << "u"; 6274 else 6275 OS << Idx; 6276 } 6277 OS << ">"; 6278 } else if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(this)) { 6279 OS << '<' << CSDN->getAPIntValue() << '>'; 6280 } else if (const ConstantFPSDNode *CSDN = dyn_cast<ConstantFPSDNode>(this)) { 6281 if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEsingle) 6282 OS << '<' << CSDN->getValueAPF().convertToFloat() << '>'; 6283 else if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEdouble) 6284 OS << '<' << CSDN->getValueAPF().convertToDouble() << '>'; 6285 else { 6286 OS << "<APFloat("; 6287 CSDN->getValueAPF().bitcastToAPInt().dump(); 6288 OS << ")>"; 6289 } 6290 } else if (const GlobalAddressSDNode *GADN = 6291 dyn_cast<GlobalAddressSDNode>(this)) { 6292 int64_t offset = GADN->getOffset(); 6293 OS << '<'; 6294 WriteAsOperand(OS, GADN->getGlobal()); 6295 OS << '>'; 6296 if (offset > 0) 6297 OS << " + " << offset; 6298 else 6299 OS << " " << offset; 6300 if (unsigned int TF = GADN->getTargetFlags()) 6301 OS << " [TF=" << TF << ']'; 6302 } else if (const FrameIndexSDNode *FIDN = dyn_cast<FrameIndexSDNode>(this)) { 6303 OS << "<" << FIDN->getIndex() << ">"; 6304 } else if (const JumpTableSDNode *JTDN = dyn_cast<JumpTableSDNode>(this)) { 6305 OS << "<" << JTDN->getIndex() << ">"; 6306 if (unsigned int TF = JTDN->getTargetFlags()) 6307 OS << " [TF=" << TF << ']'; 6308 } else if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(this)){ 6309 int offset = CP->getOffset(); 6310 if (CP->isMachineConstantPoolEntry()) 6311 OS << "<" << *CP->getMachineCPVal() << ">"; 6312 else 6313 OS << "<" << *CP->getConstVal() << ">"; 6314 if (offset > 0) 6315 OS << " + " << offset; 6316 else 6317 OS << " " << offset; 6318 if (unsigned int TF = CP->getTargetFlags()) 6319 OS << " [TF=" << TF << ']'; 6320 } else if (const BasicBlockSDNode *BBDN = dyn_cast<BasicBlockSDNode>(this)) { 6321 OS << "<"; 6322 const Value *LBB = (const Value*)BBDN->getBasicBlock()->getBasicBlock(); 6323 if (LBB) 6324 OS << LBB->getName() << " "; 6325 OS << (const void*)BBDN->getBasicBlock() << ">"; 6326 } else if (const RegisterSDNode *R = dyn_cast<RegisterSDNode>(this)) { 6327 OS << ' ' << PrintReg(R->getReg(), G ? G->getTarget().getRegisterInfo() :0); 6328 } else if (const ExternalSymbolSDNode *ES = 6329 dyn_cast<ExternalSymbolSDNode>(this)) { 6330 OS << "'" << ES->getSymbol() << "'"; 6331 if (unsigned int TF = ES->getTargetFlags()) 6332 OS << " [TF=" << TF << ']'; 6333 } else if (const SrcValueSDNode *M = dyn_cast<SrcValueSDNode>(this)) { 6334 if (M->getValue()) 6335 OS << "<" << M->getValue() << ">"; 6336 else 6337 OS << "<null>"; 6338 } else if (const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(this)) { 6339 if (MD->getMD()) 6340 OS << "<" << MD->getMD() << ">"; 6341 else 6342 OS << "<null>"; 6343 } else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) { 6344 OS << ":" << N->getVT().getEVTString(); 6345 } 6346 else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) { 6347 OS << "<" << *LD->getMemOperand(); 6348 6349 bool doExt = true; 6350 switch (LD->getExtensionType()) { 6351 default: doExt = false; break; 6352 case ISD::EXTLOAD: OS << ", anyext"; break; 6353 case ISD::SEXTLOAD: OS << ", sext"; break; 6354 case ISD::ZEXTLOAD: OS << ", zext"; break; 6355 } 6356 if (doExt) 6357 OS << " from " << LD->getMemoryVT().getEVTString(); 6358 6359 const char *AM = getIndexedModeName(LD->getAddressingMode()); 6360 if (*AM) 6361 OS << ", " << AM; 6362 6363 OS << ">"; 6364 } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(this)) { 6365 OS << "<" << *ST->getMemOperand(); 6366 6367 if (ST->isTruncatingStore()) 6368 OS << ", trunc to " << ST->getMemoryVT().getEVTString(); 6369 6370 const char *AM = getIndexedModeName(ST->getAddressingMode()); 6371 if (*AM) 6372 OS << ", " << AM; 6373 6374 OS << ">"; 6375 } else if (const MemSDNode* M = dyn_cast<MemSDNode>(this)) { 6376 OS << "<" << *M->getMemOperand() << ">"; 6377 } else if (const BlockAddressSDNode *BA = 6378 dyn_cast<BlockAddressSDNode>(this)) { 6379 OS << "<"; 6380 WriteAsOperand(OS, BA->getBlockAddress()->getFunction(), false); 6381 OS << ", "; 6382 WriteAsOperand(OS, BA->getBlockAddress()->getBasicBlock(), false); 6383 OS << ">"; 6384 if (unsigned int TF = BA->getTargetFlags()) 6385 OS << " [TF=" << TF << ']'; 6386 } 6387 6388 if (G) 6389 if (unsigned Order = G->GetOrdering(this)) 6390 OS << " [ORD=" << Order << ']'; 6391 6392 if (getNodeId() != -1) 6393 OS << " [ID=" << getNodeId() << ']'; 6394 6395 DebugLoc dl = getDebugLoc(); 6396 if (G && !dl.isUnknown()) { 6397 DIScope 6398 Scope(dl.getScope(G->getMachineFunction().getFunction()->getContext())); 6399 OS << " dbg:"; 6400 // Omit the directory, since it's usually long and uninteresting. 6401 if (Scope.Verify()) 6402 OS << Scope.getFilename(); 6403 else 6404 OS << "<unknown>"; 6405 OS << ':' << dl.getLine(); 6406 if (dl.getCol() != 0) 6407 OS << ':' << dl.getCol(); 6408 } 6409} 6410 6411void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { 6412 print_types(OS, G); 6413 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { 6414 if (i) OS << ", "; else OS << " "; 6415 OS << (void*)getOperand(i).getNode(); 6416 if (unsigned RN = getOperand(i).getResNo()) 6417 OS << ":" << RN; 6418 } 6419 print_details(OS, G); 6420} 6421 6422static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N, 6423 const SelectionDAG *G, unsigned depth, 6424 unsigned indent) { 6425 if (depth == 0) 6426 return; 6427 6428 OS.indent(indent); 6429 6430 N->print(OS, G); 6431 6432 if (depth < 1) 6433 return; 6434 6435 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6436 // Don't follow chain operands. 6437 if (N->getOperand(i).getValueType() == MVT::Other) 6438 continue; 6439 OS << '\n'; 6440 printrWithDepthHelper(OS, N->getOperand(i).getNode(), G, depth-1, indent+2); 6441 } 6442} 6443 6444void SDNode::printrWithDepth(raw_ostream &OS, const SelectionDAG *G, 6445 unsigned depth) const { 6446 printrWithDepthHelper(OS, this, G, depth, 0); 6447} 6448 6449void SDNode::printrFull(raw_ostream &OS, const SelectionDAG *G) const { 6450 // Don't print impossibly deep things. 6451 printrWithDepth(OS, G, 10); 6452} 6453 6454void SDNode::dumprWithDepth(const SelectionDAG *G, unsigned depth) const { 6455 printrWithDepth(dbgs(), G, depth); 6456} 6457 6458void SDNode::dumprFull(const SelectionDAG *G) const { 6459 // Don't print impossibly deep things. 6460 dumprWithDepth(G, 10); 6461} 6462 6463static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) { 6464 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6465 if (N->getOperand(i).getNode()->hasOneUse()) 6466 DumpNodes(N->getOperand(i).getNode(), indent+2, G); 6467 else 6468 dbgs() << "\n" << std::string(indent+2, ' ') 6469 << (void*)N->getOperand(i).getNode() << ": <multiple use>"; 6470 6471 6472 dbgs() << "\n"; 6473 dbgs().indent(indent); 6474 N->dump(G); 6475} 6476 6477SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6478 assert(N->getNumValues() == 1 && 6479 "Can't unroll a vector with multiple results!"); 6480 6481 EVT VT = N->getValueType(0); 6482 unsigned NE = VT.getVectorNumElements(); 6483 EVT EltVT = VT.getVectorElementType(); 6484 DebugLoc dl = N->getDebugLoc(); 6485 6486 SmallVector<SDValue, 8> Scalars; 6487 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6488 6489 // If ResNE is 0, fully unroll the vector op. 6490 if (ResNE == 0) 6491 ResNE = NE; 6492 else if (NE > ResNE) 6493 NE = ResNE; 6494 6495 unsigned i; 6496 for (i= 0; i != NE; ++i) { 6497 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6498 SDValue Operand = N->getOperand(j); 6499 EVT OperandVT = Operand.getValueType(); 6500 if (OperandVT.isVector()) { 6501 // A vector operand; extract a single element. 6502 EVT OperandEltVT = OperandVT.getVectorElementType(); 6503 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6504 OperandEltVT, 6505 Operand, 6506 getConstant(i, TLI.getPointerTy())); 6507 } else { 6508 // A scalar operand; just use it as is. 6509 Operands[j] = Operand; 6510 } 6511 } 6512 6513 switch (N->getOpcode()) { 6514 default: 6515 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6516 &Operands[0], Operands.size())); 6517 break; 6518 case ISD::VSELECT: 6519 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6520 &Operands[0], Operands.size())); 6521 break; 6522 case ISD::SHL: 6523 case ISD::SRA: 6524 case ISD::SRL: 6525 case ISD::ROTL: 6526 case ISD::ROTR: 6527 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6528 getShiftAmountOperand(Operands[0].getValueType(), 6529 Operands[1]))); 6530 break; 6531 case ISD::SIGN_EXTEND_INREG: 6532 case ISD::FP_ROUND_INREG: { 6533 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6534 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6535 Operands[0], 6536 getValueType(ExtVT))); 6537 } 6538 } 6539 } 6540 6541 for (; i < ResNE; ++i) 6542 Scalars.push_back(getUNDEF(EltVT)); 6543 6544 return getNode(ISD::BUILD_VECTOR, dl, 6545 EVT::getVectorVT(*getContext(), EltVT, ResNE), 6546 &Scalars[0], Scalars.size()); 6547} 6548 6549 6550/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6551/// location that is 'Dist' units away from the location that the 'Base' load 6552/// is loading from. 6553bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6554 unsigned Bytes, int Dist) const { 6555 if (LD->getChain() != Base->getChain()) 6556 return false; 6557 EVT VT = LD->getValueType(0); 6558 if (VT.getSizeInBits() / 8 != Bytes) 6559 return false; 6560 6561 SDValue Loc = LD->getOperand(1); 6562 SDValue BaseLoc = Base->getOperand(1); 6563 if (Loc.getOpcode() == ISD::FrameIndex) { 6564 if (BaseLoc.getOpcode() != ISD::FrameIndex) 6565 return false; 6566 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6567 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6568 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6569 int FS = MFI->getObjectSize(FI); 6570 int BFS = MFI->getObjectSize(BFI); 6571 if (FS != BFS || FS != (int)Bytes) return false; 6572 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6573 } 6574 6575 // Handle X+C 6576 if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6577 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6578 return true; 6579 6580 const GlobalValue *GV1 = NULL; 6581 const GlobalValue *GV2 = NULL; 6582 int64_t Offset1 = 0; 6583 int64_t Offset2 = 0; 6584 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6585 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6586 if (isGA1 && isGA2 && GV1 == GV2) 6587 return Offset1 == (Offset2 + Dist*Bytes); 6588 return false; 6589} 6590 6591 6592/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6593/// it cannot be inferred. 6594unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6595 // If this is a GlobalAddress + cst, return the alignment. 6596 const GlobalValue *GV; 6597 int64_t GVOffset = 0; 6598 if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6599 unsigned PtrWidth = TLI.getPointerTy().getSizeInBits(); 6600 APInt AllOnes = APInt::getAllOnesValue(PtrWidth); 6601 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 6602 llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), AllOnes, 6603 KnownZero, KnownOne, TLI.getTargetData()); 6604 unsigned AlignBits = KnownZero.countTrailingOnes(); 6605 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 6606 if (Align) 6607 return MinAlign(Align, GVOffset); 6608 } 6609 6610 // If this is a direct reference to a stack slot, use information about the 6611 // stack slot's alignment. 6612 int FrameIdx = 1 << 31; 6613 int64_t FrameOffset = 0; 6614 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6615 FrameIdx = FI->getIndex(); 6616 } else if (isBaseWithConstantOffset(Ptr) && 6617 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6618 // Handle FI+Cst 6619 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6620 FrameOffset = Ptr.getConstantOperandVal(1); 6621 } 6622 6623 if (FrameIdx != (1 << 31)) { 6624 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6625 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6626 FrameOffset); 6627 return FIInfoAlign; 6628 } 6629 6630 return 0; 6631} 6632 6633void SelectionDAG::dump() const { 6634 dbgs() << "SelectionDAG has " << AllNodes.size() << " nodes:"; 6635 6636 for (allnodes_const_iterator I = allnodes_begin(), E = allnodes_end(); 6637 I != E; ++I) { 6638 const SDNode *N = I; 6639 if (!N->hasOneUse() && N != getRoot().getNode()) 6640 DumpNodes(N, 2, this); 6641 } 6642 6643 if (getRoot().getNode()) DumpNodes(getRoot().getNode(), 2, this); 6644 6645 dbgs() << "\n\n"; 6646} 6647 6648void SDNode::printr(raw_ostream &OS, const SelectionDAG *G) const { 6649 print_types(OS, G); 6650 print_details(OS, G); 6651} 6652 6653typedef SmallPtrSet<const SDNode *, 128> VisitedSDNodeSet; 6654static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent, 6655 const SelectionDAG *G, VisitedSDNodeSet &once) { 6656 if (!once.insert(N)) // If we've been here before, return now. 6657 return; 6658 6659 // Dump the current SDNode, but don't end the line yet. 6660 OS.indent(indent); 6661 N->printr(OS, G); 6662 6663 // Having printed this SDNode, walk the children: 6664 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6665 const SDNode *child = N->getOperand(i).getNode(); 6666 6667 if (i) OS << ","; 6668 OS << " "; 6669 6670 if (child->getNumOperands() == 0) { 6671 // This child has no grandchildren; print it inline right here. 6672 child->printr(OS, G); 6673 once.insert(child); 6674 } else { // Just the address. FIXME: also print the child's opcode. 6675 OS << (void*)child; 6676 if (unsigned RN = N->getOperand(i).getResNo()) 6677 OS << ":" << RN; 6678 } 6679 } 6680 6681 OS << "\n"; 6682 6683 // Dump children that have grandchildren on their own line(s). 6684 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6685 const SDNode *child = N->getOperand(i).getNode(); 6686 DumpNodesr(OS, child, indent+2, G, once); 6687 } 6688} 6689 6690void SDNode::dumpr() const { 6691 VisitedSDNodeSet once; 6692 DumpNodesr(dbgs(), this, 0, 0, once); 6693} 6694 6695void SDNode::dumpr(const SelectionDAG *G) const { 6696 VisitedSDNodeSet once; 6697 DumpNodesr(dbgs(), this, 0, G, once); 6698} 6699 6700 6701// getAddressSpace - Return the address space this GlobalAddress belongs to. 6702unsigned GlobalAddressSDNode::getAddressSpace() const { 6703 return getGlobal()->getType()->getAddressSpace(); 6704} 6705 6706 6707Type *ConstantPoolSDNode::getType() const { 6708 if (isMachineConstantPoolEntry()) 6709 return Val.MachineCPVal->getType(); 6710 return Val.ConstVal->getType(); 6711} 6712 6713bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6714 APInt &SplatUndef, 6715 unsigned &SplatBitSize, 6716 bool &HasAnyUndefs, 6717 unsigned MinSplatBits, 6718 bool isBigEndian) { 6719 EVT VT = getValueType(0); 6720 assert(VT.isVector() && "Expected a vector type"); 6721 unsigned sz = VT.getSizeInBits(); 6722 if (MinSplatBits > sz) 6723 return false; 6724 6725 SplatValue = APInt(sz, 0); 6726 SplatUndef = APInt(sz, 0); 6727 6728 // Get the bits. Bits with undefined values (when the corresponding element 6729 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6730 // in SplatValue. If any of the values are not constant, give up and return 6731 // false. 6732 unsigned int nOps = getNumOperands(); 6733 assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6734 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6735 6736 for (unsigned j = 0; j < nOps; ++j) { 6737 unsigned i = isBigEndian ? nOps-1-j : j; 6738 SDValue OpVal = getOperand(i); 6739 unsigned BitPos = j * EltBitSize; 6740 6741 if (OpVal.getOpcode() == ISD::UNDEF) 6742 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6743 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6744 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6745 zextOrTrunc(sz) << BitPos; 6746 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6747 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6748 else 6749 return false; 6750 } 6751 6752 // The build_vector is all constants or undefs. Find the smallest element 6753 // size that splats the vector. 6754 6755 HasAnyUndefs = (SplatUndef != 0); 6756 while (sz > 8) { 6757 6758 unsigned HalfSize = sz / 2; 6759 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6760 APInt LowValue = SplatValue.trunc(HalfSize); 6761 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6762 APInt LowUndef = SplatUndef.trunc(HalfSize); 6763 6764 // If the two halves do not match (ignoring undef bits), stop here. 6765 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6766 MinSplatBits > HalfSize) 6767 break; 6768 6769 SplatValue = HighValue | LowValue; 6770 SplatUndef = HighUndef & LowUndef; 6771 6772 sz = HalfSize; 6773 } 6774 6775 SplatBitSize = sz; 6776 return true; 6777} 6778 6779bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6780 // Find the first non-undef value in the shuffle mask. 6781 unsigned i, e; 6782 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6783 /* search */; 6784 6785 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6786 6787 // Make sure all remaining elements are either undef or the same as the first 6788 // non-undef value. 6789 for (int Idx = Mask[i]; i != e; ++i) 6790 if (Mask[i] >= 0 && Mask[i] != Idx) 6791 return false; 6792 return true; 6793} 6794 6795#ifdef XDEBUG 6796static void checkForCyclesHelper(const SDNode *N, 6797 SmallPtrSet<const SDNode*, 32> &Visited, 6798 SmallPtrSet<const SDNode*, 32> &Checked) { 6799 // If this node has already been checked, don't check it again. 6800 if (Checked.count(N)) 6801 return; 6802 6803 // If a node has already been visited on this depth-first walk, reject it as 6804 // a cycle. 6805 if (!Visited.insert(N)) { 6806 dbgs() << "Offending node:\n"; 6807 N->dumprFull(); 6808 errs() << "Detected cycle in SelectionDAG\n"; 6809 abort(); 6810 } 6811 6812 for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6813 checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6814 6815 Checked.insert(N); 6816 Visited.erase(N); 6817} 6818#endif 6819 6820void llvm::checkForCycles(const llvm::SDNode *N) { 6821#ifdef XDEBUG 6822 assert(N && "Checking nonexistant SDNode"); 6823 SmallPtrSet<const SDNode*, 32> visited; 6824 SmallPtrSet<const SDNode*, 32> checked; 6825 checkForCyclesHelper(N, visited, checked); 6826#endif 6827} 6828 6829void llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6830 checkForCycles(DAG->getRoot().getNode()); 6831} 6832