SelectionDAG.cpp revision d752e0f7e64585839cb3a458ef52456eaebbea3c
1//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This implements the SelectionDAG class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "SDNodeOrdering.h" 16#include "SDNodeDbgValue.h" 17#include "llvm/Constants.h" 18#include "llvm/Analysis/DebugInfo.h" 19#include "llvm/Analysis/ValueTracking.h" 20#include "llvm/Function.h" 21#include "llvm/GlobalAlias.h" 22#include "llvm/GlobalVariable.h" 23#include "llvm/Intrinsics.h" 24#include "llvm/DerivedTypes.h" 25#include "llvm/Assembly/Writer.h" 26#include "llvm/CallingConv.h" 27#include "llvm/CodeGen/MachineBasicBlock.h" 28#include "llvm/CodeGen/MachineConstantPool.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineModuleInfo.h" 31#include "llvm/CodeGen/PseudoSourceValue.h" 32#include "llvm/Target/TargetRegisterInfo.h" 33#include "llvm/Target/TargetData.h" 34#include "llvm/Target/TargetLowering.h" 35#include "llvm/Target/TargetSelectionDAGInfo.h" 36#include "llvm/Target/TargetOptions.h" 37#include "llvm/Target/TargetInstrInfo.h" 38#include "llvm/Target/TargetIntrinsicInfo.h" 39#include "llvm/Target/TargetMachine.h" 40#include "llvm/Support/CommandLine.h" 41#include "llvm/Support/Debug.h" 42#include "llvm/Support/ErrorHandling.h" 43#include "llvm/Support/ManagedStatic.h" 44#include "llvm/Support/MathExtras.h" 45#include "llvm/Support/raw_ostream.h" 46#include "llvm/Support/Mutex.h" 47#include "llvm/ADT/SetVector.h" 48#include "llvm/ADT/SmallPtrSet.h" 49#include "llvm/ADT/SmallSet.h" 50#include "llvm/ADT/SmallVector.h" 51#include "llvm/ADT/StringExtras.h" 52#include <algorithm> 53#include <cmath> 54using namespace llvm; 55 56/// makeVTList - Return an instance of the SDVTList struct initialized with the 57/// specified members. 58static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 59 SDVTList Res = {VTs, NumVTs}; 60 return Res; 61} 62 63static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { 64 switch (VT.getSimpleVT().SimpleTy) { 65 default: llvm_unreachable("Unknown FP format"); 66 case MVT::f32: return &APFloat::IEEEsingle; 67 case MVT::f64: return &APFloat::IEEEdouble; 68 case MVT::f80: return &APFloat::x87DoubleExtended; 69 case MVT::f128: return &APFloat::IEEEquad; 70 case MVT::ppcf128: return &APFloat::PPCDoubleDouble; 71 } 72} 73 74SelectionDAG::DAGUpdateListener::~DAGUpdateListener() {} 75 76//===----------------------------------------------------------------------===// 77// ConstantFPSDNode Class 78//===----------------------------------------------------------------------===// 79 80/// isExactlyValue - We don't rely on operator== working on double values, as 81/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 82/// As such, this method can be used to do an exact bit-for-bit comparison of 83/// two floating point values. 84bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 85 return getValueAPF().bitwiseIsEqual(V); 86} 87 88bool ConstantFPSDNode::isValueValidForType(EVT VT, 89 const APFloat& Val) { 90 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 91 92 // PPC long double cannot be converted to any other type. 93 if (VT == MVT::ppcf128 || 94 &Val.getSemantics() == &APFloat::PPCDoubleDouble) 95 return false; 96 97 // convert modifies in place, so make a copy. 98 APFloat Val2 = APFloat(Val); 99 bool losesInfo; 100 (void) Val2.convert(*EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, 101 &losesInfo); 102 return !losesInfo; 103} 104 105//===----------------------------------------------------------------------===// 106// ISD Namespace 107//===----------------------------------------------------------------------===// 108 109/// isBuildVectorAllOnes - Return true if the specified node is a 110/// BUILD_VECTOR where all of the elements are ~0 or undef. 111bool ISD::isBuildVectorAllOnes(const SDNode *N) { 112 // Look through a bit convert. 113 if (N->getOpcode() == ISD::BITCAST) 114 N = N->getOperand(0).getNode(); 115 116 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 117 118 unsigned i = 0, e = N->getNumOperands(); 119 120 // Skip over all of the undef values. 121 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 122 ++i; 123 124 // Do not accept an all-undef vector. 125 if (i == e) return false; 126 127 // Do not accept build_vectors that aren't all constants or which have non-~0 128 // elements. 129 SDValue NotZero = N->getOperand(i); 130 if (isa<ConstantSDNode>(NotZero)) { 131 if (!cast<ConstantSDNode>(NotZero)->isAllOnesValue()) 132 return false; 133 } else if (isa<ConstantFPSDNode>(NotZero)) { 134 if (!cast<ConstantFPSDNode>(NotZero)->getValueAPF(). 135 bitcastToAPInt().isAllOnesValue()) 136 return false; 137 } else 138 return false; 139 140 // Okay, we have at least one ~0 value, check to see if the rest match or are 141 // undefs. 142 for (++i; i != e; ++i) 143 if (N->getOperand(i) != NotZero && 144 N->getOperand(i).getOpcode() != ISD::UNDEF) 145 return false; 146 return true; 147} 148 149 150/// isBuildVectorAllZeros - Return true if the specified node is a 151/// BUILD_VECTOR where all of the elements are 0 or undef. 152bool ISD::isBuildVectorAllZeros(const SDNode *N) { 153 // Look through a bit convert. 154 if (N->getOpcode() == ISD::BITCAST) 155 N = N->getOperand(0).getNode(); 156 157 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 158 159 unsigned i = 0, e = N->getNumOperands(); 160 161 // Skip over all of the undef values. 162 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 163 ++i; 164 165 // Do not accept an all-undef vector. 166 if (i == e) return false; 167 168 // Do not accept build_vectors that aren't all constants or which have non-0 169 // elements. 170 SDValue Zero = N->getOperand(i); 171 if (isa<ConstantSDNode>(Zero)) { 172 if (!cast<ConstantSDNode>(Zero)->isNullValue()) 173 return false; 174 } else if (isa<ConstantFPSDNode>(Zero)) { 175 if (!cast<ConstantFPSDNode>(Zero)->getValueAPF().isPosZero()) 176 return false; 177 } else 178 return false; 179 180 // Okay, we have at least one 0 value, check to see if the rest match or are 181 // undefs. 182 for (++i; i != e; ++i) 183 if (N->getOperand(i) != Zero && 184 N->getOperand(i).getOpcode() != ISD::UNDEF) 185 return false; 186 return true; 187} 188 189/// isScalarToVector - Return true if the specified node is a 190/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 191/// element is not an undef. 192bool ISD::isScalarToVector(const SDNode *N) { 193 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 194 return true; 195 196 if (N->getOpcode() != ISD::BUILD_VECTOR) 197 return false; 198 if (N->getOperand(0).getOpcode() == ISD::UNDEF) 199 return false; 200 unsigned NumElems = N->getNumOperands(); 201 if (NumElems == 1) 202 return false; 203 for (unsigned i = 1; i < NumElems; ++i) { 204 SDValue V = N->getOperand(i); 205 if (V.getOpcode() != ISD::UNDEF) 206 return false; 207 } 208 return true; 209} 210 211/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 212/// when given the operation for (X op Y). 213ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 214 // To perform this operation, we just need to swap the L and G bits of the 215 // operation. 216 unsigned OldL = (Operation >> 2) & 1; 217 unsigned OldG = (Operation >> 1) & 1; 218 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 219 (OldL << 1) | // New G bit 220 (OldG << 2)); // New L bit. 221} 222 223/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 224/// 'op' is a valid SetCC operation. 225ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 226 unsigned Operation = Op; 227 if (isInteger) 228 Operation ^= 7; // Flip L, G, E bits, but not U. 229 else 230 Operation ^= 15; // Flip all of the condition bits. 231 232 if (Operation > ISD::SETTRUE2) 233 Operation &= ~8; // Don't let N and U bits get set. 234 235 return ISD::CondCode(Operation); 236} 237 238 239/// isSignedOp - For an integer comparison, return 1 if the comparison is a 240/// signed operation and 2 if the result is an unsigned comparison. Return zero 241/// if the operation does not depend on the sign of the input (setne and seteq). 242static int isSignedOp(ISD::CondCode Opcode) { 243 switch (Opcode) { 244 default: llvm_unreachable("Illegal integer setcc operation!"); 245 case ISD::SETEQ: 246 case ISD::SETNE: return 0; 247 case ISD::SETLT: 248 case ISD::SETLE: 249 case ISD::SETGT: 250 case ISD::SETGE: return 1; 251 case ISD::SETULT: 252 case ISD::SETULE: 253 case ISD::SETUGT: 254 case ISD::SETUGE: return 2; 255 } 256} 257 258/// getSetCCOrOperation - Return the result of a logical OR between different 259/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 260/// returns SETCC_INVALID if it is not possible to represent the resultant 261/// comparison. 262ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 263 bool isInteger) { 264 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 265 // Cannot fold a signed integer setcc with an unsigned integer setcc. 266 return ISD::SETCC_INVALID; 267 268 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 269 270 // If the N and U bits get set then the resultant comparison DOES suddenly 271 // care about orderedness, and is true when ordered. 272 if (Op > ISD::SETTRUE2) 273 Op &= ~16; // Clear the U bit if the N bit is set. 274 275 // Canonicalize illegal integer setcc's. 276 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 277 Op = ISD::SETNE; 278 279 return ISD::CondCode(Op); 280} 281 282/// getSetCCAndOperation - Return the result of a logical AND between different 283/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 284/// function returns zero if it is not possible to represent the resultant 285/// comparison. 286ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 287 bool isInteger) { 288 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 289 // Cannot fold a signed setcc with an unsigned setcc. 290 return ISD::SETCC_INVALID; 291 292 // Combine all of the condition bits. 293 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 294 295 // Canonicalize illegal integer setcc's. 296 if (isInteger) { 297 switch (Result) { 298 default: break; 299 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 300 case ISD::SETOEQ: // SETEQ & SETU[LG]E 301 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 302 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 303 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 304 } 305 } 306 307 return Result; 308} 309 310//===----------------------------------------------------------------------===// 311// SDNode Profile Support 312//===----------------------------------------------------------------------===// 313 314/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 315/// 316static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 317 ID.AddInteger(OpC); 318} 319 320/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 321/// solely with their pointer. 322static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 323 ID.AddPointer(VTList.VTs); 324} 325 326/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 327/// 328static void AddNodeIDOperands(FoldingSetNodeID &ID, 329 const SDValue *Ops, unsigned NumOps) { 330 for (; NumOps; --NumOps, ++Ops) { 331 ID.AddPointer(Ops->getNode()); 332 ID.AddInteger(Ops->getResNo()); 333 } 334} 335 336/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 337/// 338static void AddNodeIDOperands(FoldingSetNodeID &ID, 339 const SDUse *Ops, unsigned NumOps) { 340 for (; NumOps; --NumOps, ++Ops) { 341 ID.AddPointer(Ops->getNode()); 342 ID.AddInteger(Ops->getResNo()); 343 } 344} 345 346static void AddNodeIDNode(FoldingSetNodeID &ID, 347 unsigned short OpC, SDVTList VTList, 348 const SDValue *OpList, unsigned N) { 349 AddNodeIDOpcode(ID, OpC); 350 AddNodeIDValueTypes(ID, VTList); 351 AddNodeIDOperands(ID, OpList, N); 352} 353 354/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 355/// the NodeID data. 356static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 357 switch (N->getOpcode()) { 358 case ISD::TargetExternalSymbol: 359 case ISD::ExternalSymbol: 360 llvm_unreachable("Should only be used on nodes with operands"); 361 default: break; // Normal nodes don't need extra info. 362 case ISD::TargetConstant: 363 case ISD::Constant: 364 ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 365 break; 366 case ISD::TargetConstantFP: 367 case ISD::ConstantFP: { 368 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 369 break; 370 } 371 case ISD::TargetGlobalAddress: 372 case ISD::GlobalAddress: 373 case ISD::TargetGlobalTLSAddress: 374 case ISD::GlobalTLSAddress: { 375 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 376 ID.AddPointer(GA->getGlobal()); 377 ID.AddInteger(GA->getOffset()); 378 ID.AddInteger(GA->getTargetFlags()); 379 break; 380 } 381 case ISD::BasicBlock: 382 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 383 break; 384 case ISD::Register: 385 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 386 break; 387 388 case ISD::SRCVALUE: 389 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 390 break; 391 case ISD::FrameIndex: 392 case ISD::TargetFrameIndex: 393 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 394 break; 395 case ISD::JumpTable: 396 case ISD::TargetJumpTable: 397 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 398 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 399 break; 400 case ISD::ConstantPool: 401 case ISD::TargetConstantPool: { 402 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 403 ID.AddInteger(CP->getAlignment()); 404 ID.AddInteger(CP->getOffset()); 405 if (CP->isMachineConstantPoolEntry()) 406 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 407 else 408 ID.AddPointer(CP->getConstVal()); 409 ID.AddInteger(CP->getTargetFlags()); 410 break; 411 } 412 case ISD::LOAD: { 413 const LoadSDNode *LD = cast<LoadSDNode>(N); 414 ID.AddInteger(LD->getMemoryVT().getRawBits()); 415 ID.AddInteger(LD->getRawSubclassData()); 416 break; 417 } 418 case ISD::STORE: { 419 const StoreSDNode *ST = cast<StoreSDNode>(N); 420 ID.AddInteger(ST->getMemoryVT().getRawBits()); 421 ID.AddInteger(ST->getRawSubclassData()); 422 break; 423 } 424 case ISD::ATOMIC_CMP_SWAP: 425 case ISD::ATOMIC_SWAP: 426 case ISD::ATOMIC_LOAD_ADD: 427 case ISD::ATOMIC_LOAD_SUB: 428 case ISD::ATOMIC_LOAD_AND: 429 case ISD::ATOMIC_LOAD_OR: 430 case ISD::ATOMIC_LOAD_XOR: 431 case ISD::ATOMIC_LOAD_NAND: 432 case ISD::ATOMIC_LOAD_MIN: 433 case ISD::ATOMIC_LOAD_MAX: 434 case ISD::ATOMIC_LOAD_UMIN: 435 case ISD::ATOMIC_LOAD_UMAX: 436 case ISD::ATOMIC_LOAD: 437 case ISD::ATOMIC_STORE: { 438 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 439 ID.AddInteger(AT->getMemoryVT().getRawBits()); 440 ID.AddInteger(AT->getRawSubclassData()); 441 break; 442 } 443 case ISD::VECTOR_SHUFFLE: { 444 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 445 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 446 i != e; ++i) 447 ID.AddInteger(SVN->getMaskElt(i)); 448 break; 449 } 450 case ISD::TargetBlockAddress: 451 case ISD::BlockAddress: { 452 ID.AddPointer(cast<BlockAddressSDNode>(N)->getBlockAddress()); 453 ID.AddInteger(cast<BlockAddressSDNode>(N)->getTargetFlags()); 454 break; 455 } 456 } // end switch (N->getOpcode()) 457} 458 459/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 460/// data. 461static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 462 AddNodeIDOpcode(ID, N->getOpcode()); 463 // Add the return value info. 464 AddNodeIDValueTypes(ID, N->getVTList()); 465 // Add the operand info. 466 AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 467 468 // Handle SDNode leafs with special info. 469 AddNodeIDCustom(ID, N); 470} 471 472/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 473/// the CSE map that carries volatility, temporalness, indexing mode, and 474/// extension/truncation information. 475/// 476static inline unsigned 477encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 478 bool isNonTemporal, bool isInvariant) { 479 assert((ConvType & 3) == ConvType && 480 "ConvType may not require more than 2 bits!"); 481 assert((AM & 7) == AM && 482 "AM may not require more than 3 bits!"); 483 return ConvType | 484 (AM << 2) | 485 (isVolatile << 5) | 486 (isNonTemporal << 6) | 487 (isInvariant << 7); 488} 489 490//===----------------------------------------------------------------------===// 491// SelectionDAG Class 492//===----------------------------------------------------------------------===// 493 494/// doNotCSE - Return true if CSE should not be performed for this node. 495static bool doNotCSE(SDNode *N) { 496 if (N->getValueType(0) == MVT::Glue) 497 return true; // Never CSE anything that produces a flag. 498 499 switch (N->getOpcode()) { 500 default: break; 501 case ISD::HANDLENODE: 502 case ISD::EH_LABEL: 503 return true; // Never CSE these nodes. 504 } 505 506 // Check that remaining values produced are not flags. 507 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 508 if (N->getValueType(i) == MVT::Glue) 509 return true; // Never CSE anything that produces a flag. 510 511 return false; 512} 513 514/// RemoveDeadNodes - This method deletes all unreachable nodes in the 515/// SelectionDAG. 516void SelectionDAG::RemoveDeadNodes() { 517 // Create a dummy node (which is not added to allnodes), that adds a reference 518 // to the root node, preventing it from being deleted. 519 HandleSDNode Dummy(getRoot()); 520 521 SmallVector<SDNode*, 128> DeadNodes; 522 523 // Add all obviously-dead nodes to the DeadNodes worklist. 524 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 525 if (I->use_empty()) 526 DeadNodes.push_back(I); 527 528 RemoveDeadNodes(DeadNodes); 529 530 // If the root changed (e.g. it was a dead load, update the root). 531 setRoot(Dummy.getValue()); 532} 533 534/// RemoveDeadNodes - This method deletes the unreachable nodes in the 535/// given list, and any nodes that become unreachable as a result. 536void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes, 537 DAGUpdateListener *UpdateListener) { 538 539 // Process the worklist, deleting the nodes and adding their uses to the 540 // worklist. 541 while (!DeadNodes.empty()) { 542 SDNode *N = DeadNodes.pop_back_val(); 543 544 if (UpdateListener) 545 UpdateListener->NodeDeleted(N, 0); 546 547 // Take the node out of the appropriate CSE map. 548 RemoveNodeFromCSEMaps(N); 549 550 // Next, brutally remove the operand list. This is safe to do, as there are 551 // no cycles in the graph. 552 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 553 SDUse &Use = *I++; 554 SDNode *Operand = Use.getNode(); 555 Use.set(SDValue()); 556 557 // Now that we removed this operand, see if there are no uses of it left. 558 if (Operand->use_empty()) 559 DeadNodes.push_back(Operand); 560 } 561 562 DeallocateNode(N); 563 } 564} 565 566void SelectionDAG::RemoveDeadNode(SDNode *N, DAGUpdateListener *UpdateListener){ 567 SmallVector<SDNode*, 16> DeadNodes(1, N); 568 569 // Create a dummy node that adds a reference to the root node, preventing 570 // it from being deleted. (This matters if the root is an operand of the 571 // dead node.) 572 HandleSDNode Dummy(getRoot()); 573 574 RemoveDeadNodes(DeadNodes, UpdateListener); 575} 576 577void SelectionDAG::DeleteNode(SDNode *N) { 578 // First take this out of the appropriate CSE map. 579 RemoveNodeFromCSEMaps(N); 580 581 // Finally, remove uses due to operands of this node, remove from the 582 // AllNodes list, and delete the node. 583 DeleteNodeNotInCSEMaps(N); 584} 585 586void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 587 assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 588 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 589 590 // Drop all of the operands and decrement used node's use counts. 591 N->DropOperands(); 592 593 DeallocateNode(N); 594} 595 596void SelectionDAG::DeallocateNode(SDNode *N) { 597 if (N->OperandsNeedDelete) 598 delete[] N->OperandList; 599 600 // Set the opcode to DELETED_NODE to help catch bugs when node 601 // memory is reallocated. 602 N->NodeType = ISD::DELETED_NODE; 603 604 NodeAllocator.Deallocate(AllNodes.remove(N)); 605 606 // Remove the ordering of this node. 607 Ordering->remove(N); 608 609 // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. 610 ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); 611 for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) 612 DbgVals[i]->setIsInvalidated(); 613} 614 615/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 616/// correspond to it. This is useful when we're about to delete or repurpose 617/// the node. We don't want future request for structurally identical nodes 618/// to return N anymore. 619bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 620 bool Erased = false; 621 switch (N->getOpcode()) { 622 case ISD::HANDLENODE: return false; // noop. 623 case ISD::CONDCODE: 624 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 625 "Cond code doesn't exist!"); 626 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 627 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 628 break; 629 case ISD::ExternalSymbol: 630 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 631 break; 632 case ISD::TargetExternalSymbol: { 633 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 634 Erased = TargetExternalSymbols.erase( 635 std::pair<std::string,unsigned char>(ESN->getSymbol(), 636 ESN->getTargetFlags())); 637 break; 638 } 639 case ISD::VALUETYPE: { 640 EVT VT = cast<VTSDNode>(N)->getVT(); 641 if (VT.isExtended()) { 642 Erased = ExtendedValueTypeNodes.erase(VT); 643 } else { 644 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 645 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 646 } 647 break; 648 } 649 default: 650 // Remove it from the CSE Map. 651 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 652 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 653 Erased = CSEMap.RemoveNode(N); 654 break; 655 } 656#ifndef NDEBUG 657 // Verify that the node was actually in one of the CSE maps, unless it has a 658 // flag result (which cannot be CSE'd) or is one of the special cases that are 659 // not subject to CSE. 660 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 661 !N->isMachineOpcode() && !doNotCSE(N)) { 662 N->dump(this); 663 dbgs() << "\n"; 664 llvm_unreachable("Node is not in map!"); 665 } 666#endif 667 return Erased; 668} 669 670/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 671/// maps and modified in place. Add it back to the CSE maps, unless an identical 672/// node already exists, in which case transfer all its users to the existing 673/// node. This transfer can potentially trigger recursive merging. 674/// 675void 676SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N, 677 DAGUpdateListener *UpdateListener) { 678 // For node types that aren't CSE'd, just act as if no identical node 679 // already exists. 680 if (!doNotCSE(N)) { 681 SDNode *Existing = CSEMap.GetOrInsertNode(N); 682 if (Existing != N) { 683 // If there was already an existing matching node, use ReplaceAllUsesWith 684 // to replace the dead one with the existing one. This can cause 685 // recursive merging of other unrelated nodes down the line. 686 ReplaceAllUsesWith(N, Existing, UpdateListener); 687 688 // N is now dead. Inform the listener if it exists and delete it. 689 if (UpdateListener) 690 UpdateListener->NodeDeleted(N, Existing); 691 DeleteNodeNotInCSEMaps(N); 692 return; 693 } 694 } 695 696 // If the node doesn't already exist, we updated it. Inform a listener if 697 // it exists. 698 if (UpdateListener) 699 UpdateListener->NodeUpdated(N); 700} 701 702/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 703/// were replaced with those specified. If this node is never memoized, 704/// return null, otherwise return a pointer to the slot it would take. If a 705/// node already exists with these operands, the slot will be non-null. 706SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 707 void *&InsertPos) { 708 if (doNotCSE(N)) 709 return 0; 710 711 SDValue Ops[] = { Op }; 712 FoldingSetNodeID ID; 713 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 714 AddNodeIDCustom(ID, N); 715 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 716 return Node; 717} 718 719/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 720/// were replaced with those specified. If this node is never memoized, 721/// return null, otherwise return a pointer to the slot it would take. If a 722/// node already exists with these operands, the slot will be non-null. 723SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 724 SDValue Op1, SDValue Op2, 725 void *&InsertPos) { 726 if (doNotCSE(N)) 727 return 0; 728 729 SDValue Ops[] = { Op1, Op2 }; 730 FoldingSetNodeID ID; 731 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 732 AddNodeIDCustom(ID, N); 733 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 734 return Node; 735} 736 737 738/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 739/// were replaced with those specified. If this node is never memoized, 740/// return null, otherwise return a pointer to the slot it would take. If a 741/// node already exists with these operands, the slot will be non-null. 742SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 743 const SDValue *Ops,unsigned NumOps, 744 void *&InsertPos) { 745 if (doNotCSE(N)) 746 return 0; 747 748 FoldingSetNodeID ID; 749 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 750 AddNodeIDCustom(ID, N); 751 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 752 return Node; 753} 754 755#ifndef NDEBUG 756/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 757static void VerifyNodeCommon(SDNode *N) { 758 switch (N->getOpcode()) { 759 default: 760 break; 761 case ISD::BUILD_PAIR: { 762 EVT VT = N->getValueType(0); 763 assert(N->getNumValues() == 1 && "Too many results!"); 764 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 765 "Wrong return type!"); 766 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 767 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 768 "Mismatched operand types!"); 769 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 770 "Wrong operand type!"); 771 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 772 "Wrong return type size"); 773 break; 774 } 775 case ISD::BUILD_VECTOR: { 776 assert(N->getNumValues() == 1 && "Too many results!"); 777 assert(N->getValueType(0).isVector() && "Wrong return type!"); 778 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 779 "Wrong number of operands!"); 780 EVT EltVT = N->getValueType(0).getVectorElementType(); 781 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 782 assert((I->getValueType() == EltVT || 783 (EltVT.isInteger() && I->getValueType().isInteger() && 784 EltVT.bitsLE(I->getValueType()))) && 785 "Wrong operand type!"); 786 assert(I->getValueType() == N->getOperand(0).getValueType() && 787 "Operands must all have the same type"); 788 } 789 break; 790 } 791 } 792} 793 794/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 795static void VerifySDNode(SDNode *N) { 796 // The SDNode allocators cannot be used to allocate nodes with fields that are 797 // not present in an SDNode! 798 assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 799 assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 800 assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 801 assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 802 assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 803 assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 804 assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 805 assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 806 assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 807 assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 808 assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 809 assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 810 assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 811 assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 812 assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 813 assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 814 assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 815 assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 816 assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 817 818 VerifyNodeCommon(N); 819} 820 821/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 822/// invalid. 823static void VerifyMachineNode(SDNode *N) { 824 // The MachineNode allocators cannot be used to allocate nodes with fields 825 // that are not present in a MachineNode! 826 // Currently there are no such nodes. 827 828 VerifyNodeCommon(N); 829} 830#endif // NDEBUG 831 832/// getEVTAlignment - Compute the default alignment value for the 833/// given type. 834/// 835unsigned SelectionDAG::getEVTAlignment(EVT VT) const { 836 Type *Ty = VT == MVT::iPTR ? 837 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 838 VT.getTypeForEVT(*getContext()); 839 840 return TLI.getTargetData()->getABITypeAlignment(Ty); 841} 842 843// EntryNode could meaningfully have debug info if we can find it... 844SelectionDAG::SelectionDAG(const TargetMachine &tm) 845 : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), 846 EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), 847 Root(getEntryNode()), Ordering(0) { 848 AllNodes.push_back(&EntryNode); 849 Ordering = new SDNodeOrdering(); 850 DbgInfo = new SDDbgInfo(); 851} 852 853void SelectionDAG::init(MachineFunction &mf) { 854 MF = &mf; 855 Context = &mf.getFunction()->getContext(); 856} 857 858SelectionDAG::~SelectionDAG() { 859 allnodes_clear(); 860 delete Ordering; 861 delete DbgInfo; 862} 863 864void SelectionDAG::allnodes_clear() { 865 assert(&*AllNodes.begin() == &EntryNode); 866 AllNodes.remove(AllNodes.begin()); 867 while (!AllNodes.empty()) 868 DeallocateNode(AllNodes.begin()); 869} 870 871void SelectionDAG::clear() { 872 allnodes_clear(); 873 OperandAllocator.Reset(); 874 CSEMap.clear(); 875 876 ExtendedValueTypeNodes.clear(); 877 ExternalSymbols.clear(); 878 TargetExternalSymbols.clear(); 879 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 880 static_cast<CondCodeSDNode*>(0)); 881 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 882 static_cast<SDNode*>(0)); 883 884 EntryNode.UseList = 0; 885 AllNodes.push_back(&EntryNode); 886 Root = getEntryNode(); 887 Ordering->clear(); 888 DbgInfo->clear(); 889} 890 891SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 892 return VT.bitsGT(Op.getValueType()) ? 893 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 894 getNode(ISD::TRUNCATE, DL, VT, Op); 895} 896 897SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 898 return VT.bitsGT(Op.getValueType()) ? 899 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 900 getNode(ISD::TRUNCATE, DL, VT, Op); 901} 902 903SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 904 return VT.bitsGT(Op.getValueType()) ? 905 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 906 getNode(ISD::TRUNCATE, DL, VT, Op); 907} 908 909SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, DebugLoc DL, EVT VT) { 910 assert(!VT.isVector() && 911 "getZeroExtendInReg should use the vector element type instead of " 912 "the vector type!"); 913 if (Op.getValueType() == VT) return Op; 914 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 915 APInt Imm = APInt::getLowBitsSet(BitWidth, 916 VT.getSizeInBits()); 917 return getNode(ISD::AND, DL, Op.getValueType(), Op, 918 getConstant(Imm, Op.getValueType())); 919} 920 921/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 922/// 923SDValue SelectionDAG::getNOT(DebugLoc DL, SDValue Val, EVT VT) { 924 EVT EltVT = VT.getScalarType(); 925 SDValue NegOne = 926 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 927 return getNode(ISD::XOR, DL, VT, Val, NegOne); 928} 929 930SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 931 EVT EltVT = VT.getScalarType(); 932 assert((EltVT.getSizeInBits() >= 64 || 933 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 934 "getConstant with a uint64_t value that doesn't fit in the type!"); 935 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 936} 937 938SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 939 return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 940} 941 942SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 943 assert(VT.isInteger() && "Cannot create FP integer constant!"); 944 945 EVT EltVT = VT.getScalarType(); 946 const ConstantInt *Elt = &Val; 947 948 // In some cases the vector type is legal but the element type is illegal and 949 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 950 // inserted value (the type does not need to match the vector element type). 951 // Any extra bits introduced will be truncated away. 952 if (VT.isVector() && TLI.getTypeAction(*getContext(), EltVT) == 953 TargetLowering::TypePromoteInteger) { 954 EltVT = TLI.getTypeToTransformTo(*getContext(), EltVT); 955 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 956 Elt = ConstantInt::get(*getContext(), NewVal); 957 } 958 959 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 960 "APInt size does not match type size!"); 961 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 962 FoldingSetNodeID ID; 963 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 964 ID.AddPointer(Elt); 965 void *IP = 0; 966 SDNode *N = NULL; 967 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 968 if (!VT.isVector()) 969 return SDValue(N, 0); 970 971 if (!N) { 972 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 973 CSEMap.InsertNode(N, IP); 974 AllNodes.push_back(N); 975 } 976 977 SDValue Result(N, 0); 978 if (VT.isVector()) { 979 SmallVector<SDValue, 8> Ops; 980 Ops.assign(VT.getVectorNumElements(), Result); 981 Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); 982 } 983 return Result; 984} 985 986SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 987 return getConstant(Val, TLI.getPointerTy(), isTarget); 988} 989 990 991SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 992 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 993} 994 995SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 996 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 997 998 EVT EltVT = VT.getScalarType(); 999 1000 // Do the map lookup using the actual bit pattern for the floating point 1001 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1002 // we don't have issues with SNANs. 1003 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1004 FoldingSetNodeID ID; 1005 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1006 ID.AddPointer(&V); 1007 void *IP = 0; 1008 SDNode *N = NULL; 1009 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1010 if (!VT.isVector()) 1011 return SDValue(N, 0); 1012 1013 if (!N) { 1014 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1015 CSEMap.InsertNode(N, IP); 1016 AllNodes.push_back(N); 1017 } 1018 1019 SDValue Result(N, 0); 1020 if (VT.isVector()) { 1021 SmallVector<SDValue, 8> Ops; 1022 Ops.assign(VT.getVectorNumElements(), Result); 1023 // FIXME DebugLoc info might be appropriate here 1024 Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); 1025 } 1026 return Result; 1027} 1028 1029SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1030 EVT EltVT = VT.getScalarType(); 1031 if (EltVT==MVT::f32) 1032 return getConstantFP(APFloat((float)Val), VT, isTarget); 1033 else if (EltVT==MVT::f64) 1034 return getConstantFP(APFloat(Val), VT, isTarget); 1035 else if (EltVT==MVT::f80 || EltVT==MVT::f128) { 1036 bool ignored; 1037 APFloat apf = APFloat(Val); 1038 apf.convert(*EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1039 &ignored); 1040 return getConstantFP(apf, VT, isTarget); 1041 } else { 1042 assert(0 && "Unsupported type in getConstantFP"); 1043 return SDValue(); 1044 } 1045} 1046 1047SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, DebugLoc DL, 1048 EVT VT, int64_t Offset, 1049 bool isTargetGA, 1050 unsigned char TargetFlags) { 1051 assert((TargetFlags == 0 || isTargetGA) && 1052 "Cannot set target flags on target-independent globals"); 1053 1054 // Truncate (with sign-extension) the offset value to the pointer size. 1055 EVT PTy = TLI.getPointerTy(); 1056 unsigned BitWidth = PTy.getSizeInBits(); 1057 if (BitWidth < 64) 1058 Offset = (Offset << (64 - BitWidth) >> (64 - BitWidth)); 1059 1060 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1061 if (!GVar) { 1062 // If GV is an alias then use the aliasee for determining thread-localness. 1063 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1064 GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1065 } 1066 1067 unsigned Opc; 1068 if (GVar && GVar->isThreadLocal()) 1069 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1070 else 1071 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1072 1073 FoldingSetNodeID ID; 1074 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1075 ID.AddPointer(GV); 1076 ID.AddInteger(Offset); 1077 ID.AddInteger(TargetFlags); 1078 void *IP = 0; 1079 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1080 return SDValue(E, 0); 1081 1082 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL, GV, VT, 1083 Offset, TargetFlags); 1084 CSEMap.InsertNode(N, IP); 1085 AllNodes.push_back(N); 1086 return SDValue(N, 0); 1087} 1088 1089SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1090 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1091 FoldingSetNodeID ID; 1092 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1093 ID.AddInteger(FI); 1094 void *IP = 0; 1095 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1096 return SDValue(E, 0); 1097 1098 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1099 CSEMap.InsertNode(N, IP); 1100 AllNodes.push_back(N); 1101 return SDValue(N, 0); 1102} 1103 1104SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1105 unsigned char TargetFlags) { 1106 assert((TargetFlags == 0 || isTarget) && 1107 "Cannot set target flags on target-independent jump tables"); 1108 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1109 FoldingSetNodeID ID; 1110 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1111 ID.AddInteger(JTI); 1112 ID.AddInteger(TargetFlags); 1113 void *IP = 0; 1114 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1115 return SDValue(E, 0); 1116 1117 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1118 TargetFlags); 1119 CSEMap.InsertNode(N, IP); 1120 AllNodes.push_back(N); 1121 return SDValue(N, 0); 1122} 1123 1124SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1125 unsigned Alignment, int Offset, 1126 bool isTarget, 1127 unsigned char TargetFlags) { 1128 assert((TargetFlags == 0 || isTarget) && 1129 "Cannot set target flags on target-independent globals"); 1130 if (Alignment == 0) 1131 Alignment = TLI.getTargetData()->getPrefTypeAlignment(C->getType()); 1132 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1133 FoldingSetNodeID ID; 1134 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1135 ID.AddInteger(Alignment); 1136 ID.AddInteger(Offset); 1137 ID.AddPointer(C); 1138 ID.AddInteger(TargetFlags); 1139 void *IP = 0; 1140 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1141 return SDValue(E, 0); 1142 1143 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1144 Alignment, TargetFlags); 1145 CSEMap.InsertNode(N, IP); 1146 AllNodes.push_back(N); 1147 return SDValue(N, 0); 1148} 1149 1150 1151SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1152 unsigned Alignment, int Offset, 1153 bool isTarget, 1154 unsigned char TargetFlags) { 1155 assert((TargetFlags == 0 || isTarget) && 1156 "Cannot set target flags on target-independent globals"); 1157 if (Alignment == 0) 1158 Alignment = TLI.getTargetData()->getPrefTypeAlignment(C->getType()); 1159 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1160 FoldingSetNodeID ID; 1161 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1162 ID.AddInteger(Alignment); 1163 ID.AddInteger(Offset); 1164 C->addSelectionDAGCSEId(ID); 1165 ID.AddInteger(TargetFlags); 1166 void *IP = 0; 1167 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1168 return SDValue(E, 0); 1169 1170 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1171 Alignment, TargetFlags); 1172 CSEMap.InsertNode(N, IP); 1173 AllNodes.push_back(N); 1174 return SDValue(N, 0); 1175} 1176 1177SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1178 FoldingSetNodeID ID; 1179 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1180 ID.AddPointer(MBB); 1181 void *IP = 0; 1182 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1183 return SDValue(E, 0); 1184 1185 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1186 CSEMap.InsertNode(N, IP); 1187 AllNodes.push_back(N); 1188 return SDValue(N, 0); 1189} 1190 1191SDValue SelectionDAG::getValueType(EVT VT) { 1192 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1193 ValueTypeNodes.size()) 1194 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1195 1196 SDNode *&N = VT.isExtended() ? 1197 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1198 1199 if (N) return SDValue(N, 0); 1200 N = new (NodeAllocator) VTSDNode(VT); 1201 AllNodes.push_back(N); 1202 return SDValue(N, 0); 1203} 1204 1205SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1206 SDNode *&N = ExternalSymbols[Sym]; 1207 if (N) return SDValue(N, 0); 1208 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1209 AllNodes.push_back(N); 1210 return SDValue(N, 0); 1211} 1212 1213SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1214 unsigned char TargetFlags) { 1215 SDNode *&N = 1216 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1217 TargetFlags)]; 1218 if (N) return SDValue(N, 0); 1219 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1220 AllNodes.push_back(N); 1221 return SDValue(N, 0); 1222} 1223 1224SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1225 if ((unsigned)Cond >= CondCodeNodes.size()) 1226 CondCodeNodes.resize(Cond+1); 1227 1228 if (CondCodeNodes[Cond] == 0) { 1229 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1230 CondCodeNodes[Cond] = N; 1231 AllNodes.push_back(N); 1232 } 1233 1234 return SDValue(CondCodeNodes[Cond], 0); 1235} 1236 1237// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1238// the shuffle mask M that point at N1 to point at N2, and indices that point 1239// N2 to point at N1. 1240static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1241 std::swap(N1, N2); 1242 int NElts = M.size(); 1243 for (int i = 0; i != NElts; ++i) { 1244 if (M[i] >= NElts) 1245 M[i] -= NElts; 1246 else if (M[i] >= 0) 1247 M[i] += NElts; 1248 } 1249} 1250 1251SDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1, 1252 SDValue N2, const int *Mask) { 1253 assert(N1.getValueType() == N2.getValueType() && "Invalid VECTOR_SHUFFLE"); 1254 assert(VT.isVector() && N1.getValueType().isVector() && 1255 "Vector Shuffle VTs must be a vectors"); 1256 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() 1257 && "Vector Shuffle VTs must have same element type"); 1258 1259 // Canonicalize shuffle undef, undef -> undef 1260 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1261 return getUNDEF(VT); 1262 1263 // Validate that all indices in Mask are within the range of the elements 1264 // input to the shuffle. 1265 unsigned NElts = VT.getVectorNumElements(); 1266 SmallVector<int, 8> MaskVec; 1267 for (unsigned i = 0; i != NElts; ++i) { 1268 assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1269 MaskVec.push_back(Mask[i]); 1270 } 1271 1272 // Canonicalize shuffle v, v -> v, undef 1273 if (N1 == N2) { 1274 N2 = getUNDEF(VT); 1275 for (unsigned i = 0; i != NElts; ++i) 1276 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1277 } 1278 1279 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1280 if (N1.getOpcode() == ISD::UNDEF) 1281 commuteShuffle(N1, N2, MaskVec); 1282 1283 // Canonicalize all index into lhs, -> shuffle lhs, undef 1284 // Canonicalize all index into rhs, -> shuffle rhs, undef 1285 bool AllLHS = true, AllRHS = true; 1286 bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1287 for (unsigned i = 0; i != NElts; ++i) { 1288 if (MaskVec[i] >= (int)NElts) { 1289 if (N2Undef) 1290 MaskVec[i] = -1; 1291 else 1292 AllLHS = false; 1293 } else if (MaskVec[i] >= 0) { 1294 AllRHS = false; 1295 } 1296 } 1297 if (AllLHS && AllRHS) 1298 return getUNDEF(VT); 1299 if (AllLHS && !N2Undef) 1300 N2 = getUNDEF(VT); 1301 if (AllRHS) { 1302 N1 = getUNDEF(VT); 1303 commuteShuffle(N1, N2, MaskVec); 1304 } 1305 1306 // If Identity shuffle, or all shuffle in to undef, return that node. 1307 bool AllUndef = true; 1308 bool Identity = true; 1309 for (unsigned i = 0; i != NElts; ++i) { 1310 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1311 if (MaskVec[i] >= 0) AllUndef = false; 1312 } 1313 if (Identity && NElts == N1.getValueType().getVectorNumElements()) 1314 return N1; 1315 if (AllUndef) 1316 return getUNDEF(VT); 1317 1318 FoldingSetNodeID ID; 1319 SDValue Ops[2] = { N1, N2 }; 1320 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1321 for (unsigned i = 0; i != NElts; ++i) 1322 ID.AddInteger(MaskVec[i]); 1323 1324 void* IP = 0; 1325 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1326 return SDValue(E, 0); 1327 1328 // Allocate the mask array for the node out of the BumpPtrAllocator, since 1329 // SDNode doesn't have access to it. This memory will be "leaked" when 1330 // the node is deallocated, but recovered when the NodeAllocator is released. 1331 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1332 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1333 1334 ShuffleVectorSDNode *N = 1335 new (NodeAllocator) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc); 1336 CSEMap.InsertNode(N, IP); 1337 AllNodes.push_back(N); 1338 return SDValue(N, 0); 1339} 1340 1341SDValue SelectionDAG::getConvertRndSat(EVT VT, DebugLoc dl, 1342 SDValue Val, SDValue DTy, 1343 SDValue STy, SDValue Rnd, SDValue Sat, 1344 ISD::CvtCode Code) { 1345 // If the src and dest types are the same and the conversion is between 1346 // integer types of the same sign or two floats, no conversion is necessary. 1347 if (DTy == STy && 1348 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1349 return Val; 1350 1351 FoldingSetNodeID ID; 1352 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1353 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1354 void* IP = 0; 1355 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1356 return SDValue(E, 0); 1357 1358 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl, Ops, 5, 1359 Code); 1360 CSEMap.InsertNode(N, IP); 1361 AllNodes.push_back(N); 1362 return SDValue(N, 0); 1363} 1364 1365SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1366 FoldingSetNodeID ID; 1367 AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1368 ID.AddInteger(RegNo); 1369 void *IP = 0; 1370 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1371 return SDValue(E, 0); 1372 1373 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1374 CSEMap.InsertNode(N, IP); 1375 AllNodes.push_back(N); 1376 return SDValue(N, 0); 1377} 1378 1379SDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) { 1380 FoldingSetNodeID ID; 1381 SDValue Ops[] = { Root }; 1382 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1383 ID.AddPointer(Label); 1384 void *IP = 0; 1385 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1386 return SDValue(E, 0); 1387 1388 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl, Root, Label); 1389 CSEMap.InsertNode(N, IP); 1390 AllNodes.push_back(N); 1391 return SDValue(N, 0); 1392} 1393 1394 1395SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1396 bool isTarget, 1397 unsigned char TargetFlags) { 1398 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1399 1400 FoldingSetNodeID ID; 1401 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1402 ID.AddPointer(BA); 1403 ID.AddInteger(TargetFlags); 1404 void *IP = 0; 1405 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1406 return SDValue(E, 0); 1407 1408 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, TargetFlags); 1409 CSEMap.InsertNode(N, IP); 1410 AllNodes.push_back(N); 1411 return SDValue(N, 0); 1412} 1413 1414SDValue SelectionDAG::getSrcValue(const Value *V) { 1415 assert((!V || V->getType()->isPointerTy()) && 1416 "SrcValue is not a pointer?"); 1417 1418 FoldingSetNodeID ID; 1419 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1420 ID.AddPointer(V); 1421 1422 void *IP = 0; 1423 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1424 return SDValue(E, 0); 1425 1426 SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1427 CSEMap.InsertNode(N, IP); 1428 AllNodes.push_back(N); 1429 return SDValue(N, 0); 1430} 1431 1432/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1433SDValue SelectionDAG::getMDNode(const MDNode *MD) { 1434 FoldingSetNodeID ID; 1435 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1436 ID.AddPointer(MD); 1437 1438 void *IP = 0; 1439 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1440 return SDValue(E, 0); 1441 1442 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1443 CSEMap.InsertNode(N, IP); 1444 AllNodes.push_back(N); 1445 return SDValue(N, 0); 1446} 1447 1448 1449/// getShiftAmountOperand - Return the specified value casted to 1450/// the target's desired shift amount type. 1451SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1452 EVT OpTy = Op.getValueType(); 1453 MVT ShTy = TLI.getShiftAmountTy(LHSTy); 1454 if (OpTy == ShTy || OpTy.isVector()) return Op; 1455 1456 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1457 return getNode(Opcode, Op.getDebugLoc(), ShTy, Op); 1458} 1459 1460/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1461/// specified value type. 1462SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1463 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1464 unsigned ByteSize = VT.getStoreSize(); 1465 Type *Ty = VT.getTypeForEVT(*getContext()); 1466 unsigned StackAlign = 1467 std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign); 1468 1469 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1470 return getFrameIndex(FrameIdx, TLI.getPointerTy()); 1471} 1472 1473/// CreateStackTemporary - Create a stack temporary suitable for holding 1474/// either of the specified value types. 1475SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1476 unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1477 VT2.getStoreSizeInBits())/8; 1478 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1479 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1480 const TargetData *TD = TLI.getTargetData(); 1481 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1482 TD->getPrefTypeAlignment(Ty2)); 1483 1484 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1485 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1486 return getFrameIndex(FrameIdx, TLI.getPointerTy()); 1487} 1488 1489SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1490 SDValue N2, ISD::CondCode Cond, DebugLoc dl) { 1491 // These setcc operations always fold. 1492 switch (Cond) { 1493 default: break; 1494 case ISD::SETFALSE: 1495 case ISD::SETFALSE2: return getConstant(0, VT); 1496 case ISD::SETTRUE: 1497 case ISD::SETTRUE2: return getConstant(1, VT); 1498 1499 case ISD::SETOEQ: 1500 case ISD::SETOGT: 1501 case ISD::SETOGE: 1502 case ISD::SETOLT: 1503 case ISD::SETOLE: 1504 case ISD::SETONE: 1505 case ISD::SETO: 1506 case ISD::SETUO: 1507 case ISD::SETUEQ: 1508 case ISD::SETUNE: 1509 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1510 break; 1511 } 1512 1513 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1514 const APInt &C2 = N2C->getAPIntValue(); 1515 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1516 const APInt &C1 = N1C->getAPIntValue(); 1517 1518 switch (Cond) { 1519 default: llvm_unreachable("Unknown integer setcc!"); 1520 case ISD::SETEQ: return getConstant(C1 == C2, VT); 1521 case ISD::SETNE: return getConstant(C1 != C2, VT); 1522 case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1523 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1524 case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1525 case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1526 case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1527 case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1528 case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1529 case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1530 } 1531 } 1532 } 1533 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1534 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1535 // No compile time operations on this type yet. 1536 if (N1C->getValueType(0) == MVT::ppcf128) 1537 return SDValue(); 1538 1539 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1540 switch (Cond) { 1541 default: break; 1542 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1543 return getUNDEF(VT); 1544 // fall through 1545 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1546 case ISD::SETNE: if (R==APFloat::cmpUnordered) 1547 return getUNDEF(VT); 1548 // fall through 1549 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1550 R==APFloat::cmpLessThan, VT); 1551 case ISD::SETLT: if (R==APFloat::cmpUnordered) 1552 return getUNDEF(VT); 1553 // fall through 1554 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1555 case ISD::SETGT: if (R==APFloat::cmpUnordered) 1556 return getUNDEF(VT); 1557 // fall through 1558 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1559 case ISD::SETLE: if (R==APFloat::cmpUnordered) 1560 return getUNDEF(VT); 1561 // fall through 1562 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1563 R==APFloat::cmpEqual, VT); 1564 case ISD::SETGE: if (R==APFloat::cmpUnordered) 1565 return getUNDEF(VT); 1566 // fall through 1567 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1568 R==APFloat::cmpEqual, VT); 1569 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1570 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1571 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1572 R==APFloat::cmpEqual, VT); 1573 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1574 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1575 R==APFloat::cmpLessThan, VT); 1576 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1577 R==APFloat::cmpUnordered, VT); 1578 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1579 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1580 } 1581 } else { 1582 // Ensure that the constant occurs on the RHS. 1583 return getSetCC(dl, VT, N2, N1, ISD::getSetCCSwappedOperands(Cond)); 1584 } 1585 } 1586 1587 // Could not fold it. 1588 return SDValue(); 1589} 1590 1591/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1592/// use this predicate to simplify operations downstream. 1593bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1594 // This predicate is not safe for vector operations. 1595 if (Op.getValueType().isVector()) 1596 return false; 1597 1598 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1599 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1600} 1601 1602/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1603/// this predicate to simplify operations downstream. Mask is known to be zero 1604/// for bits that V cannot have. 1605bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1606 unsigned Depth) const { 1607 APInt KnownZero, KnownOne; 1608 ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth); 1609 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1610 return (KnownZero & Mask) == Mask; 1611} 1612 1613/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1614/// known to be either zero or one and return them in the KnownZero/KnownOne 1615/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1616/// processing. 1617void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, 1618 APInt &KnownZero, APInt &KnownOne, 1619 unsigned Depth) const { 1620 unsigned BitWidth = Mask.getBitWidth(); 1621 assert(BitWidth == Op.getValueType().getScalarType().getSizeInBits() && 1622 "Mask size mismatches value type size!"); 1623 1624 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1625 if (Depth == 6 || Mask == 0) 1626 return; // Limit search depth. 1627 1628 APInt KnownZero2, KnownOne2; 1629 1630 switch (Op.getOpcode()) { 1631 case ISD::Constant: 1632 // We know all of the bits for a constant! 1633 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue() & Mask; 1634 KnownZero = ~KnownOne & Mask; 1635 return; 1636 case ISD::AND: 1637 // If either the LHS or the RHS are Zero, the result is zero. 1638 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1639 ComputeMaskedBits(Op.getOperand(0), Mask & ~KnownZero, 1640 KnownZero2, KnownOne2, Depth+1); 1641 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1642 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1643 1644 // Output known-1 bits are only known if set in both the LHS & RHS. 1645 KnownOne &= KnownOne2; 1646 // Output known-0 are known to be clear if zero in either the LHS | RHS. 1647 KnownZero |= KnownZero2; 1648 return; 1649 case ISD::OR: 1650 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1651 ComputeMaskedBits(Op.getOperand(0), Mask & ~KnownOne, 1652 KnownZero2, KnownOne2, Depth+1); 1653 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1654 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1655 1656 // Output known-0 bits are only known if clear in both the LHS & RHS. 1657 KnownZero &= KnownZero2; 1658 // Output known-1 are known to be set if set in either the LHS | RHS. 1659 KnownOne |= KnownOne2; 1660 return; 1661 case ISD::XOR: { 1662 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1663 ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1); 1664 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1665 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1666 1667 // Output known-0 bits are known if clear or set in both the LHS & RHS. 1668 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1669 // Output known-1 are known to be set if set in only one of the LHS, RHS. 1670 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1671 KnownZero = KnownZeroOut; 1672 return; 1673 } 1674 case ISD::MUL: { 1675 APInt Mask2 = APInt::getAllOnesValue(BitWidth); 1676 ComputeMaskedBits(Op.getOperand(1), Mask2, KnownZero, KnownOne, Depth+1); 1677 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1); 1678 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1679 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1680 1681 // If low bits are zero in either operand, output low known-0 bits. 1682 // Also compute a conserative estimate for high known-0 bits. 1683 // More trickiness is possible, but this is sufficient for the 1684 // interesting case of alignment computation. 1685 KnownOne.clearAllBits(); 1686 unsigned TrailZ = KnownZero.countTrailingOnes() + 1687 KnownZero2.countTrailingOnes(); 1688 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1689 KnownZero2.countLeadingOnes(), 1690 BitWidth) - BitWidth; 1691 1692 TrailZ = std::min(TrailZ, BitWidth); 1693 LeadZ = std::min(LeadZ, BitWidth); 1694 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1695 APInt::getHighBitsSet(BitWidth, LeadZ); 1696 KnownZero &= Mask; 1697 return; 1698 } 1699 case ISD::UDIV: { 1700 // For the purposes of computing leading zeros we can conservatively 1701 // treat a udiv as a logical right shift by the power of 2 known to 1702 // be less than the denominator. 1703 APInt AllOnes = APInt::getAllOnesValue(BitWidth); 1704 ComputeMaskedBits(Op.getOperand(0), 1705 AllOnes, KnownZero2, KnownOne2, Depth+1); 1706 unsigned LeadZ = KnownZero2.countLeadingOnes(); 1707 1708 KnownOne2.clearAllBits(); 1709 KnownZero2.clearAllBits(); 1710 ComputeMaskedBits(Op.getOperand(1), 1711 AllOnes, KnownZero2, KnownOne2, Depth+1); 1712 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1713 if (RHSUnknownLeadingOnes != BitWidth) 1714 LeadZ = std::min(BitWidth, 1715 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1716 1717 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ) & Mask; 1718 return; 1719 } 1720 case ISD::SELECT: 1721 ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero, KnownOne, Depth+1); 1722 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero2, KnownOne2, Depth+1); 1723 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1724 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1725 1726 // Only known if known in both the LHS and RHS. 1727 KnownOne &= KnownOne2; 1728 KnownZero &= KnownZero2; 1729 return; 1730 case ISD::SELECT_CC: 1731 ComputeMaskedBits(Op.getOperand(3), Mask, KnownZero, KnownOne, Depth+1); 1732 ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero2, KnownOne2, Depth+1); 1733 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1734 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1735 1736 // Only known if known in both the LHS and RHS. 1737 KnownOne &= KnownOne2; 1738 KnownZero &= KnownZero2; 1739 return; 1740 case ISD::SADDO: 1741 case ISD::UADDO: 1742 case ISD::SSUBO: 1743 case ISD::USUBO: 1744 case ISD::SMULO: 1745 case ISD::UMULO: 1746 if (Op.getResNo() != 1) 1747 return; 1748 // The boolean result conforms to getBooleanContents. Fall through. 1749 case ISD::SETCC: 1750 // If we know the result of a setcc has the top bits zero, use this info. 1751 if (TLI.getBooleanContents(Op.getValueType().isVector()) == 1752 TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1753 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1754 return; 1755 case ISD::SHL: 1756 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1757 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1758 unsigned ShAmt = SA->getZExtValue(); 1759 1760 // If the shift count is an invalid immediate, don't do anything. 1761 if (ShAmt >= BitWidth) 1762 return; 1763 1764 ComputeMaskedBits(Op.getOperand(0), Mask.lshr(ShAmt), 1765 KnownZero, KnownOne, Depth+1); 1766 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1767 KnownZero <<= ShAmt; 1768 KnownOne <<= ShAmt; 1769 // low bits known zero. 1770 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1771 } 1772 return; 1773 case ISD::SRL: 1774 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1775 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1776 unsigned ShAmt = SA->getZExtValue(); 1777 1778 // If the shift count is an invalid immediate, don't do anything. 1779 if (ShAmt >= BitWidth) 1780 return; 1781 1782 ComputeMaskedBits(Op.getOperand(0), (Mask << ShAmt), 1783 KnownZero, KnownOne, Depth+1); 1784 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1785 KnownZero = KnownZero.lshr(ShAmt); 1786 KnownOne = KnownOne.lshr(ShAmt); 1787 1788 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt) & Mask; 1789 KnownZero |= HighBits; // High bits known zero. 1790 } 1791 return; 1792 case ISD::SRA: 1793 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1794 unsigned ShAmt = SA->getZExtValue(); 1795 1796 // If the shift count is an invalid immediate, don't do anything. 1797 if (ShAmt >= BitWidth) 1798 return; 1799 1800 APInt InDemandedMask = (Mask << ShAmt); 1801 // If any of the demanded bits are produced by the sign extension, we also 1802 // demand the input sign bit. 1803 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt) & Mask; 1804 if (HighBits.getBoolValue()) 1805 InDemandedMask |= APInt::getSignBit(BitWidth); 1806 1807 ComputeMaskedBits(Op.getOperand(0), InDemandedMask, KnownZero, KnownOne, 1808 Depth+1); 1809 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1810 KnownZero = KnownZero.lshr(ShAmt); 1811 KnownOne = KnownOne.lshr(ShAmt); 1812 1813 // Handle the sign bits. 1814 APInt SignBit = APInt::getSignBit(BitWidth); 1815 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1816 1817 if (KnownZero.intersects(SignBit)) { 1818 KnownZero |= HighBits; // New bits are known zero. 1819 } else if (KnownOne.intersects(SignBit)) { 1820 KnownOne |= HighBits; // New bits are known one. 1821 } 1822 } 1823 return; 1824 case ISD::SIGN_EXTEND_INREG: { 1825 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1826 unsigned EBits = EVT.getScalarType().getSizeInBits(); 1827 1828 // Sign extension. Compute the demanded bits in the result that are not 1829 // present in the input. 1830 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits) & Mask; 1831 1832 APInt InSignBit = APInt::getSignBit(EBits); 1833 APInt InputDemandedBits = Mask & APInt::getLowBitsSet(BitWidth, EBits); 1834 1835 // If the sign extended bits are demanded, we know that the sign 1836 // bit is demanded. 1837 InSignBit = InSignBit.zext(BitWidth); 1838 if (NewBits.getBoolValue()) 1839 InputDemandedBits |= InSignBit; 1840 1841 ComputeMaskedBits(Op.getOperand(0), InputDemandedBits, 1842 KnownZero, KnownOne, Depth+1); 1843 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1844 1845 // If the sign bit of the input is known set or clear, then we know the 1846 // top bits of the result. 1847 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1848 KnownZero |= NewBits; 1849 KnownOne &= ~NewBits; 1850 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1851 KnownOne |= NewBits; 1852 KnownZero &= ~NewBits; 1853 } else { // Input sign bit unknown 1854 KnownZero &= ~NewBits; 1855 KnownOne &= ~NewBits; 1856 } 1857 return; 1858 } 1859 case ISD::CTTZ: 1860 case ISD::CTLZ: 1861 case ISD::CTPOP: { 1862 unsigned LowBits = Log2_32(BitWidth)+1; 1863 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 1864 KnownOne.clearAllBits(); 1865 return; 1866 } 1867 case ISD::LOAD: { 1868 if (ISD::isZEXTLoad(Op.getNode())) { 1869 LoadSDNode *LD = cast<LoadSDNode>(Op); 1870 EVT VT = LD->getMemoryVT(); 1871 unsigned MemBits = VT.getScalarType().getSizeInBits(); 1872 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits) & Mask; 1873 } 1874 return; 1875 } 1876 case ISD::ZERO_EXTEND: { 1877 EVT InVT = Op.getOperand(0).getValueType(); 1878 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1879 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; 1880 APInt InMask = Mask.trunc(InBits); 1881 KnownZero = KnownZero.trunc(InBits); 1882 KnownOne = KnownOne.trunc(InBits); 1883 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1884 KnownZero = KnownZero.zext(BitWidth); 1885 KnownOne = KnownOne.zext(BitWidth); 1886 KnownZero |= NewBits; 1887 return; 1888 } 1889 case ISD::SIGN_EXTEND: { 1890 EVT InVT = Op.getOperand(0).getValueType(); 1891 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1892 APInt InSignBit = APInt::getSignBit(InBits); 1893 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; 1894 APInt InMask = Mask.trunc(InBits); 1895 1896 // If any of the sign extended bits are demanded, we know that the sign 1897 // bit is demanded. Temporarily set this bit in the mask for our callee. 1898 if (NewBits.getBoolValue()) 1899 InMask |= InSignBit; 1900 1901 KnownZero = KnownZero.trunc(InBits); 1902 KnownOne = KnownOne.trunc(InBits); 1903 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1904 1905 // Note if the sign bit is known to be zero or one. 1906 bool SignBitKnownZero = KnownZero.isNegative(); 1907 bool SignBitKnownOne = KnownOne.isNegative(); 1908 assert(!(SignBitKnownZero && SignBitKnownOne) && 1909 "Sign bit can't be known to be both zero and one!"); 1910 1911 // If the sign bit wasn't actually demanded by our caller, we don't 1912 // want it set in the KnownZero and KnownOne result values. Reset the 1913 // mask and reapply it to the result values. 1914 InMask = Mask.trunc(InBits); 1915 KnownZero &= InMask; 1916 KnownOne &= InMask; 1917 1918 KnownZero = KnownZero.zext(BitWidth); 1919 KnownOne = KnownOne.zext(BitWidth); 1920 1921 // If the sign bit is known zero or one, the top bits match. 1922 if (SignBitKnownZero) 1923 KnownZero |= NewBits; 1924 else if (SignBitKnownOne) 1925 KnownOne |= NewBits; 1926 return; 1927 } 1928 case ISD::ANY_EXTEND: { 1929 EVT InVT = Op.getOperand(0).getValueType(); 1930 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1931 APInt InMask = Mask.trunc(InBits); 1932 KnownZero = KnownZero.trunc(InBits); 1933 KnownOne = KnownOne.trunc(InBits); 1934 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1935 KnownZero = KnownZero.zext(BitWidth); 1936 KnownOne = KnownOne.zext(BitWidth); 1937 return; 1938 } 1939 case ISD::TRUNCATE: { 1940 EVT InVT = Op.getOperand(0).getValueType(); 1941 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1942 APInt InMask = Mask.zext(InBits); 1943 KnownZero = KnownZero.zext(InBits); 1944 KnownOne = KnownOne.zext(InBits); 1945 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1946 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1947 KnownZero = KnownZero.trunc(BitWidth); 1948 KnownOne = KnownOne.trunc(BitWidth); 1949 break; 1950 } 1951 case ISD::AssertZext: { 1952 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1953 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 1954 ComputeMaskedBits(Op.getOperand(0), Mask & InMask, KnownZero, 1955 KnownOne, Depth+1); 1956 KnownZero |= (~InMask) & Mask; 1957 return; 1958 } 1959 case ISD::FGETSIGN: 1960 // All bits are zero except the low bit. 1961 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1962 return; 1963 1964 case ISD::SUB: { 1965 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 1966 // We know that the top bits of C-X are clear if X contains less bits 1967 // than C (i.e. no wrap-around can happen). For example, 20-X is 1968 // positive if we can prove that X is >= 0 and < 16. 1969 if (CLHS->getAPIntValue().isNonNegative()) { 1970 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 1971 // NLZ can't be BitWidth with no sign bit 1972 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 1973 ComputeMaskedBits(Op.getOperand(1), MaskV, KnownZero2, KnownOne2, 1974 Depth+1); 1975 1976 // If all of the MaskV bits are known to be zero, then we know the 1977 // output top bits are zero, because we now know that the output is 1978 // from [0-C]. 1979 if ((KnownZero2 & MaskV) == MaskV) { 1980 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 1981 // Top bits known zero. 1982 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2) & Mask; 1983 } 1984 } 1985 } 1986 } 1987 // fall through 1988 case ISD::ADD: 1989 case ISD::ADDE: { 1990 // Output known-0 bits are known if clear or set in both the low clear bits 1991 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 1992 // low 3 bits clear. 1993 APInt Mask2 = APInt::getLowBitsSet(BitWidth, 1994 BitWidth - Mask.countLeadingZeros()); 1995 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1); 1996 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1997 unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 1998 1999 ComputeMaskedBits(Op.getOperand(1), Mask2, KnownZero2, KnownOne2, Depth+1); 2000 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2001 KnownZeroOut = std::min(KnownZeroOut, 2002 KnownZero2.countTrailingOnes()); 2003 2004 if (Op.getOpcode() == ISD::ADD) { 2005 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2006 return; 2007 } 2008 2009 // With ADDE, a carry bit may be added in, so we can only use this 2010 // information if we know (at least) that the low two bits are clear. We 2011 // then return to the caller that the low bit is unknown but that other bits 2012 // are known zero. 2013 if (KnownZeroOut >= 2) // ADDE 2014 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2015 return; 2016 } 2017 case ISD::SREM: 2018 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2019 const APInt &RA = Rem->getAPIntValue().abs(); 2020 if (RA.isPowerOf2()) { 2021 APInt LowBits = RA - 1; 2022 APInt Mask2 = LowBits | APInt::getSignBit(BitWidth); 2023 ComputeMaskedBits(Op.getOperand(0), Mask2,KnownZero2,KnownOne2,Depth+1); 2024 2025 // The low bits of the first operand are unchanged by the srem. 2026 KnownZero = KnownZero2 & LowBits; 2027 KnownOne = KnownOne2 & LowBits; 2028 2029 // If the first operand is non-negative or has all low bits zero, then 2030 // the upper bits are all zero. 2031 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2032 KnownZero |= ~LowBits; 2033 2034 // If the first operand is negative and not all low bits are zero, then 2035 // the upper bits are all one. 2036 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2037 KnownOne |= ~LowBits; 2038 2039 KnownZero &= Mask; 2040 KnownOne &= Mask; 2041 2042 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2043 } 2044 } 2045 return; 2046 case ISD::UREM: { 2047 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2048 const APInt &RA = Rem->getAPIntValue(); 2049 if (RA.isPowerOf2()) { 2050 APInt LowBits = (RA - 1); 2051 APInt Mask2 = LowBits & Mask; 2052 KnownZero |= ~LowBits & Mask; 2053 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero, KnownOne,Depth+1); 2054 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2055 break; 2056 } 2057 } 2058 2059 // Since the result is less than or equal to either operand, any leading 2060 // zero bits in either operand must also exist in the result. 2061 APInt AllOnes = APInt::getAllOnesValue(BitWidth); 2062 ComputeMaskedBits(Op.getOperand(0), AllOnes, KnownZero, KnownOne, 2063 Depth+1); 2064 ComputeMaskedBits(Op.getOperand(1), AllOnes, KnownZero2, KnownOne2, 2065 Depth+1); 2066 2067 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2068 KnownZero2.countLeadingOnes()); 2069 KnownOne.clearAllBits(); 2070 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders) & Mask; 2071 return; 2072 } 2073 case ISD::FrameIndex: 2074 case ISD::TargetFrameIndex: 2075 if (unsigned Align = InferPtrAlignment(Op)) { 2076 // The low bits are known zero if the pointer is aligned. 2077 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2078 return; 2079 } 2080 break; 2081 2082 default: 2083 if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2084 break; 2085 // Fallthrough 2086 case ISD::INTRINSIC_WO_CHAIN: 2087 case ISD::INTRINSIC_W_CHAIN: 2088 case ISD::INTRINSIC_VOID: 2089 // Allow the target to implement this method for its nodes. 2090 TLI.computeMaskedBitsForTargetNode(Op, Mask, KnownZero, KnownOne, *this, 2091 Depth); 2092 return; 2093 } 2094} 2095 2096/// ComputeNumSignBits - Return the number of times the sign bit of the 2097/// register is replicated into the other bits. We know that at least 1 bit 2098/// is always equal to the sign bit (itself), but other cases can give us 2099/// information. For example, immediately after an "SRA X, 2", we know that 2100/// the top 3 bits are all equal to each other, so we return 3. 2101unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2102 EVT VT = Op.getValueType(); 2103 assert(VT.isInteger() && "Invalid VT!"); 2104 unsigned VTBits = VT.getScalarType().getSizeInBits(); 2105 unsigned Tmp, Tmp2; 2106 unsigned FirstAnswer = 1; 2107 2108 if (Depth == 6) 2109 return 1; // Limit search depth. 2110 2111 switch (Op.getOpcode()) { 2112 default: break; 2113 case ISD::AssertSext: 2114 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2115 return VTBits-Tmp+1; 2116 case ISD::AssertZext: 2117 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2118 return VTBits-Tmp; 2119 2120 case ISD::Constant: { 2121 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2122 return Val.getNumSignBits(); 2123 } 2124 2125 case ISD::SIGN_EXTEND: 2126 Tmp = VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2127 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2128 2129 case ISD::SIGN_EXTEND_INREG: 2130 // Max of the input and what this extends. 2131 Tmp = 2132 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2133 Tmp = VTBits-Tmp+1; 2134 2135 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2136 return std::max(Tmp, Tmp2); 2137 2138 case ISD::SRA: 2139 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2140 // SRA X, C -> adds C sign bits. 2141 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2142 Tmp += C->getZExtValue(); 2143 if (Tmp > VTBits) Tmp = VTBits; 2144 } 2145 return Tmp; 2146 case ISD::SHL: 2147 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2148 // shl destroys sign bits. 2149 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2150 if (C->getZExtValue() >= VTBits || // Bad shift. 2151 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2152 return Tmp - C->getZExtValue(); 2153 } 2154 break; 2155 case ISD::AND: 2156 case ISD::OR: 2157 case ISD::XOR: // NOT is handled here. 2158 // Logical binary ops preserve the number of sign bits at the worst. 2159 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2160 if (Tmp != 1) { 2161 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2162 FirstAnswer = std::min(Tmp, Tmp2); 2163 // We computed what we know about the sign bits as our first 2164 // answer. Now proceed to the generic code that uses 2165 // ComputeMaskedBits, and pick whichever answer is better. 2166 } 2167 break; 2168 2169 case ISD::SELECT: 2170 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2171 if (Tmp == 1) return 1; // Early out. 2172 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2173 return std::min(Tmp, Tmp2); 2174 2175 case ISD::SADDO: 2176 case ISD::UADDO: 2177 case ISD::SSUBO: 2178 case ISD::USUBO: 2179 case ISD::SMULO: 2180 case ISD::UMULO: 2181 if (Op.getResNo() != 1) 2182 break; 2183 // The boolean result conforms to getBooleanContents. Fall through. 2184 case ISD::SETCC: 2185 // If setcc returns 0/-1, all bits are sign bits. 2186 if (TLI.getBooleanContents(Op.getValueType().isVector()) == 2187 TargetLowering::ZeroOrNegativeOneBooleanContent) 2188 return VTBits; 2189 break; 2190 case ISD::ROTL: 2191 case ISD::ROTR: 2192 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2193 unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2194 2195 // Handle rotate right by N like a rotate left by 32-N. 2196 if (Op.getOpcode() == ISD::ROTR) 2197 RotAmt = (VTBits-RotAmt) & (VTBits-1); 2198 2199 // If we aren't rotating out all of the known-in sign bits, return the 2200 // number that are left. This handles rotl(sext(x), 1) for example. 2201 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2202 if (Tmp > RotAmt+1) return Tmp-RotAmt; 2203 } 2204 break; 2205 case ISD::ADD: 2206 // Add can have at most one carry bit. Thus we know that the output 2207 // is, at worst, one more bit than the inputs. 2208 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2209 if (Tmp == 1) return 1; // Early out. 2210 2211 // Special case decrementing a value (ADD X, -1): 2212 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2213 if (CRHS->isAllOnesValue()) { 2214 APInt KnownZero, KnownOne; 2215 APInt Mask = APInt::getAllOnesValue(VTBits); 2216 ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1); 2217 2218 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2219 // sign bits set. 2220 if ((KnownZero | APInt(VTBits, 1)) == Mask) 2221 return VTBits; 2222 2223 // If we are subtracting one from a positive number, there is no carry 2224 // out of the result. 2225 if (KnownZero.isNegative()) 2226 return Tmp; 2227 } 2228 2229 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2230 if (Tmp2 == 1) return 1; 2231 return std::min(Tmp, Tmp2)-1; 2232 break; 2233 2234 case ISD::SUB: 2235 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2236 if (Tmp2 == 1) return 1; 2237 2238 // Handle NEG. 2239 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2240 if (CLHS->isNullValue()) { 2241 APInt KnownZero, KnownOne; 2242 APInt Mask = APInt::getAllOnesValue(VTBits); 2243 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 2244 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2245 // sign bits set. 2246 if ((KnownZero | APInt(VTBits, 1)) == Mask) 2247 return VTBits; 2248 2249 // If the input is known to be positive (the sign bit is known clear), 2250 // the output of the NEG has the same number of sign bits as the input. 2251 if (KnownZero.isNegative()) 2252 return Tmp2; 2253 2254 // Otherwise, we treat this like a SUB. 2255 } 2256 2257 // Sub can have at most one carry bit. Thus we know that the output 2258 // is, at worst, one more bit than the inputs. 2259 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2260 if (Tmp == 1) return 1; // Early out. 2261 return std::min(Tmp, Tmp2)-1; 2262 break; 2263 case ISD::TRUNCATE: 2264 // FIXME: it's tricky to do anything useful for this, but it is an important 2265 // case for targets like X86. 2266 break; 2267 } 2268 2269 // Handle LOADX separately here. EXTLOAD case will fallthrough. 2270 if (Op.getOpcode() == ISD::LOAD) { 2271 LoadSDNode *LD = cast<LoadSDNode>(Op); 2272 unsigned ExtType = LD->getExtensionType(); 2273 switch (ExtType) { 2274 default: break; 2275 case ISD::SEXTLOAD: // '17' bits known 2276 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2277 return VTBits-Tmp+1; 2278 case ISD::ZEXTLOAD: // '16' bits known 2279 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2280 return VTBits-Tmp; 2281 } 2282 } 2283 2284 // Allow the target to implement this method for its nodes. 2285 if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2286 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2287 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2288 Op.getOpcode() == ISD::INTRINSIC_VOID) { 2289 unsigned NumBits = TLI.ComputeNumSignBitsForTargetNode(Op, Depth); 2290 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2291 } 2292 2293 // Finally, if we can prove that the top bits of the result are 0's or 1's, 2294 // use this information. 2295 APInt KnownZero, KnownOne; 2296 APInt Mask = APInt::getAllOnesValue(VTBits); 2297 ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth); 2298 2299 if (KnownZero.isNegative()) { // sign bit is 0 2300 Mask = KnownZero; 2301 } else if (KnownOne.isNegative()) { // sign bit is 1; 2302 Mask = KnownOne; 2303 } else { 2304 // Nothing known. 2305 return FirstAnswer; 2306 } 2307 2308 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2309 // the number of identical bits in the top of the input value. 2310 Mask = ~Mask; 2311 Mask <<= Mask.getBitWidth()-VTBits; 2312 // Return # leading zeros. We use 'min' here in case Val was zero before 2313 // shifting. We don't want to return '64' as for an i32 "0". 2314 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2315} 2316 2317/// isBaseWithConstantOffset - Return true if the specified operand is an 2318/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2319/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2320/// semantics as an ADD. This handles the equivalence: 2321/// X|Cst == X+Cst iff X&Cst = 0. 2322bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2323 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2324 !isa<ConstantSDNode>(Op.getOperand(1))) 2325 return false; 2326 2327 if (Op.getOpcode() == ISD::OR && 2328 !MaskedValueIsZero(Op.getOperand(0), 2329 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2330 return false; 2331 2332 return true; 2333} 2334 2335 2336bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2337 // If we're told that NaNs won't happen, assume they won't. 2338 if (NoNaNsFPMath) 2339 return true; 2340 2341 // If the value is a constant, we can obviously see if it is a NaN or not. 2342 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2343 return !C->getValueAPF().isNaN(); 2344 2345 // TODO: Recognize more cases here. 2346 2347 return false; 2348} 2349 2350bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2351 // If the value is a constant, we can obviously see if it is a zero or not. 2352 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2353 return !C->isZero(); 2354 2355 // TODO: Recognize more cases here. 2356 switch (Op.getOpcode()) { 2357 default: break; 2358 case ISD::OR: 2359 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2360 return !C->isNullValue(); 2361 break; 2362 } 2363 2364 return false; 2365} 2366 2367bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2368 // Check the obvious case. 2369 if (A == B) return true; 2370 2371 // For for negative and positive zero. 2372 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2373 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2374 if (CA->isZero() && CB->isZero()) return true; 2375 2376 // Otherwise they may not be equal. 2377 return false; 2378} 2379 2380/// getNode - Gets or creates the specified node. 2381/// 2382SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) { 2383 FoldingSetNodeID ID; 2384 AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2385 void *IP = 0; 2386 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2387 return SDValue(E, 0); 2388 2389 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL, getVTList(VT)); 2390 CSEMap.InsertNode(N, IP); 2391 2392 AllNodes.push_back(N); 2393#ifndef NDEBUG 2394 VerifySDNode(N); 2395#endif 2396 return SDValue(N, 0); 2397} 2398 2399SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 2400 EVT VT, SDValue Operand) { 2401 // Constant fold unary operations with an integer constant operand. 2402 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2403 const APInt &Val = C->getAPIntValue(); 2404 switch (Opcode) { 2405 default: break; 2406 case ISD::SIGN_EXTEND: 2407 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2408 case ISD::ANY_EXTEND: 2409 case ISD::ZERO_EXTEND: 2410 case ISD::TRUNCATE: 2411 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2412 case ISD::UINT_TO_FP: 2413 case ISD::SINT_TO_FP: { 2414 // No compile time operations on ppcf128. 2415 if (VT == MVT::ppcf128) break; 2416 APFloat apf(APInt::getNullValue(VT.getSizeInBits())); 2417 (void)apf.convertFromAPInt(Val, 2418 Opcode==ISD::SINT_TO_FP, 2419 APFloat::rmNearestTiesToEven); 2420 return getConstantFP(apf, VT); 2421 } 2422 case ISD::BITCAST: 2423 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2424 return getConstantFP(Val.bitsToFloat(), VT); 2425 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2426 return getConstantFP(Val.bitsToDouble(), VT); 2427 break; 2428 case ISD::BSWAP: 2429 return getConstant(Val.byteSwap(), VT); 2430 case ISD::CTPOP: 2431 return getConstant(Val.countPopulation(), VT); 2432 case ISD::CTLZ: 2433 return getConstant(Val.countLeadingZeros(), VT); 2434 case ISD::CTTZ: 2435 return getConstant(Val.countTrailingZeros(), VT); 2436 } 2437 } 2438 2439 // Constant fold unary operations with a floating point constant operand. 2440 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2441 APFloat V = C->getValueAPF(); // make copy 2442 if (VT != MVT::ppcf128 && Operand.getValueType() != MVT::ppcf128) { 2443 switch (Opcode) { 2444 case ISD::FNEG: 2445 V.changeSign(); 2446 return getConstantFP(V, VT); 2447 case ISD::FABS: 2448 V.clearSign(); 2449 return getConstantFP(V, VT); 2450 case ISD::FP_ROUND: 2451 case ISD::FP_EXTEND: { 2452 bool ignored; 2453 // This can return overflow, underflow, or inexact; we don't care. 2454 // FIXME need to be more flexible about rounding mode. 2455 (void)V.convert(*EVTToAPFloatSemantics(VT), 2456 APFloat::rmNearestTiesToEven, &ignored); 2457 return getConstantFP(V, VT); 2458 } 2459 case ISD::FP_TO_SINT: 2460 case ISD::FP_TO_UINT: { 2461 integerPart x[2]; 2462 bool ignored; 2463 assert(integerPartWidth >= 64); 2464 // FIXME need to be more flexible about rounding mode. 2465 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2466 Opcode==ISD::FP_TO_SINT, 2467 APFloat::rmTowardZero, &ignored); 2468 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2469 break; 2470 APInt api(VT.getSizeInBits(), x); 2471 return getConstant(api, VT); 2472 } 2473 case ISD::BITCAST: 2474 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2475 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2476 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2477 return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2478 break; 2479 } 2480 } 2481 } 2482 2483 unsigned OpOpcode = Operand.getNode()->getOpcode(); 2484 switch (Opcode) { 2485 case ISD::TokenFactor: 2486 case ISD::MERGE_VALUES: 2487 case ISD::CONCAT_VECTORS: 2488 return Operand; // Factor, merge or concat of one node? No need. 2489 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2490 case ISD::FP_EXTEND: 2491 assert(VT.isFloatingPoint() && 2492 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2493 if (Operand.getValueType() == VT) return Operand; // noop conversion. 2494 assert((!VT.isVector() || 2495 VT.getVectorNumElements() == 2496 Operand.getValueType().getVectorNumElements()) && 2497 "Vector element count mismatch!"); 2498 if (Operand.getOpcode() == ISD::UNDEF) 2499 return getUNDEF(VT); 2500 break; 2501 case ISD::SIGN_EXTEND: 2502 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2503 "Invalid SIGN_EXTEND!"); 2504 if (Operand.getValueType() == VT) return Operand; // noop extension 2505 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2506 "Invalid sext node, dst < src!"); 2507 assert((!VT.isVector() || 2508 VT.getVectorNumElements() == 2509 Operand.getValueType().getVectorNumElements()) && 2510 "Vector element count mismatch!"); 2511 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2512 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2513 else if (OpOpcode == ISD::UNDEF) 2514 // sext(undef) = 0, because the top bits will all be the same. 2515 return getConstant(0, VT); 2516 break; 2517 case ISD::ZERO_EXTEND: 2518 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2519 "Invalid ZERO_EXTEND!"); 2520 if (Operand.getValueType() == VT) return Operand; // noop extension 2521 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2522 "Invalid zext node, dst < src!"); 2523 assert((!VT.isVector() || 2524 VT.getVectorNumElements() == 2525 Operand.getValueType().getVectorNumElements()) && 2526 "Vector element count mismatch!"); 2527 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2528 return getNode(ISD::ZERO_EXTEND, DL, VT, 2529 Operand.getNode()->getOperand(0)); 2530 else if (OpOpcode == ISD::UNDEF) 2531 // zext(undef) = 0, because the top bits will be zero. 2532 return getConstant(0, VT); 2533 break; 2534 case ISD::ANY_EXTEND: 2535 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2536 "Invalid ANY_EXTEND!"); 2537 if (Operand.getValueType() == VT) return Operand; // noop extension 2538 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2539 "Invalid anyext node, dst < src!"); 2540 assert((!VT.isVector() || 2541 VT.getVectorNumElements() == 2542 Operand.getValueType().getVectorNumElements()) && 2543 "Vector element count mismatch!"); 2544 2545 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2546 OpOpcode == ISD::ANY_EXTEND) 2547 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2548 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2549 else if (OpOpcode == ISD::UNDEF) 2550 return getUNDEF(VT); 2551 2552 // (ext (trunx x)) -> x 2553 if (OpOpcode == ISD::TRUNCATE) { 2554 SDValue OpOp = Operand.getNode()->getOperand(0); 2555 if (OpOp.getValueType() == VT) 2556 return OpOp; 2557 } 2558 break; 2559 case ISD::TRUNCATE: 2560 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2561 "Invalid TRUNCATE!"); 2562 if (Operand.getValueType() == VT) return Operand; // noop truncate 2563 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2564 "Invalid truncate node, src < dst!"); 2565 assert((!VT.isVector() || 2566 VT.getVectorNumElements() == 2567 Operand.getValueType().getVectorNumElements()) && 2568 "Vector element count mismatch!"); 2569 if (OpOpcode == ISD::TRUNCATE) 2570 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2571 else if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2572 OpOpcode == ISD::ANY_EXTEND) { 2573 // If the source is smaller than the dest, we still need an extend. 2574 if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2575 .bitsLT(VT.getScalarType())) 2576 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2577 else if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2578 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2579 else 2580 return Operand.getNode()->getOperand(0); 2581 } 2582 break; 2583 case ISD::BITCAST: 2584 // Basic sanity checking. 2585 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2586 && "Cannot BITCAST between types of different sizes!"); 2587 if (VT == Operand.getValueType()) return Operand; // noop conversion. 2588 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2589 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2590 if (OpOpcode == ISD::UNDEF) 2591 return getUNDEF(VT); 2592 break; 2593 case ISD::SCALAR_TO_VECTOR: 2594 assert(VT.isVector() && !Operand.getValueType().isVector() && 2595 (VT.getVectorElementType() == Operand.getValueType() || 2596 (VT.getVectorElementType().isInteger() && 2597 Operand.getValueType().isInteger() && 2598 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2599 "Illegal SCALAR_TO_VECTOR node!"); 2600 if (OpOpcode == ISD::UNDEF) 2601 return getUNDEF(VT); 2602 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2603 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2604 isa<ConstantSDNode>(Operand.getOperand(1)) && 2605 Operand.getConstantOperandVal(1) == 0 && 2606 Operand.getOperand(0).getValueType() == VT) 2607 return Operand.getOperand(0); 2608 break; 2609 case ISD::FNEG: 2610 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2611 if (UnsafeFPMath && OpOpcode == ISD::FSUB) 2612 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2613 Operand.getNode()->getOperand(0)); 2614 if (OpOpcode == ISD::FNEG) // --X -> X 2615 return Operand.getNode()->getOperand(0); 2616 break; 2617 case ISD::FABS: 2618 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2619 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2620 break; 2621 } 2622 2623 SDNode *N; 2624 SDVTList VTs = getVTList(VT); 2625 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2626 FoldingSetNodeID ID; 2627 SDValue Ops[1] = { Operand }; 2628 AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2629 void *IP = 0; 2630 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2631 return SDValue(E, 0); 2632 2633 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); 2634 CSEMap.InsertNode(N, IP); 2635 } else { 2636 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); 2637 } 2638 2639 AllNodes.push_back(N); 2640#ifndef NDEBUG 2641 VerifySDNode(N); 2642#endif 2643 return SDValue(N, 0); 2644} 2645 2646SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, 2647 EVT VT, 2648 ConstantSDNode *Cst1, 2649 ConstantSDNode *Cst2) { 2650 const APInt &C1 = Cst1->getAPIntValue(), &C2 = Cst2->getAPIntValue(); 2651 2652 switch (Opcode) { 2653 case ISD::ADD: return getConstant(C1 + C2, VT); 2654 case ISD::SUB: return getConstant(C1 - C2, VT); 2655 case ISD::MUL: return getConstant(C1 * C2, VT); 2656 case ISD::UDIV: 2657 if (C2.getBoolValue()) return getConstant(C1.udiv(C2), VT); 2658 break; 2659 case ISD::UREM: 2660 if (C2.getBoolValue()) return getConstant(C1.urem(C2), VT); 2661 break; 2662 case ISD::SDIV: 2663 if (C2.getBoolValue()) return getConstant(C1.sdiv(C2), VT); 2664 break; 2665 case ISD::SREM: 2666 if (C2.getBoolValue()) return getConstant(C1.srem(C2), VT); 2667 break; 2668 case ISD::AND: return getConstant(C1 & C2, VT); 2669 case ISD::OR: return getConstant(C1 | C2, VT); 2670 case ISD::XOR: return getConstant(C1 ^ C2, VT); 2671 case ISD::SHL: return getConstant(C1 << C2, VT); 2672 case ISD::SRL: return getConstant(C1.lshr(C2), VT); 2673 case ISD::SRA: return getConstant(C1.ashr(C2), VT); 2674 case ISD::ROTL: return getConstant(C1.rotl(C2), VT); 2675 case ISD::ROTR: return getConstant(C1.rotr(C2), VT); 2676 default: break; 2677 } 2678 2679 return SDValue(); 2680} 2681 2682SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 2683 SDValue N1, SDValue N2) { 2684 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2685 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2686 switch (Opcode) { 2687 default: break; 2688 case ISD::TokenFactor: 2689 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2690 N2.getValueType() == MVT::Other && "Invalid token factor!"); 2691 // Fold trivial token factors. 2692 if (N1.getOpcode() == ISD::EntryToken) return N2; 2693 if (N2.getOpcode() == ISD::EntryToken) return N1; 2694 if (N1 == N2) return N1; 2695 break; 2696 case ISD::CONCAT_VECTORS: 2697 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2698 // one big BUILD_VECTOR. 2699 if (N1.getOpcode() == ISD::BUILD_VECTOR && 2700 N2.getOpcode() == ISD::BUILD_VECTOR) { 2701 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2702 N1.getNode()->op_end()); 2703 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2704 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2705 } 2706 break; 2707 case ISD::AND: 2708 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2709 assert(N1.getValueType() == N2.getValueType() && 2710 N1.getValueType() == VT && "Binary operator types must match!"); 2711 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2712 // worth handling here. 2713 if (N2C && N2C->isNullValue()) 2714 return N2; 2715 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2716 return N1; 2717 break; 2718 case ISD::OR: 2719 case ISD::XOR: 2720 case ISD::ADD: 2721 case ISD::SUB: 2722 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2723 assert(N1.getValueType() == N2.getValueType() && 2724 N1.getValueType() == VT && "Binary operator types must match!"); 2725 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2726 // it's worth handling here. 2727 if (N2C && N2C->isNullValue()) 2728 return N1; 2729 break; 2730 case ISD::UDIV: 2731 case ISD::UREM: 2732 case ISD::MULHU: 2733 case ISD::MULHS: 2734 case ISD::MUL: 2735 case ISD::SDIV: 2736 case ISD::SREM: 2737 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2738 assert(N1.getValueType() == N2.getValueType() && 2739 N1.getValueType() == VT && "Binary operator types must match!"); 2740 break; 2741 case ISD::FADD: 2742 case ISD::FSUB: 2743 case ISD::FMUL: 2744 case ISD::FDIV: 2745 case ISD::FREM: 2746 if (UnsafeFPMath) { 2747 if (Opcode == ISD::FADD) { 2748 // 0+x --> x 2749 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2750 if (CFP->getValueAPF().isZero()) 2751 return N2; 2752 // x+0 --> x 2753 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2754 if (CFP->getValueAPF().isZero()) 2755 return N1; 2756 } else if (Opcode == ISD::FSUB) { 2757 // x-0 --> x 2758 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2759 if (CFP->getValueAPF().isZero()) 2760 return N1; 2761 } 2762 } 2763 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2764 assert(N1.getValueType() == N2.getValueType() && 2765 N1.getValueType() == VT && "Binary operator types must match!"); 2766 break; 2767 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2768 assert(N1.getValueType() == VT && 2769 N1.getValueType().isFloatingPoint() && 2770 N2.getValueType().isFloatingPoint() && 2771 "Invalid FCOPYSIGN!"); 2772 break; 2773 case ISD::SHL: 2774 case ISD::SRA: 2775 case ISD::SRL: 2776 case ISD::ROTL: 2777 case ISD::ROTR: 2778 assert(VT == N1.getValueType() && 2779 "Shift operators return type must be the same as their first arg"); 2780 assert(VT.isInteger() && N2.getValueType().isInteger() && 2781 "Shifts only work on integers"); 2782 // Verify that the shift amount VT is bit enough to hold valid shift 2783 // amounts. This catches things like trying to shift an i1024 value by an 2784 // i8, which is easy to fall into in generic code that uses 2785 // TLI.getShiftAmount(). 2786 assert(N2.getValueType().getSizeInBits() >= 2787 Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 2788 "Invalid use of small shift amount with oversized value!"); 2789 2790 // Always fold shifts of i1 values so the code generator doesn't need to 2791 // handle them. Since we know the size of the shift has to be less than the 2792 // size of the value, the shift/rotate count is guaranteed to be zero. 2793 if (VT == MVT::i1) 2794 return N1; 2795 if (N2C && N2C->isNullValue()) 2796 return N1; 2797 break; 2798 case ISD::FP_ROUND_INREG: { 2799 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2800 assert(VT == N1.getValueType() && "Not an inreg round!"); 2801 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 2802 "Cannot FP_ROUND_INREG integer types"); 2803 assert(EVT.isVector() == VT.isVector() && 2804 "FP_ROUND_INREG type should be vector iff the operand " 2805 "type is vector!"); 2806 assert((!EVT.isVector() || 2807 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2808 "Vector element counts must match in FP_ROUND_INREG"); 2809 assert(EVT.bitsLE(VT) && "Not rounding down!"); 2810 (void)EVT; 2811 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 2812 break; 2813 } 2814 case ISD::FP_ROUND: 2815 assert(VT.isFloatingPoint() && 2816 N1.getValueType().isFloatingPoint() && 2817 VT.bitsLE(N1.getValueType()) && 2818 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 2819 if (N1.getValueType() == VT) return N1; // noop conversion. 2820 break; 2821 case ISD::AssertSext: 2822 case ISD::AssertZext: { 2823 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2824 assert(VT == N1.getValueType() && "Not an inreg extend!"); 2825 assert(VT.isInteger() && EVT.isInteger() && 2826 "Cannot *_EXTEND_INREG FP types"); 2827 assert(!EVT.isVector() && 2828 "AssertSExt/AssertZExt type should be the vector element type " 2829 "rather than the vector type!"); 2830 assert(EVT.bitsLE(VT) && "Not extending!"); 2831 if (VT == EVT) return N1; // noop assertion. 2832 break; 2833 } 2834 case ISD::SIGN_EXTEND_INREG: { 2835 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2836 assert(VT == N1.getValueType() && "Not an inreg extend!"); 2837 assert(VT.isInteger() && EVT.isInteger() && 2838 "Cannot *_EXTEND_INREG FP types"); 2839 assert(EVT.isVector() == VT.isVector() && 2840 "SIGN_EXTEND_INREG type should be vector iff the operand " 2841 "type is vector!"); 2842 assert((!EVT.isVector() || 2843 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2844 "Vector element counts must match in SIGN_EXTEND_INREG"); 2845 assert(EVT.bitsLE(VT) && "Not extending!"); 2846 if (EVT == VT) return N1; // Not actually extending 2847 2848 if (N1C) { 2849 APInt Val = N1C->getAPIntValue(); 2850 unsigned FromBits = EVT.getScalarType().getSizeInBits(); 2851 Val <<= Val.getBitWidth()-FromBits; 2852 Val = Val.ashr(Val.getBitWidth()-FromBits); 2853 return getConstant(Val, VT); 2854 } 2855 break; 2856 } 2857 case ISD::EXTRACT_VECTOR_ELT: 2858 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 2859 if (N1.getOpcode() == ISD::UNDEF) 2860 return getUNDEF(VT); 2861 2862 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 2863 // expanding copies of large vectors from registers. 2864 if (N2C && 2865 N1.getOpcode() == ISD::CONCAT_VECTORS && 2866 N1.getNumOperands() > 0) { 2867 unsigned Factor = 2868 N1.getOperand(0).getValueType().getVectorNumElements(); 2869 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 2870 N1.getOperand(N2C->getZExtValue() / Factor), 2871 getConstant(N2C->getZExtValue() % Factor, 2872 N2.getValueType())); 2873 } 2874 2875 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 2876 // expanding large vector constants. 2877 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 2878 SDValue Elt = N1.getOperand(N2C->getZExtValue()); 2879 EVT VEltTy = N1.getValueType().getVectorElementType(); 2880 if (Elt.getValueType() != VEltTy) { 2881 // If the vector element type is not legal, the BUILD_VECTOR operands 2882 // are promoted and implicitly truncated. Make that explicit here. 2883 Elt = getNode(ISD::TRUNCATE, DL, VEltTy, Elt); 2884 } 2885 if (VT != VEltTy) { 2886 // If the vector element type is not legal, the EXTRACT_VECTOR_ELT 2887 // result is implicitly extended. 2888 Elt = getNode(ISD::ANY_EXTEND, DL, VT, Elt); 2889 } 2890 return Elt; 2891 } 2892 2893 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 2894 // operations are lowered to scalars. 2895 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 2896 // If the indices are the same, return the inserted element else 2897 // if the indices are known different, extract the element from 2898 // the original vector. 2899 SDValue N1Op2 = N1.getOperand(2); 2900 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 2901 2902 if (N1Op2C && N2C) { 2903 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 2904 if (VT == N1.getOperand(1).getValueType()) 2905 return N1.getOperand(1); 2906 else 2907 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 2908 } 2909 2910 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 2911 } 2912 } 2913 break; 2914 case ISD::EXTRACT_ELEMENT: 2915 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 2916 assert(!N1.getValueType().isVector() && !VT.isVector() && 2917 (N1.getValueType().isInteger() == VT.isInteger()) && 2918 N1.getValueType() != VT && 2919 "Wrong types for EXTRACT_ELEMENT!"); 2920 2921 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 2922 // 64-bit integers into 32-bit parts. Instead of building the extract of 2923 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 2924 if (N1.getOpcode() == ISD::BUILD_PAIR) 2925 return N1.getOperand(N2C->getZExtValue()); 2926 2927 // EXTRACT_ELEMENT of a constant int is also very common. 2928 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 2929 unsigned ElementSize = VT.getSizeInBits(); 2930 unsigned Shift = ElementSize * N2C->getZExtValue(); 2931 APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 2932 return getConstant(ShiftedVal.trunc(ElementSize), VT); 2933 } 2934 break; 2935 case ISD::EXTRACT_SUBVECTOR: { 2936 SDValue Index = N2; 2937 if (VT.isSimple() && N1.getValueType().isSimple()) { 2938 assert(VT.isVector() && N1.getValueType().isVector() && 2939 "Extract subvector VTs must be a vectors!"); 2940 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() && 2941 "Extract subvector VTs must have the same element type!"); 2942 assert(VT.getSimpleVT() <= N1.getValueType().getSimpleVT() && 2943 "Extract subvector must be from larger vector to smaller vector!"); 2944 2945 if (isa<ConstantSDNode>(Index.getNode())) { 2946 assert((VT.getVectorNumElements() + 2947 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 2948 <= N1.getValueType().getVectorNumElements()) 2949 && "Extract subvector overflow!"); 2950 } 2951 2952 // Trivial extraction. 2953 if (VT.getSimpleVT() == N1.getValueType().getSimpleVT()) 2954 return N1; 2955 } 2956 break; 2957 } 2958 } 2959 2960 if (N1C) { 2961 if (N2C) { 2962 SDValue SV = FoldConstantArithmetic(Opcode, VT, N1C, N2C); 2963 if (SV.getNode()) return SV; 2964 } else { // Cannonicalize constant to RHS if commutative 2965 if (isCommutativeBinOp(Opcode)) { 2966 std::swap(N1C, N2C); 2967 std::swap(N1, N2); 2968 } 2969 } 2970 } 2971 2972 // Constant fold FP operations. 2973 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 2974 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 2975 if (N1CFP) { 2976 if (!N2CFP && isCommutativeBinOp(Opcode)) { 2977 // Cannonicalize constant to RHS if commutative 2978 std::swap(N1CFP, N2CFP); 2979 std::swap(N1, N2); 2980 } else if (N2CFP && VT != MVT::ppcf128) { 2981 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 2982 APFloat::opStatus s; 2983 switch (Opcode) { 2984 case ISD::FADD: 2985 s = V1.add(V2, APFloat::rmNearestTiesToEven); 2986 if (s != APFloat::opInvalidOp) 2987 return getConstantFP(V1, VT); 2988 break; 2989 case ISD::FSUB: 2990 s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 2991 if (s!=APFloat::opInvalidOp) 2992 return getConstantFP(V1, VT); 2993 break; 2994 case ISD::FMUL: 2995 s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 2996 if (s!=APFloat::opInvalidOp) 2997 return getConstantFP(V1, VT); 2998 break; 2999 case ISD::FDIV: 3000 s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3001 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3002 return getConstantFP(V1, VT); 3003 break; 3004 case ISD::FREM : 3005 s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3006 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3007 return getConstantFP(V1, VT); 3008 break; 3009 case ISD::FCOPYSIGN: 3010 V1.copySign(V2); 3011 return getConstantFP(V1, VT); 3012 default: break; 3013 } 3014 } 3015 } 3016 3017 // Canonicalize an UNDEF to the RHS, even over a constant. 3018 if (N1.getOpcode() == ISD::UNDEF) { 3019 if (isCommutativeBinOp(Opcode)) { 3020 std::swap(N1, N2); 3021 } else { 3022 switch (Opcode) { 3023 case ISD::FP_ROUND_INREG: 3024 case ISD::SIGN_EXTEND_INREG: 3025 case ISD::SUB: 3026 case ISD::FSUB: 3027 case ISD::FDIV: 3028 case ISD::FREM: 3029 case ISD::SRA: 3030 return N1; // fold op(undef, arg2) -> undef 3031 case ISD::UDIV: 3032 case ISD::SDIV: 3033 case ISD::UREM: 3034 case ISD::SREM: 3035 case ISD::SRL: 3036 case ISD::SHL: 3037 if (!VT.isVector()) 3038 return getConstant(0, VT); // fold op(undef, arg2) -> 0 3039 // For vectors, we can't easily build an all zero vector, just return 3040 // the LHS. 3041 return N2; 3042 } 3043 } 3044 } 3045 3046 // Fold a bunch of operators when the RHS is undef. 3047 if (N2.getOpcode() == ISD::UNDEF) { 3048 switch (Opcode) { 3049 case ISD::XOR: 3050 if (N1.getOpcode() == ISD::UNDEF) 3051 // Handle undef ^ undef -> 0 special case. This is a common 3052 // idiom (misuse). 3053 return getConstant(0, VT); 3054 // fallthrough 3055 case ISD::ADD: 3056 case ISD::ADDC: 3057 case ISD::ADDE: 3058 case ISD::SUB: 3059 case ISD::UDIV: 3060 case ISD::SDIV: 3061 case ISD::UREM: 3062 case ISD::SREM: 3063 return N2; // fold op(arg1, undef) -> undef 3064 case ISD::FADD: 3065 case ISD::FSUB: 3066 case ISD::FMUL: 3067 case ISD::FDIV: 3068 case ISD::FREM: 3069 if (UnsafeFPMath) 3070 return N2; 3071 break; 3072 case ISD::MUL: 3073 case ISD::AND: 3074 case ISD::SRL: 3075 case ISD::SHL: 3076 if (!VT.isVector()) 3077 return getConstant(0, VT); // fold op(arg1, undef) -> 0 3078 // For vectors, we can't easily build an all zero vector, just return 3079 // the LHS. 3080 return N1; 3081 case ISD::OR: 3082 if (!VT.isVector()) 3083 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3084 // For vectors, we can't easily build an all one vector, just return 3085 // the LHS. 3086 return N1; 3087 case ISD::SRA: 3088 return N1; 3089 } 3090 } 3091 3092 // Memoize this node if possible. 3093 SDNode *N; 3094 SDVTList VTs = getVTList(VT); 3095 if (VT != MVT::Glue) { 3096 SDValue Ops[] = { N1, N2 }; 3097 FoldingSetNodeID ID; 3098 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3099 void *IP = 0; 3100 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3101 return SDValue(E, 0); 3102 3103 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); 3104 CSEMap.InsertNode(N, IP); 3105 } else { 3106 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); 3107 } 3108 3109 AllNodes.push_back(N); 3110#ifndef NDEBUG 3111 VerifySDNode(N); 3112#endif 3113 return SDValue(N, 0); 3114} 3115 3116SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3117 SDValue N1, SDValue N2, SDValue N3) { 3118 // Perform various simplifications. 3119 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3120 switch (Opcode) { 3121 case ISD::CONCAT_VECTORS: 3122 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3123 // one big BUILD_VECTOR. 3124 if (N1.getOpcode() == ISD::BUILD_VECTOR && 3125 N2.getOpcode() == ISD::BUILD_VECTOR && 3126 N3.getOpcode() == ISD::BUILD_VECTOR) { 3127 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3128 N1.getNode()->op_end()); 3129 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3130 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3131 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3132 } 3133 break; 3134 case ISD::SETCC: { 3135 // Use FoldSetCC to simplify SETCC's. 3136 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3137 if (Simp.getNode()) return Simp; 3138 break; 3139 } 3140 case ISD::SELECT: 3141 if (N1C) { 3142 if (N1C->getZExtValue()) 3143 return N2; // select true, X, Y -> X 3144 else 3145 return N3; // select false, X, Y -> Y 3146 } 3147 3148 if (N2 == N3) return N2; // select C, X, X -> X 3149 break; 3150 case ISD::VECTOR_SHUFFLE: 3151 llvm_unreachable("should use getVectorShuffle constructor!"); 3152 break; 3153 case ISD::INSERT_SUBVECTOR: { 3154 SDValue Index = N3; 3155 if (VT.isSimple() && N1.getValueType().isSimple() 3156 && N2.getValueType().isSimple()) { 3157 assert(VT.isVector() && N1.getValueType().isVector() && 3158 N2.getValueType().isVector() && 3159 "Insert subvector VTs must be a vectors"); 3160 assert(VT == N1.getValueType() && 3161 "Dest and insert subvector source types must match!"); 3162 assert(N2.getValueType().getSimpleVT() <= N1.getValueType().getSimpleVT() && 3163 "Insert subvector must be from smaller vector to larger vector!"); 3164 if (isa<ConstantSDNode>(Index.getNode())) { 3165 assert((N2.getValueType().getVectorNumElements() + 3166 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3167 <= VT.getVectorNumElements()) 3168 && "Insert subvector overflow!"); 3169 } 3170 3171 // Trivial insertion. 3172 if (VT.getSimpleVT() == N2.getValueType().getSimpleVT()) 3173 return N2; 3174 } 3175 break; 3176 } 3177 case ISD::BITCAST: 3178 // Fold bit_convert nodes from a type to themselves. 3179 if (N1.getValueType() == VT) 3180 return N1; 3181 break; 3182 } 3183 3184 // Memoize node if it doesn't produce a flag. 3185 SDNode *N; 3186 SDVTList VTs = getVTList(VT); 3187 if (VT != MVT::Glue) { 3188 SDValue Ops[] = { N1, N2, N3 }; 3189 FoldingSetNodeID ID; 3190 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3191 void *IP = 0; 3192 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3193 return SDValue(E, 0); 3194 3195 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); 3196 CSEMap.InsertNode(N, IP); 3197 } else { 3198 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); 3199 } 3200 3201 AllNodes.push_back(N); 3202#ifndef NDEBUG 3203 VerifySDNode(N); 3204#endif 3205 return SDValue(N, 0); 3206} 3207 3208SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3209 SDValue N1, SDValue N2, SDValue N3, 3210 SDValue N4) { 3211 SDValue Ops[] = { N1, N2, N3, N4 }; 3212 return getNode(Opcode, DL, VT, Ops, 4); 3213} 3214 3215SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3216 SDValue N1, SDValue N2, SDValue N3, 3217 SDValue N4, SDValue N5) { 3218 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3219 return getNode(Opcode, DL, VT, Ops, 5); 3220} 3221 3222/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3223/// the incoming stack arguments to be loaded from the stack. 3224SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3225 SmallVector<SDValue, 8> ArgChains; 3226 3227 // Include the original chain at the beginning of the list. When this is 3228 // used by target LowerCall hooks, this helps legalize find the 3229 // CALLSEQ_BEGIN node. 3230 ArgChains.push_back(Chain); 3231 3232 // Add a chain value for each stack argument. 3233 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3234 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3235 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3236 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3237 if (FI->getIndex() < 0) 3238 ArgChains.push_back(SDValue(L, 1)); 3239 3240 // Build a tokenfactor for all the chains. 3241 return getNode(ISD::TokenFactor, Chain.getDebugLoc(), MVT::Other, 3242 &ArgChains[0], ArgChains.size()); 3243} 3244 3245/// SplatByte - Distribute ByteVal over NumBits bits. 3246static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) { 3247 APInt Val = APInt(NumBits, ByteVal); 3248 unsigned Shift = 8; 3249 for (unsigned i = NumBits; i > 8; i >>= 1) { 3250 Val = (Val << Shift) | Val; 3251 Shift <<= 1; 3252 } 3253 return Val; 3254} 3255 3256/// getMemsetValue - Vectorized representation of the memset value 3257/// operand. 3258static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3259 DebugLoc dl) { 3260 assert(Value.getOpcode() != ISD::UNDEF); 3261 3262 unsigned NumBits = VT.getScalarType().getSizeInBits(); 3263 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3264 APInt Val = SplatByte(NumBits, C->getZExtValue() & 255); 3265 if (VT.isInteger()) 3266 return DAG.getConstant(Val, VT); 3267 return DAG.getConstantFP(APFloat(Val), VT); 3268 } 3269 3270 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3271 if (NumBits > 8) { 3272 // Use a multiplication with 0x010101... to extend the input to the 3273 // required length. 3274 APInt Magic = SplatByte(NumBits, 0x01); 3275 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3276 } 3277 3278 return Value; 3279} 3280 3281/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3282/// used when a memcpy is turned into a memset when the source is a constant 3283/// string ptr. 3284static SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG, 3285 const TargetLowering &TLI, 3286 std::string &Str, unsigned Offset) { 3287 // Handle vector with all elements zero. 3288 if (Str.empty()) { 3289 if (VT.isInteger()) 3290 return DAG.getConstant(0, VT); 3291 else if (VT == MVT::f32 || VT == MVT::f64) 3292 return DAG.getConstantFP(0.0, VT); 3293 else if (VT.isVector()) { 3294 unsigned NumElts = VT.getVectorNumElements(); 3295 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3296 return DAG.getNode(ISD::BITCAST, dl, VT, 3297 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3298 EltVT, NumElts))); 3299 } else 3300 llvm_unreachable("Expected type!"); 3301 } 3302 3303 assert(!VT.isVector() && "Can't handle vector type here!"); 3304 unsigned NumBits = VT.getSizeInBits(); 3305 unsigned MSB = NumBits / 8; 3306 uint64_t Val = 0; 3307 if (TLI.isLittleEndian()) 3308 Offset = Offset + MSB - 1; 3309 for (unsigned i = 0; i != MSB; ++i) { 3310 Val = (Val << 8) | (unsigned char)Str[Offset]; 3311 Offset += TLI.isLittleEndian() ? -1 : 1; 3312 } 3313 return DAG.getConstant(Val, VT); 3314} 3315 3316/// getMemBasePlusOffset - Returns base and offset node for the 3317/// 3318static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, 3319 SelectionDAG &DAG) { 3320 EVT VT = Base.getValueType(); 3321 return DAG.getNode(ISD::ADD, Base.getDebugLoc(), 3322 VT, Base, DAG.getConstant(Offset, VT)); 3323} 3324 3325/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3326/// 3327static bool isMemSrcFromString(SDValue Src, std::string &Str) { 3328 unsigned SrcDelta = 0; 3329 GlobalAddressSDNode *G = NULL; 3330 if (Src.getOpcode() == ISD::GlobalAddress) 3331 G = cast<GlobalAddressSDNode>(Src); 3332 else if (Src.getOpcode() == ISD::ADD && 3333 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3334 Src.getOperand(1).getOpcode() == ISD::Constant) { 3335 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3336 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3337 } 3338 if (!G) 3339 return false; 3340 3341 const GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal()); 3342 if (GV && GetConstantStringInfo(GV, Str, SrcDelta, false)) 3343 return true; 3344 3345 return false; 3346} 3347 3348/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3349/// to replace the memset / memcpy. Return true if the number of memory ops 3350/// is below the threshold. It returns the types of the sequence of 3351/// memory ops to perform memset / memcpy by reference. 3352static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3353 unsigned Limit, uint64_t Size, 3354 unsigned DstAlign, unsigned SrcAlign, 3355 bool IsZeroVal, 3356 bool MemcpyStrSrc, 3357 SelectionDAG &DAG, 3358 const TargetLowering &TLI) { 3359 assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3360 "Expecting memcpy / memset source to meet alignment requirement!"); 3361 // If 'SrcAlign' is zero, that means the memory operation does not need to 3362 // load the value, i.e. memset or memcpy from constant string. Otherwise, 3363 // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3364 // is the specified alignment of the memory operation. If it is zero, that 3365 // means it's possible to change the alignment of the destination. 3366 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3367 // not need to be loaded. 3368 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3369 IsZeroVal, MemcpyStrSrc, 3370 DAG.getMachineFunction()); 3371 3372 if (VT == MVT::Other) { 3373 if (DstAlign >= TLI.getTargetData()->getPointerPrefAlignment() || 3374 TLI.allowsUnalignedMemoryAccesses(VT)) { 3375 VT = TLI.getPointerTy(); 3376 } else { 3377 switch (DstAlign & 7) { 3378 case 0: VT = MVT::i64; break; 3379 case 4: VT = MVT::i32; break; 3380 case 2: VT = MVT::i16; break; 3381 default: VT = MVT::i8; break; 3382 } 3383 } 3384 3385 MVT LVT = MVT::i64; 3386 while (!TLI.isTypeLegal(LVT)) 3387 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3388 assert(LVT.isInteger()); 3389 3390 if (VT.bitsGT(LVT)) 3391 VT = LVT; 3392 } 3393 3394 unsigned NumMemOps = 0; 3395 while (Size != 0) { 3396 unsigned VTSize = VT.getSizeInBits() / 8; 3397 while (VTSize > Size) { 3398 // For now, only use non-vector load / store's for the left-over pieces. 3399 if (VT.isVector() || VT.isFloatingPoint()) { 3400 VT = MVT::i64; 3401 while (!TLI.isTypeLegal(VT)) 3402 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); 3403 VTSize = VT.getSizeInBits() / 8; 3404 } else { 3405 // This can result in a type that is not legal on the target, e.g. 3406 // 1 or 2 bytes on PPC. 3407 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); 3408 VTSize >>= 1; 3409 } 3410 } 3411 3412 if (++NumMemOps > Limit) 3413 return false; 3414 MemOps.push_back(VT); 3415 Size -= VTSize; 3416 } 3417 3418 return true; 3419} 3420 3421static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, 3422 SDValue Chain, SDValue Dst, 3423 SDValue Src, uint64_t Size, 3424 unsigned Align, bool isVol, 3425 bool AlwaysInline, 3426 MachinePointerInfo DstPtrInfo, 3427 MachinePointerInfo SrcPtrInfo) { 3428 // Turn a memcpy of undef to nop. 3429 if (Src.getOpcode() == ISD::UNDEF) 3430 return Chain; 3431 3432 // Expand memcpy to a series of load and store ops if the size operand falls 3433 // below a certain threshold. 3434 // TODO: In the AlwaysInline case, if the size is big then generate a loop 3435 // rather than maybe a humongous number of loads and stores. 3436 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3437 std::vector<EVT> MemOps; 3438 bool DstAlignCanChange = false; 3439 MachineFunction &MF = DAG.getMachineFunction(); 3440 MachineFrameInfo *MFI = MF.getFrameInfo(); 3441 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3442 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3443 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3444 DstAlignCanChange = true; 3445 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3446 if (Align > SrcAlign) 3447 SrcAlign = Align; 3448 std::string Str; 3449 bool CopyFromStr = isMemSrcFromString(Src, Str); 3450 bool isZeroStr = CopyFromStr && Str.empty(); 3451 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3452 3453 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3454 (DstAlignCanChange ? 0 : Align), 3455 (isZeroStr ? 0 : SrcAlign), 3456 true, CopyFromStr, DAG, TLI)) 3457 return SDValue(); 3458 3459 if (DstAlignCanChange) { 3460 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3461 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3462 if (NewAlign > Align) { 3463 // Give the stack frame object a larger alignment if needed. 3464 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3465 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3466 Align = NewAlign; 3467 } 3468 } 3469 3470 SmallVector<SDValue, 8> OutChains; 3471 unsigned NumMemOps = MemOps.size(); 3472 uint64_t SrcOff = 0, DstOff = 0; 3473 for (unsigned i = 0; i != NumMemOps; ++i) { 3474 EVT VT = MemOps[i]; 3475 unsigned VTSize = VT.getSizeInBits() / 8; 3476 SDValue Value, Store; 3477 3478 if (CopyFromStr && 3479 (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3480 // It's unlikely a store of a vector immediate can be done in a single 3481 // instruction. It would require a load from a constantpool first. 3482 // We only handle zero vectors here. 3483 // FIXME: Handle other cases where store of vector immediate is done in 3484 // a single instruction. 3485 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str, SrcOff); 3486 Store = DAG.getStore(Chain, dl, Value, 3487 getMemBasePlusOffset(Dst, DstOff, DAG), 3488 DstPtrInfo.getWithOffset(DstOff), isVol, 3489 false, Align); 3490 } else { 3491 // The type might not be legal for the target. This should only happen 3492 // if the type is smaller than a legal type, as on PPC, so the right 3493 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3494 // to Load/Store if NVT==VT. 3495 // FIXME does the case above also need this? 3496 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3497 assert(NVT.bitsGE(VT)); 3498 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3499 getMemBasePlusOffset(Src, SrcOff, DAG), 3500 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3501 MinAlign(SrcAlign, SrcOff)); 3502 Store = DAG.getTruncStore(Chain, dl, Value, 3503 getMemBasePlusOffset(Dst, DstOff, DAG), 3504 DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3505 false, Align); 3506 } 3507 OutChains.push_back(Store); 3508 SrcOff += VTSize; 3509 DstOff += VTSize; 3510 } 3511 3512 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3513 &OutChains[0], OutChains.size()); 3514} 3515 3516static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, 3517 SDValue Chain, SDValue Dst, 3518 SDValue Src, uint64_t Size, 3519 unsigned Align, bool isVol, 3520 bool AlwaysInline, 3521 MachinePointerInfo DstPtrInfo, 3522 MachinePointerInfo SrcPtrInfo) { 3523 // Turn a memmove of undef to nop. 3524 if (Src.getOpcode() == ISD::UNDEF) 3525 return Chain; 3526 3527 // Expand memmove to a series of load and store ops if the size operand falls 3528 // below a certain threshold. 3529 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3530 std::vector<EVT> MemOps; 3531 bool DstAlignCanChange = false; 3532 MachineFunction &MF = DAG.getMachineFunction(); 3533 MachineFrameInfo *MFI = MF.getFrameInfo(); 3534 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3535 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3536 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3537 DstAlignCanChange = true; 3538 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3539 if (Align > SrcAlign) 3540 SrcAlign = Align; 3541 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3542 3543 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3544 (DstAlignCanChange ? 0 : Align), 3545 SrcAlign, true, false, DAG, TLI)) 3546 return SDValue(); 3547 3548 if (DstAlignCanChange) { 3549 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3550 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3551 if (NewAlign > Align) { 3552 // Give the stack frame object a larger alignment if needed. 3553 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3554 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3555 Align = NewAlign; 3556 } 3557 } 3558 3559 uint64_t SrcOff = 0, DstOff = 0; 3560 SmallVector<SDValue, 8> LoadValues; 3561 SmallVector<SDValue, 8> LoadChains; 3562 SmallVector<SDValue, 8> OutChains; 3563 unsigned NumMemOps = MemOps.size(); 3564 for (unsigned i = 0; i < NumMemOps; i++) { 3565 EVT VT = MemOps[i]; 3566 unsigned VTSize = VT.getSizeInBits() / 8; 3567 SDValue Value, Store; 3568 3569 Value = DAG.getLoad(VT, dl, Chain, 3570 getMemBasePlusOffset(Src, SrcOff, DAG), 3571 SrcPtrInfo.getWithOffset(SrcOff), isVol, 3572 false, false, SrcAlign); 3573 LoadValues.push_back(Value); 3574 LoadChains.push_back(Value.getValue(1)); 3575 SrcOff += VTSize; 3576 } 3577 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3578 &LoadChains[0], LoadChains.size()); 3579 OutChains.clear(); 3580 for (unsigned i = 0; i < NumMemOps; i++) { 3581 EVT VT = MemOps[i]; 3582 unsigned VTSize = VT.getSizeInBits() / 8; 3583 SDValue Value, Store; 3584 3585 Store = DAG.getStore(Chain, dl, LoadValues[i], 3586 getMemBasePlusOffset(Dst, DstOff, DAG), 3587 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3588 OutChains.push_back(Store); 3589 DstOff += VTSize; 3590 } 3591 3592 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3593 &OutChains[0], OutChains.size()); 3594} 3595 3596static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, 3597 SDValue Chain, SDValue Dst, 3598 SDValue Src, uint64_t Size, 3599 unsigned Align, bool isVol, 3600 MachinePointerInfo DstPtrInfo) { 3601 // Turn a memset of undef to nop. 3602 if (Src.getOpcode() == ISD::UNDEF) 3603 return Chain; 3604 3605 // Expand memset to a series of load/store ops if the size operand 3606 // falls below a certain threshold. 3607 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3608 std::vector<EVT> MemOps; 3609 bool DstAlignCanChange = false; 3610 MachineFunction &MF = DAG.getMachineFunction(); 3611 MachineFrameInfo *MFI = MF.getFrameInfo(); 3612 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3613 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3614 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3615 DstAlignCanChange = true; 3616 bool IsZeroVal = 3617 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3618 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3619 Size, (DstAlignCanChange ? 0 : Align), 0, 3620 IsZeroVal, false, DAG, TLI)) 3621 return SDValue(); 3622 3623 if (DstAlignCanChange) { 3624 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3625 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3626 if (NewAlign > Align) { 3627 // Give the stack frame object a larger alignment if needed. 3628 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3629 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3630 Align = NewAlign; 3631 } 3632 } 3633 3634 SmallVector<SDValue, 8> OutChains; 3635 uint64_t DstOff = 0; 3636 unsigned NumMemOps = MemOps.size(); 3637 3638 // Find the largest store and generate the bit pattern for it. 3639 EVT LargestVT = MemOps[0]; 3640 for (unsigned i = 1; i < NumMemOps; i++) 3641 if (MemOps[i].bitsGT(LargestVT)) 3642 LargestVT = MemOps[i]; 3643 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3644 3645 for (unsigned i = 0; i < NumMemOps; i++) { 3646 EVT VT = MemOps[i]; 3647 3648 // If this store is smaller than the largest store see whether we can get 3649 // the smaller value for free with a truncate. 3650 SDValue Value = MemSetValue; 3651 if (VT.bitsLT(LargestVT)) { 3652 if (!LargestVT.isVector() && !VT.isVector() && 3653 TLI.isTruncateFree(LargestVT, VT)) 3654 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3655 else 3656 Value = getMemsetValue(Src, VT, DAG, dl); 3657 } 3658 assert(Value.getValueType() == VT && "Value with wrong type."); 3659 SDValue Store = DAG.getStore(Chain, dl, Value, 3660 getMemBasePlusOffset(Dst, DstOff, DAG), 3661 DstPtrInfo.getWithOffset(DstOff), 3662 isVol, false, Align); 3663 OutChains.push_back(Store); 3664 DstOff += VT.getSizeInBits() / 8; 3665 } 3666 3667 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3668 &OutChains[0], OutChains.size()); 3669} 3670 3671SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, 3672 SDValue Src, SDValue Size, 3673 unsigned Align, bool isVol, bool AlwaysInline, 3674 MachinePointerInfo DstPtrInfo, 3675 MachinePointerInfo SrcPtrInfo) { 3676 3677 // Check to see if we should lower the memcpy to loads and stores first. 3678 // For cases within the target-specified limits, this is the best choice. 3679 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3680 if (ConstantSize) { 3681 // Memcpy with size zero? Just return the original chain. 3682 if (ConstantSize->isNullValue()) 3683 return Chain; 3684 3685 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3686 ConstantSize->getZExtValue(),Align, 3687 isVol, false, DstPtrInfo, SrcPtrInfo); 3688 if (Result.getNode()) 3689 return Result; 3690 } 3691 3692 // Then check to see if we should lower the memcpy with target-specific 3693 // code. If the target chooses to do this, this is the next best. 3694 SDValue Result = 3695 TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 3696 isVol, AlwaysInline, 3697 DstPtrInfo, SrcPtrInfo); 3698 if (Result.getNode()) 3699 return Result; 3700 3701 // If we really need inline code and the target declined to provide it, 3702 // use a (potentially long) sequence of loads and stores. 3703 if (AlwaysInline) { 3704 assert(ConstantSize && "AlwaysInline requires a constant size!"); 3705 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3706 ConstantSize->getZExtValue(), Align, isVol, 3707 true, DstPtrInfo, SrcPtrInfo); 3708 } 3709 3710 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 3711 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 3712 // respect volatile, so they may do things like read or write memory 3713 // beyond the given memory regions. But fixing this isn't easy, and most 3714 // people don't care. 3715 3716 // Emit a library call. 3717 TargetLowering::ArgListTy Args; 3718 TargetLowering::ArgListEntry Entry; 3719 Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext()); 3720 Entry.Node = Dst; Args.push_back(Entry); 3721 Entry.Node = Src; Args.push_back(Entry); 3722 Entry.Node = Size; Args.push_back(Entry); 3723 // FIXME: pass in DebugLoc 3724 std::pair<SDValue,SDValue> CallResult = 3725 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3726 false, false, false, false, 0, 3727 TLI.getLibcallCallingConv(RTLIB::MEMCPY), false, 3728 /*isReturnValueUsed=*/false, 3729 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMCPY), 3730 TLI.getPointerTy()), 3731 Args, *this, dl); 3732 return CallResult.second; 3733} 3734 3735SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, 3736 SDValue Src, SDValue Size, 3737 unsigned Align, bool isVol, 3738 MachinePointerInfo DstPtrInfo, 3739 MachinePointerInfo SrcPtrInfo) { 3740 3741 // Check to see if we should lower the memmove to loads and stores first. 3742 // For cases within the target-specified limits, this is the best choice. 3743 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3744 if (ConstantSize) { 3745 // Memmove with size zero? Just return the original chain. 3746 if (ConstantSize->isNullValue()) 3747 return Chain; 3748 3749 SDValue Result = 3750 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 3751 ConstantSize->getZExtValue(), Align, isVol, 3752 false, DstPtrInfo, SrcPtrInfo); 3753 if (Result.getNode()) 3754 return Result; 3755 } 3756 3757 // Then check to see if we should lower the memmove with target-specific 3758 // code. If the target chooses to do this, this is the next best. 3759 SDValue Result = 3760 TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 3761 DstPtrInfo, SrcPtrInfo); 3762 if (Result.getNode()) 3763 return Result; 3764 3765 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 3766 // not be safe. See memcpy above for more details. 3767 3768 // Emit a library call. 3769 TargetLowering::ArgListTy Args; 3770 TargetLowering::ArgListEntry Entry; 3771 Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext()); 3772 Entry.Node = Dst; Args.push_back(Entry); 3773 Entry.Node = Src; Args.push_back(Entry); 3774 Entry.Node = Size; Args.push_back(Entry); 3775 // FIXME: pass in DebugLoc 3776 std::pair<SDValue,SDValue> CallResult = 3777 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3778 false, false, false, false, 0, 3779 TLI.getLibcallCallingConv(RTLIB::MEMMOVE), false, 3780 /*isReturnValueUsed=*/false, 3781 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMMOVE), 3782 TLI.getPointerTy()), 3783 Args, *this, dl); 3784 return CallResult.second; 3785} 3786 3787SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, 3788 SDValue Src, SDValue Size, 3789 unsigned Align, bool isVol, 3790 MachinePointerInfo DstPtrInfo) { 3791 3792 // Check to see if we should lower the memset to stores first. 3793 // For cases within the target-specified limits, this is the best choice. 3794 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3795 if (ConstantSize) { 3796 // Memset with size zero? Just return the original chain. 3797 if (ConstantSize->isNullValue()) 3798 return Chain; 3799 3800 SDValue Result = 3801 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 3802 Align, isVol, DstPtrInfo); 3803 3804 if (Result.getNode()) 3805 return Result; 3806 } 3807 3808 // Then check to see if we should lower the memset with target-specific 3809 // code. If the target chooses to do this, this is the next best. 3810 SDValue Result = 3811 TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 3812 DstPtrInfo); 3813 if (Result.getNode()) 3814 return Result; 3815 3816 // Emit a library call. 3817 Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext()); 3818 TargetLowering::ArgListTy Args; 3819 TargetLowering::ArgListEntry Entry; 3820 Entry.Node = Dst; Entry.Ty = IntPtrTy; 3821 Args.push_back(Entry); 3822 // Extend or truncate the argument to be an i32 value for the call. 3823 if (Src.getValueType().bitsGT(MVT::i32)) 3824 Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 3825 else 3826 Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 3827 Entry.Node = Src; 3828 Entry.Ty = Type::getInt32Ty(*getContext()); 3829 Entry.isSExt = true; 3830 Args.push_back(Entry); 3831 Entry.Node = Size; 3832 Entry.Ty = IntPtrTy; 3833 Entry.isSExt = false; 3834 Args.push_back(Entry); 3835 // FIXME: pass in DebugLoc 3836 std::pair<SDValue,SDValue> CallResult = 3837 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3838 false, false, false, false, 0, 3839 TLI.getLibcallCallingConv(RTLIB::MEMSET), false, 3840 /*isReturnValueUsed=*/false, 3841 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET), 3842 TLI.getPointerTy()), 3843 Args, *this, dl); 3844 return CallResult.second; 3845} 3846 3847SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3848 SDValue Chain, SDValue Ptr, SDValue Cmp, 3849 SDValue Swp, MachinePointerInfo PtrInfo, 3850 unsigned Alignment, 3851 AtomicOrdering Ordering, 3852 SynchronizationScope SynchScope) { 3853 if (Alignment == 0) // Ensure that codegen never sees alignment 0 3854 Alignment = getEVTAlignment(MemVT); 3855 3856 MachineFunction &MF = getMachineFunction(); 3857 unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 3858 3859 // For now, atomics are considered to be volatile always. 3860 // FIXME: Volatile isn't really correct; we should keep track of atomic 3861 // orderings in the memoperand. 3862 Flags |= MachineMemOperand::MOVolatile; 3863 3864 MachineMemOperand *MMO = 3865 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 3866 3867 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 3868 Ordering, SynchScope); 3869} 3870 3871SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3872 SDValue Chain, 3873 SDValue Ptr, SDValue Cmp, 3874 SDValue Swp, MachineMemOperand *MMO, 3875 AtomicOrdering Ordering, 3876 SynchronizationScope SynchScope) { 3877 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 3878 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 3879 3880 EVT VT = Cmp.getValueType(); 3881 3882 SDVTList VTs = getVTList(VT, MVT::Other); 3883 FoldingSetNodeID ID; 3884 ID.AddInteger(MemVT.getRawBits()); 3885 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 3886 AddNodeIDNode(ID, Opcode, VTs, Ops, 4); 3887 void* IP = 0; 3888 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 3889 cast<AtomicSDNode>(E)->refineAlignment(MMO); 3890 return SDValue(E, 0); 3891 } 3892 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 3893 Ptr, Cmp, Swp, MMO, Ordering, 3894 SynchScope); 3895 CSEMap.InsertNode(N, IP); 3896 AllNodes.push_back(N); 3897 return SDValue(N, 0); 3898} 3899 3900SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3901 SDValue Chain, 3902 SDValue Ptr, SDValue Val, 3903 const Value* PtrVal, 3904 unsigned Alignment, 3905 AtomicOrdering Ordering, 3906 SynchronizationScope SynchScope) { 3907 if (Alignment == 0) // Ensure that codegen never sees alignment 0 3908 Alignment = getEVTAlignment(MemVT); 3909 3910 MachineFunction &MF = getMachineFunction(); 3911 // A monotonic store does not load; a release store "loads" in the sense 3912 // that other stores cannot be sunk past it. 3913 // (An atomicrmw obviously both loads and stores.) 3914 unsigned Flags = MachineMemOperand::MOStore; 3915 if (Opcode != ISD::ATOMIC_STORE || Ordering > Monotonic) 3916 Flags |= MachineMemOperand::MOLoad; 3917 3918 // For now, atomics are considered to be volatile always. 3919 // FIXME: Volatile isn't really correct; we should keep track of atomic 3920 // orderings in the memoperand. 3921 Flags |= MachineMemOperand::MOVolatile; 3922 3923 MachineMemOperand *MMO = 3924 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 3925 MemVT.getStoreSize(), Alignment); 3926 3927 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 3928 Ordering, SynchScope); 3929} 3930 3931SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3932 SDValue Chain, 3933 SDValue Ptr, SDValue Val, 3934 MachineMemOperand *MMO, 3935 AtomicOrdering Ordering, 3936 SynchronizationScope SynchScope) { 3937 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 3938 Opcode == ISD::ATOMIC_LOAD_SUB || 3939 Opcode == ISD::ATOMIC_LOAD_AND || 3940 Opcode == ISD::ATOMIC_LOAD_OR || 3941 Opcode == ISD::ATOMIC_LOAD_XOR || 3942 Opcode == ISD::ATOMIC_LOAD_NAND || 3943 Opcode == ISD::ATOMIC_LOAD_MIN || 3944 Opcode == ISD::ATOMIC_LOAD_MAX || 3945 Opcode == ISD::ATOMIC_LOAD_UMIN || 3946 Opcode == ISD::ATOMIC_LOAD_UMAX || 3947 Opcode == ISD::ATOMIC_SWAP || 3948 Opcode == ISD::ATOMIC_STORE) && 3949 "Invalid Atomic Op"); 3950 3951 EVT VT = Val.getValueType(); 3952 3953 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 3954 getVTList(VT, MVT::Other); 3955 FoldingSetNodeID ID; 3956 ID.AddInteger(MemVT.getRawBits()); 3957 SDValue Ops[] = {Chain, Ptr, Val}; 3958 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3959 void* IP = 0; 3960 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 3961 cast<AtomicSDNode>(E)->refineAlignment(MMO); 3962 return SDValue(E, 0); 3963 } 3964 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 3965 Ptr, Val, MMO, 3966 Ordering, SynchScope); 3967 CSEMap.InsertNode(N, IP); 3968 AllNodes.push_back(N); 3969 return SDValue(N, 0); 3970} 3971 3972SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3973 EVT VT, SDValue Chain, 3974 SDValue Ptr, 3975 const Value* PtrVal, 3976 unsigned Alignment, 3977 AtomicOrdering Ordering, 3978 SynchronizationScope SynchScope) { 3979 if (Alignment == 0) // Ensure that codegen never sees alignment 0 3980 Alignment = getEVTAlignment(MemVT); 3981 3982 MachineFunction &MF = getMachineFunction(); 3983 // A monotonic load does not store; an acquire load "stores" in the sense 3984 // that other loads cannot be hoisted past it. 3985 unsigned Flags = MachineMemOperand::MOLoad; 3986 if (Ordering > Monotonic) 3987 Flags |= MachineMemOperand::MOStore; 3988 3989 // For now, atomics are considered to be volatile always. 3990 // FIXME: Volatile isn't really correct; we should keep track of atomic 3991 // orderings in the memoperand. 3992 Flags |= MachineMemOperand::MOVolatile; 3993 3994 MachineMemOperand *MMO = 3995 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 3996 MemVT.getStoreSize(), Alignment); 3997 3998 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 3999 Ordering, SynchScope); 4000} 4001 4002SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4003 EVT VT, SDValue Chain, 4004 SDValue Ptr, 4005 MachineMemOperand *MMO, 4006 AtomicOrdering Ordering, 4007 SynchronizationScope SynchScope) { 4008 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4009 4010 SDVTList VTs = getVTList(VT, MVT::Other); 4011 FoldingSetNodeID ID; 4012 ID.AddInteger(MemVT.getRawBits()); 4013 SDValue Ops[] = {Chain, Ptr}; 4014 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 4015 void* IP = 0; 4016 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4017 cast<AtomicSDNode>(E)->refineAlignment(MMO); 4018 return SDValue(E, 0); 4019 } 4020 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 4021 Ptr, MMO, Ordering, SynchScope); 4022 CSEMap.InsertNode(N, IP); 4023 AllNodes.push_back(N); 4024 return SDValue(N, 0); 4025} 4026 4027/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4028SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4029 DebugLoc dl) { 4030 if (NumOps == 1) 4031 return Ops[0]; 4032 4033 SmallVector<EVT, 4> VTs; 4034 VTs.reserve(NumOps); 4035 for (unsigned i = 0; i < NumOps; ++i) 4036 VTs.push_back(Ops[i].getValueType()); 4037 return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4038 Ops, NumOps); 4039} 4040 4041SDValue 4042SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, 4043 const EVT *VTs, unsigned NumVTs, 4044 const SDValue *Ops, unsigned NumOps, 4045 EVT MemVT, MachinePointerInfo PtrInfo, 4046 unsigned Align, bool Vol, 4047 bool ReadMem, bool WriteMem) { 4048 return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4049 MemVT, PtrInfo, Align, Vol, 4050 ReadMem, WriteMem); 4051} 4052 4053SDValue 4054SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, 4055 const SDValue *Ops, unsigned NumOps, 4056 EVT MemVT, MachinePointerInfo PtrInfo, 4057 unsigned Align, bool Vol, 4058 bool ReadMem, bool WriteMem) { 4059 if (Align == 0) // Ensure that codegen never sees alignment 0 4060 Align = getEVTAlignment(MemVT); 4061 4062 MachineFunction &MF = getMachineFunction(); 4063 unsigned Flags = 0; 4064 if (WriteMem) 4065 Flags |= MachineMemOperand::MOStore; 4066 if (ReadMem) 4067 Flags |= MachineMemOperand::MOLoad; 4068 if (Vol) 4069 Flags |= MachineMemOperand::MOVolatile; 4070 MachineMemOperand *MMO = 4071 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4072 4073 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4074} 4075 4076SDValue 4077SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, 4078 const SDValue *Ops, unsigned NumOps, 4079 EVT MemVT, MachineMemOperand *MMO) { 4080 assert((Opcode == ISD::INTRINSIC_VOID || 4081 Opcode == ISD::INTRINSIC_W_CHAIN || 4082 Opcode == ISD::PREFETCH || 4083 (Opcode <= INT_MAX && 4084 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4085 "Opcode is not a memory-accessing opcode!"); 4086 4087 // Memoize the node unless it returns a flag. 4088 MemIntrinsicSDNode *N; 4089 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4090 FoldingSetNodeID ID; 4091 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4092 void *IP = 0; 4093 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4094 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4095 return SDValue(E, 0); 4096 } 4097 4098 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, 4099 MemVT, MMO); 4100 CSEMap.InsertNode(N, IP); 4101 } else { 4102 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, 4103 MemVT, MMO); 4104 } 4105 AllNodes.push_back(N); 4106 return SDValue(N, 0); 4107} 4108 4109/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4110/// MachinePointerInfo record from it. This is particularly useful because the 4111/// code generator has many cases where it doesn't bother passing in a 4112/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4113static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4114 // If this is FI+Offset, we can model it. 4115 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4116 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4117 4118 // If this is (FI+Offset1)+Offset2, we can model it. 4119 if (Ptr.getOpcode() != ISD::ADD || 4120 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4121 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4122 return MachinePointerInfo(); 4123 4124 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4125 return MachinePointerInfo::getFixedStack(FI, Offset+ 4126 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4127} 4128 4129/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4130/// MachinePointerInfo record from it. This is particularly useful because the 4131/// code generator has many cases where it doesn't bother passing in a 4132/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4133static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4134 // If the 'Offset' value isn't a constant, we can't handle this. 4135 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4136 return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4137 if (OffsetOp.getOpcode() == ISD::UNDEF) 4138 return InferPointerInfo(Ptr); 4139 return MachinePointerInfo(); 4140} 4141 4142 4143SDValue 4144SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4145 EVT VT, DebugLoc dl, SDValue Chain, 4146 SDValue Ptr, SDValue Offset, 4147 MachinePointerInfo PtrInfo, EVT MemVT, 4148 bool isVolatile, bool isNonTemporal, bool isInvariant, 4149 unsigned Alignment, const MDNode *TBAAInfo) { 4150 assert(Chain.getValueType() == MVT::Other && 4151 "Invalid chain type"); 4152 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4153 Alignment = getEVTAlignment(VT); 4154 4155 unsigned Flags = MachineMemOperand::MOLoad; 4156 if (isVolatile) 4157 Flags |= MachineMemOperand::MOVolatile; 4158 if (isNonTemporal) 4159 Flags |= MachineMemOperand::MONonTemporal; 4160 if (isInvariant) 4161 Flags |= MachineMemOperand::MOInvariant; 4162 4163 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4164 // clients. 4165 if (PtrInfo.V == 0) 4166 PtrInfo = InferPointerInfo(Ptr, Offset); 4167 4168 MachineFunction &MF = getMachineFunction(); 4169 MachineMemOperand *MMO = 4170 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4171 TBAAInfo); 4172 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4173} 4174 4175SDValue 4176SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4177 EVT VT, DebugLoc dl, SDValue Chain, 4178 SDValue Ptr, SDValue Offset, EVT MemVT, 4179 MachineMemOperand *MMO) { 4180 if (VT == MemVT) { 4181 ExtType = ISD::NON_EXTLOAD; 4182 } else if (ExtType == ISD::NON_EXTLOAD) { 4183 assert(VT == MemVT && "Non-extending load from different memory type!"); 4184 } else { 4185 // Extending load. 4186 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4187 "Should only be an extending load, not truncating!"); 4188 assert(VT.isInteger() == MemVT.isInteger() && 4189 "Cannot convert from FP to Int or Int -> FP!"); 4190 assert(VT.isVector() == MemVT.isVector() && 4191 "Cannot use trunc store to convert to or from a vector!"); 4192 assert((!VT.isVector() || 4193 VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4194 "Cannot use trunc store to change the number of vector elements!"); 4195 } 4196 4197 bool Indexed = AM != ISD::UNINDEXED; 4198 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4199 "Unindexed load with an offset!"); 4200 4201 SDVTList VTs = Indexed ? 4202 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4203 SDValue Ops[] = { Chain, Ptr, Offset }; 4204 FoldingSetNodeID ID; 4205 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4206 ID.AddInteger(MemVT.getRawBits()); 4207 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4208 MMO->isNonTemporal(), 4209 MMO->isInvariant())); 4210 void *IP = 0; 4211 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4212 cast<LoadSDNode>(E)->refineAlignment(MMO); 4213 return SDValue(E, 0); 4214 } 4215 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl, VTs, AM, ExtType, 4216 MemVT, MMO); 4217 CSEMap.InsertNode(N, IP); 4218 AllNodes.push_back(N); 4219 return SDValue(N, 0); 4220} 4221 4222SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, 4223 SDValue Chain, SDValue Ptr, 4224 MachinePointerInfo PtrInfo, 4225 bool isVolatile, bool isNonTemporal, 4226 bool isInvariant, unsigned Alignment, 4227 const MDNode *TBAAInfo) { 4228 SDValue Undef = getUNDEF(Ptr.getValueType()); 4229 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4230 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4231 TBAAInfo); 4232} 4233 4234SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, 4235 SDValue Chain, SDValue Ptr, 4236 MachinePointerInfo PtrInfo, EVT MemVT, 4237 bool isVolatile, bool isNonTemporal, 4238 unsigned Alignment, const MDNode *TBAAInfo) { 4239 SDValue Undef = getUNDEF(Ptr.getValueType()); 4240 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4241 PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4242 TBAAInfo); 4243} 4244 4245 4246SDValue 4247SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, 4248 SDValue Offset, ISD::MemIndexedMode AM) { 4249 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4250 assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4251 "Load is already a indexed load!"); 4252 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4253 LD->getChain(), Base, Offset, LD->getPointerInfo(), 4254 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4255 false, LD->getAlignment()); 4256} 4257 4258SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, 4259 SDValue Ptr, MachinePointerInfo PtrInfo, 4260 bool isVolatile, bool isNonTemporal, 4261 unsigned Alignment, const MDNode *TBAAInfo) { 4262 assert(Chain.getValueType() == MVT::Other && 4263 "Invalid chain type"); 4264 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4265 Alignment = getEVTAlignment(Val.getValueType()); 4266 4267 unsigned Flags = MachineMemOperand::MOStore; 4268 if (isVolatile) 4269 Flags |= MachineMemOperand::MOVolatile; 4270 if (isNonTemporal) 4271 Flags |= MachineMemOperand::MONonTemporal; 4272 4273 if (PtrInfo.V == 0) 4274 PtrInfo = InferPointerInfo(Ptr); 4275 4276 MachineFunction &MF = getMachineFunction(); 4277 MachineMemOperand *MMO = 4278 MF.getMachineMemOperand(PtrInfo, Flags, 4279 Val.getValueType().getStoreSize(), Alignment, 4280 TBAAInfo); 4281 4282 return getStore(Chain, dl, Val, Ptr, MMO); 4283} 4284 4285SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, 4286 SDValue Ptr, MachineMemOperand *MMO) { 4287 assert(Chain.getValueType() == MVT::Other && 4288 "Invalid chain type"); 4289 EVT VT = Val.getValueType(); 4290 SDVTList VTs = getVTList(MVT::Other); 4291 SDValue Undef = getUNDEF(Ptr.getValueType()); 4292 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4293 FoldingSetNodeID ID; 4294 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4295 ID.AddInteger(VT.getRawBits()); 4296 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4297 MMO->isNonTemporal(), MMO->isInvariant())); 4298 void *IP = 0; 4299 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4300 cast<StoreSDNode>(E)->refineAlignment(MMO); 4301 return SDValue(E, 0); 4302 } 4303 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, 4304 false, VT, MMO); 4305 CSEMap.InsertNode(N, IP); 4306 AllNodes.push_back(N); 4307 return SDValue(N, 0); 4308} 4309 4310SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, 4311 SDValue Ptr, MachinePointerInfo PtrInfo, 4312 EVT SVT,bool isVolatile, bool isNonTemporal, 4313 unsigned Alignment, 4314 const MDNode *TBAAInfo) { 4315 assert(Chain.getValueType() == MVT::Other && 4316 "Invalid chain type"); 4317 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4318 Alignment = getEVTAlignment(SVT); 4319 4320 unsigned Flags = MachineMemOperand::MOStore; 4321 if (isVolatile) 4322 Flags |= MachineMemOperand::MOVolatile; 4323 if (isNonTemporal) 4324 Flags |= MachineMemOperand::MONonTemporal; 4325 4326 if (PtrInfo.V == 0) 4327 PtrInfo = InferPointerInfo(Ptr); 4328 4329 MachineFunction &MF = getMachineFunction(); 4330 MachineMemOperand *MMO = 4331 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4332 TBAAInfo); 4333 4334 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4335} 4336 4337SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, 4338 SDValue Ptr, EVT SVT, 4339 MachineMemOperand *MMO) { 4340 EVT VT = Val.getValueType(); 4341 4342 assert(Chain.getValueType() == MVT::Other && 4343 "Invalid chain type"); 4344 if (VT == SVT) 4345 return getStore(Chain, dl, Val, Ptr, MMO); 4346 4347 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4348 "Should only be a truncating store, not extending!"); 4349 assert(VT.isInteger() == SVT.isInteger() && 4350 "Can't do FP-INT conversion!"); 4351 assert(VT.isVector() == SVT.isVector() && 4352 "Cannot use trunc store to convert to or from a vector!"); 4353 assert((!VT.isVector() || 4354 VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4355 "Cannot use trunc store to change the number of vector elements!"); 4356 4357 SDVTList VTs = getVTList(MVT::Other); 4358 SDValue Undef = getUNDEF(Ptr.getValueType()); 4359 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4360 FoldingSetNodeID ID; 4361 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4362 ID.AddInteger(SVT.getRawBits()); 4363 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4364 MMO->isNonTemporal(), MMO->isInvariant())); 4365 void *IP = 0; 4366 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4367 cast<StoreSDNode>(E)->refineAlignment(MMO); 4368 return SDValue(E, 0); 4369 } 4370 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, 4371 true, SVT, MMO); 4372 CSEMap.InsertNode(N, IP); 4373 AllNodes.push_back(N); 4374 return SDValue(N, 0); 4375} 4376 4377SDValue 4378SelectionDAG::getIndexedStore(SDValue OrigStore, DebugLoc dl, SDValue Base, 4379 SDValue Offset, ISD::MemIndexedMode AM) { 4380 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4381 assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4382 "Store is already a indexed store!"); 4383 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4384 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4385 FoldingSetNodeID ID; 4386 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4387 ID.AddInteger(ST->getMemoryVT().getRawBits()); 4388 ID.AddInteger(ST->getRawSubclassData()); 4389 void *IP = 0; 4390 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4391 return SDValue(E, 0); 4392 4393 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, AM, 4394 ST->isTruncatingStore(), 4395 ST->getMemoryVT(), 4396 ST->getMemOperand()); 4397 CSEMap.InsertNode(N, IP); 4398 AllNodes.push_back(N); 4399 return SDValue(N, 0); 4400} 4401 4402SDValue SelectionDAG::getVAArg(EVT VT, DebugLoc dl, 4403 SDValue Chain, SDValue Ptr, 4404 SDValue SV, 4405 unsigned Align) { 4406 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4407 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4408} 4409 4410SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 4411 const SDUse *Ops, unsigned NumOps) { 4412 switch (NumOps) { 4413 case 0: return getNode(Opcode, DL, VT); 4414 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4415 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4416 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4417 default: break; 4418 } 4419 4420 // Copy from an SDUse array into an SDValue array for use with 4421 // the regular getNode logic. 4422 SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4423 return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4424} 4425 4426SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 4427 const SDValue *Ops, unsigned NumOps) { 4428 switch (NumOps) { 4429 case 0: return getNode(Opcode, DL, VT); 4430 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4431 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4432 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4433 default: break; 4434 } 4435 4436 switch (Opcode) { 4437 default: break; 4438 case ISD::SELECT_CC: { 4439 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4440 assert(Ops[0].getValueType() == Ops[1].getValueType() && 4441 "LHS and RHS of condition must have same type!"); 4442 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4443 "True and False arms of SelectCC must have same type!"); 4444 assert(Ops[2].getValueType() == VT && 4445 "select_cc node must be of same type as true and false value!"); 4446 break; 4447 } 4448 case ISD::BR_CC: { 4449 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4450 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4451 "LHS/RHS of comparison should match types!"); 4452 break; 4453 } 4454 } 4455 4456 // Memoize nodes. 4457 SDNode *N; 4458 SDVTList VTs = getVTList(VT); 4459 4460 if (VT != MVT::Glue) { 4461 FoldingSetNodeID ID; 4462 AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4463 void *IP = 0; 4464 4465 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4466 return SDValue(E, 0); 4467 4468 N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); 4469 CSEMap.InsertNode(N, IP); 4470 } else { 4471 N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); 4472 } 4473 4474 AllNodes.push_back(N); 4475#ifndef NDEBUG 4476 VerifySDNode(N); 4477#endif 4478 return SDValue(N, 0); 4479} 4480 4481SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 4482 const std::vector<EVT> &ResultTys, 4483 const SDValue *Ops, unsigned NumOps) { 4484 return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4485 Ops, NumOps); 4486} 4487 4488SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 4489 const EVT *VTs, unsigned NumVTs, 4490 const SDValue *Ops, unsigned NumOps) { 4491 if (NumVTs == 1) 4492 return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4493 return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4494} 4495 4496SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4497 const SDValue *Ops, unsigned NumOps) { 4498 if (VTList.NumVTs == 1) 4499 return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4500 4501#if 0 4502 switch (Opcode) { 4503 // FIXME: figure out how to safely handle things like 4504 // int foo(int x) { return 1 << (x & 255); } 4505 // int bar() { return foo(256); } 4506 case ISD::SRA_PARTS: 4507 case ISD::SRL_PARTS: 4508 case ISD::SHL_PARTS: 4509 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4510 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4511 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4512 else if (N3.getOpcode() == ISD::AND) 4513 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4514 // If the and is only masking out bits that cannot effect the shift, 4515 // eliminate the and. 4516 unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4517 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4518 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4519 } 4520 break; 4521 } 4522#endif 4523 4524 // Memoize the node unless it returns a flag. 4525 SDNode *N; 4526 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4527 FoldingSetNodeID ID; 4528 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4529 void *IP = 0; 4530 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4531 return SDValue(E, 0); 4532 4533 if (NumOps == 1) { 4534 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); 4535 } else if (NumOps == 2) { 4536 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); 4537 } else if (NumOps == 3) { 4538 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], 4539 Ops[2]); 4540 } else { 4541 N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); 4542 } 4543 CSEMap.InsertNode(N, IP); 4544 } else { 4545 if (NumOps == 1) { 4546 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); 4547 } else if (NumOps == 2) { 4548 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); 4549 } else if (NumOps == 3) { 4550 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], 4551 Ops[2]); 4552 } else { 4553 N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); 4554 } 4555 } 4556 AllNodes.push_back(N); 4557#ifndef NDEBUG 4558 VerifySDNode(N); 4559#endif 4560 return SDValue(N, 0); 4561} 4562 4563SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList) { 4564 return getNode(Opcode, DL, VTList, 0, 0); 4565} 4566 4567SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4568 SDValue N1) { 4569 SDValue Ops[] = { N1 }; 4570 return getNode(Opcode, DL, VTList, Ops, 1); 4571} 4572 4573SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4574 SDValue N1, SDValue N2) { 4575 SDValue Ops[] = { N1, N2 }; 4576 return getNode(Opcode, DL, VTList, Ops, 2); 4577} 4578 4579SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4580 SDValue N1, SDValue N2, SDValue N3) { 4581 SDValue Ops[] = { N1, N2, N3 }; 4582 return getNode(Opcode, DL, VTList, Ops, 3); 4583} 4584 4585SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4586 SDValue N1, SDValue N2, SDValue N3, 4587 SDValue N4) { 4588 SDValue Ops[] = { N1, N2, N3, N4 }; 4589 return getNode(Opcode, DL, VTList, Ops, 4); 4590} 4591 4592SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4593 SDValue N1, SDValue N2, SDValue N3, 4594 SDValue N4, SDValue N5) { 4595 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4596 return getNode(Opcode, DL, VTList, Ops, 5); 4597} 4598 4599SDVTList SelectionDAG::getVTList(EVT VT) { 4600 return makeVTList(SDNode::getValueTypeList(VT), 1); 4601} 4602 4603SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4604 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4605 E = VTList.rend(); I != E; ++I) 4606 if (I->NumVTs == 2 && I->VTs[0] == VT1 && I->VTs[1] == VT2) 4607 return *I; 4608 4609 EVT *Array = Allocator.Allocate<EVT>(2); 4610 Array[0] = VT1; 4611 Array[1] = VT2; 4612 SDVTList Result = makeVTList(Array, 2); 4613 VTList.push_back(Result); 4614 return Result; 4615} 4616 4617SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 4618 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4619 E = VTList.rend(); I != E; ++I) 4620 if (I->NumVTs == 3 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 4621 I->VTs[2] == VT3) 4622 return *I; 4623 4624 EVT *Array = Allocator.Allocate<EVT>(3); 4625 Array[0] = VT1; 4626 Array[1] = VT2; 4627 Array[2] = VT3; 4628 SDVTList Result = makeVTList(Array, 3); 4629 VTList.push_back(Result); 4630 return Result; 4631} 4632 4633SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 4634 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4635 E = VTList.rend(); I != E; ++I) 4636 if (I->NumVTs == 4 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 4637 I->VTs[2] == VT3 && I->VTs[3] == VT4) 4638 return *I; 4639 4640 EVT *Array = Allocator.Allocate<EVT>(4); 4641 Array[0] = VT1; 4642 Array[1] = VT2; 4643 Array[2] = VT3; 4644 Array[3] = VT4; 4645 SDVTList Result = makeVTList(Array, 4); 4646 VTList.push_back(Result); 4647 return Result; 4648} 4649 4650SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 4651 switch (NumVTs) { 4652 case 0: llvm_unreachable("Cannot have nodes without results!"); 4653 case 1: return getVTList(VTs[0]); 4654 case 2: return getVTList(VTs[0], VTs[1]); 4655 case 3: return getVTList(VTs[0], VTs[1], VTs[2]); 4656 case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]); 4657 default: break; 4658 } 4659 4660 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4661 E = VTList.rend(); I != E; ++I) { 4662 if (I->NumVTs != NumVTs || VTs[0] != I->VTs[0] || VTs[1] != I->VTs[1]) 4663 continue; 4664 4665 bool NoMatch = false; 4666 for (unsigned i = 2; i != NumVTs; ++i) 4667 if (VTs[i] != I->VTs[i]) { 4668 NoMatch = true; 4669 break; 4670 } 4671 if (!NoMatch) 4672 return *I; 4673 } 4674 4675 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 4676 std::copy(VTs, VTs+NumVTs, Array); 4677 SDVTList Result = makeVTList(Array, NumVTs); 4678 VTList.push_back(Result); 4679 return Result; 4680} 4681 4682 4683/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 4684/// specified operands. If the resultant node already exists in the DAG, 4685/// this does not modify the specified node, instead it returns the node that 4686/// already exists. If the resultant node does not exist in the DAG, the 4687/// input node is returned. As a degenerate case, if you specify the same 4688/// input operands as the node already has, the input node is returned. 4689SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 4690 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 4691 4692 // Check to see if there is no change. 4693 if (Op == N->getOperand(0)) return N; 4694 4695 // See if the modified node already exists. 4696 void *InsertPos = 0; 4697 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 4698 return Existing; 4699 4700 // Nope it doesn't. Remove the node from its current place in the maps. 4701 if (InsertPos) 4702 if (!RemoveNodeFromCSEMaps(N)) 4703 InsertPos = 0; 4704 4705 // Now we update the operands. 4706 N->OperandList[0].set(Op); 4707 4708 // If this gets put into a CSE map, add it. 4709 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4710 return N; 4711} 4712 4713SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 4714 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 4715 4716 // Check to see if there is no change. 4717 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 4718 return N; // No operands changed, just return the input node. 4719 4720 // See if the modified node already exists. 4721 void *InsertPos = 0; 4722 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 4723 return Existing; 4724 4725 // Nope it doesn't. Remove the node from its current place in the maps. 4726 if (InsertPos) 4727 if (!RemoveNodeFromCSEMaps(N)) 4728 InsertPos = 0; 4729 4730 // Now we update the operands. 4731 if (N->OperandList[0] != Op1) 4732 N->OperandList[0].set(Op1); 4733 if (N->OperandList[1] != Op2) 4734 N->OperandList[1].set(Op2); 4735 4736 // If this gets put into a CSE map, add it. 4737 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4738 return N; 4739} 4740 4741SDNode *SelectionDAG:: 4742UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 4743 SDValue Ops[] = { Op1, Op2, Op3 }; 4744 return UpdateNodeOperands(N, Ops, 3); 4745} 4746 4747SDNode *SelectionDAG:: 4748UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 4749 SDValue Op3, SDValue Op4) { 4750 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 4751 return UpdateNodeOperands(N, Ops, 4); 4752} 4753 4754SDNode *SelectionDAG:: 4755UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 4756 SDValue Op3, SDValue Op4, SDValue Op5) { 4757 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 4758 return UpdateNodeOperands(N, Ops, 5); 4759} 4760 4761SDNode *SelectionDAG:: 4762UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 4763 assert(N->getNumOperands() == NumOps && 4764 "Update with wrong number of operands"); 4765 4766 // Check to see if there is no change. 4767 bool AnyChange = false; 4768 for (unsigned i = 0; i != NumOps; ++i) { 4769 if (Ops[i] != N->getOperand(i)) { 4770 AnyChange = true; 4771 break; 4772 } 4773 } 4774 4775 // No operands changed, just return the input node. 4776 if (!AnyChange) return N; 4777 4778 // See if the modified node already exists. 4779 void *InsertPos = 0; 4780 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 4781 return Existing; 4782 4783 // Nope it doesn't. Remove the node from its current place in the maps. 4784 if (InsertPos) 4785 if (!RemoveNodeFromCSEMaps(N)) 4786 InsertPos = 0; 4787 4788 // Now we update the operands. 4789 for (unsigned i = 0; i != NumOps; ++i) 4790 if (N->OperandList[i] != Ops[i]) 4791 N->OperandList[i].set(Ops[i]); 4792 4793 // If this gets put into a CSE map, add it. 4794 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4795 return N; 4796} 4797 4798/// DropOperands - Release the operands and set this node to have 4799/// zero operands. 4800void SDNode::DropOperands() { 4801 // Unlike the code in MorphNodeTo that does this, we don't need to 4802 // watch for dead nodes here. 4803 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 4804 SDUse &Use = *I++; 4805 Use.set(SDValue()); 4806 } 4807} 4808 4809/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 4810/// machine opcode. 4811/// 4812SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4813 EVT VT) { 4814 SDVTList VTs = getVTList(VT); 4815 return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 4816} 4817 4818SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4819 EVT VT, SDValue Op1) { 4820 SDVTList VTs = getVTList(VT); 4821 SDValue Ops[] = { Op1 }; 4822 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 4823} 4824 4825SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4826 EVT VT, SDValue Op1, 4827 SDValue Op2) { 4828 SDVTList VTs = getVTList(VT); 4829 SDValue Ops[] = { Op1, Op2 }; 4830 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 4831} 4832 4833SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4834 EVT VT, SDValue Op1, 4835 SDValue Op2, SDValue Op3) { 4836 SDVTList VTs = getVTList(VT); 4837 SDValue Ops[] = { Op1, Op2, Op3 }; 4838 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4839} 4840 4841SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4842 EVT VT, const SDValue *Ops, 4843 unsigned NumOps) { 4844 SDVTList VTs = getVTList(VT); 4845 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4846} 4847 4848SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4849 EVT VT1, EVT VT2, const SDValue *Ops, 4850 unsigned NumOps) { 4851 SDVTList VTs = getVTList(VT1, VT2); 4852 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4853} 4854 4855SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4856 EVT VT1, EVT VT2) { 4857 SDVTList VTs = getVTList(VT1, VT2); 4858 return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 4859} 4860 4861SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4862 EVT VT1, EVT VT2, EVT VT3, 4863 const SDValue *Ops, unsigned NumOps) { 4864 SDVTList VTs = getVTList(VT1, VT2, VT3); 4865 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4866} 4867 4868SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4869 EVT VT1, EVT VT2, EVT VT3, EVT VT4, 4870 const SDValue *Ops, unsigned NumOps) { 4871 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 4872 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4873} 4874 4875SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4876 EVT VT1, EVT VT2, 4877 SDValue Op1) { 4878 SDVTList VTs = getVTList(VT1, VT2); 4879 SDValue Ops[] = { Op1 }; 4880 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 4881} 4882 4883SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4884 EVT VT1, EVT VT2, 4885 SDValue Op1, SDValue Op2) { 4886 SDVTList VTs = getVTList(VT1, VT2); 4887 SDValue Ops[] = { Op1, Op2 }; 4888 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 4889} 4890 4891SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4892 EVT VT1, EVT VT2, 4893 SDValue Op1, SDValue Op2, 4894 SDValue Op3) { 4895 SDVTList VTs = getVTList(VT1, VT2); 4896 SDValue Ops[] = { Op1, Op2, Op3 }; 4897 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4898} 4899 4900SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4901 EVT VT1, EVT VT2, EVT VT3, 4902 SDValue Op1, SDValue Op2, 4903 SDValue Op3) { 4904 SDVTList VTs = getVTList(VT1, VT2, VT3); 4905 SDValue Ops[] = { Op1, Op2, Op3 }; 4906 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4907} 4908 4909SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4910 SDVTList VTs, const SDValue *Ops, 4911 unsigned NumOps) { 4912 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 4913 // Reset the NodeID to -1. 4914 N->setNodeId(-1); 4915 return N; 4916} 4917 4918/// MorphNodeTo - This *mutates* the specified node to have the specified 4919/// return type, opcode, and operands. 4920/// 4921/// Note that MorphNodeTo returns the resultant node. If there is already a 4922/// node of the specified opcode and operands, it returns that node instead of 4923/// the current one. Note that the DebugLoc need not be the same. 4924/// 4925/// Using MorphNodeTo is faster than creating a new node and swapping it in 4926/// with ReplaceAllUsesWith both because it often avoids allocating a new 4927/// node, and because it doesn't require CSE recalculation for any of 4928/// the node's users. 4929/// 4930SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 4931 SDVTList VTs, const SDValue *Ops, 4932 unsigned NumOps) { 4933 // If an identical node already exists, use it. 4934 void *IP = 0; 4935 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 4936 FoldingSetNodeID ID; 4937 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 4938 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 4939 return ON; 4940 } 4941 4942 if (!RemoveNodeFromCSEMaps(N)) 4943 IP = 0; 4944 4945 // Start the morphing. 4946 N->NodeType = Opc; 4947 N->ValueList = VTs.VTs; 4948 N->NumValues = VTs.NumVTs; 4949 4950 // Clear the operands list, updating used nodes to remove this from their 4951 // use list. Keep track of any operands that become dead as a result. 4952 SmallPtrSet<SDNode*, 16> DeadNodeSet; 4953 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 4954 SDUse &Use = *I++; 4955 SDNode *Used = Use.getNode(); 4956 Use.set(SDValue()); 4957 if (Used->use_empty()) 4958 DeadNodeSet.insert(Used); 4959 } 4960 4961 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 4962 // Initialize the memory references information. 4963 MN->setMemRefs(0, 0); 4964 // If NumOps is larger than the # of operands we can have in a 4965 // MachineSDNode, reallocate the operand list. 4966 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 4967 if (MN->OperandsNeedDelete) 4968 delete[] MN->OperandList; 4969 if (NumOps > array_lengthof(MN->LocalOperands)) 4970 // We're creating a final node that will live unmorphed for the 4971 // remainder of the current SelectionDAG iteration, so we can allocate 4972 // the operands directly out of a pool with no recycling metadata. 4973 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 4974 Ops, NumOps); 4975 else 4976 MN->InitOperands(MN->LocalOperands, Ops, NumOps); 4977 MN->OperandsNeedDelete = false; 4978 } else 4979 MN->InitOperands(MN->OperandList, Ops, NumOps); 4980 } else { 4981 // If NumOps is larger than the # of operands we currently have, reallocate 4982 // the operand list. 4983 if (NumOps > N->NumOperands) { 4984 if (N->OperandsNeedDelete) 4985 delete[] N->OperandList; 4986 N->InitOperands(new SDUse[NumOps], Ops, NumOps); 4987 N->OperandsNeedDelete = true; 4988 } else 4989 N->InitOperands(N->OperandList, Ops, NumOps); 4990 } 4991 4992 // Delete any nodes that are still dead after adding the uses for the 4993 // new operands. 4994 if (!DeadNodeSet.empty()) { 4995 SmallVector<SDNode *, 16> DeadNodes; 4996 for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 4997 E = DeadNodeSet.end(); I != E; ++I) 4998 if ((*I)->use_empty()) 4999 DeadNodes.push_back(*I); 5000 RemoveDeadNodes(DeadNodes); 5001 } 5002 5003 if (IP) 5004 CSEMap.InsertNode(N, IP); // Memoize the new node. 5005 return N; 5006} 5007 5008 5009/// getMachineNode - These are used for target selectors to create a new node 5010/// with specified return type(s), MachineInstr opcode, and operands. 5011/// 5012/// Note that getMachineNode returns the resultant node. If there is already a 5013/// node of the specified opcode and operands, it returns that node instead of 5014/// the current one. 5015MachineSDNode * 5016SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT) { 5017 SDVTList VTs = getVTList(VT); 5018 return getMachineNode(Opcode, dl, VTs, 0, 0); 5019} 5020 5021MachineSDNode * 5022SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, SDValue Op1) { 5023 SDVTList VTs = getVTList(VT); 5024 SDValue Ops[] = { Op1 }; 5025 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5026} 5027 5028MachineSDNode * 5029SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5030 SDValue Op1, SDValue Op2) { 5031 SDVTList VTs = getVTList(VT); 5032 SDValue Ops[] = { Op1, Op2 }; 5033 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5034} 5035 5036MachineSDNode * 5037SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5038 SDValue Op1, SDValue Op2, SDValue Op3) { 5039 SDVTList VTs = getVTList(VT); 5040 SDValue Ops[] = { Op1, Op2, Op3 }; 5041 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5042} 5043 5044MachineSDNode * 5045SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5046 const SDValue *Ops, unsigned NumOps) { 5047 SDVTList VTs = getVTList(VT); 5048 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5049} 5050 5051MachineSDNode * 5052SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, EVT VT2) { 5053 SDVTList VTs = getVTList(VT1, VT2); 5054 return getMachineNode(Opcode, dl, VTs, 0, 0); 5055} 5056 5057MachineSDNode * 5058SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5059 EVT VT1, EVT VT2, SDValue Op1) { 5060 SDVTList VTs = getVTList(VT1, VT2); 5061 SDValue Ops[] = { Op1 }; 5062 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5063} 5064 5065MachineSDNode * 5066SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5067 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5068 SDVTList VTs = getVTList(VT1, VT2); 5069 SDValue Ops[] = { Op1, Op2 }; 5070 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5071} 5072 5073MachineSDNode * 5074SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5075 EVT VT1, EVT VT2, SDValue Op1, 5076 SDValue Op2, SDValue Op3) { 5077 SDVTList VTs = getVTList(VT1, VT2); 5078 SDValue Ops[] = { Op1, Op2, Op3 }; 5079 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5080} 5081 5082MachineSDNode * 5083SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5084 EVT VT1, EVT VT2, 5085 const SDValue *Ops, unsigned NumOps) { 5086 SDVTList VTs = getVTList(VT1, VT2); 5087 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5088} 5089 5090MachineSDNode * 5091SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5092 EVT VT1, EVT VT2, EVT VT3, 5093 SDValue Op1, SDValue Op2) { 5094 SDVTList VTs = getVTList(VT1, VT2, VT3); 5095 SDValue Ops[] = { Op1, Op2 }; 5096 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5097} 5098 5099MachineSDNode * 5100SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5101 EVT VT1, EVT VT2, EVT VT3, 5102 SDValue Op1, SDValue Op2, SDValue Op3) { 5103 SDVTList VTs = getVTList(VT1, VT2, VT3); 5104 SDValue Ops[] = { Op1, Op2, Op3 }; 5105 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5106} 5107 5108MachineSDNode * 5109SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5110 EVT VT1, EVT VT2, EVT VT3, 5111 const SDValue *Ops, unsigned NumOps) { 5112 SDVTList VTs = getVTList(VT1, VT2, VT3); 5113 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5114} 5115 5116MachineSDNode * 5117SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, 5118 EVT VT2, EVT VT3, EVT VT4, 5119 const SDValue *Ops, unsigned NumOps) { 5120 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5121 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5122} 5123 5124MachineSDNode * 5125SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5126 const std::vector<EVT> &ResultTys, 5127 const SDValue *Ops, unsigned NumOps) { 5128 SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5129 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5130} 5131 5132MachineSDNode * 5133SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, 5134 const SDValue *Ops, unsigned NumOps) { 5135 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5136 MachineSDNode *N; 5137 void *IP = 0; 5138 5139 if (DoCSE) { 5140 FoldingSetNodeID ID; 5141 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5142 IP = 0; 5143 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5144 return cast<MachineSDNode>(E); 5145 } 5146 5147 // Allocate a new MachineSDNode. 5148 N = new (NodeAllocator) MachineSDNode(~Opcode, DL, VTs); 5149 5150 // Initialize the operands list. 5151 if (NumOps > array_lengthof(N->LocalOperands)) 5152 // We're creating a final node that will live unmorphed for the 5153 // remainder of the current SelectionDAG iteration, so we can allocate 5154 // the operands directly out of a pool with no recycling metadata. 5155 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5156 Ops, NumOps); 5157 else 5158 N->InitOperands(N->LocalOperands, Ops, NumOps); 5159 N->OperandsNeedDelete = false; 5160 5161 if (DoCSE) 5162 CSEMap.InsertNode(N, IP); 5163 5164 AllNodes.push_back(N); 5165#ifndef NDEBUG 5166 VerifyMachineNode(N); 5167#endif 5168 return N; 5169} 5170 5171/// getTargetExtractSubreg - A convenience function for creating 5172/// TargetOpcode::EXTRACT_SUBREG nodes. 5173SDValue 5174SelectionDAG::getTargetExtractSubreg(int SRIdx, DebugLoc DL, EVT VT, 5175 SDValue Operand) { 5176 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5177 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5178 VT, Operand, SRIdxVal); 5179 return SDValue(Subreg, 0); 5180} 5181 5182/// getTargetInsertSubreg - A convenience function for creating 5183/// TargetOpcode::INSERT_SUBREG nodes. 5184SDValue 5185SelectionDAG::getTargetInsertSubreg(int SRIdx, DebugLoc DL, EVT VT, 5186 SDValue Operand, SDValue Subreg) { 5187 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5188 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5189 VT, Operand, Subreg, SRIdxVal); 5190 return SDValue(Result, 0); 5191} 5192 5193/// getNodeIfExists - Get the specified node if it's already available, or 5194/// else return NULL. 5195SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5196 const SDValue *Ops, unsigned NumOps) { 5197 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5198 FoldingSetNodeID ID; 5199 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5200 void *IP = 0; 5201 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5202 return E; 5203 } 5204 return NULL; 5205} 5206 5207/// getDbgValue - Creates a SDDbgValue node. 5208/// 5209SDDbgValue * 5210SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5211 DebugLoc DL, unsigned O) { 5212 return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5213} 5214 5215SDDbgValue * 5216SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5217 DebugLoc DL, unsigned O) { 5218 return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5219} 5220 5221SDDbgValue * 5222SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5223 DebugLoc DL, unsigned O) { 5224 return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5225} 5226 5227namespace { 5228 5229/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5230/// pointed to by a use iterator is deleted, increment the use iterator 5231/// so that it doesn't dangle. 5232/// 5233/// This class also manages a "downlink" DAGUpdateListener, to forward 5234/// messages to ReplaceAllUsesWith's callers. 5235/// 5236class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5237 SelectionDAG::DAGUpdateListener *DownLink; 5238 SDNode::use_iterator &UI; 5239 SDNode::use_iterator &UE; 5240 5241 virtual void NodeDeleted(SDNode *N, SDNode *E) { 5242 // Increment the iterator as needed. 5243 while (UI != UE && N == *UI) 5244 ++UI; 5245 5246 // Then forward the message. 5247 if (DownLink) DownLink->NodeDeleted(N, E); 5248 } 5249 5250 virtual void NodeUpdated(SDNode *N) { 5251 // Just forward the message. 5252 if (DownLink) DownLink->NodeUpdated(N); 5253 } 5254 5255public: 5256 RAUWUpdateListener(SelectionDAG::DAGUpdateListener *dl, 5257 SDNode::use_iterator &ui, 5258 SDNode::use_iterator &ue) 5259 : DownLink(dl), UI(ui), UE(ue) {} 5260}; 5261 5262} 5263 5264/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5265/// This can cause recursive merging of nodes in the DAG. 5266/// 5267/// This version assumes From has a single result value. 5268/// 5269void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To, 5270 DAGUpdateListener *UpdateListener) { 5271 SDNode *From = FromN.getNode(); 5272 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5273 "Cannot replace with this method!"); 5274 assert(From != To.getNode() && "Cannot replace uses of with self"); 5275 5276 // Iterate over all the existing uses of From. New uses will be added 5277 // to the beginning of the use list, which we avoid visiting. 5278 // This specifically avoids visiting uses of From that arise while the 5279 // replacement is happening, because any such uses would be the result 5280 // of CSE: If an existing node looks like From after one of its operands 5281 // is replaced by To, we don't want to replace of all its users with To 5282 // too. See PR3018 for more info. 5283 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5284 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5285 while (UI != UE) { 5286 SDNode *User = *UI; 5287 5288 // This node is about to morph, remove its old self from the CSE maps. 5289 RemoveNodeFromCSEMaps(User); 5290 5291 // A user can appear in a use list multiple times, and when this 5292 // happens the uses are usually next to each other in the list. 5293 // To help reduce the number of CSE recomputations, process all 5294 // the uses of this user that we can find this way. 5295 do { 5296 SDUse &Use = UI.getUse(); 5297 ++UI; 5298 Use.set(To); 5299 } while (UI != UE && *UI == User); 5300 5301 // Now that we have modified User, add it back to the CSE maps. If it 5302 // already exists there, recursively merge the results together. 5303 AddModifiedNodeToCSEMaps(User, &Listener); 5304 } 5305 5306 // If we just RAUW'd the root, take note. 5307 if (FromN == getRoot()) 5308 setRoot(To); 5309} 5310 5311/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5312/// This can cause recursive merging of nodes in the DAG. 5313/// 5314/// This version assumes that for each value of From, there is a 5315/// corresponding value in To in the same position with the same type. 5316/// 5317void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To, 5318 DAGUpdateListener *UpdateListener) { 5319#ifndef NDEBUG 5320 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5321 assert((!From->hasAnyUseOfValue(i) || 5322 From->getValueType(i) == To->getValueType(i)) && 5323 "Cannot use this version of ReplaceAllUsesWith!"); 5324#endif 5325 5326 // Handle the trivial case. 5327 if (From == To) 5328 return; 5329 5330 // Iterate over just the existing users of From. See the comments in 5331 // the ReplaceAllUsesWith above. 5332 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5333 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5334 while (UI != UE) { 5335 SDNode *User = *UI; 5336 5337 // This node is about to morph, remove its old self from the CSE maps. 5338 RemoveNodeFromCSEMaps(User); 5339 5340 // A user can appear in a use list multiple times, and when this 5341 // happens the uses are usually next to each other in the list. 5342 // To help reduce the number of CSE recomputations, process all 5343 // the uses of this user that we can find this way. 5344 do { 5345 SDUse &Use = UI.getUse(); 5346 ++UI; 5347 Use.setNode(To); 5348 } while (UI != UE && *UI == User); 5349 5350 // Now that we have modified User, add it back to the CSE maps. If it 5351 // already exists there, recursively merge the results together. 5352 AddModifiedNodeToCSEMaps(User, &Listener); 5353 } 5354 5355 // If we just RAUW'd the root, take note. 5356 if (From == getRoot().getNode()) 5357 setRoot(SDValue(To, getRoot().getResNo())); 5358} 5359 5360/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5361/// This can cause recursive merging of nodes in the DAG. 5362/// 5363/// This version can replace From with any result values. To must match the 5364/// number and types of values returned by From. 5365void SelectionDAG::ReplaceAllUsesWith(SDNode *From, 5366 const SDValue *To, 5367 DAGUpdateListener *UpdateListener) { 5368 if (From->getNumValues() == 1) // Handle the simple case efficiently. 5369 return ReplaceAllUsesWith(SDValue(From, 0), To[0], UpdateListener); 5370 5371 // Iterate over just the existing users of From. See the comments in 5372 // the ReplaceAllUsesWith above. 5373 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5374 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5375 while (UI != UE) { 5376 SDNode *User = *UI; 5377 5378 // This node is about to morph, remove its old self from the CSE maps. 5379 RemoveNodeFromCSEMaps(User); 5380 5381 // A user can appear in a use list multiple times, and when this 5382 // happens the uses are usually next to each other in the list. 5383 // To help reduce the number of CSE recomputations, process all 5384 // the uses of this user that we can find this way. 5385 do { 5386 SDUse &Use = UI.getUse(); 5387 const SDValue &ToOp = To[Use.getResNo()]; 5388 ++UI; 5389 Use.set(ToOp); 5390 } while (UI != UE && *UI == User); 5391 5392 // Now that we have modified User, add it back to the CSE maps. If it 5393 // already exists there, recursively merge the results together. 5394 AddModifiedNodeToCSEMaps(User, &Listener); 5395 } 5396 5397 // If we just RAUW'd the root, take note. 5398 if (From == getRoot().getNode()) 5399 setRoot(SDValue(To[getRoot().getResNo()])); 5400} 5401 5402/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5403/// uses of other values produced by From.getNode() alone. The Deleted 5404/// vector is handled the same way as for ReplaceAllUsesWith. 5405void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To, 5406 DAGUpdateListener *UpdateListener){ 5407 // Handle the really simple, really trivial case efficiently. 5408 if (From == To) return; 5409 5410 // Handle the simple, trivial, case efficiently. 5411 if (From.getNode()->getNumValues() == 1) { 5412 ReplaceAllUsesWith(From, To, UpdateListener); 5413 return; 5414 } 5415 5416 // Iterate over just the existing users of From. See the comments in 5417 // the ReplaceAllUsesWith above. 5418 SDNode::use_iterator UI = From.getNode()->use_begin(), 5419 UE = From.getNode()->use_end(); 5420 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5421 while (UI != UE) { 5422 SDNode *User = *UI; 5423 bool UserRemovedFromCSEMaps = false; 5424 5425 // A user can appear in a use list multiple times, and when this 5426 // happens the uses are usually next to each other in the list. 5427 // To help reduce the number of CSE recomputations, process all 5428 // the uses of this user that we can find this way. 5429 do { 5430 SDUse &Use = UI.getUse(); 5431 5432 // Skip uses of different values from the same node. 5433 if (Use.getResNo() != From.getResNo()) { 5434 ++UI; 5435 continue; 5436 } 5437 5438 // If this node hasn't been modified yet, it's still in the CSE maps, 5439 // so remove its old self from the CSE maps. 5440 if (!UserRemovedFromCSEMaps) { 5441 RemoveNodeFromCSEMaps(User); 5442 UserRemovedFromCSEMaps = true; 5443 } 5444 5445 ++UI; 5446 Use.set(To); 5447 } while (UI != UE && *UI == User); 5448 5449 // We are iterating over all uses of the From node, so if a use 5450 // doesn't use the specific value, no changes are made. 5451 if (!UserRemovedFromCSEMaps) 5452 continue; 5453 5454 // Now that we have modified User, add it back to the CSE maps. If it 5455 // already exists there, recursively merge the results together. 5456 AddModifiedNodeToCSEMaps(User, &Listener); 5457 } 5458 5459 // If we just RAUW'd the root, take note. 5460 if (From == getRoot()) 5461 setRoot(To); 5462} 5463 5464namespace { 5465 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5466 /// to record information about a use. 5467 struct UseMemo { 5468 SDNode *User; 5469 unsigned Index; 5470 SDUse *Use; 5471 }; 5472 5473 /// operator< - Sort Memos by User. 5474 bool operator<(const UseMemo &L, const UseMemo &R) { 5475 return (intptr_t)L.User < (intptr_t)R.User; 5476 } 5477} 5478 5479/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5480/// uses of other values produced by From.getNode() alone. The same value 5481/// may appear in both the From and To list. The Deleted vector is 5482/// handled the same way as for ReplaceAllUsesWith. 5483void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5484 const SDValue *To, 5485 unsigned Num, 5486 DAGUpdateListener *UpdateListener){ 5487 // Handle the simple, trivial case efficiently. 5488 if (Num == 1) 5489 return ReplaceAllUsesOfValueWith(*From, *To, UpdateListener); 5490 5491 // Read up all the uses and make records of them. This helps 5492 // processing new uses that are introduced during the 5493 // replacement process. 5494 SmallVector<UseMemo, 4> Uses; 5495 for (unsigned i = 0; i != Num; ++i) { 5496 unsigned FromResNo = From[i].getResNo(); 5497 SDNode *FromNode = From[i].getNode(); 5498 for (SDNode::use_iterator UI = FromNode->use_begin(), 5499 E = FromNode->use_end(); UI != E; ++UI) { 5500 SDUse &Use = UI.getUse(); 5501 if (Use.getResNo() == FromResNo) { 5502 UseMemo Memo = { *UI, i, &Use }; 5503 Uses.push_back(Memo); 5504 } 5505 } 5506 } 5507 5508 // Sort the uses, so that all the uses from a given User are together. 5509 std::sort(Uses.begin(), Uses.end()); 5510 5511 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5512 UseIndex != UseIndexEnd; ) { 5513 // We know that this user uses some value of From. If it is the right 5514 // value, update it. 5515 SDNode *User = Uses[UseIndex].User; 5516 5517 // This node is about to morph, remove its old self from the CSE maps. 5518 RemoveNodeFromCSEMaps(User); 5519 5520 // The Uses array is sorted, so all the uses for a given User 5521 // are next to each other in the list. 5522 // To help reduce the number of CSE recomputations, process all 5523 // the uses of this user that we can find this way. 5524 do { 5525 unsigned i = Uses[UseIndex].Index; 5526 SDUse &Use = *Uses[UseIndex].Use; 5527 ++UseIndex; 5528 5529 Use.set(To[i]); 5530 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5531 5532 // Now that we have modified User, add it back to the CSE maps. If it 5533 // already exists there, recursively merge the results together. 5534 AddModifiedNodeToCSEMaps(User, UpdateListener); 5535 } 5536} 5537 5538/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5539/// based on their topological order. It returns the maximum id and a vector 5540/// of the SDNodes* in assigned order by reference. 5541unsigned SelectionDAG::AssignTopologicalOrder() { 5542 5543 unsigned DAGSize = 0; 5544 5545 // SortedPos tracks the progress of the algorithm. Nodes before it are 5546 // sorted, nodes after it are unsorted. When the algorithm completes 5547 // it is at the end of the list. 5548 allnodes_iterator SortedPos = allnodes_begin(); 5549 5550 // Visit all the nodes. Move nodes with no operands to the front of 5551 // the list immediately. Annotate nodes that do have operands with their 5552 // operand count. Before we do this, the Node Id fields of the nodes 5553 // may contain arbitrary values. After, the Node Id fields for nodes 5554 // before SortedPos will contain the topological sort index, and the 5555 // Node Id fields for nodes At SortedPos and after will contain the 5556 // count of outstanding operands. 5557 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5558 SDNode *N = I++; 5559 checkForCycles(N); 5560 unsigned Degree = N->getNumOperands(); 5561 if (Degree == 0) { 5562 // A node with no uses, add it to the result array immediately. 5563 N->setNodeId(DAGSize++); 5564 allnodes_iterator Q = N; 5565 if (Q != SortedPos) 5566 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5567 assert(SortedPos != AllNodes.end() && "Overran node list"); 5568 ++SortedPos; 5569 } else { 5570 // Temporarily use the Node Id as scratch space for the degree count. 5571 N->setNodeId(Degree); 5572 } 5573 } 5574 5575 // Visit all the nodes. As we iterate, moves nodes into sorted order, 5576 // such that by the time the end is reached all nodes will be sorted. 5577 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5578 SDNode *N = I; 5579 checkForCycles(N); 5580 // N is in sorted position, so all its uses have one less operand 5581 // that needs to be sorted. 5582 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5583 UI != UE; ++UI) { 5584 SDNode *P = *UI; 5585 unsigned Degree = P->getNodeId(); 5586 assert(Degree != 0 && "Invalid node degree"); 5587 --Degree; 5588 if (Degree == 0) { 5589 // All of P's operands are sorted, so P may sorted now. 5590 P->setNodeId(DAGSize++); 5591 if (P != SortedPos) 5592 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5593 assert(SortedPos != AllNodes.end() && "Overran node list"); 5594 ++SortedPos; 5595 } else { 5596 // Update P's outstanding operand count. 5597 P->setNodeId(Degree); 5598 } 5599 } 5600 if (I == SortedPos) { 5601#ifndef NDEBUG 5602 SDNode *S = ++I; 5603 dbgs() << "Overran sorted position:\n"; 5604 S->dumprFull(); 5605#endif 5606 llvm_unreachable(0); 5607 } 5608 } 5609 5610 assert(SortedPos == AllNodes.end() && 5611 "Topological sort incomplete!"); 5612 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 5613 "First node in topological sort is not the entry token!"); 5614 assert(AllNodes.front().getNodeId() == 0 && 5615 "First node in topological sort has non-zero id!"); 5616 assert(AllNodes.front().getNumOperands() == 0 && 5617 "First node in topological sort has operands!"); 5618 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 5619 "Last node in topologic sort has unexpected id!"); 5620 assert(AllNodes.back().use_empty() && 5621 "Last node in topologic sort has users!"); 5622 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 5623 return DAGSize; 5624} 5625 5626/// AssignOrdering - Assign an order to the SDNode. 5627void SelectionDAG::AssignOrdering(const SDNode *SD, unsigned Order) { 5628 assert(SD && "Trying to assign an order to a null node!"); 5629 Ordering->add(SD, Order); 5630} 5631 5632/// GetOrdering - Get the order for the SDNode. 5633unsigned SelectionDAG::GetOrdering(const SDNode *SD) const { 5634 assert(SD && "Trying to get the order of a null node!"); 5635 return Ordering->getOrder(SD); 5636} 5637 5638/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 5639/// value is produced by SD. 5640void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 5641 DbgInfo->add(DB, SD, isParameter); 5642 if (SD) 5643 SD->setHasDebugValue(true); 5644} 5645 5646/// TransferDbgValues - Transfer SDDbgValues. 5647void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 5648 if (From == To || !From.getNode()->getHasDebugValue()) 5649 return; 5650 SDNode *FromNode = From.getNode(); 5651 SDNode *ToNode = To.getNode(); 5652 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 5653 SmallVector<SDDbgValue *, 2> ClonedDVs; 5654 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 5655 I != E; ++I) { 5656 SDDbgValue *Dbg = *I; 5657 if (Dbg->getKind() == SDDbgValue::SDNODE) { 5658 SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 5659 Dbg->getOffset(), Dbg->getDebugLoc(), 5660 Dbg->getOrder()); 5661 ClonedDVs.push_back(Clone); 5662 } 5663 } 5664 for (SmallVector<SDDbgValue *, 2>::iterator I = ClonedDVs.begin(), 5665 E = ClonedDVs.end(); I != E; ++I) 5666 AddDbgValue(*I, ToNode, false); 5667} 5668 5669//===----------------------------------------------------------------------===// 5670// SDNode Class 5671//===----------------------------------------------------------------------===// 5672 5673HandleSDNode::~HandleSDNode() { 5674 DropOperands(); 5675} 5676 5677GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, DebugLoc DL, 5678 const GlobalValue *GA, 5679 EVT VT, int64_t o, unsigned char TF) 5680 : SDNode(Opc, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 5681 TheGlobal = GA; 5682} 5683 5684MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, 5685 MachineMemOperand *mmo) 5686 : SDNode(Opc, dl, VTs), MemoryVT(memvt), MMO(mmo) { 5687 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 5688 MMO->isNonTemporal(), MMO->isInvariant()); 5689 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 5690 assert(isNonTemporal() == MMO->isNonTemporal() && 5691 "Non-temporal encoding error!"); 5692 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 5693} 5694 5695MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, 5696 const SDValue *Ops, unsigned NumOps, EVT memvt, 5697 MachineMemOperand *mmo) 5698 : SDNode(Opc, dl, VTs, Ops, NumOps), 5699 MemoryVT(memvt), MMO(mmo) { 5700 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 5701 MMO->isNonTemporal(), MMO->isInvariant()); 5702 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 5703 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 5704} 5705 5706/// Profile - Gather unique data for the node. 5707/// 5708void SDNode::Profile(FoldingSetNodeID &ID) const { 5709 AddNodeIDNode(ID, this); 5710} 5711 5712namespace { 5713 struct EVTArray { 5714 std::vector<EVT> VTs; 5715 5716 EVTArray() { 5717 VTs.reserve(MVT::LAST_VALUETYPE); 5718 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 5719 VTs.push_back(MVT((MVT::SimpleValueType)i)); 5720 } 5721 }; 5722} 5723 5724static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 5725static ManagedStatic<EVTArray> SimpleVTArray; 5726static ManagedStatic<sys::SmartMutex<true> > VTMutex; 5727 5728/// getValueTypeList - Return a pointer to the specified value type. 5729/// 5730const EVT *SDNode::getValueTypeList(EVT VT) { 5731 if (VT.isExtended()) { 5732 sys::SmartScopedLock<true> Lock(*VTMutex); 5733 return &(*EVTs->insert(VT).first); 5734 } else { 5735 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 5736 "Value type out of range!"); 5737 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 5738 } 5739} 5740 5741/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 5742/// indicated value. This method ignores uses of other values defined by this 5743/// operation. 5744bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 5745 assert(Value < getNumValues() && "Bad value!"); 5746 5747 // TODO: Only iterate over uses of a given value of the node 5748 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 5749 if (UI.getUse().getResNo() == Value) { 5750 if (NUses == 0) 5751 return false; 5752 --NUses; 5753 } 5754 } 5755 5756 // Found exactly the right number of uses? 5757 return NUses == 0; 5758} 5759 5760 5761/// hasAnyUseOfValue - Return true if there are any use of the indicated 5762/// value. This method ignores uses of other values defined by this operation. 5763bool SDNode::hasAnyUseOfValue(unsigned Value) const { 5764 assert(Value < getNumValues() && "Bad value!"); 5765 5766 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 5767 if (UI.getUse().getResNo() == Value) 5768 return true; 5769 5770 return false; 5771} 5772 5773 5774/// isOnlyUserOf - Return true if this node is the only use of N. 5775/// 5776bool SDNode::isOnlyUserOf(SDNode *N) const { 5777 bool Seen = false; 5778 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 5779 SDNode *User = *I; 5780 if (User == this) 5781 Seen = true; 5782 else 5783 return false; 5784 } 5785 5786 return Seen; 5787} 5788 5789/// isOperand - Return true if this node is an operand of N. 5790/// 5791bool SDValue::isOperandOf(SDNode *N) const { 5792 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 5793 if (*this == N->getOperand(i)) 5794 return true; 5795 return false; 5796} 5797 5798bool SDNode::isOperandOf(SDNode *N) const { 5799 for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 5800 if (this == N->OperandList[i].getNode()) 5801 return true; 5802 return false; 5803} 5804 5805/// reachesChainWithoutSideEffects - Return true if this operand (which must 5806/// be a chain) reaches the specified operand without crossing any 5807/// side-effecting instructions on any chain path. In practice, this looks 5808/// through token factors and non-volatile loads. In order to remain efficient, 5809/// this only looks a couple of nodes in, it does not do an exhaustive search. 5810bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 5811 unsigned Depth) const { 5812 if (*this == Dest) return true; 5813 5814 // Don't search too deeply, we just want to be able to see through 5815 // TokenFactor's etc. 5816 if (Depth == 0) return false; 5817 5818 // If this is a token factor, all inputs to the TF happen in parallel. If any 5819 // of the operands of the TF does not reach dest, then we cannot do the xform. 5820 if (getOpcode() == ISD::TokenFactor) { 5821 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 5822 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 5823 return false; 5824 return true; 5825 } 5826 5827 // Loads don't have side effects, look through them. 5828 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 5829 if (!Ld->isVolatile()) 5830 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 5831 } 5832 return false; 5833} 5834 5835/// hasPredecessor - Return true if N is a predecessor of this node. 5836/// N is either an operand of this node, or can be reached by recursively 5837/// traversing up the operands. 5838/// NOTE: This is an expensive method. Use it carefully. 5839bool SDNode::hasPredecessor(const SDNode *N) const { 5840 SmallPtrSet<const SDNode *, 32> Visited; 5841 SmallVector<const SDNode *, 16> Worklist; 5842 return hasPredecessorHelper(N, Visited, Worklist); 5843} 5844 5845bool SDNode::hasPredecessorHelper(const SDNode *N, 5846 SmallPtrSet<const SDNode *, 32> &Visited, 5847 SmallVector<const SDNode *, 16> &Worklist) const { 5848 if (Visited.empty()) { 5849 Worklist.push_back(this); 5850 } else { 5851 // Take a look in the visited set. If we've already encountered this node 5852 // we needn't search further. 5853 if (Visited.count(N)) 5854 return true; 5855 } 5856 5857 // Haven't visited N yet. Continue the search. 5858 while (!Worklist.empty()) { 5859 const SDNode *M = Worklist.pop_back_val(); 5860 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 5861 SDNode *Op = M->getOperand(i).getNode(); 5862 if (Visited.insert(Op)) 5863 Worklist.push_back(Op); 5864 if (Op == N) 5865 return true; 5866 } 5867 } 5868 5869 return false; 5870} 5871 5872uint64_t SDNode::getConstantOperandVal(unsigned Num) const { 5873 assert(Num < NumOperands && "Invalid child # of SDNode!"); 5874 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 5875} 5876 5877std::string SDNode::getOperationName(const SelectionDAG *G) const { 5878 switch (getOpcode()) { 5879 default: 5880 if (getOpcode() < ISD::BUILTIN_OP_END) 5881 return "<<Unknown DAG Node>>"; 5882 if (isMachineOpcode()) { 5883 if (G) 5884 if (const TargetInstrInfo *TII = G->getTarget().getInstrInfo()) 5885 if (getMachineOpcode() < TII->getNumOpcodes()) 5886 return TII->get(getMachineOpcode()).getName(); 5887 return "<<Unknown Machine Node #" + utostr(getOpcode()) + ">>"; 5888 } 5889 if (G) { 5890 const TargetLowering &TLI = G->getTargetLoweringInfo(); 5891 const char *Name = TLI.getTargetNodeName(getOpcode()); 5892 if (Name) return Name; 5893 return "<<Unknown Target Node #" + utostr(getOpcode()) + ">>"; 5894 } 5895 return "<<Unknown Node #" + utostr(getOpcode()) + ">>"; 5896 5897#ifndef NDEBUG 5898 case ISD::DELETED_NODE: 5899 return "<<Deleted Node!>>"; 5900#endif 5901 case ISD::PREFETCH: return "Prefetch"; 5902 case ISD::MEMBARRIER: return "MemBarrier"; 5903 case ISD::ATOMIC_FENCE: return "AtomicFence"; 5904 case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap"; 5905 case ISD::ATOMIC_SWAP: return "AtomicSwap"; 5906 case ISD::ATOMIC_LOAD_ADD: return "AtomicLoadAdd"; 5907 case ISD::ATOMIC_LOAD_SUB: return "AtomicLoadSub"; 5908 case ISD::ATOMIC_LOAD_AND: return "AtomicLoadAnd"; 5909 case ISD::ATOMIC_LOAD_OR: return "AtomicLoadOr"; 5910 case ISD::ATOMIC_LOAD_XOR: return "AtomicLoadXor"; 5911 case ISD::ATOMIC_LOAD_NAND: return "AtomicLoadNand"; 5912 case ISD::ATOMIC_LOAD_MIN: return "AtomicLoadMin"; 5913 case ISD::ATOMIC_LOAD_MAX: return "AtomicLoadMax"; 5914 case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin"; 5915 case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax"; 5916 case ISD::ATOMIC_LOAD: return "AtomicLoad"; 5917 case ISD::ATOMIC_STORE: return "AtomicStore"; 5918 case ISD::PCMARKER: return "PCMarker"; 5919 case ISD::READCYCLECOUNTER: return "ReadCycleCounter"; 5920 case ISD::SRCVALUE: return "SrcValue"; 5921 case ISD::MDNODE_SDNODE: return "MDNode"; 5922 case ISD::EntryToken: return "EntryToken"; 5923 case ISD::TokenFactor: return "TokenFactor"; 5924 case ISD::AssertSext: return "AssertSext"; 5925 case ISD::AssertZext: return "AssertZext"; 5926 5927 case ISD::BasicBlock: return "BasicBlock"; 5928 case ISD::VALUETYPE: return "ValueType"; 5929 case ISD::Register: return "Register"; 5930 5931 case ISD::Constant: return "Constant"; 5932 case ISD::ConstantFP: return "ConstantFP"; 5933 case ISD::GlobalAddress: return "GlobalAddress"; 5934 case ISD::GlobalTLSAddress: return "GlobalTLSAddress"; 5935 case ISD::FrameIndex: return "FrameIndex"; 5936 case ISD::JumpTable: return "JumpTable"; 5937 case ISD::GLOBAL_OFFSET_TABLE: return "GLOBAL_OFFSET_TABLE"; 5938 case ISD::RETURNADDR: return "RETURNADDR"; 5939 case ISD::FRAMEADDR: return "FRAMEADDR"; 5940 case ISD::FRAME_TO_ARGS_OFFSET: return "FRAME_TO_ARGS_OFFSET"; 5941 case ISD::EXCEPTIONADDR: return "EXCEPTIONADDR"; 5942 case ISD::LSDAADDR: return "LSDAADDR"; 5943 case ISD::EHSELECTION: return "EHSELECTION"; 5944 case ISD::EH_RETURN: return "EH_RETURN"; 5945 case ISD::EH_SJLJ_SETJMP: return "EH_SJLJ_SETJMP"; 5946 case ISD::EH_SJLJ_LONGJMP: return "EH_SJLJ_LONGJMP"; 5947 case ISD::EH_SJLJ_DISPATCHSETUP: return "EH_SJLJ_DISPATCHSETUP"; 5948 case ISD::ConstantPool: return "ConstantPool"; 5949 case ISD::ExternalSymbol: return "ExternalSymbol"; 5950 case ISD::BlockAddress: return "BlockAddress"; 5951 case ISD::INTRINSIC_WO_CHAIN: 5952 case ISD::INTRINSIC_VOID: 5953 case ISD::INTRINSIC_W_CHAIN: { 5954 unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1; 5955 unsigned IID = cast<ConstantSDNode>(getOperand(OpNo))->getZExtValue(); 5956 if (IID < Intrinsic::num_intrinsics) 5957 return Intrinsic::getName((Intrinsic::ID)IID); 5958 else if (const TargetIntrinsicInfo *TII = G->getTarget().getIntrinsicInfo()) 5959 return TII->getName(IID); 5960 llvm_unreachable("Invalid intrinsic ID"); 5961 } 5962 5963 case ISD::BUILD_VECTOR: return "BUILD_VECTOR"; 5964 case ISD::TargetConstant: return "TargetConstant"; 5965 case ISD::TargetConstantFP:return "TargetConstantFP"; 5966 case ISD::TargetGlobalAddress: return "TargetGlobalAddress"; 5967 case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress"; 5968 case ISD::TargetFrameIndex: return "TargetFrameIndex"; 5969 case ISD::TargetJumpTable: return "TargetJumpTable"; 5970 case ISD::TargetConstantPool: return "TargetConstantPool"; 5971 case ISD::TargetExternalSymbol: return "TargetExternalSymbol"; 5972 case ISD::TargetBlockAddress: return "TargetBlockAddress"; 5973 5974 case ISD::CopyToReg: return "CopyToReg"; 5975 case ISD::CopyFromReg: return "CopyFromReg"; 5976 case ISD::UNDEF: return "undef"; 5977 case ISD::MERGE_VALUES: return "merge_values"; 5978 case ISD::INLINEASM: return "inlineasm"; 5979 case ISD::EH_LABEL: return "eh_label"; 5980 case ISD::HANDLENODE: return "handlenode"; 5981 5982 // Unary operators 5983 case ISD::FABS: return "fabs"; 5984 case ISD::FNEG: return "fneg"; 5985 case ISD::FSQRT: return "fsqrt"; 5986 case ISD::FSIN: return "fsin"; 5987 case ISD::FCOS: return "fcos"; 5988 case ISD::FTRUNC: return "ftrunc"; 5989 case ISD::FFLOOR: return "ffloor"; 5990 case ISD::FCEIL: return "fceil"; 5991 case ISD::FRINT: return "frint"; 5992 case ISD::FNEARBYINT: return "fnearbyint"; 5993 case ISD::FEXP: return "fexp"; 5994 case ISD::FEXP2: return "fexp2"; 5995 case ISD::FLOG: return "flog"; 5996 case ISD::FLOG2: return "flog2"; 5997 case ISD::FLOG10: return "flog10"; 5998 5999 // Binary operators 6000 case ISD::ADD: return "add"; 6001 case ISD::SUB: return "sub"; 6002 case ISD::MUL: return "mul"; 6003 case ISD::MULHU: return "mulhu"; 6004 case ISD::MULHS: return "mulhs"; 6005 case ISD::SDIV: return "sdiv"; 6006 case ISD::UDIV: return "udiv"; 6007 case ISD::SREM: return "srem"; 6008 case ISD::UREM: return "urem"; 6009 case ISD::SMUL_LOHI: return "smul_lohi"; 6010 case ISD::UMUL_LOHI: return "umul_lohi"; 6011 case ISD::SDIVREM: return "sdivrem"; 6012 case ISD::UDIVREM: return "udivrem"; 6013 case ISD::AND: return "and"; 6014 case ISD::OR: return "or"; 6015 case ISD::XOR: return "xor"; 6016 case ISD::SHL: return "shl"; 6017 case ISD::SRA: return "sra"; 6018 case ISD::SRL: return "srl"; 6019 case ISD::ROTL: return "rotl"; 6020 case ISD::ROTR: return "rotr"; 6021 case ISD::FADD: return "fadd"; 6022 case ISD::FSUB: return "fsub"; 6023 case ISD::FMUL: return "fmul"; 6024 case ISD::FDIV: return "fdiv"; 6025 case ISD::FMA: return "fma"; 6026 case ISD::FREM: return "frem"; 6027 case ISD::FCOPYSIGN: return "fcopysign"; 6028 case ISD::FGETSIGN: return "fgetsign"; 6029 case ISD::FPOW: return "fpow"; 6030 6031 case ISD::FPOWI: return "fpowi"; 6032 case ISD::SETCC: return "setcc"; 6033 case ISD::SELECT: return "select"; 6034 case ISD::VSELECT: return "vselect"; 6035 case ISD::SELECT_CC: return "select_cc"; 6036 case ISD::INSERT_VECTOR_ELT: return "insert_vector_elt"; 6037 case ISD::EXTRACT_VECTOR_ELT: return "extract_vector_elt"; 6038 case ISD::CONCAT_VECTORS: return "concat_vectors"; 6039 case ISD::INSERT_SUBVECTOR: return "insert_subvector"; 6040 case ISD::EXTRACT_SUBVECTOR: return "extract_subvector"; 6041 case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector"; 6042 case ISD::VECTOR_SHUFFLE: return "vector_shuffle"; 6043 case ISD::CARRY_FALSE: return "carry_false"; 6044 case ISD::ADDC: return "addc"; 6045 case ISD::ADDE: return "adde"; 6046 case ISD::SADDO: return "saddo"; 6047 case ISD::UADDO: return "uaddo"; 6048 case ISD::SSUBO: return "ssubo"; 6049 case ISD::USUBO: return "usubo"; 6050 case ISD::SMULO: return "smulo"; 6051 case ISD::UMULO: return "umulo"; 6052 case ISD::SUBC: return "subc"; 6053 case ISD::SUBE: return "sube"; 6054 case ISD::SHL_PARTS: return "shl_parts"; 6055 case ISD::SRA_PARTS: return "sra_parts"; 6056 case ISD::SRL_PARTS: return "srl_parts"; 6057 6058 // Conversion operators. 6059 case ISD::SIGN_EXTEND: return "sign_extend"; 6060 case ISD::ZERO_EXTEND: return "zero_extend"; 6061 case ISD::ANY_EXTEND: return "any_extend"; 6062 case ISD::SIGN_EXTEND_INREG: return "sign_extend_inreg"; 6063 case ISD::TRUNCATE: return "truncate"; 6064 case ISD::FP_ROUND: return "fp_round"; 6065 case ISD::FLT_ROUNDS_: return "flt_rounds"; 6066 case ISD::FP_ROUND_INREG: return "fp_round_inreg"; 6067 case ISD::FP_EXTEND: return "fp_extend"; 6068 6069 case ISD::SINT_TO_FP: return "sint_to_fp"; 6070 case ISD::UINT_TO_FP: return "uint_to_fp"; 6071 case ISD::FP_TO_SINT: return "fp_to_sint"; 6072 case ISD::FP_TO_UINT: return "fp_to_uint"; 6073 case ISD::BITCAST: return "bitcast"; 6074 case ISD::FP16_TO_FP32: return "fp16_to_fp32"; 6075 case ISD::FP32_TO_FP16: return "fp32_to_fp16"; 6076 6077 case ISD::CONVERT_RNDSAT: { 6078 switch (cast<CvtRndSatSDNode>(this)->getCvtCode()) { 6079 default: llvm_unreachable("Unknown cvt code!"); 6080 case ISD::CVT_FF: return "cvt_ff"; 6081 case ISD::CVT_FS: return "cvt_fs"; 6082 case ISD::CVT_FU: return "cvt_fu"; 6083 case ISD::CVT_SF: return "cvt_sf"; 6084 case ISD::CVT_UF: return "cvt_uf"; 6085 case ISD::CVT_SS: return "cvt_ss"; 6086 case ISD::CVT_SU: return "cvt_su"; 6087 case ISD::CVT_US: return "cvt_us"; 6088 case ISD::CVT_UU: return "cvt_uu"; 6089 } 6090 } 6091 6092 // Control flow instructions 6093 case ISD::BR: return "br"; 6094 case ISD::BRIND: return "brind"; 6095 case ISD::BR_JT: return "br_jt"; 6096 case ISD::BRCOND: return "brcond"; 6097 case ISD::BR_CC: return "br_cc"; 6098 case ISD::CALLSEQ_START: return "callseq_start"; 6099 case ISD::CALLSEQ_END: return "callseq_end"; 6100 6101 // Other operators 6102 case ISD::LOAD: return "load"; 6103 case ISD::STORE: return "store"; 6104 case ISD::VAARG: return "vaarg"; 6105 case ISD::VACOPY: return "vacopy"; 6106 case ISD::VAEND: return "vaend"; 6107 case ISD::VASTART: return "vastart"; 6108 case ISD::DYNAMIC_STACKALLOC: return "dynamic_stackalloc"; 6109 case ISD::EXTRACT_ELEMENT: return "extract_element"; 6110 case ISD::BUILD_PAIR: return "build_pair"; 6111 case ISD::STACKSAVE: return "stacksave"; 6112 case ISD::STACKRESTORE: return "stackrestore"; 6113 case ISD::TRAP: return "trap"; 6114 6115 // Bit manipulation 6116 case ISD::BSWAP: return "bswap"; 6117 case ISD::CTPOP: return "ctpop"; 6118 case ISD::CTTZ: return "cttz"; 6119 case ISD::CTLZ: return "ctlz"; 6120 6121 // Trampolines 6122 case ISD::INIT_TRAMPOLINE: return "init_trampoline"; 6123 case ISD::ADJUST_TRAMPOLINE: return "adjust_trampoline"; 6124 6125 case ISD::CONDCODE: 6126 switch (cast<CondCodeSDNode>(this)->get()) { 6127 default: llvm_unreachable("Unknown setcc condition!"); 6128 case ISD::SETOEQ: return "setoeq"; 6129 case ISD::SETOGT: return "setogt"; 6130 case ISD::SETOGE: return "setoge"; 6131 case ISD::SETOLT: return "setolt"; 6132 case ISD::SETOLE: return "setole"; 6133 case ISD::SETONE: return "setone"; 6134 6135 case ISD::SETO: return "seto"; 6136 case ISD::SETUO: return "setuo"; 6137 case ISD::SETUEQ: return "setue"; 6138 case ISD::SETUGT: return "setugt"; 6139 case ISD::SETUGE: return "setuge"; 6140 case ISD::SETULT: return "setult"; 6141 case ISD::SETULE: return "setule"; 6142 case ISD::SETUNE: return "setune"; 6143 6144 case ISD::SETEQ: return "seteq"; 6145 case ISD::SETGT: return "setgt"; 6146 case ISD::SETGE: return "setge"; 6147 case ISD::SETLT: return "setlt"; 6148 case ISD::SETLE: return "setle"; 6149 case ISD::SETNE: return "setne"; 6150 } 6151 } 6152} 6153 6154const char *SDNode::getIndexedModeName(ISD::MemIndexedMode AM) { 6155 switch (AM) { 6156 default: 6157 return ""; 6158 case ISD::PRE_INC: 6159 return "<pre-inc>"; 6160 case ISD::PRE_DEC: 6161 return "<pre-dec>"; 6162 case ISD::POST_INC: 6163 return "<post-inc>"; 6164 case ISD::POST_DEC: 6165 return "<post-dec>"; 6166 } 6167} 6168 6169std::string ISD::ArgFlagsTy::getArgFlagsString() { 6170 std::string S = "< "; 6171 6172 if (isZExt()) 6173 S += "zext "; 6174 if (isSExt()) 6175 S += "sext "; 6176 if (isInReg()) 6177 S += "inreg "; 6178 if (isSRet()) 6179 S += "sret "; 6180 if (isByVal()) 6181 S += "byval "; 6182 if (isNest()) 6183 S += "nest "; 6184 if (getByValAlign()) 6185 S += "byval-align:" + utostr(getByValAlign()) + " "; 6186 if (getOrigAlign()) 6187 S += "orig-align:" + utostr(getOrigAlign()) + " "; 6188 if (getByValSize()) 6189 S += "byval-size:" + utostr(getByValSize()) + " "; 6190 return S + ">"; 6191} 6192 6193void SDNode::dump() const { dump(0); } 6194void SDNode::dump(const SelectionDAG *G) const { 6195 print(dbgs(), G); 6196 dbgs() << '\n'; 6197} 6198 6199void SDNode::print_types(raw_ostream &OS, const SelectionDAG *G) const { 6200 OS << (void*)this << ": "; 6201 6202 for (unsigned i = 0, e = getNumValues(); i != e; ++i) { 6203 if (i) OS << ","; 6204 if (getValueType(i) == MVT::Other) 6205 OS << "ch"; 6206 else 6207 OS << getValueType(i).getEVTString(); 6208 } 6209 OS << " = " << getOperationName(G); 6210} 6211 6212void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { 6213 if (const MachineSDNode *MN = dyn_cast<MachineSDNode>(this)) { 6214 if (!MN->memoperands_empty()) { 6215 OS << "<"; 6216 OS << "Mem:"; 6217 for (MachineSDNode::mmo_iterator i = MN->memoperands_begin(), 6218 e = MN->memoperands_end(); i != e; ++i) { 6219 OS << **i; 6220 if (llvm::next(i) != e) 6221 OS << " "; 6222 } 6223 OS << ">"; 6224 } 6225 } else if (const ShuffleVectorSDNode *SVN = 6226 dyn_cast<ShuffleVectorSDNode>(this)) { 6227 OS << "<"; 6228 for (unsigned i = 0, e = ValueList[0].getVectorNumElements(); i != e; ++i) { 6229 int Idx = SVN->getMaskElt(i); 6230 if (i) OS << ","; 6231 if (Idx < 0) 6232 OS << "u"; 6233 else 6234 OS << Idx; 6235 } 6236 OS << ">"; 6237 } else if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(this)) { 6238 OS << '<' << CSDN->getAPIntValue() << '>'; 6239 } else if (const ConstantFPSDNode *CSDN = dyn_cast<ConstantFPSDNode>(this)) { 6240 if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEsingle) 6241 OS << '<' << CSDN->getValueAPF().convertToFloat() << '>'; 6242 else if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEdouble) 6243 OS << '<' << CSDN->getValueAPF().convertToDouble() << '>'; 6244 else { 6245 OS << "<APFloat("; 6246 CSDN->getValueAPF().bitcastToAPInt().dump(); 6247 OS << ")>"; 6248 } 6249 } else if (const GlobalAddressSDNode *GADN = 6250 dyn_cast<GlobalAddressSDNode>(this)) { 6251 int64_t offset = GADN->getOffset(); 6252 OS << '<'; 6253 WriteAsOperand(OS, GADN->getGlobal()); 6254 OS << '>'; 6255 if (offset > 0) 6256 OS << " + " << offset; 6257 else 6258 OS << " " << offset; 6259 if (unsigned int TF = GADN->getTargetFlags()) 6260 OS << " [TF=" << TF << ']'; 6261 } else if (const FrameIndexSDNode *FIDN = dyn_cast<FrameIndexSDNode>(this)) { 6262 OS << "<" << FIDN->getIndex() << ">"; 6263 } else if (const JumpTableSDNode *JTDN = dyn_cast<JumpTableSDNode>(this)) { 6264 OS << "<" << JTDN->getIndex() << ">"; 6265 if (unsigned int TF = JTDN->getTargetFlags()) 6266 OS << " [TF=" << TF << ']'; 6267 } else if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(this)){ 6268 int offset = CP->getOffset(); 6269 if (CP->isMachineConstantPoolEntry()) 6270 OS << "<" << *CP->getMachineCPVal() << ">"; 6271 else 6272 OS << "<" << *CP->getConstVal() << ">"; 6273 if (offset > 0) 6274 OS << " + " << offset; 6275 else 6276 OS << " " << offset; 6277 if (unsigned int TF = CP->getTargetFlags()) 6278 OS << " [TF=" << TF << ']'; 6279 } else if (const BasicBlockSDNode *BBDN = dyn_cast<BasicBlockSDNode>(this)) { 6280 OS << "<"; 6281 const Value *LBB = (const Value*)BBDN->getBasicBlock()->getBasicBlock(); 6282 if (LBB) 6283 OS << LBB->getName() << " "; 6284 OS << (const void*)BBDN->getBasicBlock() << ">"; 6285 } else if (const RegisterSDNode *R = dyn_cast<RegisterSDNode>(this)) { 6286 OS << ' ' << PrintReg(R->getReg(), G ? G->getTarget().getRegisterInfo() :0); 6287 } else if (const ExternalSymbolSDNode *ES = 6288 dyn_cast<ExternalSymbolSDNode>(this)) { 6289 OS << "'" << ES->getSymbol() << "'"; 6290 if (unsigned int TF = ES->getTargetFlags()) 6291 OS << " [TF=" << TF << ']'; 6292 } else if (const SrcValueSDNode *M = dyn_cast<SrcValueSDNode>(this)) { 6293 if (M->getValue()) 6294 OS << "<" << M->getValue() << ">"; 6295 else 6296 OS << "<null>"; 6297 } else if (const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(this)) { 6298 if (MD->getMD()) 6299 OS << "<" << MD->getMD() << ">"; 6300 else 6301 OS << "<null>"; 6302 } else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) { 6303 OS << ":" << N->getVT().getEVTString(); 6304 } 6305 else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) { 6306 OS << "<" << *LD->getMemOperand(); 6307 6308 bool doExt = true; 6309 switch (LD->getExtensionType()) { 6310 default: doExt = false; break; 6311 case ISD::EXTLOAD: OS << ", anyext"; break; 6312 case ISD::SEXTLOAD: OS << ", sext"; break; 6313 case ISD::ZEXTLOAD: OS << ", zext"; break; 6314 } 6315 if (doExt) 6316 OS << " from " << LD->getMemoryVT().getEVTString(); 6317 6318 const char *AM = getIndexedModeName(LD->getAddressingMode()); 6319 if (*AM) 6320 OS << ", " << AM; 6321 6322 OS << ">"; 6323 } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(this)) { 6324 OS << "<" << *ST->getMemOperand(); 6325 6326 if (ST->isTruncatingStore()) 6327 OS << ", trunc to " << ST->getMemoryVT().getEVTString(); 6328 6329 const char *AM = getIndexedModeName(ST->getAddressingMode()); 6330 if (*AM) 6331 OS << ", " << AM; 6332 6333 OS << ">"; 6334 } else if (const MemSDNode* M = dyn_cast<MemSDNode>(this)) { 6335 OS << "<" << *M->getMemOperand() << ">"; 6336 } else if (const BlockAddressSDNode *BA = 6337 dyn_cast<BlockAddressSDNode>(this)) { 6338 OS << "<"; 6339 WriteAsOperand(OS, BA->getBlockAddress()->getFunction(), false); 6340 OS << ", "; 6341 WriteAsOperand(OS, BA->getBlockAddress()->getBasicBlock(), false); 6342 OS << ">"; 6343 if (unsigned int TF = BA->getTargetFlags()) 6344 OS << " [TF=" << TF << ']'; 6345 } 6346 6347 if (G) 6348 if (unsigned Order = G->GetOrdering(this)) 6349 OS << " [ORD=" << Order << ']'; 6350 6351 if (getNodeId() != -1) 6352 OS << " [ID=" << getNodeId() << ']'; 6353 6354 DebugLoc dl = getDebugLoc(); 6355 if (G && !dl.isUnknown()) { 6356 DIScope 6357 Scope(dl.getScope(G->getMachineFunction().getFunction()->getContext())); 6358 OS << " dbg:"; 6359 // Omit the directory, since it's usually long and uninteresting. 6360 if (Scope.Verify()) 6361 OS << Scope.getFilename(); 6362 else 6363 OS << "<unknown>"; 6364 OS << ':' << dl.getLine(); 6365 if (dl.getCol() != 0) 6366 OS << ':' << dl.getCol(); 6367 } 6368} 6369 6370void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { 6371 print_types(OS, G); 6372 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { 6373 if (i) OS << ", "; else OS << " "; 6374 OS << (void*)getOperand(i).getNode(); 6375 if (unsigned RN = getOperand(i).getResNo()) 6376 OS << ":" << RN; 6377 } 6378 print_details(OS, G); 6379} 6380 6381static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N, 6382 const SelectionDAG *G, unsigned depth, 6383 unsigned indent) { 6384 if (depth == 0) 6385 return; 6386 6387 OS.indent(indent); 6388 6389 N->print(OS, G); 6390 6391 if (depth < 1) 6392 return; 6393 6394 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6395 // Don't follow chain operands. 6396 if (N->getOperand(i).getValueType() == MVT::Other) 6397 continue; 6398 OS << '\n'; 6399 printrWithDepthHelper(OS, N->getOperand(i).getNode(), G, depth-1, indent+2); 6400 } 6401} 6402 6403void SDNode::printrWithDepth(raw_ostream &OS, const SelectionDAG *G, 6404 unsigned depth) const { 6405 printrWithDepthHelper(OS, this, G, depth, 0); 6406} 6407 6408void SDNode::printrFull(raw_ostream &OS, const SelectionDAG *G) const { 6409 // Don't print impossibly deep things. 6410 printrWithDepth(OS, G, 10); 6411} 6412 6413void SDNode::dumprWithDepth(const SelectionDAG *G, unsigned depth) const { 6414 printrWithDepth(dbgs(), G, depth); 6415} 6416 6417void SDNode::dumprFull(const SelectionDAG *G) const { 6418 // Don't print impossibly deep things. 6419 dumprWithDepth(G, 10); 6420} 6421 6422static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) { 6423 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6424 if (N->getOperand(i).getNode()->hasOneUse()) 6425 DumpNodes(N->getOperand(i).getNode(), indent+2, G); 6426 else 6427 dbgs() << "\n" << std::string(indent+2, ' ') 6428 << (void*)N->getOperand(i).getNode() << ": <multiple use>"; 6429 6430 6431 dbgs() << "\n"; 6432 dbgs().indent(indent); 6433 N->dump(G); 6434} 6435 6436SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6437 assert(N->getNumValues() == 1 && 6438 "Can't unroll a vector with multiple results!"); 6439 6440 EVT VT = N->getValueType(0); 6441 unsigned NE = VT.getVectorNumElements(); 6442 EVT EltVT = VT.getVectorElementType(); 6443 DebugLoc dl = N->getDebugLoc(); 6444 6445 SmallVector<SDValue, 8> Scalars; 6446 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6447 6448 // If ResNE is 0, fully unroll the vector op. 6449 if (ResNE == 0) 6450 ResNE = NE; 6451 else if (NE > ResNE) 6452 NE = ResNE; 6453 6454 unsigned i; 6455 for (i= 0; i != NE; ++i) { 6456 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6457 SDValue Operand = N->getOperand(j); 6458 EVT OperandVT = Operand.getValueType(); 6459 if (OperandVT.isVector()) { 6460 // A vector operand; extract a single element. 6461 EVT OperandEltVT = OperandVT.getVectorElementType(); 6462 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6463 OperandEltVT, 6464 Operand, 6465 getConstant(i, TLI.getPointerTy())); 6466 } else { 6467 // A scalar operand; just use it as is. 6468 Operands[j] = Operand; 6469 } 6470 } 6471 6472 switch (N->getOpcode()) { 6473 default: 6474 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6475 &Operands[0], Operands.size())); 6476 break; 6477 case ISD::VSELECT: 6478 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6479 &Operands[0], Operands.size())); 6480 break; 6481 case ISD::SHL: 6482 case ISD::SRA: 6483 case ISD::SRL: 6484 case ISD::ROTL: 6485 case ISD::ROTR: 6486 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6487 getShiftAmountOperand(Operands[0].getValueType(), 6488 Operands[1]))); 6489 break; 6490 case ISD::SIGN_EXTEND_INREG: 6491 case ISD::FP_ROUND_INREG: { 6492 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6493 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6494 Operands[0], 6495 getValueType(ExtVT))); 6496 } 6497 } 6498 } 6499 6500 for (; i < ResNE; ++i) 6501 Scalars.push_back(getUNDEF(EltVT)); 6502 6503 return getNode(ISD::BUILD_VECTOR, dl, 6504 EVT::getVectorVT(*getContext(), EltVT, ResNE), 6505 &Scalars[0], Scalars.size()); 6506} 6507 6508 6509/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6510/// location that is 'Dist' units away from the location that the 'Base' load 6511/// is loading from. 6512bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6513 unsigned Bytes, int Dist) const { 6514 if (LD->getChain() != Base->getChain()) 6515 return false; 6516 EVT VT = LD->getValueType(0); 6517 if (VT.getSizeInBits() / 8 != Bytes) 6518 return false; 6519 6520 SDValue Loc = LD->getOperand(1); 6521 SDValue BaseLoc = Base->getOperand(1); 6522 if (Loc.getOpcode() == ISD::FrameIndex) { 6523 if (BaseLoc.getOpcode() != ISD::FrameIndex) 6524 return false; 6525 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6526 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6527 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6528 int FS = MFI->getObjectSize(FI); 6529 int BFS = MFI->getObjectSize(BFI); 6530 if (FS != BFS || FS != (int)Bytes) return false; 6531 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6532 } 6533 6534 // Handle X+C 6535 if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6536 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6537 return true; 6538 6539 const GlobalValue *GV1 = NULL; 6540 const GlobalValue *GV2 = NULL; 6541 int64_t Offset1 = 0; 6542 int64_t Offset2 = 0; 6543 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6544 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6545 if (isGA1 && isGA2 && GV1 == GV2) 6546 return Offset1 == (Offset2 + Dist*Bytes); 6547 return false; 6548} 6549 6550 6551/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6552/// it cannot be inferred. 6553unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6554 // If this is a GlobalAddress + cst, return the alignment. 6555 const GlobalValue *GV; 6556 int64_t GVOffset = 0; 6557 if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6558 // If GV has specified alignment, then use it. Otherwise, use the preferred 6559 // alignment. 6560 unsigned Align = GV->getAlignment(); 6561 if (!Align) { 6562 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV)) { 6563 if (GVar->hasInitializer()) { 6564 const TargetData *TD = TLI.getTargetData(); 6565 Align = TD->getPreferredAlignment(GVar); 6566 } 6567 } 6568 if (!Align) 6569 Align = TLI.getTargetData()->getABITypeAlignment(GV->getType()); 6570 } 6571 return MinAlign(Align, GVOffset); 6572 } 6573 6574 // If this is a direct reference to a stack slot, use information about the 6575 // stack slot's alignment. 6576 int FrameIdx = 1 << 31; 6577 int64_t FrameOffset = 0; 6578 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6579 FrameIdx = FI->getIndex(); 6580 } else if (isBaseWithConstantOffset(Ptr) && 6581 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6582 // Handle FI+Cst 6583 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6584 FrameOffset = Ptr.getConstantOperandVal(1); 6585 } 6586 6587 if (FrameIdx != (1 << 31)) { 6588 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6589 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6590 FrameOffset); 6591 return FIInfoAlign; 6592 } 6593 6594 return 0; 6595} 6596 6597void SelectionDAG::dump() const { 6598 dbgs() << "SelectionDAG has " << AllNodes.size() << " nodes:"; 6599 6600 for (allnodes_const_iterator I = allnodes_begin(), E = allnodes_end(); 6601 I != E; ++I) { 6602 const SDNode *N = I; 6603 if (!N->hasOneUse() && N != getRoot().getNode()) 6604 DumpNodes(N, 2, this); 6605 } 6606 6607 if (getRoot().getNode()) DumpNodes(getRoot().getNode(), 2, this); 6608 6609 dbgs() << "\n\n"; 6610} 6611 6612void SDNode::printr(raw_ostream &OS, const SelectionDAG *G) const { 6613 print_types(OS, G); 6614 print_details(OS, G); 6615} 6616 6617typedef SmallPtrSet<const SDNode *, 128> VisitedSDNodeSet; 6618static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent, 6619 const SelectionDAG *G, VisitedSDNodeSet &once) { 6620 if (!once.insert(N)) // If we've been here before, return now. 6621 return; 6622 6623 // Dump the current SDNode, but don't end the line yet. 6624 OS.indent(indent); 6625 N->printr(OS, G); 6626 6627 // Having printed this SDNode, walk the children: 6628 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6629 const SDNode *child = N->getOperand(i).getNode(); 6630 6631 if (i) OS << ","; 6632 OS << " "; 6633 6634 if (child->getNumOperands() == 0) { 6635 // This child has no grandchildren; print it inline right here. 6636 child->printr(OS, G); 6637 once.insert(child); 6638 } else { // Just the address. FIXME: also print the child's opcode. 6639 OS << (void*)child; 6640 if (unsigned RN = N->getOperand(i).getResNo()) 6641 OS << ":" << RN; 6642 } 6643 } 6644 6645 OS << "\n"; 6646 6647 // Dump children that have grandchildren on their own line(s). 6648 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6649 const SDNode *child = N->getOperand(i).getNode(); 6650 DumpNodesr(OS, child, indent+2, G, once); 6651 } 6652} 6653 6654void SDNode::dumpr() const { 6655 VisitedSDNodeSet once; 6656 DumpNodesr(dbgs(), this, 0, 0, once); 6657} 6658 6659void SDNode::dumpr(const SelectionDAG *G) const { 6660 VisitedSDNodeSet once; 6661 DumpNodesr(dbgs(), this, 0, G, once); 6662} 6663 6664 6665// getAddressSpace - Return the address space this GlobalAddress belongs to. 6666unsigned GlobalAddressSDNode::getAddressSpace() const { 6667 return getGlobal()->getType()->getAddressSpace(); 6668} 6669 6670 6671Type *ConstantPoolSDNode::getType() const { 6672 if (isMachineConstantPoolEntry()) 6673 return Val.MachineCPVal->getType(); 6674 return Val.ConstVal->getType(); 6675} 6676 6677bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6678 APInt &SplatUndef, 6679 unsigned &SplatBitSize, 6680 bool &HasAnyUndefs, 6681 unsigned MinSplatBits, 6682 bool isBigEndian) { 6683 EVT VT = getValueType(0); 6684 assert(VT.isVector() && "Expected a vector type"); 6685 unsigned sz = VT.getSizeInBits(); 6686 if (MinSplatBits > sz) 6687 return false; 6688 6689 SplatValue = APInt(sz, 0); 6690 SplatUndef = APInt(sz, 0); 6691 6692 // Get the bits. Bits with undefined values (when the corresponding element 6693 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6694 // in SplatValue. If any of the values are not constant, give up and return 6695 // false. 6696 unsigned int nOps = getNumOperands(); 6697 assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6698 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6699 6700 for (unsigned j = 0; j < nOps; ++j) { 6701 unsigned i = isBigEndian ? nOps-1-j : j; 6702 SDValue OpVal = getOperand(i); 6703 unsigned BitPos = j * EltBitSize; 6704 6705 if (OpVal.getOpcode() == ISD::UNDEF) 6706 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6707 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6708 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6709 zextOrTrunc(sz) << BitPos; 6710 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6711 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6712 else 6713 return false; 6714 } 6715 6716 // The build_vector is all constants or undefs. Find the smallest element 6717 // size that splats the vector. 6718 6719 HasAnyUndefs = (SplatUndef != 0); 6720 while (sz > 8) { 6721 6722 unsigned HalfSize = sz / 2; 6723 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6724 APInt LowValue = SplatValue.trunc(HalfSize); 6725 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6726 APInt LowUndef = SplatUndef.trunc(HalfSize); 6727 6728 // If the two halves do not match (ignoring undef bits), stop here. 6729 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6730 MinSplatBits > HalfSize) 6731 break; 6732 6733 SplatValue = HighValue | LowValue; 6734 SplatUndef = HighUndef & LowUndef; 6735 6736 sz = HalfSize; 6737 } 6738 6739 SplatBitSize = sz; 6740 return true; 6741} 6742 6743bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6744 // Find the first non-undef value in the shuffle mask. 6745 unsigned i, e; 6746 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6747 /* search */; 6748 6749 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6750 6751 // Make sure all remaining elements are either undef or the same as the first 6752 // non-undef value. 6753 for (int Idx = Mask[i]; i != e; ++i) 6754 if (Mask[i] >= 0 && Mask[i] != Idx) 6755 return false; 6756 return true; 6757} 6758 6759#ifdef XDEBUG 6760static void checkForCyclesHelper(const SDNode *N, 6761 SmallPtrSet<const SDNode*, 32> &Visited, 6762 SmallPtrSet<const SDNode*, 32> &Checked) { 6763 // If this node has already been checked, don't check it again. 6764 if (Checked.count(N)) 6765 return; 6766 6767 // If a node has already been visited on this depth-first walk, reject it as 6768 // a cycle. 6769 if (!Visited.insert(N)) { 6770 dbgs() << "Offending node:\n"; 6771 N->dumprFull(); 6772 errs() << "Detected cycle in SelectionDAG\n"; 6773 abort(); 6774 } 6775 6776 for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6777 checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6778 6779 Checked.insert(N); 6780 Visited.erase(N); 6781} 6782#endif 6783 6784void llvm::checkForCycles(const llvm::SDNode *N) { 6785#ifdef XDEBUG 6786 assert(N && "Checking nonexistant SDNode"); 6787 SmallPtrSet<const SDNode*, 32> visited; 6788 SmallPtrSet<const SDNode*, 32> checked; 6789 checkForCyclesHelper(N, visited, checked); 6790#endif 6791} 6792 6793void llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6794 checkForCycles(DAG->getRoot().getNode()); 6795} 6796