SelectionDAG.cpp revision 799ea5ccc0a1d6b00f45191855056fb07cd641bc
1//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This implements the SelectionDAG class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "SDNodeOrdering.h" 16#include "SDNodeDbgValue.h" 17#include "llvm/Constants.h" 18#include "llvm/Analysis/DebugInfo.h" 19#include "llvm/Analysis/ValueTracking.h" 20#include "llvm/Function.h" 21#include "llvm/GlobalAlias.h" 22#include "llvm/GlobalVariable.h" 23#include "llvm/Intrinsics.h" 24#include "llvm/DerivedTypes.h" 25#include "llvm/Assembly/Writer.h" 26#include "llvm/CallingConv.h" 27#include "llvm/CodeGen/MachineBasicBlock.h" 28#include "llvm/CodeGen/MachineConstantPool.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineModuleInfo.h" 31#include "llvm/Target/TargetRegisterInfo.h" 32#include "llvm/Target/TargetData.h" 33#include "llvm/Target/TargetLowering.h" 34#include "llvm/Target/TargetSelectionDAGInfo.h" 35#include "llvm/Target/TargetOptions.h" 36#include "llvm/Target/TargetInstrInfo.h" 37#include "llvm/Target/TargetIntrinsicInfo.h" 38#include "llvm/Target/TargetMachine.h" 39#include "llvm/Support/CommandLine.h" 40#include "llvm/Support/Debug.h" 41#include "llvm/Support/ErrorHandling.h" 42#include "llvm/Support/ManagedStatic.h" 43#include "llvm/Support/MathExtras.h" 44#include "llvm/Support/raw_ostream.h" 45#include "llvm/Support/Mutex.h" 46#include "llvm/ADT/SetVector.h" 47#include "llvm/ADT/SmallPtrSet.h" 48#include "llvm/ADT/SmallSet.h" 49#include "llvm/ADT/SmallVector.h" 50#include "llvm/ADT/StringExtras.h" 51#include <algorithm> 52#include <cmath> 53using namespace llvm; 54 55/// makeVTList - Return an instance of the SDVTList struct initialized with the 56/// specified members. 57static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 58 SDVTList Res = {VTs, NumVTs}; 59 return Res; 60} 61 62static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { 63 switch (VT.getSimpleVT().SimpleTy) { 64 default: llvm_unreachable("Unknown FP format"); 65 case MVT::f32: return &APFloat::IEEEsingle; 66 case MVT::f64: return &APFloat::IEEEdouble; 67 case MVT::f80: return &APFloat::x87DoubleExtended; 68 case MVT::f128: return &APFloat::IEEEquad; 69 case MVT::ppcf128: return &APFloat::PPCDoubleDouble; 70 } 71} 72 73SelectionDAG::DAGUpdateListener::~DAGUpdateListener() {} 74 75//===----------------------------------------------------------------------===// 76// ConstantFPSDNode Class 77//===----------------------------------------------------------------------===// 78 79/// isExactlyValue - We don't rely on operator== working on double values, as 80/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 81/// As such, this method can be used to do an exact bit-for-bit comparison of 82/// two floating point values. 83bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 84 return getValueAPF().bitwiseIsEqual(V); 85} 86 87bool ConstantFPSDNode::isValueValidForType(EVT VT, 88 const APFloat& Val) { 89 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 90 91 // PPC long double cannot be converted to any other type. 92 if (VT == MVT::ppcf128 || 93 &Val.getSemantics() == &APFloat::PPCDoubleDouble) 94 return false; 95 96 // convert modifies in place, so make a copy. 97 APFloat Val2 = APFloat(Val); 98 bool losesInfo; 99 (void) Val2.convert(*EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, 100 &losesInfo); 101 return !losesInfo; 102} 103 104//===----------------------------------------------------------------------===// 105// ISD Namespace 106//===----------------------------------------------------------------------===// 107 108/// isBuildVectorAllOnes - Return true if the specified node is a 109/// BUILD_VECTOR where all of the elements are ~0 or undef. 110bool ISD::isBuildVectorAllOnes(const SDNode *N) { 111 // Look through a bit convert. 112 if (N->getOpcode() == ISD::BITCAST) 113 N = N->getOperand(0).getNode(); 114 115 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 116 117 unsigned i = 0, e = N->getNumOperands(); 118 119 // Skip over all of the undef values. 120 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 121 ++i; 122 123 // Do not accept an all-undef vector. 124 if (i == e) return false; 125 126 // Do not accept build_vectors that aren't all constants or which have non-~0 127 // elements. 128 SDValue NotZero = N->getOperand(i); 129 if (isa<ConstantSDNode>(NotZero)) { 130 if (!cast<ConstantSDNode>(NotZero)->isAllOnesValue()) 131 return false; 132 } else if (isa<ConstantFPSDNode>(NotZero)) { 133 if (!cast<ConstantFPSDNode>(NotZero)->getValueAPF(). 134 bitcastToAPInt().isAllOnesValue()) 135 return false; 136 } else 137 return false; 138 139 // Okay, we have at least one ~0 value, check to see if the rest match or are 140 // undefs. 141 for (++i; i != e; ++i) 142 if (N->getOperand(i) != NotZero && 143 N->getOperand(i).getOpcode() != ISD::UNDEF) 144 return false; 145 return true; 146} 147 148 149/// isBuildVectorAllZeros - Return true if the specified node is a 150/// BUILD_VECTOR where all of the elements are 0 or undef. 151bool ISD::isBuildVectorAllZeros(const SDNode *N) { 152 // Look through a bit convert. 153 if (N->getOpcode() == ISD::BITCAST) 154 N = N->getOperand(0).getNode(); 155 156 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 157 158 unsigned i = 0, e = N->getNumOperands(); 159 160 // Skip over all of the undef values. 161 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 162 ++i; 163 164 // Do not accept an all-undef vector. 165 if (i == e) return false; 166 167 // Do not accept build_vectors that aren't all constants or which have non-0 168 // elements. 169 SDValue Zero = N->getOperand(i); 170 if (isa<ConstantSDNode>(Zero)) { 171 if (!cast<ConstantSDNode>(Zero)->isNullValue()) 172 return false; 173 } else if (isa<ConstantFPSDNode>(Zero)) { 174 if (!cast<ConstantFPSDNode>(Zero)->getValueAPF().isPosZero()) 175 return false; 176 } else 177 return false; 178 179 // Okay, we have at least one 0 value, check to see if the rest match or are 180 // undefs. 181 for (++i; i != e; ++i) 182 if (N->getOperand(i) != Zero && 183 N->getOperand(i).getOpcode() != ISD::UNDEF) 184 return false; 185 return true; 186} 187 188/// isScalarToVector - Return true if the specified node is a 189/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 190/// element is not an undef. 191bool ISD::isScalarToVector(const SDNode *N) { 192 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 193 return true; 194 195 if (N->getOpcode() != ISD::BUILD_VECTOR) 196 return false; 197 if (N->getOperand(0).getOpcode() == ISD::UNDEF) 198 return false; 199 unsigned NumElems = N->getNumOperands(); 200 if (NumElems == 1) 201 return false; 202 for (unsigned i = 1; i < NumElems; ++i) { 203 SDValue V = N->getOperand(i); 204 if (V.getOpcode() != ISD::UNDEF) 205 return false; 206 } 207 return true; 208} 209 210/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 211/// when given the operation for (X op Y). 212ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 213 // To perform this operation, we just need to swap the L and G bits of the 214 // operation. 215 unsigned OldL = (Operation >> 2) & 1; 216 unsigned OldG = (Operation >> 1) & 1; 217 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 218 (OldL << 1) | // New G bit 219 (OldG << 2)); // New L bit. 220} 221 222/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 223/// 'op' is a valid SetCC operation. 224ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 225 unsigned Operation = Op; 226 if (isInteger) 227 Operation ^= 7; // Flip L, G, E bits, but not U. 228 else 229 Operation ^= 15; // Flip all of the condition bits. 230 231 if (Operation > ISD::SETTRUE2) 232 Operation &= ~8; // Don't let N and U bits get set. 233 234 return ISD::CondCode(Operation); 235} 236 237 238/// isSignedOp - For an integer comparison, return 1 if the comparison is a 239/// signed operation and 2 if the result is an unsigned comparison. Return zero 240/// if the operation does not depend on the sign of the input (setne and seteq). 241static int isSignedOp(ISD::CondCode Opcode) { 242 switch (Opcode) { 243 default: llvm_unreachable("Illegal integer setcc operation!"); 244 case ISD::SETEQ: 245 case ISD::SETNE: return 0; 246 case ISD::SETLT: 247 case ISD::SETLE: 248 case ISD::SETGT: 249 case ISD::SETGE: return 1; 250 case ISD::SETULT: 251 case ISD::SETULE: 252 case ISD::SETUGT: 253 case ISD::SETUGE: return 2; 254 } 255} 256 257/// getSetCCOrOperation - Return the result of a logical OR between different 258/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 259/// returns SETCC_INVALID if it is not possible to represent the resultant 260/// comparison. 261ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 262 bool isInteger) { 263 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 264 // Cannot fold a signed integer setcc with an unsigned integer setcc. 265 return ISD::SETCC_INVALID; 266 267 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 268 269 // If the N and U bits get set then the resultant comparison DOES suddenly 270 // care about orderedness, and is true when ordered. 271 if (Op > ISD::SETTRUE2) 272 Op &= ~16; // Clear the U bit if the N bit is set. 273 274 // Canonicalize illegal integer setcc's. 275 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 276 Op = ISD::SETNE; 277 278 return ISD::CondCode(Op); 279} 280 281/// getSetCCAndOperation - Return the result of a logical AND between different 282/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 283/// function returns zero if it is not possible to represent the resultant 284/// comparison. 285ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 286 bool isInteger) { 287 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 288 // Cannot fold a signed setcc with an unsigned setcc. 289 return ISD::SETCC_INVALID; 290 291 // Combine all of the condition bits. 292 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 293 294 // Canonicalize illegal integer setcc's. 295 if (isInteger) { 296 switch (Result) { 297 default: break; 298 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 299 case ISD::SETOEQ: // SETEQ & SETU[LG]E 300 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 301 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 302 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 303 } 304 } 305 306 return Result; 307} 308 309//===----------------------------------------------------------------------===// 310// SDNode Profile Support 311//===----------------------------------------------------------------------===// 312 313/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 314/// 315static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 316 ID.AddInteger(OpC); 317} 318 319/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 320/// solely with their pointer. 321static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 322 ID.AddPointer(VTList.VTs); 323} 324 325/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 326/// 327static void AddNodeIDOperands(FoldingSetNodeID &ID, 328 const SDValue *Ops, unsigned NumOps) { 329 for (; NumOps; --NumOps, ++Ops) { 330 ID.AddPointer(Ops->getNode()); 331 ID.AddInteger(Ops->getResNo()); 332 } 333} 334 335/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 336/// 337static void AddNodeIDOperands(FoldingSetNodeID &ID, 338 const SDUse *Ops, unsigned NumOps) { 339 for (; NumOps; --NumOps, ++Ops) { 340 ID.AddPointer(Ops->getNode()); 341 ID.AddInteger(Ops->getResNo()); 342 } 343} 344 345static void AddNodeIDNode(FoldingSetNodeID &ID, 346 unsigned short OpC, SDVTList VTList, 347 const SDValue *OpList, unsigned N) { 348 AddNodeIDOpcode(ID, OpC); 349 AddNodeIDValueTypes(ID, VTList); 350 AddNodeIDOperands(ID, OpList, N); 351} 352 353/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 354/// the NodeID data. 355static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 356 switch (N->getOpcode()) { 357 case ISD::TargetExternalSymbol: 358 case ISD::ExternalSymbol: 359 llvm_unreachable("Should only be used on nodes with operands"); 360 default: break; // Normal nodes don't need extra info. 361 case ISD::TargetConstant: 362 case ISD::Constant: 363 ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 364 break; 365 case ISD::TargetConstantFP: 366 case ISD::ConstantFP: { 367 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 368 break; 369 } 370 case ISD::TargetGlobalAddress: 371 case ISD::GlobalAddress: 372 case ISD::TargetGlobalTLSAddress: 373 case ISD::GlobalTLSAddress: { 374 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 375 ID.AddPointer(GA->getGlobal()); 376 ID.AddInteger(GA->getOffset()); 377 ID.AddInteger(GA->getTargetFlags()); 378 break; 379 } 380 case ISD::BasicBlock: 381 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 382 break; 383 case ISD::Register: 384 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 385 break; 386 387 case ISD::SRCVALUE: 388 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 389 break; 390 case ISD::FrameIndex: 391 case ISD::TargetFrameIndex: 392 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 393 break; 394 case ISD::JumpTable: 395 case ISD::TargetJumpTable: 396 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 397 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 398 break; 399 case ISD::ConstantPool: 400 case ISD::TargetConstantPool: { 401 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 402 ID.AddInteger(CP->getAlignment()); 403 ID.AddInteger(CP->getOffset()); 404 if (CP->isMachineConstantPoolEntry()) 405 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 406 else 407 ID.AddPointer(CP->getConstVal()); 408 ID.AddInteger(CP->getTargetFlags()); 409 break; 410 } 411 case ISD::LOAD: { 412 const LoadSDNode *LD = cast<LoadSDNode>(N); 413 ID.AddInteger(LD->getMemoryVT().getRawBits()); 414 ID.AddInteger(LD->getRawSubclassData()); 415 break; 416 } 417 case ISD::STORE: { 418 const StoreSDNode *ST = cast<StoreSDNode>(N); 419 ID.AddInteger(ST->getMemoryVT().getRawBits()); 420 ID.AddInteger(ST->getRawSubclassData()); 421 break; 422 } 423 case ISD::ATOMIC_CMP_SWAP: 424 case ISD::ATOMIC_SWAP: 425 case ISD::ATOMIC_LOAD_ADD: 426 case ISD::ATOMIC_LOAD_SUB: 427 case ISD::ATOMIC_LOAD_AND: 428 case ISD::ATOMIC_LOAD_OR: 429 case ISD::ATOMIC_LOAD_XOR: 430 case ISD::ATOMIC_LOAD_NAND: 431 case ISD::ATOMIC_LOAD_MIN: 432 case ISD::ATOMIC_LOAD_MAX: 433 case ISD::ATOMIC_LOAD_UMIN: 434 case ISD::ATOMIC_LOAD_UMAX: 435 case ISD::ATOMIC_LOAD: 436 case ISD::ATOMIC_STORE: { 437 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 438 ID.AddInteger(AT->getMemoryVT().getRawBits()); 439 ID.AddInteger(AT->getRawSubclassData()); 440 break; 441 } 442 case ISD::VECTOR_SHUFFLE: { 443 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 444 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 445 i != e; ++i) 446 ID.AddInteger(SVN->getMaskElt(i)); 447 break; 448 } 449 case ISD::TargetBlockAddress: 450 case ISD::BlockAddress: { 451 ID.AddPointer(cast<BlockAddressSDNode>(N)->getBlockAddress()); 452 ID.AddInteger(cast<BlockAddressSDNode>(N)->getTargetFlags()); 453 break; 454 } 455 } // end switch (N->getOpcode()) 456} 457 458/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 459/// data. 460static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 461 AddNodeIDOpcode(ID, N->getOpcode()); 462 // Add the return value info. 463 AddNodeIDValueTypes(ID, N->getVTList()); 464 // Add the operand info. 465 AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 466 467 // Handle SDNode leafs with special info. 468 AddNodeIDCustom(ID, N); 469} 470 471/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 472/// the CSE map that carries volatility, temporalness, indexing mode, and 473/// extension/truncation information. 474/// 475static inline unsigned 476encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 477 bool isNonTemporal, bool isInvariant) { 478 assert((ConvType & 3) == ConvType && 479 "ConvType may not require more than 2 bits!"); 480 assert((AM & 7) == AM && 481 "AM may not require more than 3 bits!"); 482 return ConvType | 483 (AM << 2) | 484 (isVolatile << 5) | 485 (isNonTemporal << 6) | 486 (isInvariant << 7); 487} 488 489//===----------------------------------------------------------------------===// 490// SelectionDAG Class 491//===----------------------------------------------------------------------===// 492 493/// doNotCSE - Return true if CSE should not be performed for this node. 494static bool doNotCSE(SDNode *N) { 495 if (N->getValueType(0) == MVT::Glue) 496 return true; // Never CSE anything that produces a flag. 497 498 switch (N->getOpcode()) { 499 default: break; 500 case ISD::HANDLENODE: 501 case ISD::EH_LABEL: 502 return true; // Never CSE these nodes. 503 } 504 505 // Check that remaining values produced are not flags. 506 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 507 if (N->getValueType(i) == MVT::Glue) 508 return true; // Never CSE anything that produces a flag. 509 510 return false; 511} 512 513/// RemoveDeadNodes - This method deletes all unreachable nodes in the 514/// SelectionDAG. 515void SelectionDAG::RemoveDeadNodes() { 516 // Create a dummy node (which is not added to allnodes), that adds a reference 517 // to the root node, preventing it from being deleted. 518 HandleSDNode Dummy(getRoot()); 519 520 SmallVector<SDNode*, 128> DeadNodes; 521 522 // Add all obviously-dead nodes to the DeadNodes worklist. 523 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 524 if (I->use_empty()) 525 DeadNodes.push_back(I); 526 527 RemoveDeadNodes(DeadNodes); 528 529 // If the root changed (e.g. it was a dead load, update the root). 530 setRoot(Dummy.getValue()); 531} 532 533/// RemoveDeadNodes - This method deletes the unreachable nodes in the 534/// given list, and any nodes that become unreachable as a result. 535void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes, 536 DAGUpdateListener *UpdateListener) { 537 538 // Process the worklist, deleting the nodes and adding their uses to the 539 // worklist. 540 while (!DeadNodes.empty()) { 541 SDNode *N = DeadNodes.pop_back_val(); 542 543 if (UpdateListener) 544 UpdateListener->NodeDeleted(N, 0); 545 546 // Take the node out of the appropriate CSE map. 547 RemoveNodeFromCSEMaps(N); 548 549 // Next, brutally remove the operand list. This is safe to do, as there are 550 // no cycles in the graph. 551 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 552 SDUse &Use = *I++; 553 SDNode *Operand = Use.getNode(); 554 Use.set(SDValue()); 555 556 // Now that we removed this operand, see if there are no uses of it left. 557 if (Operand->use_empty()) 558 DeadNodes.push_back(Operand); 559 } 560 561 DeallocateNode(N); 562 } 563} 564 565void SelectionDAG::RemoveDeadNode(SDNode *N, DAGUpdateListener *UpdateListener){ 566 SmallVector<SDNode*, 16> DeadNodes(1, N); 567 568 // Create a dummy node that adds a reference to the root node, preventing 569 // it from being deleted. (This matters if the root is an operand of the 570 // dead node.) 571 HandleSDNode Dummy(getRoot()); 572 573 RemoveDeadNodes(DeadNodes, UpdateListener); 574} 575 576void SelectionDAG::DeleteNode(SDNode *N) { 577 // First take this out of the appropriate CSE map. 578 RemoveNodeFromCSEMaps(N); 579 580 // Finally, remove uses due to operands of this node, remove from the 581 // AllNodes list, and delete the node. 582 DeleteNodeNotInCSEMaps(N); 583} 584 585void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 586 assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 587 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 588 589 // Drop all of the operands and decrement used node's use counts. 590 N->DropOperands(); 591 592 DeallocateNode(N); 593} 594 595void SelectionDAG::DeallocateNode(SDNode *N) { 596 if (N->OperandsNeedDelete) 597 delete[] N->OperandList; 598 599 // Set the opcode to DELETED_NODE to help catch bugs when node 600 // memory is reallocated. 601 N->NodeType = ISD::DELETED_NODE; 602 603 NodeAllocator.Deallocate(AllNodes.remove(N)); 604 605 // Remove the ordering of this node. 606 Ordering->remove(N); 607 608 // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. 609 ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); 610 for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) 611 DbgVals[i]->setIsInvalidated(); 612} 613 614/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 615/// correspond to it. This is useful when we're about to delete or repurpose 616/// the node. We don't want future request for structurally identical nodes 617/// to return N anymore. 618bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 619 bool Erased = false; 620 switch (N->getOpcode()) { 621 case ISD::HANDLENODE: return false; // noop. 622 case ISD::CONDCODE: 623 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 624 "Cond code doesn't exist!"); 625 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 626 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 627 break; 628 case ISD::ExternalSymbol: 629 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 630 break; 631 case ISD::TargetExternalSymbol: { 632 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 633 Erased = TargetExternalSymbols.erase( 634 std::pair<std::string,unsigned char>(ESN->getSymbol(), 635 ESN->getTargetFlags())); 636 break; 637 } 638 case ISD::VALUETYPE: { 639 EVT VT = cast<VTSDNode>(N)->getVT(); 640 if (VT.isExtended()) { 641 Erased = ExtendedValueTypeNodes.erase(VT); 642 } else { 643 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 644 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 645 } 646 break; 647 } 648 default: 649 // Remove it from the CSE Map. 650 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 651 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 652 Erased = CSEMap.RemoveNode(N); 653 break; 654 } 655#ifndef NDEBUG 656 // Verify that the node was actually in one of the CSE maps, unless it has a 657 // flag result (which cannot be CSE'd) or is one of the special cases that are 658 // not subject to CSE. 659 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 660 !N->isMachineOpcode() && !doNotCSE(N)) { 661 N->dump(this); 662 dbgs() << "\n"; 663 llvm_unreachable("Node is not in map!"); 664 } 665#endif 666 return Erased; 667} 668 669/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 670/// maps and modified in place. Add it back to the CSE maps, unless an identical 671/// node already exists, in which case transfer all its users to the existing 672/// node. This transfer can potentially trigger recursive merging. 673/// 674void 675SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N, 676 DAGUpdateListener *UpdateListener) { 677 // For node types that aren't CSE'd, just act as if no identical node 678 // already exists. 679 if (!doNotCSE(N)) { 680 SDNode *Existing = CSEMap.GetOrInsertNode(N); 681 if (Existing != N) { 682 // If there was already an existing matching node, use ReplaceAllUsesWith 683 // to replace the dead one with the existing one. This can cause 684 // recursive merging of other unrelated nodes down the line. 685 ReplaceAllUsesWith(N, Existing, UpdateListener); 686 687 // N is now dead. Inform the listener if it exists and delete it. 688 if (UpdateListener) 689 UpdateListener->NodeDeleted(N, Existing); 690 DeleteNodeNotInCSEMaps(N); 691 return; 692 } 693 } 694 695 // If the node doesn't already exist, we updated it. Inform a listener if 696 // it exists. 697 if (UpdateListener) 698 UpdateListener->NodeUpdated(N); 699} 700 701/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 702/// were replaced with those specified. If this node is never memoized, 703/// return null, otherwise return a pointer to the slot it would take. If a 704/// node already exists with these operands, the slot will be non-null. 705SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 706 void *&InsertPos) { 707 if (doNotCSE(N)) 708 return 0; 709 710 SDValue Ops[] = { Op }; 711 FoldingSetNodeID ID; 712 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 713 AddNodeIDCustom(ID, N); 714 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 715 return Node; 716} 717 718/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 719/// were replaced with those specified. If this node is never memoized, 720/// return null, otherwise return a pointer to the slot it would take. If a 721/// node already exists with these operands, the slot will be non-null. 722SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 723 SDValue Op1, SDValue Op2, 724 void *&InsertPos) { 725 if (doNotCSE(N)) 726 return 0; 727 728 SDValue Ops[] = { Op1, Op2 }; 729 FoldingSetNodeID ID; 730 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 731 AddNodeIDCustom(ID, N); 732 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 733 return Node; 734} 735 736 737/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 738/// were replaced with those specified. If this node is never memoized, 739/// return null, otherwise return a pointer to the slot it would take. If a 740/// node already exists with these operands, the slot will be non-null. 741SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 742 const SDValue *Ops,unsigned NumOps, 743 void *&InsertPos) { 744 if (doNotCSE(N)) 745 return 0; 746 747 FoldingSetNodeID ID; 748 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 749 AddNodeIDCustom(ID, N); 750 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 751 return Node; 752} 753 754#ifndef NDEBUG 755/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 756static void VerifyNodeCommon(SDNode *N) { 757 switch (N->getOpcode()) { 758 default: 759 break; 760 case ISD::BUILD_PAIR: { 761 EVT VT = N->getValueType(0); 762 assert(N->getNumValues() == 1 && "Too many results!"); 763 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 764 "Wrong return type!"); 765 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 766 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 767 "Mismatched operand types!"); 768 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 769 "Wrong operand type!"); 770 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 771 "Wrong return type size"); 772 break; 773 } 774 case ISD::BUILD_VECTOR: { 775 assert(N->getNumValues() == 1 && "Too many results!"); 776 assert(N->getValueType(0).isVector() && "Wrong return type!"); 777 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 778 "Wrong number of operands!"); 779 EVT EltVT = N->getValueType(0).getVectorElementType(); 780 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 781 assert((I->getValueType() == EltVT || 782 (EltVT.isInteger() && I->getValueType().isInteger() && 783 EltVT.bitsLE(I->getValueType()))) && 784 "Wrong operand type!"); 785 assert(I->getValueType() == N->getOperand(0).getValueType() && 786 "Operands must all have the same type"); 787 } 788 break; 789 } 790 } 791} 792 793/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 794static void VerifySDNode(SDNode *N) { 795 // The SDNode allocators cannot be used to allocate nodes with fields that are 796 // not present in an SDNode! 797 assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 798 assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 799 assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 800 assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 801 assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 802 assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 803 assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 804 assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 805 assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 806 assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 807 assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 808 assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 809 assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 810 assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 811 assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 812 assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 813 assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 814 assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 815 assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 816 817 VerifyNodeCommon(N); 818} 819 820/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 821/// invalid. 822static void VerifyMachineNode(SDNode *N) { 823 // The MachineNode allocators cannot be used to allocate nodes with fields 824 // that are not present in a MachineNode! 825 // Currently there are no such nodes. 826 827 VerifyNodeCommon(N); 828} 829#endif // NDEBUG 830 831/// getEVTAlignment - Compute the default alignment value for the 832/// given type. 833/// 834unsigned SelectionDAG::getEVTAlignment(EVT VT) const { 835 Type *Ty = VT == MVT::iPTR ? 836 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 837 VT.getTypeForEVT(*getContext()); 838 839 return TLI.getTargetData()->getABITypeAlignment(Ty); 840} 841 842// EntryNode could meaningfully have debug info if we can find it... 843SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 844 : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), 845 OptLevel(OL), EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), 846 Root(getEntryNode()), Ordering(0) { 847 AllNodes.push_back(&EntryNode); 848 Ordering = new SDNodeOrdering(); 849 DbgInfo = new SDDbgInfo(); 850} 851 852void SelectionDAG::init(MachineFunction &mf) { 853 MF = &mf; 854 Context = &mf.getFunction()->getContext(); 855} 856 857SelectionDAG::~SelectionDAG() { 858 allnodes_clear(); 859 delete Ordering; 860 delete DbgInfo; 861} 862 863void SelectionDAG::allnodes_clear() { 864 assert(&*AllNodes.begin() == &EntryNode); 865 AllNodes.remove(AllNodes.begin()); 866 while (!AllNodes.empty()) 867 DeallocateNode(AllNodes.begin()); 868} 869 870void SelectionDAG::clear() { 871 allnodes_clear(); 872 OperandAllocator.Reset(); 873 CSEMap.clear(); 874 875 ExtendedValueTypeNodes.clear(); 876 ExternalSymbols.clear(); 877 TargetExternalSymbols.clear(); 878 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 879 static_cast<CondCodeSDNode*>(0)); 880 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 881 static_cast<SDNode*>(0)); 882 883 EntryNode.UseList = 0; 884 AllNodes.push_back(&EntryNode); 885 Root = getEntryNode(); 886 Ordering->clear(); 887 DbgInfo->clear(); 888} 889 890SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 891 return VT.bitsGT(Op.getValueType()) ? 892 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 893 getNode(ISD::TRUNCATE, DL, VT, Op); 894} 895 896SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 897 return VT.bitsGT(Op.getValueType()) ? 898 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 899 getNode(ISD::TRUNCATE, DL, VT, Op); 900} 901 902SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 903 return VT.bitsGT(Op.getValueType()) ? 904 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 905 getNode(ISD::TRUNCATE, DL, VT, Op); 906} 907 908SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, DebugLoc DL, EVT VT) { 909 assert(!VT.isVector() && 910 "getZeroExtendInReg should use the vector element type instead of " 911 "the vector type!"); 912 if (Op.getValueType() == VT) return Op; 913 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 914 APInt Imm = APInt::getLowBitsSet(BitWidth, 915 VT.getSizeInBits()); 916 return getNode(ISD::AND, DL, Op.getValueType(), Op, 917 getConstant(Imm, Op.getValueType())); 918} 919 920/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 921/// 922SDValue SelectionDAG::getNOT(DebugLoc DL, SDValue Val, EVT VT) { 923 EVT EltVT = VT.getScalarType(); 924 SDValue NegOne = 925 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 926 return getNode(ISD::XOR, DL, VT, Val, NegOne); 927} 928 929SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 930 EVT EltVT = VT.getScalarType(); 931 assert((EltVT.getSizeInBits() >= 64 || 932 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 933 "getConstant with a uint64_t value that doesn't fit in the type!"); 934 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 935} 936 937SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 938 return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 939} 940 941SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 942 assert(VT.isInteger() && "Cannot create FP integer constant!"); 943 944 EVT EltVT = VT.getScalarType(); 945 const ConstantInt *Elt = &Val; 946 947 // In some cases the vector type is legal but the element type is illegal and 948 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 949 // inserted value (the type does not need to match the vector element type). 950 // Any extra bits introduced will be truncated away. 951 if (VT.isVector() && TLI.getTypeAction(*getContext(), EltVT) == 952 TargetLowering::TypePromoteInteger) { 953 EltVT = TLI.getTypeToTransformTo(*getContext(), EltVT); 954 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 955 Elt = ConstantInt::get(*getContext(), NewVal); 956 } 957 958 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 959 "APInt size does not match type size!"); 960 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 961 FoldingSetNodeID ID; 962 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 963 ID.AddPointer(Elt); 964 void *IP = 0; 965 SDNode *N = NULL; 966 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 967 if (!VT.isVector()) 968 return SDValue(N, 0); 969 970 if (!N) { 971 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 972 CSEMap.InsertNode(N, IP); 973 AllNodes.push_back(N); 974 } 975 976 SDValue Result(N, 0); 977 if (VT.isVector()) { 978 SmallVector<SDValue, 8> Ops; 979 Ops.assign(VT.getVectorNumElements(), Result); 980 Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); 981 } 982 return Result; 983} 984 985SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 986 return getConstant(Val, TLI.getPointerTy(), isTarget); 987} 988 989 990SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 991 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 992} 993 994SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 995 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 996 997 EVT EltVT = VT.getScalarType(); 998 999 // Do the map lookup using the actual bit pattern for the floating point 1000 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1001 // we don't have issues with SNANs. 1002 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1003 FoldingSetNodeID ID; 1004 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1005 ID.AddPointer(&V); 1006 void *IP = 0; 1007 SDNode *N = NULL; 1008 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1009 if (!VT.isVector()) 1010 return SDValue(N, 0); 1011 1012 if (!N) { 1013 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1014 CSEMap.InsertNode(N, IP); 1015 AllNodes.push_back(N); 1016 } 1017 1018 SDValue Result(N, 0); 1019 if (VT.isVector()) { 1020 SmallVector<SDValue, 8> Ops; 1021 Ops.assign(VT.getVectorNumElements(), Result); 1022 // FIXME DebugLoc info might be appropriate here 1023 Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); 1024 } 1025 return Result; 1026} 1027 1028SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1029 EVT EltVT = VT.getScalarType(); 1030 if (EltVT==MVT::f32) 1031 return getConstantFP(APFloat((float)Val), VT, isTarget); 1032 else if (EltVT==MVT::f64) 1033 return getConstantFP(APFloat(Val), VT, isTarget); 1034 else if (EltVT==MVT::f80 || EltVT==MVT::f128) { 1035 bool ignored; 1036 APFloat apf = APFloat(Val); 1037 apf.convert(*EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1038 &ignored); 1039 return getConstantFP(apf, VT, isTarget); 1040 } else { 1041 assert(0 && "Unsupported type in getConstantFP"); 1042 return SDValue(); 1043 } 1044} 1045 1046SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, DebugLoc DL, 1047 EVT VT, int64_t Offset, 1048 bool isTargetGA, 1049 unsigned char TargetFlags) { 1050 assert((TargetFlags == 0 || isTargetGA) && 1051 "Cannot set target flags on target-independent globals"); 1052 1053 // Truncate (with sign-extension) the offset value to the pointer size. 1054 EVT PTy = TLI.getPointerTy(); 1055 unsigned BitWidth = PTy.getSizeInBits(); 1056 if (BitWidth < 64) 1057 Offset = (Offset << (64 - BitWidth) >> (64 - BitWidth)); 1058 1059 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1060 if (!GVar) { 1061 // If GV is an alias then use the aliasee for determining thread-localness. 1062 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1063 GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1064 } 1065 1066 unsigned Opc; 1067 if (GVar && GVar->isThreadLocal()) 1068 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1069 else 1070 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1071 1072 FoldingSetNodeID ID; 1073 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1074 ID.AddPointer(GV); 1075 ID.AddInteger(Offset); 1076 ID.AddInteger(TargetFlags); 1077 void *IP = 0; 1078 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1079 return SDValue(E, 0); 1080 1081 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL, GV, VT, 1082 Offset, TargetFlags); 1083 CSEMap.InsertNode(N, IP); 1084 AllNodes.push_back(N); 1085 return SDValue(N, 0); 1086} 1087 1088SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1089 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1090 FoldingSetNodeID ID; 1091 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1092 ID.AddInteger(FI); 1093 void *IP = 0; 1094 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1095 return SDValue(E, 0); 1096 1097 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1098 CSEMap.InsertNode(N, IP); 1099 AllNodes.push_back(N); 1100 return SDValue(N, 0); 1101} 1102 1103SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1104 unsigned char TargetFlags) { 1105 assert((TargetFlags == 0 || isTarget) && 1106 "Cannot set target flags on target-independent jump tables"); 1107 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1108 FoldingSetNodeID ID; 1109 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1110 ID.AddInteger(JTI); 1111 ID.AddInteger(TargetFlags); 1112 void *IP = 0; 1113 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1114 return SDValue(E, 0); 1115 1116 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1117 TargetFlags); 1118 CSEMap.InsertNode(N, IP); 1119 AllNodes.push_back(N); 1120 return SDValue(N, 0); 1121} 1122 1123SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1124 unsigned Alignment, int Offset, 1125 bool isTarget, 1126 unsigned char TargetFlags) { 1127 assert((TargetFlags == 0 || isTarget) && 1128 "Cannot set target flags on target-independent globals"); 1129 if (Alignment == 0) 1130 Alignment = TLI.getTargetData()->getPrefTypeAlignment(C->getType()); 1131 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1132 FoldingSetNodeID ID; 1133 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1134 ID.AddInteger(Alignment); 1135 ID.AddInteger(Offset); 1136 ID.AddPointer(C); 1137 ID.AddInteger(TargetFlags); 1138 void *IP = 0; 1139 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1140 return SDValue(E, 0); 1141 1142 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1143 Alignment, TargetFlags); 1144 CSEMap.InsertNode(N, IP); 1145 AllNodes.push_back(N); 1146 return SDValue(N, 0); 1147} 1148 1149 1150SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1151 unsigned Alignment, int Offset, 1152 bool isTarget, 1153 unsigned char TargetFlags) { 1154 assert((TargetFlags == 0 || isTarget) && 1155 "Cannot set target flags on target-independent globals"); 1156 if (Alignment == 0) 1157 Alignment = TLI.getTargetData()->getPrefTypeAlignment(C->getType()); 1158 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1159 FoldingSetNodeID ID; 1160 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1161 ID.AddInteger(Alignment); 1162 ID.AddInteger(Offset); 1163 C->addSelectionDAGCSEId(ID); 1164 ID.AddInteger(TargetFlags); 1165 void *IP = 0; 1166 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1167 return SDValue(E, 0); 1168 1169 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1170 Alignment, TargetFlags); 1171 CSEMap.InsertNode(N, IP); 1172 AllNodes.push_back(N); 1173 return SDValue(N, 0); 1174} 1175 1176SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1177 FoldingSetNodeID ID; 1178 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1179 ID.AddPointer(MBB); 1180 void *IP = 0; 1181 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1182 return SDValue(E, 0); 1183 1184 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1185 CSEMap.InsertNode(N, IP); 1186 AllNodes.push_back(N); 1187 return SDValue(N, 0); 1188} 1189 1190SDValue SelectionDAG::getValueType(EVT VT) { 1191 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1192 ValueTypeNodes.size()) 1193 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1194 1195 SDNode *&N = VT.isExtended() ? 1196 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1197 1198 if (N) return SDValue(N, 0); 1199 N = new (NodeAllocator) VTSDNode(VT); 1200 AllNodes.push_back(N); 1201 return SDValue(N, 0); 1202} 1203 1204SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1205 SDNode *&N = ExternalSymbols[Sym]; 1206 if (N) return SDValue(N, 0); 1207 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1208 AllNodes.push_back(N); 1209 return SDValue(N, 0); 1210} 1211 1212SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1213 unsigned char TargetFlags) { 1214 SDNode *&N = 1215 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1216 TargetFlags)]; 1217 if (N) return SDValue(N, 0); 1218 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1219 AllNodes.push_back(N); 1220 return SDValue(N, 0); 1221} 1222 1223SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1224 if ((unsigned)Cond >= CondCodeNodes.size()) 1225 CondCodeNodes.resize(Cond+1); 1226 1227 if (CondCodeNodes[Cond] == 0) { 1228 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1229 CondCodeNodes[Cond] = N; 1230 AllNodes.push_back(N); 1231 } 1232 1233 return SDValue(CondCodeNodes[Cond], 0); 1234} 1235 1236// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1237// the shuffle mask M that point at N1 to point at N2, and indices that point 1238// N2 to point at N1. 1239static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1240 std::swap(N1, N2); 1241 int NElts = M.size(); 1242 for (int i = 0; i != NElts; ++i) { 1243 if (M[i] >= NElts) 1244 M[i] -= NElts; 1245 else if (M[i] >= 0) 1246 M[i] += NElts; 1247 } 1248} 1249 1250SDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1, 1251 SDValue N2, const int *Mask) { 1252 assert(N1.getValueType() == N2.getValueType() && "Invalid VECTOR_SHUFFLE"); 1253 assert(VT.isVector() && N1.getValueType().isVector() && 1254 "Vector Shuffle VTs must be a vectors"); 1255 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() 1256 && "Vector Shuffle VTs must have same element type"); 1257 1258 // Canonicalize shuffle undef, undef -> undef 1259 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1260 return getUNDEF(VT); 1261 1262 // Validate that all indices in Mask are within the range of the elements 1263 // input to the shuffle. 1264 unsigned NElts = VT.getVectorNumElements(); 1265 SmallVector<int, 8> MaskVec; 1266 for (unsigned i = 0; i != NElts; ++i) { 1267 assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1268 MaskVec.push_back(Mask[i]); 1269 } 1270 1271 // Canonicalize shuffle v, v -> v, undef 1272 if (N1 == N2) { 1273 N2 = getUNDEF(VT); 1274 for (unsigned i = 0; i != NElts; ++i) 1275 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1276 } 1277 1278 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1279 if (N1.getOpcode() == ISD::UNDEF) 1280 commuteShuffle(N1, N2, MaskVec); 1281 1282 // Canonicalize all index into lhs, -> shuffle lhs, undef 1283 // Canonicalize all index into rhs, -> shuffle rhs, undef 1284 bool AllLHS = true, AllRHS = true; 1285 bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1286 for (unsigned i = 0; i != NElts; ++i) { 1287 if (MaskVec[i] >= (int)NElts) { 1288 if (N2Undef) 1289 MaskVec[i] = -1; 1290 else 1291 AllLHS = false; 1292 } else if (MaskVec[i] >= 0) { 1293 AllRHS = false; 1294 } 1295 } 1296 if (AllLHS && AllRHS) 1297 return getUNDEF(VT); 1298 if (AllLHS && !N2Undef) 1299 N2 = getUNDEF(VT); 1300 if (AllRHS) { 1301 N1 = getUNDEF(VT); 1302 commuteShuffle(N1, N2, MaskVec); 1303 } 1304 1305 // If Identity shuffle, or all shuffle in to undef, return that node. 1306 bool AllUndef = true; 1307 bool Identity = true; 1308 for (unsigned i = 0; i != NElts; ++i) { 1309 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1310 if (MaskVec[i] >= 0) AllUndef = false; 1311 } 1312 if (Identity && NElts == N1.getValueType().getVectorNumElements()) 1313 return N1; 1314 if (AllUndef) 1315 return getUNDEF(VT); 1316 1317 FoldingSetNodeID ID; 1318 SDValue Ops[2] = { N1, N2 }; 1319 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1320 for (unsigned i = 0; i != NElts; ++i) 1321 ID.AddInteger(MaskVec[i]); 1322 1323 void* IP = 0; 1324 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1325 return SDValue(E, 0); 1326 1327 // Allocate the mask array for the node out of the BumpPtrAllocator, since 1328 // SDNode doesn't have access to it. This memory will be "leaked" when 1329 // the node is deallocated, but recovered when the NodeAllocator is released. 1330 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1331 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1332 1333 ShuffleVectorSDNode *N = 1334 new (NodeAllocator) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc); 1335 CSEMap.InsertNode(N, IP); 1336 AllNodes.push_back(N); 1337 return SDValue(N, 0); 1338} 1339 1340SDValue SelectionDAG::getConvertRndSat(EVT VT, DebugLoc dl, 1341 SDValue Val, SDValue DTy, 1342 SDValue STy, SDValue Rnd, SDValue Sat, 1343 ISD::CvtCode Code) { 1344 // If the src and dest types are the same and the conversion is between 1345 // integer types of the same sign or two floats, no conversion is necessary. 1346 if (DTy == STy && 1347 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1348 return Val; 1349 1350 FoldingSetNodeID ID; 1351 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1352 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1353 void* IP = 0; 1354 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1355 return SDValue(E, 0); 1356 1357 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl, Ops, 5, 1358 Code); 1359 CSEMap.InsertNode(N, IP); 1360 AllNodes.push_back(N); 1361 return SDValue(N, 0); 1362} 1363 1364SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1365 FoldingSetNodeID ID; 1366 AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1367 ID.AddInteger(RegNo); 1368 void *IP = 0; 1369 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1370 return SDValue(E, 0); 1371 1372 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1373 CSEMap.InsertNode(N, IP); 1374 AllNodes.push_back(N); 1375 return SDValue(N, 0); 1376} 1377 1378SDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) { 1379 FoldingSetNodeID ID; 1380 SDValue Ops[] = { Root }; 1381 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1382 ID.AddPointer(Label); 1383 void *IP = 0; 1384 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1385 return SDValue(E, 0); 1386 1387 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl, Root, Label); 1388 CSEMap.InsertNode(N, IP); 1389 AllNodes.push_back(N); 1390 return SDValue(N, 0); 1391} 1392 1393 1394SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1395 bool isTarget, 1396 unsigned char TargetFlags) { 1397 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1398 1399 FoldingSetNodeID ID; 1400 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1401 ID.AddPointer(BA); 1402 ID.AddInteger(TargetFlags); 1403 void *IP = 0; 1404 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1405 return SDValue(E, 0); 1406 1407 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, TargetFlags); 1408 CSEMap.InsertNode(N, IP); 1409 AllNodes.push_back(N); 1410 return SDValue(N, 0); 1411} 1412 1413SDValue SelectionDAG::getSrcValue(const Value *V) { 1414 assert((!V || V->getType()->isPointerTy()) && 1415 "SrcValue is not a pointer?"); 1416 1417 FoldingSetNodeID ID; 1418 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1419 ID.AddPointer(V); 1420 1421 void *IP = 0; 1422 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1423 return SDValue(E, 0); 1424 1425 SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1426 CSEMap.InsertNode(N, IP); 1427 AllNodes.push_back(N); 1428 return SDValue(N, 0); 1429} 1430 1431/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1432SDValue SelectionDAG::getMDNode(const MDNode *MD) { 1433 FoldingSetNodeID ID; 1434 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1435 ID.AddPointer(MD); 1436 1437 void *IP = 0; 1438 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1439 return SDValue(E, 0); 1440 1441 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1442 CSEMap.InsertNode(N, IP); 1443 AllNodes.push_back(N); 1444 return SDValue(N, 0); 1445} 1446 1447 1448/// getShiftAmountOperand - Return the specified value casted to 1449/// the target's desired shift amount type. 1450SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1451 EVT OpTy = Op.getValueType(); 1452 MVT ShTy = TLI.getShiftAmountTy(LHSTy); 1453 if (OpTy == ShTy || OpTy.isVector()) return Op; 1454 1455 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1456 return getNode(Opcode, Op.getDebugLoc(), ShTy, Op); 1457} 1458 1459/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1460/// specified value type. 1461SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1462 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1463 unsigned ByteSize = VT.getStoreSize(); 1464 Type *Ty = VT.getTypeForEVT(*getContext()); 1465 unsigned StackAlign = 1466 std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign); 1467 1468 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1469 return getFrameIndex(FrameIdx, TLI.getPointerTy()); 1470} 1471 1472/// CreateStackTemporary - Create a stack temporary suitable for holding 1473/// either of the specified value types. 1474SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1475 unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1476 VT2.getStoreSizeInBits())/8; 1477 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1478 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1479 const TargetData *TD = TLI.getTargetData(); 1480 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1481 TD->getPrefTypeAlignment(Ty2)); 1482 1483 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1484 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1485 return getFrameIndex(FrameIdx, TLI.getPointerTy()); 1486} 1487 1488SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1489 SDValue N2, ISD::CondCode Cond, DebugLoc dl) { 1490 // These setcc operations always fold. 1491 switch (Cond) { 1492 default: break; 1493 case ISD::SETFALSE: 1494 case ISD::SETFALSE2: return getConstant(0, VT); 1495 case ISD::SETTRUE: 1496 case ISD::SETTRUE2: return getConstant(1, VT); 1497 1498 case ISD::SETOEQ: 1499 case ISD::SETOGT: 1500 case ISD::SETOGE: 1501 case ISD::SETOLT: 1502 case ISD::SETOLE: 1503 case ISD::SETONE: 1504 case ISD::SETO: 1505 case ISD::SETUO: 1506 case ISD::SETUEQ: 1507 case ISD::SETUNE: 1508 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1509 break; 1510 } 1511 1512 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1513 const APInt &C2 = N2C->getAPIntValue(); 1514 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1515 const APInt &C1 = N1C->getAPIntValue(); 1516 1517 switch (Cond) { 1518 default: llvm_unreachable("Unknown integer setcc!"); 1519 case ISD::SETEQ: return getConstant(C1 == C2, VT); 1520 case ISD::SETNE: return getConstant(C1 != C2, VT); 1521 case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1522 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1523 case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1524 case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1525 case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1526 case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1527 case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1528 case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1529 } 1530 } 1531 } 1532 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1533 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1534 // No compile time operations on this type yet. 1535 if (N1C->getValueType(0) == MVT::ppcf128) 1536 return SDValue(); 1537 1538 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1539 switch (Cond) { 1540 default: break; 1541 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1542 return getUNDEF(VT); 1543 // fall through 1544 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1545 case ISD::SETNE: if (R==APFloat::cmpUnordered) 1546 return getUNDEF(VT); 1547 // fall through 1548 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1549 R==APFloat::cmpLessThan, VT); 1550 case ISD::SETLT: if (R==APFloat::cmpUnordered) 1551 return getUNDEF(VT); 1552 // fall through 1553 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1554 case ISD::SETGT: if (R==APFloat::cmpUnordered) 1555 return getUNDEF(VT); 1556 // fall through 1557 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1558 case ISD::SETLE: if (R==APFloat::cmpUnordered) 1559 return getUNDEF(VT); 1560 // fall through 1561 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1562 R==APFloat::cmpEqual, VT); 1563 case ISD::SETGE: if (R==APFloat::cmpUnordered) 1564 return getUNDEF(VT); 1565 // fall through 1566 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1567 R==APFloat::cmpEqual, VT); 1568 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1569 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1570 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1571 R==APFloat::cmpEqual, VT); 1572 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1573 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1574 R==APFloat::cmpLessThan, VT); 1575 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1576 R==APFloat::cmpUnordered, VT); 1577 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1578 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1579 } 1580 } else { 1581 // Ensure that the constant occurs on the RHS. 1582 return getSetCC(dl, VT, N2, N1, ISD::getSetCCSwappedOperands(Cond)); 1583 } 1584 } 1585 1586 // Could not fold it. 1587 return SDValue(); 1588} 1589 1590/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1591/// use this predicate to simplify operations downstream. 1592bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1593 // This predicate is not safe for vector operations. 1594 if (Op.getValueType().isVector()) 1595 return false; 1596 1597 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1598 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1599} 1600 1601/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1602/// this predicate to simplify operations downstream. Mask is known to be zero 1603/// for bits that V cannot have. 1604bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1605 unsigned Depth) const { 1606 APInt KnownZero, KnownOne; 1607 ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth); 1608 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1609 return (KnownZero & Mask) == Mask; 1610} 1611 1612/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1613/// known to be either zero or one and return them in the KnownZero/KnownOne 1614/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1615/// processing. 1616void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, 1617 APInt &KnownZero, APInt &KnownOne, 1618 unsigned Depth) const { 1619 unsigned BitWidth = Mask.getBitWidth(); 1620 assert(BitWidth == Op.getValueType().getScalarType().getSizeInBits() && 1621 "Mask size mismatches value type size!"); 1622 1623 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1624 if (Depth == 6 || Mask == 0) 1625 return; // Limit search depth. 1626 1627 APInt KnownZero2, KnownOne2; 1628 1629 switch (Op.getOpcode()) { 1630 case ISD::Constant: 1631 // We know all of the bits for a constant! 1632 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue() & Mask; 1633 KnownZero = ~KnownOne & Mask; 1634 return; 1635 case ISD::AND: 1636 // If either the LHS or the RHS are Zero, the result is zero. 1637 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1638 ComputeMaskedBits(Op.getOperand(0), Mask & ~KnownZero, 1639 KnownZero2, KnownOne2, Depth+1); 1640 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1641 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1642 1643 // Output known-1 bits are only known if set in both the LHS & RHS. 1644 KnownOne &= KnownOne2; 1645 // Output known-0 are known to be clear if zero in either the LHS | RHS. 1646 KnownZero |= KnownZero2; 1647 return; 1648 case ISD::OR: 1649 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1650 ComputeMaskedBits(Op.getOperand(0), Mask & ~KnownOne, 1651 KnownZero2, KnownOne2, Depth+1); 1652 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1653 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1654 1655 // Output known-0 bits are only known if clear in both the LHS & RHS. 1656 KnownZero &= KnownZero2; 1657 // Output known-1 are known to be set if set in either the LHS | RHS. 1658 KnownOne |= KnownOne2; 1659 return; 1660 case ISD::XOR: { 1661 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 1662 ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1); 1663 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1664 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1665 1666 // Output known-0 bits are known if clear or set in both the LHS & RHS. 1667 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1668 // Output known-1 are known to be set if set in only one of the LHS, RHS. 1669 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1670 KnownZero = KnownZeroOut; 1671 return; 1672 } 1673 case ISD::MUL: { 1674 APInt Mask2 = APInt::getAllOnesValue(BitWidth); 1675 ComputeMaskedBits(Op.getOperand(1), Mask2, KnownZero, KnownOne, Depth+1); 1676 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1); 1677 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1678 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1679 1680 // If low bits are zero in either operand, output low known-0 bits. 1681 // Also compute a conserative estimate for high known-0 bits. 1682 // More trickiness is possible, but this is sufficient for the 1683 // interesting case of alignment computation. 1684 KnownOne.clearAllBits(); 1685 unsigned TrailZ = KnownZero.countTrailingOnes() + 1686 KnownZero2.countTrailingOnes(); 1687 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1688 KnownZero2.countLeadingOnes(), 1689 BitWidth) - BitWidth; 1690 1691 TrailZ = std::min(TrailZ, BitWidth); 1692 LeadZ = std::min(LeadZ, BitWidth); 1693 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1694 APInt::getHighBitsSet(BitWidth, LeadZ); 1695 KnownZero &= Mask; 1696 return; 1697 } 1698 case ISD::UDIV: { 1699 // For the purposes of computing leading zeros we can conservatively 1700 // treat a udiv as a logical right shift by the power of 2 known to 1701 // be less than the denominator. 1702 APInt AllOnes = APInt::getAllOnesValue(BitWidth); 1703 ComputeMaskedBits(Op.getOperand(0), 1704 AllOnes, KnownZero2, KnownOne2, Depth+1); 1705 unsigned LeadZ = KnownZero2.countLeadingOnes(); 1706 1707 KnownOne2.clearAllBits(); 1708 KnownZero2.clearAllBits(); 1709 ComputeMaskedBits(Op.getOperand(1), 1710 AllOnes, KnownZero2, KnownOne2, Depth+1); 1711 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1712 if (RHSUnknownLeadingOnes != BitWidth) 1713 LeadZ = std::min(BitWidth, 1714 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1715 1716 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ) & Mask; 1717 return; 1718 } 1719 case ISD::SELECT: 1720 ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero, KnownOne, Depth+1); 1721 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero2, KnownOne2, Depth+1); 1722 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1723 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1724 1725 // Only known if known in both the LHS and RHS. 1726 KnownOne &= KnownOne2; 1727 KnownZero &= KnownZero2; 1728 return; 1729 case ISD::SELECT_CC: 1730 ComputeMaskedBits(Op.getOperand(3), Mask, KnownZero, KnownOne, Depth+1); 1731 ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero2, KnownOne2, Depth+1); 1732 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1733 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1734 1735 // Only known if known in both the LHS and RHS. 1736 KnownOne &= KnownOne2; 1737 KnownZero &= KnownZero2; 1738 return; 1739 case ISD::SADDO: 1740 case ISD::UADDO: 1741 case ISD::SSUBO: 1742 case ISD::USUBO: 1743 case ISD::SMULO: 1744 case ISD::UMULO: 1745 if (Op.getResNo() != 1) 1746 return; 1747 // The boolean result conforms to getBooleanContents. Fall through. 1748 case ISD::SETCC: 1749 // If we know the result of a setcc has the top bits zero, use this info. 1750 if (TLI.getBooleanContents(Op.getValueType().isVector()) == 1751 TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1752 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1753 return; 1754 case ISD::SHL: 1755 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1756 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1757 unsigned ShAmt = SA->getZExtValue(); 1758 1759 // If the shift count is an invalid immediate, don't do anything. 1760 if (ShAmt >= BitWidth) 1761 return; 1762 1763 ComputeMaskedBits(Op.getOperand(0), Mask.lshr(ShAmt), 1764 KnownZero, KnownOne, Depth+1); 1765 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1766 KnownZero <<= ShAmt; 1767 KnownOne <<= ShAmt; 1768 // low bits known zero. 1769 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1770 } 1771 return; 1772 case ISD::SRL: 1773 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1774 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1775 unsigned ShAmt = SA->getZExtValue(); 1776 1777 // If the shift count is an invalid immediate, don't do anything. 1778 if (ShAmt >= BitWidth) 1779 return; 1780 1781 ComputeMaskedBits(Op.getOperand(0), (Mask << ShAmt), 1782 KnownZero, KnownOne, Depth+1); 1783 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1784 KnownZero = KnownZero.lshr(ShAmt); 1785 KnownOne = KnownOne.lshr(ShAmt); 1786 1787 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt) & Mask; 1788 KnownZero |= HighBits; // High bits known zero. 1789 } 1790 return; 1791 case ISD::SRA: 1792 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1793 unsigned ShAmt = SA->getZExtValue(); 1794 1795 // If the shift count is an invalid immediate, don't do anything. 1796 if (ShAmt >= BitWidth) 1797 return; 1798 1799 APInt InDemandedMask = (Mask << ShAmt); 1800 // If any of the demanded bits are produced by the sign extension, we also 1801 // demand the input sign bit. 1802 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt) & Mask; 1803 if (HighBits.getBoolValue()) 1804 InDemandedMask |= APInt::getSignBit(BitWidth); 1805 1806 ComputeMaskedBits(Op.getOperand(0), InDemandedMask, KnownZero, KnownOne, 1807 Depth+1); 1808 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1809 KnownZero = KnownZero.lshr(ShAmt); 1810 KnownOne = KnownOne.lshr(ShAmt); 1811 1812 // Handle the sign bits. 1813 APInt SignBit = APInt::getSignBit(BitWidth); 1814 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1815 1816 if (KnownZero.intersects(SignBit)) { 1817 KnownZero |= HighBits; // New bits are known zero. 1818 } else if (KnownOne.intersects(SignBit)) { 1819 KnownOne |= HighBits; // New bits are known one. 1820 } 1821 } 1822 return; 1823 case ISD::SIGN_EXTEND_INREG: { 1824 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1825 unsigned EBits = EVT.getScalarType().getSizeInBits(); 1826 1827 // Sign extension. Compute the demanded bits in the result that are not 1828 // present in the input. 1829 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits) & Mask; 1830 1831 APInt InSignBit = APInt::getSignBit(EBits); 1832 APInt InputDemandedBits = Mask & APInt::getLowBitsSet(BitWidth, EBits); 1833 1834 // If the sign extended bits are demanded, we know that the sign 1835 // bit is demanded. 1836 InSignBit = InSignBit.zext(BitWidth); 1837 if (NewBits.getBoolValue()) 1838 InputDemandedBits |= InSignBit; 1839 1840 ComputeMaskedBits(Op.getOperand(0), InputDemandedBits, 1841 KnownZero, KnownOne, Depth+1); 1842 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1843 1844 // If the sign bit of the input is known set or clear, then we know the 1845 // top bits of the result. 1846 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1847 KnownZero |= NewBits; 1848 KnownOne &= ~NewBits; 1849 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1850 KnownOne |= NewBits; 1851 KnownZero &= ~NewBits; 1852 } else { // Input sign bit unknown 1853 KnownZero &= ~NewBits; 1854 KnownOne &= ~NewBits; 1855 } 1856 return; 1857 } 1858 case ISD::CTTZ: 1859 case ISD::CTTZ_ZERO_UNDEF: 1860 case ISD::CTLZ: 1861 case ISD::CTLZ_ZERO_UNDEF: 1862 case ISD::CTPOP: { 1863 unsigned LowBits = Log2_32(BitWidth)+1; 1864 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 1865 KnownOne.clearAllBits(); 1866 return; 1867 } 1868 case ISD::LOAD: { 1869 if (ISD::isZEXTLoad(Op.getNode())) { 1870 LoadSDNode *LD = cast<LoadSDNode>(Op); 1871 EVT VT = LD->getMemoryVT(); 1872 unsigned MemBits = VT.getScalarType().getSizeInBits(); 1873 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits) & Mask; 1874 } 1875 return; 1876 } 1877 case ISD::ZERO_EXTEND: { 1878 EVT InVT = Op.getOperand(0).getValueType(); 1879 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1880 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; 1881 APInt InMask = Mask.trunc(InBits); 1882 KnownZero = KnownZero.trunc(InBits); 1883 KnownOne = KnownOne.trunc(InBits); 1884 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1885 KnownZero = KnownZero.zext(BitWidth); 1886 KnownOne = KnownOne.zext(BitWidth); 1887 KnownZero |= NewBits; 1888 return; 1889 } 1890 case ISD::SIGN_EXTEND: { 1891 EVT InVT = Op.getOperand(0).getValueType(); 1892 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1893 APInt InSignBit = APInt::getSignBit(InBits); 1894 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; 1895 APInt InMask = Mask.trunc(InBits); 1896 1897 // If any of the sign extended bits are demanded, we know that the sign 1898 // bit is demanded. Temporarily set this bit in the mask for our callee. 1899 if (NewBits.getBoolValue()) 1900 InMask |= InSignBit; 1901 1902 KnownZero = KnownZero.trunc(InBits); 1903 KnownOne = KnownOne.trunc(InBits); 1904 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1905 1906 // Note if the sign bit is known to be zero or one. 1907 bool SignBitKnownZero = KnownZero.isNegative(); 1908 bool SignBitKnownOne = KnownOne.isNegative(); 1909 assert(!(SignBitKnownZero && SignBitKnownOne) && 1910 "Sign bit can't be known to be both zero and one!"); 1911 1912 // If the sign bit wasn't actually demanded by our caller, we don't 1913 // want it set in the KnownZero and KnownOne result values. Reset the 1914 // mask and reapply it to the result values. 1915 InMask = Mask.trunc(InBits); 1916 KnownZero &= InMask; 1917 KnownOne &= InMask; 1918 1919 KnownZero = KnownZero.zext(BitWidth); 1920 KnownOne = KnownOne.zext(BitWidth); 1921 1922 // If the sign bit is known zero or one, the top bits match. 1923 if (SignBitKnownZero) 1924 KnownZero |= NewBits; 1925 else if (SignBitKnownOne) 1926 KnownOne |= NewBits; 1927 return; 1928 } 1929 case ISD::ANY_EXTEND: { 1930 EVT InVT = Op.getOperand(0).getValueType(); 1931 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1932 APInt InMask = Mask.trunc(InBits); 1933 KnownZero = KnownZero.trunc(InBits); 1934 KnownOne = KnownOne.trunc(InBits); 1935 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1936 KnownZero = KnownZero.zext(BitWidth); 1937 KnownOne = KnownOne.zext(BitWidth); 1938 return; 1939 } 1940 case ISD::TRUNCATE: { 1941 EVT InVT = Op.getOperand(0).getValueType(); 1942 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1943 APInt InMask = Mask.zext(InBits); 1944 KnownZero = KnownZero.zext(InBits); 1945 KnownOne = KnownOne.zext(InBits); 1946 ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); 1947 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1948 KnownZero = KnownZero.trunc(BitWidth); 1949 KnownOne = KnownOne.trunc(BitWidth); 1950 break; 1951 } 1952 case ISD::AssertZext: { 1953 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1954 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 1955 ComputeMaskedBits(Op.getOperand(0), Mask & InMask, KnownZero, 1956 KnownOne, Depth+1); 1957 KnownZero |= (~InMask) & Mask; 1958 return; 1959 } 1960 case ISD::FGETSIGN: 1961 // All bits are zero except the low bit. 1962 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1963 return; 1964 1965 case ISD::SUB: { 1966 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 1967 // We know that the top bits of C-X are clear if X contains less bits 1968 // than C (i.e. no wrap-around can happen). For example, 20-X is 1969 // positive if we can prove that X is >= 0 and < 16. 1970 if (CLHS->getAPIntValue().isNonNegative()) { 1971 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 1972 // NLZ can't be BitWidth with no sign bit 1973 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 1974 ComputeMaskedBits(Op.getOperand(1), MaskV, KnownZero2, KnownOne2, 1975 Depth+1); 1976 1977 // If all of the MaskV bits are known to be zero, then we know the 1978 // output top bits are zero, because we now know that the output is 1979 // from [0-C]. 1980 if ((KnownZero2 & MaskV) == MaskV) { 1981 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 1982 // Top bits known zero. 1983 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2) & Mask; 1984 } 1985 } 1986 } 1987 } 1988 // fall through 1989 case ISD::ADD: 1990 case ISD::ADDE: { 1991 // Output known-0 bits are known if clear or set in both the low clear bits 1992 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 1993 // low 3 bits clear. 1994 APInt Mask2 = APInt::getLowBitsSet(BitWidth, 1995 BitWidth - Mask.countLeadingZeros()); 1996 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1); 1997 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1998 unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 1999 2000 ComputeMaskedBits(Op.getOperand(1), Mask2, KnownZero2, KnownOne2, Depth+1); 2001 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2002 KnownZeroOut = std::min(KnownZeroOut, 2003 KnownZero2.countTrailingOnes()); 2004 2005 if (Op.getOpcode() == ISD::ADD) { 2006 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2007 return; 2008 } 2009 2010 // With ADDE, a carry bit may be added in, so we can only use this 2011 // information if we know (at least) that the low two bits are clear. We 2012 // then return to the caller that the low bit is unknown but that other bits 2013 // are known zero. 2014 if (KnownZeroOut >= 2) // ADDE 2015 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2016 return; 2017 } 2018 case ISD::SREM: 2019 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2020 const APInt &RA = Rem->getAPIntValue().abs(); 2021 if (RA.isPowerOf2()) { 2022 APInt LowBits = RA - 1; 2023 APInt Mask2 = LowBits | APInt::getSignBit(BitWidth); 2024 ComputeMaskedBits(Op.getOperand(0), Mask2,KnownZero2,KnownOne2,Depth+1); 2025 2026 // The low bits of the first operand are unchanged by the srem. 2027 KnownZero = KnownZero2 & LowBits; 2028 KnownOne = KnownOne2 & LowBits; 2029 2030 // If the first operand is non-negative or has all low bits zero, then 2031 // the upper bits are all zero. 2032 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2033 KnownZero |= ~LowBits; 2034 2035 // If the first operand is negative and not all low bits are zero, then 2036 // the upper bits are all one. 2037 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2038 KnownOne |= ~LowBits; 2039 2040 KnownZero &= Mask; 2041 KnownOne &= Mask; 2042 2043 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2044 } 2045 } 2046 return; 2047 case ISD::UREM: { 2048 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2049 const APInt &RA = Rem->getAPIntValue(); 2050 if (RA.isPowerOf2()) { 2051 APInt LowBits = (RA - 1); 2052 APInt Mask2 = LowBits & Mask; 2053 KnownZero |= ~LowBits & Mask; 2054 ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero, KnownOne,Depth+1); 2055 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2056 break; 2057 } 2058 } 2059 2060 // Since the result is less than or equal to either operand, any leading 2061 // zero bits in either operand must also exist in the result. 2062 APInt AllOnes = APInt::getAllOnesValue(BitWidth); 2063 ComputeMaskedBits(Op.getOperand(0), AllOnes, KnownZero, KnownOne, 2064 Depth+1); 2065 ComputeMaskedBits(Op.getOperand(1), AllOnes, KnownZero2, KnownOne2, 2066 Depth+1); 2067 2068 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2069 KnownZero2.countLeadingOnes()); 2070 KnownOne.clearAllBits(); 2071 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders) & Mask; 2072 return; 2073 } 2074 case ISD::FrameIndex: 2075 case ISD::TargetFrameIndex: 2076 if (unsigned Align = InferPtrAlignment(Op)) { 2077 // The low bits are known zero if the pointer is aligned. 2078 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2079 return; 2080 } 2081 break; 2082 2083 default: 2084 if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2085 break; 2086 // Fallthrough 2087 case ISD::INTRINSIC_WO_CHAIN: 2088 case ISD::INTRINSIC_W_CHAIN: 2089 case ISD::INTRINSIC_VOID: 2090 // Allow the target to implement this method for its nodes. 2091 TLI.computeMaskedBitsForTargetNode(Op, Mask, KnownZero, KnownOne, *this, 2092 Depth); 2093 return; 2094 } 2095} 2096 2097/// ComputeNumSignBits - Return the number of times the sign bit of the 2098/// register is replicated into the other bits. We know that at least 1 bit 2099/// is always equal to the sign bit (itself), but other cases can give us 2100/// information. For example, immediately after an "SRA X, 2", we know that 2101/// the top 3 bits are all equal to each other, so we return 3. 2102unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2103 EVT VT = Op.getValueType(); 2104 assert(VT.isInteger() && "Invalid VT!"); 2105 unsigned VTBits = VT.getScalarType().getSizeInBits(); 2106 unsigned Tmp, Tmp2; 2107 unsigned FirstAnswer = 1; 2108 2109 if (Depth == 6) 2110 return 1; // Limit search depth. 2111 2112 switch (Op.getOpcode()) { 2113 default: break; 2114 case ISD::AssertSext: 2115 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2116 return VTBits-Tmp+1; 2117 case ISD::AssertZext: 2118 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2119 return VTBits-Tmp; 2120 2121 case ISD::Constant: { 2122 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2123 return Val.getNumSignBits(); 2124 } 2125 2126 case ISD::SIGN_EXTEND: 2127 Tmp = VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2128 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2129 2130 case ISD::SIGN_EXTEND_INREG: 2131 // Max of the input and what this extends. 2132 Tmp = 2133 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2134 Tmp = VTBits-Tmp+1; 2135 2136 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2137 return std::max(Tmp, Tmp2); 2138 2139 case ISD::SRA: 2140 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2141 // SRA X, C -> adds C sign bits. 2142 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2143 Tmp += C->getZExtValue(); 2144 if (Tmp > VTBits) Tmp = VTBits; 2145 } 2146 return Tmp; 2147 case ISD::SHL: 2148 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2149 // shl destroys sign bits. 2150 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2151 if (C->getZExtValue() >= VTBits || // Bad shift. 2152 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2153 return Tmp - C->getZExtValue(); 2154 } 2155 break; 2156 case ISD::AND: 2157 case ISD::OR: 2158 case ISD::XOR: // NOT is handled here. 2159 // Logical binary ops preserve the number of sign bits at the worst. 2160 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2161 if (Tmp != 1) { 2162 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2163 FirstAnswer = std::min(Tmp, Tmp2); 2164 // We computed what we know about the sign bits as our first 2165 // answer. Now proceed to the generic code that uses 2166 // ComputeMaskedBits, and pick whichever answer is better. 2167 } 2168 break; 2169 2170 case ISD::SELECT: 2171 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2172 if (Tmp == 1) return 1; // Early out. 2173 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2174 return std::min(Tmp, Tmp2); 2175 2176 case ISD::SADDO: 2177 case ISD::UADDO: 2178 case ISD::SSUBO: 2179 case ISD::USUBO: 2180 case ISD::SMULO: 2181 case ISD::UMULO: 2182 if (Op.getResNo() != 1) 2183 break; 2184 // The boolean result conforms to getBooleanContents. Fall through. 2185 case ISD::SETCC: 2186 // If setcc returns 0/-1, all bits are sign bits. 2187 if (TLI.getBooleanContents(Op.getValueType().isVector()) == 2188 TargetLowering::ZeroOrNegativeOneBooleanContent) 2189 return VTBits; 2190 break; 2191 case ISD::ROTL: 2192 case ISD::ROTR: 2193 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2194 unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2195 2196 // Handle rotate right by N like a rotate left by 32-N. 2197 if (Op.getOpcode() == ISD::ROTR) 2198 RotAmt = (VTBits-RotAmt) & (VTBits-1); 2199 2200 // If we aren't rotating out all of the known-in sign bits, return the 2201 // number that are left. This handles rotl(sext(x), 1) for example. 2202 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2203 if (Tmp > RotAmt+1) return Tmp-RotAmt; 2204 } 2205 break; 2206 case ISD::ADD: 2207 // Add can have at most one carry bit. Thus we know that the output 2208 // is, at worst, one more bit than the inputs. 2209 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2210 if (Tmp == 1) return 1; // Early out. 2211 2212 // Special case decrementing a value (ADD X, -1): 2213 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2214 if (CRHS->isAllOnesValue()) { 2215 APInt KnownZero, KnownOne; 2216 APInt Mask = APInt::getAllOnesValue(VTBits); 2217 ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1); 2218 2219 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2220 // sign bits set. 2221 if ((KnownZero | APInt(VTBits, 1)) == Mask) 2222 return VTBits; 2223 2224 // If we are subtracting one from a positive number, there is no carry 2225 // out of the result. 2226 if (KnownZero.isNegative()) 2227 return Tmp; 2228 } 2229 2230 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2231 if (Tmp2 == 1) return 1; 2232 return std::min(Tmp, Tmp2)-1; 2233 break; 2234 2235 case ISD::SUB: 2236 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2237 if (Tmp2 == 1) return 1; 2238 2239 // Handle NEG. 2240 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2241 if (CLHS->isNullValue()) { 2242 APInt KnownZero, KnownOne; 2243 APInt Mask = APInt::getAllOnesValue(VTBits); 2244 ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); 2245 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2246 // sign bits set. 2247 if ((KnownZero | APInt(VTBits, 1)) == Mask) 2248 return VTBits; 2249 2250 // If the input is known to be positive (the sign bit is known clear), 2251 // the output of the NEG has the same number of sign bits as the input. 2252 if (KnownZero.isNegative()) 2253 return Tmp2; 2254 2255 // Otherwise, we treat this like a SUB. 2256 } 2257 2258 // Sub can have at most one carry bit. Thus we know that the output 2259 // is, at worst, one more bit than the inputs. 2260 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2261 if (Tmp == 1) return 1; // Early out. 2262 return std::min(Tmp, Tmp2)-1; 2263 break; 2264 case ISD::TRUNCATE: 2265 // FIXME: it's tricky to do anything useful for this, but it is an important 2266 // case for targets like X86. 2267 break; 2268 } 2269 2270 // Handle LOADX separately here. EXTLOAD case will fallthrough. 2271 if (Op.getOpcode() == ISD::LOAD) { 2272 LoadSDNode *LD = cast<LoadSDNode>(Op); 2273 unsigned ExtType = LD->getExtensionType(); 2274 switch (ExtType) { 2275 default: break; 2276 case ISD::SEXTLOAD: // '17' bits known 2277 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2278 return VTBits-Tmp+1; 2279 case ISD::ZEXTLOAD: // '16' bits known 2280 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2281 return VTBits-Tmp; 2282 } 2283 } 2284 2285 // Allow the target to implement this method for its nodes. 2286 if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2287 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2288 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2289 Op.getOpcode() == ISD::INTRINSIC_VOID) { 2290 unsigned NumBits = TLI.ComputeNumSignBitsForTargetNode(Op, Depth); 2291 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2292 } 2293 2294 // Finally, if we can prove that the top bits of the result are 0's or 1's, 2295 // use this information. 2296 APInt KnownZero, KnownOne; 2297 APInt Mask = APInt::getAllOnesValue(VTBits); 2298 ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth); 2299 2300 if (KnownZero.isNegative()) { // sign bit is 0 2301 Mask = KnownZero; 2302 } else if (KnownOne.isNegative()) { // sign bit is 1; 2303 Mask = KnownOne; 2304 } else { 2305 // Nothing known. 2306 return FirstAnswer; 2307 } 2308 2309 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2310 // the number of identical bits in the top of the input value. 2311 Mask = ~Mask; 2312 Mask <<= Mask.getBitWidth()-VTBits; 2313 // Return # leading zeros. We use 'min' here in case Val was zero before 2314 // shifting. We don't want to return '64' as for an i32 "0". 2315 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2316} 2317 2318/// isBaseWithConstantOffset - Return true if the specified operand is an 2319/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2320/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2321/// semantics as an ADD. This handles the equivalence: 2322/// X|Cst == X+Cst iff X&Cst = 0. 2323bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2324 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2325 !isa<ConstantSDNode>(Op.getOperand(1))) 2326 return false; 2327 2328 if (Op.getOpcode() == ISD::OR && 2329 !MaskedValueIsZero(Op.getOperand(0), 2330 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2331 return false; 2332 2333 return true; 2334} 2335 2336 2337bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2338 // If we're told that NaNs won't happen, assume they won't. 2339 if (getTarget().Options.NoNaNsFPMath) 2340 return true; 2341 2342 // If the value is a constant, we can obviously see if it is a NaN or not. 2343 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2344 return !C->getValueAPF().isNaN(); 2345 2346 // TODO: Recognize more cases here. 2347 2348 return false; 2349} 2350 2351bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2352 // If the value is a constant, we can obviously see if it is a zero or not. 2353 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2354 return !C->isZero(); 2355 2356 // TODO: Recognize more cases here. 2357 switch (Op.getOpcode()) { 2358 default: break; 2359 case ISD::OR: 2360 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2361 return !C->isNullValue(); 2362 break; 2363 } 2364 2365 return false; 2366} 2367 2368bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2369 // Check the obvious case. 2370 if (A == B) return true; 2371 2372 // For for negative and positive zero. 2373 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2374 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2375 if (CA->isZero() && CB->isZero()) return true; 2376 2377 // Otherwise they may not be equal. 2378 return false; 2379} 2380 2381/// getNode - Gets or creates the specified node. 2382/// 2383SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) { 2384 FoldingSetNodeID ID; 2385 AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2386 void *IP = 0; 2387 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2388 return SDValue(E, 0); 2389 2390 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL, getVTList(VT)); 2391 CSEMap.InsertNode(N, IP); 2392 2393 AllNodes.push_back(N); 2394#ifndef NDEBUG 2395 VerifySDNode(N); 2396#endif 2397 return SDValue(N, 0); 2398} 2399 2400SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 2401 EVT VT, SDValue Operand) { 2402 // Constant fold unary operations with an integer constant operand. 2403 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2404 const APInt &Val = C->getAPIntValue(); 2405 switch (Opcode) { 2406 default: break; 2407 case ISD::SIGN_EXTEND: 2408 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2409 case ISD::ANY_EXTEND: 2410 case ISD::ZERO_EXTEND: 2411 case ISD::TRUNCATE: 2412 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2413 case ISD::UINT_TO_FP: 2414 case ISD::SINT_TO_FP: { 2415 // No compile time operations on ppcf128. 2416 if (VT == MVT::ppcf128) break; 2417 APFloat apf(APInt::getNullValue(VT.getSizeInBits())); 2418 (void)apf.convertFromAPInt(Val, 2419 Opcode==ISD::SINT_TO_FP, 2420 APFloat::rmNearestTiesToEven); 2421 return getConstantFP(apf, VT); 2422 } 2423 case ISD::BITCAST: 2424 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2425 return getConstantFP(Val.bitsToFloat(), VT); 2426 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2427 return getConstantFP(Val.bitsToDouble(), VT); 2428 break; 2429 case ISD::BSWAP: 2430 return getConstant(Val.byteSwap(), VT); 2431 case ISD::CTPOP: 2432 return getConstant(Val.countPopulation(), VT); 2433 case ISD::CTLZ: 2434 case ISD::CTLZ_ZERO_UNDEF: 2435 return getConstant(Val.countLeadingZeros(), VT); 2436 case ISD::CTTZ: 2437 case ISD::CTTZ_ZERO_UNDEF: 2438 return getConstant(Val.countTrailingZeros(), VT); 2439 } 2440 } 2441 2442 // Constant fold unary operations with a floating point constant operand. 2443 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2444 APFloat V = C->getValueAPF(); // make copy 2445 if (VT != MVT::ppcf128 && Operand.getValueType() != MVT::ppcf128) { 2446 switch (Opcode) { 2447 case ISD::FNEG: 2448 V.changeSign(); 2449 return getConstantFP(V, VT); 2450 case ISD::FABS: 2451 V.clearSign(); 2452 return getConstantFP(V, VT); 2453 case ISD::FP_ROUND: 2454 case ISD::FP_EXTEND: { 2455 bool ignored; 2456 // This can return overflow, underflow, or inexact; we don't care. 2457 // FIXME need to be more flexible about rounding mode. 2458 (void)V.convert(*EVTToAPFloatSemantics(VT), 2459 APFloat::rmNearestTiesToEven, &ignored); 2460 return getConstantFP(V, VT); 2461 } 2462 case ISD::FP_TO_SINT: 2463 case ISD::FP_TO_UINT: { 2464 integerPart x[2]; 2465 bool ignored; 2466 assert(integerPartWidth >= 64); 2467 // FIXME need to be more flexible about rounding mode. 2468 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2469 Opcode==ISD::FP_TO_SINT, 2470 APFloat::rmTowardZero, &ignored); 2471 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2472 break; 2473 APInt api(VT.getSizeInBits(), x); 2474 return getConstant(api, VT); 2475 } 2476 case ISD::BITCAST: 2477 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2478 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2479 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2480 return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2481 break; 2482 } 2483 } 2484 } 2485 2486 unsigned OpOpcode = Operand.getNode()->getOpcode(); 2487 switch (Opcode) { 2488 case ISD::TokenFactor: 2489 case ISD::MERGE_VALUES: 2490 case ISD::CONCAT_VECTORS: 2491 return Operand; // Factor, merge or concat of one node? No need. 2492 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2493 case ISD::FP_EXTEND: 2494 assert(VT.isFloatingPoint() && 2495 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2496 if (Operand.getValueType() == VT) return Operand; // noop conversion. 2497 assert((!VT.isVector() || 2498 VT.getVectorNumElements() == 2499 Operand.getValueType().getVectorNumElements()) && 2500 "Vector element count mismatch!"); 2501 if (Operand.getOpcode() == ISD::UNDEF) 2502 return getUNDEF(VT); 2503 break; 2504 case ISD::SIGN_EXTEND: 2505 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2506 "Invalid SIGN_EXTEND!"); 2507 if (Operand.getValueType() == VT) return Operand; // noop extension 2508 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2509 "Invalid sext node, dst < src!"); 2510 assert((!VT.isVector() || 2511 VT.getVectorNumElements() == 2512 Operand.getValueType().getVectorNumElements()) && 2513 "Vector element count mismatch!"); 2514 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2515 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2516 else if (OpOpcode == ISD::UNDEF) 2517 // sext(undef) = 0, because the top bits will all be the same. 2518 return getConstant(0, VT); 2519 break; 2520 case ISD::ZERO_EXTEND: 2521 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2522 "Invalid ZERO_EXTEND!"); 2523 if (Operand.getValueType() == VT) return Operand; // noop extension 2524 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2525 "Invalid zext node, dst < src!"); 2526 assert((!VT.isVector() || 2527 VT.getVectorNumElements() == 2528 Operand.getValueType().getVectorNumElements()) && 2529 "Vector element count mismatch!"); 2530 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2531 return getNode(ISD::ZERO_EXTEND, DL, VT, 2532 Operand.getNode()->getOperand(0)); 2533 else if (OpOpcode == ISD::UNDEF) 2534 // zext(undef) = 0, because the top bits will be zero. 2535 return getConstant(0, VT); 2536 break; 2537 case ISD::ANY_EXTEND: 2538 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2539 "Invalid ANY_EXTEND!"); 2540 if (Operand.getValueType() == VT) return Operand; // noop extension 2541 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2542 "Invalid anyext node, dst < src!"); 2543 assert((!VT.isVector() || 2544 VT.getVectorNumElements() == 2545 Operand.getValueType().getVectorNumElements()) && 2546 "Vector element count mismatch!"); 2547 2548 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2549 OpOpcode == ISD::ANY_EXTEND) 2550 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2551 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2552 else if (OpOpcode == ISD::UNDEF) 2553 return getUNDEF(VT); 2554 2555 // (ext (trunx x)) -> x 2556 if (OpOpcode == ISD::TRUNCATE) { 2557 SDValue OpOp = Operand.getNode()->getOperand(0); 2558 if (OpOp.getValueType() == VT) 2559 return OpOp; 2560 } 2561 break; 2562 case ISD::TRUNCATE: 2563 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2564 "Invalid TRUNCATE!"); 2565 if (Operand.getValueType() == VT) return Operand; // noop truncate 2566 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2567 "Invalid truncate node, src < dst!"); 2568 assert((!VT.isVector() || 2569 VT.getVectorNumElements() == 2570 Operand.getValueType().getVectorNumElements()) && 2571 "Vector element count mismatch!"); 2572 if (OpOpcode == ISD::TRUNCATE) 2573 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2574 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2575 OpOpcode == ISD::ANY_EXTEND) { 2576 // If the source is smaller than the dest, we still need an extend. 2577 if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2578 .bitsLT(VT.getScalarType())) 2579 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2580 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2581 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2582 return Operand.getNode()->getOperand(0); 2583 } 2584 if (OpOpcode == ISD::UNDEF) 2585 return getUNDEF(VT); 2586 break; 2587 case ISD::BITCAST: 2588 // Basic sanity checking. 2589 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2590 && "Cannot BITCAST between types of different sizes!"); 2591 if (VT == Operand.getValueType()) return Operand; // noop conversion. 2592 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2593 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2594 if (OpOpcode == ISD::UNDEF) 2595 return getUNDEF(VT); 2596 break; 2597 case ISD::SCALAR_TO_VECTOR: 2598 assert(VT.isVector() && !Operand.getValueType().isVector() && 2599 (VT.getVectorElementType() == Operand.getValueType() || 2600 (VT.getVectorElementType().isInteger() && 2601 Operand.getValueType().isInteger() && 2602 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2603 "Illegal SCALAR_TO_VECTOR node!"); 2604 if (OpOpcode == ISD::UNDEF) 2605 return getUNDEF(VT); 2606 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2607 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2608 isa<ConstantSDNode>(Operand.getOperand(1)) && 2609 Operand.getConstantOperandVal(1) == 0 && 2610 Operand.getOperand(0).getValueType() == VT) 2611 return Operand.getOperand(0); 2612 break; 2613 case ISD::FNEG: 2614 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2615 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 2616 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2617 Operand.getNode()->getOperand(0)); 2618 if (OpOpcode == ISD::FNEG) // --X -> X 2619 return Operand.getNode()->getOperand(0); 2620 break; 2621 case ISD::FABS: 2622 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2623 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2624 break; 2625 } 2626 2627 SDNode *N; 2628 SDVTList VTs = getVTList(VT); 2629 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2630 FoldingSetNodeID ID; 2631 SDValue Ops[1] = { Operand }; 2632 AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2633 void *IP = 0; 2634 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2635 return SDValue(E, 0); 2636 2637 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); 2638 CSEMap.InsertNode(N, IP); 2639 } else { 2640 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); 2641 } 2642 2643 AllNodes.push_back(N); 2644#ifndef NDEBUG 2645 VerifySDNode(N); 2646#endif 2647 return SDValue(N, 0); 2648} 2649 2650SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, 2651 EVT VT, 2652 ConstantSDNode *Cst1, 2653 ConstantSDNode *Cst2) { 2654 const APInt &C1 = Cst1->getAPIntValue(), &C2 = Cst2->getAPIntValue(); 2655 2656 switch (Opcode) { 2657 case ISD::ADD: return getConstant(C1 + C2, VT); 2658 case ISD::SUB: return getConstant(C1 - C2, VT); 2659 case ISD::MUL: return getConstant(C1 * C2, VT); 2660 case ISD::UDIV: 2661 if (C2.getBoolValue()) return getConstant(C1.udiv(C2), VT); 2662 break; 2663 case ISD::UREM: 2664 if (C2.getBoolValue()) return getConstant(C1.urem(C2), VT); 2665 break; 2666 case ISD::SDIV: 2667 if (C2.getBoolValue()) return getConstant(C1.sdiv(C2), VT); 2668 break; 2669 case ISD::SREM: 2670 if (C2.getBoolValue()) return getConstant(C1.srem(C2), VT); 2671 break; 2672 case ISD::AND: return getConstant(C1 & C2, VT); 2673 case ISD::OR: return getConstant(C1 | C2, VT); 2674 case ISD::XOR: return getConstant(C1 ^ C2, VT); 2675 case ISD::SHL: return getConstant(C1 << C2, VT); 2676 case ISD::SRL: return getConstant(C1.lshr(C2), VT); 2677 case ISD::SRA: return getConstant(C1.ashr(C2), VT); 2678 case ISD::ROTL: return getConstant(C1.rotl(C2), VT); 2679 case ISD::ROTR: return getConstant(C1.rotr(C2), VT); 2680 default: break; 2681 } 2682 2683 return SDValue(); 2684} 2685 2686SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 2687 SDValue N1, SDValue N2) { 2688 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2689 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2690 switch (Opcode) { 2691 default: break; 2692 case ISD::TokenFactor: 2693 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2694 N2.getValueType() == MVT::Other && "Invalid token factor!"); 2695 // Fold trivial token factors. 2696 if (N1.getOpcode() == ISD::EntryToken) return N2; 2697 if (N2.getOpcode() == ISD::EntryToken) return N1; 2698 if (N1 == N2) return N1; 2699 break; 2700 case ISD::CONCAT_VECTORS: 2701 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2702 // one big BUILD_VECTOR. 2703 if (N1.getOpcode() == ISD::BUILD_VECTOR && 2704 N2.getOpcode() == ISD::BUILD_VECTOR) { 2705 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2706 N1.getNode()->op_end()); 2707 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2708 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2709 } 2710 break; 2711 case ISD::AND: 2712 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2713 assert(N1.getValueType() == N2.getValueType() && 2714 N1.getValueType() == VT && "Binary operator types must match!"); 2715 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2716 // worth handling here. 2717 if (N2C && N2C->isNullValue()) 2718 return N2; 2719 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2720 return N1; 2721 break; 2722 case ISD::OR: 2723 case ISD::XOR: 2724 case ISD::ADD: 2725 case ISD::SUB: 2726 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2727 assert(N1.getValueType() == N2.getValueType() && 2728 N1.getValueType() == VT && "Binary operator types must match!"); 2729 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2730 // it's worth handling here. 2731 if (N2C && N2C->isNullValue()) 2732 return N1; 2733 break; 2734 case ISD::UDIV: 2735 case ISD::UREM: 2736 case ISD::MULHU: 2737 case ISD::MULHS: 2738 case ISD::MUL: 2739 case ISD::SDIV: 2740 case ISD::SREM: 2741 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2742 assert(N1.getValueType() == N2.getValueType() && 2743 N1.getValueType() == VT && "Binary operator types must match!"); 2744 break; 2745 case ISD::FADD: 2746 case ISD::FSUB: 2747 case ISD::FMUL: 2748 case ISD::FDIV: 2749 case ISD::FREM: 2750 if (getTarget().Options.UnsafeFPMath) { 2751 if (Opcode == ISD::FADD) { 2752 // 0+x --> x 2753 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2754 if (CFP->getValueAPF().isZero()) 2755 return N2; 2756 // x+0 --> x 2757 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2758 if (CFP->getValueAPF().isZero()) 2759 return N1; 2760 } else if (Opcode == ISD::FSUB) { 2761 // x-0 --> x 2762 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2763 if (CFP->getValueAPF().isZero()) 2764 return N1; 2765 } 2766 } 2767 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2768 assert(N1.getValueType() == N2.getValueType() && 2769 N1.getValueType() == VT && "Binary operator types must match!"); 2770 break; 2771 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2772 assert(N1.getValueType() == VT && 2773 N1.getValueType().isFloatingPoint() && 2774 N2.getValueType().isFloatingPoint() && 2775 "Invalid FCOPYSIGN!"); 2776 break; 2777 case ISD::SHL: 2778 case ISD::SRA: 2779 case ISD::SRL: 2780 case ISD::ROTL: 2781 case ISD::ROTR: 2782 assert(VT == N1.getValueType() && 2783 "Shift operators return type must be the same as their first arg"); 2784 assert(VT.isInteger() && N2.getValueType().isInteger() && 2785 "Shifts only work on integers"); 2786 // Verify that the shift amount VT is bit enough to hold valid shift 2787 // amounts. This catches things like trying to shift an i1024 value by an 2788 // i8, which is easy to fall into in generic code that uses 2789 // TLI.getShiftAmount(). 2790 assert(N2.getValueType().getSizeInBits() >= 2791 Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 2792 "Invalid use of small shift amount with oversized value!"); 2793 2794 // Always fold shifts of i1 values so the code generator doesn't need to 2795 // handle them. Since we know the size of the shift has to be less than the 2796 // size of the value, the shift/rotate count is guaranteed to be zero. 2797 if (VT == MVT::i1) 2798 return N1; 2799 if (N2C && N2C->isNullValue()) 2800 return N1; 2801 break; 2802 case ISD::FP_ROUND_INREG: { 2803 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2804 assert(VT == N1.getValueType() && "Not an inreg round!"); 2805 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 2806 "Cannot FP_ROUND_INREG integer types"); 2807 assert(EVT.isVector() == VT.isVector() && 2808 "FP_ROUND_INREG type should be vector iff the operand " 2809 "type is vector!"); 2810 assert((!EVT.isVector() || 2811 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2812 "Vector element counts must match in FP_ROUND_INREG"); 2813 assert(EVT.bitsLE(VT) && "Not rounding down!"); 2814 (void)EVT; 2815 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 2816 break; 2817 } 2818 case ISD::FP_ROUND: 2819 assert(VT.isFloatingPoint() && 2820 N1.getValueType().isFloatingPoint() && 2821 VT.bitsLE(N1.getValueType()) && 2822 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 2823 if (N1.getValueType() == VT) return N1; // noop conversion. 2824 break; 2825 case ISD::AssertSext: 2826 case ISD::AssertZext: { 2827 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2828 assert(VT == N1.getValueType() && "Not an inreg extend!"); 2829 assert(VT.isInteger() && EVT.isInteger() && 2830 "Cannot *_EXTEND_INREG FP types"); 2831 assert(!EVT.isVector() && 2832 "AssertSExt/AssertZExt type should be the vector element type " 2833 "rather than the vector type!"); 2834 assert(EVT.bitsLE(VT) && "Not extending!"); 2835 if (VT == EVT) return N1; // noop assertion. 2836 break; 2837 } 2838 case ISD::SIGN_EXTEND_INREG: { 2839 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2840 assert(VT == N1.getValueType() && "Not an inreg extend!"); 2841 assert(VT.isInteger() && EVT.isInteger() && 2842 "Cannot *_EXTEND_INREG FP types"); 2843 assert(EVT.isVector() == VT.isVector() && 2844 "SIGN_EXTEND_INREG type should be vector iff the operand " 2845 "type is vector!"); 2846 assert((!EVT.isVector() || 2847 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2848 "Vector element counts must match in SIGN_EXTEND_INREG"); 2849 assert(EVT.bitsLE(VT) && "Not extending!"); 2850 if (EVT == VT) return N1; // Not actually extending 2851 2852 if (N1C) { 2853 APInt Val = N1C->getAPIntValue(); 2854 unsigned FromBits = EVT.getScalarType().getSizeInBits(); 2855 Val <<= Val.getBitWidth()-FromBits; 2856 Val = Val.ashr(Val.getBitWidth()-FromBits); 2857 return getConstant(Val, VT); 2858 } 2859 break; 2860 } 2861 case ISD::EXTRACT_VECTOR_ELT: 2862 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 2863 if (N1.getOpcode() == ISD::UNDEF) 2864 return getUNDEF(VT); 2865 2866 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 2867 // expanding copies of large vectors from registers. 2868 if (N2C && 2869 N1.getOpcode() == ISD::CONCAT_VECTORS && 2870 N1.getNumOperands() > 0) { 2871 unsigned Factor = 2872 N1.getOperand(0).getValueType().getVectorNumElements(); 2873 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 2874 N1.getOperand(N2C->getZExtValue() / Factor), 2875 getConstant(N2C->getZExtValue() % Factor, 2876 N2.getValueType())); 2877 } 2878 2879 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 2880 // expanding large vector constants. 2881 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 2882 SDValue Elt = N1.getOperand(N2C->getZExtValue()); 2883 EVT VEltTy = N1.getValueType().getVectorElementType(); 2884 if (Elt.getValueType() != VEltTy) { 2885 // If the vector element type is not legal, the BUILD_VECTOR operands 2886 // are promoted and implicitly truncated. Make that explicit here. 2887 Elt = getNode(ISD::TRUNCATE, DL, VEltTy, Elt); 2888 } 2889 if (VT != VEltTy) { 2890 // If the vector element type is not legal, the EXTRACT_VECTOR_ELT 2891 // result is implicitly extended. 2892 Elt = getNode(ISD::ANY_EXTEND, DL, VT, Elt); 2893 } 2894 return Elt; 2895 } 2896 2897 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 2898 // operations are lowered to scalars. 2899 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 2900 // If the indices are the same, return the inserted element else 2901 // if the indices are known different, extract the element from 2902 // the original vector. 2903 SDValue N1Op2 = N1.getOperand(2); 2904 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 2905 2906 if (N1Op2C && N2C) { 2907 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 2908 if (VT == N1.getOperand(1).getValueType()) 2909 return N1.getOperand(1); 2910 else 2911 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 2912 } 2913 2914 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 2915 } 2916 } 2917 break; 2918 case ISD::EXTRACT_ELEMENT: 2919 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 2920 assert(!N1.getValueType().isVector() && !VT.isVector() && 2921 (N1.getValueType().isInteger() == VT.isInteger()) && 2922 N1.getValueType() != VT && 2923 "Wrong types for EXTRACT_ELEMENT!"); 2924 2925 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 2926 // 64-bit integers into 32-bit parts. Instead of building the extract of 2927 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 2928 if (N1.getOpcode() == ISD::BUILD_PAIR) 2929 return N1.getOperand(N2C->getZExtValue()); 2930 2931 // EXTRACT_ELEMENT of a constant int is also very common. 2932 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 2933 unsigned ElementSize = VT.getSizeInBits(); 2934 unsigned Shift = ElementSize * N2C->getZExtValue(); 2935 APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 2936 return getConstant(ShiftedVal.trunc(ElementSize), VT); 2937 } 2938 break; 2939 case ISD::EXTRACT_SUBVECTOR: { 2940 SDValue Index = N2; 2941 if (VT.isSimple() && N1.getValueType().isSimple()) { 2942 assert(VT.isVector() && N1.getValueType().isVector() && 2943 "Extract subvector VTs must be a vectors!"); 2944 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() && 2945 "Extract subvector VTs must have the same element type!"); 2946 assert(VT.getSimpleVT() <= N1.getValueType().getSimpleVT() && 2947 "Extract subvector must be from larger vector to smaller vector!"); 2948 2949 if (isa<ConstantSDNode>(Index.getNode())) { 2950 assert((VT.getVectorNumElements() + 2951 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 2952 <= N1.getValueType().getVectorNumElements()) 2953 && "Extract subvector overflow!"); 2954 } 2955 2956 // Trivial extraction. 2957 if (VT.getSimpleVT() == N1.getValueType().getSimpleVT()) 2958 return N1; 2959 } 2960 break; 2961 } 2962 } 2963 2964 if (N1C) { 2965 if (N2C) { 2966 SDValue SV = FoldConstantArithmetic(Opcode, VT, N1C, N2C); 2967 if (SV.getNode()) return SV; 2968 } else { // Cannonicalize constant to RHS if commutative 2969 if (isCommutativeBinOp(Opcode)) { 2970 std::swap(N1C, N2C); 2971 std::swap(N1, N2); 2972 } 2973 } 2974 } 2975 2976 // Constant fold FP operations. 2977 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 2978 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 2979 if (N1CFP) { 2980 if (!N2CFP && isCommutativeBinOp(Opcode)) { 2981 // Cannonicalize constant to RHS if commutative 2982 std::swap(N1CFP, N2CFP); 2983 std::swap(N1, N2); 2984 } else if (N2CFP && VT != MVT::ppcf128) { 2985 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 2986 APFloat::opStatus s; 2987 switch (Opcode) { 2988 case ISD::FADD: 2989 s = V1.add(V2, APFloat::rmNearestTiesToEven); 2990 if (s != APFloat::opInvalidOp) 2991 return getConstantFP(V1, VT); 2992 break; 2993 case ISD::FSUB: 2994 s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 2995 if (s!=APFloat::opInvalidOp) 2996 return getConstantFP(V1, VT); 2997 break; 2998 case ISD::FMUL: 2999 s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 3000 if (s!=APFloat::opInvalidOp) 3001 return getConstantFP(V1, VT); 3002 break; 3003 case ISD::FDIV: 3004 s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3005 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3006 return getConstantFP(V1, VT); 3007 break; 3008 case ISD::FREM : 3009 s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3010 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3011 return getConstantFP(V1, VT); 3012 break; 3013 case ISD::FCOPYSIGN: 3014 V1.copySign(V2); 3015 return getConstantFP(V1, VT); 3016 default: break; 3017 } 3018 } 3019 } 3020 3021 // Canonicalize an UNDEF to the RHS, even over a constant. 3022 if (N1.getOpcode() == ISD::UNDEF) { 3023 if (isCommutativeBinOp(Opcode)) { 3024 std::swap(N1, N2); 3025 } else { 3026 switch (Opcode) { 3027 case ISD::FP_ROUND_INREG: 3028 case ISD::SIGN_EXTEND_INREG: 3029 case ISD::SUB: 3030 case ISD::FSUB: 3031 case ISD::FDIV: 3032 case ISD::FREM: 3033 case ISD::SRA: 3034 return N1; // fold op(undef, arg2) -> undef 3035 case ISD::UDIV: 3036 case ISD::SDIV: 3037 case ISD::UREM: 3038 case ISD::SREM: 3039 case ISD::SRL: 3040 case ISD::SHL: 3041 if (!VT.isVector()) 3042 return getConstant(0, VT); // fold op(undef, arg2) -> 0 3043 // For vectors, we can't easily build an all zero vector, just return 3044 // the LHS. 3045 return N2; 3046 } 3047 } 3048 } 3049 3050 // Fold a bunch of operators when the RHS is undef. 3051 if (N2.getOpcode() == ISD::UNDEF) { 3052 switch (Opcode) { 3053 case ISD::XOR: 3054 if (N1.getOpcode() == ISD::UNDEF) 3055 // Handle undef ^ undef -> 0 special case. This is a common 3056 // idiom (misuse). 3057 return getConstant(0, VT); 3058 // fallthrough 3059 case ISD::ADD: 3060 case ISD::ADDC: 3061 case ISD::ADDE: 3062 case ISD::SUB: 3063 case ISD::UDIV: 3064 case ISD::SDIV: 3065 case ISD::UREM: 3066 case ISD::SREM: 3067 return N2; // fold op(arg1, undef) -> undef 3068 case ISD::FADD: 3069 case ISD::FSUB: 3070 case ISD::FMUL: 3071 case ISD::FDIV: 3072 case ISD::FREM: 3073 if (getTarget().Options.UnsafeFPMath) 3074 return N2; 3075 break; 3076 case ISD::MUL: 3077 case ISD::AND: 3078 case ISD::SRL: 3079 case ISD::SHL: 3080 if (!VT.isVector()) 3081 return getConstant(0, VT); // fold op(arg1, undef) -> 0 3082 // For vectors, we can't easily build an all zero vector, just return 3083 // the LHS. 3084 return N1; 3085 case ISD::OR: 3086 if (!VT.isVector()) 3087 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3088 // For vectors, we can't easily build an all one vector, just return 3089 // the LHS. 3090 return N1; 3091 case ISD::SRA: 3092 return N1; 3093 } 3094 } 3095 3096 // Memoize this node if possible. 3097 SDNode *N; 3098 SDVTList VTs = getVTList(VT); 3099 if (VT != MVT::Glue) { 3100 SDValue Ops[] = { N1, N2 }; 3101 FoldingSetNodeID ID; 3102 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3103 void *IP = 0; 3104 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3105 return SDValue(E, 0); 3106 3107 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); 3108 CSEMap.InsertNode(N, IP); 3109 } else { 3110 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); 3111 } 3112 3113 AllNodes.push_back(N); 3114#ifndef NDEBUG 3115 VerifySDNode(N); 3116#endif 3117 return SDValue(N, 0); 3118} 3119 3120SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3121 SDValue N1, SDValue N2, SDValue N3) { 3122 // Perform various simplifications. 3123 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3124 switch (Opcode) { 3125 case ISD::CONCAT_VECTORS: 3126 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3127 // one big BUILD_VECTOR. 3128 if (N1.getOpcode() == ISD::BUILD_VECTOR && 3129 N2.getOpcode() == ISD::BUILD_VECTOR && 3130 N3.getOpcode() == ISD::BUILD_VECTOR) { 3131 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3132 N1.getNode()->op_end()); 3133 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3134 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3135 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3136 } 3137 break; 3138 case ISD::SETCC: { 3139 // Use FoldSetCC to simplify SETCC's. 3140 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3141 if (Simp.getNode()) return Simp; 3142 break; 3143 } 3144 case ISD::SELECT: 3145 if (N1C) { 3146 if (N1C->getZExtValue()) 3147 return N2; // select true, X, Y -> X 3148 else 3149 return N3; // select false, X, Y -> Y 3150 } 3151 3152 if (N2 == N3) return N2; // select C, X, X -> X 3153 break; 3154 case ISD::VECTOR_SHUFFLE: 3155 llvm_unreachable("should use getVectorShuffle constructor!"); 3156 break; 3157 case ISD::INSERT_SUBVECTOR: { 3158 SDValue Index = N3; 3159 if (VT.isSimple() && N1.getValueType().isSimple() 3160 && N2.getValueType().isSimple()) { 3161 assert(VT.isVector() && N1.getValueType().isVector() && 3162 N2.getValueType().isVector() && 3163 "Insert subvector VTs must be a vectors"); 3164 assert(VT == N1.getValueType() && 3165 "Dest and insert subvector source types must match!"); 3166 assert(N2.getValueType().getSimpleVT() <= N1.getValueType().getSimpleVT() && 3167 "Insert subvector must be from smaller vector to larger vector!"); 3168 if (isa<ConstantSDNode>(Index.getNode())) { 3169 assert((N2.getValueType().getVectorNumElements() + 3170 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3171 <= VT.getVectorNumElements()) 3172 && "Insert subvector overflow!"); 3173 } 3174 3175 // Trivial insertion. 3176 if (VT.getSimpleVT() == N2.getValueType().getSimpleVT()) 3177 return N2; 3178 } 3179 break; 3180 } 3181 case ISD::BITCAST: 3182 // Fold bit_convert nodes from a type to themselves. 3183 if (N1.getValueType() == VT) 3184 return N1; 3185 break; 3186 } 3187 3188 // Memoize node if it doesn't produce a flag. 3189 SDNode *N; 3190 SDVTList VTs = getVTList(VT); 3191 if (VT != MVT::Glue) { 3192 SDValue Ops[] = { N1, N2, N3 }; 3193 FoldingSetNodeID ID; 3194 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3195 void *IP = 0; 3196 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3197 return SDValue(E, 0); 3198 3199 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); 3200 CSEMap.InsertNode(N, IP); 3201 } else { 3202 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); 3203 } 3204 3205 AllNodes.push_back(N); 3206#ifndef NDEBUG 3207 VerifySDNode(N); 3208#endif 3209 return SDValue(N, 0); 3210} 3211 3212SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3213 SDValue N1, SDValue N2, SDValue N3, 3214 SDValue N4) { 3215 SDValue Ops[] = { N1, N2, N3, N4 }; 3216 return getNode(Opcode, DL, VT, Ops, 4); 3217} 3218 3219SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3220 SDValue N1, SDValue N2, SDValue N3, 3221 SDValue N4, SDValue N5) { 3222 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3223 return getNode(Opcode, DL, VT, Ops, 5); 3224} 3225 3226/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3227/// the incoming stack arguments to be loaded from the stack. 3228SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3229 SmallVector<SDValue, 8> ArgChains; 3230 3231 // Include the original chain at the beginning of the list. When this is 3232 // used by target LowerCall hooks, this helps legalize find the 3233 // CALLSEQ_BEGIN node. 3234 ArgChains.push_back(Chain); 3235 3236 // Add a chain value for each stack argument. 3237 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3238 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3239 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3240 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3241 if (FI->getIndex() < 0) 3242 ArgChains.push_back(SDValue(L, 1)); 3243 3244 // Build a tokenfactor for all the chains. 3245 return getNode(ISD::TokenFactor, Chain.getDebugLoc(), MVT::Other, 3246 &ArgChains[0], ArgChains.size()); 3247} 3248 3249/// SplatByte - Distribute ByteVal over NumBits bits. 3250static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) { 3251 APInt Val = APInt(NumBits, ByteVal); 3252 unsigned Shift = 8; 3253 for (unsigned i = NumBits; i > 8; i >>= 1) { 3254 Val = (Val << Shift) | Val; 3255 Shift <<= 1; 3256 } 3257 return Val; 3258} 3259 3260/// getMemsetValue - Vectorized representation of the memset value 3261/// operand. 3262static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3263 DebugLoc dl) { 3264 assert(Value.getOpcode() != ISD::UNDEF); 3265 3266 unsigned NumBits = VT.getScalarType().getSizeInBits(); 3267 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3268 APInt Val = SplatByte(NumBits, C->getZExtValue() & 255); 3269 if (VT.isInteger()) 3270 return DAG.getConstant(Val, VT); 3271 return DAG.getConstantFP(APFloat(Val), VT); 3272 } 3273 3274 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3275 if (NumBits > 8) { 3276 // Use a multiplication with 0x010101... to extend the input to the 3277 // required length. 3278 APInt Magic = SplatByte(NumBits, 0x01); 3279 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3280 } 3281 3282 return Value; 3283} 3284 3285/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3286/// used when a memcpy is turned into a memset when the source is a constant 3287/// string ptr. 3288static SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG, 3289 const TargetLowering &TLI, 3290 std::string &Str, unsigned Offset) { 3291 // Handle vector with all elements zero. 3292 if (Str.empty()) { 3293 if (VT.isInteger()) 3294 return DAG.getConstant(0, VT); 3295 else if (VT == MVT::f32 || VT == MVT::f64) 3296 return DAG.getConstantFP(0.0, VT); 3297 else if (VT.isVector()) { 3298 unsigned NumElts = VT.getVectorNumElements(); 3299 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3300 return DAG.getNode(ISD::BITCAST, dl, VT, 3301 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3302 EltVT, NumElts))); 3303 } else 3304 llvm_unreachable("Expected type!"); 3305 } 3306 3307 assert(!VT.isVector() && "Can't handle vector type here!"); 3308 unsigned NumBits = VT.getSizeInBits(); 3309 unsigned MSB = NumBits / 8; 3310 uint64_t Val = 0; 3311 if (TLI.isLittleEndian()) 3312 Offset = Offset + MSB - 1; 3313 for (unsigned i = 0; i != MSB; ++i) { 3314 Val = (Val << 8) | (unsigned char)Str[Offset]; 3315 Offset += TLI.isLittleEndian() ? -1 : 1; 3316 } 3317 return DAG.getConstant(Val, VT); 3318} 3319 3320/// getMemBasePlusOffset - Returns base and offset node for the 3321/// 3322static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, 3323 SelectionDAG &DAG) { 3324 EVT VT = Base.getValueType(); 3325 return DAG.getNode(ISD::ADD, Base.getDebugLoc(), 3326 VT, Base, DAG.getConstant(Offset, VT)); 3327} 3328 3329/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3330/// 3331static bool isMemSrcFromString(SDValue Src, std::string &Str) { 3332 unsigned SrcDelta = 0; 3333 GlobalAddressSDNode *G = NULL; 3334 if (Src.getOpcode() == ISD::GlobalAddress) 3335 G = cast<GlobalAddressSDNode>(Src); 3336 else if (Src.getOpcode() == ISD::ADD && 3337 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3338 Src.getOperand(1).getOpcode() == ISD::Constant) { 3339 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3340 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3341 } 3342 if (!G) 3343 return false; 3344 3345 const GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal()); 3346 if (GV && GetConstantStringInfo(GV, Str, SrcDelta, false)) 3347 return true; 3348 3349 return false; 3350} 3351 3352/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3353/// to replace the memset / memcpy. Return true if the number of memory ops 3354/// is below the threshold. It returns the types of the sequence of 3355/// memory ops to perform memset / memcpy by reference. 3356static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3357 unsigned Limit, uint64_t Size, 3358 unsigned DstAlign, unsigned SrcAlign, 3359 bool IsZeroVal, 3360 bool MemcpyStrSrc, 3361 SelectionDAG &DAG, 3362 const TargetLowering &TLI) { 3363 assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3364 "Expecting memcpy / memset source to meet alignment requirement!"); 3365 // If 'SrcAlign' is zero, that means the memory operation does not need to 3366 // load the value, i.e. memset or memcpy from constant string. Otherwise, 3367 // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3368 // is the specified alignment of the memory operation. If it is zero, that 3369 // means it's possible to change the alignment of the destination. 3370 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3371 // not need to be loaded. 3372 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3373 IsZeroVal, MemcpyStrSrc, 3374 DAG.getMachineFunction()); 3375 3376 if (VT == MVT::Other) { 3377 if (DstAlign >= TLI.getTargetData()->getPointerPrefAlignment() || 3378 TLI.allowsUnalignedMemoryAccesses(VT)) { 3379 VT = TLI.getPointerTy(); 3380 } else { 3381 switch (DstAlign & 7) { 3382 case 0: VT = MVT::i64; break; 3383 case 4: VT = MVT::i32; break; 3384 case 2: VT = MVT::i16; break; 3385 default: VT = MVT::i8; break; 3386 } 3387 } 3388 3389 MVT LVT = MVT::i64; 3390 while (!TLI.isTypeLegal(LVT)) 3391 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3392 assert(LVT.isInteger()); 3393 3394 if (VT.bitsGT(LVT)) 3395 VT = LVT; 3396 } 3397 3398 unsigned NumMemOps = 0; 3399 while (Size != 0) { 3400 unsigned VTSize = VT.getSizeInBits() / 8; 3401 while (VTSize > Size) { 3402 // For now, only use non-vector load / store's for the left-over pieces. 3403 if (VT.isVector() || VT.isFloatingPoint()) { 3404 VT = MVT::i64; 3405 while (!TLI.isTypeLegal(VT)) 3406 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); 3407 VTSize = VT.getSizeInBits() / 8; 3408 } else { 3409 // This can result in a type that is not legal on the target, e.g. 3410 // 1 or 2 bytes on PPC. 3411 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); 3412 VTSize >>= 1; 3413 } 3414 } 3415 3416 if (++NumMemOps > Limit) 3417 return false; 3418 MemOps.push_back(VT); 3419 Size -= VTSize; 3420 } 3421 3422 return true; 3423} 3424 3425static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, 3426 SDValue Chain, SDValue Dst, 3427 SDValue Src, uint64_t Size, 3428 unsigned Align, bool isVol, 3429 bool AlwaysInline, 3430 MachinePointerInfo DstPtrInfo, 3431 MachinePointerInfo SrcPtrInfo) { 3432 // Turn a memcpy of undef to nop. 3433 if (Src.getOpcode() == ISD::UNDEF) 3434 return Chain; 3435 3436 // Expand memcpy to a series of load and store ops if the size operand falls 3437 // below a certain threshold. 3438 // TODO: In the AlwaysInline case, if the size is big then generate a loop 3439 // rather than maybe a humongous number of loads and stores. 3440 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3441 std::vector<EVT> MemOps; 3442 bool DstAlignCanChange = false; 3443 MachineFunction &MF = DAG.getMachineFunction(); 3444 MachineFrameInfo *MFI = MF.getFrameInfo(); 3445 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3446 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3447 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3448 DstAlignCanChange = true; 3449 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3450 if (Align > SrcAlign) 3451 SrcAlign = Align; 3452 std::string Str; 3453 bool CopyFromStr = isMemSrcFromString(Src, Str); 3454 bool isZeroStr = CopyFromStr && Str.empty(); 3455 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3456 3457 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3458 (DstAlignCanChange ? 0 : Align), 3459 (isZeroStr ? 0 : SrcAlign), 3460 true, CopyFromStr, DAG, TLI)) 3461 return SDValue(); 3462 3463 if (DstAlignCanChange) { 3464 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3465 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3466 if (NewAlign > Align) { 3467 // Give the stack frame object a larger alignment if needed. 3468 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3469 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3470 Align = NewAlign; 3471 } 3472 } 3473 3474 SmallVector<SDValue, 8> OutChains; 3475 unsigned NumMemOps = MemOps.size(); 3476 uint64_t SrcOff = 0, DstOff = 0; 3477 for (unsigned i = 0; i != NumMemOps; ++i) { 3478 EVT VT = MemOps[i]; 3479 unsigned VTSize = VT.getSizeInBits() / 8; 3480 SDValue Value, Store; 3481 3482 if (CopyFromStr && 3483 (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3484 // It's unlikely a store of a vector immediate can be done in a single 3485 // instruction. It would require a load from a constantpool first. 3486 // We only handle zero vectors here. 3487 // FIXME: Handle other cases where store of vector immediate is done in 3488 // a single instruction. 3489 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str, SrcOff); 3490 Store = DAG.getStore(Chain, dl, Value, 3491 getMemBasePlusOffset(Dst, DstOff, DAG), 3492 DstPtrInfo.getWithOffset(DstOff), isVol, 3493 false, Align); 3494 } else { 3495 // The type might not be legal for the target. This should only happen 3496 // if the type is smaller than a legal type, as on PPC, so the right 3497 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3498 // to Load/Store if NVT==VT. 3499 // FIXME does the case above also need this? 3500 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3501 assert(NVT.bitsGE(VT)); 3502 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3503 getMemBasePlusOffset(Src, SrcOff, DAG), 3504 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3505 MinAlign(SrcAlign, SrcOff)); 3506 Store = DAG.getTruncStore(Chain, dl, Value, 3507 getMemBasePlusOffset(Dst, DstOff, DAG), 3508 DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3509 false, Align); 3510 } 3511 OutChains.push_back(Store); 3512 SrcOff += VTSize; 3513 DstOff += VTSize; 3514 } 3515 3516 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3517 &OutChains[0], OutChains.size()); 3518} 3519 3520static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, 3521 SDValue Chain, SDValue Dst, 3522 SDValue Src, uint64_t Size, 3523 unsigned Align, bool isVol, 3524 bool AlwaysInline, 3525 MachinePointerInfo DstPtrInfo, 3526 MachinePointerInfo SrcPtrInfo) { 3527 // Turn a memmove of undef to nop. 3528 if (Src.getOpcode() == ISD::UNDEF) 3529 return Chain; 3530 3531 // Expand memmove to a series of load and store ops if the size operand falls 3532 // below a certain threshold. 3533 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3534 std::vector<EVT> MemOps; 3535 bool DstAlignCanChange = false; 3536 MachineFunction &MF = DAG.getMachineFunction(); 3537 MachineFrameInfo *MFI = MF.getFrameInfo(); 3538 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3539 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3540 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3541 DstAlignCanChange = true; 3542 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3543 if (Align > SrcAlign) 3544 SrcAlign = Align; 3545 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3546 3547 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3548 (DstAlignCanChange ? 0 : Align), 3549 SrcAlign, true, false, DAG, TLI)) 3550 return SDValue(); 3551 3552 if (DstAlignCanChange) { 3553 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3554 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3555 if (NewAlign > Align) { 3556 // Give the stack frame object a larger alignment if needed. 3557 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3558 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3559 Align = NewAlign; 3560 } 3561 } 3562 3563 uint64_t SrcOff = 0, DstOff = 0; 3564 SmallVector<SDValue, 8> LoadValues; 3565 SmallVector<SDValue, 8> LoadChains; 3566 SmallVector<SDValue, 8> OutChains; 3567 unsigned NumMemOps = MemOps.size(); 3568 for (unsigned i = 0; i < NumMemOps; i++) { 3569 EVT VT = MemOps[i]; 3570 unsigned VTSize = VT.getSizeInBits() / 8; 3571 SDValue Value, Store; 3572 3573 Value = DAG.getLoad(VT, dl, Chain, 3574 getMemBasePlusOffset(Src, SrcOff, DAG), 3575 SrcPtrInfo.getWithOffset(SrcOff), isVol, 3576 false, false, SrcAlign); 3577 LoadValues.push_back(Value); 3578 LoadChains.push_back(Value.getValue(1)); 3579 SrcOff += VTSize; 3580 } 3581 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3582 &LoadChains[0], LoadChains.size()); 3583 OutChains.clear(); 3584 for (unsigned i = 0; i < NumMemOps; i++) { 3585 EVT VT = MemOps[i]; 3586 unsigned VTSize = VT.getSizeInBits() / 8; 3587 SDValue Value, Store; 3588 3589 Store = DAG.getStore(Chain, dl, LoadValues[i], 3590 getMemBasePlusOffset(Dst, DstOff, DAG), 3591 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3592 OutChains.push_back(Store); 3593 DstOff += VTSize; 3594 } 3595 3596 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3597 &OutChains[0], OutChains.size()); 3598} 3599 3600static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, 3601 SDValue Chain, SDValue Dst, 3602 SDValue Src, uint64_t Size, 3603 unsigned Align, bool isVol, 3604 MachinePointerInfo DstPtrInfo) { 3605 // Turn a memset of undef to nop. 3606 if (Src.getOpcode() == ISD::UNDEF) 3607 return Chain; 3608 3609 // Expand memset to a series of load/store ops if the size operand 3610 // falls below a certain threshold. 3611 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3612 std::vector<EVT> MemOps; 3613 bool DstAlignCanChange = false; 3614 MachineFunction &MF = DAG.getMachineFunction(); 3615 MachineFrameInfo *MFI = MF.getFrameInfo(); 3616 bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); 3617 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3618 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3619 DstAlignCanChange = true; 3620 bool IsZeroVal = 3621 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3622 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3623 Size, (DstAlignCanChange ? 0 : Align), 0, 3624 IsZeroVal, false, DAG, TLI)) 3625 return SDValue(); 3626 3627 if (DstAlignCanChange) { 3628 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3629 unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); 3630 if (NewAlign > Align) { 3631 // Give the stack frame object a larger alignment if needed. 3632 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3633 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3634 Align = NewAlign; 3635 } 3636 } 3637 3638 SmallVector<SDValue, 8> OutChains; 3639 uint64_t DstOff = 0; 3640 unsigned NumMemOps = MemOps.size(); 3641 3642 // Find the largest store and generate the bit pattern for it. 3643 EVT LargestVT = MemOps[0]; 3644 for (unsigned i = 1; i < NumMemOps; i++) 3645 if (MemOps[i].bitsGT(LargestVT)) 3646 LargestVT = MemOps[i]; 3647 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3648 3649 for (unsigned i = 0; i < NumMemOps; i++) { 3650 EVT VT = MemOps[i]; 3651 3652 // If this store is smaller than the largest store see whether we can get 3653 // the smaller value for free with a truncate. 3654 SDValue Value = MemSetValue; 3655 if (VT.bitsLT(LargestVT)) { 3656 if (!LargestVT.isVector() && !VT.isVector() && 3657 TLI.isTruncateFree(LargestVT, VT)) 3658 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3659 else 3660 Value = getMemsetValue(Src, VT, DAG, dl); 3661 } 3662 assert(Value.getValueType() == VT && "Value with wrong type."); 3663 SDValue Store = DAG.getStore(Chain, dl, Value, 3664 getMemBasePlusOffset(Dst, DstOff, DAG), 3665 DstPtrInfo.getWithOffset(DstOff), 3666 isVol, false, Align); 3667 OutChains.push_back(Store); 3668 DstOff += VT.getSizeInBits() / 8; 3669 } 3670 3671 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3672 &OutChains[0], OutChains.size()); 3673} 3674 3675SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, 3676 SDValue Src, SDValue Size, 3677 unsigned Align, bool isVol, bool AlwaysInline, 3678 MachinePointerInfo DstPtrInfo, 3679 MachinePointerInfo SrcPtrInfo) { 3680 3681 // Check to see if we should lower the memcpy to loads and stores first. 3682 // For cases within the target-specified limits, this is the best choice. 3683 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3684 if (ConstantSize) { 3685 // Memcpy with size zero? Just return the original chain. 3686 if (ConstantSize->isNullValue()) 3687 return Chain; 3688 3689 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3690 ConstantSize->getZExtValue(),Align, 3691 isVol, false, DstPtrInfo, SrcPtrInfo); 3692 if (Result.getNode()) 3693 return Result; 3694 } 3695 3696 // Then check to see if we should lower the memcpy with target-specific 3697 // code. If the target chooses to do this, this is the next best. 3698 SDValue Result = 3699 TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 3700 isVol, AlwaysInline, 3701 DstPtrInfo, SrcPtrInfo); 3702 if (Result.getNode()) 3703 return Result; 3704 3705 // If we really need inline code and the target declined to provide it, 3706 // use a (potentially long) sequence of loads and stores. 3707 if (AlwaysInline) { 3708 assert(ConstantSize && "AlwaysInline requires a constant size!"); 3709 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3710 ConstantSize->getZExtValue(), Align, isVol, 3711 true, DstPtrInfo, SrcPtrInfo); 3712 } 3713 3714 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 3715 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 3716 // respect volatile, so they may do things like read or write memory 3717 // beyond the given memory regions. But fixing this isn't easy, and most 3718 // people don't care. 3719 3720 // Emit a library call. 3721 TargetLowering::ArgListTy Args; 3722 TargetLowering::ArgListEntry Entry; 3723 Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext()); 3724 Entry.Node = Dst; Args.push_back(Entry); 3725 Entry.Node = Src; Args.push_back(Entry); 3726 Entry.Node = Size; Args.push_back(Entry); 3727 // FIXME: pass in DebugLoc 3728 std::pair<SDValue,SDValue> CallResult = 3729 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3730 false, false, false, false, 0, 3731 TLI.getLibcallCallingConv(RTLIB::MEMCPY), false, 3732 /*isReturnValueUsed=*/false, 3733 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMCPY), 3734 TLI.getPointerTy()), 3735 Args, *this, dl); 3736 return CallResult.second; 3737} 3738 3739SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, 3740 SDValue Src, SDValue Size, 3741 unsigned Align, bool isVol, 3742 MachinePointerInfo DstPtrInfo, 3743 MachinePointerInfo SrcPtrInfo) { 3744 3745 // Check to see if we should lower the memmove to loads and stores first. 3746 // For cases within the target-specified limits, this is the best choice. 3747 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3748 if (ConstantSize) { 3749 // Memmove with size zero? Just return the original chain. 3750 if (ConstantSize->isNullValue()) 3751 return Chain; 3752 3753 SDValue Result = 3754 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 3755 ConstantSize->getZExtValue(), Align, isVol, 3756 false, DstPtrInfo, SrcPtrInfo); 3757 if (Result.getNode()) 3758 return Result; 3759 } 3760 3761 // Then check to see if we should lower the memmove with target-specific 3762 // code. If the target chooses to do this, this is the next best. 3763 SDValue Result = 3764 TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 3765 DstPtrInfo, SrcPtrInfo); 3766 if (Result.getNode()) 3767 return Result; 3768 3769 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 3770 // not be safe. See memcpy above for more details. 3771 3772 // Emit a library call. 3773 TargetLowering::ArgListTy Args; 3774 TargetLowering::ArgListEntry Entry; 3775 Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext()); 3776 Entry.Node = Dst; Args.push_back(Entry); 3777 Entry.Node = Src; Args.push_back(Entry); 3778 Entry.Node = Size; Args.push_back(Entry); 3779 // FIXME: pass in DebugLoc 3780 std::pair<SDValue,SDValue> CallResult = 3781 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3782 false, false, false, false, 0, 3783 TLI.getLibcallCallingConv(RTLIB::MEMMOVE), false, 3784 /*isReturnValueUsed=*/false, 3785 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMMOVE), 3786 TLI.getPointerTy()), 3787 Args, *this, dl); 3788 return CallResult.second; 3789} 3790 3791SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, 3792 SDValue Src, SDValue Size, 3793 unsigned Align, bool isVol, 3794 MachinePointerInfo DstPtrInfo) { 3795 3796 // Check to see if we should lower the memset to stores first. 3797 // For cases within the target-specified limits, this is the best choice. 3798 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3799 if (ConstantSize) { 3800 // Memset with size zero? Just return the original chain. 3801 if (ConstantSize->isNullValue()) 3802 return Chain; 3803 3804 SDValue Result = 3805 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 3806 Align, isVol, DstPtrInfo); 3807 3808 if (Result.getNode()) 3809 return Result; 3810 } 3811 3812 // Then check to see if we should lower the memset with target-specific 3813 // code. If the target chooses to do this, this is the next best. 3814 SDValue Result = 3815 TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 3816 DstPtrInfo); 3817 if (Result.getNode()) 3818 return Result; 3819 3820 // Emit a library call. 3821 Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext()); 3822 TargetLowering::ArgListTy Args; 3823 TargetLowering::ArgListEntry Entry; 3824 Entry.Node = Dst; Entry.Ty = IntPtrTy; 3825 Args.push_back(Entry); 3826 // Extend or truncate the argument to be an i32 value for the call. 3827 if (Src.getValueType().bitsGT(MVT::i32)) 3828 Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 3829 else 3830 Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 3831 Entry.Node = Src; 3832 Entry.Ty = Type::getInt32Ty(*getContext()); 3833 Entry.isSExt = true; 3834 Args.push_back(Entry); 3835 Entry.Node = Size; 3836 Entry.Ty = IntPtrTy; 3837 Entry.isSExt = false; 3838 Args.push_back(Entry); 3839 // FIXME: pass in DebugLoc 3840 std::pair<SDValue,SDValue> CallResult = 3841 TLI.LowerCallTo(Chain, Type::getVoidTy(*getContext()), 3842 false, false, false, false, 0, 3843 TLI.getLibcallCallingConv(RTLIB::MEMSET), false, 3844 /*isReturnValueUsed=*/false, 3845 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET), 3846 TLI.getPointerTy()), 3847 Args, *this, dl); 3848 return CallResult.second; 3849} 3850 3851SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3852 SDValue Chain, SDValue Ptr, SDValue Cmp, 3853 SDValue Swp, MachinePointerInfo PtrInfo, 3854 unsigned Alignment, 3855 AtomicOrdering Ordering, 3856 SynchronizationScope SynchScope) { 3857 if (Alignment == 0) // Ensure that codegen never sees alignment 0 3858 Alignment = getEVTAlignment(MemVT); 3859 3860 MachineFunction &MF = getMachineFunction(); 3861 unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 3862 3863 // For now, atomics are considered to be volatile always. 3864 // FIXME: Volatile isn't really correct; we should keep track of atomic 3865 // orderings in the memoperand. 3866 Flags |= MachineMemOperand::MOVolatile; 3867 3868 MachineMemOperand *MMO = 3869 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 3870 3871 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 3872 Ordering, SynchScope); 3873} 3874 3875SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3876 SDValue Chain, 3877 SDValue Ptr, SDValue Cmp, 3878 SDValue Swp, MachineMemOperand *MMO, 3879 AtomicOrdering Ordering, 3880 SynchronizationScope SynchScope) { 3881 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 3882 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 3883 3884 EVT VT = Cmp.getValueType(); 3885 3886 SDVTList VTs = getVTList(VT, MVT::Other); 3887 FoldingSetNodeID ID; 3888 ID.AddInteger(MemVT.getRawBits()); 3889 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 3890 AddNodeIDNode(ID, Opcode, VTs, Ops, 4); 3891 void* IP = 0; 3892 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 3893 cast<AtomicSDNode>(E)->refineAlignment(MMO); 3894 return SDValue(E, 0); 3895 } 3896 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 3897 Ptr, Cmp, Swp, MMO, Ordering, 3898 SynchScope); 3899 CSEMap.InsertNode(N, IP); 3900 AllNodes.push_back(N); 3901 return SDValue(N, 0); 3902} 3903 3904SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3905 SDValue Chain, 3906 SDValue Ptr, SDValue Val, 3907 const Value* PtrVal, 3908 unsigned Alignment, 3909 AtomicOrdering Ordering, 3910 SynchronizationScope SynchScope) { 3911 if (Alignment == 0) // Ensure that codegen never sees alignment 0 3912 Alignment = getEVTAlignment(MemVT); 3913 3914 MachineFunction &MF = getMachineFunction(); 3915 // A monotonic store does not load; a release store "loads" in the sense 3916 // that other stores cannot be sunk past it. 3917 // (An atomicrmw obviously both loads and stores.) 3918 unsigned Flags = MachineMemOperand::MOStore; 3919 if (Opcode != ISD::ATOMIC_STORE || Ordering > Monotonic) 3920 Flags |= MachineMemOperand::MOLoad; 3921 3922 // For now, atomics are considered to be volatile always. 3923 // FIXME: Volatile isn't really correct; we should keep track of atomic 3924 // orderings in the memoperand. 3925 Flags |= MachineMemOperand::MOVolatile; 3926 3927 MachineMemOperand *MMO = 3928 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 3929 MemVT.getStoreSize(), Alignment); 3930 3931 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 3932 Ordering, SynchScope); 3933} 3934 3935SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3936 SDValue Chain, 3937 SDValue Ptr, SDValue Val, 3938 MachineMemOperand *MMO, 3939 AtomicOrdering Ordering, 3940 SynchronizationScope SynchScope) { 3941 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 3942 Opcode == ISD::ATOMIC_LOAD_SUB || 3943 Opcode == ISD::ATOMIC_LOAD_AND || 3944 Opcode == ISD::ATOMIC_LOAD_OR || 3945 Opcode == ISD::ATOMIC_LOAD_XOR || 3946 Opcode == ISD::ATOMIC_LOAD_NAND || 3947 Opcode == ISD::ATOMIC_LOAD_MIN || 3948 Opcode == ISD::ATOMIC_LOAD_MAX || 3949 Opcode == ISD::ATOMIC_LOAD_UMIN || 3950 Opcode == ISD::ATOMIC_LOAD_UMAX || 3951 Opcode == ISD::ATOMIC_SWAP || 3952 Opcode == ISD::ATOMIC_STORE) && 3953 "Invalid Atomic Op"); 3954 3955 EVT VT = Val.getValueType(); 3956 3957 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 3958 getVTList(VT, MVT::Other); 3959 FoldingSetNodeID ID; 3960 ID.AddInteger(MemVT.getRawBits()); 3961 SDValue Ops[] = {Chain, Ptr, Val}; 3962 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3963 void* IP = 0; 3964 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 3965 cast<AtomicSDNode>(E)->refineAlignment(MMO); 3966 return SDValue(E, 0); 3967 } 3968 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 3969 Ptr, Val, MMO, 3970 Ordering, SynchScope); 3971 CSEMap.InsertNode(N, IP); 3972 AllNodes.push_back(N); 3973 return SDValue(N, 0); 3974} 3975 3976SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 3977 EVT VT, SDValue Chain, 3978 SDValue Ptr, 3979 const Value* PtrVal, 3980 unsigned Alignment, 3981 AtomicOrdering Ordering, 3982 SynchronizationScope SynchScope) { 3983 if (Alignment == 0) // Ensure that codegen never sees alignment 0 3984 Alignment = getEVTAlignment(MemVT); 3985 3986 MachineFunction &MF = getMachineFunction(); 3987 // A monotonic load does not store; an acquire load "stores" in the sense 3988 // that other loads cannot be hoisted past it. 3989 unsigned Flags = MachineMemOperand::MOLoad; 3990 if (Ordering > Monotonic) 3991 Flags |= MachineMemOperand::MOStore; 3992 3993 // For now, atomics are considered to be volatile always. 3994 // FIXME: Volatile isn't really correct; we should keep track of atomic 3995 // orderings in the memoperand. 3996 Flags |= MachineMemOperand::MOVolatile; 3997 3998 MachineMemOperand *MMO = 3999 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4000 MemVT.getStoreSize(), Alignment); 4001 4002 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 4003 Ordering, SynchScope); 4004} 4005 4006SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4007 EVT VT, SDValue Chain, 4008 SDValue Ptr, 4009 MachineMemOperand *MMO, 4010 AtomicOrdering Ordering, 4011 SynchronizationScope SynchScope) { 4012 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4013 4014 SDVTList VTs = getVTList(VT, MVT::Other); 4015 FoldingSetNodeID ID; 4016 ID.AddInteger(MemVT.getRawBits()); 4017 SDValue Ops[] = {Chain, Ptr}; 4018 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 4019 void* IP = 0; 4020 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4021 cast<AtomicSDNode>(E)->refineAlignment(MMO); 4022 return SDValue(E, 0); 4023 } 4024 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 4025 Ptr, MMO, Ordering, SynchScope); 4026 CSEMap.InsertNode(N, IP); 4027 AllNodes.push_back(N); 4028 return SDValue(N, 0); 4029} 4030 4031/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4032SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4033 DebugLoc dl) { 4034 if (NumOps == 1) 4035 return Ops[0]; 4036 4037 SmallVector<EVT, 4> VTs; 4038 VTs.reserve(NumOps); 4039 for (unsigned i = 0; i < NumOps; ++i) 4040 VTs.push_back(Ops[i].getValueType()); 4041 return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4042 Ops, NumOps); 4043} 4044 4045SDValue 4046SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, 4047 const EVT *VTs, unsigned NumVTs, 4048 const SDValue *Ops, unsigned NumOps, 4049 EVT MemVT, MachinePointerInfo PtrInfo, 4050 unsigned Align, bool Vol, 4051 bool ReadMem, bool WriteMem) { 4052 return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4053 MemVT, PtrInfo, Align, Vol, 4054 ReadMem, WriteMem); 4055} 4056 4057SDValue 4058SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, 4059 const SDValue *Ops, unsigned NumOps, 4060 EVT MemVT, MachinePointerInfo PtrInfo, 4061 unsigned Align, bool Vol, 4062 bool ReadMem, bool WriteMem) { 4063 if (Align == 0) // Ensure that codegen never sees alignment 0 4064 Align = getEVTAlignment(MemVT); 4065 4066 MachineFunction &MF = getMachineFunction(); 4067 unsigned Flags = 0; 4068 if (WriteMem) 4069 Flags |= MachineMemOperand::MOStore; 4070 if (ReadMem) 4071 Flags |= MachineMemOperand::MOLoad; 4072 if (Vol) 4073 Flags |= MachineMemOperand::MOVolatile; 4074 MachineMemOperand *MMO = 4075 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4076 4077 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4078} 4079 4080SDValue 4081SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, 4082 const SDValue *Ops, unsigned NumOps, 4083 EVT MemVT, MachineMemOperand *MMO) { 4084 assert((Opcode == ISD::INTRINSIC_VOID || 4085 Opcode == ISD::INTRINSIC_W_CHAIN || 4086 Opcode == ISD::PREFETCH || 4087 (Opcode <= INT_MAX && 4088 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4089 "Opcode is not a memory-accessing opcode!"); 4090 4091 // Memoize the node unless it returns a flag. 4092 MemIntrinsicSDNode *N; 4093 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4094 FoldingSetNodeID ID; 4095 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4096 void *IP = 0; 4097 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4098 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4099 return SDValue(E, 0); 4100 } 4101 4102 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, 4103 MemVT, MMO); 4104 CSEMap.InsertNode(N, IP); 4105 } else { 4106 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, 4107 MemVT, MMO); 4108 } 4109 AllNodes.push_back(N); 4110 return SDValue(N, 0); 4111} 4112 4113/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4114/// MachinePointerInfo record from it. This is particularly useful because the 4115/// code generator has many cases where it doesn't bother passing in a 4116/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4117static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4118 // If this is FI+Offset, we can model it. 4119 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4120 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4121 4122 // If this is (FI+Offset1)+Offset2, we can model it. 4123 if (Ptr.getOpcode() != ISD::ADD || 4124 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4125 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4126 return MachinePointerInfo(); 4127 4128 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4129 return MachinePointerInfo::getFixedStack(FI, Offset+ 4130 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4131} 4132 4133/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4134/// MachinePointerInfo record from it. This is particularly useful because the 4135/// code generator has many cases where it doesn't bother passing in a 4136/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4137static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4138 // If the 'Offset' value isn't a constant, we can't handle this. 4139 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4140 return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4141 if (OffsetOp.getOpcode() == ISD::UNDEF) 4142 return InferPointerInfo(Ptr); 4143 return MachinePointerInfo(); 4144} 4145 4146 4147SDValue 4148SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4149 EVT VT, DebugLoc dl, SDValue Chain, 4150 SDValue Ptr, SDValue Offset, 4151 MachinePointerInfo PtrInfo, EVT MemVT, 4152 bool isVolatile, bool isNonTemporal, bool isInvariant, 4153 unsigned Alignment, const MDNode *TBAAInfo) { 4154 assert(Chain.getValueType() == MVT::Other && 4155 "Invalid chain type"); 4156 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4157 Alignment = getEVTAlignment(VT); 4158 4159 unsigned Flags = MachineMemOperand::MOLoad; 4160 if (isVolatile) 4161 Flags |= MachineMemOperand::MOVolatile; 4162 if (isNonTemporal) 4163 Flags |= MachineMemOperand::MONonTemporal; 4164 if (isInvariant) 4165 Flags |= MachineMemOperand::MOInvariant; 4166 4167 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4168 // clients. 4169 if (PtrInfo.V == 0) 4170 PtrInfo = InferPointerInfo(Ptr, Offset); 4171 4172 MachineFunction &MF = getMachineFunction(); 4173 MachineMemOperand *MMO = 4174 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4175 TBAAInfo); 4176 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4177} 4178 4179SDValue 4180SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4181 EVT VT, DebugLoc dl, SDValue Chain, 4182 SDValue Ptr, SDValue Offset, EVT MemVT, 4183 MachineMemOperand *MMO) { 4184 if (VT == MemVT) { 4185 ExtType = ISD::NON_EXTLOAD; 4186 } else if (ExtType == ISD::NON_EXTLOAD) { 4187 assert(VT == MemVT && "Non-extending load from different memory type!"); 4188 } else { 4189 // Extending load. 4190 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4191 "Should only be an extending load, not truncating!"); 4192 assert(VT.isInteger() == MemVT.isInteger() && 4193 "Cannot convert from FP to Int or Int -> FP!"); 4194 assert(VT.isVector() == MemVT.isVector() && 4195 "Cannot use trunc store to convert to or from a vector!"); 4196 assert((!VT.isVector() || 4197 VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4198 "Cannot use trunc store to change the number of vector elements!"); 4199 } 4200 4201 bool Indexed = AM != ISD::UNINDEXED; 4202 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4203 "Unindexed load with an offset!"); 4204 4205 SDVTList VTs = Indexed ? 4206 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4207 SDValue Ops[] = { Chain, Ptr, Offset }; 4208 FoldingSetNodeID ID; 4209 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4210 ID.AddInteger(MemVT.getRawBits()); 4211 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4212 MMO->isNonTemporal(), 4213 MMO->isInvariant())); 4214 void *IP = 0; 4215 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4216 cast<LoadSDNode>(E)->refineAlignment(MMO); 4217 return SDValue(E, 0); 4218 } 4219 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl, VTs, AM, ExtType, 4220 MemVT, MMO); 4221 CSEMap.InsertNode(N, IP); 4222 AllNodes.push_back(N); 4223 return SDValue(N, 0); 4224} 4225 4226SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, 4227 SDValue Chain, SDValue Ptr, 4228 MachinePointerInfo PtrInfo, 4229 bool isVolatile, bool isNonTemporal, 4230 bool isInvariant, unsigned Alignment, 4231 const MDNode *TBAAInfo) { 4232 SDValue Undef = getUNDEF(Ptr.getValueType()); 4233 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4234 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4235 TBAAInfo); 4236} 4237 4238SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, 4239 SDValue Chain, SDValue Ptr, 4240 MachinePointerInfo PtrInfo, EVT MemVT, 4241 bool isVolatile, bool isNonTemporal, 4242 unsigned Alignment, const MDNode *TBAAInfo) { 4243 SDValue Undef = getUNDEF(Ptr.getValueType()); 4244 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4245 PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4246 TBAAInfo); 4247} 4248 4249 4250SDValue 4251SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, 4252 SDValue Offset, ISD::MemIndexedMode AM) { 4253 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4254 assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4255 "Load is already a indexed load!"); 4256 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4257 LD->getChain(), Base, Offset, LD->getPointerInfo(), 4258 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4259 false, LD->getAlignment()); 4260} 4261 4262SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, 4263 SDValue Ptr, MachinePointerInfo PtrInfo, 4264 bool isVolatile, bool isNonTemporal, 4265 unsigned Alignment, const MDNode *TBAAInfo) { 4266 assert(Chain.getValueType() == MVT::Other && 4267 "Invalid chain type"); 4268 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4269 Alignment = getEVTAlignment(Val.getValueType()); 4270 4271 unsigned Flags = MachineMemOperand::MOStore; 4272 if (isVolatile) 4273 Flags |= MachineMemOperand::MOVolatile; 4274 if (isNonTemporal) 4275 Flags |= MachineMemOperand::MONonTemporal; 4276 4277 if (PtrInfo.V == 0) 4278 PtrInfo = InferPointerInfo(Ptr); 4279 4280 MachineFunction &MF = getMachineFunction(); 4281 MachineMemOperand *MMO = 4282 MF.getMachineMemOperand(PtrInfo, Flags, 4283 Val.getValueType().getStoreSize(), Alignment, 4284 TBAAInfo); 4285 4286 return getStore(Chain, dl, Val, Ptr, MMO); 4287} 4288 4289SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, 4290 SDValue Ptr, MachineMemOperand *MMO) { 4291 assert(Chain.getValueType() == MVT::Other && 4292 "Invalid chain type"); 4293 EVT VT = Val.getValueType(); 4294 SDVTList VTs = getVTList(MVT::Other); 4295 SDValue Undef = getUNDEF(Ptr.getValueType()); 4296 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4297 FoldingSetNodeID ID; 4298 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4299 ID.AddInteger(VT.getRawBits()); 4300 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4301 MMO->isNonTemporal(), MMO->isInvariant())); 4302 void *IP = 0; 4303 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4304 cast<StoreSDNode>(E)->refineAlignment(MMO); 4305 return SDValue(E, 0); 4306 } 4307 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, 4308 false, VT, MMO); 4309 CSEMap.InsertNode(N, IP); 4310 AllNodes.push_back(N); 4311 return SDValue(N, 0); 4312} 4313 4314SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, 4315 SDValue Ptr, MachinePointerInfo PtrInfo, 4316 EVT SVT,bool isVolatile, bool isNonTemporal, 4317 unsigned Alignment, 4318 const MDNode *TBAAInfo) { 4319 assert(Chain.getValueType() == MVT::Other && 4320 "Invalid chain type"); 4321 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4322 Alignment = getEVTAlignment(SVT); 4323 4324 unsigned Flags = MachineMemOperand::MOStore; 4325 if (isVolatile) 4326 Flags |= MachineMemOperand::MOVolatile; 4327 if (isNonTemporal) 4328 Flags |= MachineMemOperand::MONonTemporal; 4329 4330 if (PtrInfo.V == 0) 4331 PtrInfo = InferPointerInfo(Ptr); 4332 4333 MachineFunction &MF = getMachineFunction(); 4334 MachineMemOperand *MMO = 4335 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4336 TBAAInfo); 4337 4338 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4339} 4340 4341SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, 4342 SDValue Ptr, EVT SVT, 4343 MachineMemOperand *MMO) { 4344 EVT VT = Val.getValueType(); 4345 4346 assert(Chain.getValueType() == MVT::Other && 4347 "Invalid chain type"); 4348 if (VT == SVT) 4349 return getStore(Chain, dl, Val, Ptr, MMO); 4350 4351 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4352 "Should only be a truncating store, not extending!"); 4353 assert(VT.isInteger() == SVT.isInteger() && 4354 "Can't do FP-INT conversion!"); 4355 assert(VT.isVector() == SVT.isVector() && 4356 "Cannot use trunc store to convert to or from a vector!"); 4357 assert((!VT.isVector() || 4358 VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4359 "Cannot use trunc store to change the number of vector elements!"); 4360 4361 SDVTList VTs = getVTList(MVT::Other); 4362 SDValue Undef = getUNDEF(Ptr.getValueType()); 4363 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4364 FoldingSetNodeID ID; 4365 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4366 ID.AddInteger(SVT.getRawBits()); 4367 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4368 MMO->isNonTemporal(), MMO->isInvariant())); 4369 void *IP = 0; 4370 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4371 cast<StoreSDNode>(E)->refineAlignment(MMO); 4372 return SDValue(E, 0); 4373 } 4374 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, 4375 true, SVT, MMO); 4376 CSEMap.InsertNode(N, IP); 4377 AllNodes.push_back(N); 4378 return SDValue(N, 0); 4379} 4380 4381SDValue 4382SelectionDAG::getIndexedStore(SDValue OrigStore, DebugLoc dl, SDValue Base, 4383 SDValue Offset, ISD::MemIndexedMode AM) { 4384 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4385 assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4386 "Store is already a indexed store!"); 4387 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4388 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4389 FoldingSetNodeID ID; 4390 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4391 ID.AddInteger(ST->getMemoryVT().getRawBits()); 4392 ID.AddInteger(ST->getRawSubclassData()); 4393 void *IP = 0; 4394 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4395 return SDValue(E, 0); 4396 4397 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, AM, 4398 ST->isTruncatingStore(), 4399 ST->getMemoryVT(), 4400 ST->getMemOperand()); 4401 CSEMap.InsertNode(N, IP); 4402 AllNodes.push_back(N); 4403 return SDValue(N, 0); 4404} 4405 4406SDValue SelectionDAG::getVAArg(EVT VT, DebugLoc dl, 4407 SDValue Chain, SDValue Ptr, 4408 SDValue SV, 4409 unsigned Align) { 4410 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4411 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4412} 4413 4414SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 4415 const SDUse *Ops, unsigned NumOps) { 4416 switch (NumOps) { 4417 case 0: return getNode(Opcode, DL, VT); 4418 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4419 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4420 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4421 default: break; 4422 } 4423 4424 // Copy from an SDUse array into an SDValue array for use with 4425 // the regular getNode logic. 4426 SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4427 return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4428} 4429 4430SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 4431 const SDValue *Ops, unsigned NumOps) { 4432 switch (NumOps) { 4433 case 0: return getNode(Opcode, DL, VT); 4434 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4435 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4436 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4437 default: break; 4438 } 4439 4440 switch (Opcode) { 4441 default: break; 4442 case ISD::SELECT_CC: { 4443 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4444 assert(Ops[0].getValueType() == Ops[1].getValueType() && 4445 "LHS and RHS of condition must have same type!"); 4446 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4447 "True and False arms of SelectCC must have same type!"); 4448 assert(Ops[2].getValueType() == VT && 4449 "select_cc node must be of same type as true and false value!"); 4450 break; 4451 } 4452 case ISD::BR_CC: { 4453 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4454 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4455 "LHS/RHS of comparison should match types!"); 4456 break; 4457 } 4458 } 4459 4460 // Memoize nodes. 4461 SDNode *N; 4462 SDVTList VTs = getVTList(VT); 4463 4464 if (VT != MVT::Glue) { 4465 FoldingSetNodeID ID; 4466 AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4467 void *IP = 0; 4468 4469 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4470 return SDValue(E, 0); 4471 4472 N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); 4473 CSEMap.InsertNode(N, IP); 4474 } else { 4475 N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); 4476 } 4477 4478 AllNodes.push_back(N); 4479#ifndef NDEBUG 4480 VerifySDNode(N); 4481#endif 4482 return SDValue(N, 0); 4483} 4484 4485SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 4486 const std::vector<EVT> &ResultTys, 4487 const SDValue *Ops, unsigned NumOps) { 4488 return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4489 Ops, NumOps); 4490} 4491 4492SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 4493 const EVT *VTs, unsigned NumVTs, 4494 const SDValue *Ops, unsigned NumOps) { 4495 if (NumVTs == 1) 4496 return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4497 return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4498} 4499 4500SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4501 const SDValue *Ops, unsigned NumOps) { 4502 if (VTList.NumVTs == 1) 4503 return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4504 4505#if 0 4506 switch (Opcode) { 4507 // FIXME: figure out how to safely handle things like 4508 // int foo(int x) { return 1 << (x & 255); } 4509 // int bar() { return foo(256); } 4510 case ISD::SRA_PARTS: 4511 case ISD::SRL_PARTS: 4512 case ISD::SHL_PARTS: 4513 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4514 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4515 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4516 else if (N3.getOpcode() == ISD::AND) 4517 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4518 // If the and is only masking out bits that cannot effect the shift, 4519 // eliminate the and. 4520 unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4521 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4522 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4523 } 4524 break; 4525 } 4526#endif 4527 4528 // Memoize the node unless it returns a flag. 4529 SDNode *N; 4530 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4531 FoldingSetNodeID ID; 4532 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4533 void *IP = 0; 4534 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4535 return SDValue(E, 0); 4536 4537 if (NumOps == 1) { 4538 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); 4539 } else if (NumOps == 2) { 4540 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); 4541 } else if (NumOps == 3) { 4542 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], 4543 Ops[2]); 4544 } else { 4545 N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); 4546 } 4547 CSEMap.InsertNode(N, IP); 4548 } else { 4549 if (NumOps == 1) { 4550 N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); 4551 } else if (NumOps == 2) { 4552 N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); 4553 } else if (NumOps == 3) { 4554 N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], 4555 Ops[2]); 4556 } else { 4557 N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); 4558 } 4559 } 4560 AllNodes.push_back(N); 4561#ifndef NDEBUG 4562 VerifySDNode(N); 4563#endif 4564 return SDValue(N, 0); 4565} 4566 4567SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList) { 4568 return getNode(Opcode, DL, VTList, 0, 0); 4569} 4570 4571SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4572 SDValue N1) { 4573 SDValue Ops[] = { N1 }; 4574 return getNode(Opcode, DL, VTList, Ops, 1); 4575} 4576 4577SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4578 SDValue N1, SDValue N2) { 4579 SDValue Ops[] = { N1, N2 }; 4580 return getNode(Opcode, DL, VTList, Ops, 2); 4581} 4582 4583SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4584 SDValue N1, SDValue N2, SDValue N3) { 4585 SDValue Ops[] = { N1, N2, N3 }; 4586 return getNode(Opcode, DL, VTList, Ops, 3); 4587} 4588 4589SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4590 SDValue N1, SDValue N2, SDValue N3, 4591 SDValue N4) { 4592 SDValue Ops[] = { N1, N2, N3, N4 }; 4593 return getNode(Opcode, DL, VTList, Ops, 4); 4594} 4595 4596SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4597 SDValue N1, SDValue N2, SDValue N3, 4598 SDValue N4, SDValue N5) { 4599 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4600 return getNode(Opcode, DL, VTList, Ops, 5); 4601} 4602 4603SDVTList SelectionDAG::getVTList(EVT VT) { 4604 return makeVTList(SDNode::getValueTypeList(VT), 1); 4605} 4606 4607SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4608 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4609 E = VTList.rend(); I != E; ++I) 4610 if (I->NumVTs == 2 && I->VTs[0] == VT1 && I->VTs[1] == VT2) 4611 return *I; 4612 4613 EVT *Array = Allocator.Allocate<EVT>(2); 4614 Array[0] = VT1; 4615 Array[1] = VT2; 4616 SDVTList Result = makeVTList(Array, 2); 4617 VTList.push_back(Result); 4618 return Result; 4619} 4620 4621SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 4622 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4623 E = VTList.rend(); I != E; ++I) 4624 if (I->NumVTs == 3 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 4625 I->VTs[2] == VT3) 4626 return *I; 4627 4628 EVT *Array = Allocator.Allocate<EVT>(3); 4629 Array[0] = VT1; 4630 Array[1] = VT2; 4631 Array[2] = VT3; 4632 SDVTList Result = makeVTList(Array, 3); 4633 VTList.push_back(Result); 4634 return Result; 4635} 4636 4637SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 4638 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4639 E = VTList.rend(); I != E; ++I) 4640 if (I->NumVTs == 4 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 4641 I->VTs[2] == VT3 && I->VTs[3] == VT4) 4642 return *I; 4643 4644 EVT *Array = Allocator.Allocate<EVT>(4); 4645 Array[0] = VT1; 4646 Array[1] = VT2; 4647 Array[2] = VT3; 4648 Array[3] = VT4; 4649 SDVTList Result = makeVTList(Array, 4); 4650 VTList.push_back(Result); 4651 return Result; 4652} 4653 4654SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 4655 switch (NumVTs) { 4656 case 0: llvm_unreachable("Cannot have nodes without results!"); 4657 case 1: return getVTList(VTs[0]); 4658 case 2: return getVTList(VTs[0], VTs[1]); 4659 case 3: return getVTList(VTs[0], VTs[1], VTs[2]); 4660 case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]); 4661 default: break; 4662 } 4663 4664 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4665 E = VTList.rend(); I != E; ++I) { 4666 if (I->NumVTs != NumVTs || VTs[0] != I->VTs[0] || VTs[1] != I->VTs[1]) 4667 continue; 4668 4669 bool NoMatch = false; 4670 for (unsigned i = 2; i != NumVTs; ++i) 4671 if (VTs[i] != I->VTs[i]) { 4672 NoMatch = true; 4673 break; 4674 } 4675 if (!NoMatch) 4676 return *I; 4677 } 4678 4679 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 4680 std::copy(VTs, VTs+NumVTs, Array); 4681 SDVTList Result = makeVTList(Array, NumVTs); 4682 VTList.push_back(Result); 4683 return Result; 4684} 4685 4686 4687/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 4688/// specified operands. If the resultant node already exists in the DAG, 4689/// this does not modify the specified node, instead it returns the node that 4690/// already exists. If the resultant node does not exist in the DAG, the 4691/// input node is returned. As a degenerate case, if you specify the same 4692/// input operands as the node already has, the input node is returned. 4693SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 4694 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 4695 4696 // Check to see if there is no change. 4697 if (Op == N->getOperand(0)) return N; 4698 4699 // See if the modified node already exists. 4700 void *InsertPos = 0; 4701 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 4702 return Existing; 4703 4704 // Nope it doesn't. Remove the node from its current place in the maps. 4705 if (InsertPos) 4706 if (!RemoveNodeFromCSEMaps(N)) 4707 InsertPos = 0; 4708 4709 // Now we update the operands. 4710 N->OperandList[0].set(Op); 4711 4712 // If this gets put into a CSE map, add it. 4713 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4714 return N; 4715} 4716 4717SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 4718 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 4719 4720 // Check to see if there is no change. 4721 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 4722 return N; // No operands changed, just return the input node. 4723 4724 // See if the modified node already exists. 4725 void *InsertPos = 0; 4726 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 4727 return Existing; 4728 4729 // Nope it doesn't. Remove the node from its current place in the maps. 4730 if (InsertPos) 4731 if (!RemoveNodeFromCSEMaps(N)) 4732 InsertPos = 0; 4733 4734 // Now we update the operands. 4735 if (N->OperandList[0] != Op1) 4736 N->OperandList[0].set(Op1); 4737 if (N->OperandList[1] != Op2) 4738 N->OperandList[1].set(Op2); 4739 4740 // If this gets put into a CSE map, add it. 4741 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4742 return N; 4743} 4744 4745SDNode *SelectionDAG:: 4746UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 4747 SDValue Ops[] = { Op1, Op2, Op3 }; 4748 return UpdateNodeOperands(N, Ops, 3); 4749} 4750 4751SDNode *SelectionDAG:: 4752UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 4753 SDValue Op3, SDValue Op4) { 4754 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 4755 return UpdateNodeOperands(N, Ops, 4); 4756} 4757 4758SDNode *SelectionDAG:: 4759UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 4760 SDValue Op3, SDValue Op4, SDValue Op5) { 4761 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 4762 return UpdateNodeOperands(N, Ops, 5); 4763} 4764 4765SDNode *SelectionDAG:: 4766UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 4767 assert(N->getNumOperands() == NumOps && 4768 "Update with wrong number of operands"); 4769 4770 // Check to see if there is no change. 4771 bool AnyChange = false; 4772 for (unsigned i = 0; i != NumOps; ++i) { 4773 if (Ops[i] != N->getOperand(i)) { 4774 AnyChange = true; 4775 break; 4776 } 4777 } 4778 4779 // No operands changed, just return the input node. 4780 if (!AnyChange) return N; 4781 4782 // See if the modified node already exists. 4783 void *InsertPos = 0; 4784 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 4785 return Existing; 4786 4787 // Nope it doesn't. Remove the node from its current place in the maps. 4788 if (InsertPos) 4789 if (!RemoveNodeFromCSEMaps(N)) 4790 InsertPos = 0; 4791 4792 // Now we update the operands. 4793 for (unsigned i = 0; i != NumOps; ++i) 4794 if (N->OperandList[i] != Ops[i]) 4795 N->OperandList[i].set(Ops[i]); 4796 4797 // If this gets put into a CSE map, add it. 4798 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4799 return N; 4800} 4801 4802/// DropOperands - Release the operands and set this node to have 4803/// zero operands. 4804void SDNode::DropOperands() { 4805 // Unlike the code in MorphNodeTo that does this, we don't need to 4806 // watch for dead nodes here. 4807 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 4808 SDUse &Use = *I++; 4809 Use.set(SDValue()); 4810 } 4811} 4812 4813/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 4814/// machine opcode. 4815/// 4816SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4817 EVT VT) { 4818 SDVTList VTs = getVTList(VT); 4819 return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 4820} 4821 4822SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4823 EVT VT, SDValue Op1) { 4824 SDVTList VTs = getVTList(VT); 4825 SDValue Ops[] = { Op1 }; 4826 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 4827} 4828 4829SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4830 EVT VT, SDValue Op1, 4831 SDValue Op2) { 4832 SDVTList VTs = getVTList(VT); 4833 SDValue Ops[] = { Op1, Op2 }; 4834 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 4835} 4836 4837SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4838 EVT VT, SDValue Op1, 4839 SDValue Op2, SDValue Op3) { 4840 SDVTList VTs = getVTList(VT); 4841 SDValue Ops[] = { Op1, Op2, Op3 }; 4842 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4843} 4844 4845SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4846 EVT VT, const SDValue *Ops, 4847 unsigned NumOps) { 4848 SDVTList VTs = getVTList(VT); 4849 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4850} 4851 4852SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4853 EVT VT1, EVT VT2, const SDValue *Ops, 4854 unsigned NumOps) { 4855 SDVTList VTs = getVTList(VT1, VT2); 4856 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4857} 4858 4859SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4860 EVT VT1, EVT VT2) { 4861 SDVTList VTs = getVTList(VT1, VT2); 4862 return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 4863} 4864 4865SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4866 EVT VT1, EVT VT2, EVT VT3, 4867 const SDValue *Ops, unsigned NumOps) { 4868 SDVTList VTs = getVTList(VT1, VT2, VT3); 4869 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4870} 4871 4872SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4873 EVT VT1, EVT VT2, EVT VT3, EVT VT4, 4874 const SDValue *Ops, unsigned NumOps) { 4875 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 4876 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 4877} 4878 4879SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4880 EVT VT1, EVT VT2, 4881 SDValue Op1) { 4882 SDVTList VTs = getVTList(VT1, VT2); 4883 SDValue Ops[] = { Op1 }; 4884 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 4885} 4886 4887SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4888 EVT VT1, EVT VT2, 4889 SDValue Op1, SDValue Op2) { 4890 SDVTList VTs = getVTList(VT1, VT2); 4891 SDValue Ops[] = { Op1, Op2 }; 4892 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 4893} 4894 4895SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4896 EVT VT1, EVT VT2, 4897 SDValue Op1, SDValue Op2, 4898 SDValue Op3) { 4899 SDVTList VTs = getVTList(VT1, VT2); 4900 SDValue Ops[] = { Op1, Op2, Op3 }; 4901 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4902} 4903 4904SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4905 EVT VT1, EVT VT2, EVT VT3, 4906 SDValue Op1, SDValue Op2, 4907 SDValue Op3) { 4908 SDVTList VTs = getVTList(VT1, VT2, VT3); 4909 SDValue Ops[] = { Op1, Op2, Op3 }; 4910 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 4911} 4912 4913SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 4914 SDVTList VTs, const SDValue *Ops, 4915 unsigned NumOps) { 4916 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 4917 // Reset the NodeID to -1. 4918 N->setNodeId(-1); 4919 return N; 4920} 4921 4922/// UpdadeDebugLocOnMergedSDNode - If the opt level is -O0 then it throws away 4923/// the line number information on the merged node since it is not possible to 4924/// preserve the information that operation is associated with multiple lines. 4925/// This will make the debugger working better at -O0, were there is a higher 4926/// probability having other instructions associated with that line. 4927/// 4928SDNode *SelectionDAG::UpdadeDebugLocOnMergedSDNode(SDNode *N, DebugLoc OLoc) { 4929 DebugLoc NLoc = N->getDebugLoc(); 4930 if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && (OLoc != NLoc)) { 4931 N->setDebugLoc(DebugLoc()); 4932 } 4933 return N; 4934} 4935 4936/// MorphNodeTo - This *mutates* the specified node to have the specified 4937/// return type, opcode, and operands. 4938/// 4939/// Note that MorphNodeTo returns the resultant node. If there is already a 4940/// node of the specified opcode and operands, it returns that node instead of 4941/// the current one. Note that the DebugLoc need not be the same. 4942/// 4943/// Using MorphNodeTo is faster than creating a new node and swapping it in 4944/// with ReplaceAllUsesWith both because it often avoids allocating a new 4945/// node, and because it doesn't require CSE recalculation for any of 4946/// the node's users. 4947/// 4948SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 4949 SDVTList VTs, const SDValue *Ops, 4950 unsigned NumOps) { 4951 // If an identical node already exists, use it. 4952 void *IP = 0; 4953 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 4954 FoldingSetNodeID ID; 4955 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 4956 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 4957 return UpdadeDebugLocOnMergedSDNode(ON, N->getDebugLoc()); 4958 } 4959 4960 if (!RemoveNodeFromCSEMaps(N)) 4961 IP = 0; 4962 4963 // Start the morphing. 4964 N->NodeType = Opc; 4965 N->ValueList = VTs.VTs; 4966 N->NumValues = VTs.NumVTs; 4967 4968 // Clear the operands list, updating used nodes to remove this from their 4969 // use list. Keep track of any operands that become dead as a result. 4970 SmallPtrSet<SDNode*, 16> DeadNodeSet; 4971 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 4972 SDUse &Use = *I++; 4973 SDNode *Used = Use.getNode(); 4974 Use.set(SDValue()); 4975 if (Used->use_empty()) 4976 DeadNodeSet.insert(Used); 4977 } 4978 4979 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 4980 // Initialize the memory references information. 4981 MN->setMemRefs(0, 0); 4982 // If NumOps is larger than the # of operands we can have in a 4983 // MachineSDNode, reallocate the operand list. 4984 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 4985 if (MN->OperandsNeedDelete) 4986 delete[] MN->OperandList; 4987 if (NumOps > array_lengthof(MN->LocalOperands)) 4988 // We're creating a final node that will live unmorphed for the 4989 // remainder of the current SelectionDAG iteration, so we can allocate 4990 // the operands directly out of a pool with no recycling metadata. 4991 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 4992 Ops, NumOps); 4993 else 4994 MN->InitOperands(MN->LocalOperands, Ops, NumOps); 4995 MN->OperandsNeedDelete = false; 4996 } else 4997 MN->InitOperands(MN->OperandList, Ops, NumOps); 4998 } else { 4999 // If NumOps is larger than the # of operands we currently have, reallocate 5000 // the operand list. 5001 if (NumOps > N->NumOperands) { 5002 if (N->OperandsNeedDelete) 5003 delete[] N->OperandList; 5004 N->InitOperands(new SDUse[NumOps], Ops, NumOps); 5005 N->OperandsNeedDelete = true; 5006 } else 5007 N->InitOperands(N->OperandList, Ops, NumOps); 5008 } 5009 5010 // Delete any nodes that are still dead after adding the uses for the 5011 // new operands. 5012 if (!DeadNodeSet.empty()) { 5013 SmallVector<SDNode *, 16> DeadNodes; 5014 for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 5015 E = DeadNodeSet.end(); I != E; ++I) 5016 if ((*I)->use_empty()) 5017 DeadNodes.push_back(*I); 5018 RemoveDeadNodes(DeadNodes); 5019 } 5020 5021 if (IP) 5022 CSEMap.InsertNode(N, IP); // Memoize the new node. 5023 return N; 5024} 5025 5026 5027/// getMachineNode - These are used for target selectors to create a new node 5028/// with specified return type(s), MachineInstr opcode, and operands. 5029/// 5030/// Note that getMachineNode returns the resultant node. If there is already a 5031/// node of the specified opcode and operands, it returns that node instead of 5032/// the current one. 5033MachineSDNode * 5034SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT) { 5035 SDVTList VTs = getVTList(VT); 5036 return getMachineNode(Opcode, dl, VTs, 0, 0); 5037} 5038 5039MachineSDNode * 5040SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, SDValue Op1) { 5041 SDVTList VTs = getVTList(VT); 5042 SDValue Ops[] = { Op1 }; 5043 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5044} 5045 5046MachineSDNode * 5047SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5048 SDValue Op1, SDValue Op2) { 5049 SDVTList VTs = getVTList(VT); 5050 SDValue Ops[] = { Op1, Op2 }; 5051 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5052} 5053 5054MachineSDNode * 5055SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5056 SDValue Op1, SDValue Op2, SDValue Op3) { 5057 SDVTList VTs = getVTList(VT); 5058 SDValue Ops[] = { Op1, Op2, Op3 }; 5059 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5060} 5061 5062MachineSDNode * 5063SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5064 const SDValue *Ops, unsigned NumOps) { 5065 SDVTList VTs = getVTList(VT); 5066 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5067} 5068 5069MachineSDNode * 5070SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, EVT VT2) { 5071 SDVTList VTs = getVTList(VT1, VT2); 5072 return getMachineNode(Opcode, dl, VTs, 0, 0); 5073} 5074 5075MachineSDNode * 5076SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5077 EVT VT1, EVT VT2, SDValue Op1) { 5078 SDVTList VTs = getVTList(VT1, VT2); 5079 SDValue Ops[] = { Op1 }; 5080 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5081} 5082 5083MachineSDNode * 5084SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5085 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5086 SDVTList VTs = getVTList(VT1, VT2); 5087 SDValue Ops[] = { Op1, Op2 }; 5088 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5089} 5090 5091MachineSDNode * 5092SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5093 EVT VT1, EVT VT2, SDValue Op1, 5094 SDValue Op2, SDValue Op3) { 5095 SDVTList VTs = getVTList(VT1, VT2); 5096 SDValue Ops[] = { Op1, Op2, Op3 }; 5097 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5098} 5099 5100MachineSDNode * 5101SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5102 EVT VT1, EVT VT2, 5103 const SDValue *Ops, unsigned NumOps) { 5104 SDVTList VTs = getVTList(VT1, VT2); 5105 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5106} 5107 5108MachineSDNode * 5109SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5110 EVT VT1, EVT VT2, EVT VT3, 5111 SDValue Op1, SDValue Op2) { 5112 SDVTList VTs = getVTList(VT1, VT2, VT3); 5113 SDValue Ops[] = { Op1, Op2 }; 5114 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5115} 5116 5117MachineSDNode * 5118SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5119 EVT VT1, EVT VT2, EVT VT3, 5120 SDValue Op1, SDValue Op2, SDValue Op3) { 5121 SDVTList VTs = getVTList(VT1, VT2, VT3); 5122 SDValue Ops[] = { Op1, Op2, Op3 }; 5123 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); 5124} 5125 5126MachineSDNode * 5127SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5128 EVT VT1, EVT VT2, EVT VT3, 5129 const SDValue *Ops, unsigned NumOps) { 5130 SDVTList VTs = getVTList(VT1, VT2, VT3); 5131 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5132} 5133 5134MachineSDNode * 5135SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, 5136 EVT VT2, EVT VT3, EVT VT4, 5137 const SDValue *Ops, unsigned NumOps) { 5138 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5139 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5140} 5141 5142MachineSDNode * 5143SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5144 const std::vector<EVT> &ResultTys, 5145 const SDValue *Ops, unsigned NumOps) { 5146 SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5147 return getMachineNode(Opcode, dl, VTs, Ops, NumOps); 5148} 5149 5150MachineSDNode * 5151SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, 5152 const SDValue *Ops, unsigned NumOps) { 5153 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5154 MachineSDNode *N; 5155 void *IP = 0; 5156 5157 if (DoCSE) { 5158 FoldingSetNodeID ID; 5159 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5160 IP = 0; 5161 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 5162 return cast<MachineSDNode>(UpdadeDebugLocOnMergedSDNode(E, DL)); 5163 } 5164 } 5165 5166 // Allocate a new MachineSDNode. 5167 N = new (NodeAllocator) MachineSDNode(~Opcode, DL, VTs); 5168 5169 // Initialize the operands list. 5170 if (NumOps > array_lengthof(N->LocalOperands)) 5171 // We're creating a final node that will live unmorphed for the 5172 // remainder of the current SelectionDAG iteration, so we can allocate 5173 // the operands directly out of a pool with no recycling metadata. 5174 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5175 Ops, NumOps); 5176 else 5177 N->InitOperands(N->LocalOperands, Ops, NumOps); 5178 N->OperandsNeedDelete = false; 5179 5180 if (DoCSE) 5181 CSEMap.InsertNode(N, IP); 5182 5183 AllNodes.push_back(N); 5184#ifndef NDEBUG 5185 VerifyMachineNode(N); 5186#endif 5187 return N; 5188} 5189 5190/// getTargetExtractSubreg - A convenience function for creating 5191/// TargetOpcode::EXTRACT_SUBREG nodes. 5192SDValue 5193SelectionDAG::getTargetExtractSubreg(int SRIdx, DebugLoc DL, EVT VT, 5194 SDValue Operand) { 5195 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5196 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5197 VT, Operand, SRIdxVal); 5198 return SDValue(Subreg, 0); 5199} 5200 5201/// getTargetInsertSubreg - A convenience function for creating 5202/// TargetOpcode::INSERT_SUBREG nodes. 5203SDValue 5204SelectionDAG::getTargetInsertSubreg(int SRIdx, DebugLoc DL, EVT VT, 5205 SDValue Operand, SDValue Subreg) { 5206 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5207 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5208 VT, Operand, Subreg, SRIdxVal); 5209 return SDValue(Result, 0); 5210} 5211 5212/// getNodeIfExists - Get the specified node if it's already available, or 5213/// else return NULL. 5214SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5215 const SDValue *Ops, unsigned NumOps) { 5216 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5217 FoldingSetNodeID ID; 5218 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5219 void *IP = 0; 5220 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5221 return E; 5222 } 5223 return NULL; 5224} 5225 5226/// getDbgValue - Creates a SDDbgValue node. 5227/// 5228SDDbgValue * 5229SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5230 DebugLoc DL, unsigned O) { 5231 return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5232} 5233 5234SDDbgValue * 5235SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5236 DebugLoc DL, unsigned O) { 5237 return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5238} 5239 5240SDDbgValue * 5241SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5242 DebugLoc DL, unsigned O) { 5243 return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5244} 5245 5246namespace { 5247 5248/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5249/// pointed to by a use iterator is deleted, increment the use iterator 5250/// so that it doesn't dangle. 5251/// 5252/// This class also manages a "downlink" DAGUpdateListener, to forward 5253/// messages to ReplaceAllUsesWith's callers. 5254/// 5255class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5256 SelectionDAG::DAGUpdateListener *DownLink; 5257 SDNode::use_iterator &UI; 5258 SDNode::use_iterator &UE; 5259 5260 virtual void NodeDeleted(SDNode *N, SDNode *E) { 5261 // Increment the iterator as needed. 5262 while (UI != UE && N == *UI) 5263 ++UI; 5264 5265 // Then forward the message. 5266 if (DownLink) DownLink->NodeDeleted(N, E); 5267 } 5268 5269 virtual void NodeUpdated(SDNode *N) { 5270 // Just forward the message. 5271 if (DownLink) DownLink->NodeUpdated(N); 5272 } 5273 5274public: 5275 RAUWUpdateListener(SelectionDAG::DAGUpdateListener *dl, 5276 SDNode::use_iterator &ui, 5277 SDNode::use_iterator &ue) 5278 : DownLink(dl), UI(ui), UE(ue) {} 5279}; 5280 5281} 5282 5283/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5284/// This can cause recursive merging of nodes in the DAG. 5285/// 5286/// This version assumes From has a single result value. 5287/// 5288void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To, 5289 DAGUpdateListener *UpdateListener) { 5290 SDNode *From = FromN.getNode(); 5291 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5292 "Cannot replace with this method!"); 5293 assert(From != To.getNode() && "Cannot replace uses of with self"); 5294 5295 // Iterate over all the existing uses of From. New uses will be added 5296 // to the beginning of the use list, which we avoid visiting. 5297 // This specifically avoids visiting uses of From that arise while the 5298 // replacement is happening, because any such uses would be the result 5299 // of CSE: If an existing node looks like From after one of its operands 5300 // is replaced by To, we don't want to replace of all its users with To 5301 // too. See PR3018 for more info. 5302 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5303 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5304 while (UI != UE) { 5305 SDNode *User = *UI; 5306 5307 // This node is about to morph, remove its old self from the CSE maps. 5308 RemoveNodeFromCSEMaps(User); 5309 5310 // A user can appear in a use list multiple times, and when this 5311 // happens the uses are usually next to each other in the list. 5312 // To help reduce the number of CSE recomputations, process all 5313 // the uses of this user that we can find this way. 5314 do { 5315 SDUse &Use = UI.getUse(); 5316 ++UI; 5317 Use.set(To); 5318 } while (UI != UE && *UI == User); 5319 5320 // Now that we have modified User, add it back to the CSE maps. If it 5321 // already exists there, recursively merge the results together. 5322 AddModifiedNodeToCSEMaps(User, &Listener); 5323 } 5324 5325 // If we just RAUW'd the root, take note. 5326 if (FromN == getRoot()) 5327 setRoot(To); 5328} 5329 5330/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5331/// This can cause recursive merging of nodes in the DAG. 5332/// 5333/// This version assumes that for each value of From, there is a 5334/// corresponding value in To in the same position with the same type. 5335/// 5336void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To, 5337 DAGUpdateListener *UpdateListener) { 5338#ifndef NDEBUG 5339 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5340 assert((!From->hasAnyUseOfValue(i) || 5341 From->getValueType(i) == To->getValueType(i)) && 5342 "Cannot use this version of ReplaceAllUsesWith!"); 5343#endif 5344 5345 // Handle the trivial case. 5346 if (From == To) 5347 return; 5348 5349 // Iterate over just the existing users of From. See the comments in 5350 // the ReplaceAllUsesWith above. 5351 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5352 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5353 while (UI != UE) { 5354 SDNode *User = *UI; 5355 5356 // This node is about to morph, remove its old self from the CSE maps. 5357 RemoveNodeFromCSEMaps(User); 5358 5359 // A user can appear in a use list multiple times, and when this 5360 // happens the uses are usually next to each other in the list. 5361 // To help reduce the number of CSE recomputations, process all 5362 // the uses of this user that we can find this way. 5363 do { 5364 SDUse &Use = UI.getUse(); 5365 ++UI; 5366 Use.setNode(To); 5367 } while (UI != UE && *UI == User); 5368 5369 // Now that we have modified User, add it back to the CSE maps. If it 5370 // already exists there, recursively merge the results together. 5371 AddModifiedNodeToCSEMaps(User, &Listener); 5372 } 5373 5374 // If we just RAUW'd the root, take note. 5375 if (From == getRoot().getNode()) 5376 setRoot(SDValue(To, getRoot().getResNo())); 5377} 5378 5379/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5380/// This can cause recursive merging of nodes in the DAG. 5381/// 5382/// This version can replace From with any result values. To must match the 5383/// number and types of values returned by From. 5384void SelectionDAG::ReplaceAllUsesWith(SDNode *From, 5385 const SDValue *To, 5386 DAGUpdateListener *UpdateListener) { 5387 if (From->getNumValues() == 1) // Handle the simple case efficiently. 5388 return ReplaceAllUsesWith(SDValue(From, 0), To[0], UpdateListener); 5389 5390 // Iterate over just the existing users of From. See the comments in 5391 // the ReplaceAllUsesWith above. 5392 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5393 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5394 while (UI != UE) { 5395 SDNode *User = *UI; 5396 5397 // This node is about to morph, remove its old self from the CSE maps. 5398 RemoveNodeFromCSEMaps(User); 5399 5400 // A user can appear in a use list multiple times, and when this 5401 // happens the uses are usually next to each other in the list. 5402 // To help reduce the number of CSE recomputations, process all 5403 // the uses of this user that we can find this way. 5404 do { 5405 SDUse &Use = UI.getUse(); 5406 const SDValue &ToOp = To[Use.getResNo()]; 5407 ++UI; 5408 Use.set(ToOp); 5409 } while (UI != UE && *UI == User); 5410 5411 // Now that we have modified User, add it back to the CSE maps. If it 5412 // already exists there, recursively merge the results together. 5413 AddModifiedNodeToCSEMaps(User, &Listener); 5414 } 5415 5416 // If we just RAUW'd the root, take note. 5417 if (From == getRoot().getNode()) 5418 setRoot(SDValue(To[getRoot().getResNo()])); 5419} 5420 5421/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5422/// uses of other values produced by From.getNode() alone. The Deleted 5423/// vector is handled the same way as for ReplaceAllUsesWith. 5424void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To, 5425 DAGUpdateListener *UpdateListener){ 5426 // Handle the really simple, really trivial case efficiently. 5427 if (From == To) return; 5428 5429 // Handle the simple, trivial, case efficiently. 5430 if (From.getNode()->getNumValues() == 1) { 5431 ReplaceAllUsesWith(From, To, UpdateListener); 5432 return; 5433 } 5434 5435 // Iterate over just the existing users of From. See the comments in 5436 // the ReplaceAllUsesWith above. 5437 SDNode::use_iterator UI = From.getNode()->use_begin(), 5438 UE = From.getNode()->use_end(); 5439 RAUWUpdateListener Listener(UpdateListener, UI, UE); 5440 while (UI != UE) { 5441 SDNode *User = *UI; 5442 bool UserRemovedFromCSEMaps = false; 5443 5444 // A user can appear in a use list multiple times, and when this 5445 // happens the uses are usually next to each other in the list. 5446 // To help reduce the number of CSE recomputations, process all 5447 // the uses of this user that we can find this way. 5448 do { 5449 SDUse &Use = UI.getUse(); 5450 5451 // Skip uses of different values from the same node. 5452 if (Use.getResNo() != From.getResNo()) { 5453 ++UI; 5454 continue; 5455 } 5456 5457 // If this node hasn't been modified yet, it's still in the CSE maps, 5458 // so remove its old self from the CSE maps. 5459 if (!UserRemovedFromCSEMaps) { 5460 RemoveNodeFromCSEMaps(User); 5461 UserRemovedFromCSEMaps = true; 5462 } 5463 5464 ++UI; 5465 Use.set(To); 5466 } while (UI != UE && *UI == User); 5467 5468 // We are iterating over all uses of the From node, so if a use 5469 // doesn't use the specific value, no changes are made. 5470 if (!UserRemovedFromCSEMaps) 5471 continue; 5472 5473 // Now that we have modified User, add it back to the CSE maps. If it 5474 // already exists there, recursively merge the results together. 5475 AddModifiedNodeToCSEMaps(User, &Listener); 5476 } 5477 5478 // If we just RAUW'd the root, take note. 5479 if (From == getRoot()) 5480 setRoot(To); 5481} 5482 5483namespace { 5484 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5485 /// to record information about a use. 5486 struct UseMemo { 5487 SDNode *User; 5488 unsigned Index; 5489 SDUse *Use; 5490 }; 5491 5492 /// operator< - Sort Memos by User. 5493 bool operator<(const UseMemo &L, const UseMemo &R) { 5494 return (intptr_t)L.User < (intptr_t)R.User; 5495 } 5496} 5497 5498/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5499/// uses of other values produced by From.getNode() alone. The same value 5500/// may appear in both the From and To list. The Deleted vector is 5501/// handled the same way as for ReplaceAllUsesWith. 5502void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5503 const SDValue *To, 5504 unsigned Num, 5505 DAGUpdateListener *UpdateListener){ 5506 // Handle the simple, trivial case efficiently. 5507 if (Num == 1) 5508 return ReplaceAllUsesOfValueWith(*From, *To, UpdateListener); 5509 5510 // Read up all the uses and make records of them. This helps 5511 // processing new uses that are introduced during the 5512 // replacement process. 5513 SmallVector<UseMemo, 4> Uses; 5514 for (unsigned i = 0; i != Num; ++i) { 5515 unsigned FromResNo = From[i].getResNo(); 5516 SDNode *FromNode = From[i].getNode(); 5517 for (SDNode::use_iterator UI = FromNode->use_begin(), 5518 E = FromNode->use_end(); UI != E; ++UI) { 5519 SDUse &Use = UI.getUse(); 5520 if (Use.getResNo() == FromResNo) { 5521 UseMemo Memo = { *UI, i, &Use }; 5522 Uses.push_back(Memo); 5523 } 5524 } 5525 } 5526 5527 // Sort the uses, so that all the uses from a given User are together. 5528 std::sort(Uses.begin(), Uses.end()); 5529 5530 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5531 UseIndex != UseIndexEnd; ) { 5532 // We know that this user uses some value of From. If it is the right 5533 // value, update it. 5534 SDNode *User = Uses[UseIndex].User; 5535 5536 // This node is about to morph, remove its old self from the CSE maps. 5537 RemoveNodeFromCSEMaps(User); 5538 5539 // The Uses array is sorted, so all the uses for a given User 5540 // are next to each other in the list. 5541 // To help reduce the number of CSE recomputations, process all 5542 // the uses of this user that we can find this way. 5543 do { 5544 unsigned i = Uses[UseIndex].Index; 5545 SDUse &Use = *Uses[UseIndex].Use; 5546 ++UseIndex; 5547 5548 Use.set(To[i]); 5549 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5550 5551 // Now that we have modified User, add it back to the CSE maps. If it 5552 // already exists there, recursively merge the results together. 5553 AddModifiedNodeToCSEMaps(User, UpdateListener); 5554 } 5555} 5556 5557/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5558/// based on their topological order. It returns the maximum id and a vector 5559/// of the SDNodes* in assigned order by reference. 5560unsigned SelectionDAG::AssignTopologicalOrder() { 5561 5562 unsigned DAGSize = 0; 5563 5564 // SortedPos tracks the progress of the algorithm. Nodes before it are 5565 // sorted, nodes after it are unsorted. When the algorithm completes 5566 // it is at the end of the list. 5567 allnodes_iterator SortedPos = allnodes_begin(); 5568 5569 // Visit all the nodes. Move nodes with no operands to the front of 5570 // the list immediately. Annotate nodes that do have operands with their 5571 // operand count. Before we do this, the Node Id fields of the nodes 5572 // may contain arbitrary values. After, the Node Id fields for nodes 5573 // before SortedPos will contain the topological sort index, and the 5574 // Node Id fields for nodes At SortedPos and after will contain the 5575 // count of outstanding operands. 5576 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5577 SDNode *N = I++; 5578 checkForCycles(N); 5579 unsigned Degree = N->getNumOperands(); 5580 if (Degree == 0) { 5581 // A node with no uses, add it to the result array immediately. 5582 N->setNodeId(DAGSize++); 5583 allnodes_iterator Q = N; 5584 if (Q != SortedPos) 5585 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5586 assert(SortedPos != AllNodes.end() && "Overran node list"); 5587 ++SortedPos; 5588 } else { 5589 // Temporarily use the Node Id as scratch space for the degree count. 5590 N->setNodeId(Degree); 5591 } 5592 } 5593 5594 // Visit all the nodes. As we iterate, moves nodes into sorted order, 5595 // such that by the time the end is reached all nodes will be sorted. 5596 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5597 SDNode *N = I; 5598 checkForCycles(N); 5599 // N is in sorted position, so all its uses have one less operand 5600 // that needs to be sorted. 5601 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5602 UI != UE; ++UI) { 5603 SDNode *P = *UI; 5604 unsigned Degree = P->getNodeId(); 5605 assert(Degree != 0 && "Invalid node degree"); 5606 --Degree; 5607 if (Degree == 0) { 5608 // All of P's operands are sorted, so P may sorted now. 5609 P->setNodeId(DAGSize++); 5610 if (P != SortedPos) 5611 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5612 assert(SortedPos != AllNodes.end() && "Overran node list"); 5613 ++SortedPos; 5614 } else { 5615 // Update P's outstanding operand count. 5616 P->setNodeId(Degree); 5617 } 5618 } 5619 if (I == SortedPos) { 5620#ifndef NDEBUG 5621 SDNode *S = ++I; 5622 dbgs() << "Overran sorted position:\n"; 5623 S->dumprFull(); 5624#endif 5625 llvm_unreachable(0); 5626 } 5627 } 5628 5629 assert(SortedPos == AllNodes.end() && 5630 "Topological sort incomplete!"); 5631 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 5632 "First node in topological sort is not the entry token!"); 5633 assert(AllNodes.front().getNodeId() == 0 && 5634 "First node in topological sort has non-zero id!"); 5635 assert(AllNodes.front().getNumOperands() == 0 && 5636 "First node in topological sort has operands!"); 5637 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 5638 "Last node in topologic sort has unexpected id!"); 5639 assert(AllNodes.back().use_empty() && 5640 "Last node in topologic sort has users!"); 5641 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 5642 return DAGSize; 5643} 5644 5645/// AssignOrdering - Assign an order to the SDNode. 5646void SelectionDAG::AssignOrdering(const SDNode *SD, unsigned Order) { 5647 assert(SD && "Trying to assign an order to a null node!"); 5648 Ordering->add(SD, Order); 5649} 5650 5651/// GetOrdering - Get the order for the SDNode. 5652unsigned SelectionDAG::GetOrdering(const SDNode *SD) const { 5653 assert(SD && "Trying to get the order of a null node!"); 5654 return Ordering->getOrder(SD); 5655} 5656 5657/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 5658/// value is produced by SD. 5659void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 5660 DbgInfo->add(DB, SD, isParameter); 5661 if (SD) 5662 SD->setHasDebugValue(true); 5663} 5664 5665/// TransferDbgValues - Transfer SDDbgValues. 5666void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 5667 if (From == To || !From.getNode()->getHasDebugValue()) 5668 return; 5669 SDNode *FromNode = From.getNode(); 5670 SDNode *ToNode = To.getNode(); 5671 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 5672 SmallVector<SDDbgValue *, 2> ClonedDVs; 5673 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 5674 I != E; ++I) { 5675 SDDbgValue *Dbg = *I; 5676 if (Dbg->getKind() == SDDbgValue::SDNODE) { 5677 SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 5678 Dbg->getOffset(), Dbg->getDebugLoc(), 5679 Dbg->getOrder()); 5680 ClonedDVs.push_back(Clone); 5681 } 5682 } 5683 for (SmallVector<SDDbgValue *, 2>::iterator I = ClonedDVs.begin(), 5684 E = ClonedDVs.end(); I != E; ++I) 5685 AddDbgValue(*I, ToNode, false); 5686} 5687 5688//===----------------------------------------------------------------------===// 5689// SDNode Class 5690//===----------------------------------------------------------------------===// 5691 5692HandleSDNode::~HandleSDNode() { 5693 DropOperands(); 5694} 5695 5696GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, DebugLoc DL, 5697 const GlobalValue *GA, 5698 EVT VT, int64_t o, unsigned char TF) 5699 : SDNode(Opc, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 5700 TheGlobal = GA; 5701} 5702 5703MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, 5704 MachineMemOperand *mmo) 5705 : SDNode(Opc, dl, VTs), MemoryVT(memvt), MMO(mmo) { 5706 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 5707 MMO->isNonTemporal(), MMO->isInvariant()); 5708 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 5709 assert(isNonTemporal() == MMO->isNonTemporal() && 5710 "Non-temporal encoding error!"); 5711 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 5712} 5713 5714MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, 5715 const SDValue *Ops, unsigned NumOps, EVT memvt, 5716 MachineMemOperand *mmo) 5717 : SDNode(Opc, dl, VTs, Ops, NumOps), 5718 MemoryVT(memvt), MMO(mmo) { 5719 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 5720 MMO->isNonTemporal(), MMO->isInvariant()); 5721 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 5722 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 5723} 5724 5725/// Profile - Gather unique data for the node. 5726/// 5727void SDNode::Profile(FoldingSetNodeID &ID) const { 5728 AddNodeIDNode(ID, this); 5729} 5730 5731namespace { 5732 struct EVTArray { 5733 std::vector<EVT> VTs; 5734 5735 EVTArray() { 5736 VTs.reserve(MVT::LAST_VALUETYPE); 5737 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 5738 VTs.push_back(MVT((MVT::SimpleValueType)i)); 5739 } 5740 }; 5741} 5742 5743static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 5744static ManagedStatic<EVTArray> SimpleVTArray; 5745static ManagedStatic<sys::SmartMutex<true> > VTMutex; 5746 5747/// getValueTypeList - Return a pointer to the specified value type. 5748/// 5749const EVT *SDNode::getValueTypeList(EVT VT) { 5750 if (VT.isExtended()) { 5751 sys::SmartScopedLock<true> Lock(*VTMutex); 5752 return &(*EVTs->insert(VT).first); 5753 } else { 5754 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 5755 "Value type out of range!"); 5756 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 5757 } 5758} 5759 5760/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 5761/// indicated value. This method ignores uses of other values defined by this 5762/// operation. 5763bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 5764 assert(Value < getNumValues() && "Bad value!"); 5765 5766 // TODO: Only iterate over uses of a given value of the node 5767 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 5768 if (UI.getUse().getResNo() == Value) { 5769 if (NUses == 0) 5770 return false; 5771 --NUses; 5772 } 5773 } 5774 5775 // Found exactly the right number of uses? 5776 return NUses == 0; 5777} 5778 5779 5780/// hasAnyUseOfValue - Return true if there are any use of the indicated 5781/// value. This method ignores uses of other values defined by this operation. 5782bool SDNode::hasAnyUseOfValue(unsigned Value) const { 5783 assert(Value < getNumValues() && "Bad value!"); 5784 5785 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 5786 if (UI.getUse().getResNo() == Value) 5787 return true; 5788 5789 return false; 5790} 5791 5792 5793/// isOnlyUserOf - Return true if this node is the only use of N. 5794/// 5795bool SDNode::isOnlyUserOf(SDNode *N) const { 5796 bool Seen = false; 5797 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 5798 SDNode *User = *I; 5799 if (User == this) 5800 Seen = true; 5801 else 5802 return false; 5803 } 5804 5805 return Seen; 5806} 5807 5808/// isOperand - Return true if this node is an operand of N. 5809/// 5810bool SDValue::isOperandOf(SDNode *N) const { 5811 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 5812 if (*this == N->getOperand(i)) 5813 return true; 5814 return false; 5815} 5816 5817bool SDNode::isOperandOf(SDNode *N) const { 5818 for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 5819 if (this == N->OperandList[i].getNode()) 5820 return true; 5821 return false; 5822} 5823 5824/// reachesChainWithoutSideEffects - Return true if this operand (which must 5825/// be a chain) reaches the specified operand without crossing any 5826/// side-effecting instructions on any chain path. In practice, this looks 5827/// through token factors and non-volatile loads. In order to remain efficient, 5828/// this only looks a couple of nodes in, it does not do an exhaustive search. 5829bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 5830 unsigned Depth) const { 5831 if (*this == Dest) return true; 5832 5833 // Don't search too deeply, we just want to be able to see through 5834 // TokenFactor's etc. 5835 if (Depth == 0) return false; 5836 5837 // If this is a token factor, all inputs to the TF happen in parallel. If any 5838 // of the operands of the TF does not reach dest, then we cannot do the xform. 5839 if (getOpcode() == ISD::TokenFactor) { 5840 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 5841 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 5842 return false; 5843 return true; 5844 } 5845 5846 // Loads don't have side effects, look through them. 5847 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 5848 if (!Ld->isVolatile()) 5849 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 5850 } 5851 return false; 5852} 5853 5854/// hasPredecessor - Return true if N is a predecessor of this node. 5855/// N is either an operand of this node, or can be reached by recursively 5856/// traversing up the operands. 5857/// NOTE: This is an expensive method. Use it carefully. 5858bool SDNode::hasPredecessor(const SDNode *N) const { 5859 SmallPtrSet<const SDNode *, 32> Visited; 5860 SmallVector<const SDNode *, 16> Worklist; 5861 return hasPredecessorHelper(N, Visited, Worklist); 5862} 5863 5864bool SDNode::hasPredecessorHelper(const SDNode *N, 5865 SmallPtrSet<const SDNode *, 32> &Visited, 5866 SmallVector<const SDNode *, 16> &Worklist) const { 5867 if (Visited.empty()) { 5868 Worklist.push_back(this); 5869 } else { 5870 // Take a look in the visited set. If we've already encountered this node 5871 // we needn't search further. 5872 if (Visited.count(N)) 5873 return true; 5874 } 5875 5876 // Haven't visited N yet. Continue the search. 5877 while (!Worklist.empty()) { 5878 const SDNode *M = Worklist.pop_back_val(); 5879 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 5880 SDNode *Op = M->getOperand(i).getNode(); 5881 if (Visited.insert(Op)) 5882 Worklist.push_back(Op); 5883 if (Op == N) 5884 return true; 5885 } 5886 } 5887 5888 return false; 5889} 5890 5891uint64_t SDNode::getConstantOperandVal(unsigned Num) const { 5892 assert(Num < NumOperands && "Invalid child # of SDNode!"); 5893 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 5894} 5895 5896std::string SDNode::getOperationName(const SelectionDAG *G) const { 5897 switch (getOpcode()) { 5898 default: 5899 if (getOpcode() < ISD::BUILTIN_OP_END) 5900 return "<<Unknown DAG Node>>"; 5901 if (isMachineOpcode()) { 5902 if (G) 5903 if (const TargetInstrInfo *TII = G->getTarget().getInstrInfo()) 5904 if (getMachineOpcode() < TII->getNumOpcodes()) 5905 return TII->get(getMachineOpcode()).getName(); 5906 return "<<Unknown Machine Node #" + utostr(getOpcode()) + ">>"; 5907 } 5908 if (G) { 5909 const TargetLowering &TLI = G->getTargetLoweringInfo(); 5910 const char *Name = TLI.getTargetNodeName(getOpcode()); 5911 if (Name) return Name; 5912 return "<<Unknown Target Node #" + utostr(getOpcode()) + ">>"; 5913 } 5914 return "<<Unknown Node #" + utostr(getOpcode()) + ">>"; 5915 5916#ifndef NDEBUG 5917 case ISD::DELETED_NODE: 5918 return "<<Deleted Node!>>"; 5919#endif 5920 case ISD::PREFETCH: return "Prefetch"; 5921 case ISD::MEMBARRIER: return "MemBarrier"; 5922 case ISD::ATOMIC_FENCE: return "AtomicFence"; 5923 case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap"; 5924 case ISD::ATOMIC_SWAP: return "AtomicSwap"; 5925 case ISD::ATOMIC_LOAD_ADD: return "AtomicLoadAdd"; 5926 case ISD::ATOMIC_LOAD_SUB: return "AtomicLoadSub"; 5927 case ISD::ATOMIC_LOAD_AND: return "AtomicLoadAnd"; 5928 case ISD::ATOMIC_LOAD_OR: return "AtomicLoadOr"; 5929 case ISD::ATOMIC_LOAD_XOR: return "AtomicLoadXor"; 5930 case ISD::ATOMIC_LOAD_NAND: return "AtomicLoadNand"; 5931 case ISD::ATOMIC_LOAD_MIN: return "AtomicLoadMin"; 5932 case ISD::ATOMIC_LOAD_MAX: return "AtomicLoadMax"; 5933 case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin"; 5934 case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax"; 5935 case ISD::ATOMIC_LOAD: return "AtomicLoad"; 5936 case ISD::ATOMIC_STORE: return "AtomicStore"; 5937 case ISD::PCMARKER: return "PCMarker"; 5938 case ISD::READCYCLECOUNTER: return "ReadCycleCounter"; 5939 case ISD::SRCVALUE: return "SrcValue"; 5940 case ISD::MDNODE_SDNODE: return "MDNode"; 5941 case ISD::EntryToken: return "EntryToken"; 5942 case ISD::TokenFactor: return "TokenFactor"; 5943 case ISD::AssertSext: return "AssertSext"; 5944 case ISD::AssertZext: return "AssertZext"; 5945 5946 case ISD::BasicBlock: return "BasicBlock"; 5947 case ISD::VALUETYPE: return "ValueType"; 5948 case ISD::Register: return "Register"; 5949 5950 case ISD::Constant: return "Constant"; 5951 case ISD::ConstantFP: return "ConstantFP"; 5952 case ISD::GlobalAddress: return "GlobalAddress"; 5953 case ISD::GlobalTLSAddress: return "GlobalTLSAddress"; 5954 case ISD::FrameIndex: return "FrameIndex"; 5955 case ISD::JumpTable: return "JumpTable"; 5956 case ISD::GLOBAL_OFFSET_TABLE: return "GLOBAL_OFFSET_TABLE"; 5957 case ISD::RETURNADDR: return "RETURNADDR"; 5958 case ISD::FRAMEADDR: return "FRAMEADDR"; 5959 case ISD::FRAME_TO_ARGS_OFFSET: return "FRAME_TO_ARGS_OFFSET"; 5960 case ISD::EXCEPTIONADDR: return "EXCEPTIONADDR"; 5961 case ISD::LSDAADDR: return "LSDAADDR"; 5962 case ISD::EHSELECTION: return "EHSELECTION"; 5963 case ISD::EH_RETURN: return "EH_RETURN"; 5964 case ISD::EH_SJLJ_SETJMP: return "EH_SJLJ_SETJMP"; 5965 case ISD::EH_SJLJ_LONGJMP: return "EH_SJLJ_LONGJMP"; 5966 case ISD::ConstantPool: return "ConstantPool"; 5967 case ISD::ExternalSymbol: return "ExternalSymbol"; 5968 case ISD::BlockAddress: return "BlockAddress"; 5969 case ISD::INTRINSIC_WO_CHAIN: 5970 case ISD::INTRINSIC_VOID: 5971 case ISD::INTRINSIC_W_CHAIN: { 5972 unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1; 5973 unsigned IID = cast<ConstantSDNode>(getOperand(OpNo))->getZExtValue(); 5974 if (IID < Intrinsic::num_intrinsics) 5975 return Intrinsic::getName((Intrinsic::ID)IID); 5976 else if (const TargetIntrinsicInfo *TII = G->getTarget().getIntrinsicInfo()) 5977 return TII->getName(IID); 5978 llvm_unreachable("Invalid intrinsic ID"); 5979 } 5980 5981 case ISD::BUILD_VECTOR: return "BUILD_VECTOR"; 5982 case ISD::TargetConstant: return "TargetConstant"; 5983 case ISD::TargetConstantFP:return "TargetConstantFP"; 5984 case ISD::TargetGlobalAddress: return "TargetGlobalAddress"; 5985 case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress"; 5986 case ISD::TargetFrameIndex: return "TargetFrameIndex"; 5987 case ISD::TargetJumpTable: return "TargetJumpTable"; 5988 case ISD::TargetConstantPool: return "TargetConstantPool"; 5989 case ISD::TargetExternalSymbol: return "TargetExternalSymbol"; 5990 case ISD::TargetBlockAddress: return "TargetBlockAddress"; 5991 5992 case ISD::CopyToReg: return "CopyToReg"; 5993 case ISD::CopyFromReg: return "CopyFromReg"; 5994 case ISD::UNDEF: return "undef"; 5995 case ISD::MERGE_VALUES: return "merge_values"; 5996 case ISD::INLINEASM: return "inlineasm"; 5997 case ISD::EH_LABEL: return "eh_label"; 5998 case ISD::HANDLENODE: return "handlenode"; 5999 6000 // Unary operators 6001 case ISD::FABS: return "fabs"; 6002 case ISD::FNEG: return "fneg"; 6003 case ISD::FSQRT: return "fsqrt"; 6004 case ISD::FSIN: return "fsin"; 6005 case ISD::FCOS: return "fcos"; 6006 case ISD::FTRUNC: return "ftrunc"; 6007 case ISD::FFLOOR: return "ffloor"; 6008 case ISD::FCEIL: return "fceil"; 6009 case ISD::FRINT: return "frint"; 6010 case ISD::FNEARBYINT: return "fnearbyint"; 6011 case ISD::FEXP: return "fexp"; 6012 case ISD::FEXP2: return "fexp2"; 6013 case ISD::FLOG: return "flog"; 6014 case ISD::FLOG2: return "flog2"; 6015 case ISD::FLOG10: return "flog10"; 6016 6017 // Binary operators 6018 case ISD::ADD: return "add"; 6019 case ISD::SUB: return "sub"; 6020 case ISD::MUL: return "mul"; 6021 case ISD::MULHU: return "mulhu"; 6022 case ISD::MULHS: return "mulhs"; 6023 case ISD::SDIV: return "sdiv"; 6024 case ISD::UDIV: return "udiv"; 6025 case ISD::SREM: return "srem"; 6026 case ISD::UREM: return "urem"; 6027 case ISD::SMUL_LOHI: return "smul_lohi"; 6028 case ISD::UMUL_LOHI: return "umul_lohi"; 6029 case ISD::SDIVREM: return "sdivrem"; 6030 case ISD::UDIVREM: return "udivrem"; 6031 case ISD::AND: return "and"; 6032 case ISD::OR: return "or"; 6033 case ISD::XOR: return "xor"; 6034 case ISD::SHL: return "shl"; 6035 case ISD::SRA: return "sra"; 6036 case ISD::SRL: return "srl"; 6037 case ISD::ROTL: return "rotl"; 6038 case ISD::ROTR: return "rotr"; 6039 case ISD::FADD: return "fadd"; 6040 case ISD::FSUB: return "fsub"; 6041 case ISD::FMUL: return "fmul"; 6042 case ISD::FDIV: return "fdiv"; 6043 case ISD::FMA: return "fma"; 6044 case ISD::FREM: return "frem"; 6045 case ISD::FCOPYSIGN: return "fcopysign"; 6046 case ISD::FGETSIGN: return "fgetsign"; 6047 case ISD::FPOW: return "fpow"; 6048 6049 case ISD::FPOWI: return "fpowi"; 6050 case ISD::SETCC: return "setcc"; 6051 case ISD::SELECT: return "select"; 6052 case ISD::VSELECT: return "vselect"; 6053 case ISD::SELECT_CC: return "select_cc"; 6054 case ISD::INSERT_VECTOR_ELT: return "insert_vector_elt"; 6055 case ISD::EXTRACT_VECTOR_ELT: return "extract_vector_elt"; 6056 case ISD::CONCAT_VECTORS: return "concat_vectors"; 6057 case ISD::INSERT_SUBVECTOR: return "insert_subvector"; 6058 case ISD::EXTRACT_SUBVECTOR: return "extract_subvector"; 6059 case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector"; 6060 case ISD::VECTOR_SHUFFLE: return "vector_shuffle"; 6061 case ISD::CARRY_FALSE: return "carry_false"; 6062 case ISD::ADDC: return "addc"; 6063 case ISD::ADDE: return "adde"; 6064 case ISD::SADDO: return "saddo"; 6065 case ISD::UADDO: return "uaddo"; 6066 case ISD::SSUBO: return "ssubo"; 6067 case ISD::USUBO: return "usubo"; 6068 case ISD::SMULO: return "smulo"; 6069 case ISD::UMULO: return "umulo"; 6070 case ISD::SUBC: return "subc"; 6071 case ISD::SUBE: return "sube"; 6072 case ISD::SHL_PARTS: return "shl_parts"; 6073 case ISD::SRA_PARTS: return "sra_parts"; 6074 case ISD::SRL_PARTS: return "srl_parts"; 6075 6076 // Conversion operators. 6077 case ISD::SIGN_EXTEND: return "sign_extend"; 6078 case ISD::ZERO_EXTEND: return "zero_extend"; 6079 case ISD::ANY_EXTEND: return "any_extend"; 6080 case ISD::SIGN_EXTEND_INREG: return "sign_extend_inreg"; 6081 case ISD::TRUNCATE: return "truncate"; 6082 case ISD::FP_ROUND: return "fp_round"; 6083 case ISD::FLT_ROUNDS_: return "flt_rounds"; 6084 case ISD::FP_ROUND_INREG: return "fp_round_inreg"; 6085 case ISD::FP_EXTEND: return "fp_extend"; 6086 6087 case ISD::SINT_TO_FP: return "sint_to_fp"; 6088 case ISD::UINT_TO_FP: return "uint_to_fp"; 6089 case ISD::FP_TO_SINT: return "fp_to_sint"; 6090 case ISD::FP_TO_UINT: return "fp_to_uint"; 6091 case ISD::BITCAST: return "bitcast"; 6092 case ISD::FP16_TO_FP32: return "fp16_to_fp32"; 6093 case ISD::FP32_TO_FP16: return "fp32_to_fp16"; 6094 6095 case ISD::CONVERT_RNDSAT: { 6096 switch (cast<CvtRndSatSDNode>(this)->getCvtCode()) { 6097 default: llvm_unreachable("Unknown cvt code!"); 6098 case ISD::CVT_FF: return "cvt_ff"; 6099 case ISD::CVT_FS: return "cvt_fs"; 6100 case ISD::CVT_FU: return "cvt_fu"; 6101 case ISD::CVT_SF: return "cvt_sf"; 6102 case ISD::CVT_UF: return "cvt_uf"; 6103 case ISD::CVT_SS: return "cvt_ss"; 6104 case ISD::CVT_SU: return "cvt_su"; 6105 case ISD::CVT_US: return "cvt_us"; 6106 case ISD::CVT_UU: return "cvt_uu"; 6107 } 6108 } 6109 6110 // Control flow instructions 6111 case ISD::BR: return "br"; 6112 case ISD::BRIND: return "brind"; 6113 case ISD::BR_JT: return "br_jt"; 6114 case ISD::BRCOND: return "brcond"; 6115 case ISD::BR_CC: return "br_cc"; 6116 case ISD::CALLSEQ_START: return "callseq_start"; 6117 case ISD::CALLSEQ_END: return "callseq_end"; 6118 6119 // Other operators 6120 case ISD::LOAD: return "load"; 6121 case ISD::STORE: return "store"; 6122 case ISD::VAARG: return "vaarg"; 6123 case ISD::VACOPY: return "vacopy"; 6124 case ISD::VAEND: return "vaend"; 6125 case ISD::VASTART: return "vastart"; 6126 case ISD::DYNAMIC_STACKALLOC: return "dynamic_stackalloc"; 6127 case ISD::EXTRACT_ELEMENT: return "extract_element"; 6128 case ISD::BUILD_PAIR: return "build_pair"; 6129 case ISD::STACKSAVE: return "stacksave"; 6130 case ISD::STACKRESTORE: return "stackrestore"; 6131 case ISD::TRAP: return "trap"; 6132 6133 // Bit manipulation 6134 case ISD::BSWAP: return "bswap"; 6135 case ISD::CTPOP: return "ctpop"; 6136 case ISD::CTTZ: return "cttz"; 6137 case ISD::CTTZ_ZERO_UNDEF: return "cttz_zero_undef"; 6138 case ISD::CTLZ: return "ctlz"; 6139 case ISD::CTLZ_ZERO_UNDEF: return "ctlz_zero_undef"; 6140 6141 // Trampolines 6142 case ISD::INIT_TRAMPOLINE: return "init_trampoline"; 6143 case ISD::ADJUST_TRAMPOLINE: return "adjust_trampoline"; 6144 6145 case ISD::CONDCODE: 6146 switch (cast<CondCodeSDNode>(this)->get()) { 6147 default: llvm_unreachable("Unknown setcc condition!"); 6148 case ISD::SETOEQ: return "setoeq"; 6149 case ISD::SETOGT: return "setogt"; 6150 case ISD::SETOGE: return "setoge"; 6151 case ISD::SETOLT: return "setolt"; 6152 case ISD::SETOLE: return "setole"; 6153 case ISD::SETONE: return "setone"; 6154 6155 case ISD::SETO: return "seto"; 6156 case ISD::SETUO: return "setuo"; 6157 case ISD::SETUEQ: return "setue"; 6158 case ISD::SETUGT: return "setugt"; 6159 case ISD::SETUGE: return "setuge"; 6160 case ISD::SETULT: return "setult"; 6161 case ISD::SETULE: return "setule"; 6162 case ISD::SETUNE: return "setune"; 6163 6164 case ISD::SETEQ: return "seteq"; 6165 case ISD::SETGT: return "setgt"; 6166 case ISD::SETGE: return "setge"; 6167 case ISD::SETLT: return "setlt"; 6168 case ISD::SETLE: return "setle"; 6169 case ISD::SETNE: return "setne"; 6170 6171 case ISD::SETTRUE: return "settrue"; 6172 case ISD::SETTRUE2: return "settrue2"; 6173 case ISD::SETFALSE: return "setfalse"; 6174 case ISD::SETFALSE2: return "setfalse2"; 6175 } 6176 } 6177} 6178 6179const char *SDNode::getIndexedModeName(ISD::MemIndexedMode AM) { 6180 switch (AM) { 6181 default: 6182 return ""; 6183 case ISD::PRE_INC: 6184 return "<pre-inc>"; 6185 case ISD::PRE_DEC: 6186 return "<pre-dec>"; 6187 case ISD::POST_INC: 6188 return "<post-inc>"; 6189 case ISD::POST_DEC: 6190 return "<post-dec>"; 6191 } 6192} 6193 6194std::string ISD::ArgFlagsTy::getArgFlagsString() { 6195 std::string S = "< "; 6196 6197 if (isZExt()) 6198 S += "zext "; 6199 if (isSExt()) 6200 S += "sext "; 6201 if (isInReg()) 6202 S += "inreg "; 6203 if (isSRet()) 6204 S += "sret "; 6205 if (isByVal()) 6206 S += "byval "; 6207 if (isNest()) 6208 S += "nest "; 6209 if (getByValAlign()) 6210 S += "byval-align:" + utostr(getByValAlign()) + " "; 6211 if (getOrigAlign()) 6212 S += "orig-align:" + utostr(getOrigAlign()) + " "; 6213 if (getByValSize()) 6214 S += "byval-size:" + utostr(getByValSize()) + " "; 6215 return S + ">"; 6216} 6217 6218void SDNode::dump() const { dump(0); } 6219void SDNode::dump(const SelectionDAG *G) const { 6220 print(dbgs(), G); 6221 dbgs() << '\n'; 6222} 6223 6224void SDNode::print_types(raw_ostream &OS, const SelectionDAG *G) const { 6225 OS << (void*)this << ": "; 6226 6227 for (unsigned i = 0, e = getNumValues(); i != e; ++i) { 6228 if (i) OS << ","; 6229 if (getValueType(i) == MVT::Other) 6230 OS << "ch"; 6231 else 6232 OS << getValueType(i).getEVTString(); 6233 } 6234 OS << " = " << getOperationName(G); 6235} 6236 6237void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { 6238 if (const MachineSDNode *MN = dyn_cast<MachineSDNode>(this)) { 6239 if (!MN->memoperands_empty()) { 6240 OS << "<"; 6241 OS << "Mem:"; 6242 for (MachineSDNode::mmo_iterator i = MN->memoperands_begin(), 6243 e = MN->memoperands_end(); i != e; ++i) { 6244 OS << **i; 6245 if (llvm::next(i) != e) 6246 OS << " "; 6247 } 6248 OS << ">"; 6249 } 6250 } else if (const ShuffleVectorSDNode *SVN = 6251 dyn_cast<ShuffleVectorSDNode>(this)) { 6252 OS << "<"; 6253 for (unsigned i = 0, e = ValueList[0].getVectorNumElements(); i != e; ++i) { 6254 int Idx = SVN->getMaskElt(i); 6255 if (i) OS << ","; 6256 if (Idx < 0) 6257 OS << "u"; 6258 else 6259 OS << Idx; 6260 } 6261 OS << ">"; 6262 } else if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(this)) { 6263 OS << '<' << CSDN->getAPIntValue() << '>'; 6264 } else if (const ConstantFPSDNode *CSDN = dyn_cast<ConstantFPSDNode>(this)) { 6265 if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEsingle) 6266 OS << '<' << CSDN->getValueAPF().convertToFloat() << '>'; 6267 else if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEdouble) 6268 OS << '<' << CSDN->getValueAPF().convertToDouble() << '>'; 6269 else { 6270 OS << "<APFloat("; 6271 CSDN->getValueAPF().bitcastToAPInt().dump(); 6272 OS << ")>"; 6273 } 6274 } else if (const GlobalAddressSDNode *GADN = 6275 dyn_cast<GlobalAddressSDNode>(this)) { 6276 int64_t offset = GADN->getOffset(); 6277 OS << '<'; 6278 WriteAsOperand(OS, GADN->getGlobal()); 6279 OS << '>'; 6280 if (offset > 0) 6281 OS << " + " << offset; 6282 else 6283 OS << " " << offset; 6284 if (unsigned int TF = GADN->getTargetFlags()) 6285 OS << " [TF=" << TF << ']'; 6286 } else if (const FrameIndexSDNode *FIDN = dyn_cast<FrameIndexSDNode>(this)) { 6287 OS << "<" << FIDN->getIndex() << ">"; 6288 } else if (const JumpTableSDNode *JTDN = dyn_cast<JumpTableSDNode>(this)) { 6289 OS << "<" << JTDN->getIndex() << ">"; 6290 if (unsigned int TF = JTDN->getTargetFlags()) 6291 OS << " [TF=" << TF << ']'; 6292 } else if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(this)){ 6293 int offset = CP->getOffset(); 6294 if (CP->isMachineConstantPoolEntry()) 6295 OS << "<" << *CP->getMachineCPVal() << ">"; 6296 else 6297 OS << "<" << *CP->getConstVal() << ">"; 6298 if (offset > 0) 6299 OS << " + " << offset; 6300 else 6301 OS << " " << offset; 6302 if (unsigned int TF = CP->getTargetFlags()) 6303 OS << " [TF=" << TF << ']'; 6304 } else if (const BasicBlockSDNode *BBDN = dyn_cast<BasicBlockSDNode>(this)) { 6305 OS << "<"; 6306 const Value *LBB = (const Value*)BBDN->getBasicBlock()->getBasicBlock(); 6307 if (LBB) 6308 OS << LBB->getName() << " "; 6309 OS << (const void*)BBDN->getBasicBlock() << ">"; 6310 } else if (const RegisterSDNode *R = dyn_cast<RegisterSDNode>(this)) { 6311 OS << ' ' << PrintReg(R->getReg(), G ? G->getTarget().getRegisterInfo() :0); 6312 } else if (const ExternalSymbolSDNode *ES = 6313 dyn_cast<ExternalSymbolSDNode>(this)) { 6314 OS << "'" << ES->getSymbol() << "'"; 6315 if (unsigned int TF = ES->getTargetFlags()) 6316 OS << " [TF=" << TF << ']'; 6317 } else if (const SrcValueSDNode *M = dyn_cast<SrcValueSDNode>(this)) { 6318 if (M->getValue()) 6319 OS << "<" << M->getValue() << ">"; 6320 else 6321 OS << "<null>"; 6322 } else if (const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(this)) { 6323 if (MD->getMD()) 6324 OS << "<" << MD->getMD() << ">"; 6325 else 6326 OS << "<null>"; 6327 } else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) { 6328 OS << ":" << N->getVT().getEVTString(); 6329 } 6330 else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) { 6331 OS << "<" << *LD->getMemOperand(); 6332 6333 bool doExt = true; 6334 switch (LD->getExtensionType()) { 6335 default: doExt = false; break; 6336 case ISD::EXTLOAD: OS << ", anyext"; break; 6337 case ISD::SEXTLOAD: OS << ", sext"; break; 6338 case ISD::ZEXTLOAD: OS << ", zext"; break; 6339 } 6340 if (doExt) 6341 OS << " from " << LD->getMemoryVT().getEVTString(); 6342 6343 const char *AM = getIndexedModeName(LD->getAddressingMode()); 6344 if (*AM) 6345 OS << ", " << AM; 6346 6347 OS << ">"; 6348 } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(this)) { 6349 OS << "<" << *ST->getMemOperand(); 6350 6351 if (ST->isTruncatingStore()) 6352 OS << ", trunc to " << ST->getMemoryVT().getEVTString(); 6353 6354 const char *AM = getIndexedModeName(ST->getAddressingMode()); 6355 if (*AM) 6356 OS << ", " << AM; 6357 6358 OS << ">"; 6359 } else if (const MemSDNode* M = dyn_cast<MemSDNode>(this)) { 6360 OS << "<" << *M->getMemOperand() << ">"; 6361 } else if (const BlockAddressSDNode *BA = 6362 dyn_cast<BlockAddressSDNode>(this)) { 6363 OS << "<"; 6364 WriteAsOperand(OS, BA->getBlockAddress()->getFunction(), false); 6365 OS << ", "; 6366 WriteAsOperand(OS, BA->getBlockAddress()->getBasicBlock(), false); 6367 OS << ">"; 6368 if (unsigned int TF = BA->getTargetFlags()) 6369 OS << " [TF=" << TF << ']'; 6370 } 6371 6372 if (G) 6373 if (unsigned Order = G->GetOrdering(this)) 6374 OS << " [ORD=" << Order << ']'; 6375 6376 if (getNodeId() != -1) 6377 OS << " [ID=" << getNodeId() << ']'; 6378 6379 DebugLoc dl = getDebugLoc(); 6380 if (G && !dl.isUnknown()) { 6381 DIScope 6382 Scope(dl.getScope(G->getMachineFunction().getFunction()->getContext())); 6383 OS << " dbg:"; 6384 // Omit the directory, since it's usually long and uninteresting. 6385 if (Scope.Verify()) 6386 OS << Scope.getFilename(); 6387 else 6388 OS << "<unknown>"; 6389 OS << ':' << dl.getLine(); 6390 if (dl.getCol() != 0) 6391 OS << ':' << dl.getCol(); 6392 } 6393} 6394 6395void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { 6396 print_types(OS, G); 6397 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { 6398 if (i) OS << ", "; else OS << " "; 6399 OS << (void*)getOperand(i).getNode(); 6400 if (unsigned RN = getOperand(i).getResNo()) 6401 OS << ":" << RN; 6402 } 6403 print_details(OS, G); 6404} 6405 6406static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N, 6407 const SelectionDAG *G, unsigned depth, 6408 unsigned indent) { 6409 if (depth == 0) 6410 return; 6411 6412 OS.indent(indent); 6413 6414 N->print(OS, G); 6415 6416 if (depth < 1) 6417 return; 6418 6419 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6420 // Don't follow chain operands. 6421 if (N->getOperand(i).getValueType() == MVT::Other) 6422 continue; 6423 OS << '\n'; 6424 printrWithDepthHelper(OS, N->getOperand(i).getNode(), G, depth-1, indent+2); 6425 } 6426} 6427 6428void SDNode::printrWithDepth(raw_ostream &OS, const SelectionDAG *G, 6429 unsigned depth) const { 6430 printrWithDepthHelper(OS, this, G, depth, 0); 6431} 6432 6433void SDNode::printrFull(raw_ostream &OS, const SelectionDAG *G) const { 6434 // Don't print impossibly deep things. 6435 printrWithDepth(OS, G, 10); 6436} 6437 6438void SDNode::dumprWithDepth(const SelectionDAG *G, unsigned depth) const { 6439 printrWithDepth(dbgs(), G, depth); 6440} 6441 6442void SDNode::dumprFull(const SelectionDAG *G) const { 6443 // Don't print impossibly deep things. 6444 dumprWithDepth(G, 10); 6445} 6446 6447static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) { 6448 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6449 if (N->getOperand(i).getNode()->hasOneUse()) 6450 DumpNodes(N->getOperand(i).getNode(), indent+2, G); 6451 else 6452 dbgs() << "\n" << std::string(indent+2, ' ') 6453 << (void*)N->getOperand(i).getNode() << ": <multiple use>"; 6454 6455 6456 dbgs() << "\n"; 6457 dbgs().indent(indent); 6458 N->dump(G); 6459} 6460 6461SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6462 assert(N->getNumValues() == 1 && 6463 "Can't unroll a vector with multiple results!"); 6464 6465 EVT VT = N->getValueType(0); 6466 unsigned NE = VT.getVectorNumElements(); 6467 EVT EltVT = VT.getVectorElementType(); 6468 DebugLoc dl = N->getDebugLoc(); 6469 6470 SmallVector<SDValue, 8> Scalars; 6471 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6472 6473 // If ResNE is 0, fully unroll the vector op. 6474 if (ResNE == 0) 6475 ResNE = NE; 6476 else if (NE > ResNE) 6477 NE = ResNE; 6478 6479 unsigned i; 6480 for (i= 0; i != NE; ++i) { 6481 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6482 SDValue Operand = N->getOperand(j); 6483 EVT OperandVT = Operand.getValueType(); 6484 if (OperandVT.isVector()) { 6485 // A vector operand; extract a single element. 6486 EVT OperandEltVT = OperandVT.getVectorElementType(); 6487 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6488 OperandEltVT, 6489 Operand, 6490 getConstant(i, TLI.getPointerTy())); 6491 } else { 6492 // A scalar operand; just use it as is. 6493 Operands[j] = Operand; 6494 } 6495 } 6496 6497 switch (N->getOpcode()) { 6498 default: 6499 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6500 &Operands[0], Operands.size())); 6501 break; 6502 case ISD::VSELECT: 6503 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6504 &Operands[0], Operands.size())); 6505 break; 6506 case ISD::SHL: 6507 case ISD::SRA: 6508 case ISD::SRL: 6509 case ISD::ROTL: 6510 case ISD::ROTR: 6511 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6512 getShiftAmountOperand(Operands[0].getValueType(), 6513 Operands[1]))); 6514 break; 6515 case ISD::SIGN_EXTEND_INREG: 6516 case ISD::FP_ROUND_INREG: { 6517 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6518 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6519 Operands[0], 6520 getValueType(ExtVT))); 6521 } 6522 } 6523 } 6524 6525 for (; i < ResNE; ++i) 6526 Scalars.push_back(getUNDEF(EltVT)); 6527 6528 return getNode(ISD::BUILD_VECTOR, dl, 6529 EVT::getVectorVT(*getContext(), EltVT, ResNE), 6530 &Scalars[0], Scalars.size()); 6531} 6532 6533 6534/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6535/// location that is 'Dist' units away from the location that the 'Base' load 6536/// is loading from. 6537bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6538 unsigned Bytes, int Dist) const { 6539 if (LD->getChain() != Base->getChain()) 6540 return false; 6541 EVT VT = LD->getValueType(0); 6542 if (VT.getSizeInBits() / 8 != Bytes) 6543 return false; 6544 6545 SDValue Loc = LD->getOperand(1); 6546 SDValue BaseLoc = Base->getOperand(1); 6547 if (Loc.getOpcode() == ISD::FrameIndex) { 6548 if (BaseLoc.getOpcode() != ISD::FrameIndex) 6549 return false; 6550 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6551 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6552 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6553 int FS = MFI->getObjectSize(FI); 6554 int BFS = MFI->getObjectSize(BFI); 6555 if (FS != BFS || FS != (int)Bytes) return false; 6556 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6557 } 6558 6559 // Handle X+C 6560 if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6561 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6562 return true; 6563 6564 const GlobalValue *GV1 = NULL; 6565 const GlobalValue *GV2 = NULL; 6566 int64_t Offset1 = 0; 6567 int64_t Offset2 = 0; 6568 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6569 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6570 if (isGA1 && isGA2 && GV1 == GV2) 6571 return Offset1 == (Offset2 + Dist*Bytes); 6572 return false; 6573} 6574 6575 6576/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6577/// it cannot be inferred. 6578unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6579 // If this is a GlobalAddress + cst, return the alignment. 6580 const GlobalValue *GV; 6581 int64_t GVOffset = 0; 6582 if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6583 unsigned PtrWidth = TLI.getPointerTy().getSizeInBits(); 6584 APInt AllOnes = APInt::getAllOnesValue(PtrWidth); 6585 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 6586 llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), AllOnes, 6587 KnownZero, KnownOne, TLI.getTargetData()); 6588 unsigned AlignBits = KnownZero.countTrailingOnes(); 6589 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 6590 if (Align) 6591 return MinAlign(Align, GVOffset); 6592 } 6593 6594 // If this is a direct reference to a stack slot, use information about the 6595 // stack slot's alignment. 6596 int FrameIdx = 1 << 31; 6597 int64_t FrameOffset = 0; 6598 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6599 FrameIdx = FI->getIndex(); 6600 } else if (isBaseWithConstantOffset(Ptr) && 6601 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6602 // Handle FI+Cst 6603 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6604 FrameOffset = Ptr.getConstantOperandVal(1); 6605 } 6606 6607 if (FrameIdx != (1 << 31)) { 6608 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6609 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6610 FrameOffset); 6611 return FIInfoAlign; 6612 } 6613 6614 return 0; 6615} 6616 6617void SelectionDAG::dump() const { 6618 dbgs() << "SelectionDAG has " << AllNodes.size() << " nodes:"; 6619 6620 for (allnodes_const_iterator I = allnodes_begin(), E = allnodes_end(); 6621 I != E; ++I) { 6622 const SDNode *N = I; 6623 if (!N->hasOneUse() && N != getRoot().getNode()) 6624 DumpNodes(N, 2, this); 6625 } 6626 6627 if (getRoot().getNode()) DumpNodes(getRoot().getNode(), 2, this); 6628 6629 dbgs() << "\n\n"; 6630} 6631 6632void SDNode::printr(raw_ostream &OS, const SelectionDAG *G) const { 6633 print_types(OS, G); 6634 print_details(OS, G); 6635} 6636 6637typedef SmallPtrSet<const SDNode *, 128> VisitedSDNodeSet; 6638static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent, 6639 const SelectionDAG *G, VisitedSDNodeSet &once) { 6640 if (!once.insert(N)) // If we've been here before, return now. 6641 return; 6642 6643 // Dump the current SDNode, but don't end the line yet. 6644 OS.indent(indent); 6645 N->printr(OS, G); 6646 6647 // Having printed this SDNode, walk the children: 6648 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6649 const SDNode *child = N->getOperand(i).getNode(); 6650 6651 if (i) OS << ","; 6652 OS << " "; 6653 6654 if (child->getNumOperands() == 0) { 6655 // This child has no grandchildren; print it inline right here. 6656 child->printr(OS, G); 6657 once.insert(child); 6658 } else { // Just the address. FIXME: also print the child's opcode. 6659 OS << (void*)child; 6660 if (unsigned RN = N->getOperand(i).getResNo()) 6661 OS << ":" << RN; 6662 } 6663 } 6664 6665 OS << "\n"; 6666 6667 // Dump children that have grandchildren on their own line(s). 6668 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 6669 const SDNode *child = N->getOperand(i).getNode(); 6670 DumpNodesr(OS, child, indent+2, G, once); 6671 } 6672} 6673 6674void SDNode::dumpr() const { 6675 VisitedSDNodeSet once; 6676 DumpNodesr(dbgs(), this, 0, 0, once); 6677} 6678 6679void SDNode::dumpr(const SelectionDAG *G) const { 6680 VisitedSDNodeSet once; 6681 DumpNodesr(dbgs(), this, 0, G, once); 6682} 6683 6684 6685// getAddressSpace - Return the address space this GlobalAddress belongs to. 6686unsigned GlobalAddressSDNode::getAddressSpace() const { 6687 return getGlobal()->getType()->getAddressSpace(); 6688} 6689 6690 6691Type *ConstantPoolSDNode::getType() const { 6692 if (isMachineConstantPoolEntry()) 6693 return Val.MachineCPVal->getType(); 6694 return Val.ConstVal->getType(); 6695} 6696 6697bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6698 APInt &SplatUndef, 6699 unsigned &SplatBitSize, 6700 bool &HasAnyUndefs, 6701 unsigned MinSplatBits, 6702 bool isBigEndian) { 6703 EVT VT = getValueType(0); 6704 assert(VT.isVector() && "Expected a vector type"); 6705 unsigned sz = VT.getSizeInBits(); 6706 if (MinSplatBits > sz) 6707 return false; 6708 6709 SplatValue = APInt(sz, 0); 6710 SplatUndef = APInt(sz, 0); 6711 6712 // Get the bits. Bits with undefined values (when the corresponding element 6713 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6714 // in SplatValue. If any of the values are not constant, give up and return 6715 // false. 6716 unsigned int nOps = getNumOperands(); 6717 assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6718 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6719 6720 for (unsigned j = 0; j < nOps; ++j) { 6721 unsigned i = isBigEndian ? nOps-1-j : j; 6722 SDValue OpVal = getOperand(i); 6723 unsigned BitPos = j * EltBitSize; 6724 6725 if (OpVal.getOpcode() == ISD::UNDEF) 6726 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6727 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6728 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6729 zextOrTrunc(sz) << BitPos; 6730 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6731 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6732 else 6733 return false; 6734 } 6735 6736 // The build_vector is all constants or undefs. Find the smallest element 6737 // size that splats the vector. 6738 6739 HasAnyUndefs = (SplatUndef != 0); 6740 while (sz > 8) { 6741 6742 unsigned HalfSize = sz / 2; 6743 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6744 APInt LowValue = SplatValue.trunc(HalfSize); 6745 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6746 APInt LowUndef = SplatUndef.trunc(HalfSize); 6747 6748 // If the two halves do not match (ignoring undef bits), stop here. 6749 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6750 MinSplatBits > HalfSize) 6751 break; 6752 6753 SplatValue = HighValue | LowValue; 6754 SplatUndef = HighUndef & LowUndef; 6755 6756 sz = HalfSize; 6757 } 6758 6759 SplatBitSize = sz; 6760 return true; 6761} 6762 6763bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6764 // Find the first non-undef value in the shuffle mask. 6765 unsigned i, e; 6766 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6767 /* search */; 6768 6769 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6770 6771 // Make sure all remaining elements are either undef or the same as the first 6772 // non-undef value. 6773 for (int Idx = Mask[i]; i != e; ++i) 6774 if (Mask[i] >= 0 && Mask[i] != Idx) 6775 return false; 6776 return true; 6777} 6778 6779#ifdef XDEBUG 6780static void checkForCyclesHelper(const SDNode *N, 6781 SmallPtrSet<const SDNode*, 32> &Visited, 6782 SmallPtrSet<const SDNode*, 32> &Checked) { 6783 // If this node has already been checked, don't check it again. 6784 if (Checked.count(N)) 6785 return; 6786 6787 // If a node has already been visited on this depth-first walk, reject it as 6788 // a cycle. 6789 if (!Visited.insert(N)) { 6790 dbgs() << "Offending node:\n"; 6791 N->dumprFull(); 6792 errs() << "Detected cycle in SelectionDAG\n"; 6793 abort(); 6794 } 6795 6796 for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6797 checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6798 6799 Checked.insert(N); 6800 Visited.erase(N); 6801} 6802#endif 6803 6804void llvm::checkForCycles(const llvm::SDNode *N) { 6805#ifdef XDEBUG 6806 assert(N && "Checking nonexistant SDNode"); 6807 SmallPtrSet<const SDNode*, 32> visited; 6808 SmallPtrSet<const SDNode*, 32> checked; 6809 checkForCyclesHelper(N, visited, checked); 6810#endif 6811} 6812 6813void llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6814 checkForCycles(DAG->getRoot().getNode()); 6815} 6816