LegalizeDAG.cpp revision 40b2c32475572d33ede7cf202f2d49cab9b43bb4
1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SelectionDAG::Legalize method. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "llvm/ADT/DenseMap.h" 16#include "llvm/ADT/SmallPtrSet.h" 17#include "llvm/ADT/SmallVector.h" 18#include "llvm/CodeGen/Analysis.h" 19#include "llvm/CodeGen/MachineFunction.h" 20#include "llvm/CodeGen/MachineJumpTableInfo.h" 21#include "llvm/DebugInfo.h" 22#include "llvm/IR/CallingConv.h" 23#include "llvm/IR/Constants.h" 24#include "llvm/IR/DataLayout.h" 25#include "llvm/IR/DerivedTypes.h" 26#include "llvm/IR/Function.h" 27#include "llvm/IR/LLVMContext.h" 28#include "llvm/Support/Debug.h" 29#include "llvm/Support/ErrorHandling.h" 30#include "llvm/Support/MathExtras.h" 31#include "llvm/Support/raw_ostream.h" 32#include "llvm/Target/TargetFrameLowering.h" 33#include "llvm/Target/TargetLowering.h" 34#include "llvm/Target/TargetMachine.h" 35using namespace llvm; 36 37//===----------------------------------------------------------------------===// 38/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and 39/// hacks on it until the target machine can handle it. This involves 40/// eliminating value sizes the machine cannot handle (promoting small sizes to 41/// large sizes or splitting up large values into small values) as well as 42/// eliminating operations the machine cannot handle. 43/// 44/// This code also does a small amount of optimization and recognition of idioms 45/// as part of its processing. For example, if a target does not support a 46/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 47/// will attempt merge setcc and brc instructions into brcc's. 48/// 49namespace { 50class SelectionDAGLegalize : public SelectionDAG::DAGUpdateListener { 51 const TargetMachine &TM; 52 const TargetLowering &TLI; 53 SelectionDAG &DAG; 54 55 /// LegalizePosition - The iterator for walking through the node list. 56 SelectionDAG::allnodes_iterator LegalizePosition; 57 58 /// LegalizedNodes - The set of nodes which have already been legalized. 59 SmallPtrSet<SDNode *, 16> LegalizedNodes; 60 61 // Libcall insertion helpers. 62 63public: 64 explicit SelectionDAGLegalize(SelectionDAG &DAG); 65 66 void LegalizeDAG(); 67 68private: 69 /// LegalizeOp - Legalizes the given operation. 70 void LegalizeOp(SDNode *Node); 71 72 SDValue OptimizeFloatStore(StoreSDNode *ST); 73 74 void LegalizeLoadOps(SDNode *Node); 75 void LegalizeStoreOps(SDNode *Node); 76 77 /// PerformInsertVectorEltInMemory - Some target cannot handle a variable 78 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 79 /// is necessary to spill the vector being inserted into to memory, perform 80 /// the insert there, and then read the result back. 81 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, 82 SDValue Idx, DebugLoc dl); 83 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, 84 SDValue Idx, DebugLoc dl); 85 86 /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 87 /// performs the same shuffe in terms of order or result bytes, but on a type 88 /// whose vector element type is narrower than the original shuffle type. 89 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 90 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl, 91 SDValue N1, SDValue N2, 92 ArrayRef<int> Mask) const; 93 94 void LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 95 DebugLoc dl); 96 97 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); 98 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, 99 unsigned NumOps, bool isSigned, DebugLoc dl); 100 101 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, 102 SDNode *Node, bool isSigned); 103 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, 104 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, 105 RTLIB::Libcall Call_PPCF128); 106 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, 107 RTLIB::Libcall Call_I8, 108 RTLIB::Libcall Call_I16, 109 RTLIB::Libcall Call_I32, 110 RTLIB::Libcall Call_I64, 111 RTLIB::Libcall Call_I128); 112 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 113 114 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, DebugLoc dl); 115 SDValue ExpandBUILD_VECTOR(SDNode *Node); 116 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 117 void ExpandDYNAMIC_STACKALLOC(SDNode *Node, 118 SmallVectorImpl<SDValue> &Results); 119 SDValue ExpandFCOPYSIGN(SDNode *Node); 120 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT, 121 DebugLoc dl); 122 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, 123 DebugLoc dl); 124 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, 125 DebugLoc dl); 126 127 SDValue ExpandBSWAP(SDValue Op, DebugLoc dl); 128 SDValue ExpandBitCount(unsigned Opc, SDValue Op, DebugLoc dl); 129 130 SDValue ExpandExtractFromVectorThroughStack(SDValue Op); 131 SDValue ExpandInsertToVectorThroughStack(SDValue Op); 132 SDValue ExpandVectorBuildThroughStack(SDNode* Node); 133 134 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); 135 136 std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); 137 138 void ExpandNode(SDNode *Node); 139 void PromoteNode(SDNode *Node); 140 141 void ForgetNode(SDNode *N) { 142 LegalizedNodes.erase(N); 143 if (LegalizePosition == SelectionDAG::allnodes_iterator(N)) 144 ++LegalizePosition; 145 } 146 147public: 148 // DAGUpdateListener implementation. 149 virtual void NodeDeleted(SDNode *N, SDNode *E) { 150 ForgetNode(N); 151 } 152 virtual void NodeUpdated(SDNode *N) {} 153 154 // Node replacement helpers 155 void ReplacedNode(SDNode *N) { 156 if (N->use_empty()) { 157 DAG.RemoveDeadNode(N); 158 } else { 159 ForgetNode(N); 160 } 161 } 162 void ReplaceNode(SDNode *Old, SDNode *New) { 163 DAG.ReplaceAllUsesWith(Old, New); 164 ReplacedNode(Old); 165 } 166 void ReplaceNode(SDValue Old, SDValue New) { 167 DAG.ReplaceAllUsesWith(Old, New); 168 ReplacedNode(Old.getNode()); 169 } 170 void ReplaceNode(SDNode *Old, const SDValue *New) { 171 DAG.ReplaceAllUsesWith(Old, New); 172 ReplacedNode(Old); 173 } 174}; 175} 176 177/// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 178/// performs the same shuffe in terms of order or result bytes, but on a type 179/// whose vector element type is narrower than the original shuffle type. 180/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 181SDValue 182SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl, 183 SDValue N1, SDValue N2, 184 ArrayRef<int> Mask) const { 185 unsigned NumMaskElts = VT.getVectorNumElements(); 186 unsigned NumDestElts = NVT.getVectorNumElements(); 187 unsigned NumEltsGrowth = NumDestElts / NumMaskElts; 188 189 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 190 191 if (NumEltsGrowth == 1) 192 return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]); 193 194 SmallVector<int, 8> NewMask; 195 for (unsigned i = 0; i != NumMaskElts; ++i) { 196 int Idx = Mask[i]; 197 for (unsigned j = 0; j != NumEltsGrowth; ++j) { 198 if (Idx < 0) 199 NewMask.push_back(-1); 200 else 201 NewMask.push_back(Idx * NumEltsGrowth + j); 202 } 203 } 204 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); 205 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); 206 return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); 207} 208 209SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) 210 : SelectionDAG::DAGUpdateListener(dag), 211 TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()), 212 DAG(dag) { 213} 214 215void SelectionDAGLegalize::LegalizeDAG() { 216 DAG.AssignTopologicalOrder(); 217 218 // Visit all the nodes. We start in topological order, so that we see 219 // nodes with their original operands intact. Legalization can produce 220 // new nodes which may themselves need to be legalized. Iterate until all 221 // nodes have been legalized. 222 for (;;) { 223 bool AnyLegalized = false; 224 for (LegalizePosition = DAG.allnodes_end(); 225 LegalizePosition != DAG.allnodes_begin(); ) { 226 --LegalizePosition; 227 228 SDNode *N = LegalizePosition; 229 if (LegalizedNodes.insert(N)) { 230 AnyLegalized = true; 231 LegalizeOp(N); 232 } 233 } 234 if (!AnyLegalized) 235 break; 236 237 } 238 239 // Remove dead nodes now. 240 DAG.RemoveDeadNodes(); 241} 242 243/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or 244/// a load from the constant pool. 245SDValue 246SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { 247 bool Extend = false; 248 DebugLoc dl = CFP->getDebugLoc(); 249 250 // If a FP immediate is precise when represented as a float and if the 251 // target can do an extending load from float to double, we put it into 252 // the constant pool as a float, even if it's is statically typed as a 253 // double. This shrinks FP constants and canonicalizes them for targets where 254 // an FP extending load is the same cost as a normal load (such as on the x87 255 // fp stack or PPC FP unit). 256 EVT VT = CFP->getValueType(0); 257 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 258 if (!UseCP) { 259 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); 260 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), 261 (VT == MVT::f64) ? MVT::i64 : MVT::i32); 262 } 263 264 EVT OrigVT = VT; 265 EVT SVT = VT; 266 while (SVT != MVT::f32) { 267 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); 268 if (ConstantFPSDNode::isValueValidForType(SVT, CFP->getValueAPF()) && 269 // Only do this if the target has a native EXTLOAD instruction from 270 // smaller type. 271 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && 272 TLI.ShouldShrinkFPConstant(OrigVT)) { 273 Type *SType = SVT.getTypeForEVT(*DAG.getContext()); 274 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 275 VT = SVT; 276 Extend = true; 277 } 278 } 279 280 SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); 281 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 282 if (Extend) { 283 SDValue Result = 284 DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT, 285 DAG.getEntryNode(), 286 CPIdx, MachinePointerInfo::getConstantPool(), 287 VT, false, false, Alignment); 288 return Result; 289 } 290 SDValue Result = 291 DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx, 292 MachinePointerInfo::getConstantPool(), false, false, false, 293 Alignment); 294 return Result; 295} 296 297/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. 298static void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, 299 const TargetLowering &TLI, 300 SelectionDAGLegalize *DAGLegalize) { 301 assert(ST->getAddressingMode() == ISD::UNINDEXED && 302 "unaligned indexed stores not implemented!"); 303 SDValue Chain = ST->getChain(); 304 SDValue Ptr = ST->getBasePtr(); 305 SDValue Val = ST->getValue(); 306 EVT VT = Val.getValueType(); 307 int Alignment = ST->getAlignment(); 308 DebugLoc dl = ST->getDebugLoc(); 309 if (ST->getMemoryVT().isFloatingPoint() || 310 ST->getMemoryVT().isVector()) { 311 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 312 if (TLI.isTypeLegal(intVT)) { 313 // Expand to a bitconvert of the value to the integer type of the 314 // same size, then a (misaligned) int store. 315 // FIXME: Does not handle truncating floating point stores! 316 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); 317 Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(), 318 ST->isVolatile(), ST->isNonTemporal(), Alignment); 319 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 320 return; 321 } 322 // Do a (aligned) store to a stack slot, then copy from the stack slot 323 // to the final destination using (unaligned) integer loads and stores. 324 EVT StoredVT = ST->getMemoryVT(); 325 MVT RegVT = 326 TLI.getRegisterType(*DAG.getContext(), 327 EVT::getIntegerVT(*DAG.getContext(), 328 StoredVT.getSizeInBits())); 329 unsigned StoredBytes = StoredVT.getSizeInBits() / 8; 330 unsigned RegBytes = RegVT.getSizeInBits() / 8; 331 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; 332 333 // Make sure the stack slot is also aligned for the register type. 334 SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); 335 336 // Perform the original store, only redirected to the stack slot. 337 SDValue Store = DAG.getTruncStore(Chain, dl, 338 Val, StackPtr, MachinePointerInfo(), 339 StoredVT, false, false, 0); 340 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 341 SmallVector<SDValue, 8> Stores; 342 unsigned Offset = 0; 343 344 // Do all but one copies using the full register width. 345 for (unsigned i = 1; i < NumRegs; i++) { 346 // Load one integer register's worth from the stack slot. 347 SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr, 348 MachinePointerInfo(), 349 false, false, false, 0); 350 // Store it to the final location. Remember the store. 351 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, 352 ST->getPointerInfo().getWithOffset(Offset), 353 ST->isVolatile(), ST->isNonTemporal(), 354 MinAlign(ST->getAlignment(), Offset))); 355 // Increment the pointers. 356 Offset += RegBytes; 357 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 358 Increment); 359 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 360 } 361 362 // The last store may be partial. Do a truncating store. On big-endian 363 // machines this requires an extending load from the stack slot to ensure 364 // that the bits are in the right place. 365 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 366 8 * (StoredBytes - Offset)); 367 368 // Load from the stack slot. 369 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, 370 MachinePointerInfo(), 371 MemVT, false, false, 0); 372 373 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, 374 ST->getPointerInfo() 375 .getWithOffset(Offset), 376 MemVT, ST->isVolatile(), 377 ST->isNonTemporal(), 378 MinAlign(ST->getAlignment(), Offset))); 379 // The order of the stores doesn't matter - say it with a TokenFactor. 380 SDValue Result = 381 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 382 Stores.size()); 383 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 384 return; 385 } 386 assert(ST->getMemoryVT().isInteger() && 387 !ST->getMemoryVT().isVector() && 388 "Unaligned store of unknown type."); 389 // Get the half-size VT 390 EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext()); 391 int NumBits = NewStoredVT.getSizeInBits(); 392 int IncrementSize = NumBits / 8; 393 394 // Divide the stored value in two parts. 395 SDValue ShiftAmount = DAG.getConstant(NumBits, 396 TLI.getShiftAmountTy(Val.getValueType())); 397 SDValue Lo = Val; 398 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); 399 400 // Store the two parts 401 SDValue Store1, Store2; 402 Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr, 403 ST->getPointerInfo(), NewStoredVT, 404 ST->isVolatile(), ST->isNonTemporal(), Alignment); 405 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 406 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 407 Alignment = MinAlign(Alignment, IncrementSize); 408 Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr, 409 ST->getPointerInfo().getWithOffset(IncrementSize), 410 NewStoredVT, ST->isVolatile(), ST->isNonTemporal(), 411 Alignment); 412 413 SDValue Result = 414 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); 415 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 416} 417 418/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. 419static void 420ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, 421 const TargetLowering &TLI, 422 SDValue &ValResult, SDValue &ChainResult) { 423 assert(LD->getAddressingMode() == ISD::UNINDEXED && 424 "unaligned indexed loads not implemented!"); 425 SDValue Chain = LD->getChain(); 426 SDValue Ptr = LD->getBasePtr(); 427 EVT VT = LD->getValueType(0); 428 EVT LoadedVT = LD->getMemoryVT(); 429 DebugLoc dl = LD->getDebugLoc(); 430 if (VT.isFloatingPoint() || VT.isVector()) { 431 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits()); 432 if (TLI.isTypeLegal(intVT) && TLI.isTypeLegal(LoadedVT)) { 433 // Expand to a (misaligned) integer load of the same size, 434 // then bitconvert to floating point or vector. 435 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, LD->getPointerInfo(), 436 LD->isVolatile(), 437 LD->isNonTemporal(), 438 LD->isInvariant(), LD->getAlignment()); 439 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); 440 if (LoadedVT != VT) 441 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : 442 ISD::ANY_EXTEND, dl, VT, Result); 443 444 ValResult = Result; 445 ChainResult = Chain; 446 return; 447 } 448 449 // Copy the value to a (aligned) stack slot using (unaligned) integer 450 // loads and stores, then do a (aligned) load from the stack slot. 451 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT); 452 unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; 453 unsigned RegBytes = RegVT.getSizeInBits() / 8; 454 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; 455 456 // Make sure the stack slot is also aligned for the register type. 457 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT); 458 459 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 460 SmallVector<SDValue, 8> Stores; 461 SDValue StackPtr = StackBase; 462 unsigned Offset = 0; 463 464 // Do all but one copies using the full register width. 465 for (unsigned i = 1; i < NumRegs; i++) { 466 // Load one integer register's worth from the original location. 467 SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr, 468 LD->getPointerInfo().getWithOffset(Offset), 469 LD->isVolatile(), LD->isNonTemporal(), 470 LD->isInvariant(), 471 MinAlign(LD->getAlignment(), Offset)); 472 // Follow the load with a store to the stack slot. Remember the store. 473 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr, 474 MachinePointerInfo(), false, false, 0)); 475 // Increment the pointers. 476 Offset += RegBytes; 477 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 478 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 479 Increment); 480 } 481 482 // The last copy may be partial. Do an extending load. 483 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 484 8 * (LoadedBytes - Offset)); 485 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, 486 LD->getPointerInfo().getWithOffset(Offset), 487 MemVT, LD->isVolatile(), 488 LD->isNonTemporal(), 489 MinAlign(LD->getAlignment(), Offset)); 490 // Follow the load with a store to the stack slot. Remember the store. 491 // On big-endian machines this requires a truncating store to ensure 492 // that the bits end up in the right place. 493 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr, 494 MachinePointerInfo(), MemVT, 495 false, false, 0)); 496 497 // The order of the stores doesn't matter - say it with a TokenFactor. 498 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 499 Stores.size()); 500 501 // Finally, perform the original load only redirected to the stack slot. 502 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, 503 MachinePointerInfo(), LoadedVT, false, false, 0); 504 505 // Callers expect a MERGE_VALUES node. 506 ValResult = Load; 507 ChainResult = TF; 508 return; 509 } 510 assert(LoadedVT.isInteger() && !LoadedVT.isVector() && 511 "Unaligned load of unsupported type."); 512 513 // Compute the new VT that is half the size of the old one. This is an 514 // integer MVT. 515 unsigned NumBits = LoadedVT.getSizeInBits(); 516 EVT NewLoadedVT; 517 NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2); 518 NumBits >>= 1; 519 520 unsigned Alignment = LD->getAlignment(); 521 unsigned IncrementSize = NumBits / 8; 522 ISD::LoadExtType HiExtType = LD->getExtensionType(); 523 524 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. 525 if (HiExtType == ISD::NON_EXTLOAD) 526 HiExtType = ISD::ZEXTLOAD; 527 528 // Load the value in two parts 529 SDValue Lo, Hi; 530 if (TLI.isLittleEndian()) { 531 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), 532 NewLoadedVT, LD->isVolatile(), 533 LD->isNonTemporal(), Alignment); 534 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 535 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 536 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, 537 LD->getPointerInfo().getWithOffset(IncrementSize), 538 NewLoadedVT, LD->isVolatile(), 539 LD->isNonTemporal(), MinAlign(Alignment,IncrementSize)); 540 } else { 541 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), 542 NewLoadedVT, LD->isVolatile(), 543 LD->isNonTemporal(), Alignment); 544 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 545 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 546 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, 547 LD->getPointerInfo().getWithOffset(IncrementSize), 548 NewLoadedVT, LD->isVolatile(), 549 LD->isNonTemporal(), MinAlign(Alignment,IncrementSize)); 550 } 551 552 // aggregate the two parts 553 SDValue ShiftAmount = DAG.getConstant(NumBits, 554 TLI.getShiftAmountTy(Hi.getValueType())); 555 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); 556 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); 557 558 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 559 Hi.getValue(1)); 560 561 ValResult = Result; 562 ChainResult = TF; 563} 564 565/// PerformInsertVectorEltInMemory - Some target cannot handle a variable 566/// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 567/// is necessary to spill the vector being inserted into to memory, perform 568/// the insert there, and then read the result back. 569SDValue SelectionDAGLegalize:: 570PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 571 DebugLoc dl) { 572 SDValue Tmp1 = Vec; 573 SDValue Tmp2 = Val; 574 SDValue Tmp3 = Idx; 575 576 // If the target doesn't support this, we have to spill the input vector 577 // to a temporary stack slot, update the element, then reload it. This is 578 // badness. We could also load the value into a vector register (either 579 // with a "move to register" or "extload into register" instruction, then 580 // permute it into place, if the idx is a constant and if the idx is 581 // supported by the target. 582 EVT VT = Tmp1.getValueType(); 583 EVT EltVT = VT.getVectorElementType(); 584 EVT IdxVT = Tmp3.getValueType(); 585 EVT PtrVT = TLI.getPointerTy(); 586 SDValue StackPtr = DAG.CreateStackTemporary(VT); 587 588 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 589 590 // Store the vector. 591 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr, 592 MachinePointerInfo::getFixedStack(SPFI), 593 false, false, 0); 594 595 // Truncate or zero extend offset to target pointer type. 596 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 597 Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3); 598 // Add the offset to the index. 599 unsigned EltSize = EltVT.getSizeInBits()/8; 600 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); 601 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); 602 // Store the scalar value. 603 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT, 604 false, false, 0); 605 // Load the updated vector. 606 return DAG.getLoad(VT, dl, Ch, StackPtr, 607 MachinePointerInfo::getFixedStack(SPFI), false, false, 608 false, 0); 609} 610 611 612SDValue SelectionDAGLegalize:: 613ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, DebugLoc dl) { 614 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { 615 // SCALAR_TO_VECTOR requires that the type of the value being inserted 616 // match the element type of the vector being created, except for 617 // integers in which case the inserted value can be over width. 618 EVT EltVT = Vec.getValueType().getVectorElementType(); 619 if (Val.getValueType() == EltVT || 620 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { 621 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 622 Vec.getValueType(), Val); 623 624 unsigned NumElts = Vec.getValueType().getVectorNumElements(); 625 // We generate a shuffle of InVec and ScVec, so the shuffle mask 626 // should be 0,1,2,3,4,5... with the appropriate element replaced with 627 // elt 0 of the RHS. 628 SmallVector<int, 8> ShufOps; 629 for (unsigned i = 0; i != NumElts; ++i) 630 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); 631 632 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, 633 &ShufOps[0]); 634 } 635 } 636 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); 637} 638 639SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { 640 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 641 // FIXME: We shouldn't do this for TargetConstantFP's. 642 // FIXME: move this to the DAG Combiner! Note that we can't regress due 643 // to phase ordering between legalized code and the dag combiner. This 644 // probably means that we need to integrate dag combiner and legalizer 645 // together. 646 // We generally can't do this one for long doubles. 647 SDValue Chain = ST->getChain(); 648 SDValue Ptr = ST->getBasePtr(); 649 unsigned Alignment = ST->getAlignment(); 650 bool isVolatile = ST->isVolatile(); 651 bool isNonTemporal = ST->isNonTemporal(); 652 DebugLoc dl = ST->getDebugLoc(); 653 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 654 if (CFP->getValueType(0) == MVT::f32 && 655 TLI.isTypeLegal(MVT::i32)) { 656 SDValue Con = DAG.getConstant(CFP->getValueAPF(). 657 bitcastToAPInt().zextOrTrunc(32), 658 MVT::i32); 659 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 660 isVolatile, isNonTemporal, Alignment); 661 } 662 663 if (CFP->getValueType(0) == MVT::f64) { 664 // If this target supports 64-bit registers, do a single 64-bit store. 665 if (TLI.isTypeLegal(MVT::i64)) { 666 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 667 zextOrTrunc(64), MVT::i64); 668 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 669 isVolatile, isNonTemporal, Alignment); 670 } 671 672 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { 673 // Otherwise, if the target supports 32-bit registers, use 2 32-bit 674 // stores. If the target supports neither 32- nor 64-bits, this 675 // xform is certainly not worth it. 676 const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); 677 SDValue Lo = DAG.getConstant(IntVal.trunc(32), MVT::i32); 678 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); 679 if (TLI.isBigEndian()) std::swap(Lo, Hi); 680 681 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), isVolatile, 682 isNonTemporal, Alignment); 683 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 684 DAG.getIntPtrConstant(4)); 685 Hi = DAG.getStore(Chain, dl, Hi, Ptr, 686 ST->getPointerInfo().getWithOffset(4), 687 isVolatile, isNonTemporal, MinAlign(Alignment, 4U)); 688 689 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 690 } 691 } 692 } 693 return SDValue(0, 0); 694} 695 696void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { 697 StoreSDNode *ST = cast<StoreSDNode>(Node); 698 SDValue Chain = ST->getChain(); 699 SDValue Ptr = ST->getBasePtr(); 700 DebugLoc dl = Node->getDebugLoc(); 701 702 unsigned Alignment = ST->getAlignment(); 703 bool isVolatile = ST->isVolatile(); 704 bool isNonTemporal = ST->isNonTemporal(); 705 706 if (!ST->isTruncatingStore()) { 707 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { 708 ReplaceNode(ST, OptStore); 709 return; 710 } 711 712 { 713 SDValue Value = ST->getValue(); 714 MVT VT = Value.getSimpleValueType(); 715 switch (TLI.getOperationAction(ISD::STORE, VT)) { 716 default: llvm_unreachable("This action is not supported yet!"); 717 case TargetLowering::Legal: 718 // If this is an unaligned store and the target doesn't support it, 719 // expand it. 720 if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { 721 Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 722 unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); 723 if (ST->getAlignment() < ABIAlignment) 724 ExpandUnalignedStore(cast<StoreSDNode>(Node), 725 DAG, TLI, this); 726 } 727 break; 728 case TargetLowering::Custom: { 729 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 730 if (Res.getNode()) 731 ReplaceNode(SDValue(Node, 0), Res); 732 return; 733 } 734 case TargetLowering::Promote: { 735 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); 736 assert(NVT.getSizeInBits() == VT.getSizeInBits() && 737 "Can only promote stores to same size type"); 738 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); 739 SDValue Result = 740 DAG.getStore(Chain, dl, Value, Ptr, 741 ST->getPointerInfo(), isVolatile, 742 isNonTemporal, Alignment); 743 ReplaceNode(SDValue(Node, 0), Result); 744 break; 745 } 746 } 747 return; 748 } 749 } else { 750 SDValue Value = ST->getValue(); 751 752 EVT StVT = ST->getMemoryVT(); 753 unsigned StWidth = StVT.getSizeInBits(); 754 755 if (StWidth != StVT.getStoreSizeInBits()) { 756 // Promote to a byte-sized store with upper bits zero if not 757 // storing an integral number of bytes. For example, promote 758 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 759 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), 760 StVT.getStoreSizeInBits()); 761 Value = DAG.getZeroExtendInReg(Value, dl, StVT); 762 SDValue Result = 763 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 764 NVT, isVolatile, isNonTemporal, Alignment); 765 ReplaceNode(SDValue(Node, 0), Result); 766 } else if (StWidth & (StWidth - 1)) { 767 // If not storing a power-of-2 number of bits, expand as two stores. 768 assert(!StVT.isVector() && "Unsupported truncstore!"); 769 unsigned RoundWidth = 1 << Log2_32(StWidth); 770 assert(RoundWidth < StWidth); 771 unsigned ExtraWidth = StWidth - RoundWidth; 772 assert(ExtraWidth < RoundWidth); 773 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 774 "Store size not an integral number of bytes!"); 775 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 776 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 777 SDValue Lo, Hi; 778 unsigned IncrementSize; 779 780 if (TLI.isLittleEndian()) { 781 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 782 // Store the bottom RoundWidth bits. 783 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 784 RoundVT, 785 isVolatile, isNonTemporal, Alignment); 786 787 // Store the remaining ExtraWidth bits. 788 IncrementSize = RoundWidth / 8; 789 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 790 DAG.getIntPtrConstant(IncrementSize)); 791 Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, 792 DAG.getConstant(RoundWidth, 793 TLI.getShiftAmountTy(Value.getValueType()))); 794 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, 795 ST->getPointerInfo().getWithOffset(IncrementSize), 796 ExtraVT, isVolatile, isNonTemporal, 797 MinAlign(Alignment, IncrementSize)); 798 } else { 799 // Big endian - avoid unaligned stores. 800 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 801 // Store the top RoundWidth bits. 802 Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, 803 DAG.getConstant(ExtraWidth, 804 TLI.getShiftAmountTy(Value.getValueType()))); 805 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), 806 RoundVT, isVolatile, isNonTemporal, Alignment); 807 808 // Store the remaining ExtraWidth bits. 809 IncrementSize = RoundWidth / 8; 810 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 811 DAG.getIntPtrConstant(IncrementSize)); 812 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, 813 ST->getPointerInfo().getWithOffset(IncrementSize), 814 ExtraVT, isVolatile, isNonTemporal, 815 MinAlign(Alignment, IncrementSize)); 816 } 817 818 // The order of the stores doesn't matter. 819 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 820 ReplaceNode(SDValue(Node, 0), Result); 821 } else { 822 switch (TLI.getTruncStoreAction(ST->getValue().getSimpleValueType(), 823 StVT.getSimpleVT())) { 824 default: llvm_unreachable("This action is not supported yet!"); 825 case TargetLowering::Legal: 826 // If this is an unaligned store and the target doesn't support it, 827 // expand it. 828 if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { 829 Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 830 unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); 831 if (ST->getAlignment() < ABIAlignment) 832 ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this); 833 } 834 break; 835 case TargetLowering::Custom: { 836 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 837 if (Res.getNode()) 838 ReplaceNode(SDValue(Node, 0), Res); 839 return; 840 } 841 case TargetLowering::Expand: 842 assert(!StVT.isVector() && 843 "Vector Stores are handled in LegalizeVectorOps"); 844 845 // TRUNCSTORE:i16 i32 -> STORE i16 846 assert(TLI.isTypeLegal(StVT) && 847 "Do not know how to expand this store!"); 848 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); 849 SDValue Result = 850 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 851 isVolatile, isNonTemporal, Alignment); 852 ReplaceNode(SDValue(Node, 0), Result); 853 break; 854 } 855 } 856 } 857} 858 859void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { 860 LoadSDNode *LD = cast<LoadSDNode>(Node); 861 SDValue Chain = LD->getChain(); // The chain. 862 SDValue Ptr = LD->getBasePtr(); // The base pointer. 863 SDValue Value; // The value returned by the load op. 864 DebugLoc dl = Node->getDebugLoc(); 865 866 ISD::LoadExtType ExtType = LD->getExtensionType(); 867 if (ExtType == ISD::NON_EXTLOAD) { 868 MVT VT = Node->getSimpleValueType(0); 869 SDValue RVal = SDValue(Node, 0); 870 SDValue RChain = SDValue(Node, 1); 871 872 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 873 default: llvm_unreachable("This action is not supported yet!"); 874 case TargetLowering::Legal: 875 // If this is an unaligned load and the target doesn't support it, 876 // expand it. 877 if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { 878 Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 879 unsigned ABIAlignment = 880 TLI.getDataLayout()->getABITypeAlignment(Ty); 881 if (LD->getAlignment() < ABIAlignment){ 882 ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain); 883 } 884 } 885 break; 886 case TargetLowering::Custom: { 887 SDValue Res = TLI.LowerOperation(RVal, DAG); 888 if (Res.getNode()) { 889 RVal = Res; 890 RChain = Res.getValue(1); 891 } 892 break; 893 } 894 case TargetLowering::Promote: { 895 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 896 assert(NVT.getSizeInBits() == VT.getSizeInBits() && 897 "Can only promote loads to same size type"); 898 899 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), 900 LD->isVolatile(), LD->isNonTemporal(), 901 LD->isInvariant(), LD->getAlignment()); 902 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); 903 RChain = Res.getValue(1); 904 break; 905 } 906 } 907 if (RChain.getNode() != Node) { 908 assert(RVal.getNode() != Node && "Load must be completely replaced"); 909 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); 910 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); 911 ReplacedNode(Node); 912 } 913 return; 914 } 915 916 EVT SrcVT = LD->getMemoryVT(); 917 unsigned SrcWidth = SrcVT.getSizeInBits(); 918 unsigned Alignment = LD->getAlignment(); 919 bool isVolatile = LD->isVolatile(); 920 bool isNonTemporal = LD->isNonTemporal(); 921 922 if (SrcWidth != SrcVT.getStoreSizeInBits() && 923 // Some targets pretend to have an i1 loading operation, and actually 924 // load an i8. This trick is correct for ZEXTLOAD because the top 7 925 // bits are guaranteed to be zero; it helps the optimizers understand 926 // that these bits are zero. It is also useful for EXTLOAD, since it 927 // tells the optimizers that those bits are undefined. It would be 928 // nice to have an effective generic way of getting these benefits... 929 // Until such a way is found, don't insist on promoting i1 here. 930 (SrcVT != MVT::i1 || 931 TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { 932 // Promote to a byte-sized load if not loading an integral number of 933 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 934 unsigned NewWidth = SrcVT.getStoreSizeInBits(); 935 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); 936 SDValue Ch; 937 938 // The extra bits are guaranteed to be zero, since we stored them that 939 // way. A zext load from NVT thus automatically gives zext from SrcVT. 940 941 ISD::LoadExtType NewExtType = 942 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 943 944 SDValue Result = 945 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), 946 Chain, Ptr, LD->getPointerInfo(), 947 NVT, isVolatile, isNonTemporal, Alignment); 948 949 Ch = Result.getValue(1); // The chain. 950 951 if (ExtType == ISD::SEXTLOAD) 952 // Having the top bits zero doesn't help when sign extending. 953 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 954 Result.getValueType(), 955 Result, DAG.getValueType(SrcVT)); 956 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 957 // All the top bits are guaranteed to be zero - inform the optimizers. 958 Result = DAG.getNode(ISD::AssertZext, dl, 959 Result.getValueType(), Result, 960 DAG.getValueType(SrcVT)); 961 962 Value = Result; 963 Chain = Ch; 964 } else if (SrcWidth & (SrcWidth - 1)) { 965 // If not loading a power-of-2 number of bits, expand as two loads. 966 assert(!SrcVT.isVector() && "Unsupported extload!"); 967 unsigned RoundWidth = 1 << Log2_32(SrcWidth); 968 assert(RoundWidth < SrcWidth); 969 unsigned ExtraWidth = SrcWidth - RoundWidth; 970 assert(ExtraWidth < RoundWidth); 971 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 972 "Load size not an integral number of bytes!"); 973 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 974 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 975 SDValue Lo, Hi, Ch; 976 unsigned IncrementSize; 977 978 if (TLI.isLittleEndian()) { 979 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 980 // Load the bottom RoundWidth bits. 981 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), 982 Chain, Ptr, 983 LD->getPointerInfo(), RoundVT, isVolatile, 984 isNonTemporal, Alignment); 985 986 // Load the remaining ExtraWidth bits. 987 IncrementSize = RoundWidth / 8; 988 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 989 DAG.getIntPtrConstant(IncrementSize)); 990 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 991 LD->getPointerInfo().getWithOffset(IncrementSize), 992 ExtraVT, isVolatile, isNonTemporal, 993 MinAlign(Alignment, IncrementSize)); 994 995 // Build a factor node to remember that this load is independent of 996 // the other one. 997 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 998 Hi.getValue(1)); 999 1000 // Move the top bits to the right place. 1001 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1002 DAG.getConstant(RoundWidth, 1003 TLI.getShiftAmountTy(Hi.getValueType()))); 1004 1005 // Join the hi and lo parts. 1006 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1007 } else { 1008 // Big endian - avoid unaligned loads. 1009 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 1010 // Load the top RoundWidth bits. 1011 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 1012 LD->getPointerInfo(), RoundVT, isVolatile, 1013 isNonTemporal, Alignment); 1014 1015 // Load the remaining ExtraWidth bits. 1016 IncrementSize = RoundWidth / 8; 1017 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1018 DAG.getIntPtrConstant(IncrementSize)); 1019 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, 1020 dl, Node->getValueType(0), Chain, Ptr, 1021 LD->getPointerInfo().getWithOffset(IncrementSize), 1022 ExtraVT, isVolatile, isNonTemporal, 1023 MinAlign(Alignment, IncrementSize)); 1024 1025 // Build a factor node to remember that this load is independent of 1026 // the other one. 1027 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1028 Hi.getValue(1)); 1029 1030 // Move the top bits to the right place. 1031 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1032 DAG.getConstant(ExtraWidth, 1033 TLI.getShiftAmountTy(Hi.getValueType()))); 1034 1035 // Join the hi and lo parts. 1036 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1037 } 1038 1039 Chain = Ch; 1040 } else { 1041 bool isCustom = false; 1042 switch (TLI.getLoadExtAction(ExtType, SrcVT.getSimpleVT())) { 1043 default: llvm_unreachable("This action is not supported yet!"); 1044 case TargetLowering::Custom: 1045 isCustom = true; 1046 // FALLTHROUGH 1047 case TargetLowering::Legal: { 1048 Value = SDValue(Node, 0); 1049 Chain = SDValue(Node, 1); 1050 1051 if (isCustom) { 1052 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 1053 if (Res.getNode()) { 1054 Value = Res; 1055 Chain = Res.getValue(1); 1056 } 1057 } else { 1058 // If this is an unaligned load and the target doesn't support it, 1059 // expand it. 1060 if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { 1061 Type *Ty = 1062 LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 1063 unsigned ABIAlignment = 1064 TLI.getDataLayout()->getABITypeAlignment(Ty); 1065 if (LD->getAlignment() < ABIAlignment){ 1066 ExpandUnalignedLoad(cast<LoadSDNode>(Node), 1067 DAG, TLI, Value, Chain); 1068 } 1069 } 1070 } 1071 break; 1072 } 1073 case TargetLowering::Expand: 1074 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, SrcVT) && TLI.isTypeLegal(SrcVT)) { 1075 SDValue Load = DAG.getLoad(SrcVT, dl, Chain, Ptr, 1076 LD->getPointerInfo(), 1077 LD->isVolatile(), LD->isNonTemporal(), 1078 LD->isInvariant(), LD->getAlignment()); 1079 unsigned ExtendOp; 1080 switch (ExtType) { 1081 case ISD::EXTLOAD: 1082 ExtendOp = (SrcVT.isFloatingPoint() ? 1083 ISD::FP_EXTEND : ISD::ANY_EXTEND); 1084 break; 1085 case ISD::SEXTLOAD: ExtendOp = ISD::SIGN_EXTEND; break; 1086 case ISD::ZEXTLOAD: ExtendOp = ISD::ZERO_EXTEND; break; 1087 default: llvm_unreachable("Unexpected extend load type!"); 1088 } 1089 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); 1090 Chain = Load.getValue(1); 1091 break; 1092 } 1093 1094 assert(!SrcVT.isVector() && 1095 "Vector Loads are handled in LegalizeVectorOps"); 1096 1097 // FIXME: This does not work for vectors on most targets. Sign- and 1098 // zero-extend operations are currently folded into extending loads, 1099 // whether they are legal or not, and then we end up here without any 1100 // support for legalizing them. 1101 assert(ExtType != ISD::EXTLOAD && 1102 "EXTLOAD should always be supported!"); 1103 // Turn the unsupported load into an EXTLOAD followed by an explicit 1104 // zero/sign extend inreg. 1105 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0), 1106 Chain, Ptr, LD->getPointerInfo(), SrcVT, 1107 LD->isVolatile(), LD->isNonTemporal(), 1108 LD->getAlignment()); 1109 SDValue ValRes; 1110 if (ExtType == ISD::SEXTLOAD) 1111 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 1112 Result.getValueType(), 1113 Result, DAG.getValueType(SrcVT)); 1114 else 1115 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); 1116 Value = ValRes; 1117 Chain = Result.getValue(1); 1118 break; 1119 } 1120 } 1121 1122 // Since loads produce two values, make sure to remember that we legalized 1123 // both of them. 1124 if (Chain.getNode() != Node) { 1125 assert(Value.getNode() != Node && "Load must be completely replaced"); 1126 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); 1127 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 1128 ReplacedNode(Node); 1129 } 1130} 1131 1132/// LegalizeOp - Return a legal replacement for the given operation, with 1133/// all legal operands. 1134void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { 1135 if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. 1136 return; 1137 1138 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1139 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == 1140 TargetLowering::TypeLegal && 1141 "Unexpected illegal type!"); 1142 1143 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1144 assert((TLI.getTypeAction(*DAG.getContext(), 1145 Node->getOperand(i).getValueType()) == 1146 TargetLowering::TypeLegal || 1147 Node->getOperand(i).getOpcode() == ISD::TargetConstant) && 1148 "Unexpected illegal type!"); 1149 1150 // Figure out the correct action; the way to query this varies by opcode 1151 TargetLowering::LegalizeAction Action = TargetLowering::Legal; 1152 bool SimpleFinishLegalizing = true; 1153 switch (Node->getOpcode()) { 1154 case ISD::INTRINSIC_W_CHAIN: 1155 case ISD::INTRINSIC_WO_CHAIN: 1156 case ISD::INTRINSIC_VOID: 1157 case ISD::STACKSAVE: 1158 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1159 break; 1160 case ISD::VAARG: 1161 Action = TLI.getOperationAction(Node->getOpcode(), 1162 Node->getValueType(0)); 1163 if (Action != TargetLowering::Promote) 1164 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1165 break; 1166 case ISD::SINT_TO_FP: 1167 case ISD::UINT_TO_FP: 1168 case ISD::EXTRACT_VECTOR_ELT: 1169 Action = TLI.getOperationAction(Node->getOpcode(), 1170 Node->getOperand(0).getValueType()); 1171 break; 1172 case ISD::FP_ROUND_INREG: 1173 case ISD::SIGN_EXTEND_INREG: { 1174 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 1175 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); 1176 break; 1177 } 1178 case ISD::ATOMIC_STORE: { 1179 Action = TLI.getOperationAction(Node->getOpcode(), 1180 Node->getOperand(2).getValueType()); 1181 break; 1182 } 1183 case ISD::SELECT_CC: 1184 case ISD::SETCC: 1185 case ISD::BR_CC: { 1186 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : 1187 Node->getOpcode() == ISD::SETCC ? 2 : 1; 1188 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; 1189 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); 1190 ISD::CondCode CCCode = 1191 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); 1192 Action = TLI.getCondCodeAction(CCCode, OpVT); 1193 if (Action == TargetLowering::Legal) { 1194 if (Node->getOpcode() == ISD::SELECT_CC) 1195 Action = TLI.getOperationAction(Node->getOpcode(), 1196 Node->getValueType(0)); 1197 else 1198 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); 1199 } 1200 break; 1201 } 1202 case ISD::LOAD: 1203 case ISD::STORE: 1204 // FIXME: Model these properly. LOAD and STORE are complicated, and 1205 // STORE expects the unlegalized operand in some cases. 1206 SimpleFinishLegalizing = false; 1207 break; 1208 case ISD::CALLSEQ_START: 1209 case ISD::CALLSEQ_END: 1210 // FIXME: This shouldn't be necessary. These nodes have special properties 1211 // dealing with the recursive nature of legalization. Removing this 1212 // special case should be done as part of making LegalizeDAG non-recursive. 1213 SimpleFinishLegalizing = false; 1214 break; 1215 case ISD::EXTRACT_ELEMENT: 1216 case ISD::FLT_ROUNDS_: 1217 case ISD::SADDO: 1218 case ISD::SSUBO: 1219 case ISD::UADDO: 1220 case ISD::USUBO: 1221 case ISD::SMULO: 1222 case ISD::UMULO: 1223 case ISD::FPOWI: 1224 case ISD::MERGE_VALUES: 1225 case ISD::EH_RETURN: 1226 case ISD::FRAME_TO_ARGS_OFFSET: 1227 case ISD::EH_SJLJ_SETJMP: 1228 case ISD::EH_SJLJ_LONGJMP: 1229 // These operations lie about being legal: when they claim to be legal, 1230 // they should actually be expanded. 1231 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1232 if (Action == TargetLowering::Legal) 1233 Action = TargetLowering::Expand; 1234 break; 1235 case ISD::INIT_TRAMPOLINE: 1236 case ISD::ADJUST_TRAMPOLINE: 1237 case ISD::FRAMEADDR: 1238 case ISD::RETURNADDR: 1239 // These operations lie about being legal: when they claim to be legal, 1240 // they should actually be custom-lowered. 1241 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1242 if (Action == TargetLowering::Legal) 1243 Action = TargetLowering::Custom; 1244 break; 1245 case ISD::DEBUGTRAP: 1246 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1247 if (Action == TargetLowering::Expand) { 1248 // replace ISD::DEBUGTRAP with ISD::TRAP 1249 SDValue NewVal; 1250 NewVal = DAG.getNode(ISD::TRAP, Node->getDebugLoc(), Node->getVTList(), 1251 Node->getOperand(0)); 1252 ReplaceNode(Node, NewVal.getNode()); 1253 LegalizeOp(NewVal.getNode()); 1254 return; 1255 } 1256 break; 1257 1258 default: 1259 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 1260 Action = TargetLowering::Legal; 1261 } else { 1262 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1263 } 1264 break; 1265 } 1266 1267 if (SimpleFinishLegalizing) { 1268 SDNode *NewNode = Node; 1269 switch (Node->getOpcode()) { 1270 default: break; 1271 case ISD::SHL: 1272 case ISD::SRL: 1273 case ISD::SRA: 1274 case ISD::ROTL: 1275 case ISD::ROTR: 1276 // Legalizing shifts/rotates requires adjusting the shift amount 1277 // to the appropriate width. 1278 if (!Node->getOperand(1).getValueType().isVector()) { 1279 SDValue SAO = 1280 DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), 1281 Node->getOperand(1)); 1282 HandleSDNode Handle(SAO); 1283 LegalizeOp(SAO.getNode()); 1284 NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), 1285 Handle.getValue()); 1286 } 1287 break; 1288 case ISD::SRL_PARTS: 1289 case ISD::SRA_PARTS: 1290 case ISD::SHL_PARTS: 1291 // Legalizing shifts/rotates requires adjusting the shift amount 1292 // to the appropriate width. 1293 if (!Node->getOperand(2).getValueType().isVector()) { 1294 SDValue SAO = 1295 DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), 1296 Node->getOperand(2)); 1297 HandleSDNode Handle(SAO); 1298 LegalizeOp(SAO.getNode()); 1299 NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), 1300 Node->getOperand(1), 1301 Handle.getValue()); 1302 } 1303 break; 1304 } 1305 1306 if (NewNode != Node) { 1307 DAG.ReplaceAllUsesWith(Node, NewNode); 1308 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1309 DAG.TransferDbgValues(SDValue(Node, i), SDValue(NewNode, i)); 1310 ReplacedNode(Node); 1311 Node = NewNode; 1312 } 1313 switch (Action) { 1314 case TargetLowering::Legal: 1315 return; 1316 case TargetLowering::Custom: { 1317 // FIXME: The handling for custom lowering with multiple results is 1318 // a complete mess. 1319 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 1320 if (Res.getNode()) { 1321 SmallVector<SDValue, 8> ResultVals; 1322 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { 1323 if (e == 1) 1324 ResultVals.push_back(Res); 1325 else 1326 ResultVals.push_back(Res.getValue(i)); 1327 } 1328 if (Res.getNode() != Node || Res.getResNo() != 0) { 1329 DAG.ReplaceAllUsesWith(Node, ResultVals.data()); 1330 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1331 DAG.TransferDbgValues(SDValue(Node, i), ResultVals[i]); 1332 ReplacedNode(Node); 1333 } 1334 return; 1335 } 1336 } 1337 // FALL THROUGH 1338 case TargetLowering::Expand: 1339 ExpandNode(Node); 1340 return; 1341 case TargetLowering::Promote: 1342 PromoteNode(Node); 1343 return; 1344 } 1345 } 1346 1347 switch (Node->getOpcode()) { 1348 default: 1349#ifndef NDEBUG 1350 dbgs() << "NODE: "; 1351 Node->dump( &DAG); 1352 dbgs() << "\n"; 1353#endif 1354 llvm_unreachable("Do not know how to legalize this operator!"); 1355 1356 case ISD::CALLSEQ_START: 1357 case ISD::CALLSEQ_END: 1358 break; 1359 case ISD::LOAD: { 1360 return LegalizeLoadOps(Node); 1361 } 1362 case ISD::STORE: { 1363 return LegalizeStoreOps(Node); 1364 } 1365 } 1366} 1367 1368SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { 1369 SDValue Vec = Op.getOperand(0); 1370 SDValue Idx = Op.getOperand(1); 1371 DebugLoc dl = Op.getDebugLoc(); 1372 // Store the value to a temporary stack slot, then LOAD the returned part. 1373 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1374 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1375 MachinePointerInfo(), false, false, 0); 1376 1377 // Add the offset to the index. 1378 unsigned EltSize = 1379 Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1380 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1381 DAG.getConstant(EltSize, Idx.getValueType())); 1382 1383 if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 1384 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx); 1385 else 1386 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 1387 1388 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); 1389 1390 if (Op.getValueType().isVector()) 1391 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,MachinePointerInfo(), 1392 false, false, false, 0); 1393 return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, 1394 MachinePointerInfo(), 1395 Vec.getValueType().getVectorElementType(), 1396 false, false, 0); 1397} 1398 1399SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { 1400 assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); 1401 1402 SDValue Vec = Op.getOperand(0); 1403 SDValue Part = Op.getOperand(1); 1404 SDValue Idx = Op.getOperand(2); 1405 DebugLoc dl = Op.getDebugLoc(); 1406 1407 // Store the value to a temporary stack slot, then LOAD the returned part. 1408 1409 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1410 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1411 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1412 1413 // First store the whole vector. 1414 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1415 false, false, 0); 1416 1417 // Then store the inserted part. 1418 1419 // Add the offset to the index. 1420 unsigned EltSize = 1421 Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1422 1423 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1424 DAG.getConstant(EltSize, Idx.getValueType())); 1425 1426 if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 1427 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx); 1428 else 1429 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 1430 1431 SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1432 StackPtr); 1433 1434 // Store the subvector. 1435 Ch = DAG.getStore(DAG.getEntryNode(), dl, Part, SubStackPtr, 1436 MachinePointerInfo(), false, false, 0); 1437 1438 // Finally, load the updated vector. 1439 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo, 1440 false, false, false, 0); 1441} 1442 1443SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { 1444 // We can't handle this case efficiently. Allocate a sufficiently 1445 // aligned object on the stack, store each element into it, then load 1446 // the result as a vector. 1447 // Create the stack frame object. 1448 EVT VT = Node->getValueType(0); 1449 EVT EltVT = VT.getVectorElementType(); 1450 DebugLoc dl = Node->getDebugLoc(); 1451 SDValue FIPtr = DAG.CreateStackTemporary(VT); 1452 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1453 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1454 1455 // Emit a store of each element to the stack slot. 1456 SmallVector<SDValue, 8> Stores; 1457 unsigned TypeByteSize = EltVT.getSizeInBits() / 8; 1458 // Store (in the right endianness) the elements to memory. 1459 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1460 // Ignore undef elements. 1461 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1462 1463 unsigned Offset = TypeByteSize*i; 1464 1465 SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); 1466 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); 1467 1468 // If the destination vector element type is narrower than the source 1469 // element type, only store the bits necessary. 1470 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { 1471 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, 1472 Node->getOperand(i), Idx, 1473 PtrInfo.getWithOffset(Offset), 1474 EltVT, false, false, 0)); 1475 } else 1476 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, 1477 Node->getOperand(i), Idx, 1478 PtrInfo.getWithOffset(Offset), 1479 false, false, 0)); 1480 } 1481 1482 SDValue StoreChain; 1483 if (!Stores.empty()) // Not all undef elements? 1484 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1485 &Stores[0], Stores.size()); 1486 else 1487 StoreChain = DAG.getEntryNode(); 1488 1489 // Result is a load from the stack slot. 1490 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo, 1491 false, false, false, 0); 1492} 1493 1494SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) { 1495 DebugLoc dl = Node->getDebugLoc(); 1496 SDValue Tmp1 = Node->getOperand(0); 1497 SDValue Tmp2 = Node->getOperand(1); 1498 1499 // Get the sign bit of the RHS. First obtain a value that has the same 1500 // sign as the sign bit, i.e. negative if and only if the sign bit is 1. 1501 SDValue SignBit; 1502 EVT FloatVT = Tmp2.getValueType(); 1503 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), FloatVT.getSizeInBits()); 1504 if (TLI.isTypeLegal(IVT)) { 1505 // Convert to an integer with the same sign bit. 1506 SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2); 1507 } else { 1508 // Store the float to memory, then load the sign part out as an integer. 1509 MVT LoadTy = TLI.getPointerTy(); 1510 // First create a temporary that is aligned for both the load and store. 1511 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); 1512 // Then store the float to it. 1513 SDValue Ch = 1514 DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(), 1515 false, false, 0); 1516 if (TLI.isBigEndian()) { 1517 assert(FloatVT.isByteSized() && "Unsupported floating point type!"); 1518 // Load out a legal integer with the same sign bit as the float. 1519 SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(), 1520 false, false, false, 0); 1521 } else { // Little endian 1522 SDValue LoadPtr = StackPtr; 1523 // The float may be wider than the integer we are going to load. Advance 1524 // the pointer so that the loaded integer will contain the sign bit. 1525 unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits(); 1526 unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8; 1527 LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), 1528 LoadPtr, DAG.getIntPtrConstant(ByteOffset)); 1529 // Load a legal integer containing the sign bit. 1530 SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(), 1531 false, false, false, 0); 1532 // Move the sign bit to the top bit of the loaded integer. 1533 unsigned BitShift = LoadTy.getSizeInBits() - 1534 (FloatVT.getSizeInBits() - 8 * ByteOffset); 1535 assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?"); 1536 if (BitShift) 1537 SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit, 1538 DAG.getConstant(BitShift, 1539 TLI.getShiftAmountTy(SignBit.getValueType()))); 1540 } 1541 } 1542 // Now get the sign bit proper, by seeing whether the value is negative. 1543 SignBit = DAG.getSetCC(dl, TLI.getSetCCResultType(SignBit.getValueType()), 1544 SignBit, DAG.getConstant(0, SignBit.getValueType()), 1545 ISD::SETLT); 1546 // Get the absolute value of the result. 1547 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); 1548 // Select between the nabs and abs value based on the sign bit of 1549 // the input. 1550 return DAG.getNode(ISD::SELECT, dl, AbsVal.getValueType(), SignBit, 1551 DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal), 1552 AbsVal); 1553} 1554 1555void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, 1556 SmallVectorImpl<SDValue> &Results) { 1557 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1558 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1559 " not tell us which reg is the stack pointer!"); 1560 DebugLoc dl = Node->getDebugLoc(); 1561 EVT VT = Node->getValueType(0); 1562 SDValue Tmp1 = SDValue(Node, 0); 1563 SDValue Tmp2 = SDValue(Node, 1); 1564 SDValue Tmp3 = Node->getOperand(2); 1565 SDValue Chain = Tmp1.getOperand(0); 1566 1567 // Chain the dynamic stack allocation so that it doesn't modify the stack 1568 // pointer when other instructions are using the stack. 1569 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); 1570 1571 SDValue Size = Tmp2.getOperand(1); 1572 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1573 Chain = SP.getValue(1); 1574 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1575 unsigned StackAlign = TM.getFrameLowering()->getStackAlignment(); 1576 if (Align > StackAlign) 1577 SP = DAG.getNode(ISD::AND, dl, VT, SP, 1578 DAG.getConstant(-(uint64_t)Align, VT)); 1579 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value 1580 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1581 1582 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), 1583 DAG.getIntPtrConstant(0, true), SDValue()); 1584 1585 Results.push_back(Tmp1); 1586 Results.push_back(Tmp2); 1587} 1588 1589/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and 1590/// condition code CC on the current target. This routine expands SETCC with 1591/// illegal condition code into AND / OR of multiple SETCC values. 1592void SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, 1593 SDValue &LHS, SDValue &RHS, 1594 SDValue &CC, 1595 DebugLoc dl) { 1596 MVT OpVT = LHS.getSimpleValueType(); 1597 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 1598 switch (TLI.getCondCodeAction(CCCode, OpVT)) { 1599 default: llvm_unreachable("Unknown condition code action!"); 1600 case TargetLowering::Legal: 1601 // Nothing to do. 1602 break; 1603 case TargetLowering::Expand: { 1604 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 1605 ISD::CondCode InvCC = ISD::SETCC_INVALID; 1606 unsigned Opc = 0; 1607 switch (CCCode) { 1608 default: llvm_unreachable("Don't know how to expand this condition!"); 1609 case ISD::SETO: 1610 assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT) 1611 == TargetLowering::Legal 1612 && "If SETO is expanded, SETOEQ must be legal!"); 1613 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; 1614 case ISD::SETUO: 1615 assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT) 1616 == TargetLowering::Legal 1617 && "If SETUO is expanded, SETUNE must be legal!"); 1618 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; 1619 case ISD::SETOEQ: 1620 case ISD::SETOGT: 1621 case ISD::SETOGE: 1622 case ISD::SETOLT: 1623 case ISD::SETOLE: 1624 case ISD::SETONE: 1625 case ISD::SETUEQ: 1626 case ISD::SETUNE: 1627 case ISD::SETUGT: 1628 case ISD::SETUGE: 1629 case ISD::SETULT: 1630 case ISD::SETULE: 1631 // If we are floating point, assign and break, otherwise fall through. 1632 if (!OpVT.isInteger()) { 1633 // We can use the 4th bit to tell if we are the unordered 1634 // or ordered version of the opcode. 1635 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; 1636 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; 1637 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); 1638 break; 1639 } 1640 // Fallthrough if we are unsigned integer. 1641 case ISD::SETLE: 1642 case ISD::SETGT: 1643 case ISD::SETGE: 1644 case ISD::SETLT: 1645 case ISD::SETNE: 1646 case ISD::SETEQ: 1647 InvCC = ISD::getSetCCSwappedOperands(CCCode); 1648 if (TLI.getCondCodeAction(InvCC, OpVT) == TargetLowering::Expand) { 1649 // We only support using the inverted operation and not a 1650 // different manner of supporting expanding these cases. 1651 llvm_unreachable("Don't know how to expand this condition!"); 1652 } 1653 LHS = DAG.getSetCC(dl, VT, RHS, LHS, InvCC); 1654 RHS = SDValue(); 1655 CC = SDValue(); 1656 return; 1657 } 1658 1659 SDValue SetCC1, SetCC2; 1660 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { 1661 // If we aren't the ordered or unorder operation, 1662 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). 1663 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); 1664 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); 1665 } else { 1666 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) 1667 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); 1668 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); 1669 } 1670 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 1671 RHS = SDValue(); 1672 CC = SDValue(); 1673 break; 1674 } 1675 } 1676} 1677 1678/// EmitStackConvert - Emit a store/load combination to the stack. This stores 1679/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 1680/// a load from the stack slot to DestVT, extending it if needed. 1681/// The resultant code need not be legal. 1682SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, 1683 EVT SlotVT, 1684 EVT DestVT, 1685 DebugLoc dl) { 1686 // Create the stack frame object. 1687 unsigned SrcAlign = 1688 TLI.getDataLayout()->getPrefTypeAlignment(SrcOp.getValueType(). 1689 getTypeForEVT(*DAG.getContext())); 1690 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 1691 1692 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 1693 int SPFI = StackPtrFI->getIndex(); 1694 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI); 1695 1696 unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); 1697 unsigned SlotSize = SlotVT.getSizeInBits(); 1698 unsigned DestSize = DestVT.getSizeInBits(); 1699 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); 1700 unsigned DestAlign = TLI.getDataLayout()->getPrefTypeAlignment(DestType); 1701 1702 // Emit a store to the stack slot. Use a truncstore if the input value is 1703 // later than DestVT. 1704 SDValue Store; 1705 1706 if (SrcSize > SlotSize) 1707 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1708 PtrInfo, SlotVT, false, false, SrcAlign); 1709 else { 1710 assert(SrcSize == SlotSize && "Invalid store"); 1711 Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1712 PtrInfo, false, false, SrcAlign); 1713 } 1714 1715 // Result is a load from the stack slot. 1716 if (SlotSize == DestSize) 1717 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, 1718 false, false, false, DestAlign); 1719 1720 assert(SlotSize < DestSize && "Unknown extension!"); 1721 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, 1722 PtrInfo, SlotVT, false, false, DestAlign); 1723} 1724 1725SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 1726 DebugLoc dl = Node->getDebugLoc(); 1727 // Create a vector sized/aligned stack slot, store the value to element #0, 1728 // then load the whole vector back out. 1729 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 1730 1731 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 1732 int SPFI = StackPtrFI->getIndex(); 1733 1734 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0), 1735 StackPtr, 1736 MachinePointerInfo::getFixedStack(SPFI), 1737 Node->getValueType(0).getVectorElementType(), 1738 false, false, 0); 1739 return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr, 1740 MachinePointerInfo::getFixedStack(SPFI), 1741 false, false, false, 0); 1742} 1743 1744 1745/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't 1746/// support the operation, but do support the resultant vector type. 1747SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 1748 unsigned NumElems = Node->getNumOperands(); 1749 SDValue Value1, Value2; 1750 DebugLoc dl = Node->getDebugLoc(); 1751 EVT VT = Node->getValueType(0); 1752 EVT OpVT = Node->getOperand(0).getValueType(); 1753 EVT EltVT = VT.getVectorElementType(); 1754 1755 // If the only non-undef value is the low element, turn this into a 1756 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 1757 bool isOnlyLowElement = true; 1758 bool MoreThanTwoValues = false; 1759 bool isConstant = true; 1760 for (unsigned i = 0; i < NumElems; ++i) { 1761 SDValue V = Node->getOperand(i); 1762 if (V.getOpcode() == ISD::UNDEF) 1763 continue; 1764 if (i > 0) 1765 isOnlyLowElement = false; 1766 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 1767 isConstant = false; 1768 1769 if (!Value1.getNode()) { 1770 Value1 = V; 1771 } else if (!Value2.getNode()) { 1772 if (V != Value1) 1773 Value2 = V; 1774 } else if (V != Value1 && V != Value2) { 1775 MoreThanTwoValues = true; 1776 } 1777 } 1778 1779 if (!Value1.getNode()) 1780 return DAG.getUNDEF(VT); 1781 1782 if (isOnlyLowElement) 1783 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); 1784 1785 // If all elements are constants, create a load from the constant pool. 1786 if (isConstant) { 1787 SmallVector<Constant*, 16> CV; 1788 for (unsigned i = 0, e = NumElems; i != e; ++i) { 1789 if (ConstantFPSDNode *V = 1790 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 1791 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 1792 } else if (ConstantSDNode *V = 1793 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 1794 if (OpVT==EltVT) 1795 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 1796 else { 1797 // If OpVT and EltVT don't match, EltVT is not legal and the 1798 // element values have been promoted/truncated earlier. Undo this; 1799 // we don't want a v16i8 to become a v16i32 for example. 1800 const ConstantInt *CI = V->getConstantIntValue(); 1801 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), 1802 CI->getZExtValue())); 1803 } 1804 } else { 1805 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); 1806 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); 1807 CV.push_back(UndefValue::get(OpNTy)); 1808 } 1809 } 1810 Constant *CP = ConstantVector::get(CV); 1811 SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); 1812 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 1813 return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 1814 MachinePointerInfo::getConstantPool(), 1815 false, false, false, Alignment); 1816 } 1817 1818 if (!MoreThanTwoValues) { 1819 SmallVector<int, 8> ShuffleVec(NumElems, -1); 1820 for (unsigned i = 0; i < NumElems; ++i) { 1821 SDValue V = Node->getOperand(i); 1822 if (V.getOpcode() == ISD::UNDEF) 1823 continue; 1824 ShuffleVec[i] = V == Value1 ? 0 : NumElems; 1825 } 1826 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { 1827 // Get the splatted value into the low element of a vector register. 1828 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); 1829 SDValue Vec2; 1830 if (Value2.getNode()) 1831 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); 1832 else 1833 Vec2 = DAG.getUNDEF(VT); 1834 1835 // Return shuffle(LowValVec, undef, <0,0,0,0>) 1836 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data()); 1837 } 1838 } 1839 1840 // Otherwise, we can't handle this case efficiently. 1841 return ExpandVectorBuildThroughStack(Node); 1842} 1843 1844static bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, 1845 SDValue &Chain, const TargetLowering &TLI) { 1846 const Function *F = DAG.getMachineFunction().getFunction(); 1847 1848 // Conservatively require the attributes of the call to match those of 1849 // the return. Ignore noalias because it doesn't affect the call sequence. 1850 Attribute CallerRetAttr = F->getAttributes().getRetAttributes(); 1851 if (AttrBuilder(CallerRetAttr) 1852 .removeAttribute(Attribute::NoAlias).hasAttributes()) 1853 return false; 1854 1855 // It's not safe to eliminate the sign / zero extension of the return value. 1856 if (CallerRetAttr.hasAttribute(Attribute::ZExt) || 1857 CallerRetAttr.hasAttribute(Attribute::SExt)) 1858 return false; 1859 1860 // Check if the only use is a function return node. 1861 return TLI.isUsedByReturnOnly(Node, Chain); 1862} 1863 1864// ExpandLibCall - Expand a node into a call to a libcall. If the result value 1865// does not fit into a register, return the lo part and set the hi part to the 1866// by-reg argument. If it does fit into a single register, return the result 1867// and leave the Hi part unset. 1868SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 1869 bool isSigned) { 1870 TargetLowering::ArgListTy Args; 1871 TargetLowering::ArgListEntry Entry; 1872 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1873 EVT ArgVT = Node->getOperand(i).getValueType(); 1874 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1875 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 1876 Entry.isSExt = isSigned; 1877 Entry.isZExt = !isSigned; 1878 Args.push_back(Entry); 1879 } 1880 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1881 TLI.getPointerTy()); 1882 1883 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 1884 1885 // By default, the input chain to this libcall is the entry node of the 1886 // function. If the libcall is going to be emitted as a tail call then 1887 // TLI.isUsedByReturnOnly will change it to the right chain if the return 1888 // node which is being folded has a non-entry input chain. 1889 SDValue InChain = DAG.getEntryNode(); 1890 1891 // isTailCall may be true since the callee does not reference caller stack 1892 // frame. Check if it's in the right position. 1893 SDValue TCChain = InChain; 1894 bool isTailCall = isInTailCallPosition(DAG, Node, TCChain, TLI); 1895 if (isTailCall) 1896 InChain = TCChain; 1897 1898 TargetLowering:: 1899 CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 1900 0, TLI.getLibcallCallingConv(LC), isTailCall, 1901 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1902 Callee, Args, DAG, Node->getDebugLoc()); 1903 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 1904 1905 1906 if (!CallInfo.second.getNode()) 1907 // It's a tailcall, return the chain (which is the DAG root). 1908 return DAG.getRoot(); 1909 1910 return CallInfo.first; 1911} 1912 1913/// ExpandLibCall - Generate a libcall taking the given operands as arguments 1914/// and returning a result of type RetVT. 1915SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, 1916 const SDValue *Ops, unsigned NumOps, 1917 bool isSigned, DebugLoc dl) { 1918 TargetLowering::ArgListTy Args; 1919 Args.reserve(NumOps); 1920 1921 TargetLowering::ArgListEntry Entry; 1922 for (unsigned i = 0; i != NumOps; ++i) { 1923 Entry.Node = Ops[i]; 1924 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); 1925 Entry.isSExt = isSigned; 1926 Entry.isZExt = !isSigned; 1927 Args.push_back(Entry); 1928 } 1929 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1930 TLI.getPointerTy()); 1931 1932 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 1933 TargetLowering:: 1934 CallLoweringInfo CLI(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, 1935 false, 0, TLI.getLibcallCallingConv(LC), 1936 /*isTailCall=*/false, 1937 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1938 Callee, Args, DAG, dl); 1939 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI); 1940 1941 return CallInfo.first; 1942} 1943 1944// ExpandChainLibCall - Expand a node into a call to a libcall. Similar to 1945// ExpandLibCall except that the first operand is the in-chain. 1946std::pair<SDValue, SDValue> 1947SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, 1948 SDNode *Node, 1949 bool isSigned) { 1950 SDValue InChain = Node->getOperand(0); 1951 1952 TargetLowering::ArgListTy Args; 1953 TargetLowering::ArgListEntry Entry; 1954 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { 1955 EVT ArgVT = Node->getOperand(i).getValueType(); 1956 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1957 Entry.Node = Node->getOperand(i); 1958 Entry.Ty = ArgTy; 1959 Entry.isSExt = isSigned; 1960 Entry.isZExt = !isSigned; 1961 Args.push_back(Entry); 1962 } 1963 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1964 TLI.getPointerTy()); 1965 1966 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 1967 TargetLowering:: 1968 CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 1969 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 1970 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1971 Callee, Args, DAG, Node->getDebugLoc()); 1972 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 1973 1974 return CallInfo; 1975} 1976 1977SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, 1978 RTLIB::Libcall Call_F32, 1979 RTLIB::Libcall Call_F64, 1980 RTLIB::Libcall Call_F80, 1981 RTLIB::Libcall Call_PPCF128) { 1982 RTLIB::Libcall LC; 1983 switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 1984 default: llvm_unreachable("Unexpected request for libcall!"); 1985 case MVT::f32: LC = Call_F32; break; 1986 case MVT::f64: LC = Call_F64; break; 1987 case MVT::f80: LC = Call_F80; break; 1988 case MVT::ppcf128: LC = Call_PPCF128; break; 1989 } 1990 return ExpandLibCall(LC, Node, false); 1991} 1992 1993SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, 1994 RTLIB::Libcall Call_I8, 1995 RTLIB::Libcall Call_I16, 1996 RTLIB::Libcall Call_I32, 1997 RTLIB::Libcall Call_I64, 1998 RTLIB::Libcall Call_I128) { 1999 RTLIB::Libcall LC; 2000 switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 2001 default: llvm_unreachable("Unexpected request for libcall!"); 2002 case MVT::i8: LC = Call_I8; break; 2003 case MVT::i16: LC = Call_I16; break; 2004 case MVT::i32: LC = Call_I32; break; 2005 case MVT::i64: LC = Call_I64; break; 2006 case MVT::i128: LC = Call_I128; break; 2007 } 2008 return ExpandLibCall(LC, Node, isSigned); 2009} 2010 2011/// isDivRemLibcallAvailable - Return true if divmod libcall is available. 2012static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, 2013 const TargetLowering &TLI) { 2014 RTLIB::Libcall LC; 2015 switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 2016 default: llvm_unreachable("Unexpected request for libcall!"); 2017 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2018 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2019 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2020 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2021 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2022 } 2023 2024 return TLI.getLibcallName(LC) != 0; 2025} 2026 2027/// useDivRem - Only issue divrem libcall if both quotient and remainder are 2028/// needed. 2029static bool useDivRem(SDNode *Node, bool isSigned, bool isDIV) { 2030 // The other use might have been replaced with a divrem already. 2031 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 2032 unsigned OtherOpcode = 0; 2033 if (isSigned) 2034 OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV; 2035 else 2036 OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV; 2037 2038 SDValue Op0 = Node->getOperand(0); 2039 SDValue Op1 = Node->getOperand(1); 2040 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2041 UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2042 SDNode *User = *UI; 2043 if (User == Node) 2044 continue; 2045 if ((User->getOpcode() == OtherOpcode || User->getOpcode() == DivRemOpc) && 2046 User->getOperand(0) == Op0 && 2047 User->getOperand(1) == Op1) 2048 return true; 2049 } 2050 return false; 2051} 2052 2053/// ExpandDivRemLibCall - Issue libcalls to __{u}divmod to compute div / rem 2054/// pairs. 2055void 2056SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, 2057 SmallVectorImpl<SDValue> &Results) { 2058 unsigned Opcode = Node->getOpcode(); 2059 bool isSigned = Opcode == ISD::SDIVREM; 2060 2061 RTLIB::Libcall LC; 2062 switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 2063 default: llvm_unreachable("Unexpected request for libcall!"); 2064 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2065 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2066 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2067 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2068 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2069 } 2070 2071 // The input chain to this libcall is the entry node of the function. 2072 // Legalizing the call will automatically add the previous call to the 2073 // dependence. 2074 SDValue InChain = DAG.getEntryNode(); 2075 2076 EVT RetVT = Node->getValueType(0); 2077 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2078 2079 TargetLowering::ArgListTy Args; 2080 TargetLowering::ArgListEntry Entry; 2081 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 2082 EVT ArgVT = Node->getOperand(i).getValueType(); 2083 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2084 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 2085 Entry.isSExt = isSigned; 2086 Entry.isZExt = !isSigned; 2087 Args.push_back(Entry); 2088 } 2089 2090 // Also pass the return address of the remainder. 2091 SDValue FIPtr = DAG.CreateStackTemporary(RetVT); 2092 Entry.Node = FIPtr; 2093 Entry.Ty = RetTy->getPointerTo(); 2094 Entry.isSExt = isSigned; 2095 Entry.isZExt = !isSigned; 2096 Args.push_back(Entry); 2097 2098 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2099 TLI.getPointerTy()); 2100 2101 DebugLoc dl = Node->getDebugLoc(); 2102 TargetLowering:: 2103 CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 2104 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 2105 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2106 Callee, Args, DAG, dl); 2107 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2108 2109 // Remainder is loaded back from the stack frame. 2110 SDValue Rem = DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, 2111 MachinePointerInfo(), false, false, false, 0); 2112 Results.push_back(CallInfo.first); 2113 Results.push_back(Rem); 2114} 2115 2116/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a 2117/// INT_TO_FP operation of the specified operand when the target requests that 2118/// we expand it. At this point, we know that the result and operand types are 2119/// legal for the target. 2120SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, 2121 SDValue Op0, 2122 EVT DestVT, 2123 DebugLoc dl) { 2124 if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) { 2125 // simple 32-bit [signed|unsigned] integer to float/double expansion 2126 2127 // Get the stack frame index of a 8 byte buffer. 2128 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 2129 2130 // word offset constant for Hi/Lo address computation 2131 SDValue WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy()); 2132 // set up Hi and Lo (into buffer) address based on endian 2133 SDValue Hi = StackSlot; 2134 SDValue Lo = DAG.getNode(ISD::ADD, dl, 2135 TLI.getPointerTy(), StackSlot, WordOff); 2136 if (TLI.isLittleEndian()) 2137 std::swap(Hi, Lo); 2138 2139 // if signed map to unsigned space 2140 SDValue Op0Mapped; 2141 if (isSigned) { 2142 // constant used to invert sign bit (signed to unsigned mapping) 2143 SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); 2144 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); 2145 } else { 2146 Op0Mapped = Op0; 2147 } 2148 // store the lo of the constructed double - based on integer input 2149 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, 2150 Op0Mapped, Lo, MachinePointerInfo(), 2151 false, false, 0); 2152 // initial hi portion of constructed double 2153 SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); 2154 // store the hi of the constructed double - biased exponent 2155 SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi, 2156 MachinePointerInfo(), 2157 false, false, 0); 2158 // load the constructed double 2159 SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot, 2160 MachinePointerInfo(), false, false, false, 0); 2161 // FP constant to bias correct the final result 2162 SDValue Bias = DAG.getConstantFP(isSigned ? 2163 BitsToDouble(0x4330000080000000ULL) : 2164 BitsToDouble(0x4330000000000000ULL), 2165 MVT::f64); 2166 // subtract the bias 2167 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 2168 // final result 2169 SDValue Result; 2170 // handle final rounding 2171 if (DestVT == MVT::f64) { 2172 // do nothing 2173 Result = Sub; 2174 } else if (DestVT.bitsLT(MVT::f64)) { 2175 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 2176 DAG.getIntPtrConstant(0)); 2177 } else if (DestVT.bitsGT(MVT::f64)) { 2178 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 2179 } 2180 return Result; 2181 } 2182 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 2183 // Code below here assumes !isSigned without checking again. 2184 2185 // Implementation of unsigned i64 to f64 following the algorithm in 2186 // __floatundidf in compiler_rt. This implementation has the advantage 2187 // of performing rounding correctly, both in the default rounding mode 2188 // and in all alternate rounding modes. 2189 // TODO: Generalize this for use with other types. 2190 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) { 2191 SDValue TwoP52 = 2192 DAG.getConstant(UINT64_C(0x4330000000000000), MVT::i64); 2193 SDValue TwoP84PlusTwoP52 = 2194 DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), MVT::f64); 2195 SDValue TwoP84 = 2196 DAG.getConstant(UINT64_C(0x4530000000000000), MVT::i64); 2197 2198 SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32); 2199 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, 2200 DAG.getConstant(32, MVT::i64)); 2201 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52); 2202 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84); 2203 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr); 2204 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr); 2205 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt, 2206 TwoP84PlusTwoP52); 2207 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); 2208 } 2209 2210 // Implementation of unsigned i64 to f32. 2211 // TODO: Generalize this for use with other types. 2212 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) { 2213 // For unsigned conversions, convert them to signed conversions using the 2214 // algorithm from the x86_64 __floatundidf in compiler_rt. 2215 if (!isSigned) { 2216 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0); 2217 2218 SDValue ShiftConst = 2219 DAG.getConstant(1, TLI.getShiftAmountTy(Op0.getValueType())); 2220 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst); 2221 SDValue AndConst = DAG.getConstant(1, MVT::i64); 2222 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst); 2223 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr); 2224 2225 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or); 2226 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt); 2227 2228 // TODO: This really should be implemented using a branch rather than a 2229 // select. We happen to get lucky and machinesink does the right 2230 // thing most of the time. This would be a good candidate for a 2231 //pseudo-op, or, even better, for whole-function isel. 2232 SDValue SignBitTest = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2233 Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT); 2234 return DAG.getNode(ISD::SELECT, dl, MVT::f32, SignBitTest, Slow, Fast); 2235 } 2236 2237 // Otherwise, implement the fully general conversion. 2238 2239 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2240 DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64)); 2241 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, 2242 DAG.getConstant(UINT64_C(0x800), MVT::i64)); 2243 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2244 DAG.getConstant(UINT64_C(0x7ff), MVT::i64)); 2245 SDValue Ne = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2246 And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE); 2247 SDValue Sel = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ne, Or, Op0); 2248 SDValue Ge = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2249 Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64), 2250 ISD::SETUGE); 2251 SDValue Sel2 = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ge, Sel, Op0); 2252 EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType()); 2253 2254 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, 2255 DAG.getConstant(32, SHVT)); 2256 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); 2257 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); 2258 SDValue TwoP32 = 2259 DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64); 2260 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); 2261 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); 2262 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); 2263 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); 2264 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, 2265 DAG.getIntPtrConstant(0)); 2266 } 2267 2268 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 2269 2270 SDValue SignSet = DAG.getSetCC(dl, TLI.getSetCCResultType(Op0.getValueType()), 2271 Op0, DAG.getConstant(0, Op0.getValueType()), 2272 ISD::SETLT); 2273 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 2274 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), 2275 SignSet, Four, Zero); 2276 2277 // If the sign bit of the integer is set, the large number will be treated 2278 // as a negative number. To counteract this, the dynamic code adds an 2279 // offset depending on the data type. 2280 uint64_t FF; 2281 switch (Op0.getValueType().getSimpleVT().SimpleTy) { 2282 default: llvm_unreachable("Unsupported integer type!"); 2283 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 2284 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 2285 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 2286 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 2287 } 2288 if (TLI.isLittleEndian()) FF <<= 32; 2289 Constant *FudgeFactor = ConstantInt::get( 2290 Type::getInt64Ty(*DAG.getContext()), FF); 2291 2292 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 2293 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 2294 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset); 2295 Alignment = std::min(Alignment, 4u); 2296 SDValue FudgeInReg; 2297 if (DestVT == MVT::f32) 2298 FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 2299 MachinePointerInfo::getConstantPool(), 2300 false, false, false, Alignment); 2301 else { 2302 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, 2303 DAG.getEntryNode(), CPIdx, 2304 MachinePointerInfo::getConstantPool(), 2305 MVT::f32, false, false, Alignment); 2306 HandleSDNode Handle(Load); 2307 LegalizeOp(Load.getNode()); 2308 FudgeInReg = Handle.getValue(); 2309 } 2310 2311 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 2312} 2313 2314/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a 2315/// *INT_TO_FP operation of the specified operand when the target requests that 2316/// we promote it. At this point, we know that the result and operand types are 2317/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 2318/// operation that takes a larger input. 2319SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, 2320 EVT DestVT, 2321 bool isSigned, 2322 DebugLoc dl) { 2323 // First step, figure out the appropriate *INT_TO_FP operation to use. 2324 EVT NewInTy = LegalOp.getValueType(); 2325 2326 unsigned OpToUse = 0; 2327 2328 // Scan for the appropriate larger type to use. 2329 while (1) { 2330 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); 2331 assert(NewInTy.isInteger() && "Ran out of possibilities!"); 2332 2333 // If the target supports SINT_TO_FP of this type, use it. 2334 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { 2335 OpToUse = ISD::SINT_TO_FP; 2336 break; 2337 } 2338 if (isSigned) continue; 2339 2340 // If the target supports UINT_TO_FP of this type, use it. 2341 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { 2342 OpToUse = ISD::UINT_TO_FP; 2343 break; 2344 } 2345 2346 // Otherwise, try a larger type. 2347 } 2348 2349 // Okay, we found the operation and type to use. Zero extend our input to the 2350 // desired type then run the operation on it. 2351 return DAG.getNode(OpToUse, dl, DestVT, 2352 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 2353 dl, NewInTy, LegalOp)); 2354} 2355 2356/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a 2357/// FP_TO_*INT operation of the specified operand when the target requests that 2358/// we promote it. At this point, we know that the result and operand types are 2359/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 2360/// operation that returns a larger result. 2361SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, 2362 EVT DestVT, 2363 bool isSigned, 2364 DebugLoc dl) { 2365 // First step, figure out the appropriate FP_TO*INT operation to use. 2366 EVT NewOutTy = DestVT; 2367 2368 unsigned OpToUse = 0; 2369 2370 // Scan for the appropriate larger type to use. 2371 while (1) { 2372 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); 2373 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 2374 2375 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { 2376 OpToUse = ISD::FP_TO_SINT; 2377 break; 2378 } 2379 2380 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { 2381 OpToUse = ISD::FP_TO_UINT; 2382 break; 2383 } 2384 2385 // Otherwise, try a larger type. 2386 } 2387 2388 2389 // Okay, we found the operation and type to use. 2390 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 2391 2392 // Truncate the result of the extended FP_TO_*INT operation to the desired 2393 // size. 2394 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 2395} 2396 2397/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. 2398/// 2399SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) { 2400 EVT VT = Op.getValueType(); 2401 EVT SHVT = TLI.getShiftAmountTy(VT); 2402 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 2403 switch (VT.getSimpleVT().SimpleTy) { 2404 default: llvm_unreachable("Unhandled Expand type in BSWAP!"); 2405 case MVT::i16: 2406 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2407 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2408 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 2409 case MVT::i32: 2410 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2411 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2412 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2413 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2414 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); 2415 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); 2416 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2417 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2418 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2419 case MVT::i64: 2420 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2421 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2422 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2423 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2424 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2425 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2426 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2427 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2428 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); 2429 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); 2430 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); 2431 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); 2432 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); 2433 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); 2434 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 2435 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 2436 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2437 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2438 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 2439 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2440 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 2441 } 2442} 2443 2444/// SplatByte - Distribute ByteVal over NumBits bits. 2445// FIXME: Move this helper to a common place. 2446static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) { 2447 APInt Val = APInt(NumBits, ByteVal); 2448 unsigned Shift = 8; 2449 for (unsigned i = NumBits; i > 8; i >>= 1) { 2450 Val = (Val << Shift) | Val; 2451 Shift <<= 1; 2452 } 2453 return Val; 2454} 2455 2456/// ExpandBitCount - Expand the specified bitcount instruction into operations. 2457/// 2458SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, 2459 DebugLoc dl) { 2460 switch (Opc) { 2461 default: llvm_unreachable("Cannot expand this yet!"); 2462 case ISD::CTPOP: { 2463 EVT VT = Op.getValueType(); 2464 EVT ShVT = TLI.getShiftAmountTy(VT); 2465 unsigned Len = VT.getSizeInBits(); 2466 2467 assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 && 2468 "CTPOP not implemented for this type."); 2469 2470 // This is the "best" algorithm from 2471 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel 2472 2473 SDValue Mask55 = DAG.getConstant(SplatByte(Len, 0x55), VT); 2474 SDValue Mask33 = DAG.getConstant(SplatByte(Len, 0x33), VT); 2475 SDValue Mask0F = DAG.getConstant(SplatByte(Len, 0x0F), VT); 2476 SDValue Mask01 = DAG.getConstant(SplatByte(Len, 0x01), VT); 2477 2478 // v = v - ((v >> 1) & 0x55555555...) 2479 Op = DAG.getNode(ISD::SUB, dl, VT, Op, 2480 DAG.getNode(ISD::AND, dl, VT, 2481 DAG.getNode(ISD::SRL, dl, VT, Op, 2482 DAG.getConstant(1, ShVT)), 2483 Mask55)); 2484 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...) 2485 Op = DAG.getNode(ISD::ADD, dl, VT, 2486 DAG.getNode(ISD::AND, dl, VT, Op, Mask33), 2487 DAG.getNode(ISD::AND, dl, VT, 2488 DAG.getNode(ISD::SRL, dl, VT, Op, 2489 DAG.getConstant(2, ShVT)), 2490 Mask33)); 2491 // v = (v + (v >> 4)) & 0x0F0F0F0F... 2492 Op = DAG.getNode(ISD::AND, dl, VT, 2493 DAG.getNode(ISD::ADD, dl, VT, Op, 2494 DAG.getNode(ISD::SRL, dl, VT, Op, 2495 DAG.getConstant(4, ShVT))), 2496 Mask0F); 2497 // v = (v * 0x01010101...) >> (Len - 8) 2498 Op = DAG.getNode(ISD::SRL, dl, VT, 2499 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), 2500 DAG.getConstant(Len - 8, ShVT)); 2501 2502 return Op; 2503 } 2504 case ISD::CTLZ_ZERO_UNDEF: 2505 // This trivially expands to CTLZ. 2506 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); 2507 case ISD::CTLZ: { 2508 // for now, we do this: 2509 // x = x | (x >> 1); 2510 // x = x | (x >> 2); 2511 // ... 2512 // x = x | (x >>16); 2513 // x = x | (x >>32); // for 64-bit input 2514 // return popcount(~x); 2515 // 2516 // but see also: http://www.hackersdelight.org/HDcode/nlz.cc 2517 EVT VT = Op.getValueType(); 2518 EVT ShVT = TLI.getShiftAmountTy(VT); 2519 unsigned len = VT.getSizeInBits(); 2520 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 2521 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 2522 Op = DAG.getNode(ISD::OR, dl, VT, Op, 2523 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); 2524 } 2525 Op = DAG.getNOT(dl, Op, VT); 2526 return DAG.getNode(ISD::CTPOP, dl, VT, Op); 2527 } 2528 case ISD::CTTZ_ZERO_UNDEF: 2529 // This trivially expands to CTTZ. 2530 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); 2531 case ISD::CTTZ: { 2532 // for now, we use: { return popcount(~x & (x - 1)); } 2533 // unless the target has ctlz but not ctpop, in which case we use: 2534 // { return 32 - nlz(~x & (x-1)); } 2535 // see also http://www.hackersdelight.org/HDcode/ntz.cc 2536 EVT VT = Op.getValueType(); 2537 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, 2538 DAG.getNOT(dl, Op, VT), 2539 DAG.getNode(ISD::SUB, dl, VT, Op, 2540 DAG.getConstant(1, VT))); 2541 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. 2542 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && 2543 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) 2544 return DAG.getNode(ISD::SUB, dl, VT, 2545 DAG.getConstant(VT.getSizeInBits(), VT), 2546 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); 2547 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); 2548 } 2549 } 2550} 2551 2552std::pair <SDValue, SDValue> SelectionDAGLegalize::ExpandAtomic(SDNode *Node) { 2553 unsigned Opc = Node->getOpcode(); 2554 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2555 RTLIB::Libcall LC; 2556 2557 switch (Opc) { 2558 default: 2559 llvm_unreachable("Unhandled atomic intrinsic Expand!"); 2560 case ISD::ATOMIC_SWAP: 2561 switch (VT.SimpleTy) { 2562 default: llvm_unreachable("Unexpected value type for atomic!"); 2563 case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 2564 case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 2565 case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 2566 case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 2567 } 2568 break; 2569 case ISD::ATOMIC_CMP_SWAP: 2570 switch (VT.SimpleTy) { 2571 default: llvm_unreachable("Unexpected value type for atomic!"); 2572 case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 2573 case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 2574 case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 2575 case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 2576 } 2577 break; 2578 case ISD::ATOMIC_LOAD_ADD: 2579 switch (VT.SimpleTy) { 2580 default: llvm_unreachable("Unexpected value type for atomic!"); 2581 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 2582 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 2583 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 2584 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 2585 } 2586 break; 2587 case ISD::ATOMIC_LOAD_SUB: 2588 switch (VT.SimpleTy) { 2589 default: llvm_unreachable("Unexpected value type for atomic!"); 2590 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 2591 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 2592 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 2593 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 2594 } 2595 break; 2596 case ISD::ATOMIC_LOAD_AND: 2597 switch (VT.SimpleTy) { 2598 default: llvm_unreachable("Unexpected value type for atomic!"); 2599 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 2600 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 2601 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 2602 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 2603 } 2604 break; 2605 case ISD::ATOMIC_LOAD_OR: 2606 switch (VT.SimpleTy) { 2607 default: llvm_unreachable("Unexpected value type for atomic!"); 2608 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 2609 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 2610 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 2611 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 2612 } 2613 break; 2614 case ISD::ATOMIC_LOAD_XOR: 2615 switch (VT.SimpleTy) { 2616 default: llvm_unreachable("Unexpected value type for atomic!"); 2617 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 2618 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 2619 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 2620 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 2621 } 2622 break; 2623 case ISD::ATOMIC_LOAD_NAND: 2624 switch (VT.SimpleTy) { 2625 default: llvm_unreachable("Unexpected value type for atomic!"); 2626 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 2627 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 2628 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 2629 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 2630 } 2631 break; 2632 } 2633 2634 return ExpandChainLibCall(LC, Node, false); 2635} 2636 2637void SelectionDAGLegalize::ExpandNode(SDNode *Node) { 2638 SmallVector<SDValue, 8> Results; 2639 DebugLoc dl = Node->getDebugLoc(); 2640 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 2641 switch (Node->getOpcode()) { 2642 case ISD::CTPOP: 2643 case ISD::CTLZ: 2644 case ISD::CTLZ_ZERO_UNDEF: 2645 case ISD::CTTZ: 2646 case ISD::CTTZ_ZERO_UNDEF: 2647 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl); 2648 Results.push_back(Tmp1); 2649 break; 2650 case ISD::BSWAP: 2651 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); 2652 break; 2653 case ISD::FRAMEADDR: 2654 case ISD::RETURNADDR: 2655 case ISD::FRAME_TO_ARGS_OFFSET: 2656 Results.push_back(DAG.getConstant(0, Node->getValueType(0))); 2657 break; 2658 case ISD::FLT_ROUNDS_: 2659 Results.push_back(DAG.getConstant(1, Node->getValueType(0))); 2660 break; 2661 case ISD::EH_RETURN: 2662 case ISD::EH_LABEL: 2663 case ISD::PREFETCH: 2664 case ISD::VAEND: 2665 case ISD::EH_SJLJ_LONGJMP: 2666 // If the target didn't expand these, there's nothing to do, so just 2667 // preserve the chain and be done. 2668 Results.push_back(Node->getOperand(0)); 2669 break; 2670 case ISD::EH_SJLJ_SETJMP: 2671 // If the target didn't expand this, just return 'zero' and preserve the 2672 // chain. 2673 Results.push_back(DAG.getConstant(0, MVT::i32)); 2674 Results.push_back(Node->getOperand(0)); 2675 break; 2676 case ISD::ATOMIC_FENCE: 2677 case ISD::MEMBARRIER: { 2678 // If the target didn't lower this, lower it to '__sync_synchronize()' call 2679 // FIXME: handle "fence singlethread" more efficiently. 2680 TargetLowering::ArgListTy Args; 2681 TargetLowering:: 2682 CallLoweringInfo CLI(Node->getOperand(0), 2683 Type::getVoidTy(*DAG.getContext()), 2684 false, false, false, false, 0, CallingConv::C, 2685 /*isTailCall=*/false, 2686 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2687 DAG.getExternalSymbol("__sync_synchronize", 2688 TLI.getPointerTy()), 2689 Args, DAG, dl); 2690 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 2691 2692 Results.push_back(CallResult.second); 2693 break; 2694 } 2695 case ISD::ATOMIC_LOAD: { 2696 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. 2697 SDValue Zero = DAG.getConstant(0, Node->getValueType(0)); 2698 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, 2699 cast<AtomicSDNode>(Node)->getMemoryVT(), 2700 Node->getOperand(0), 2701 Node->getOperand(1), Zero, Zero, 2702 cast<AtomicSDNode>(Node)->getMemOperand(), 2703 cast<AtomicSDNode>(Node)->getOrdering(), 2704 cast<AtomicSDNode>(Node)->getSynchScope()); 2705 Results.push_back(Swap.getValue(0)); 2706 Results.push_back(Swap.getValue(1)); 2707 break; 2708 } 2709 case ISD::ATOMIC_STORE: { 2710 // There is no libcall for atomic store; fake it with ATOMIC_SWAP. 2711 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 2712 cast<AtomicSDNode>(Node)->getMemoryVT(), 2713 Node->getOperand(0), 2714 Node->getOperand(1), Node->getOperand(2), 2715 cast<AtomicSDNode>(Node)->getMemOperand(), 2716 cast<AtomicSDNode>(Node)->getOrdering(), 2717 cast<AtomicSDNode>(Node)->getSynchScope()); 2718 Results.push_back(Swap.getValue(1)); 2719 break; 2720 } 2721 // By default, atomic intrinsics are marked Legal and lowered. Targets 2722 // which don't support them directly, however, may want libcalls, in which 2723 // case they mark them Expand, and we get here. 2724 case ISD::ATOMIC_SWAP: 2725 case ISD::ATOMIC_LOAD_ADD: 2726 case ISD::ATOMIC_LOAD_SUB: 2727 case ISD::ATOMIC_LOAD_AND: 2728 case ISD::ATOMIC_LOAD_OR: 2729 case ISD::ATOMIC_LOAD_XOR: 2730 case ISD::ATOMIC_LOAD_NAND: 2731 case ISD::ATOMIC_LOAD_MIN: 2732 case ISD::ATOMIC_LOAD_MAX: 2733 case ISD::ATOMIC_LOAD_UMIN: 2734 case ISD::ATOMIC_LOAD_UMAX: 2735 case ISD::ATOMIC_CMP_SWAP: { 2736 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node); 2737 Results.push_back(Tmp.first); 2738 Results.push_back(Tmp.second); 2739 break; 2740 } 2741 case ISD::DYNAMIC_STACKALLOC: 2742 ExpandDYNAMIC_STACKALLOC(Node, Results); 2743 break; 2744 case ISD::MERGE_VALUES: 2745 for (unsigned i = 0; i < Node->getNumValues(); i++) 2746 Results.push_back(Node->getOperand(i)); 2747 break; 2748 case ISD::UNDEF: { 2749 EVT VT = Node->getValueType(0); 2750 if (VT.isInteger()) 2751 Results.push_back(DAG.getConstant(0, VT)); 2752 else { 2753 assert(VT.isFloatingPoint() && "Unknown value type!"); 2754 Results.push_back(DAG.getConstantFP(0, VT)); 2755 } 2756 break; 2757 } 2758 case ISD::TRAP: { 2759 // If this operation is not supported, lower it to 'abort()' call 2760 TargetLowering::ArgListTy Args; 2761 TargetLowering:: 2762 CallLoweringInfo CLI(Node->getOperand(0), 2763 Type::getVoidTy(*DAG.getContext()), 2764 false, false, false, false, 0, CallingConv::C, 2765 /*isTailCall=*/false, 2766 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2767 DAG.getExternalSymbol("abort", TLI.getPointerTy()), 2768 Args, DAG, dl); 2769 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 2770 2771 Results.push_back(CallResult.second); 2772 break; 2773 } 2774 case ISD::FP_ROUND: 2775 case ISD::BITCAST: 2776 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 2777 Node->getValueType(0), dl); 2778 Results.push_back(Tmp1); 2779 break; 2780 case ISD::FP_EXTEND: 2781 Tmp1 = EmitStackConvert(Node->getOperand(0), 2782 Node->getOperand(0).getValueType(), 2783 Node->getValueType(0), dl); 2784 Results.push_back(Tmp1); 2785 break; 2786 case ISD::SIGN_EXTEND_INREG: { 2787 // NOTE: we could fall back on load/store here too for targets without 2788 // SAR. However, it is doubtful that any exist. 2789 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2790 EVT VT = Node->getValueType(0); 2791 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT); 2792 if (VT.isVector()) 2793 ShiftAmountTy = VT; 2794 unsigned BitsDiff = VT.getScalarType().getSizeInBits() - 2795 ExtraVT.getScalarType().getSizeInBits(); 2796 SDValue ShiftCst = DAG.getConstant(BitsDiff, ShiftAmountTy); 2797 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 2798 Node->getOperand(0), ShiftCst); 2799 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); 2800 Results.push_back(Tmp1); 2801 break; 2802 } 2803 case ISD::FP_ROUND_INREG: { 2804 // The only way we can lower this is to turn it into a TRUNCSTORE, 2805 // EXTLOAD pair, targeting a temporary location (a stack slot). 2806 2807 // NOTE: there is a choice here between constantly creating new stack 2808 // slots and always reusing the same one. We currently always create 2809 // new ones, as reuse may inhibit scheduling. 2810 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2811 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, 2812 Node->getValueType(0), dl); 2813 Results.push_back(Tmp1); 2814 break; 2815 } 2816 case ISD::SINT_TO_FP: 2817 case ISD::UINT_TO_FP: 2818 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, 2819 Node->getOperand(0), Node->getValueType(0), dl); 2820 Results.push_back(Tmp1); 2821 break; 2822 case ISD::FP_TO_UINT: { 2823 SDValue True, False; 2824 EVT VT = Node->getOperand(0).getValueType(); 2825 EVT NVT = Node->getValueType(0); 2826 APFloat apf(APInt::getNullValue(VT.getSizeInBits())); 2827 APInt x = APInt::getSignBit(NVT.getSizeInBits()); 2828 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); 2829 Tmp1 = DAG.getConstantFP(apf, VT); 2830 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), 2831 Node->getOperand(0), 2832 Tmp1, ISD::SETLT); 2833 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); 2834 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, 2835 DAG.getNode(ISD::FSUB, dl, VT, 2836 Node->getOperand(0), Tmp1)); 2837 False = DAG.getNode(ISD::XOR, dl, NVT, False, 2838 DAG.getConstant(x, NVT)); 2839 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, True, False); 2840 Results.push_back(Tmp1); 2841 break; 2842 } 2843 case ISD::VAARG: { 2844 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 2845 EVT VT = Node->getValueType(0); 2846 Tmp1 = Node->getOperand(0); 2847 Tmp2 = Node->getOperand(1); 2848 unsigned Align = Node->getConstantOperandVal(3); 2849 2850 SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, 2851 MachinePointerInfo(V), 2852 false, false, false, 0); 2853 SDValue VAList = VAListLoad; 2854 2855 if (Align > TLI.getMinStackArgumentAlignment()) { 2856 assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); 2857 2858 VAList = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 2859 DAG.getConstant(Align - 1, 2860 TLI.getPointerTy())); 2861 2862 VAList = DAG.getNode(ISD::AND, dl, TLI.getPointerTy(), VAList, 2863 DAG.getConstant(-(int64_t)Align, 2864 TLI.getPointerTy())); 2865 } 2866 2867 // Increment the pointer, VAList, to the next vaarg 2868 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 2869 DAG.getConstant(TLI.getDataLayout()-> 2870 getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), 2871 TLI.getPointerTy())); 2872 // Store the incremented VAList to the legalized pointer 2873 Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2, 2874 MachinePointerInfo(V), false, false, 0); 2875 // Load the actual argument out of the pointer VAList 2876 Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(), 2877 false, false, false, 0)); 2878 Results.push_back(Results[0].getValue(1)); 2879 break; 2880 } 2881 case ISD::VACOPY: { 2882 // This defaults to loading a pointer from the input and storing it to the 2883 // output, returning the chain. 2884 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 2885 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 2886 Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0), 2887 Node->getOperand(2), MachinePointerInfo(VS), 2888 false, false, false, 0); 2889 Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), 2890 MachinePointerInfo(VD), false, false, 0); 2891 Results.push_back(Tmp1); 2892 break; 2893 } 2894 case ISD::EXTRACT_VECTOR_ELT: 2895 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) 2896 // This must be an access of the only element. Return it. 2897 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), 2898 Node->getOperand(0)); 2899 else 2900 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); 2901 Results.push_back(Tmp1); 2902 break; 2903 case ISD::EXTRACT_SUBVECTOR: 2904 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); 2905 break; 2906 case ISD::INSERT_SUBVECTOR: 2907 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); 2908 break; 2909 case ISD::CONCAT_VECTORS: { 2910 Results.push_back(ExpandVectorBuildThroughStack(Node)); 2911 break; 2912 } 2913 case ISD::SCALAR_TO_VECTOR: 2914 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); 2915 break; 2916 case ISD::INSERT_VECTOR_ELT: 2917 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), 2918 Node->getOperand(1), 2919 Node->getOperand(2), dl)); 2920 break; 2921 case ISD::VECTOR_SHUFFLE: { 2922 SmallVector<int, 32> NewMask; 2923 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 2924 2925 EVT VT = Node->getValueType(0); 2926 EVT EltVT = VT.getVectorElementType(); 2927 SDValue Op0 = Node->getOperand(0); 2928 SDValue Op1 = Node->getOperand(1); 2929 if (!TLI.isTypeLegal(EltVT)) { 2930 2931 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); 2932 2933 // BUILD_VECTOR operands are allowed to be wider than the element type. 2934 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept it 2935 if (NewEltVT.bitsLT(EltVT)) { 2936 2937 // Convert shuffle node. 2938 // If original node was v4i64 and the new EltVT is i32, 2939 // cast operands to v8i32 and re-build the mask. 2940 2941 // Calculate new VT, the size of the new VT should be equal to original. 2942 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, 2943 VT.getSizeInBits()/NewEltVT.getSizeInBits()); 2944 assert(NewVT.bitsEq(VT)); 2945 2946 // cast operands to new VT 2947 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); 2948 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); 2949 2950 // Convert the shuffle mask 2951 unsigned int factor = NewVT.getVectorNumElements()/VT.getVectorNumElements(); 2952 2953 // EltVT gets smaller 2954 assert(factor > 0); 2955 2956 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 2957 if (Mask[i] < 0) { 2958 for (unsigned fi = 0; fi < factor; ++fi) 2959 NewMask.push_back(Mask[i]); 2960 } 2961 else { 2962 for (unsigned fi = 0; fi < factor; ++fi) 2963 NewMask.push_back(Mask[i]*factor+fi); 2964 } 2965 } 2966 Mask = NewMask; 2967 VT = NewVT; 2968 } 2969 EltVT = NewEltVT; 2970 } 2971 unsigned NumElems = VT.getVectorNumElements(); 2972 SmallVector<SDValue, 16> Ops; 2973 for (unsigned i = 0; i != NumElems; ++i) { 2974 if (Mask[i] < 0) { 2975 Ops.push_back(DAG.getUNDEF(EltVT)); 2976 continue; 2977 } 2978 unsigned Idx = Mask[i]; 2979 if (Idx < NumElems) 2980 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 2981 Op0, 2982 DAG.getIntPtrConstant(Idx))); 2983 else 2984 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 2985 Op1, 2986 DAG.getIntPtrConstant(Idx - NumElems))); 2987 } 2988 2989 Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size()); 2990 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. 2991 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); 2992 Results.push_back(Tmp1); 2993 break; 2994 } 2995 case ISD::EXTRACT_ELEMENT: { 2996 EVT OpTy = Node->getOperand(0).getValueType(); 2997 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 2998 // 1 -> Hi 2999 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 3000 DAG.getConstant(OpTy.getSizeInBits()/2, 3001 TLI.getShiftAmountTy(Node->getOperand(0).getValueType()))); 3002 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 3003 } else { 3004 // 0 -> Lo 3005 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 3006 Node->getOperand(0)); 3007 } 3008 Results.push_back(Tmp1); 3009 break; 3010 } 3011 case ISD::STACKSAVE: 3012 // Expand to CopyFromReg if the target set 3013 // StackPointerRegisterToSaveRestore. 3014 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3015 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, 3016 Node->getValueType(0))); 3017 Results.push_back(Results[0].getValue(1)); 3018 } else { 3019 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); 3020 Results.push_back(Node->getOperand(0)); 3021 } 3022 break; 3023 case ISD::STACKRESTORE: 3024 // Expand to CopyToReg if the target set 3025 // StackPointerRegisterToSaveRestore. 3026 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3027 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, 3028 Node->getOperand(1))); 3029 } else { 3030 Results.push_back(Node->getOperand(0)); 3031 } 3032 break; 3033 case ISD::FCOPYSIGN: 3034 Results.push_back(ExpandFCOPYSIGN(Node)); 3035 break; 3036 case ISD::FNEG: 3037 // Expand Y = FNEG(X) -> Y = SUB -0.0, X 3038 Tmp1 = DAG.getConstantFP(-0.0, Node->getValueType(0)); 3039 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, 3040 Node->getOperand(0)); 3041 Results.push_back(Tmp1); 3042 break; 3043 case ISD::FABS: { 3044 // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). 3045 EVT VT = Node->getValueType(0); 3046 Tmp1 = Node->getOperand(0); 3047 Tmp2 = DAG.getConstantFP(0.0, VT); 3048 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), 3049 Tmp1, Tmp2, ISD::SETUGT); 3050 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); 3051 Tmp1 = DAG.getNode(ISD::SELECT, dl, VT, Tmp2, Tmp1, Tmp3); 3052 Results.push_back(Tmp1); 3053 break; 3054 } 3055 case ISD::FSQRT: 3056 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 3057 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128)); 3058 break; 3059 case ISD::FSIN: 3060 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, 3061 RTLIB::SIN_F80, RTLIB::SIN_PPCF128)); 3062 break; 3063 case ISD::FCOS: 3064 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, 3065 RTLIB::COS_F80, RTLIB::COS_PPCF128)); 3066 break; 3067 case ISD::FLOG: 3068 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, 3069 RTLIB::LOG_F80, RTLIB::LOG_PPCF128)); 3070 break; 3071 case ISD::FLOG2: 3072 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 3073 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128)); 3074 break; 3075 case ISD::FLOG10: 3076 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 3077 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128)); 3078 break; 3079 case ISD::FEXP: 3080 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, 3081 RTLIB::EXP_F80, RTLIB::EXP_PPCF128)); 3082 break; 3083 case ISD::FEXP2: 3084 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 3085 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128)); 3086 break; 3087 case ISD::FTRUNC: 3088 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 3089 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128)); 3090 break; 3091 case ISD::FFLOOR: 3092 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 3093 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128)); 3094 break; 3095 case ISD::FCEIL: 3096 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 3097 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128)); 3098 break; 3099 case ISD::FRINT: 3100 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, 3101 RTLIB::RINT_F80, RTLIB::RINT_PPCF128)); 3102 break; 3103 case ISD::FNEARBYINT: 3104 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, 3105 RTLIB::NEARBYINT_F64, 3106 RTLIB::NEARBYINT_F80, 3107 RTLIB::NEARBYINT_PPCF128)); 3108 break; 3109 case ISD::FPOWI: 3110 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, 3111 RTLIB::POWI_F80, RTLIB::POWI_PPCF128)); 3112 break; 3113 case ISD::FPOW: 3114 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, 3115 RTLIB::POW_F80, RTLIB::POW_PPCF128)); 3116 break; 3117 case ISD::FDIV: 3118 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, 3119 RTLIB::DIV_F80, RTLIB::DIV_PPCF128)); 3120 break; 3121 case ISD::FREM: 3122 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, 3123 RTLIB::REM_F80, RTLIB::REM_PPCF128)); 3124 break; 3125 case ISD::FMA: 3126 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, 3127 RTLIB::FMA_F80, RTLIB::FMA_PPCF128)); 3128 break; 3129 case ISD::FP16_TO_FP32: 3130 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); 3131 break; 3132 case ISD::FP32_TO_FP16: 3133 Results.push_back(ExpandLibCall(RTLIB::FPROUND_F32_F16, Node, false)); 3134 break; 3135 case ISD::ConstantFP: { 3136 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 3137 // Check to see if this FP immediate is already legal. 3138 // If this is a legal constant, turn it into a TargetConstantFP node. 3139 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0))) 3140 Results.push_back(ExpandConstantFP(CFP, true)); 3141 break; 3142 } 3143 case ISD::EHSELECTION: { 3144 unsigned Reg = TLI.getExceptionSelectorRegister(); 3145 assert(Reg && "Can't expand to unknown register!"); 3146 Results.push_back(DAG.getCopyFromReg(Node->getOperand(1), dl, Reg, 3147 Node->getValueType(0))); 3148 Results.push_back(Results[0].getValue(1)); 3149 break; 3150 } 3151 case ISD::EXCEPTIONADDR: { 3152 unsigned Reg = TLI.getExceptionPointerRegister(); 3153 assert(Reg && "Can't expand to unknown register!"); 3154 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, Reg, 3155 Node->getValueType(0))); 3156 Results.push_back(Results[0].getValue(1)); 3157 break; 3158 } 3159 case ISD::FSUB: { 3160 EVT VT = Node->getValueType(0); 3161 assert(TLI.isOperationLegalOrCustom(ISD::FADD, VT) && 3162 TLI.isOperationLegalOrCustom(ISD::FNEG, VT) && 3163 "Don't know how to expand this FP subtraction!"); 3164 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); 3165 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1); 3166 Results.push_back(Tmp1); 3167 break; 3168 } 3169 case ISD::SUB: { 3170 EVT VT = Node->getValueType(0); 3171 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && 3172 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && 3173 "Don't know how to expand this subtraction!"); 3174 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), 3175 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT)); 3176 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, VT)); 3177 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); 3178 break; 3179 } 3180 case ISD::UREM: 3181 case ISD::SREM: { 3182 EVT VT = Node->getValueType(0); 3183 SDVTList VTs = DAG.getVTList(VT, VT); 3184 bool isSigned = Node->getOpcode() == ISD::SREM; 3185 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; 3186 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3187 Tmp2 = Node->getOperand(0); 3188 Tmp3 = Node->getOperand(1); 3189 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3190 (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3191 // If div is legal, it's better to do the normal expansion 3192 !TLI.isOperationLegalOrCustom(DivOpc, Node->getValueType(0)) && 3193 useDivRem(Node, isSigned, false))) { 3194 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); 3195 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { 3196 // X % Y -> X-X/Y*Y 3197 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); 3198 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); 3199 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); 3200 } else if (isSigned) 3201 Tmp1 = ExpandIntLibCall(Node, true, 3202 RTLIB::SREM_I8, 3203 RTLIB::SREM_I16, RTLIB::SREM_I32, 3204 RTLIB::SREM_I64, RTLIB::SREM_I128); 3205 else 3206 Tmp1 = ExpandIntLibCall(Node, false, 3207 RTLIB::UREM_I8, 3208 RTLIB::UREM_I16, RTLIB::UREM_I32, 3209 RTLIB::UREM_I64, RTLIB::UREM_I128); 3210 Results.push_back(Tmp1); 3211 break; 3212 } 3213 case ISD::UDIV: 3214 case ISD::SDIV: { 3215 bool isSigned = Node->getOpcode() == ISD::SDIV; 3216 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3217 EVT VT = Node->getValueType(0); 3218 SDVTList VTs = DAG.getVTList(VT, VT); 3219 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3220 (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3221 useDivRem(Node, isSigned, true))) 3222 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), 3223 Node->getOperand(1)); 3224 else if (isSigned) 3225 Tmp1 = ExpandIntLibCall(Node, true, 3226 RTLIB::SDIV_I8, 3227 RTLIB::SDIV_I16, RTLIB::SDIV_I32, 3228 RTLIB::SDIV_I64, RTLIB::SDIV_I128); 3229 else 3230 Tmp1 = ExpandIntLibCall(Node, false, 3231 RTLIB::UDIV_I8, 3232 RTLIB::UDIV_I16, RTLIB::UDIV_I32, 3233 RTLIB::UDIV_I64, RTLIB::UDIV_I128); 3234 Results.push_back(Tmp1); 3235 break; 3236 } 3237 case ISD::MULHU: 3238 case ISD::MULHS: { 3239 unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : 3240 ISD::SMUL_LOHI; 3241 EVT VT = Node->getValueType(0); 3242 SDVTList VTs = DAG.getVTList(VT, VT); 3243 assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) && 3244 "If this wasn't legal, it shouldn't have been created!"); 3245 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), 3246 Node->getOperand(1)); 3247 Results.push_back(Tmp1.getValue(1)); 3248 break; 3249 } 3250 case ISD::SDIVREM: 3251 case ISD::UDIVREM: 3252 // Expand into divrem libcall 3253 ExpandDivRemLibCall(Node, Results); 3254 break; 3255 case ISD::MUL: { 3256 EVT VT = Node->getValueType(0); 3257 SDVTList VTs = DAG.getVTList(VT, VT); 3258 // See if multiply or divide can be lowered using two-result operations. 3259 // We just need the low half of the multiply; try both the signed 3260 // and unsigned forms. If the target supports both SMUL_LOHI and 3261 // UMUL_LOHI, form a preference by checking which forms of plain 3262 // MULH it supports. 3263 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3264 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3265 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3266 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3267 unsigned OpToUse = 0; 3268 if (HasSMUL_LOHI && !HasMULHS) { 3269 OpToUse = ISD::SMUL_LOHI; 3270 } else if (HasUMUL_LOHI && !HasMULHU) { 3271 OpToUse = ISD::UMUL_LOHI; 3272 } else if (HasSMUL_LOHI) { 3273 OpToUse = ISD::SMUL_LOHI; 3274 } else if (HasUMUL_LOHI) { 3275 OpToUse = ISD::UMUL_LOHI; 3276 } 3277 if (OpToUse) { 3278 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), 3279 Node->getOperand(1))); 3280 break; 3281 } 3282 Tmp1 = ExpandIntLibCall(Node, false, 3283 RTLIB::MUL_I8, 3284 RTLIB::MUL_I16, RTLIB::MUL_I32, 3285 RTLIB::MUL_I64, RTLIB::MUL_I128); 3286 Results.push_back(Tmp1); 3287 break; 3288 } 3289 case ISD::SADDO: 3290 case ISD::SSUBO: { 3291 SDValue LHS = Node->getOperand(0); 3292 SDValue RHS = Node->getOperand(1); 3293 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3294 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3295 LHS, RHS); 3296 Results.push_back(Sum); 3297 EVT OType = Node->getValueType(1); 3298 3299 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 3300 3301 // LHSSign -> LHS >= 0 3302 // RHSSign -> RHS >= 0 3303 // SumSign -> Sum >= 0 3304 // 3305 // Add: 3306 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 3307 // Sub: 3308 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 3309 // 3310 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 3311 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 3312 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 3313 Node->getOpcode() == ISD::SADDO ? 3314 ISD::SETEQ : ISD::SETNE); 3315 3316 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 3317 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 3318 3319 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 3320 Results.push_back(Cmp); 3321 break; 3322 } 3323 case ISD::UADDO: 3324 case ISD::USUBO: { 3325 SDValue LHS = Node->getOperand(0); 3326 SDValue RHS = Node->getOperand(1); 3327 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? 3328 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3329 LHS, RHS); 3330 Results.push_back(Sum); 3331 Results.push_back(DAG.getSetCC(dl, Node->getValueType(1), Sum, LHS, 3332 Node->getOpcode () == ISD::UADDO ? 3333 ISD::SETULT : ISD::SETUGT)); 3334 break; 3335 } 3336 case ISD::UMULO: 3337 case ISD::SMULO: { 3338 EVT VT = Node->getValueType(0); 3339 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); 3340 SDValue LHS = Node->getOperand(0); 3341 SDValue RHS = Node->getOperand(1); 3342 SDValue BottomHalf; 3343 SDValue TopHalf; 3344 static const unsigned Ops[2][3] = 3345 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, 3346 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; 3347 bool isSigned = Node->getOpcode() == ISD::SMULO; 3348 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) { 3349 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3350 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); 3351 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) { 3352 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, 3353 RHS); 3354 TopHalf = BottomHalf.getValue(1); 3355 } else if (TLI.isTypeLegal(EVT::getIntegerVT(*DAG.getContext(), 3356 VT.getSizeInBits() * 2))) { 3357 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); 3358 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); 3359 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 3360 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3361 DAG.getIntPtrConstant(0)); 3362 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3363 DAG.getIntPtrConstant(1)); 3364 } else { 3365 // We can fall back to a libcall with an illegal type for the MUL if we 3366 // have a libcall big enough. 3367 // Also, we can fall back to a division in some cases, but that's a big 3368 // performance hit in the general case. 3369 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3370 if (WideVT == MVT::i16) 3371 LC = RTLIB::MUL_I16; 3372 else if (WideVT == MVT::i32) 3373 LC = RTLIB::MUL_I32; 3374 else if (WideVT == MVT::i64) 3375 LC = RTLIB::MUL_I64; 3376 else if (WideVT == MVT::i128) 3377 LC = RTLIB::MUL_I128; 3378 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); 3379 3380 // The high part is obtained by SRA'ing all but one of the bits of low 3381 // part. 3382 unsigned LoSize = VT.getSizeInBits(); 3383 SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS, 3384 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3385 SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS, 3386 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3387 3388 // Here we're passing the 2 arguments explicitly as 4 arguments that are 3389 // pre-lowered to the correct types. This all depends upon WideVT not 3390 // being a legal type for the architecture and thus has to be split to 3391 // two arguments. 3392 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; 3393 SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); 3394 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3395 DAG.getIntPtrConstant(0)); 3396 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3397 DAG.getIntPtrConstant(1)); 3398 // Ret is a node with an illegal type. Because such things are not 3399 // generally permitted during this phase of legalization, delete the 3400 // node. The above EXTRACT_ELEMENT nodes should have been folded. 3401 DAG.DeleteNode(Ret.getNode()); 3402 } 3403 3404 if (isSigned) { 3405 Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1, 3406 TLI.getShiftAmountTy(BottomHalf.getValueType())); 3407 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); 3408 TopHalf = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), TopHalf, Tmp1, 3409 ISD::SETNE); 3410 } else { 3411 TopHalf = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), TopHalf, 3412 DAG.getConstant(0, VT), ISD::SETNE); 3413 } 3414 Results.push_back(BottomHalf); 3415 Results.push_back(TopHalf); 3416 break; 3417 } 3418 case ISD::BUILD_PAIR: { 3419 EVT PairTy = Node->getValueType(0); 3420 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); 3421 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); 3422 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, 3423 DAG.getConstant(PairTy.getSizeInBits()/2, 3424 TLI.getShiftAmountTy(PairTy))); 3425 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); 3426 break; 3427 } 3428 case ISD::SELECT: 3429 Tmp1 = Node->getOperand(0); 3430 Tmp2 = Node->getOperand(1); 3431 Tmp3 = Node->getOperand(2); 3432 if (Tmp1.getOpcode() == ISD::SETCC) { 3433 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 3434 Tmp2, Tmp3, 3435 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 3436 } else { 3437 Tmp1 = DAG.getSelectCC(dl, Tmp1, 3438 DAG.getConstant(0, Tmp1.getValueType()), 3439 Tmp2, Tmp3, ISD::SETNE); 3440 } 3441 Results.push_back(Tmp1); 3442 break; 3443 case ISD::BR_JT: { 3444 SDValue Chain = Node->getOperand(0); 3445 SDValue Table = Node->getOperand(1); 3446 SDValue Index = Node->getOperand(2); 3447 3448 EVT PTy = TLI.getPointerTy(); 3449 3450 const DataLayout &TD = *TLI.getDataLayout(); 3451 unsigned EntrySize = 3452 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); 3453 3454 Index = DAG.getNode(ISD::MUL, dl, PTy, 3455 Index, DAG.getConstant(EntrySize, PTy)); 3456 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table); 3457 3458 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); 3459 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, 3460 MachinePointerInfo::getJumpTable(), MemVT, 3461 false, false, 0); 3462 Addr = LD; 3463 if (TM.getRelocationModel() == Reloc::PIC_) { 3464 // For PIC, the sequence is: 3465 // BRIND(load(Jumptable + index) + RelocBase) 3466 // RelocBase can be JumpTable, GOT or some sort of global base. 3467 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 3468 TLI.getPICJumpTableRelocBase(Table, DAG)); 3469 } 3470 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); 3471 Results.push_back(Tmp1); 3472 break; 3473 } 3474 case ISD::BRCOND: 3475 // Expand brcond's setcc into its constituent parts and create a BR_CC 3476 // Node. 3477 Tmp1 = Node->getOperand(0); 3478 Tmp2 = Node->getOperand(1); 3479 if (Tmp2.getOpcode() == ISD::SETCC) { 3480 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 3481 Tmp1, Tmp2.getOperand(2), 3482 Tmp2.getOperand(0), Tmp2.getOperand(1), 3483 Node->getOperand(2)); 3484 } else { 3485 // We test only the i1 bit. Skip the AND if UNDEF. 3486 Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 : 3487 DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, 3488 DAG.getConstant(1, Tmp2.getValueType())); 3489 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 3490 DAG.getCondCode(ISD::SETNE), Tmp3, 3491 DAG.getConstant(0, Tmp3.getValueType()), 3492 Node->getOperand(2)); 3493 } 3494 Results.push_back(Tmp1); 3495 break; 3496 case ISD::SETCC: { 3497 Tmp1 = Node->getOperand(0); 3498 Tmp2 = Node->getOperand(1); 3499 Tmp3 = Node->getOperand(2); 3500 LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, Tmp3, dl); 3501 3502 // If we expanded the SETCC into an AND/OR, return the new node 3503 if (Tmp2.getNode() == 0) { 3504 Results.push_back(Tmp1); 3505 break; 3506 } 3507 3508 // Otherwise, SETCC for the given comparison type must be completely 3509 // illegal; expand it into a SELECT_CC. 3510 EVT VT = Node->getValueType(0); 3511 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3512 DAG.getConstant(1, VT), DAG.getConstant(0, VT), Tmp3); 3513 Results.push_back(Tmp1); 3514 break; 3515 } 3516 case ISD::SELECT_CC: { 3517 Tmp1 = Node->getOperand(0); // LHS 3518 Tmp2 = Node->getOperand(1); // RHS 3519 Tmp3 = Node->getOperand(2); // True 3520 Tmp4 = Node->getOperand(3); // False 3521 SDValue CC = Node->getOperand(4); 3522 3523 LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp1.getValueType()), 3524 Tmp1, Tmp2, CC, dl); 3525 3526 assert(!Tmp2.getNode() && "Can't legalize SELECT_CC with legal condition!"); 3527 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 3528 CC = DAG.getCondCode(ISD::SETNE); 3529 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, Tmp2, 3530 Tmp3, Tmp4, CC); 3531 Results.push_back(Tmp1); 3532 break; 3533 } 3534 case ISD::BR_CC: { 3535 Tmp1 = Node->getOperand(0); // Chain 3536 Tmp2 = Node->getOperand(2); // LHS 3537 Tmp3 = Node->getOperand(3); // RHS 3538 Tmp4 = Node->getOperand(1); // CC 3539 3540 LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp2.getValueType()), 3541 Tmp2, Tmp3, Tmp4, dl); 3542 3543 assert(!Tmp3.getNode() && "Can't legalize BR_CC with legal condition!"); 3544 Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); 3545 Tmp4 = DAG.getCondCode(ISD::SETNE); 3546 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, Tmp2, 3547 Tmp3, Node->getOperand(4)); 3548 Results.push_back(Tmp1); 3549 break; 3550 } 3551 case ISD::BUILD_VECTOR: 3552 Results.push_back(ExpandBUILD_VECTOR(Node)); 3553 break; 3554 case ISD::SRA: 3555 case ISD::SRL: 3556 case ISD::SHL: { 3557 // Scalarize vector SRA/SRL/SHL. 3558 EVT VT = Node->getValueType(0); 3559 assert(VT.isVector() && "Unable to legalize non-vector shift"); 3560 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); 3561 unsigned NumElem = VT.getVectorNumElements(); 3562 3563 SmallVector<SDValue, 8> Scalars; 3564 for (unsigned Idx = 0; Idx < NumElem; Idx++) { 3565 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3566 VT.getScalarType(), 3567 Node->getOperand(0), DAG.getIntPtrConstant(Idx)); 3568 SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3569 VT.getScalarType(), 3570 Node->getOperand(1), DAG.getIntPtrConstant(Idx)); 3571 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, 3572 VT.getScalarType(), Ex, Sh)); 3573 } 3574 SDValue Result = 3575 DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), 3576 &Scalars[0], Scalars.size()); 3577 ReplaceNode(SDValue(Node, 0), Result); 3578 break; 3579 } 3580 case ISD::GLOBAL_OFFSET_TABLE: 3581 case ISD::GlobalAddress: 3582 case ISD::GlobalTLSAddress: 3583 case ISD::ExternalSymbol: 3584 case ISD::ConstantPool: 3585 case ISD::JumpTable: 3586 case ISD::INTRINSIC_W_CHAIN: 3587 case ISD::INTRINSIC_WO_CHAIN: 3588 case ISD::INTRINSIC_VOID: 3589 // FIXME: Custom lowering for these operations shouldn't return null! 3590 break; 3591 } 3592 3593 // Replace the original node with the legalized result. 3594 if (!Results.empty()) 3595 ReplaceNode(Node, Results.data()); 3596} 3597 3598void SelectionDAGLegalize::PromoteNode(SDNode *Node) { 3599 SmallVector<SDValue, 8> Results; 3600 MVT OVT = Node->getSimpleValueType(0); 3601 if (Node->getOpcode() == ISD::UINT_TO_FP || 3602 Node->getOpcode() == ISD::SINT_TO_FP || 3603 Node->getOpcode() == ISD::SETCC) { 3604 OVT = Node->getOperand(0).getSimpleValueType(); 3605 } 3606 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3607 DebugLoc dl = Node->getDebugLoc(); 3608 SDValue Tmp1, Tmp2, Tmp3; 3609 switch (Node->getOpcode()) { 3610 case ISD::CTTZ: 3611 case ISD::CTTZ_ZERO_UNDEF: 3612 case ISD::CTLZ: 3613 case ISD::CTLZ_ZERO_UNDEF: 3614 case ISD::CTPOP: 3615 // Zero extend the argument. 3616 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 3617 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is 3618 // already the correct result. 3619 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 3620 if (Node->getOpcode() == ISD::CTTZ) { 3621 // FIXME: This should set a bit in the zero extended value instead. 3622 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), 3623 Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), 3624 ISD::SETEQ); 3625 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, 3626 DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); 3627 } else if (Node->getOpcode() == ISD::CTLZ || 3628 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { 3629 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 3630 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 3631 DAG.getConstant(NVT.getSizeInBits() - 3632 OVT.getSizeInBits(), NVT)); 3633 } 3634 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 3635 break; 3636 case ISD::BSWAP: { 3637 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 3638 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 3639 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 3640 Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 3641 DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT))); 3642 Results.push_back(Tmp1); 3643 break; 3644 } 3645 case ISD::FP_TO_UINT: 3646 case ISD::FP_TO_SINT: 3647 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), 3648 Node->getOpcode() == ISD::FP_TO_SINT, dl); 3649 Results.push_back(Tmp1); 3650 break; 3651 case ISD::UINT_TO_FP: 3652 case ISD::SINT_TO_FP: 3653 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), 3654 Node->getOpcode() == ISD::SINT_TO_FP, dl); 3655 Results.push_back(Tmp1); 3656 break; 3657 case ISD::VAARG: { 3658 SDValue Chain = Node->getOperand(0); // Get the chain. 3659 SDValue Ptr = Node->getOperand(1); // Get the pointer. 3660 3661 unsigned TruncOp; 3662 if (OVT.isVector()) { 3663 TruncOp = ISD::BITCAST; 3664 } else { 3665 assert(OVT.isInteger() 3666 && "VAARG promotion is supported only for vectors or integer types"); 3667 TruncOp = ISD::TRUNCATE; 3668 } 3669 3670 // Perform the larger operation, then convert back 3671 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), 3672 Node->getConstantOperandVal(3)); 3673 Chain = Tmp1.getValue(1); 3674 3675 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); 3676 3677 // Modified the chain result - switch anything that used the old chain to 3678 // use the new one. 3679 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); 3680 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 3681 ReplacedNode(Node); 3682 break; 3683 } 3684 case ISD::AND: 3685 case ISD::OR: 3686 case ISD::XOR: { 3687 unsigned ExtOp, TruncOp; 3688 if (OVT.isVector()) { 3689 ExtOp = ISD::BITCAST; 3690 TruncOp = ISD::BITCAST; 3691 } else { 3692 assert(OVT.isInteger() && "Cannot promote logic operation"); 3693 ExtOp = ISD::ANY_EXTEND; 3694 TruncOp = ISD::TRUNCATE; 3695 } 3696 // Promote each of the values to the new type. 3697 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 3698 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3699 // Perform the larger operation, then convert back 3700 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3701 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); 3702 break; 3703 } 3704 case ISD::SELECT: { 3705 unsigned ExtOp, TruncOp; 3706 if (Node->getValueType(0).isVector()) { 3707 ExtOp = ISD::BITCAST; 3708 TruncOp = ISD::BITCAST; 3709 } else if (Node->getValueType(0).isInteger()) { 3710 ExtOp = ISD::ANY_EXTEND; 3711 TruncOp = ISD::TRUNCATE; 3712 } else { 3713 ExtOp = ISD::FP_EXTEND; 3714 TruncOp = ISD::FP_ROUND; 3715 } 3716 Tmp1 = Node->getOperand(0); 3717 // Promote each of the values to the new type. 3718 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3719 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 3720 // Perform the larger operation, then round down. 3721 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2, Tmp3); 3722 if (TruncOp != ISD::FP_ROUND) 3723 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); 3724 else 3725 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, 3726 DAG.getIntPtrConstant(0)); 3727 Results.push_back(Tmp1); 3728 break; 3729 } 3730 case ISD::VECTOR_SHUFFLE: { 3731 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 3732 3733 // Cast the two input vectors. 3734 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); 3735 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); 3736 3737 // Convert the shuffle mask to the right # elements. 3738 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); 3739 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); 3740 Results.push_back(Tmp1); 3741 break; 3742 } 3743 case ISD::SETCC: { 3744 unsigned ExtOp = ISD::FP_EXTEND; 3745 if (NVT.isInteger()) { 3746 ISD::CondCode CCCode = 3747 cast<CondCodeSDNode>(Node->getOperand(2))->get(); 3748 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 3749 } 3750 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 3751 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3752 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 3753 Tmp1, Tmp2, Node->getOperand(2))); 3754 break; 3755 } 3756 case ISD::FDIV: 3757 case ISD::FREM: 3758 case ISD::FPOW: { 3759 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 3760 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 3761 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3762 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 3763 Tmp3, DAG.getIntPtrConstant(0))); 3764 break; 3765 } 3766 case ISD::FLOG2: 3767 case ISD::FEXP2: 3768 case ISD::FLOG: 3769 case ISD::FEXP: { 3770 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 3771 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 3772 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 3773 Tmp2, DAG.getIntPtrConstant(0))); 3774 break; 3775 } 3776 } 3777 3778 // Replace the original node with the legalized result. 3779 if (!Results.empty()) 3780 ReplaceNode(Node, Results.data()); 3781} 3782 3783// SelectionDAG::Legalize - This is the entry point for the file. 3784// 3785void SelectionDAG::Legalize() { 3786 /// run - This is the main entry point to this class. 3787 /// 3788 SelectionDAGLegalize(*this).LegalizeDAG(); 3789} 3790