LegalizeDAG.cpp revision bc7d448f242b1bbc1031fb87cd69c285ff9aaffa
1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SelectionDAG::Legalize method. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/Analysis/DebugInfo.h" 15#include "llvm/CodeGen/Analysis.h" 16#include "llvm/CodeGen/MachineFunction.h" 17#include "llvm/CodeGen/MachineJumpTableInfo.h" 18#include "llvm/CodeGen/SelectionDAG.h" 19#include "llvm/Target/TargetFrameLowering.h" 20#include "llvm/Target/TargetLowering.h" 21#include "llvm/Target/TargetData.h" 22#include "llvm/Target/TargetMachine.h" 23#include "llvm/CallingConv.h" 24#include "llvm/Constants.h" 25#include "llvm/DerivedTypes.h" 26#include "llvm/LLVMContext.h" 27#include "llvm/Support/Debug.h" 28#include "llvm/Support/ErrorHandling.h" 29#include "llvm/Support/MathExtras.h" 30#include "llvm/Support/raw_ostream.h" 31#include "llvm/ADT/DenseMap.h" 32#include "llvm/ADT/SmallVector.h" 33#include "llvm/ADT/SmallPtrSet.h" 34using namespace llvm; 35 36//===----------------------------------------------------------------------===// 37/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and 38/// hacks on it until the target machine can handle it. This involves 39/// eliminating value sizes the machine cannot handle (promoting small sizes to 40/// large sizes or splitting up large values into small values) as well as 41/// eliminating operations the machine cannot handle. 42/// 43/// This code also does a small amount of optimization and recognition of idioms 44/// as part of its processing. For example, if a target does not support a 45/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 46/// will attempt merge setcc and brc instructions into brcc's. 47/// 48namespace { 49class SelectionDAGLegalize : public SelectionDAG::DAGUpdateListener { 50 const TargetMachine &TM; 51 const TargetLowering &TLI; 52 SelectionDAG &DAG; 53 54 /// LegalizePosition - The iterator for walking through the node list. 55 SelectionDAG::allnodes_iterator LegalizePosition; 56 57 /// LegalizedNodes - The set of nodes which have already been legalized. 58 SmallPtrSet<SDNode *, 16> LegalizedNodes; 59 60 // Libcall insertion helpers. 61 62public: 63 explicit SelectionDAGLegalize(SelectionDAG &DAG); 64 65 void LegalizeDAG(); 66 67private: 68 /// LegalizeOp - Legalizes the given operation. 69 void LegalizeOp(SDNode *Node); 70 71 SDValue OptimizeFloatStore(StoreSDNode *ST); 72 73 /// PerformInsertVectorEltInMemory - Some target cannot handle a variable 74 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 75 /// is necessary to spill the vector being inserted into to memory, perform 76 /// the insert there, and then read the result back. 77 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, 78 SDValue Idx, DebugLoc dl); 79 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, 80 SDValue Idx, DebugLoc dl); 81 82 /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 83 /// performs the same shuffe in terms of order or result bytes, but on a type 84 /// whose vector element type is narrower than the original shuffle type. 85 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 86 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl, 87 SDValue N1, SDValue N2, 88 ArrayRef<int> Mask) const; 89 90 void LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 91 DebugLoc dl); 92 93 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); 94 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, 95 unsigned NumOps, bool isSigned, DebugLoc dl); 96 97 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, 98 SDNode *Node, bool isSigned); 99 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, 100 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, 101 RTLIB::Libcall Call_PPCF128); 102 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, 103 RTLIB::Libcall Call_I8, 104 RTLIB::Libcall Call_I16, 105 RTLIB::Libcall Call_I32, 106 RTLIB::Libcall Call_I64, 107 RTLIB::Libcall Call_I128); 108 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 109 110 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, DebugLoc dl); 111 SDValue ExpandBUILD_VECTOR(SDNode *Node); 112 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 113 void ExpandDYNAMIC_STACKALLOC(SDNode *Node, 114 SmallVectorImpl<SDValue> &Results); 115 SDValue ExpandFCOPYSIGN(SDNode *Node); 116 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT, 117 DebugLoc dl); 118 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, 119 DebugLoc dl); 120 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, 121 DebugLoc dl); 122 123 SDValue ExpandBSWAP(SDValue Op, DebugLoc dl); 124 SDValue ExpandBitCount(unsigned Opc, SDValue Op, DebugLoc dl); 125 126 SDValue ExpandExtractFromVectorThroughStack(SDValue Op); 127 SDValue ExpandInsertToVectorThroughStack(SDValue Op); 128 SDValue ExpandVectorBuildThroughStack(SDNode* Node); 129 130 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); 131 132 std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); 133 134 void ExpandNode(SDNode *Node); 135 void PromoteNode(SDNode *Node); 136 137 void ForgetNode(SDNode *N) { 138 LegalizedNodes.erase(N); 139 if (LegalizePosition == SelectionDAG::allnodes_iterator(N)) 140 ++LegalizePosition; 141 } 142 143public: 144 // DAGUpdateListener implementation. 145 virtual void NodeDeleted(SDNode *N, SDNode *E) { 146 ForgetNode(N); 147 } 148 virtual void NodeUpdated(SDNode *N) {} 149 150 // Node replacement helpers 151 void ReplacedNode(SDNode *N) { 152 if (N->use_empty()) { 153 DAG.RemoveDeadNode(N); 154 } else { 155 ForgetNode(N); 156 } 157 } 158 void ReplaceNode(SDNode *Old, SDNode *New) { 159 DAG.ReplaceAllUsesWith(Old, New); 160 ReplacedNode(Old); 161 } 162 void ReplaceNode(SDValue Old, SDValue New) { 163 DAG.ReplaceAllUsesWith(Old, New); 164 ReplacedNode(Old.getNode()); 165 } 166 void ReplaceNode(SDNode *Old, const SDValue *New) { 167 DAG.ReplaceAllUsesWith(Old, New); 168 ReplacedNode(Old); 169 } 170}; 171} 172 173/// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 174/// performs the same shuffe in terms of order or result bytes, but on a type 175/// whose vector element type is narrower than the original shuffle type. 176/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 177SDValue 178SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl, 179 SDValue N1, SDValue N2, 180 ArrayRef<int> Mask) const { 181 unsigned NumMaskElts = VT.getVectorNumElements(); 182 unsigned NumDestElts = NVT.getVectorNumElements(); 183 unsigned NumEltsGrowth = NumDestElts / NumMaskElts; 184 185 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 186 187 if (NumEltsGrowth == 1) 188 return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]); 189 190 SmallVector<int, 8> NewMask; 191 for (unsigned i = 0; i != NumMaskElts; ++i) { 192 int Idx = Mask[i]; 193 for (unsigned j = 0; j != NumEltsGrowth; ++j) { 194 if (Idx < 0) 195 NewMask.push_back(-1); 196 else 197 NewMask.push_back(Idx * NumEltsGrowth + j); 198 } 199 } 200 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); 201 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); 202 return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); 203} 204 205SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) 206 : SelectionDAG::DAGUpdateListener(dag), 207 TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()), 208 DAG(dag) { 209} 210 211void SelectionDAGLegalize::LegalizeDAG() { 212 DAG.AssignTopologicalOrder(); 213 214 // Visit all the nodes. We start in topological order, so that we see 215 // nodes with their original operands intact. Legalization can produce 216 // new nodes which may themselves need to be legalized. Iterate until all 217 // nodes have been legalized. 218 for (;;) { 219 bool AnyLegalized = false; 220 for (LegalizePosition = DAG.allnodes_end(); 221 LegalizePosition != DAG.allnodes_begin(); ) { 222 --LegalizePosition; 223 224 SDNode *N = LegalizePosition; 225 if (LegalizedNodes.insert(N)) { 226 AnyLegalized = true; 227 LegalizeOp(N); 228 } 229 } 230 if (!AnyLegalized) 231 break; 232 233 } 234 235 // Remove dead nodes now. 236 DAG.RemoveDeadNodes(); 237} 238 239/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or 240/// a load from the constant pool. 241SDValue 242SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { 243 bool Extend = false; 244 DebugLoc dl = CFP->getDebugLoc(); 245 246 // If a FP immediate is precise when represented as a float and if the 247 // target can do an extending load from float to double, we put it into 248 // the constant pool as a float, even if it's is statically typed as a 249 // double. This shrinks FP constants and canonicalizes them for targets where 250 // an FP extending load is the same cost as a normal load (such as on the x87 251 // fp stack or PPC FP unit). 252 EVT VT = CFP->getValueType(0); 253 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 254 if (!UseCP) { 255 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); 256 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), 257 (VT == MVT::f64) ? MVT::i64 : MVT::i32); 258 } 259 260 EVT OrigVT = VT; 261 EVT SVT = VT; 262 while (SVT != MVT::f32) { 263 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); 264 if (ConstantFPSDNode::isValueValidForType(SVT, CFP->getValueAPF()) && 265 // Only do this if the target has a native EXTLOAD instruction from 266 // smaller type. 267 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && 268 TLI.ShouldShrinkFPConstant(OrigVT)) { 269 Type *SType = SVT.getTypeForEVT(*DAG.getContext()); 270 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 271 VT = SVT; 272 Extend = true; 273 } 274 } 275 276 SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); 277 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 278 if (Extend) { 279 SDValue Result = 280 DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT, 281 DAG.getEntryNode(), 282 CPIdx, MachinePointerInfo::getConstantPool(), 283 VT, false, false, Alignment); 284 return Result; 285 } 286 SDValue Result = 287 DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx, 288 MachinePointerInfo::getConstantPool(), false, false, false, 289 Alignment); 290 return Result; 291} 292 293/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. 294static void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, 295 const TargetLowering &TLI, 296 SelectionDAGLegalize *DAGLegalize) { 297 assert(ST->getAddressingMode() == ISD::UNINDEXED && 298 "unaligned indexed stores not implemented!"); 299 SDValue Chain = ST->getChain(); 300 SDValue Ptr = ST->getBasePtr(); 301 SDValue Val = ST->getValue(); 302 EVT VT = Val.getValueType(); 303 int Alignment = ST->getAlignment(); 304 DebugLoc dl = ST->getDebugLoc(); 305 if (ST->getMemoryVT().isFloatingPoint() || 306 ST->getMemoryVT().isVector()) { 307 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 308 if (TLI.isTypeLegal(intVT)) { 309 // Expand to a bitconvert of the value to the integer type of the 310 // same size, then a (misaligned) int store. 311 // FIXME: Does not handle truncating floating point stores! 312 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); 313 Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(), 314 ST->isVolatile(), ST->isNonTemporal(), Alignment); 315 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 316 return; 317 } 318 // Do a (aligned) store to a stack slot, then copy from the stack slot 319 // to the final destination using (unaligned) integer loads and stores. 320 EVT StoredVT = ST->getMemoryVT(); 321 EVT RegVT = 322 TLI.getRegisterType(*DAG.getContext(), 323 EVT::getIntegerVT(*DAG.getContext(), 324 StoredVT.getSizeInBits())); 325 unsigned StoredBytes = StoredVT.getSizeInBits() / 8; 326 unsigned RegBytes = RegVT.getSizeInBits() / 8; 327 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; 328 329 // Make sure the stack slot is also aligned for the register type. 330 SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); 331 332 // Perform the original store, only redirected to the stack slot. 333 SDValue Store = DAG.getTruncStore(Chain, dl, 334 Val, StackPtr, MachinePointerInfo(), 335 StoredVT, false, false, 0); 336 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 337 SmallVector<SDValue, 8> Stores; 338 unsigned Offset = 0; 339 340 // Do all but one copies using the full register width. 341 for (unsigned i = 1; i < NumRegs; i++) { 342 // Load one integer register's worth from the stack slot. 343 SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr, 344 MachinePointerInfo(), 345 false, false, false, 0); 346 // Store it to the final location. Remember the store. 347 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, 348 ST->getPointerInfo().getWithOffset(Offset), 349 ST->isVolatile(), ST->isNonTemporal(), 350 MinAlign(ST->getAlignment(), Offset))); 351 // Increment the pointers. 352 Offset += RegBytes; 353 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 354 Increment); 355 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 356 } 357 358 // The last store may be partial. Do a truncating store. On big-endian 359 // machines this requires an extending load from the stack slot to ensure 360 // that the bits are in the right place. 361 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 362 8 * (StoredBytes - Offset)); 363 364 // Load from the stack slot. 365 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, 366 MachinePointerInfo(), 367 MemVT, false, false, 0); 368 369 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, 370 ST->getPointerInfo() 371 .getWithOffset(Offset), 372 MemVT, ST->isVolatile(), 373 ST->isNonTemporal(), 374 MinAlign(ST->getAlignment(), Offset))); 375 // The order of the stores doesn't matter - say it with a TokenFactor. 376 SDValue Result = 377 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 378 Stores.size()); 379 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 380 return; 381 } 382 assert(ST->getMemoryVT().isInteger() && 383 !ST->getMemoryVT().isVector() && 384 "Unaligned store of unknown type."); 385 // Get the half-size VT 386 EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext()); 387 int NumBits = NewStoredVT.getSizeInBits(); 388 int IncrementSize = NumBits / 8; 389 390 // Divide the stored value in two parts. 391 SDValue ShiftAmount = DAG.getConstant(NumBits, 392 TLI.getShiftAmountTy(Val.getValueType())); 393 SDValue Lo = Val; 394 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); 395 396 // Store the two parts 397 SDValue Store1, Store2; 398 Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr, 399 ST->getPointerInfo(), NewStoredVT, 400 ST->isVolatile(), ST->isNonTemporal(), Alignment); 401 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 402 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 403 Alignment = MinAlign(Alignment, IncrementSize); 404 Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr, 405 ST->getPointerInfo().getWithOffset(IncrementSize), 406 NewStoredVT, ST->isVolatile(), ST->isNonTemporal(), 407 Alignment); 408 409 SDValue Result = 410 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); 411 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 412} 413 414/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. 415static void 416ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, 417 const TargetLowering &TLI, 418 SDValue &ValResult, SDValue &ChainResult) { 419 assert(LD->getAddressingMode() == ISD::UNINDEXED && 420 "unaligned indexed loads not implemented!"); 421 SDValue Chain = LD->getChain(); 422 SDValue Ptr = LD->getBasePtr(); 423 EVT VT = LD->getValueType(0); 424 EVT LoadedVT = LD->getMemoryVT(); 425 DebugLoc dl = LD->getDebugLoc(); 426 if (VT.isFloatingPoint() || VT.isVector()) { 427 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits()); 428 if (TLI.isTypeLegal(intVT)) { 429 // Expand to a (misaligned) integer load of the same size, 430 // then bitconvert to floating point or vector. 431 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, LD->getPointerInfo(), 432 LD->isVolatile(), 433 LD->isNonTemporal(), 434 LD->isInvariant(), LD->getAlignment()); 435 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); 436 if (VT.isFloatingPoint() && LoadedVT != VT) 437 Result = DAG.getNode(ISD::FP_EXTEND, dl, VT, Result); 438 439 ValResult = Result; 440 ChainResult = Chain; 441 return; 442 } 443 444 // Copy the value to a (aligned) stack slot using (unaligned) integer 445 // loads and stores, then do a (aligned) load from the stack slot. 446 EVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT); 447 unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; 448 unsigned RegBytes = RegVT.getSizeInBits() / 8; 449 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; 450 451 // Make sure the stack slot is also aligned for the register type. 452 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT); 453 454 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 455 SmallVector<SDValue, 8> Stores; 456 SDValue StackPtr = StackBase; 457 unsigned Offset = 0; 458 459 // Do all but one copies using the full register width. 460 for (unsigned i = 1; i < NumRegs; i++) { 461 // Load one integer register's worth from the original location. 462 SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr, 463 LD->getPointerInfo().getWithOffset(Offset), 464 LD->isVolatile(), LD->isNonTemporal(), 465 LD->isInvariant(), 466 MinAlign(LD->getAlignment(), Offset)); 467 // Follow the load with a store to the stack slot. Remember the store. 468 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr, 469 MachinePointerInfo(), false, false, 0)); 470 // Increment the pointers. 471 Offset += RegBytes; 472 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 473 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 474 Increment); 475 } 476 477 // The last copy may be partial. Do an extending load. 478 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 479 8 * (LoadedBytes - Offset)); 480 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, 481 LD->getPointerInfo().getWithOffset(Offset), 482 MemVT, LD->isVolatile(), 483 LD->isNonTemporal(), 484 MinAlign(LD->getAlignment(), Offset)); 485 // Follow the load with a store to the stack slot. Remember the store. 486 // On big-endian machines this requires a truncating store to ensure 487 // that the bits end up in the right place. 488 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr, 489 MachinePointerInfo(), MemVT, 490 false, false, 0)); 491 492 // The order of the stores doesn't matter - say it with a TokenFactor. 493 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 494 Stores.size()); 495 496 // Finally, perform the original load only redirected to the stack slot. 497 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, 498 MachinePointerInfo(), LoadedVT, false, false, 0); 499 500 // Callers expect a MERGE_VALUES node. 501 ValResult = Load; 502 ChainResult = TF; 503 return; 504 } 505 assert(LoadedVT.isInteger() && !LoadedVT.isVector() && 506 "Unaligned load of unsupported type."); 507 508 // Compute the new VT that is half the size of the old one. This is an 509 // integer MVT. 510 unsigned NumBits = LoadedVT.getSizeInBits(); 511 EVT NewLoadedVT; 512 NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2); 513 NumBits >>= 1; 514 515 unsigned Alignment = LD->getAlignment(); 516 unsigned IncrementSize = NumBits / 8; 517 ISD::LoadExtType HiExtType = LD->getExtensionType(); 518 519 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. 520 if (HiExtType == ISD::NON_EXTLOAD) 521 HiExtType = ISD::ZEXTLOAD; 522 523 // Load the value in two parts 524 SDValue Lo, Hi; 525 if (TLI.isLittleEndian()) { 526 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), 527 NewLoadedVT, LD->isVolatile(), 528 LD->isNonTemporal(), Alignment); 529 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 530 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 531 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, 532 LD->getPointerInfo().getWithOffset(IncrementSize), 533 NewLoadedVT, LD->isVolatile(), 534 LD->isNonTemporal(), MinAlign(Alignment,IncrementSize)); 535 } else { 536 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), 537 NewLoadedVT, LD->isVolatile(), 538 LD->isNonTemporal(), Alignment); 539 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 540 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 541 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, 542 LD->getPointerInfo().getWithOffset(IncrementSize), 543 NewLoadedVT, LD->isVolatile(), 544 LD->isNonTemporal(), MinAlign(Alignment,IncrementSize)); 545 } 546 547 // aggregate the two parts 548 SDValue ShiftAmount = DAG.getConstant(NumBits, 549 TLI.getShiftAmountTy(Hi.getValueType())); 550 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); 551 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); 552 553 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 554 Hi.getValue(1)); 555 556 ValResult = Result; 557 ChainResult = TF; 558} 559 560/// PerformInsertVectorEltInMemory - Some target cannot handle a variable 561/// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 562/// is necessary to spill the vector being inserted into to memory, perform 563/// the insert there, and then read the result back. 564SDValue SelectionDAGLegalize:: 565PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 566 DebugLoc dl) { 567 SDValue Tmp1 = Vec; 568 SDValue Tmp2 = Val; 569 SDValue Tmp3 = Idx; 570 571 // If the target doesn't support this, we have to spill the input vector 572 // to a temporary stack slot, update the element, then reload it. This is 573 // badness. We could also load the value into a vector register (either 574 // with a "move to register" or "extload into register" instruction, then 575 // permute it into place, if the idx is a constant and if the idx is 576 // supported by the target. 577 EVT VT = Tmp1.getValueType(); 578 EVT EltVT = VT.getVectorElementType(); 579 EVT IdxVT = Tmp3.getValueType(); 580 EVT PtrVT = TLI.getPointerTy(); 581 SDValue StackPtr = DAG.CreateStackTemporary(VT); 582 583 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 584 585 // Store the vector. 586 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr, 587 MachinePointerInfo::getFixedStack(SPFI), 588 false, false, 0); 589 590 // Truncate or zero extend offset to target pointer type. 591 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 592 Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3); 593 // Add the offset to the index. 594 unsigned EltSize = EltVT.getSizeInBits()/8; 595 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); 596 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); 597 // Store the scalar value. 598 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT, 599 false, false, 0); 600 // Load the updated vector. 601 return DAG.getLoad(VT, dl, Ch, StackPtr, 602 MachinePointerInfo::getFixedStack(SPFI), false, false, 603 false, 0); 604} 605 606 607SDValue SelectionDAGLegalize:: 608ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, DebugLoc dl) { 609 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { 610 // SCALAR_TO_VECTOR requires that the type of the value being inserted 611 // match the element type of the vector being created, except for 612 // integers in which case the inserted value can be over width. 613 EVT EltVT = Vec.getValueType().getVectorElementType(); 614 if (Val.getValueType() == EltVT || 615 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { 616 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 617 Vec.getValueType(), Val); 618 619 unsigned NumElts = Vec.getValueType().getVectorNumElements(); 620 // We generate a shuffle of InVec and ScVec, so the shuffle mask 621 // should be 0,1,2,3,4,5... with the appropriate element replaced with 622 // elt 0 of the RHS. 623 SmallVector<int, 8> ShufOps; 624 for (unsigned i = 0; i != NumElts; ++i) 625 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); 626 627 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, 628 &ShufOps[0]); 629 } 630 } 631 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); 632} 633 634SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { 635 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 636 // FIXME: We shouldn't do this for TargetConstantFP's. 637 // FIXME: move this to the DAG Combiner! Note that we can't regress due 638 // to phase ordering between legalized code and the dag combiner. This 639 // probably means that we need to integrate dag combiner and legalizer 640 // together. 641 // We generally can't do this one for long doubles. 642 SDValue Tmp1 = ST->getChain(); 643 SDValue Tmp2 = ST->getBasePtr(); 644 SDValue Tmp3; 645 unsigned Alignment = ST->getAlignment(); 646 bool isVolatile = ST->isVolatile(); 647 bool isNonTemporal = ST->isNonTemporal(); 648 DebugLoc dl = ST->getDebugLoc(); 649 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 650 if (CFP->getValueType(0) == MVT::f32 && 651 TLI.isTypeLegal(MVT::i32)) { 652 Tmp3 = DAG.getConstant(CFP->getValueAPF(). 653 bitcastToAPInt().zextOrTrunc(32), 654 MVT::i32); 655 return DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), 656 isVolatile, isNonTemporal, Alignment); 657 } 658 659 if (CFP->getValueType(0) == MVT::f64) { 660 // If this target supports 64-bit registers, do a single 64-bit store. 661 if (TLI.isTypeLegal(MVT::i64)) { 662 Tmp3 = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 663 zextOrTrunc(64), MVT::i64); 664 return DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), 665 isVolatile, isNonTemporal, Alignment); 666 } 667 668 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { 669 // Otherwise, if the target supports 32-bit registers, use 2 32-bit 670 // stores. If the target supports neither 32- nor 64-bits, this 671 // xform is certainly not worth it. 672 const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); 673 SDValue Lo = DAG.getConstant(IntVal.trunc(32), MVT::i32); 674 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); 675 if (TLI.isBigEndian()) std::swap(Lo, Hi); 676 677 Lo = DAG.getStore(Tmp1, dl, Lo, Tmp2, ST->getPointerInfo(), isVolatile, 678 isNonTemporal, Alignment); 679 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 680 DAG.getIntPtrConstant(4)); 681 Hi = DAG.getStore(Tmp1, dl, Hi, Tmp2, 682 ST->getPointerInfo().getWithOffset(4), 683 isVolatile, isNonTemporal, MinAlign(Alignment, 4U)); 684 685 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 686 } 687 } 688 } 689 return SDValue(0, 0); 690} 691 692/// LegalizeOp - Return a legal replacement for the given operation, with 693/// all legal operands. 694void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { 695 if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. 696 return; 697 698 DebugLoc dl = Node->getDebugLoc(); 699 700 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 701 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == 702 TargetLowering::TypeLegal && 703 "Unexpected illegal type!"); 704 705 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 706 assert((TLI.getTypeAction(*DAG.getContext(), 707 Node->getOperand(i).getValueType()) == 708 TargetLowering::TypeLegal || 709 Node->getOperand(i).getOpcode() == ISD::TargetConstant) && 710 "Unexpected illegal type!"); 711 712 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 713 bool isCustom = false; 714 715 // Figure out the correct action; the way to query this varies by opcode 716 TargetLowering::LegalizeAction Action = TargetLowering::Legal; 717 bool SimpleFinishLegalizing = true; 718 switch (Node->getOpcode()) { 719 case ISD::INTRINSIC_W_CHAIN: 720 case ISD::INTRINSIC_WO_CHAIN: 721 case ISD::INTRINSIC_VOID: 722 case ISD::STACKSAVE: 723 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 724 break; 725 case ISD::VAARG: 726 Action = TLI.getOperationAction(Node->getOpcode(), 727 Node->getValueType(0)); 728 if (Action != TargetLowering::Promote) 729 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 730 break; 731 case ISD::SINT_TO_FP: 732 case ISD::UINT_TO_FP: 733 case ISD::EXTRACT_VECTOR_ELT: 734 Action = TLI.getOperationAction(Node->getOpcode(), 735 Node->getOperand(0).getValueType()); 736 break; 737 case ISD::FP_ROUND_INREG: 738 case ISD::SIGN_EXTEND_INREG: { 739 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 740 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); 741 break; 742 } 743 case ISD::ATOMIC_STORE: { 744 Action = TLI.getOperationAction(Node->getOpcode(), 745 Node->getOperand(2).getValueType()); 746 break; 747 } 748 case ISD::SELECT_CC: 749 case ISD::SETCC: 750 case ISD::BR_CC: { 751 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : 752 Node->getOpcode() == ISD::SETCC ? 2 : 1; 753 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; 754 EVT OpVT = Node->getOperand(CompareOperand).getValueType(); 755 ISD::CondCode CCCode = 756 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); 757 Action = TLI.getCondCodeAction(CCCode, OpVT); 758 if (Action == TargetLowering::Legal) { 759 if (Node->getOpcode() == ISD::SELECT_CC) 760 Action = TLI.getOperationAction(Node->getOpcode(), 761 Node->getValueType(0)); 762 else 763 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); 764 } 765 break; 766 } 767 case ISD::LOAD: 768 case ISD::STORE: 769 // FIXME: Model these properly. LOAD and STORE are complicated, and 770 // STORE expects the unlegalized operand in some cases. 771 SimpleFinishLegalizing = false; 772 break; 773 case ISD::CALLSEQ_START: 774 case ISD::CALLSEQ_END: 775 // FIXME: This shouldn't be necessary. These nodes have special properties 776 // dealing with the recursive nature of legalization. Removing this 777 // special case should be done as part of making LegalizeDAG non-recursive. 778 SimpleFinishLegalizing = false; 779 break; 780 case ISD::EXTRACT_ELEMENT: 781 case ISD::FLT_ROUNDS_: 782 case ISD::SADDO: 783 case ISD::SSUBO: 784 case ISD::UADDO: 785 case ISD::USUBO: 786 case ISD::SMULO: 787 case ISD::UMULO: 788 case ISD::FPOWI: 789 case ISD::MERGE_VALUES: 790 case ISD::EH_RETURN: 791 case ISD::FRAME_TO_ARGS_OFFSET: 792 case ISD::EH_SJLJ_SETJMP: 793 case ISD::EH_SJLJ_LONGJMP: 794 // These operations lie about being legal: when they claim to be legal, 795 // they should actually be expanded. 796 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 797 if (Action == TargetLowering::Legal) 798 Action = TargetLowering::Expand; 799 break; 800 case ISD::INIT_TRAMPOLINE: 801 case ISD::ADJUST_TRAMPOLINE: 802 case ISD::FRAMEADDR: 803 case ISD::RETURNADDR: 804 // These operations lie about being legal: when they claim to be legal, 805 // they should actually be custom-lowered. 806 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 807 if (Action == TargetLowering::Legal) 808 Action = TargetLowering::Custom; 809 break; 810 default: 811 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 812 Action = TargetLowering::Legal; 813 } else { 814 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 815 } 816 break; 817 } 818 819 if (SimpleFinishLegalizing) { 820 SmallVector<SDValue, 8> Ops; 821 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 822 Ops.push_back(Node->getOperand(i)); 823 switch (Node->getOpcode()) { 824 default: break; 825 case ISD::SHL: 826 case ISD::SRL: 827 case ISD::SRA: 828 case ISD::ROTL: 829 case ISD::ROTR: 830 // Legalizing shifts/rotates requires adjusting the shift amount 831 // to the appropriate width. 832 if (!Ops[1].getValueType().isVector()) { 833 SDValue SAO = DAG.getShiftAmountOperand(Ops[0].getValueType(), Ops[1]); 834 HandleSDNode Handle(SAO); 835 LegalizeOp(SAO.getNode()); 836 Ops[1] = Handle.getValue(); 837 } 838 break; 839 case ISD::SRL_PARTS: 840 case ISD::SRA_PARTS: 841 case ISD::SHL_PARTS: 842 // Legalizing shifts/rotates requires adjusting the shift amount 843 // to the appropriate width. 844 if (!Ops[2].getValueType().isVector()) { 845 SDValue SAO = DAG.getShiftAmountOperand(Ops[0].getValueType(), Ops[2]); 846 HandleSDNode Handle(SAO); 847 LegalizeOp(SAO.getNode()); 848 Ops[2] = Handle.getValue(); 849 } 850 break; 851 } 852 853 SDNode *NewNode = DAG.UpdateNodeOperands(Node, Ops.data(), Ops.size()); 854 if (NewNode != Node) { 855 DAG.ReplaceAllUsesWith(Node, NewNode); 856 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 857 DAG.TransferDbgValues(SDValue(Node, i), SDValue(NewNode, i)); 858 ReplacedNode(Node); 859 Node = NewNode; 860 } 861 switch (Action) { 862 case TargetLowering::Legal: 863 return; 864 case TargetLowering::Custom: 865 // FIXME: The handling for custom lowering with multiple results is 866 // a complete mess. 867 Tmp1 = TLI.LowerOperation(SDValue(Node, 0), DAG); 868 if (Tmp1.getNode()) { 869 SmallVector<SDValue, 8> ResultVals; 870 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { 871 if (e == 1) 872 ResultVals.push_back(Tmp1); 873 else 874 ResultVals.push_back(Tmp1.getValue(i)); 875 } 876 if (Tmp1.getNode() != Node || Tmp1.getResNo() != 0) { 877 DAG.ReplaceAllUsesWith(Node, ResultVals.data()); 878 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 879 DAG.TransferDbgValues(SDValue(Node, i), ResultVals[i]); 880 ReplacedNode(Node); 881 } 882 return; 883 } 884 885 // FALL THROUGH 886 case TargetLowering::Expand: 887 ExpandNode(Node); 888 return; 889 case TargetLowering::Promote: 890 PromoteNode(Node); 891 return; 892 } 893 } 894 895 switch (Node->getOpcode()) { 896 default: 897#ifndef NDEBUG 898 dbgs() << "NODE: "; 899 Node->dump( &DAG); 900 dbgs() << "\n"; 901#endif 902 llvm_unreachable("Do not know how to legalize this operator!"); 903 904 case ISD::CALLSEQ_START: 905 case ISD::CALLSEQ_END: 906 break; 907 case ISD::LOAD: { 908 LoadSDNode *LD = cast<LoadSDNode>(Node); 909 Tmp1 = LD->getChain(); // Legalize the chain. 910 Tmp2 = LD->getBasePtr(); // Legalize the base pointer. 911 912 ISD::LoadExtType ExtType = LD->getExtensionType(); 913 if (ExtType == ISD::NON_EXTLOAD) { 914 EVT VT = Node->getValueType(0); 915 Tmp3 = SDValue(Node, 0); 916 Tmp4 = SDValue(Node, 1); 917 918 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 919 default: llvm_unreachable("This action is not supported yet!"); 920 case TargetLowering::Legal: 921 // If this is an unaligned load and the target doesn't support it, 922 // expand it. 923 if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { 924 Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 925 unsigned ABIAlignment = TLI.getTargetData()->getABITypeAlignment(Ty); 926 if (LD->getAlignment() < ABIAlignment){ 927 ExpandUnalignedLoad(cast<LoadSDNode>(Node), 928 DAG, TLI, Tmp3, Tmp4); 929 } 930 } 931 break; 932 case TargetLowering::Custom: 933 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 934 if (Tmp1.getNode()) { 935 Tmp3 = Tmp1; 936 Tmp4 = Tmp1.getValue(1); 937 } 938 break; 939 case TargetLowering::Promote: { 940 // Only promote a load of vector type to another. 941 assert(VT.isVector() && "Cannot promote this load!"); 942 // Change base type to a different vector type. 943 EVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 944 945 Tmp1 = DAG.getLoad(NVT, dl, Tmp1, Tmp2, LD->getPointerInfo(), 946 LD->isVolatile(), LD->isNonTemporal(), 947 LD->isInvariant(), LD->getAlignment()); 948 Tmp3 = DAG.getNode(ISD::BITCAST, dl, VT, Tmp1); 949 Tmp4 = Tmp1.getValue(1); 950 break; 951 } 952 } 953 if (Tmp4.getNode() != Node) { 954 assert(Tmp3.getNode() != Node && "Load must be completely replaced"); 955 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp3); 956 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Tmp4); 957 ReplacedNode(Node); 958 } 959 return; 960 } 961 962 EVT SrcVT = LD->getMemoryVT(); 963 unsigned SrcWidth = SrcVT.getSizeInBits(); 964 unsigned Alignment = LD->getAlignment(); 965 bool isVolatile = LD->isVolatile(); 966 bool isNonTemporal = LD->isNonTemporal(); 967 968 if (SrcWidth != SrcVT.getStoreSizeInBits() && 969 // Some targets pretend to have an i1 loading operation, and actually 970 // load an i8. This trick is correct for ZEXTLOAD because the top 7 971 // bits are guaranteed to be zero; it helps the optimizers understand 972 // that these bits are zero. It is also useful for EXTLOAD, since it 973 // tells the optimizers that those bits are undefined. It would be 974 // nice to have an effective generic way of getting these benefits... 975 // Until such a way is found, don't insist on promoting i1 here. 976 (SrcVT != MVT::i1 || 977 TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { 978 // Promote to a byte-sized load if not loading an integral number of 979 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 980 unsigned NewWidth = SrcVT.getStoreSizeInBits(); 981 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); 982 SDValue Ch; 983 984 // The extra bits are guaranteed to be zero, since we stored them that 985 // way. A zext load from NVT thus automatically gives zext from SrcVT. 986 987 ISD::LoadExtType NewExtType = 988 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 989 990 SDValue Result = 991 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), 992 Tmp1, Tmp2, LD->getPointerInfo(), 993 NVT, isVolatile, isNonTemporal, Alignment); 994 995 Ch = Result.getValue(1); // The chain. 996 997 if (ExtType == ISD::SEXTLOAD) 998 // Having the top bits zero doesn't help when sign extending. 999 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 1000 Result.getValueType(), 1001 Result, DAG.getValueType(SrcVT)); 1002 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 1003 // All the top bits are guaranteed to be zero - inform the optimizers. 1004 Result = DAG.getNode(ISD::AssertZext, dl, 1005 Result.getValueType(), Result, 1006 DAG.getValueType(SrcVT)); 1007 1008 Tmp1 = Result; 1009 Tmp2 = Ch; 1010 } else if (SrcWidth & (SrcWidth - 1)) { 1011 // If not loading a power-of-2 number of bits, expand as two loads. 1012 assert(!SrcVT.isVector() && "Unsupported extload!"); 1013 unsigned RoundWidth = 1 << Log2_32(SrcWidth); 1014 assert(RoundWidth < SrcWidth); 1015 unsigned ExtraWidth = SrcWidth - RoundWidth; 1016 assert(ExtraWidth < RoundWidth); 1017 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 1018 "Load size not an integral number of bytes!"); 1019 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 1020 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 1021 SDValue Lo, Hi, Ch; 1022 unsigned IncrementSize; 1023 1024 if (TLI.isLittleEndian()) { 1025 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 1026 // Load the bottom RoundWidth bits. 1027 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), 1028 Tmp1, Tmp2, 1029 LD->getPointerInfo(), RoundVT, isVolatile, 1030 isNonTemporal, Alignment); 1031 1032 // Load the remaining ExtraWidth bits. 1033 IncrementSize = RoundWidth / 8; 1034 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 1035 DAG.getIntPtrConstant(IncrementSize)); 1036 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Tmp1, Tmp2, 1037 LD->getPointerInfo().getWithOffset(IncrementSize), 1038 ExtraVT, isVolatile, isNonTemporal, 1039 MinAlign(Alignment, IncrementSize)); 1040 1041 // Build a factor node to remember that this load is independent of 1042 // the other one. 1043 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1044 Hi.getValue(1)); 1045 1046 // Move the top bits to the right place. 1047 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1048 DAG.getConstant(RoundWidth, 1049 TLI.getShiftAmountTy(Hi.getValueType()))); 1050 1051 // Join the hi and lo parts. 1052 Tmp1 = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1053 } else { 1054 // Big endian - avoid unaligned loads. 1055 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 1056 // Load the top RoundWidth bits. 1057 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Tmp1, Tmp2, 1058 LD->getPointerInfo(), RoundVT, isVolatile, 1059 isNonTemporal, Alignment); 1060 1061 // Load the remaining ExtraWidth bits. 1062 IncrementSize = RoundWidth / 8; 1063 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 1064 DAG.getIntPtrConstant(IncrementSize)); 1065 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, 1066 dl, Node->getValueType(0), Tmp1, Tmp2, 1067 LD->getPointerInfo().getWithOffset(IncrementSize), 1068 ExtraVT, isVolatile, isNonTemporal, 1069 MinAlign(Alignment, IncrementSize)); 1070 1071 // Build a factor node to remember that this load is independent of 1072 // the other one. 1073 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1074 Hi.getValue(1)); 1075 1076 // Move the top bits to the right place. 1077 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1078 DAG.getConstant(ExtraWidth, 1079 TLI.getShiftAmountTy(Hi.getValueType()))); 1080 1081 // Join the hi and lo parts. 1082 Tmp1 = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1083 } 1084 1085 Tmp2 = Ch; 1086 } else { 1087 switch (TLI.getLoadExtAction(ExtType, SrcVT)) { 1088 default: llvm_unreachable("This action is not supported yet!"); 1089 case TargetLowering::Custom: 1090 isCustom = true; 1091 // FALLTHROUGH 1092 case TargetLowering::Legal: 1093 Tmp1 = SDValue(Node, 0); 1094 Tmp2 = SDValue(Node, 1); 1095 1096 if (isCustom) { 1097 Tmp3 = TLI.LowerOperation(SDValue(Node, 0), DAG); 1098 if (Tmp3.getNode()) { 1099 Tmp1 = Tmp3; 1100 Tmp2 = Tmp3.getValue(1); 1101 } 1102 } else { 1103 // If this is an unaligned load and the target doesn't support it, 1104 // expand it. 1105 if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { 1106 Type *Ty = 1107 LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 1108 unsigned ABIAlignment = 1109 TLI.getTargetData()->getABITypeAlignment(Ty); 1110 if (LD->getAlignment() < ABIAlignment){ 1111 ExpandUnalignedLoad(cast<LoadSDNode>(Node), 1112 DAG, TLI, Tmp1, Tmp2); 1113 } 1114 } 1115 } 1116 break; 1117 case TargetLowering::Expand: 1118 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, SrcVT) && TLI.isTypeLegal(SrcVT)) { 1119 SDValue Load = DAG.getLoad(SrcVT, dl, Tmp1, Tmp2, 1120 LD->getPointerInfo(), 1121 LD->isVolatile(), LD->isNonTemporal(), 1122 LD->isInvariant(), LD->getAlignment()); 1123 unsigned ExtendOp; 1124 switch (ExtType) { 1125 case ISD::EXTLOAD: 1126 ExtendOp = (SrcVT.isFloatingPoint() ? 1127 ISD::FP_EXTEND : ISD::ANY_EXTEND); 1128 break; 1129 case ISD::SEXTLOAD: ExtendOp = ISD::SIGN_EXTEND; break; 1130 case ISD::ZEXTLOAD: ExtendOp = ISD::ZERO_EXTEND; break; 1131 default: llvm_unreachable("Unexpected extend load type!"); 1132 } 1133 Tmp1 = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); 1134 Tmp2 = Load.getValue(1); 1135 break; 1136 } 1137 1138 assert(!SrcVT.isVector() && 1139 "Vector Loads are handled in LegalizeVectorOps"); 1140 1141 // FIXME: This does not work for vectors on most targets. Sign- and 1142 // zero-extend operations are currently folded into extending loads, 1143 // whether they are legal or not, and then we end up here without any 1144 // support for legalizing them. 1145 assert(ExtType != ISD::EXTLOAD && 1146 "EXTLOAD should always be supported!"); 1147 // Turn the unsupported load into an EXTLOAD followed by an explicit 1148 // zero/sign extend inreg. 1149 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0), 1150 Tmp1, Tmp2, LD->getPointerInfo(), SrcVT, 1151 LD->isVolatile(), LD->isNonTemporal(), 1152 LD->getAlignment()); 1153 SDValue ValRes; 1154 if (ExtType == ISD::SEXTLOAD) 1155 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 1156 Result.getValueType(), 1157 Result, DAG.getValueType(SrcVT)); 1158 else 1159 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); 1160 Tmp1 = ValRes; 1161 Tmp2 = Result.getValue(1); 1162 break; 1163 } 1164 } 1165 1166 // Since loads produce two values, make sure to remember that we legalized 1167 // both of them. 1168 if (Tmp2.getNode() != Node) { 1169 assert(Tmp1.getNode() != Node && "Load must be completely replaced"); 1170 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp1); 1171 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Tmp2); 1172 ReplacedNode(Node); 1173 } 1174 break; 1175 } 1176 case ISD::STORE: { 1177 StoreSDNode *ST = cast<StoreSDNode>(Node); 1178 Tmp1 = ST->getChain(); 1179 Tmp2 = ST->getBasePtr(); 1180 unsigned Alignment = ST->getAlignment(); 1181 bool isVolatile = ST->isVolatile(); 1182 bool isNonTemporal = ST->isNonTemporal(); 1183 1184 if (!ST->isTruncatingStore()) { 1185 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { 1186 ReplaceNode(ST, OptStore); 1187 break; 1188 } 1189 1190 { 1191 Tmp3 = ST->getValue(); 1192 EVT VT = Tmp3.getValueType(); 1193 switch (TLI.getOperationAction(ISD::STORE, VT)) { 1194 default: llvm_unreachable("This action is not supported yet!"); 1195 case TargetLowering::Legal: 1196 // If this is an unaligned store and the target doesn't support it, 1197 // expand it. 1198 if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { 1199 Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 1200 unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty); 1201 if (ST->getAlignment() < ABIAlignment) 1202 ExpandUnalignedStore(cast<StoreSDNode>(Node), 1203 DAG, TLI, this); 1204 } 1205 break; 1206 case TargetLowering::Custom: 1207 Tmp1 = TLI.LowerOperation(SDValue(Node, 0), DAG); 1208 if (Tmp1.getNode()) 1209 ReplaceNode(SDValue(Node, 0), Tmp1); 1210 break; 1211 case TargetLowering::Promote: { 1212 assert(VT.isVector() && "Unknown legal promote case!"); 1213 Tmp3 = DAG.getNode(ISD::BITCAST, dl, 1214 TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3); 1215 SDValue Result = 1216 DAG.getStore(Tmp1, dl, Tmp3, Tmp2, 1217 ST->getPointerInfo(), isVolatile, 1218 isNonTemporal, Alignment); 1219 ReplaceNode(SDValue(Node, 0), Result); 1220 break; 1221 } 1222 } 1223 break; 1224 } 1225 } else { 1226 Tmp3 = ST->getValue(); 1227 1228 EVT StVT = ST->getMemoryVT(); 1229 unsigned StWidth = StVT.getSizeInBits(); 1230 1231 if (StWidth != StVT.getStoreSizeInBits()) { 1232 // Promote to a byte-sized store with upper bits zero if not 1233 // storing an integral number of bytes. For example, promote 1234 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 1235 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), 1236 StVT.getStoreSizeInBits()); 1237 Tmp3 = DAG.getZeroExtendInReg(Tmp3, dl, StVT); 1238 SDValue Result = 1239 DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), 1240 NVT, isVolatile, isNonTemporal, Alignment); 1241 ReplaceNode(SDValue(Node, 0), Result); 1242 } else if (StWidth & (StWidth - 1)) { 1243 // If not storing a power-of-2 number of bits, expand as two stores. 1244 assert(!StVT.isVector() && "Unsupported truncstore!"); 1245 unsigned RoundWidth = 1 << Log2_32(StWidth); 1246 assert(RoundWidth < StWidth); 1247 unsigned ExtraWidth = StWidth - RoundWidth; 1248 assert(ExtraWidth < RoundWidth); 1249 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 1250 "Store size not an integral number of bytes!"); 1251 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 1252 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 1253 SDValue Lo, Hi; 1254 unsigned IncrementSize; 1255 1256 if (TLI.isLittleEndian()) { 1257 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 1258 // Store the bottom RoundWidth bits. 1259 Lo = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), 1260 RoundVT, 1261 isVolatile, isNonTemporal, Alignment); 1262 1263 // Store the remaining ExtraWidth bits. 1264 IncrementSize = RoundWidth / 8; 1265 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 1266 DAG.getIntPtrConstant(IncrementSize)); 1267 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3, 1268 DAG.getConstant(RoundWidth, 1269 TLI.getShiftAmountTy(Tmp3.getValueType()))); 1270 Hi = DAG.getTruncStore(Tmp1, dl, Hi, Tmp2, 1271 ST->getPointerInfo().getWithOffset(IncrementSize), 1272 ExtraVT, isVolatile, isNonTemporal, 1273 MinAlign(Alignment, IncrementSize)); 1274 } else { 1275 // Big endian - avoid unaligned stores. 1276 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 1277 // Store the top RoundWidth bits. 1278 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3, 1279 DAG.getConstant(ExtraWidth, 1280 TLI.getShiftAmountTy(Tmp3.getValueType()))); 1281 Hi = DAG.getTruncStore(Tmp1, dl, Hi, Tmp2, ST->getPointerInfo(), 1282 RoundVT, isVolatile, isNonTemporal, Alignment); 1283 1284 // Store the remaining ExtraWidth bits. 1285 IncrementSize = RoundWidth / 8; 1286 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 1287 DAG.getIntPtrConstant(IncrementSize)); 1288 Lo = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, 1289 ST->getPointerInfo().getWithOffset(IncrementSize), 1290 ExtraVT, isVolatile, isNonTemporal, 1291 MinAlign(Alignment, IncrementSize)); 1292 } 1293 1294 // The order of the stores doesn't matter. 1295 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1296 ReplaceNode(SDValue(Node, 0), Result); 1297 } else { 1298 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { 1299 default: llvm_unreachable("This action is not supported yet!"); 1300 case TargetLowering::Legal: 1301 // If this is an unaligned store and the target doesn't support it, 1302 // expand it. 1303 if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { 1304 Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 1305 unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty); 1306 if (ST->getAlignment() < ABIAlignment) 1307 ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this); 1308 } 1309 break; 1310 case TargetLowering::Custom: 1311 ReplaceNode(SDValue(Node, 0), 1312 TLI.LowerOperation(SDValue(Node, 0), DAG)); 1313 break; 1314 case TargetLowering::Expand: 1315 assert(!StVT.isVector() && 1316 "Vector Stores are handled in LegalizeVectorOps"); 1317 1318 // TRUNCSTORE:i16 i32 -> STORE i16 1319 assert(TLI.isTypeLegal(StVT) && "Do not know how to expand this store!"); 1320 Tmp3 = DAG.getNode(ISD::TRUNCATE, dl, StVT, Tmp3); 1321 SDValue Result = 1322 DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), 1323 isVolatile, isNonTemporal, Alignment); 1324 ReplaceNode(SDValue(Node, 0), Result); 1325 break; 1326 } 1327 } 1328 } 1329 break; 1330 } 1331 } 1332} 1333 1334SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { 1335 SDValue Vec = Op.getOperand(0); 1336 SDValue Idx = Op.getOperand(1); 1337 DebugLoc dl = Op.getDebugLoc(); 1338 // Store the value to a temporary stack slot, then LOAD the returned part. 1339 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1340 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1341 MachinePointerInfo(), false, false, 0); 1342 1343 // Add the offset to the index. 1344 unsigned EltSize = 1345 Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1346 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1347 DAG.getConstant(EltSize, Idx.getValueType())); 1348 1349 if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 1350 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx); 1351 else 1352 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 1353 1354 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); 1355 1356 if (Op.getValueType().isVector()) 1357 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,MachinePointerInfo(), 1358 false, false, false, 0); 1359 return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, 1360 MachinePointerInfo(), 1361 Vec.getValueType().getVectorElementType(), 1362 false, false, 0); 1363} 1364 1365SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { 1366 assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); 1367 1368 SDValue Vec = Op.getOperand(0); 1369 SDValue Part = Op.getOperand(1); 1370 SDValue Idx = Op.getOperand(2); 1371 DebugLoc dl = Op.getDebugLoc(); 1372 1373 // Store the value to a temporary stack slot, then LOAD the returned part. 1374 1375 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1376 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1377 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1378 1379 // First store the whole vector. 1380 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1381 false, false, 0); 1382 1383 // Then store the inserted part. 1384 1385 // Add the offset to the index. 1386 unsigned EltSize = 1387 Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1388 1389 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1390 DAG.getConstant(EltSize, Idx.getValueType())); 1391 1392 if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 1393 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx); 1394 else 1395 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 1396 1397 SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1398 StackPtr); 1399 1400 // Store the subvector. 1401 Ch = DAG.getStore(DAG.getEntryNode(), dl, Part, SubStackPtr, 1402 MachinePointerInfo(), false, false, 0); 1403 1404 // Finally, load the updated vector. 1405 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo, 1406 false, false, false, 0); 1407} 1408 1409SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { 1410 // We can't handle this case efficiently. Allocate a sufficiently 1411 // aligned object on the stack, store each element into it, then load 1412 // the result as a vector. 1413 // Create the stack frame object. 1414 EVT VT = Node->getValueType(0); 1415 EVT EltVT = VT.getVectorElementType(); 1416 DebugLoc dl = Node->getDebugLoc(); 1417 SDValue FIPtr = DAG.CreateStackTemporary(VT); 1418 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1419 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1420 1421 // Emit a store of each element to the stack slot. 1422 SmallVector<SDValue, 8> Stores; 1423 unsigned TypeByteSize = EltVT.getSizeInBits() / 8; 1424 // Store (in the right endianness) the elements to memory. 1425 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1426 // Ignore undef elements. 1427 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1428 1429 unsigned Offset = TypeByteSize*i; 1430 1431 SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); 1432 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); 1433 1434 // If the destination vector element type is narrower than the source 1435 // element type, only store the bits necessary. 1436 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { 1437 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, 1438 Node->getOperand(i), Idx, 1439 PtrInfo.getWithOffset(Offset), 1440 EltVT, false, false, 0)); 1441 } else 1442 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, 1443 Node->getOperand(i), Idx, 1444 PtrInfo.getWithOffset(Offset), 1445 false, false, 0)); 1446 } 1447 1448 SDValue StoreChain; 1449 if (!Stores.empty()) // Not all undef elements? 1450 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1451 &Stores[0], Stores.size()); 1452 else 1453 StoreChain = DAG.getEntryNode(); 1454 1455 // Result is a load from the stack slot. 1456 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo, 1457 false, false, false, 0); 1458} 1459 1460SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) { 1461 DebugLoc dl = Node->getDebugLoc(); 1462 SDValue Tmp1 = Node->getOperand(0); 1463 SDValue Tmp2 = Node->getOperand(1); 1464 1465 // Get the sign bit of the RHS. First obtain a value that has the same 1466 // sign as the sign bit, i.e. negative if and only if the sign bit is 1. 1467 SDValue SignBit; 1468 EVT FloatVT = Tmp2.getValueType(); 1469 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), FloatVT.getSizeInBits()); 1470 if (TLI.isTypeLegal(IVT)) { 1471 // Convert to an integer with the same sign bit. 1472 SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2); 1473 } else { 1474 // Store the float to memory, then load the sign part out as an integer. 1475 MVT LoadTy = TLI.getPointerTy(); 1476 // First create a temporary that is aligned for both the load and store. 1477 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); 1478 // Then store the float to it. 1479 SDValue Ch = 1480 DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(), 1481 false, false, 0); 1482 if (TLI.isBigEndian()) { 1483 assert(FloatVT.isByteSized() && "Unsupported floating point type!"); 1484 // Load out a legal integer with the same sign bit as the float. 1485 SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(), 1486 false, false, false, 0); 1487 } else { // Little endian 1488 SDValue LoadPtr = StackPtr; 1489 // The float may be wider than the integer we are going to load. Advance 1490 // the pointer so that the loaded integer will contain the sign bit. 1491 unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits(); 1492 unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8; 1493 LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), 1494 LoadPtr, DAG.getIntPtrConstant(ByteOffset)); 1495 // Load a legal integer containing the sign bit. 1496 SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(), 1497 false, false, false, 0); 1498 // Move the sign bit to the top bit of the loaded integer. 1499 unsigned BitShift = LoadTy.getSizeInBits() - 1500 (FloatVT.getSizeInBits() - 8 * ByteOffset); 1501 assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?"); 1502 if (BitShift) 1503 SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit, 1504 DAG.getConstant(BitShift, 1505 TLI.getShiftAmountTy(SignBit.getValueType()))); 1506 } 1507 } 1508 // Now get the sign bit proper, by seeing whether the value is negative. 1509 SignBit = DAG.getSetCC(dl, TLI.getSetCCResultType(SignBit.getValueType()), 1510 SignBit, DAG.getConstant(0, SignBit.getValueType()), 1511 ISD::SETLT); 1512 // Get the absolute value of the result. 1513 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); 1514 // Select between the nabs and abs value based on the sign bit of 1515 // the input. 1516 return DAG.getNode(ISD::SELECT, dl, AbsVal.getValueType(), SignBit, 1517 DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal), 1518 AbsVal); 1519} 1520 1521void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, 1522 SmallVectorImpl<SDValue> &Results) { 1523 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1524 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1525 " not tell us which reg is the stack pointer!"); 1526 DebugLoc dl = Node->getDebugLoc(); 1527 EVT VT = Node->getValueType(0); 1528 SDValue Tmp1 = SDValue(Node, 0); 1529 SDValue Tmp2 = SDValue(Node, 1); 1530 SDValue Tmp3 = Node->getOperand(2); 1531 SDValue Chain = Tmp1.getOperand(0); 1532 1533 // Chain the dynamic stack allocation so that it doesn't modify the stack 1534 // pointer when other instructions are using the stack. 1535 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); 1536 1537 SDValue Size = Tmp2.getOperand(1); 1538 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1539 Chain = SP.getValue(1); 1540 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1541 unsigned StackAlign = TM.getFrameLowering()->getStackAlignment(); 1542 if (Align > StackAlign) 1543 SP = DAG.getNode(ISD::AND, dl, VT, SP, 1544 DAG.getConstant(-(uint64_t)Align, VT)); 1545 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value 1546 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1547 1548 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), 1549 DAG.getIntPtrConstant(0, true), SDValue()); 1550 1551 Results.push_back(Tmp1); 1552 Results.push_back(Tmp2); 1553} 1554 1555/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and 1556/// condition code CC on the current target. This routine expands SETCC with 1557/// illegal condition code into AND / OR of multiple SETCC values. 1558void SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, 1559 SDValue &LHS, SDValue &RHS, 1560 SDValue &CC, 1561 DebugLoc dl) { 1562 EVT OpVT = LHS.getValueType(); 1563 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 1564 switch (TLI.getCondCodeAction(CCCode, OpVT)) { 1565 default: llvm_unreachable("Unknown condition code action!"); 1566 case TargetLowering::Legal: 1567 // Nothing to do. 1568 break; 1569 case TargetLowering::Expand: { 1570 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 1571 unsigned Opc = 0; 1572 switch (CCCode) { 1573 default: llvm_unreachable("Don't know how to expand this condition!"); 1574 case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break; 1575 case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break; 1576 case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break; 1577 case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break; 1578 case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break; 1579 case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break; 1580 case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break; 1581 case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break; 1582 case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 1583 case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break; 1584 case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 1585 case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 1586 // FIXME: Implement more expansions. 1587 } 1588 1589 SDValue SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); 1590 SDValue SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); 1591 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 1592 RHS = SDValue(); 1593 CC = SDValue(); 1594 break; 1595 } 1596 } 1597} 1598 1599/// EmitStackConvert - Emit a store/load combination to the stack. This stores 1600/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 1601/// a load from the stack slot to DestVT, extending it if needed. 1602/// The resultant code need not be legal. 1603SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, 1604 EVT SlotVT, 1605 EVT DestVT, 1606 DebugLoc dl) { 1607 // Create the stack frame object. 1608 unsigned SrcAlign = 1609 TLI.getTargetData()->getPrefTypeAlignment(SrcOp.getValueType(). 1610 getTypeForEVT(*DAG.getContext())); 1611 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 1612 1613 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 1614 int SPFI = StackPtrFI->getIndex(); 1615 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI); 1616 1617 unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); 1618 unsigned SlotSize = SlotVT.getSizeInBits(); 1619 unsigned DestSize = DestVT.getSizeInBits(); 1620 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); 1621 unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment(DestType); 1622 1623 // Emit a store to the stack slot. Use a truncstore if the input value is 1624 // later than DestVT. 1625 SDValue Store; 1626 1627 if (SrcSize > SlotSize) 1628 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1629 PtrInfo, SlotVT, false, false, SrcAlign); 1630 else { 1631 assert(SrcSize == SlotSize && "Invalid store"); 1632 Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1633 PtrInfo, false, false, SrcAlign); 1634 } 1635 1636 // Result is a load from the stack slot. 1637 if (SlotSize == DestSize) 1638 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, 1639 false, false, false, DestAlign); 1640 1641 assert(SlotSize < DestSize && "Unknown extension!"); 1642 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, 1643 PtrInfo, SlotVT, false, false, DestAlign); 1644} 1645 1646SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 1647 DebugLoc dl = Node->getDebugLoc(); 1648 // Create a vector sized/aligned stack slot, store the value to element #0, 1649 // then load the whole vector back out. 1650 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 1651 1652 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 1653 int SPFI = StackPtrFI->getIndex(); 1654 1655 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0), 1656 StackPtr, 1657 MachinePointerInfo::getFixedStack(SPFI), 1658 Node->getValueType(0).getVectorElementType(), 1659 false, false, 0); 1660 return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr, 1661 MachinePointerInfo::getFixedStack(SPFI), 1662 false, false, false, 0); 1663} 1664 1665 1666/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't 1667/// support the operation, but do support the resultant vector type. 1668SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 1669 unsigned NumElems = Node->getNumOperands(); 1670 SDValue Value1, Value2; 1671 DebugLoc dl = Node->getDebugLoc(); 1672 EVT VT = Node->getValueType(0); 1673 EVT OpVT = Node->getOperand(0).getValueType(); 1674 EVT EltVT = VT.getVectorElementType(); 1675 1676 // If the only non-undef value is the low element, turn this into a 1677 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 1678 bool isOnlyLowElement = true; 1679 bool MoreThanTwoValues = false; 1680 bool isConstant = true; 1681 for (unsigned i = 0; i < NumElems; ++i) { 1682 SDValue V = Node->getOperand(i); 1683 if (V.getOpcode() == ISD::UNDEF) 1684 continue; 1685 if (i > 0) 1686 isOnlyLowElement = false; 1687 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 1688 isConstant = false; 1689 1690 if (!Value1.getNode()) { 1691 Value1 = V; 1692 } else if (!Value2.getNode()) { 1693 if (V != Value1) 1694 Value2 = V; 1695 } else if (V != Value1 && V != Value2) { 1696 MoreThanTwoValues = true; 1697 } 1698 } 1699 1700 if (!Value1.getNode()) 1701 return DAG.getUNDEF(VT); 1702 1703 if (isOnlyLowElement) 1704 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); 1705 1706 // If all elements are constants, create a load from the constant pool. 1707 if (isConstant) { 1708 SmallVector<Constant*, 16> CV; 1709 for (unsigned i = 0, e = NumElems; i != e; ++i) { 1710 if (ConstantFPSDNode *V = 1711 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 1712 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 1713 } else if (ConstantSDNode *V = 1714 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 1715 if (OpVT==EltVT) 1716 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 1717 else { 1718 // If OpVT and EltVT don't match, EltVT is not legal and the 1719 // element values have been promoted/truncated earlier. Undo this; 1720 // we don't want a v16i8 to become a v16i32 for example. 1721 const ConstantInt *CI = V->getConstantIntValue(); 1722 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), 1723 CI->getZExtValue())); 1724 } 1725 } else { 1726 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); 1727 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); 1728 CV.push_back(UndefValue::get(OpNTy)); 1729 } 1730 } 1731 Constant *CP = ConstantVector::get(CV); 1732 SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); 1733 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 1734 return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 1735 MachinePointerInfo::getConstantPool(), 1736 false, false, false, Alignment); 1737 } 1738 1739 if (!MoreThanTwoValues) { 1740 SmallVector<int, 8> ShuffleVec(NumElems, -1); 1741 for (unsigned i = 0; i < NumElems; ++i) { 1742 SDValue V = Node->getOperand(i); 1743 if (V.getOpcode() == ISD::UNDEF) 1744 continue; 1745 ShuffleVec[i] = V == Value1 ? 0 : NumElems; 1746 } 1747 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { 1748 // Get the splatted value into the low element of a vector register. 1749 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); 1750 SDValue Vec2; 1751 if (Value2.getNode()) 1752 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); 1753 else 1754 Vec2 = DAG.getUNDEF(VT); 1755 1756 // Return shuffle(LowValVec, undef, <0,0,0,0>) 1757 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data()); 1758 } 1759 } 1760 1761 // Otherwise, we can't handle this case efficiently. 1762 return ExpandVectorBuildThroughStack(Node); 1763} 1764 1765// ExpandLibCall - Expand a node into a call to a libcall. If the result value 1766// does not fit into a register, return the lo part and set the hi part to the 1767// by-reg argument. If it does fit into a single register, return the result 1768// and leave the Hi part unset. 1769SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 1770 bool isSigned) { 1771 TargetLowering::ArgListTy Args; 1772 TargetLowering::ArgListEntry Entry; 1773 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1774 EVT ArgVT = Node->getOperand(i).getValueType(); 1775 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1776 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 1777 Entry.isSExt = isSigned; 1778 Entry.isZExt = !isSigned; 1779 Args.push_back(Entry); 1780 } 1781 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1782 TLI.getPointerTy()); 1783 1784 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 1785 1786 // By default, the input chain to this libcall is the entry node of the 1787 // function. If the libcall is going to be emitted as a tail call then 1788 // TLI.isUsedByReturnOnly will change it to the right chain if the return 1789 // node which is being folded has a non-entry input chain. 1790 SDValue InChain = DAG.getEntryNode(); 1791 1792 // isTailCall may be true since the callee does not reference caller stack 1793 // frame. Check if it's in the right position. 1794 SDValue TCChain = InChain; 1795 bool isTailCall = isInTailCallPosition(DAG, Node, TCChain, TLI); 1796 if (isTailCall) 1797 InChain = TCChain; 1798 1799 std::pair<SDValue, SDValue> CallInfo = 1800 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, 1801 0, TLI.getLibcallCallingConv(LC), isTailCall, 1802 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1803 Callee, Args, DAG, Node->getDebugLoc()); 1804 1805 if (!CallInfo.second.getNode()) 1806 // It's a tailcall, return the chain (which is the DAG root). 1807 return DAG.getRoot(); 1808 1809 return CallInfo.first; 1810} 1811 1812/// ExpandLibCall - Generate a libcall taking the given operands as arguments 1813/// and returning a result of type RetVT. 1814SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, 1815 const SDValue *Ops, unsigned NumOps, 1816 bool isSigned, DebugLoc dl) { 1817 TargetLowering::ArgListTy Args; 1818 Args.reserve(NumOps); 1819 1820 TargetLowering::ArgListEntry Entry; 1821 for (unsigned i = 0; i != NumOps; ++i) { 1822 Entry.Node = Ops[i]; 1823 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); 1824 Entry.isSExt = isSigned; 1825 Entry.isZExt = !isSigned; 1826 Args.push_back(Entry); 1827 } 1828 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1829 TLI.getPointerTy()); 1830 1831 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 1832 std::pair<SDValue,SDValue> CallInfo = 1833 TLI.LowerCallTo(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, 1834 false, 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 1835 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1836 Callee, Args, DAG, dl); 1837 1838 return CallInfo.first; 1839} 1840 1841// ExpandChainLibCall - Expand a node into a call to a libcall. Similar to 1842// ExpandLibCall except that the first operand is the in-chain. 1843std::pair<SDValue, SDValue> 1844SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, 1845 SDNode *Node, 1846 bool isSigned) { 1847 SDValue InChain = Node->getOperand(0); 1848 1849 TargetLowering::ArgListTy Args; 1850 TargetLowering::ArgListEntry Entry; 1851 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { 1852 EVT ArgVT = Node->getOperand(i).getValueType(); 1853 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1854 Entry.Node = Node->getOperand(i); 1855 Entry.Ty = ArgTy; 1856 Entry.isSExt = isSigned; 1857 Entry.isZExt = !isSigned; 1858 Args.push_back(Entry); 1859 } 1860 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1861 TLI.getPointerTy()); 1862 1863 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 1864 std::pair<SDValue, SDValue> CallInfo = 1865 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, 1866 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 1867 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1868 Callee, Args, DAG, Node->getDebugLoc()); 1869 1870 return CallInfo; 1871} 1872 1873SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, 1874 RTLIB::Libcall Call_F32, 1875 RTLIB::Libcall Call_F64, 1876 RTLIB::Libcall Call_F80, 1877 RTLIB::Libcall Call_PPCF128) { 1878 RTLIB::Libcall LC; 1879 switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 1880 default: llvm_unreachable("Unexpected request for libcall!"); 1881 case MVT::f32: LC = Call_F32; break; 1882 case MVT::f64: LC = Call_F64; break; 1883 case MVT::f80: LC = Call_F80; break; 1884 case MVT::ppcf128: LC = Call_PPCF128; break; 1885 } 1886 return ExpandLibCall(LC, Node, false); 1887} 1888 1889SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, 1890 RTLIB::Libcall Call_I8, 1891 RTLIB::Libcall Call_I16, 1892 RTLIB::Libcall Call_I32, 1893 RTLIB::Libcall Call_I64, 1894 RTLIB::Libcall Call_I128) { 1895 RTLIB::Libcall LC; 1896 switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 1897 default: llvm_unreachable("Unexpected request for libcall!"); 1898 case MVT::i8: LC = Call_I8; break; 1899 case MVT::i16: LC = Call_I16; break; 1900 case MVT::i32: LC = Call_I32; break; 1901 case MVT::i64: LC = Call_I64; break; 1902 case MVT::i128: LC = Call_I128; break; 1903 } 1904 return ExpandLibCall(LC, Node, isSigned); 1905} 1906 1907/// isDivRemLibcallAvailable - Return true if divmod libcall is available. 1908static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, 1909 const TargetLowering &TLI) { 1910 RTLIB::Libcall LC; 1911 switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 1912 default: llvm_unreachable("Unexpected request for libcall!"); 1913 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 1914 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 1915 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 1916 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 1917 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 1918 } 1919 1920 return TLI.getLibcallName(LC) != 0; 1921} 1922 1923/// UseDivRem - Only issue divrem libcall if both quotient and remainder are 1924/// needed. 1925static bool UseDivRem(SDNode *Node, bool isSigned, bool isDIV) { 1926 unsigned OtherOpcode = 0; 1927 if (isSigned) 1928 OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV; 1929 else 1930 OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV; 1931 1932 SDValue Op0 = Node->getOperand(0); 1933 SDValue Op1 = Node->getOperand(1); 1934 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 1935 UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 1936 SDNode *User = *UI; 1937 if (User == Node) 1938 continue; 1939 if (User->getOpcode() == OtherOpcode && 1940 User->getOperand(0) == Op0 && 1941 User->getOperand(1) == Op1) 1942 return true; 1943 } 1944 return false; 1945} 1946 1947/// ExpandDivRemLibCall - Issue libcalls to __{u}divmod to compute div / rem 1948/// pairs. 1949void 1950SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, 1951 SmallVectorImpl<SDValue> &Results) { 1952 unsigned Opcode = Node->getOpcode(); 1953 bool isSigned = Opcode == ISD::SDIVREM; 1954 1955 RTLIB::Libcall LC; 1956 switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 1957 default: llvm_unreachable("Unexpected request for libcall!"); 1958 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 1959 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 1960 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 1961 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 1962 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 1963 } 1964 1965 // The input chain to this libcall is the entry node of the function. 1966 // Legalizing the call will automatically add the previous call to the 1967 // dependence. 1968 SDValue InChain = DAG.getEntryNode(); 1969 1970 EVT RetVT = Node->getValueType(0); 1971 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 1972 1973 TargetLowering::ArgListTy Args; 1974 TargetLowering::ArgListEntry Entry; 1975 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1976 EVT ArgVT = Node->getOperand(i).getValueType(); 1977 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1978 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 1979 Entry.isSExt = isSigned; 1980 Entry.isZExt = !isSigned; 1981 Args.push_back(Entry); 1982 } 1983 1984 // Also pass the return address of the remainder. 1985 SDValue FIPtr = DAG.CreateStackTemporary(RetVT); 1986 Entry.Node = FIPtr; 1987 Entry.Ty = RetTy->getPointerTo(); 1988 Entry.isSExt = isSigned; 1989 Entry.isZExt = !isSigned; 1990 Args.push_back(Entry); 1991 1992 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1993 TLI.getPointerTy()); 1994 1995 DebugLoc dl = Node->getDebugLoc(); 1996 std::pair<SDValue, SDValue> CallInfo = 1997 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, 1998 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 1999 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2000 Callee, Args, DAG, dl); 2001 2002 // Remainder is loaded back from the stack frame. 2003 SDValue Rem = DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, 2004 MachinePointerInfo(), false, false, false, 0); 2005 Results.push_back(CallInfo.first); 2006 Results.push_back(Rem); 2007} 2008 2009/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a 2010/// INT_TO_FP operation of the specified operand when the target requests that 2011/// we expand it. At this point, we know that the result and operand types are 2012/// legal for the target. 2013SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, 2014 SDValue Op0, 2015 EVT DestVT, 2016 DebugLoc dl) { 2017 if (Op0.getValueType() == MVT::i32) { 2018 // simple 32-bit [signed|unsigned] integer to float/double expansion 2019 2020 // Get the stack frame index of a 8 byte buffer. 2021 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 2022 2023 // word offset constant for Hi/Lo address computation 2024 SDValue WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy()); 2025 // set up Hi and Lo (into buffer) address based on endian 2026 SDValue Hi = StackSlot; 2027 SDValue Lo = DAG.getNode(ISD::ADD, dl, 2028 TLI.getPointerTy(), StackSlot, WordOff); 2029 if (TLI.isLittleEndian()) 2030 std::swap(Hi, Lo); 2031 2032 // if signed map to unsigned space 2033 SDValue Op0Mapped; 2034 if (isSigned) { 2035 // constant used to invert sign bit (signed to unsigned mapping) 2036 SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); 2037 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); 2038 } else { 2039 Op0Mapped = Op0; 2040 } 2041 // store the lo of the constructed double - based on integer input 2042 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, 2043 Op0Mapped, Lo, MachinePointerInfo(), 2044 false, false, 0); 2045 // initial hi portion of constructed double 2046 SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); 2047 // store the hi of the constructed double - biased exponent 2048 SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi, 2049 MachinePointerInfo(), 2050 false, false, 0); 2051 // load the constructed double 2052 SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot, 2053 MachinePointerInfo(), false, false, false, 0); 2054 // FP constant to bias correct the final result 2055 SDValue Bias = DAG.getConstantFP(isSigned ? 2056 BitsToDouble(0x4330000080000000ULL) : 2057 BitsToDouble(0x4330000000000000ULL), 2058 MVT::f64); 2059 // subtract the bias 2060 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 2061 // final result 2062 SDValue Result; 2063 // handle final rounding 2064 if (DestVT == MVT::f64) { 2065 // do nothing 2066 Result = Sub; 2067 } else if (DestVT.bitsLT(MVT::f64)) { 2068 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 2069 DAG.getIntPtrConstant(0)); 2070 } else if (DestVT.bitsGT(MVT::f64)) { 2071 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 2072 } 2073 return Result; 2074 } 2075 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 2076 // Code below here assumes !isSigned without checking again. 2077 2078 // Implementation of unsigned i64 to f64 following the algorithm in 2079 // __floatundidf in compiler_rt. This implementation has the advantage 2080 // of performing rounding correctly, both in the default rounding mode 2081 // and in all alternate rounding modes. 2082 // TODO: Generalize this for use with other types. 2083 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) { 2084 SDValue TwoP52 = 2085 DAG.getConstant(UINT64_C(0x4330000000000000), MVT::i64); 2086 SDValue TwoP84PlusTwoP52 = 2087 DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), MVT::f64); 2088 SDValue TwoP84 = 2089 DAG.getConstant(UINT64_C(0x4530000000000000), MVT::i64); 2090 2091 SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32); 2092 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, 2093 DAG.getConstant(32, MVT::i64)); 2094 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52); 2095 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84); 2096 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr); 2097 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr); 2098 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt, 2099 TwoP84PlusTwoP52); 2100 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); 2101 } 2102 2103 // Implementation of unsigned i64 to f32. 2104 // TODO: Generalize this for use with other types. 2105 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) { 2106 // For unsigned conversions, convert them to signed conversions using the 2107 // algorithm from the x86_64 __floatundidf in compiler_rt. 2108 if (!isSigned) { 2109 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0); 2110 2111 SDValue ShiftConst = 2112 DAG.getConstant(1, TLI.getShiftAmountTy(Op0.getValueType())); 2113 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst); 2114 SDValue AndConst = DAG.getConstant(1, MVT::i64); 2115 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst); 2116 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr); 2117 2118 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or); 2119 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt); 2120 2121 // TODO: This really should be implemented using a branch rather than a 2122 // select. We happen to get lucky and machinesink does the right 2123 // thing most of the time. This would be a good candidate for a 2124 //pseudo-op, or, even better, for whole-function isel. 2125 SDValue SignBitTest = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2126 Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT); 2127 return DAG.getNode(ISD::SELECT, dl, MVT::f32, SignBitTest, Slow, Fast); 2128 } 2129 2130 // Otherwise, implement the fully general conversion. 2131 2132 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2133 DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64)); 2134 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, 2135 DAG.getConstant(UINT64_C(0x800), MVT::i64)); 2136 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2137 DAG.getConstant(UINT64_C(0x7ff), MVT::i64)); 2138 SDValue Ne = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2139 And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE); 2140 SDValue Sel = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ne, Or, Op0); 2141 SDValue Ge = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2142 Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64), 2143 ISD::SETUGE); 2144 SDValue Sel2 = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ge, Sel, Op0); 2145 EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType()); 2146 2147 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, 2148 DAG.getConstant(32, SHVT)); 2149 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); 2150 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); 2151 SDValue TwoP32 = 2152 DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64); 2153 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); 2154 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); 2155 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); 2156 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); 2157 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, 2158 DAG.getIntPtrConstant(0)); 2159 } 2160 2161 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 2162 2163 SDValue SignSet = DAG.getSetCC(dl, TLI.getSetCCResultType(Op0.getValueType()), 2164 Op0, DAG.getConstant(0, Op0.getValueType()), 2165 ISD::SETLT); 2166 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 2167 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), 2168 SignSet, Four, Zero); 2169 2170 // If the sign bit of the integer is set, the large number will be treated 2171 // as a negative number. To counteract this, the dynamic code adds an 2172 // offset depending on the data type. 2173 uint64_t FF; 2174 switch (Op0.getValueType().getSimpleVT().SimpleTy) { 2175 default: llvm_unreachable("Unsupported integer type!"); 2176 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 2177 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 2178 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 2179 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 2180 } 2181 if (TLI.isLittleEndian()) FF <<= 32; 2182 Constant *FudgeFactor = ConstantInt::get( 2183 Type::getInt64Ty(*DAG.getContext()), FF); 2184 2185 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 2186 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 2187 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset); 2188 Alignment = std::min(Alignment, 4u); 2189 SDValue FudgeInReg; 2190 if (DestVT == MVT::f32) 2191 FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 2192 MachinePointerInfo::getConstantPool(), 2193 false, false, false, Alignment); 2194 else { 2195 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, 2196 DAG.getEntryNode(), CPIdx, 2197 MachinePointerInfo::getConstantPool(), 2198 MVT::f32, false, false, Alignment); 2199 HandleSDNode Handle(Load); 2200 LegalizeOp(Load.getNode()); 2201 FudgeInReg = Handle.getValue(); 2202 } 2203 2204 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 2205} 2206 2207/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a 2208/// *INT_TO_FP operation of the specified operand when the target requests that 2209/// we promote it. At this point, we know that the result and operand types are 2210/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 2211/// operation that takes a larger input. 2212SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, 2213 EVT DestVT, 2214 bool isSigned, 2215 DebugLoc dl) { 2216 // First step, figure out the appropriate *INT_TO_FP operation to use. 2217 EVT NewInTy = LegalOp.getValueType(); 2218 2219 unsigned OpToUse = 0; 2220 2221 // Scan for the appropriate larger type to use. 2222 while (1) { 2223 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); 2224 assert(NewInTy.isInteger() && "Ran out of possibilities!"); 2225 2226 // If the target supports SINT_TO_FP of this type, use it. 2227 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { 2228 OpToUse = ISD::SINT_TO_FP; 2229 break; 2230 } 2231 if (isSigned) continue; 2232 2233 // If the target supports UINT_TO_FP of this type, use it. 2234 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { 2235 OpToUse = ISD::UINT_TO_FP; 2236 break; 2237 } 2238 2239 // Otherwise, try a larger type. 2240 } 2241 2242 // Okay, we found the operation and type to use. Zero extend our input to the 2243 // desired type then run the operation on it. 2244 return DAG.getNode(OpToUse, dl, DestVT, 2245 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 2246 dl, NewInTy, LegalOp)); 2247} 2248 2249/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a 2250/// FP_TO_*INT operation of the specified operand when the target requests that 2251/// we promote it. At this point, we know that the result and operand types are 2252/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 2253/// operation that returns a larger result. 2254SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, 2255 EVT DestVT, 2256 bool isSigned, 2257 DebugLoc dl) { 2258 // First step, figure out the appropriate FP_TO*INT operation to use. 2259 EVT NewOutTy = DestVT; 2260 2261 unsigned OpToUse = 0; 2262 2263 // Scan for the appropriate larger type to use. 2264 while (1) { 2265 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); 2266 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 2267 2268 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { 2269 OpToUse = ISD::FP_TO_SINT; 2270 break; 2271 } 2272 2273 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { 2274 OpToUse = ISD::FP_TO_UINT; 2275 break; 2276 } 2277 2278 // Otherwise, try a larger type. 2279 } 2280 2281 2282 // Okay, we found the operation and type to use. 2283 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 2284 2285 // Truncate the result of the extended FP_TO_*INT operation to the desired 2286 // size. 2287 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 2288} 2289 2290/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. 2291/// 2292SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) { 2293 EVT VT = Op.getValueType(); 2294 EVT SHVT = TLI.getShiftAmountTy(VT); 2295 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 2296 switch (VT.getSimpleVT().SimpleTy) { 2297 default: llvm_unreachable("Unhandled Expand type in BSWAP!"); 2298 case MVT::i16: 2299 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2300 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2301 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 2302 case MVT::i32: 2303 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2304 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2305 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2306 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2307 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); 2308 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); 2309 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2310 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2311 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2312 case MVT::i64: 2313 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2314 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2315 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2316 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2317 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2318 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2319 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2320 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2321 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); 2322 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); 2323 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); 2324 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); 2325 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); 2326 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); 2327 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 2328 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 2329 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2330 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2331 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 2332 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2333 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 2334 } 2335} 2336 2337/// SplatByte - Distribute ByteVal over NumBits bits. 2338// FIXME: Move this helper to a common place. 2339static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) { 2340 APInt Val = APInt(NumBits, ByteVal); 2341 unsigned Shift = 8; 2342 for (unsigned i = NumBits; i > 8; i >>= 1) { 2343 Val = (Val << Shift) | Val; 2344 Shift <<= 1; 2345 } 2346 return Val; 2347} 2348 2349/// ExpandBitCount - Expand the specified bitcount instruction into operations. 2350/// 2351SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, 2352 DebugLoc dl) { 2353 switch (Opc) { 2354 default: llvm_unreachable("Cannot expand this yet!"); 2355 case ISD::CTPOP: { 2356 EVT VT = Op.getValueType(); 2357 EVT ShVT = TLI.getShiftAmountTy(VT); 2358 unsigned Len = VT.getSizeInBits(); 2359 2360 assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 && 2361 "CTPOP not implemented for this type."); 2362 2363 // This is the "best" algorithm from 2364 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel 2365 2366 SDValue Mask55 = DAG.getConstant(SplatByte(Len, 0x55), VT); 2367 SDValue Mask33 = DAG.getConstant(SplatByte(Len, 0x33), VT); 2368 SDValue Mask0F = DAG.getConstant(SplatByte(Len, 0x0F), VT); 2369 SDValue Mask01 = DAG.getConstant(SplatByte(Len, 0x01), VT); 2370 2371 // v = v - ((v >> 1) & 0x55555555...) 2372 Op = DAG.getNode(ISD::SUB, dl, VT, Op, 2373 DAG.getNode(ISD::AND, dl, VT, 2374 DAG.getNode(ISD::SRL, dl, VT, Op, 2375 DAG.getConstant(1, ShVT)), 2376 Mask55)); 2377 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...) 2378 Op = DAG.getNode(ISD::ADD, dl, VT, 2379 DAG.getNode(ISD::AND, dl, VT, Op, Mask33), 2380 DAG.getNode(ISD::AND, dl, VT, 2381 DAG.getNode(ISD::SRL, dl, VT, Op, 2382 DAG.getConstant(2, ShVT)), 2383 Mask33)); 2384 // v = (v + (v >> 4)) & 0x0F0F0F0F... 2385 Op = DAG.getNode(ISD::AND, dl, VT, 2386 DAG.getNode(ISD::ADD, dl, VT, Op, 2387 DAG.getNode(ISD::SRL, dl, VT, Op, 2388 DAG.getConstant(4, ShVT))), 2389 Mask0F); 2390 // v = (v * 0x01010101...) >> (Len - 8) 2391 Op = DAG.getNode(ISD::SRL, dl, VT, 2392 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), 2393 DAG.getConstant(Len - 8, ShVT)); 2394 2395 return Op; 2396 } 2397 case ISD::CTLZ_ZERO_UNDEF: 2398 // This trivially expands to CTLZ. 2399 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); 2400 case ISD::CTLZ: { 2401 // for now, we do this: 2402 // x = x | (x >> 1); 2403 // x = x | (x >> 2); 2404 // ... 2405 // x = x | (x >>16); 2406 // x = x | (x >>32); // for 64-bit input 2407 // return popcount(~x); 2408 // 2409 // but see also: http://www.hackersdelight.org/HDcode/nlz.cc 2410 EVT VT = Op.getValueType(); 2411 EVT ShVT = TLI.getShiftAmountTy(VT); 2412 unsigned len = VT.getSizeInBits(); 2413 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 2414 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 2415 Op = DAG.getNode(ISD::OR, dl, VT, Op, 2416 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); 2417 } 2418 Op = DAG.getNOT(dl, Op, VT); 2419 return DAG.getNode(ISD::CTPOP, dl, VT, Op); 2420 } 2421 case ISD::CTTZ_ZERO_UNDEF: 2422 // This trivially expands to CTTZ. 2423 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); 2424 case ISD::CTTZ: { 2425 // for now, we use: { return popcount(~x & (x - 1)); } 2426 // unless the target has ctlz but not ctpop, in which case we use: 2427 // { return 32 - nlz(~x & (x-1)); } 2428 // see also http://www.hackersdelight.org/HDcode/ntz.cc 2429 EVT VT = Op.getValueType(); 2430 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, 2431 DAG.getNOT(dl, Op, VT), 2432 DAG.getNode(ISD::SUB, dl, VT, Op, 2433 DAG.getConstant(1, VT))); 2434 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. 2435 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && 2436 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) 2437 return DAG.getNode(ISD::SUB, dl, VT, 2438 DAG.getConstant(VT.getSizeInBits(), VT), 2439 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); 2440 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); 2441 } 2442 } 2443} 2444 2445std::pair <SDValue, SDValue> SelectionDAGLegalize::ExpandAtomic(SDNode *Node) { 2446 unsigned Opc = Node->getOpcode(); 2447 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2448 RTLIB::Libcall LC; 2449 2450 switch (Opc) { 2451 default: 2452 llvm_unreachable("Unhandled atomic intrinsic Expand!"); 2453 case ISD::ATOMIC_SWAP: 2454 switch (VT.SimpleTy) { 2455 default: llvm_unreachable("Unexpected value type for atomic!"); 2456 case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 2457 case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 2458 case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 2459 case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 2460 } 2461 break; 2462 case ISD::ATOMIC_CMP_SWAP: 2463 switch (VT.SimpleTy) { 2464 default: llvm_unreachable("Unexpected value type for atomic!"); 2465 case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 2466 case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 2467 case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 2468 case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 2469 } 2470 break; 2471 case ISD::ATOMIC_LOAD_ADD: 2472 switch (VT.SimpleTy) { 2473 default: llvm_unreachable("Unexpected value type for atomic!"); 2474 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 2475 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 2476 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 2477 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 2478 } 2479 break; 2480 case ISD::ATOMIC_LOAD_SUB: 2481 switch (VT.SimpleTy) { 2482 default: llvm_unreachable("Unexpected value type for atomic!"); 2483 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 2484 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 2485 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 2486 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 2487 } 2488 break; 2489 case ISD::ATOMIC_LOAD_AND: 2490 switch (VT.SimpleTy) { 2491 default: llvm_unreachable("Unexpected value type for atomic!"); 2492 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 2493 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 2494 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 2495 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 2496 } 2497 break; 2498 case ISD::ATOMIC_LOAD_OR: 2499 switch (VT.SimpleTy) { 2500 default: llvm_unreachable("Unexpected value type for atomic!"); 2501 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 2502 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 2503 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 2504 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 2505 } 2506 break; 2507 case ISD::ATOMIC_LOAD_XOR: 2508 switch (VT.SimpleTy) { 2509 default: llvm_unreachable("Unexpected value type for atomic!"); 2510 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 2511 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 2512 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 2513 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 2514 } 2515 break; 2516 case ISD::ATOMIC_LOAD_NAND: 2517 switch (VT.SimpleTy) { 2518 default: llvm_unreachable("Unexpected value type for atomic!"); 2519 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 2520 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 2521 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 2522 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 2523 } 2524 break; 2525 } 2526 2527 return ExpandChainLibCall(LC, Node, false); 2528} 2529 2530void SelectionDAGLegalize::ExpandNode(SDNode *Node) { 2531 SmallVector<SDValue, 8> Results; 2532 DebugLoc dl = Node->getDebugLoc(); 2533 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 2534 switch (Node->getOpcode()) { 2535 case ISD::CTPOP: 2536 case ISD::CTLZ: 2537 case ISD::CTLZ_ZERO_UNDEF: 2538 case ISD::CTTZ: 2539 case ISD::CTTZ_ZERO_UNDEF: 2540 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl); 2541 Results.push_back(Tmp1); 2542 break; 2543 case ISD::BSWAP: 2544 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); 2545 break; 2546 case ISD::FRAMEADDR: 2547 case ISD::RETURNADDR: 2548 case ISD::FRAME_TO_ARGS_OFFSET: 2549 Results.push_back(DAG.getConstant(0, Node->getValueType(0))); 2550 break; 2551 case ISD::FLT_ROUNDS_: 2552 Results.push_back(DAG.getConstant(1, Node->getValueType(0))); 2553 break; 2554 case ISD::EH_RETURN: 2555 case ISD::EH_LABEL: 2556 case ISD::PREFETCH: 2557 case ISD::VAEND: 2558 case ISD::EH_SJLJ_LONGJMP: 2559 // If the target didn't expand these, there's nothing to do, so just 2560 // preserve the chain and be done. 2561 Results.push_back(Node->getOperand(0)); 2562 break; 2563 case ISD::EH_SJLJ_SETJMP: 2564 // If the target didn't expand this, just return 'zero' and preserve the 2565 // chain. 2566 Results.push_back(DAG.getConstant(0, MVT::i32)); 2567 Results.push_back(Node->getOperand(0)); 2568 break; 2569 case ISD::ATOMIC_FENCE: 2570 case ISD::MEMBARRIER: { 2571 // If the target didn't lower this, lower it to '__sync_synchronize()' call 2572 // FIXME: handle "fence singlethread" more efficiently. 2573 TargetLowering::ArgListTy Args; 2574 std::pair<SDValue, SDValue> CallResult = 2575 TLI.LowerCallTo(Node->getOperand(0), Type::getVoidTy(*DAG.getContext()), 2576 false, false, false, false, 0, CallingConv::C, 2577 /*isTailCall=*/false, 2578 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2579 DAG.getExternalSymbol("__sync_synchronize", 2580 TLI.getPointerTy()), 2581 Args, DAG, dl); 2582 Results.push_back(CallResult.second); 2583 break; 2584 } 2585 case ISD::ATOMIC_LOAD: { 2586 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. 2587 SDValue Zero = DAG.getConstant(0, Node->getValueType(0)); 2588 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, 2589 cast<AtomicSDNode>(Node)->getMemoryVT(), 2590 Node->getOperand(0), 2591 Node->getOperand(1), Zero, Zero, 2592 cast<AtomicSDNode>(Node)->getMemOperand(), 2593 cast<AtomicSDNode>(Node)->getOrdering(), 2594 cast<AtomicSDNode>(Node)->getSynchScope()); 2595 Results.push_back(Swap.getValue(0)); 2596 Results.push_back(Swap.getValue(1)); 2597 break; 2598 } 2599 case ISD::ATOMIC_STORE: { 2600 // There is no libcall for atomic store; fake it with ATOMIC_SWAP. 2601 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 2602 cast<AtomicSDNode>(Node)->getMemoryVT(), 2603 Node->getOperand(0), 2604 Node->getOperand(1), Node->getOperand(2), 2605 cast<AtomicSDNode>(Node)->getMemOperand(), 2606 cast<AtomicSDNode>(Node)->getOrdering(), 2607 cast<AtomicSDNode>(Node)->getSynchScope()); 2608 Results.push_back(Swap.getValue(1)); 2609 break; 2610 } 2611 // By default, atomic intrinsics are marked Legal and lowered. Targets 2612 // which don't support them directly, however, may want libcalls, in which 2613 // case they mark them Expand, and we get here. 2614 case ISD::ATOMIC_SWAP: 2615 case ISD::ATOMIC_LOAD_ADD: 2616 case ISD::ATOMIC_LOAD_SUB: 2617 case ISD::ATOMIC_LOAD_AND: 2618 case ISD::ATOMIC_LOAD_OR: 2619 case ISD::ATOMIC_LOAD_XOR: 2620 case ISD::ATOMIC_LOAD_NAND: 2621 case ISD::ATOMIC_LOAD_MIN: 2622 case ISD::ATOMIC_LOAD_MAX: 2623 case ISD::ATOMIC_LOAD_UMIN: 2624 case ISD::ATOMIC_LOAD_UMAX: 2625 case ISD::ATOMIC_CMP_SWAP: { 2626 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node); 2627 Results.push_back(Tmp.first); 2628 Results.push_back(Tmp.second); 2629 break; 2630 } 2631 case ISD::DYNAMIC_STACKALLOC: 2632 ExpandDYNAMIC_STACKALLOC(Node, Results); 2633 break; 2634 case ISD::MERGE_VALUES: 2635 for (unsigned i = 0; i < Node->getNumValues(); i++) 2636 Results.push_back(Node->getOperand(i)); 2637 break; 2638 case ISD::UNDEF: { 2639 EVT VT = Node->getValueType(0); 2640 if (VT.isInteger()) 2641 Results.push_back(DAG.getConstant(0, VT)); 2642 else { 2643 assert(VT.isFloatingPoint() && "Unknown value type!"); 2644 Results.push_back(DAG.getConstantFP(0, VT)); 2645 } 2646 break; 2647 } 2648 case ISD::TRAP: { 2649 // If this operation is not supported, lower it to 'abort()' call 2650 TargetLowering::ArgListTy Args; 2651 std::pair<SDValue, SDValue> CallResult = 2652 TLI.LowerCallTo(Node->getOperand(0), Type::getVoidTy(*DAG.getContext()), 2653 false, false, false, false, 0, CallingConv::C, 2654 /*isTailCall=*/false, 2655 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2656 DAG.getExternalSymbol("abort", TLI.getPointerTy()), 2657 Args, DAG, dl); 2658 Results.push_back(CallResult.second); 2659 break; 2660 } 2661 case ISD::FP_ROUND: 2662 case ISD::BITCAST: 2663 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 2664 Node->getValueType(0), dl); 2665 Results.push_back(Tmp1); 2666 break; 2667 case ISD::FP_EXTEND: 2668 Tmp1 = EmitStackConvert(Node->getOperand(0), 2669 Node->getOperand(0).getValueType(), 2670 Node->getValueType(0), dl); 2671 Results.push_back(Tmp1); 2672 break; 2673 case ISD::SIGN_EXTEND_INREG: { 2674 // NOTE: we could fall back on load/store here too for targets without 2675 // SAR. However, it is doubtful that any exist. 2676 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2677 EVT VT = Node->getValueType(0); 2678 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT); 2679 if (VT.isVector()) 2680 ShiftAmountTy = VT; 2681 unsigned BitsDiff = VT.getScalarType().getSizeInBits() - 2682 ExtraVT.getScalarType().getSizeInBits(); 2683 SDValue ShiftCst = DAG.getConstant(BitsDiff, ShiftAmountTy); 2684 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 2685 Node->getOperand(0), ShiftCst); 2686 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); 2687 Results.push_back(Tmp1); 2688 break; 2689 } 2690 case ISD::FP_ROUND_INREG: { 2691 // The only way we can lower this is to turn it into a TRUNCSTORE, 2692 // EXTLOAD pair, targeting a temporary location (a stack slot). 2693 2694 // NOTE: there is a choice here between constantly creating new stack 2695 // slots and always reusing the same one. We currently always create 2696 // new ones, as reuse may inhibit scheduling. 2697 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2698 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, 2699 Node->getValueType(0), dl); 2700 Results.push_back(Tmp1); 2701 break; 2702 } 2703 case ISD::SINT_TO_FP: 2704 case ISD::UINT_TO_FP: 2705 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, 2706 Node->getOperand(0), Node->getValueType(0), dl); 2707 Results.push_back(Tmp1); 2708 break; 2709 case ISD::FP_TO_UINT: { 2710 SDValue True, False; 2711 EVT VT = Node->getOperand(0).getValueType(); 2712 EVT NVT = Node->getValueType(0); 2713 APFloat apf(APInt::getNullValue(VT.getSizeInBits())); 2714 APInt x = APInt::getSignBit(NVT.getSizeInBits()); 2715 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); 2716 Tmp1 = DAG.getConstantFP(apf, VT); 2717 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), 2718 Node->getOperand(0), 2719 Tmp1, ISD::SETLT); 2720 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); 2721 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, 2722 DAG.getNode(ISD::FSUB, dl, VT, 2723 Node->getOperand(0), Tmp1)); 2724 False = DAG.getNode(ISD::XOR, dl, NVT, False, 2725 DAG.getConstant(x, NVT)); 2726 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, True, False); 2727 Results.push_back(Tmp1); 2728 break; 2729 } 2730 case ISD::VAARG: { 2731 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 2732 EVT VT = Node->getValueType(0); 2733 Tmp1 = Node->getOperand(0); 2734 Tmp2 = Node->getOperand(1); 2735 unsigned Align = Node->getConstantOperandVal(3); 2736 2737 SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, 2738 MachinePointerInfo(V), 2739 false, false, false, 0); 2740 SDValue VAList = VAListLoad; 2741 2742 if (Align > TLI.getMinStackArgumentAlignment()) { 2743 assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); 2744 2745 VAList = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 2746 DAG.getConstant(Align - 1, 2747 TLI.getPointerTy())); 2748 2749 VAList = DAG.getNode(ISD::AND, dl, TLI.getPointerTy(), VAList, 2750 DAG.getConstant(-(int64_t)Align, 2751 TLI.getPointerTy())); 2752 } 2753 2754 // Increment the pointer, VAList, to the next vaarg 2755 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 2756 DAG.getConstant(TLI.getTargetData()-> 2757 getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), 2758 TLI.getPointerTy())); 2759 // Store the incremented VAList to the legalized pointer 2760 Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2, 2761 MachinePointerInfo(V), false, false, 0); 2762 // Load the actual argument out of the pointer VAList 2763 Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(), 2764 false, false, false, 0)); 2765 Results.push_back(Results[0].getValue(1)); 2766 break; 2767 } 2768 case ISD::VACOPY: { 2769 // This defaults to loading a pointer from the input and storing it to the 2770 // output, returning the chain. 2771 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 2772 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 2773 Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0), 2774 Node->getOperand(2), MachinePointerInfo(VS), 2775 false, false, false, 0); 2776 Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), 2777 MachinePointerInfo(VD), false, false, 0); 2778 Results.push_back(Tmp1); 2779 break; 2780 } 2781 case ISD::EXTRACT_VECTOR_ELT: 2782 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) 2783 // This must be an access of the only element. Return it. 2784 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), 2785 Node->getOperand(0)); 2786 else 2787 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); 2788 Results.push_back(Tmp1); 2789 break; 2790 case ISD::EXTRACT_SUBVECTOR: 2791 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); 2792 break; 2793 case ISD::INSERT_SUBVECTOR: 2794 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); 2795 break; 2796 case ISD::CONCAT_VECTORS: { 2797 Results.push_back(ExpandVectorBuildThroughStack(Node)); 2798 break; 2799 } 2800 case ISD::SCALAR_TO_VECTOR: 2801 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); 2802 break; 2803 case ISD::INSERT_VECTOR_ELT: 2804 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), 2805 Node->getOperand(1), 2806 Node->getOperand(2), dl)); 2807 break; 2808 case ISD::VECTOR_SHUFFLE: { 2809 SmallVector<int, 32> NewMask; 2810 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 2811 2812 EVT VT = Node->getValueType(0); 2813 EVT EltVT = VT.getVectorElementType(); 2814 SDValue Op0 = Node->getOperand(0); 2815 SDValue Op1 = Node->getOperand(1); 2816 if (!TLI.isTypeLegal(EltVT)) { 2817 2818 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); 2819 2820 // BUILD_VECTOR operands are allowed to be wider than the element type. 2821 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept it 2822 if (NewEltVT.bitsLT(EltVT)) { 2823 2824 // Convert shuffle node. 2825 // If original node was v4i64 and the new EltVT is i32, 2826 // cast operands to v8i32 and re-build the mask. 2827 2828 // Calculate new VT, the size of the new VT should be equal to original. 2829 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, 2830 VT.getSizeInBits()/NewEltVT.getSizeInBits()); 2831 assert(NewVT.bitsEq(VT)); 2832 2833 // cast operands to new VT 2834 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); 2835 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); 2836 2837 // Convert the shuffle mask 2838 unsigned int factor = NewVT.getVectorNumElements()/VT.getVectorNumElements(); 2839 2840 // EltVT gets smaller 2841 assert(factor > 0); 2842 2843 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 2844 if (Mask[i] < 0) { 2845 for (unsigned fi = 0; fi < factor; ++fi) 2846 NewMask.push_back(Mask[i]); 2847 } 2848 else { 2849 for (unsigned fi = 0; fi < factor; ++fi) 2850 NewMask.push_back(Mask[i]*factor+fi); 2851 } 2852 } 2853 Mask = NewMask; 2854 VT = NewVT; 2855 } 2856 EltVT = NewEltVT; 2857 } 2858 unsigned NumElems = VT.getVectorNumElements(); 2859 SmallVector<SDValue, 16> Ops; 2860 for (unsigned i = 0; i != NumElems; ++i) { 2861 if (Mask[i] < 0) { 2862 Ops.push_back(DAG.getUNDEF(EltVT)); 2863 continue; 2864 } 2865 unsigned Idx = Mask[i]; 2866 if (Idx < NumElems) 2867 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 2868 Op0, 2869 DAG.getIntPtrConstant(Idx))); 2870 else 2871 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 2872 Op1, 2873 DAG.getIntPtrConstant(Idx - NumElems))); 2874 } 2875 2876 Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size()); 2877 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. 2878 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); 2879 Results.push_back(Tmp1); 2880 break; 2881 } 2882 case ISD::EXTRACT_ELEMENT: { 2883 EVT OpTy = Node->getOperand(0).getValueType(); 2884 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 2885 // 1 -> Hi 2886 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 2887 DAG.getConstant(OpTy.getSizeInBits()/2, 2888 TLI.getShiftAmountTy(Node->getOperand(0).getValueType()))); 2889 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 2890 } else { 2891 // 0 -> Lo 2892 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 2893 Node->getOperand(0)); 2894 } 2895 Results.push_back(Tmp1); 2896 break; 2897 } 2898 case ISD::STACKSAVE: 2899 // Expand to CopyFromReg if the target set 2900 // StackPointerRegisterToSaveRestore. 2901 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 2902 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, 2903 Node->getValueType(0))); 2904 Results.push_back(Results[0].getValue(1)); 2905 } else { 2906 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); 2907 Results.push_back(Node->getOperand(0)); 2908 } 2909 break; 2910 case ISD::STACKRESTORE: 2911 // Expand to CopyToReg if the target set 2912 // StackPointerRegisterToSaveRestore. 2913 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 2914 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, 2915 Node->getOperand(1))); 2916 } else { 2917 Results.push_back(Node->getOperand(0)); 2918 } 2919 break; 2920 case ISD::FCOPYSIGN: 2921 Results.push_back(ExpandFCOPYSIGN(Node)); 2922 break; 2923 case ISD::FNEG: 2924 // Expand Y = FNEG(X) -> Y = SUB -0.0, X 2925 Tmp1 = DAG.getConstantFP(-0.0, Node->getValueType(0)); 2926 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, 2927 Node->getOperand(0)); 2928 Results.push_back(Tmp1); 2929 break; 2930 case ISD::FABS: { 2931 // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). 2932 EVT VT = Node->getValueType(0); 2933 Tmp1 = Node->getOperand(0); 2934 Tmp2 = DAG.getConstantFP(0.0, VT); 2935 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), 2936 Tmp1, Tmp2, ISD::SETUGT); 2937 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); 2938 Tmp1 = DAG.getNode(ISD::SELECT, dl, VT, Tmp2, Tmp1, Tmp3); 2939 Results.push_back(Tmp1); 2940 break; 2941 } 2942 case ISD::FSQRT: 2943 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 2944 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128)); 2945 break; 2946 case ISD::FSIN: 2947 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, 2948 RTLIB::SIN_F80, RTLIB::SIN_PPCF128)); 2949 break; 2950 case ISD::FCOS: 2951 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, 2952 RTLIB::COS_F80, RTLIB::COS_PPCF128)); 2953 break; 2954 case ISD::FLOG: 2955 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, 2956 RTLIB::LOG_F80, RTLIB::LOG_PPCF128)); 2957 break; 2958 case ISD::FLOG2: 2959 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 2960 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128)); 2961 break; 2962 case ISD::FLOG10: 2963 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 2964 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128)); 2965 break; 2966 case ISD::FEXP: 2967 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, 2968 RTLIB::EXP_F80, RTLIB::EXP_PPCF128)); 2969 break; 2970 case ISD::FEXP2: 2971 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 2972 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128)); 2973 break; 2974 case ISD::FTRUNC: 2975 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 2976 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128)); 2977 break; 2978 case ISD::FFLOOR: 2979 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 2980 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128)); 2981 break; 2982 case ISD::FCEIL: 2983 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 2984 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128)); 2985 break; 2986 case ISD::FRINT: 2987 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, 2988 RTLIB::RINT_F80, RTLIB::RINT_PPCF128)); 2989 break; 2990 case ISD::FNEARBYINT: 2991 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, 2992 RTLIB::NEARBYINT_F64, 2993 RTLIB::NEARBYINT_F80, 2994 RTLIB::NEARBYINT_PPCF128)); 2995 break; 2996 case ISD::FPOWI: 2997 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, 2998 RTLIB::POWI_F80, RTLIB::POWI_PPCF128)); 2999 break; 3000 case ISD::FPOW: 3001 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, 3002 RTLIB::POW_F80, RTLIB::POW_PPCF128)); 3003 break; 3004 case ISD::FDIV: 3005 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, 3006 RTLIB::DIV_F80, RTLIB::DIV_PPCF128)); 3007 break; 3008 case ISD::FREM: 3009 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, 3010 RTLIB::REM_F80, RTLIB::REM_PPCF128)); 3011 break; 3012 case ISD::FMA: 3013 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, 3014 RTLIB::FMA_F80, RTLIB::FMA_PPCF128)); 3015 break; 3016 case ISD::FP16_TO_FP32: 3017 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); 3018 break; 3019 case ISD::FP32_TO_FP16: 3020 Results.push_back(ExpandLibCall(RTLIB::FPROUND_F32_F16, Node, false)); 3021 break; 3022 case ISD::ConstantFP: { 3023 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 3024 // Check to see if this FP immediate is already legal. 3025 // If this is a legal constant, turn it into a TargetConstantFP node. 3026 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0))) 3027 Results.push_back(ExpandConstantFP(CFP, true)); 3028 break; 3029 } 3030 case ISD::EHSELECTION: { 3031 unsigned Reg = TLI.getExceptionSelectorRegister(); 3032 assert(Reg && "Can't expand to unknown register!"); 3033 Results.push_back(DAG.getCopyFromReg(Node->getOperand(1), dl, Reg, 3034 Node->getValueType(0))); 3035 Results.push_back(Results[0].getValue(1)); 3036 break; 3037 } 3038 case ISD::EXCEPTIONADDR: { 3039 unsigned Reg = TLI.getExceptionPointerRegister(); 3040 assert(Reg && "Can't expand to unknown register!"); 3041 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, Reg, 3042 Node->getValueType(0))); 3043 Results.push_back(Results[0].getValue(1)); 3044 break; 3045 } 3046 case ISD::FSUB: { 3047 EVT VT = Node->getValueType(0); 3048 assert(TLI.isOperationLegalOrCustom(ISD::FADD, VT) && 3049 TLI.isOperationLegalOrCustom(ISD::FNEG, VT) && 3050 "Don't know how to expand this FP subtraction!"); 3051 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); 3052 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1); 3053 Results.push_back(Tmp1); 3054 break; 3055 } 3056 case ISD::SUB: { 3057 EVT VT = Node->getValueType(0); 3058 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && 3059 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && 3060 "Don't know how to expand this subtraction!"); 3061 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), 3062 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT)); 3063 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp2, DAG.getConstant(1, VT)); 3064 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); 3065 break; 3066 } 3067 case ISD::UREM: 3068 case ISD::SREM: { 3069 EVT VT = Node->getValueType(0); 3070 SDVTList VTs = DAG.getVTList(VT, VT); 3071 bool isSigned = Node->getOpcode() == ISD::SREM; 3072 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; 3073 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3074 Tmp2 = Node->getOperand(0); 3075 Tmp3 = Node->getOperand(1); 3076 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3077 (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3078 UseDivRem(Node, isSigned, false))) { 3079 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); 3080 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { 3081 // X % Y -> X-X/Y*Y 3082 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); 3083 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); 3084 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); 3085 } else if (isSigned) 3086 Tmp1 = ExpandIntLibCall(Node, true, 3087 RTLIB::SREM_I8, 3088 RTLIB::SREM_I16, RTLIB::SREM_I32, 3089 RTLIB::SREM_I64, RTLIB::SREM_I128); 3090 else 3091 Tmp1 = ExpandIntLibCall(Node, false, 3092 RTLIB::UREM_I8, 3093 RTLIB::UREM_I16, RTLIB::UREM_I32, 3094 RTLIB::UREM_I64, RTLIB::UREM_I128); 3095 Results.push_back(Tmp1); 3096 break; 3097 } 3098 case ISD::UDIV: 3099 case ISD::SDIV: { 3100 bool isSigned = Node->getOpcode() == ISD::SDIV; 3101 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3102 EVT VT = Node->getValueType(0); 3103 SDVTList VTs = DAG.getVTList(VT, VT); 3104 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3105 (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3106 UseDivRem(Node, isSigned, true))) 3107 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), 3108 Node->getOperand(1)); 3109 else if (isSigned) 3110 Tmp1 = ExpandIntLibCall(Node, true, 3111 RTLIB::SDIV_I8, 3112 RTLIB::SDIV_I16, RTLIB::SDIV_I32, 3113 RTLIB::SDIV_I64, RTLIB::SDIV_I128); 3114 else 3115 Tmp1 = ExpandIntLibCall(Node, false, 3116 RTLIB::UDIV_I8, 3117 RTLIB::UDIV_I16, RTLIB::UDIV_I32, 3118 RTLIB::UDIV_I64, RTLIB::UDIV_I128); 3119 Results.push_back(Tmp1); 3120 break; 3121 } 3122 case ISD::MULHU: 3123 case ISD::MULHS: { 3124 unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : 3125 ISD::SMUL_LOHI; 3126 EVT VT = Node->getValueType(0); 3127 SDVTList VTs = DAG.getVTList(VT, VT); 3128 assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) && 3129 "If this wasn't legal, it shouldn't have been created!"); 3130 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), 3131 Node->getOperand(1)); 3132 Results.push_back(Tmp1.getValue(1)); 3133 break; 3134 } 3135 case ISD::SDIVREM: 3136 case ISD::UDIVREM: 3137 // Expand into divrem libcall 3138 ExpandDivRemLibCall(Node, Results); 3139 break; 3140 case ISD::MUL: { 3141 EVT VT = Node->getValueType(0); 3142 SDVTList VTs = DAG.getVTList(VT, VT); 3143 // See if multiply or divide can be lowered using two-result operations. 3144 // We just need the low half of the multiply; try both the signed 3145 // and unsigned forms. If the target supports both SMUL_LOHI and 3146 // UMUL_LOHI, form a preference by checking which forms of plain 3147 // MULH it supports. 3148 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3149 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3150 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3151 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3152 unsigned OpToUse = 0; 3153 if (HasSMUL_LOHI && !HasMULHS) { 3154 OpToUse = ISD::SMUL_LOHI; 3155 } else if (HasUMUL_LOHI && !HasMULHU) { 3156 OpToUse = ISD::UMUL_LOHI; 3157 } else if (HasSMUL_LOHI) { 3158 OpToUse = ISD::SMUL_LOHI; 3159 } else if (HasUMUL_LOHI) { 3160 OpToUse = ISD::UMUL_LOHI; 3161 } 3162 if (OpToUse) { 3163 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), 3164 Node->getOperand(1))); 3165 break; 3166 } 3167 Tmp1 = ExpandIntLibCall(Node, false, 3168 RTLIB::MUL_I8, 3169 RTLIB::MUL_I16, RTLIB::MUL_I32, 3170 RTLIB::MUL_I64, RTLIB::MUL_I128); 3171 Results.push_back(Tmp1); 3172 break; 3173 } 3174 case ISD::SADDO: 3175 case ISD::SSUBO: { 3176 SDValue LHS = Node->getOperand(0); 3177 SDValue RHS = Node->getOperand(1); 3178 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3179 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3180 LHS, RHS); 3181 Results.push_back(Sum); 3182 EVT OType = Node->getValueType(1); 3183 3184 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 3185 3186 // LHSSign -> LHS >= 0 3187 // RHSSign -> RHS >= 0 3188 // SumSign -> Sum >= 0 3189 // 3190 // Add: 3191 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 3192 // Sub: 3193 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 3194 // 3195 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 3196 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 3197 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 3198 Node->getOpcode() == ISD::SADDO ? 3199 ISD::SETEQ : ISD::SETNE); 3200 3201 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 3202 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 3203 3204 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 3205 Results.push_back(Cmp); 3206 break; 3207 } 3208 case ISD::UADDO: 3209 case ISD::USUBO: { 3210 SDValue LHS = Node->getOperand(0); 3211 SDValue RHS = Node->getOperand(1); 3212 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? 3213 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3214 LHS, RHS); 3215 Results.push_back(Sum); 3216 Results.push_back(DAG.getSetCC(dl, Node->getValueType(1), Sum, LHS, 3217 Node->getOpcode () == ISD::UADDO ? 3218 ISD::SETULT : ISD::SETUGT)); 3219 break; 3220 } 3221 case ISD::UMULO: 3222 case ISD::SMULO: { 3223 EVT VT = Node->getValueType(0); 3224 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); 3225 SDValue LHS = Node->getOperand(0); 3226 SDValue RHS = Node->getOperand(1); 3227 SDValue BottomHalf; 3228 SDValue TopHalf; 3229 static const unsigned Ops[2][3] = 3230 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, 3231 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; 3232 bool isSigned = Node->getOpcode() == ISD::SMULO; 3233 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) { 3234 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3235 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); 3236 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) { 3237 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, 3238 RHS); 3239 TopHalf = BottomHalf.getValue(1); 3240 } else if (TLI.isTypeLegal(EVT::getIntegerVT(*DAG.getContext(), 3241 VT.getSizeInBits() * 2))) { 3242 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); 3243 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); 3244 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 3245 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3246 DAG.getIntPtrConstant(0)); 3247 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3248 DAG.getIntPtrConstant(1)); 3249 } else { 3250 // We can fall back to a libcall with an illegal type for the MUL if we 3251 // have a libcall big enough. 3252 // Also, we can fall back to a division in some cases, but that's a big 3253 // performance hit in the general case. 3254 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3255 if (WideVT == MVT::i16) 3256 LC = RTLIB::MUL_I16; 3257 else if (WideVT == MVT::i32) 3258 LC = RTLIB::MUL_I32; 3259 else if (WideVT == MVT::i64) 3260 LC = RTLIB::MUL_I64; 3261 else if (WideVT == MVT::i128) 3262 LC = RTLIB::MUL_I128; 3263 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); 3264 3265 // The high part is obtained by SRA'ing all but one of the bits of low 3266 // part. 3267 unsigned LoSize = VT.getSizeInBits(); 3268 SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS, 3269 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3270 SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS, 3271 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3272 3273 // Here we're passing the 2 arguments explicitly as 4 arguments that are 3274 // pre-lowered to the correct types. This all depends upon WideVT not 3275 // being a legal type for the architecture and thus has to be split to 3276 // two arguments. 3277 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; 3278 SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); 3279 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3280 DAG.getIntPtrConstant(0)); 3281 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3282 DAG.getIntPtrConstant(1)); 3283 // Ret is a node with an illegal type. Because such things are not 3284 // generally permitted during this phase of legalization, delete the 3285 // node. The above EXTRACT_ELEMENT nodes should have been folded. 3286 DAG.DeleteNode(Ret.getNode()); 3287 } 3288 3289 if (isSigned) { 3290 Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1, 3291 TLI.getShiftAmountTy(BottomHalf.getValueType())); 3292 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); 3293 TopHalf = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), TopHalf, Tmp1, 3294 ISD::SETNE); 3295 } else { 3296 TopHalf = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), TopHalf, 3297 DAG.getConstant(0, VT), ISD::SETNE); 3298 } 3299 Results.push_back(BottomHalf); 3300 Results.push_back(TopHalf); 3301 break; 3302 } 3303 case ISD::BUILD_PAIR: { 3304 EVT PairTy = Node->getValueType(0); 3305 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); 3306 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); 3307 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, 3308 DAG.getConstant(PairTy.getSizeInBits()/2, 3309 TLI.getShiftAmountTy(PairTy))); 3310 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); 3311 break; 3312 } 3313 case ISD::SELECT: 3314 Tmp1 = Node->getOperand(0); 3315 Tmp2 = Node->getOperand(1); 3316 Tmp3 = Node->getOperand(2); 3317 if (Tmp1.getOpcode() == ISD::SETCC) { 3318 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 3319 Tmp2, Tmp3, 3320 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 3321 } else { 3322 Tmp1 = DAG.getSelectCC(dl, Tmp1, 3323 DAG.getConstant(0, Tmp1.getValueType()), 3324 Tmp2, Tmp3, ISD::SETNE); 3325 } 3326 Results.push_back(Tmp1); 3327 break; 3328 case ISD::BR_JT: { 3329 SDValue Chain = Node->getOperand(0); 3330 SDValue Table = Node->getOperand(1); 3331 SDValue Index = Node->getOperand(2); 3332 3333 EVT PTy = TLI.getPointerTy(); 3334 3335 const TargetData &TD = *TLI.getTargetData(); 3336 unsigned EntrySize = 3337 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); 3338 3339 Index = DAG.getNode(ISD::MUL, dl, PTy, 3340 Index, DAG.getConstant(EntrySize, PTy)); 3341 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table); 3342 3343 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); 3344 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, 3345 MachinePointerInfo::getJumpTable(), MemVT, 3346 false, false, 0); 3347 Addr = LD; 3348 if (TM.getRelocationModel() == Reloc::PIC_) { 3349 // For PIC, the sequence is: 3350 // BRIND(load(Jumptable + index) + RelocBase) 3351 // RelocBase can be JumpTable, GOT or some sort of global base. 3352 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 3353 TLI.getPICJumpTableRelocBase(Table, DAG)); 3354 } 3355 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); 3356 Results.push_back(Tmp1); 3357 break; 3358 } 3359 case ISD::BRCOND: 3360 // Expand brcond's setcc into its constituent parts and create a BR_CC 3361 // Node. 3362 Tmp1 = Node->getOperand(0); 3363 Tmp2 = Node->getOperand(1); 3364 if (Tmp2.getOpcode() == ISD::SETCC) { 3365 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 3366 Tmp1, Tmp2.getOperand(2), 3367 Tmp2.getOperand(0), Tmp2.getOperand(1), 3368 Node->getOperand(2)); 3369 } else { 3370 // We test only the i1 bit. Skip the AND if UNDEF. 3371 Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 : 3372 DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, 3373 DAG.getConstant(1, Tmp2.getValueType())); 3374 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 3375 DAG.getCondCode(ISD::SETNE), Tmp3, 3376 DAG.getConstant(0, Tmp3.getValueType()), 3377 Node->getOperand(2)); 3378 } 3379 Results.push_back(Tmp1); 3380 break; 3381 case ISD::SETCC: { 3382 Tmp1 = Node->getOperand(0); 3383 Tmp2 = Node->getOperand(1); 3384 Tmp3 = Node->getOperand(2); 3385 LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, Tmp3, dl); 3386 3387 // If we expanded the SETCC into an AND/OR, return the new node 3388 if (Tmp2.getNode() == 0) { 3389 Results.push_back(Tmp1); 3390 break; 3391 } 3392 3393 // Otherwise, SETCC for the given comparison type must be completely 3394 // illegal; expand it into a SELECT_CC. 3395 EVT VT = Node->getValueType(0); 3396 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3397 DAG.getConstant(1, VT), DAG.getConstant(0, VT), Tmp3); 3398 Results.push_back(Tmp1); 3399 break; 3400 } 3401 case ISD::SELECT_CC: { 3402 Tmp1 = Node->getOperand(0); // LHS 3403 Tmp2 = Node->getOperand(1); // RHS 3404 Tmp3 = Node->getOperand(2); // True 3405 Tmp4 = Node->getOperand(3); // False 3406 SDValue CC = Node->getOperand(4); 3407 3408 LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp1.getValueType()), 3409 Tmp1, Tmp2, CC, dl); 3410 3411 assert(!Tmp2.getNode() && "Can't legalize SELECT_CC with legal condition!"); 3412 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 3413 CC = DAG.getCondCode(ISD::SETNE); 3414 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, Tmp2, 3415 Tmp3, Tmp4, CC); 3416 Results.push_back(Tmp1); 3417 break; 3418 } 3419 case ISD::BR_CC: { 3420 Tmp1 = Node->getOperand(0); // Chain 3421 Tmp2 = Node->getOperand(2); // LHS 3422 Tmp3 = Node->getOperand(3); // RHS 3423 Tmp4 = Node->getOperand(1); // CC 3424 3425 LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp2.getValueType()), 3426 Tmp2, Tmp3, Tmp4, dl); 3427 3428 assert(!Tmp3.getNode() && "Can't legalize BR_CC with legal condition!"); 3429 Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); 3430 Tmp4 = DAG.getCondCode(ISD::SETNE); 3431 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, Tmp2, 3432 Tmp3, Node->getOperand(4)); 3433 Results.push_back(Tmp1); 3434 break; 3435 } 3436 case ISD::BUILD_VECTOR: 3437 Results.push_back(ExpandBUILD_VECTOR(Node)); 3438 break; 3439 case ISD::SRA: 3440 case ISD::SRL: 3441 case ISD::SHL: { 3442 // Scalarize vector SRA/SRL/SHL. 3443 EVT VT = Node->getValueType(0); 3444 assert(VT.isVector() && "Unable to legalize non-vector shift"); 3445 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); 3446 unsigned NumElem = VT.getVectorNumElements(); 3447 3448 SmallVector<SDValue, 8> Scalars; 3449 for (unsigned Idx = 0; Idx < NumElem; Idx++) { 3450 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3451 VT.getScalarType(), 3452 Node->getOperand(0), DAG.getIntPtrConstant(Idx)); 3453 SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3454 VT.getScalarType(), 3455 Node->getOperand(1), DAG.getIntPtrConstant(Idx)); 3456 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, 3457 VT.getScalarType(), Ex, Sh)); 3458 } 3459 SDValue Result = 3460 DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), 3461 &Scalars[0], Scalars.size()); 3462 ReplaceNode(SDValue(Node, 0), Result); 3463 break; 3464 } 3465 case ISD::GLOBAL_OFFSET_TABLE: 3466 case ISD::GlobalAddress: 3467 case ISD::GlobalTLSAddress: 3468 case ISD::ExternalSymbol: 3469 case ISD::ConstantPool: 3470 case ISD::JumpTable: 3471 case ISD::INTRINSIC_W_CHAIN: 3472 case ISD::INTRINSIC_WO_CHAIN: 3473 case ISD::INTRINSIC_VOID: 3474 // FIXME: Custom lowering for these operations shouldn't return null! 3475 break; 3476 } 3477 3478 // Replace the original node with the legalized result. 3479 if (!Results.empty()) 3480 ReplaceNode(Node, Results.data()); 3481} 3482 3483void SelectionDAGLegalize::PromoteNode(SDNode *Node) { 3484 SmallVector<SDValue, 8> Results; 3485 EVT OVT = Node->getValueType(0); 3486 if (Node->getOpcode() == ISD::UINT_TO_FP || 3487 Node->getOpcode() == ISD::SINT_TO_FP || 3488 Node->getOpcode() == ISD::SETCC) { 3489 OVT = Node->getOperand(0).getValueType(); 3490 } 3491 EVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3492 DebugLoc dl = Node->getDebugLoc(); 3493 SDValue Tmp1, Tmp2, Tmp3; 3494 switch (Node->getOpcode()) { 3495 case ISD::CTTZ: 3496 case ISD::CTTZ_ZERO_UNDEF: 3497 case ISD::CTLZ: 3498 case ISD::CTLZ_ZERO_UNDEF: 3499 case ISD::CTPOP: 3500 // Zero extend the argument. 3501 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 3502 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is 3503 // already the correct result. 3504 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 3505 if (Node->getOpcode() == ISD::CTTZ) { 3506 // FIXME: This should set a bit in the zero extended value instead. 3507 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), 3508 Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), 3509 ISD::SETEQ); 3510 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, 3511 DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); 3512 } else if (Node->getOpcode() == ISD::CTLZ || 3513 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { 3514 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 3515 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 3516 DAG.getConstant(NVT.getSizeInBits() - 3517 OVT.getSizeInBits(), NVT)); 3518 } 3519 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 3520 break; 3521 case ISD::BSWAP: { 3522 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 3523 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 3524 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 3525 Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 3526 DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT))); 3527 Results.push_back(Tmp1); 3528 break; 3529 } 3530 case ISD::FP_TO_UINT: 3531 case ISD::FP_TO_SINT: 3532 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), 3533 Node->getOpcode() == ISD::FP_TO_SINT, dl); 3534 Results.push_back(Tmp1); 3535 break; 3536 case ISD::UINT_TO_FP: 3537 case ISD::SINT_TO_FP: 3538 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), 3539 Node->getOpcode() == ISD::SINT_TO_FP, dl); 3540 Results.push_back(Tmp1); 3541 break; 3542 case ISD::VAARG: { 3543 SDValue Chain = Node->getOperand(0); // Get the chain. 3544 SDValue Ptr = Node->getOperand(1); // Get the pointer. 3545 3546 unsigned TruncOp; 3547 if (OVT.isVector()) { 3548 TruncOp = ISD::BITCAST; 3549 } else { 3550 assert(OVT.isInteger() 3551 && "VAARG promotion is supported only for vectors or integer types"); 3552 TruncOp = ISD::TRUNCATE; 3553 } 3554 3555 // Perform the larger operation, then convert back 3556 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), 3557 Node->getConstantOperandVal(3)); 3558 Chain = Tmp1.getValue(1); 3559 3560 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); 3561 3562 // Modified the chain result - switch anything that used the old chain to 3563 // use the new one. 3564 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); 3565 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 3566 ReplacedNode(Node); 3567 break; 3568 } 3569 case ISD::AND: 3570 case ISD::OR: 3571 case ISD::XOR: { 3572 unsigned ExtOp, TruncOp; 3573 if (OVT.isVector()) { 3574 ExtOp = ISD::BITCAST; 3575 TruncOp = ISD::BITCAST; 3576 } else { 3577 assert(OVT.isInteger() && "Cannot promote logic operation"); 3578 ExtOp = ISD::ANY_EXTEND; 3579 TruncOp = ISD::TRUNCATE; 3580 } 3581 // Promote each of the values to the new type. 3582 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 3583 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3584 // Perform the larger operation, then convert back 3585 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3586 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); 3587 break; 3588 } 3589 case ISD::SELECT: { 3590 unsigned ExtOp, TruncOp; 3591 if (Node->getValueType(0).isVector()) { 3592 ExtOp = ISD::BITCAST; 3593 TruncOp = ISD::BITCAST; 3594 } else if (Node->getValueType(0).isInteger()) { 3595 ExtOp = ISD::ANY_EXTEND; 3596 TruncOp = ISD::TRUNCATE; 3597 } else { 3598 ExtOp = ISD::FP_EXTEND; 3599 TruncOp = ISD::FP_ROUND; 3600 } 3601 Tmp1 = Node->getOperand(0); 3602 // Promote each of the values to the new type. 3603 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3604 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 3605 // Perform the larger operation, then round down. 3606 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2, Tmp3); 3607 if (TruncOp != ISD::FP_ROUND) 3608 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); 3609 else 3610 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, 3611 DAG.getIntPtrConstant(0)); 3612 Results.push_back(Tmp1); 3613 break; 3614 } 3615 case ISD::VECTOR_SHUFFLE: { 3616 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 3617 3618 // Cast the two input vectors. 3619 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); 3620 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); 3621 3622 // Convert the shuffle mask to the right # elements. 3623 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); 3624 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); 3625 Results.push_back(Tmp1); 3626 break; 3627 } 3628 case ISD::SETCC: { 3629 unsigned ExtOp = ISD::FP_EXTEND; 3630 if (NVT.isInteger()) { 3631 ISD::CondCode CCCode = 3632 cast<CondCodeSDNode>(Node->getOperand(2))->get(); 3633 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 3634 } 3635 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 3636 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3637 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 3638 Tmp1, Tmp2, Node->getOperand(2))); 3639 break; 3640 } 3641 case ISD::FDIV: 3642 case ISD::FREM: 3643 case ISD::FPOW: { 3644 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 3645 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 3646 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3647 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 3648 Tmp3, DAG.getIntPtrConstant(0))); 3649 break; 3650 } 3651 case ISD::FLOG2: 3652 case ISD::FEXP2: 3653 case ISD::FLOG: 3654 case ISD::FEXP: { 3655 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 3656 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 3657 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 3658 Tmp2, DAG.getIntPtrConstant(0))); 3659 break; 3660 } 3661 } 3662 3663 // Replace the original node with the legalized result. 3664 if (!Results.empty()) 3665 ReplaceNode(Node, Results.data()); 3666} 3667 3668// SelectionDAG::Legalize - This is the entry point for the file. 3669// 3670void SelectionDAG::Legalize() { 3671 /// run - This is the main entry point to this class. 3672 /// 3673 SelectionDAGLegalize(*this).LegalizeDAG(); 3674} 3675