LegalizeDAG.cpp revision bfc55eea7e43acf7d966c433e85d633095a11440
1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SelectionDAG::Legalize method. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "llvm/CodeGen/MachineFunction.h" 16#include "llvm/CodeGen/MachineFrameInfo.h" 17#include "llvm/CodeGen/MachineJumpTableInfo.h" 18#include "llvm/CodeGen/MachineModuleInfo.h" 19#include "llvm/CodeGen/PseudoSourceValue.h" 20#include "llvm/Target/TargetFrameInfo.h" 21#include "llvm/Target/TargetLowering.h" 22#include "llvm/Target/TargetData.h" 23#include "llvm/Target/TargetMachine.h" 24#include "llvm/Target/TargetOptions.h" 25#include "llvm/Target/TargetSubtarget.h" 26#include "llvm/CallingConv.h" 27#include "llvm/Constants.h" 28#include "llvm/DerivedTypes.h" 29#include "llvm/Support/CommandLine.h" 30#include "llvm/Support/Compiler.h" 31#include "llvm/Support/MathExtras.h" 32#include "llvm/ADT/DenseMap.h" 33#include "llvm/ADT/SmallVector.h" 34#include "llvm/ADT/SmallPtrSet.h" 35#include <map> 36using namespace llvm; 37 38//===----------------------------------------------------------------------===// 39/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and 40/// hacks on it until the target machine can handle it. This involves 41/// eliminating value sizes the machine cannot handle (promoting small sizes to 42/// large sizes or splitting up large values into small values) as well as 43/// eliminating operations the machine cannot handle. 44/// 45/// This code also does a small amount of optimization and recognition of idioms 46/// as part of its processing. For example, if a target does not support a 47/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 48/// will attempt merge setcc and brc instructions into brcc's. 49/// 50namespace { 51class VISIBILITY_HIDDEN SelectionDAGLegalize { 52 TargetLowering &TLI; 53 SelectionDAG &DAG; 54 55 // Libcall insertion helpers. 56 57 /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been 58 /// legalized. We use this to ensure that calls are properly serialized 59 /// against each other, including inserted libcalls. 60 SDValue LastCALLSEQ_END; 61 62 /// IsLegalizingCall - This member is used *only* for purposes of providing 63 /// helpful assertions that a libcall isn't created while another call is 64 /// being legalized (which could lead to non-serialized call sequences). 65 bool IsLegalizingCall; 66 67 enum LegalizeAction { 68 Legal, // The target natively supports this operation. 69 Promote, // This operation should be executed in a larger type. 70 Expand // Try to expand this to other ops, otherwise use a libcall. 71 }; 72 73 /// ValueTypeActions - This is a bitvector that contains two bits for each 74 /// value type, where the two bits correspond to the LegalizeAction enum. 75 /// This can be queried with "getTypeAction(VT)". 76 TargetLowering::ValueTypeActionImpl ValueTypeActions; 77 78 /// LegalizedNodes - For nodes that are of legal width, and that have more 79 /// than one use, this map indicates what regularized operand to use. This 80 /// allows us to avoid legalizing the same thing more than once. 81 DenseMap<SDValue, SDValue> LegalizedNodes; 82 83 /// PromotedNodes - For nodes that are below legal width, and that have more 84 /// than one use, this map indicates what promoted value to use. This allows 85 /// us to avoid promoting the same thing more than once. 86 DenseMap<SDValue, SDValue> PromotedNodes; 87 88 /// ExpandedNodes - For nodes that need to be expanded this map indicates 89 /// which operands are the expanded version of the input. This allows 90 /// us to avoid expanding the same node more than once. 91 DenseMap<SDValue, std::pair<SDValue, SDValue> > ExpandedNodes; 92 93 /// SplitNodes - For vector nodes that need to be split, this map indicates 94 /// which operands are the split version of the input. This allows us 95 /// to avoid splitting the same node more than once. 96 std::map<SDValue, std::pair<SDValue, SDValue> > SplitNodes; 97 98 /// ScalarizedNodes - For nodes that need to be converted from vector types to 99 /// scalar types, this contains the mapping of ones we have already 100 /// processed to the result. 101 std::map<SDValue, SDValue> ScalarizedNodes; 102 103 /// WidenNodes - For nodes that need to be widened from one vector type to 104 /// another, this contains the mapping of those that we have already widen. 105 /// This allows us to avoid widening more than once. 106 std::map<SDValue, SDValue> WidenNodes; 107 108 void AddLegalizedOperand(SDValue From, SDValue To) { 109 LegalizedNodes.insert(std::make_pair(From, To)); 110 // If someone requests legalization of the new node, return itself. 111 if (From != To) 112 LegalizedNodes.insert(std::make_pair(To, To)); 113 } 114 void AddPromotedOperand(SDValue From, SDValue To) { 115 bool isNew = PromotedNodes.insert(std::make_pair(From, To)).second; 116 assert(isNew && "Got into the map somehow?"); 117 isNew = isNew; 118 // If someone requests legalization of the new node, return itself. 119 LegalizedNodes.insert(std::make_pair(To, To)); 120 } 121 void AddWidenedOperand(SDValue From, SDValue To) { 122 bool isNew = WidenNodes.insert(std::make_pair(From, To)).second; 123 assert(isNew && "Got into the map somehow?"); 124 isNew = isNew; 125 // If someone requests legalization of the new node, return itself. 126 LegalizedNodes.insert(std::make_pair(To, To)); 127 } 128 129public: 130 explicit SelectionDAGLegalize(SelectionDAG &DAG); 131 132 /// getTypeAction - Return how we should legalize values of this type, either 133 /// it is already legal or we need to expand it into multiple registers of 134 /// smaller integer type, or we need to promote it to a larger type. 135 LegalizeAction getTypeAction(MVT VT) const { 136 return (LegalizeAction)ValueTypeActions.getTypeAction(VT); 137 } 138 139 /// isTypeLegal - Return true if this type is legal on this target. 140 /// 141 bool isTypeLegal(MVT VT) const { 142 return getTypeAction(VT) == Legal; 143 } 144 145 void LegalizeDAG(); 146 147private: 148 /// HandleOp - Legalize, Promote, or Expand the specified operand as 149 /// appropriate for its type. 150 void HandleOp(SDValue Op); 151 152 /// LegalizeOp - We know that the specified value has a legal type. 153 /// Recursively ensure that the operands have legal types, then return the 154 /// result. 155 SDValue LegalizeOp(SDValue O); 156 157 /// UnrollVectorOp - We know that the given vector has a legal type, however 158 /// the operation it performs is not legal and is an operation that we have 159 /// no way of lowering. "Unroll" the vector, splitting out the scalars and 160 /// operating on each element individually. 161 SDValue UnrollVectorOp(SDValue O); 162 163 /// PerformInsertVectorEltInMemory - Some target cannot handle a variable 164 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 165 /// is necessary to spill the vector being inserted into to memory, perform 166 /// the insert there, and then read the result back. 167 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, 168 SDValue Idx); 169 170 /// PromoteOp - Given an operation that produces a value in an invalid type, 171 /// promote it to compute the value into a larger type. The produced value 172 /// will have the correct bits for the low portion of the register, but no 173 /// guarantee is made about the top bits: it may be zero, sign-extended, or 174 /// garbage. 175 SDValue PromoteOp(SDValue O); 176 177 /// ExpandOp - Expand the specified SDValue into its two component pieces 178 /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, 179 /// the LegalizedNodes map is filled in for any results that are not expanded, 180 /// the ExpandedNodes map is filled in for any results that are expanded, and 181 /// the Lo/Hi values are returned. This applies to integer types and Vector 182 /// types. 183 void ExpandOp(SDValue O, SDValue &Lo, SDValue &Hi); 184 185 /// WidenVectorOp - Widen a vector operation to a wider type given by WidenVT 186 /// (e.g., v3i32 to v4i32). The produced value will have the correct value 187 /// for the existing elements but no guarantee is made about the new elements 188 /// at the end of the vector: it may be zero, ones, or garbage. This is useful 189 /// when we have an instruction operating on an illegal vector type and we 190 /// want to widen it to do the computation on a legal wider vector type. 191 SDValue WidenVectorOp(SDValue Op, MVT WidenVT); 192 193 /// SplitVectorOp - Given an operand of vector type, break it down into 194 /// two smaller values. 195 void SplitVectorOp(SDValue O, SDValue &Lo, SDValue &Hi); 196 197 /// ScalarizeVectorOp - Given an operand of single-element vector type 198 /// (e.g. v1f32), convert it into the equivalent operation that returns a 199 /// scalar (e.g. f32) value. 200 SDValue ScalarizeVectorOp(SDValue O); 201 202 /// Useful 16 element vector type that is used to pass operands for widening. 203 typedef SmallVector<SDValue, 16> SDValueVector; 204 205 /// LoadWidenVectorOp - Load a vector for a wider type. Returns true if 206 /// the LdChain contains a single load and false if it contains a token 207 /// factor for multiple loads. It takes 208 /// Result: location to return the result 209 /// LdChain: location to return the load chain 210 /// Op: load operation to widen 211 /// NVT: widen vector result type we want for the load 212 bool LoadWidenVectorOp(SDValue& Result, SDValue& LdChain, 213 SDValue Op, MVT NVT); 214 215 /// Helper genWidenVectorLoads - Helper function to generate a set of 216 /// loads to load a vector with a resulting wider type. It takes 217 /// LdChain: list of chains for the load we have generated 218 /// Chain: incoming chain for the ld vector 219 /// BasePtr: base pointer to load from 220 /// SV: memory disambiguation source value 221 /// SVOffset: memory disambiugation offset 222 /// Alignment: alignment of the memory 223 /// isVolatile: volatile load 224 /// LdWidth: width of memory that we want to load 225 /// ResType: the wider result result type for the resulting loaded vector 226 SDValue genWidenVectorLoads(SDValueVector& LdChain, SDValue Chain, 227 SDValue BasePtr, const Value *SV, 228 int SVOffset, unsigned Alignment, 229 bool isVolatile, unsigned LdWidth, 230 MVT ResType); 231 232 /// StoreWidenVectorOp - Stores a widen vector into non widen memory 233 /// location. It takes 234 /// ST: store node that we want to replace 235 /// Chain: incoming store chain 236 /// BasePtr: base address of where we want to store into 237 SDValue StoreWidenVectorOp(StoreSDNode *ST, SDValue Chain, 238 SDValue BasePtr); 239 240 /// Helper genWidenVectorStores - Helper function to generate a set of 241 /// stores to store a widen vector into non widen memory 242 // It takes 243 // StChain: list of chains for the stores we have generated 244 // Chain: incoming chain for the ld vector 245 // BasePtr: base pointer to load from 246 // SV: memory disambiguation source value 247 // SVOffset: memory disambiugation offset 248 // Alignment: alignment of the memory 249 // isVolatile: volatile lod 250 // ValOp: value to store 251 // StWidth: width of memory that we want to store 252 void genWidenVectorStores(SDValueVector& StChain, SDValue Chain, 253 SDValue BasePtr, const Value *SV, 254 int SVOffset, unsigned Alignment, 255 bool isVolatile, SDValue ValOp, 256 unsigned StWidth); 257 258 /// isShuffleLegal - Return non-null if a vector shuffle is legal with the 259 /// specified mask and type. Targets can specify exactly which masks they 260 /// support and the code generator is tasked with not creating illegal masks. 261 /// 262 /// Note that this will also return true for shuffles that are promoted to a 263 /// different type. 264 /// 265 /// If this is a legal shuffle, this method returns the (possibly promoted) 266 /// build_vector Mask. If it's not a legal shuffle, it returns null. 267 SDNode *isShuffleLegal(MVT VT, SDValue Mask) const; 268 269 bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, 270 SmallPtrSet<SDNode*, 32> &NodesLeadingTo); 271 272 void LegalizeSetCCOperands(SDValue &LHS, SDValue &RHS, SDValue &CC); 273 void LegalizeSetCCCondCode(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC); 274 void LegalizeSetCC(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC) { 275 LegalizeSetCCOperands(LHS, RHS, CC); 276 LegalizeSetCCCondCode(VT, LHS, RHS, CC); 277 } 278 279 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned, 280 SDValue &Hi); 281 SDValue ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source); 282 283 SDValue EmitStackConvert(SDValue SrcOp, MVT SlotVT, MVT DestVT); 284 SDValue ExpandBUILD_VECTOR(SDNode *Node); 285 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 286 SDValue LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op); 287 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, MVT DestVT); 288 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, MVT DestVT, bool isSigned); 289 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, MVT DestVT, bool isSigned); 290 291 SDValue ExpandBSWAP(SDValue Op); 292 SDValue ExpandBitCount(unsigned Opc, SDValue Op); 293 bool ExpandShift(unsigned Opc, SDValue Op, SDValue Amt, 294 SDValue &Lo, SDValue &Hi); 295 void ExpandShiftParts(unsigned NodeOp, SDValue Op, SDValue Amt, 296 SDValue &Lo, SDValue &Hi); 297 298 SDValue ExpandEXTRACT_SUBVECTOR(SDValue Op); 299 SDValue ExpandEXTRACT_VECTOR_ELT(SDValue Op); 300}; 301} 302 303/// isVectorShuffleLegal - Return true if a vector shuffle is legal with the 304/// specified mask and type. Targets can specify exactly which masks they 305/// support and the code generator is tasked with not creating illegal masks. 306/// 307/// Note that this will also return true for shuffles that are promoted to a 308/// different type. 309SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDValue Mask) const { 310 switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) { 311 default: return 0; 312 case TargetLowering::Legal: 313 case TargetLowering::Custom: 314 break; 315 case TargetLowering::Promote: { 316 // If this is promoted to a different type, convert the shuffle mask and 317 // ask if it is legal in the promoted type! 318 MVT NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT); 319 MVT EltVT = NVT.getVectorElementType(); 320 321 // If we changed # elements, change the shuffle mask. 322 unsigned NumEltsGrowth = 323 NVT.getVectorNumElements() / VT.getVectorNumElements(); 324 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 325 if (NumEltsGrowth > 1) { 326 // Renumber the elements. 327 SmallVector<SDValue, 8> Ops; 328 for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) { 329 SDValue InOp = Mask.getOperand(i); 330 for (unsigned j = 0; j != NumEltsGrowth; ++j) { 331 if (InOp.getOpcode() == ISD::UNDEF) 332 Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 333 else { 334 unsigned InEltNo = cast<ConstantSDNode>(InOp)->getZExtValue(); 335 Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, EltVT)); 336 } 337 } 338 } 339 Mask = DAG.getNode(ISD::BUILD_VECTOR, NVT, &Ops[0], Ops.size()); 340 } 341 VT = NVT; 342 break; 343 } 344 } 345 return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.getNode() : 0; 346} 347 348SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) 349 : TLI(dag.getTargetLoweringInfo()), DAG(dag), 350 ValueTypeActions(TLI.getValueTypeActions()) { 351 assert(MVT::LAST_VALUETYPE <= 32 && 352 "Too many value types for ValueTypeActions to hold!"); 353} 354 355void SelectionDAGLegalize::LegalizeDAG() { 356 LastCALLSEQ_END = DAG.getEntryNode(); 357 IsLegalizingCall = false; 358 359 // The legalize process is inherently a bottom-up recursive process (users 360 // legalize their uses before themselves). Given infinite stack space, we 361 // could just start legalizing on the root and traverse the whole graph. In 362 // practice however, this causes us to run out of stack space on large basic 363 // blocks. To avoid this problem, compute an ordering of the nodes where each 364 // node is only legalized after all of its operands are legalized. 365 DAG.AssignTopologicalOrder(); 366 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 367 E = prior(DAG.allnodes_end()); I != next(E); ++I) 368 HandleOp(SDValue(I, 0)); 369 370 // Finally, it's possible the root changed. Get the new root. 371 SDValue OldRoot = DAG.getRoot(); 372 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?"); 373 DAG.setRoot(LegalizedNodes[OldRoot]); 374 375 ExpandedNodes.clear(); 376 LegalizedNodes.clear(); 377 PromotedNodes.clear(); 378 SplitNodes.clear(); 379 ScalarizedNodes.clear(); 380 WidenNodes.clear(); 381 382 // Remove dead nodes now. 383 DAG.RemoveDeadNodes(); 384} 385 386 387/// FindCallEndFromCallStart - Given a chained node that is part of a call 388/// sequence, find the CALLSEQ_END node that terminates the call sequence. 389static SDNode *FindCallEndFromCallStart(SDNode *Node) { 390 if (Node->getOpcode() == ISD::CALLSEQ_END) 391 return Node; 392 if (Node->use_empty()) 393 return 0; // No CallSeqEnd 394 395 // The chain is usually at the end. 396 SDValue TheChain(Node, Node->getNumValues()-1); 397 if (TheChain.getValueType() != MVT::Other) { 398 // Sometimes it's at the beginning. 399 TheChain = SDValue(Node, 0); 400 if (TheChain.getValueType() != MVT::Other) { 401 // Otherwise, hunt for it. 402 for (unsigned i = 1, e = Node->getNumValues(); i != e; ++i) 403 if (Node->getValueType(i) == MVT::Other) { 404 TheChain = SDValue(Node, i); 405 break; 406 } 407 408 // Otherwise, we walked into a node without a chain. 409 if (TheChain.getValueType() != MVT::Other) 410 return 0; 411 } 412 } 413 414 for (SDNode::use_iterator UI = Node->use_begin(), 415 E = Node->use_end(); UI != E; ++UI) { 416 417 // Make sure to only follow users of our token chain. 418 SDNode *User = *UI; 419 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) 420 if (User->getOperand(i) == TheChain) 421 if (SDNode *Result = FindCallEndFromCallStart(User)) 422 return Result; 423 } 424 return 0; 425} 426 427/// FindCallStartFromCallEnd - Given a chained node that is part of a call 428/// sequence, find the CALLSEQ_START node that initiates the call sequence. 429static SDNode *FindCallStartFromCallEnd(SDNode *Node) { 430 assert(Node && "Didn't find callseq_start for a call??"); 431 if (Node->getOpcode() == ISD::CALLSEQ_START) return Node; 432 433 assert(Node->getOperand(0).getValueType() == MVT::Other && 434 "Node doesn't have a token chain argument!"); 435 return FindCallStartFromCallEnd(Node->getOperand(0).getNode()); 436} 437 438/// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to 439/// see if any uses can reach Dest. If no dest operands can get to dest, 440/// legalize them, legalize ourself, and return false, otherwise, return true. 441/// 442/// Keep track of the nodes we fine that actually do lead to Dest in 443/// NodesLeadingTo. This avoids retraversing them exponential number of times. 444/// 445bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, 446 SmallPtrSet<SDNode*, 32> &NodesLeadingTo) { 447 if (N == Dest) return true; // N certainly leads to Dest :) 448 449 // If we've already processed this node and it does lead to Dest, there is no 450 // need to reprocess it. 451 if (NodesLeadingTo.count(N)) return true; 452 453 // If the first result of this node has been already legalized, then it cannot 454 // reach N. 455 switch (getTypeAction(N->getValueType(0))) { 456 case Legal: 457 if (LegalizedNodes.count(SDValue(N, 0))) return false; 458 break; 459 case Promote: 460 if (PromotedNodes.count(SDValue(N, 0))) return false; 461 break; 462 case Expand: 463 if (ExpandedNodes.count(SDValue(N, 0))) return false; 464 break; 465 } 466 467 // Okay, this node has not already been legalized. Check and legalize all 468 // operands. If none lead to Dest, then we can legalize this node. 469 bool OperandsLeadToDest = false; 470 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 471 OperandsLeadToDest |= // If an operand leads to Dest, so do we. 472 LegalizeAllNodesNotLeadingTo(N->getOperand(i).getNode(), Dest, NodesLeadingTo); 473 474 if (OperandsLeadToDest) { 475 NodesLeadingTo.insert(N); 476 return true; 477 } 478 479 // Okay, this node looks safe, legalize it and return false. 480 HandleOp(SDValue(N, 0)); 481 return false; 482} 483 484/// HandleOp - Legalize, Promote, Widen, or Expand the specified operand as 485/// appropriate for its type. 486void SelectionDAGLegalize::HandleOp(SDValue Op) { 487 MVT VT = Op.getValueType(); 488 switch (getTypeAction(VT)) { 489 default: assert(0 && "Bad type action!"); 490 case Legal: (void)LegalizeOp(Op); break; 491 case Promote: 492 if (!VT.isVector()) { 493 (void)PromoteOp(Op); 494 break; 495 } 496 else { 497 // See if we can widen otherwise use Expand to either scalarize or split 498 MVT WidenVT = TLI.getWidenVectorType(VT); 499 if (WidenVT != MVT::Other) { 500 (void) WidenVectorOp(Op, WidenVT); 501 break; 502 } 503 // else fall thru to expand since we can't widen the vector 504 } 505 case Expand: 506 if (!VT.isVector()) { 507 // If this is an illegal scalar, expand it into its two component 508 // pieces. 509 SDValue X, Y; 510 if (Op.getOpcode() == ISD::TargetConstant) 511 break; // Allow illegal target nodes. 512 ExpandOp(Op, X, Y); 513 } else if (VT.getVectorNumElements() == 1) { 514 // If this is an illegal single element vector, convert it to a 515 // scalar operation. 516 (void)ScalarizeVectorOp(Op); 517 } else { 518 // This is an illegal multiple element vector. 519 // Split it in half and legalize both parts. 520 SDValue X, Y; 521 SplitVectorOp(Op, X, Y); 522 } 523 break; 524 } 525} 526 527/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or 528/// a load from the constant pool. 529static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, 530 SelectionDAG &DAG, TargetLowering &TLI) { 531 bool Extend = false; 532 533 // If a FP immediate is precise when represented as a float and if the 534 // target can do an extending load from float to double, we put it into 535 // the constant pool as a float, even if it's is statically typed as a 536 // double. This shrinks FP constants and canonicalizes them for targets where 537 // an FP extending load is the same cost as a normal load (such as on the x87 538 // fp stack or PPC FP unit). 539 MVT VT = CFP->getValueType(0); 540 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 541 if (!UseCP) { 542 if (VT!=MVT::f64 && VT!=MVT::f32) 543 assert(0 && "Invalid type expansion"); 544 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), 545 (VT == MVT::f64) ? MVT::i64 : MVT::i32); 546 } 547 548 MVT OrigVT = VT; 549 MVT SVT = VT; 550 while (SVT != MVT::f32) { 551 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT() - 1); 552 if (CFP->isValueValidForType(SVT, CFP->getValueAPF()) && 553 // Only do this if the target has a native EXTLOAD instruction from 554 // smaller type. 555 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && 556 TLI.ShouldShrinkFPConstant(OrigVT)) { 557 const Type *SType = SVT.getTypeForMVT(); 558 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 559 VT = SVT; 560 Extend = true; 561 } 562 } 563 564 SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); 565 unsigned Alignment = 1 << cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 566 if (Extend) 567 return DAG.getExtLoad(ISD::EXTLOAD, OrigVT, DAG.getEntryNode(), 568 CPIdx, PseudoSourceValue::getConstantPool(), 569 0, VT, false, Alignment); 570 return DAG.getLoad(OrigVT, DAG.getEntryNode(), CPIdx, 571 PseudoSourceValue::getConstantPool(), 0, false, Alignment); 572} 573 574 575/// ExpandFCOPYSIGNToBitwiseOps - Expands fcopysign to a series of bitwise 576/// operations. 577static 578SDValue ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, 579 SelectionDAG &DAG, TargetLowering &TLI) { 580 MVT VT = Node->getValueType(0); 581 MVT SrcVT = Node->getOperand(1).getValueType(); 582 assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) && 583 "fcopysign expansion only supported for f32 and f64"); 584 MVT SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32; 585 586 // First get the sign bit of second operand. 587 SDValue Mask1 = (SrcVT == MVT::f64) 588 ? DAG.getConstantFP(BitsToDouble(1ULL << 63), SrcVT) 589 : DAG.getConstantFP(BitsToFloat(1U << 31), SrcVT); 590 Mask1 = DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Mask1); 591 SDValue SignBit= DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Node->getOperand(1)); 592 SignBit = DAG.getNode(ISD::AND, SrcNVT, SignBit, Mask1); 593 // Shift right or sign-extend it if the two operands have different types. 594 int SizeDiff = SrcNVT.getSizeInBits() - NVT.getSizeInBits(); 595 if (SizeDiff > 0) { 596 SignBit = DAG.getNode(ISD::SRL, SrcNVT, SignBit, 597 DAG.getConstant(SizeDiff, TLI.getShiftAmountTy())); 598 SignBit = DAG.getNode(ISD::TRUNCATE, NVT, SignBit); 599 } else if (SizeDiff < 0) { 600 SignBit = DAG.getNode(ISD::ZERO_EXTEND, NVT, SignBit); 601 SignBit = DAG.getNode(ISD::SHL, NVT, SignBit, 602 DAG.getConstant(-SizeDiff, TLI.getShiftAmountTy())); 603 } 604 605 // Clear the sign bit of first operand. 606 SDValue Mask2 = (VT == MVT::f64) 607 ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) 608 : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); 609 Mask2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask2); 610 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0)); 611 Result = DAG.getNode(ISD::AND, NVT, Result, Mask2); 612 613 // Or the value with the sign bit. 614 Result = DAG.getNode(ISD::OR, NVT, Result, SignBit); 615 return Result; 616} 617 618/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. 619static 620SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, 621 TargetLowering &TLI) { 622 SDValue Chain = ST->getChain(); 623 SDValue Ptr = ST->getBasePtr(); 624 SDValue Val = ST->getValue(); 625 MVT VT = Val.getValueType(); 626 int Alignment = ST->getAlignment(); 627 int SVOffset = ST->getSrcValueOffset(); 628 if (ST->getMemoryVT().isFloatingPoint() || 629 ST->getMemoryVT().isVector()) { 630 // Expand to a bitconvert of the value to the integer type of the 631 // same size, then a (misaligned) int store. 632 MVT intVT; 633 if (VT.is128BitVector() || VT == MVT::ppcf128 || VT == MVT::f128) 634 intVT = MVT::i128; 635 else if (VT.is64BitVector() || VT==MVT::f64) 636 intVT = MVT::i64; 637 else if (VT==MVT::f32) 638 intVT = MVT::i32; 639 else 640 assert(0 && "Unaligned store of unsupported type"); 641 642 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, intVT, Val); 643 return DAG.getStore(Chain, Result, Ptr, ST->getSrcValue(), 644 SVOffset, ST->isVolatile(), Alignment); 645 } 646 assert(ST->getMemoryVT().isInteger() && 647 !ST->getMemoryVT().isVector() && 648 "Unaligned store of unknown type."); 649 // Get the half-size VT 650 MVT NewStoredVT = 651 (MVT::SimpleValueType)(ST->getMemoryVT().getSimpleVT() - 1); 652 int NumBits = NewStoredVT.getSizeInBits(); 653 int IncrementSize = NumBits / 8; 654 655 // Divide the stored value in two parts. 656 SDValue ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); 657 SDValue Lo = Val; 658 SDValue Hi = DAG.getNode(ISD::SRL, VT, Val, ShiftAmount); 659 660 // Store the two parts 661 SDValue Store1, Store2; 662 Store1 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Lo:Hi, Ptr, 663 ST->getSrcValue(), SVOffset, NewStoredVT, 664 ST->isVolatile(), Alignment); 665 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 666 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 667 Alignment = MinAlign(Alignment, IncrementSize); 668 Store2 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Hi:Lo, Ptr, 669 ST->getSrcValue(), SVOffset + IncrementSize, 670 NewStoredVT, ST->isVolatile(), Alignment); 671 672 return DAG.getNode(ISD::TokenFactor, MVT::Other, Store1, Store2); 673} 674 675/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. 676static 677SDValue ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, 678 TargetLowering &TLI) { 679 int SVOffset = LD->getSrcValueOffset(); 680 SDValue Chain = LD->getChain(); 681 SDValue Ptr = LD->getBasePtr(); 682 MVT VT = LD->getValueType(0); 683 MVT LoadedVT = LD->getMemoryVT(); 684 if (VT.isFloatingPoint() || VT.isVector()) { 685 // Expand to a (misaligned) integer load of the same size, 686 // then bitconvert to floating point or vector. 687 MVT intVT; 688 if (LoadedVT.is128BitVector() || 689 LoadedVT == MVT::ppcf128 || LoadedVT == MVT::f128) 690 intVT = MVT::i128; 691 else if (LoadedVT.is64BitVector() || LoadedVT == MVT::f64) 692 intVT = MVT::i64; 693 else if (LoadedVT == MVT::f32) 694 intVT = MVT::i32; 695 else 696 assert(0 && "Unaligned load of unsupported type"); 697 698 SDValue newLoad = DAG.getLoad(intVT, Chain, Ptr, LD->getSrcValue(), 699 SVOffset, LD->isVolatile(), 700 LD->getAlignment()); 701 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, LoadedVT, newLoad); 702 if (VT.isFloatingPoint() && LoadedVT != VT) 703 Result = DAG.getNode(ISD::FP_EXTEND, VT, Result); 704 705 SDValue Ops[] = { Result, Chain }; 706 return DAG.getMergeValues(Ops, 2); 707 } 708 assert(LoadedVT.isInteger() && !LoadedVT.isVector() && 709 "Unaligned load of unsupported type."); 710 711 // Compute the new VT that is half the size of the old one. This is an 712 // integer MVT. 713 unsigned NumBits = LoadedVT.getSizeInBits(); 714 MVT NewLoadedVT; 715 NewLoadedVT = MVT::getIntegerVT(NumBits/2); 716 NumBits >>= 1; 717 718 unsigned Alignment = LD->getAlignment(); 719 unsigned IncrementSize = NumBits / 8; 720 ISD::LoadExtType HiExtType = LD->getExtensionType(); 721 722 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. 723 if (HiExtType == ISD::NON_EXTLOAD) 724 HiExtType = ISD::ZEXTLOAD; 725 726 // Load the value in two parts 727 SDValue Lo, Hi; 728 if (TLI.isLittleEndian()) { 729 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(), 730 SVOffset, NewLoadedVT, LD->isVolatile(), Alignment); 731 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 732 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 733 Hi = DAG.getExtLoad(HiExtType, VT, Chain, Ptr, LD->getSrcValue(), 734 SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(), 735 MinAlign(Alignment, IncrementSize)); 736 } else { 737 Hi = DAG.getExtLoad(HiExtType, VT, Chain, Ptr, LD->getSrcValue(), SVOffset, 738 NewLoadedVT,LD->isVolatile(), Alignment); 739 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 740 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 741 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(), 742 SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(), 743 MinAlign(Alignment, IncrementSize)); 744 } 745 746 // aggregate the two parts 747 SDValue ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); 748 SDValue Result = DAG.getNode(ISD::SHL, VT, Hi, ShiftAmount); 749 Result = DAG.getNode(ISD::OR, VT, Result, Lo); 750 751 SDValue TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 752 Hi.getValue(1)); 753 754 SDValue Ops[] = { Result, TF }; 755 return DAG.getMergeValues(Ops, 2); 756} 757 758/// UnrollVectorOp - We know that the given vector has a legal type, however 759/// the operation it performs is not legal and is an operation that we have 760/// no way of lowering. "Unroll" the vector, splitting out the scalars and 761/// operating on each element individually. 762SDValue SelectionDAGLegalize::UnrollVectorOp(SDValue Op) { 763 MVT VT = Op.getValueType(); 764 assert(isTypeLegal(VT) && 765 "Caller should expand or promote operands that are not legal!"); 766 assert(Op.getNode()->getNumValues() == 1 && 767 "Can't unroll a vector with multiple results!"); 768 unsigned NE = VT.getVectorNumElements(); 769 MVT EltVT = VT.getVectorElementType(); 770 771 SmallVector<SDValue, 8> Scalars; 772 SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 773 for (unsigned i = 0; i != NE; ++i) { 774 for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 775 SDValue Operand = Op.getOperand(j); 776 MVT OperandVT = Operand.getValueType(); 777 if (OperandVT.isVector()) { 778 // A vector operand; extract a single element. 779 MVT OperandEltVT = OperandVT.getVectorElementType(); 780 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, 781 OperandEltVT, 782 Operand, 783 DAG.getConstant(i, MVT::i32)); 784 } else { 785 // A scalar operand; just use it as is. 786 Operands[j] = Operand; 787 } 788 } 789 Scalars.push_back(DAG.getNode(Op.getOpcode(), EltVT, 790 &Operands[0], Operands.size())); 791 } 792 793 return DAG.getNode(ISD::BUILD_VECTOR, VT, &Scalars[0], Scalars.size()); 794} 795 796/// GetFPLibCall - Return the right libcall for the given floating point type. 797static RTLIB::Libcall GetFPLibCall(MVT VT, 798 RTLIB::Libcall Call_F32, 799 RTLIB::Libcall Call_F64, 800 RTLIB::Libcall Call_F80, 801 RTLIB::Libcall Call_PPCF128) { 802 return 803 VT == MVT::f32 ? Call_F32 : 804 VT == MVT::f64 ? Call_F64 : 805 VT == MVT::f80 ? Call_F80 : 806 VT == MVT::ppcf128 ? Call_PPCF128 : 807 RTLIB::UNKNOWN_LIBCALL; 808} 809 810/// PerformInsertVectorEltInMemory - Some target cannot handle a variable 811/// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 812/// is necessary to spill the vector being inserted into to memory, perform 813/// the insert there, and then read the result back. 814SDValue SelectionDAGLegalize:: 815PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx) { 816 SDValue Tmp1 = Vec; 817 SDValue Tmp2 = Val; 818 SDValue Tmp3 = Idx; 819 820 // If the target doesn't support this, we have to spill the input vector 821 // to a temporary stack slot, update the element, then reload it. This is 822 // badness. We could also load the value into a vector register (either 823 // with a "move to register" or "extload into register" instruction, then 824 // permute it into place, if the idx is a constant and if the idx is 825 // supported by the target. 826 MVT VT = Tmp1.getValueType(); 827 MVT EltVT = VT.getVectorElementType(); 828 MVT IdxVT = Tmp3.getValueType(); 829 MVT PtrVT = TLI.getPointerTy(); 830 SDValue StackPtr = DAG.CreateStackTemporary(VT); 831 832 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 833 834 // Store the vector. 835 SDValue Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr, 836 PseudoSourceValue::getFixedStack(SPFI), 0); 837 838 // Truncate or zero extend offset to target pointer type. 839 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 840 Tmp3 = DAG.getNode(CastOpc, PtrVT, Tmp3); 841 // Add the offset to the index. 842 unsigned EltSize = EltVT.getSizeInBits()/8; 843 Tmp3 = DAG.getNode(ISD::MUL, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); 844 SDValue StackPtr2 = DAG.getNode(ISD::ADD, IdxVT, Tmp3, StackPtr); 845 // Store the scalar value. 846 Ch = DAG.getTruncStore(Ch, Tmp2, StackPtr2, 847 PseudoSourceValue::getFixedStack(SPFI), 0, EltVT); 848 // Load the updated vector. 849 return DAG.getLoad(VT, Ch, StackPtr, 850 PseudoSourceValue::getFixedStack(SPFI), 0); 851} 852 853/// LegalizeOp - We know that the specified value has a legal type, and 854/// that its operands are legal. Now ensure that the operation itself 855/// is legal, recursively ensuring that the operands' operations remain 856/// legal. 857SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { 858 if (Op.getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. 859 return Op; 860 861 assert(isTypeLegal(Op.getValueType()) && 862 "Caller should expand or promote operands that are not legal!"); 863 SDNode *Node = Op.getNode(); 864 865 // If this operation defines any values that cannot be represented in a 866 // register on this target, make sure to expand or promote them. 867 if (Node->getNumValues() > 1) { 868 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 869 if (getTypeAction(Node->getValueType(i)) != Legal) { 870 HandleOp(Op.getValue(i)); 871 assert(LegalizedNodes.count(Op) && 872 "Handling didn't add legal operands!"); 873 return LegalizedNodes[Op]; 874 } 875 } 876 877 // Note that LegalizeOp may be reentered even from single-use nodes, which 878 // means that we always must cache transformed nodes. 879 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 880 if (I != LegalizedNodes.end()) return I->second; 881 882 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 883 SDValue Result = Op; 884 bool isCustom = false; 885 886 switch (Node->getOpcode()) { 887 case ISD::FrameIndex: 888 case ISD::EntryToken: 889 case ISD::Register: 890 case ISD::BasicBlock: 891 case ISD::TargetFrameIndex: 892 case ISD::TargetJumpTable: 893 case ISD::TargetConstant: 894 case ISD::TargetConstantFP: 895 case ISD::TargetConstantPool: 896 case ISD::TargetGlobalAddress: 897 case ISD::TargetGlobalTLSAddress: 898 case ISD::TargetExternalSymbol: 899 case ISD::VALUETYPE: 900 case ISD::SRCVALUE: 901 case ISD::MEMOPERAND: 902 case ISD::CONDCODE: 903 case ISD::ARG_FLAGS: 904 // Primitives must all be legal. 905 assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) && 906 "This must be legal!"); 907 break; 908 default: 909 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 910 // If this is a target node, legalize it by legalizing the operands then 911 // passing it through. 912 SmallVector<SDValue, 8> Ops; 913 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 914 Ops.push_back(LegalizeOp(Node->getOperand(i))); 915 916 Result = DAG.UpdateNodeOperands(Result.getValue(0), &Ops[0], Ops.size()); 917 918 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 919 AddLegalizedOperand(Op.getValue(i), Result.getValue(i)); 920 return Result.getValue(Op.getResNo()); 921 } 922 // Otherwise this is an unhandled builtin node. splat. 923#ifndef NDEBUG 924 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 925#endif 926 assert(0 && "Do not know how to legalize this operator!"); 927 abort(); 928 case ISD::GLOBAL_OFFSET_TABLE: 929 case ISD::GlobalAddress: 930 case ISD::GlobalTLSAddress: 931 case ISD::ExternalSymbol: 932 case ISD::ConstantPool: 933 case ISD::JumpTable: // Nothing to do. 934 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 935 default: assert(0 && "This action is not supported yet!"); 936 case TargetLowering::Custom: 937 Tmp1 = TLI.LowerOperation(Op, DAG); 938 if (Tmp1.getNode()) Result = Tmp1; 939 // FALLTHROUGH if the target doesn't want to lower this op after all. 940 case TargetLowering::Legal: 941 break; 942 } 943 break; 944 case ISD::FRAMEADDR: 945 case ISD::RETURNADDR: 946 // The only option for these nodes is to custom lower them. If the target 947 // does not custom lower them, then return zero. 948 Tmp1 = TLI.LowerOperation(Op, DAG); 949 if (Tmp1.getNode()) 950 Result = Tmp1; 951 else 952 Result = DAG.getConstant(0, TLI.getPointerTy()); 953 break; 954 case ISD::FRAME_TO_ARGS_OFFSET: { 955 MVT VT = Node->getValueType(0); 956 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 957 default: assert(0 && "This action is not supported yet!"); 958 case TargetLowering::Custom: 959 Result = TLI.LowerOperation(Op, DAG); 960 if (Result.getNode()) break; 961 // Fall Thru 962 case TargetLowering::Legal: 963 Result = DAG.getConstant(0, VT); 964 break; 965 } 966 } 967 break; 968 case ISD::EXCEPTIONADDR: { 969 Tmp1 = LegalizeOp(Node->getOperand(0)); 970 MVT VT = Node->getValueType(0); 971 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 972 default: assert(0 && "This action is not supported yet!"); 973 case TargetLowering::Expand: { 974 unsigned Reg = TLI.getExceptionAddressRegister(); 975 Result = DAG.getCopyFromReg(Tmp1, Reg, VT); 976 } 977 break; 978 case TargetLowering::Custom: 979 Result = TLI.LowerOperation(Op, DAG); 980 if (Result.getNode()) break; 981 // Fall Thru 982 case TargetLowering::Legal: { 983 SDValue Ops[] = { DAG.getConstant(0, VT), Tmp1 }; 984 Result = DAG.getMergeValues(Ops, 2); 985 break; 986 } 987 } 988 } 989 if (Result.getNode()->getNumValues() == 1) break; 990 991 assert(Result.getNode()->getNumValues() == 2 && 992 "Cannot return more than two values!"); 993 994 // Since we produced two values, make sure to remember that we 995 // legalized both of them. 996 Tmp1 = LegalizeOp(Result); 997 Tmp2 = LegalizeOp(Result.getValue(1)); 998 AddLegalizedOperand(Op.getValue(0), Tmp1); 999 AddLegalizedOperand(Op.getValue(1), Tmp2); 1000 return Op.getResNo() ? Tmp2 : Tmp1; 1001 case ISD::EHSELECTION: { 1002 Tmp1 = LegalizeOp(Node->getOperand(0)); 1003 Tmp2 = LegalizeOp(Node->getOperand(1)); 1004 MVT VT = Node->getValueType(0); 1005 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1006 default: assert(0 && "This action is not supported yet!"); 1007 case TargetLowering::Expand: { 1008 unsigned Reg = TLI.getExceptionSelectorRegister(); 1009 Result = DAG.getCopyFromReg(Tmp2, Reg, VT); 1010 } 1011 break; 1012 case TargetLowering::Custom: 1013 Result = TLI.LowerOperation(Op, DAG); 1014 if (Result.getNode()) break; 1015 // Fall Thru 1016 case TargetLowering::Legal: { 1017 SDValue Ops[] = { DAG.getConstant(0, VT), Tmp2 }; 1018 Result = DAG.getMergeValues(Ops, 2); 1019 break; 1020 } 1021 } 1022 } 1023 if (Result.getNode()->getNumValues() == 1) break; 1024 1025 assert(Result.getNode()->getNumValues() == 2 && 1026 "Cannot return more than two values!"); 1027 1028 // Since we produced two values, make sure to remember that we 1029 // legalized both of them. 1030 Tmp1 = LegalizeOp(Result); 1031 Tmp2 = LegalizeOp(Result.getValue(1)); 1032 AddLegalizedOperand(Op.getValue(0), Tmp1); 1033 AddLegalizedOperand(Op.getValue(1), Tmp2); 1034 return Op.getResNo() ? Tmp2 : Tmp1; 1035 case ISD::EH_RETURN: { 1036 MVT VT = Node->getValueType(0); 1037 // The only "good" option for this node is to custom lower it. 1038 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1039 default: assert(0 && "This action is not supported at all!"); 1040 case TargetLowering::Custom: 1041 Result = TLI.LowerOperation(Op, DAG); 1042 if (Result.getNode()) break; 1043 // Fall Thru 1044 case TargetLowering::Legal: 1045 // Target does not know, how to lower this, lower to noop 1046 Result = LegalizeOp(Node->getOperand(0)); 1047 break; 1048 } 1049 } 1050 break; 1051 case ISD::AssertSext: 1052 case ISD::AssertZext: 1053 Tmp1 = LegalizeOp(Node->getOperand(0)); 1054 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1055 break; 1056 case ISD::MERGE_VALUES: 1057 // Legalize eliminates MERGE_VALUES nodes. 1058 Result = Node->getOperand(Op.getResNo()); 1059 break; 1060 case ISD::CopyFromReg: 1061 Tmp1 = LegalizeOp(Node->getOperand(0)); 1062 Result = Op.getValue(0); 1063 if (Node->getNumValues() == 2) { 1064 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1065 } else { 1066 assert(Node->getNumValues() == 3 && "Invalid copyfromreg!"); 1067 if (Node->getNumOperands() == 3) { 1068 Tmp2 = LegalizeOp(Node->getOperand(2)); 1069 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2); 1070 } else { 1071 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1072 } 1073 AddLegalizedOperand(Op.getValue(2), Result.getValue(2)); 1074 } 1075 // Since CopyFromReg produces two values, make sure to remember that we 1076 // legalized both of them. 1077 AddLegalizedOperand(Op.getValue(0), Result); 1078 AddLegalizedOperand(Op.getValue(1), Result.getValue(1)); 1079 return Result.getValue(Op.getResNo()); 1080 case ISD::UNDEF: { 1081 MVT VT = Op.getValueType(); 1082 switch (TLI.getOperationAction(ISD::UNDEF, VT)) { 1083 default: assert(0 && "This action is not supported yet!"); 1084 case TargetLowering::Expand: 1085 if (VT.isInteger()) 1086 Result = DAG.getConstant(0, VT); 1087 else if (VT.isFloatingPoint()) 1088 Result = DAG.getConstantFP(APFloat(APInt(VT.getSizeInBits(), 0)), 1089 VT); 1090 else 1091 assert(0 && "Unknown value type!"); 1092 break; 1093 case TargetLowering::Legal: 1094 break; 1095 } 1096 break; 1097 } 1098 1099 case ISD::INTRINSIC_W_CHAIN: 1100 case ISD::INTRINSIC_WO_CHAIN: 1101 case ISD::INTRINSIC_VOID: { 1102 SmallVector<SDValue, 8> Ops; 1103 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1104 Ops.push_back(LegalizeOp(Node->getOperand(i))); 1105 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1106 1107 // Allow the target to custom lower its intrinsics if it wants to. 1108 if (TLI.getOperationAction(Node->getOpcode(), MVT::Other) == 1109 TargetLowering::Custom) { 1110 Tmp3 = TLI.LowerOperation(Result, DAG); 1111 if (Tmp3.getNode()) Result = Tmp3; 1112 } 1113 1114 if (Result.getNode()->getNumValues() == 1) break; 1115 1116 // Must have return value and chain result. 1117 assert(Result.getNode()->getNumValues() == 2 && 1118 "Cannot return more than two values!"); 1119 1120 // Since loads produce two values, make sure to remember that we 1121 // legalized both of them. 1122 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1123 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1124 return Result.getValue(Op.getResNo()); 1125 } 1126 1127 case ISD::DBG_STOPPOINT: 1128 assert(Node->getNumOperands() == 1 && "Invalid DBG_STOPPOINT node!"); 1129 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input chain. 1130 1131 switch (TLI.getOperationAction(ISD::DBG_STOPPOINT, MVT::Other)) { 1132 case TargetLowering::Promote: 1133 default: assert(0 && "This action is not supported yet!"); 1134 case TargetLowering::Expand: { 1135 MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); 1136 bool useDEBUG_LOC = TLI.isOperationLegal(ISD::DEBUG_LOC, MVT::Other); 1137 bool useLABEL = TLI.isOperationLegal(ISD::DBG_LABEL, MVT::Other); 1138 1139 const DbgStopPointSDNode *DSP = cast<DbgStopPointSDNode>(Node); 1140 if (MMI && (useDEBUG_LOC || useLABEL)) { 1141 const CompileUnitDesc *CompileUnit = DSP->getCompileUnit(); 1142 unsigned SrcFile = MMI->RecordSource(CompileUnit); 1143 1144 unsigned Line = DSP->getLine(); 1145 unsigned Col = DSP->getColumn(); 1146 1147 if (useDEBUG_LOC) { 1148 SDValue Ops[] = { Tmp1, DAG.getConstant(Line, MVT::i32), 1149 DAG.getConstant(Col, MVT::i32), 1150 DAG.getConstant(SrcFile, MVT::i32) }; 1151 Result = DAG.getNode(ISD::DEBUG_LOC, MVT::Other, Ops, 4); 1152 } else { 1153 unsigned ID = MMI->RecordSourceLine(Line, Col, SrcFile); 1154 Result = DAG.getLabel(ISD::DBG_LABEL, Tmp1, ID); 1155 } 1156 } else { 1157 Result = Tmp1; // chain 1158 } 1159 break; 1160 } 1161 case TargetLowering::Legal: { 1162 LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); 1163 if (Action == Legal && Tmp1 == Node->getOperand(0)) 1164 break; 1165 1166 SmallVector<SDValue, 8> Ops; 1167 Ops.push_back(Tmp1); 1168 if (Action == Legal) { 1169 Ops.push_back(Node->getOperand(1)); // line # must be legal. 1170 Ops.push_back(Node->getOperand(2)); // col # must be legal. 1171 } else { 1172 // Otherwise promote them. 1173 Ops.push_back(PromoteOp(Node->getOperand(1))); 1174 Ops.push_back(PromoteOp(Node->getOperand(2))); 1175 } 1176 Ops.push_back(Node->getOperand(3)); // filename must be legal. 1177 Ops.push_back(Node->getOperand(4)); // working dir # must be legal. 1178 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1179 break; 1180 } 1181 } 1182 break; 1183 1184 case ISD::DECLARE: 1185 assert(Node->getNumOperands() == 3 && "Invalid DECLARE node!"); 1186 switch (TLI.getOperationAction(ISD::DECLARE, MVT::Other)) { 1187 default: assert(0 && "This action is not supported yet!"); 1188 case TargetLowering::Legal: 1189 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1190 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the address. 1191 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the variable. 1192 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1193 break; 1194 case TargetLowering::Expand: 1195 Result = LegalizeOp(Node->getOperand(0)); 1196 break; 1197 } 1198 break; 1199 1200 case ISD::DEBUG_LOC: 1201 assert(Node->getNumOperands() == 4 && "Invalid DEBUG_LOC node!"); 1202 switch (TLI.getOperationAction(ISD::DEBUG_LOC, MVT::Other)) { 1203 default: assert(0 && "This action is not supported yet!"); 1204 case TargetLowering::Legal: { 1205 LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); 1206 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1207 if (Action == Legal && Tmp1 == Node->getOperand(0)) 1208 break; 1209 if (Action == Legal) { 1210 Tmp2 = Node->getOperand(1); 1211 Tmp3 = Node->getOperand(2); 1212 Tmp4 = Node->getOperand(3); 1213 } else { 1214 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the line #. 1215 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the col #. 1216 Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize the source file id. 1217 } 1218 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4); 1219 break; 1220 } 1221 } 1222 break; 1223 1224 case ISD::DBG_LABEL: 1225 case ISD::EH_LABEL: 1226 assert(Node->getNumOperands() == 1 && "Invalid LABEL node!"); 1227 switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) { 1228 default: assert(0 && "This action is not supported yet!"); 1229 case TargetLowering::Legal: 1230 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1231 Result = DAG.UpdateNodeOperands(Result, Tmp1); 1232 break; 1233 case TargetLowering::Expand: 1234 Result = LegalizeOp(Node->getOperand(0)); 1235 break; 1236 } 1237 break; 1238 1239 case ISD::PREFETCH: 1240 assert(Node->getNumOperands() == 4 && "Invalid Prefetch node!"); 1241 switch (TLI.getOperationAction(ISD::PREFETCH, MVT::Other)) { 1242 default: assert(0 && "This action is not supported yet!"); 1243 case TargetLowering::Legal: 1244 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1245 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the address. 1246 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the rw specifier. 1247 Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize locality specifier. 1248 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4); 1249 break; 1250 case TargetLowering::Expand: 1251 // It's a noop. 1252 Result = LegalizeOp(Node->getOperand(0)); 1253 break; 1254 } 1255 break; 1256 1257 case ISD::MEMBARRIER: { 1258 assert(Node->getNumOperands() == 6 && "Invalid MemBarrier node!"); 1259 switch (TLI.getOperationAction(ISD::MEMBARRIER, MVT::Other)) { 1260 default: assert(0 && "This action is not supported yet!"); 1261 case TargetLowering::Legal: { 1262 SDValue Ops[6]; 1263 Ops[0] = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1264 for (int x = 1; x < 6; ++x) { 1265 Ops[x] = Node->getOperand(x); 1266 if (!isTypeLegal(Ops[x].getValueType())) 1267 Ops[x] = PromoteOp(Ops[x]); 1268 } 1269 Result = DAG.UpdateNodeOperands(Result, &Ops[0], 6); 1270 break; 1271 } 1272 case TargetLowering::Expand: 1273 //There is no libgcc call for this op 1274 Result = Node->getOperand(0); // Noop 1275 break; 1276 } 1277 break; 1278 } 1279 1280 case ISD::ATOMIC_CMP_SWAP_8: 1281 case ISD::ATOMIC_CMP_SWAP_16: 1282 case ISD::ATOMIC_CMP_SWAP_32: 1283 case ISD::ATOMIC_CMP_SWAP_64: { 1284 unsigned int num_operands = 4; 1285 assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!"); 1286 SDValue Ops[4]; 1287 for (unsigned int x = 0; x < num_operands; ++x) 1288 Ops[x] = LegalizeOp(Node->getOperand(x)); 1289 Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands); 1290 1291 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 1292 default: assert(0 && "This action is not supported yet!"); 1293 case TargetLowering::Custom: 1294 Result = TLI.LowerOperation(Result, DAG); 1295 break; 1296 case TargetLowering::Legal: 1297 break; 1298 } 1299 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1300 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1301 return Result.getValue(Op.getResNo()); 1302 } 1303 case ISD::ATOMIC_LOAD_ADD_8: 1304 case ISD::ATOMIC_LOAD_SUB_8: 1305 case ISD::ATOMIC_LOAD_AND_8: 1306 case ISD::ATOMIC_LOAD_OR_8: 1307 case ISD::ATOMIC_LOAD_XOR_8: 1308 case ISD::ATOMIC_LOAD_NAND_8: 1309 case ISD::ATOMIC_LOAD_MIN_8: 1310 case ISD::ATOMIC_LOAD_MAX_8: 1311 case ISD::ATOMIC_LOAD_UMIN_8: 1312 case ISD::ATOMIC_LOAD_UMAX_8: 1313 case ISD::ATOMIC_SWAP_8: 1314 case ISD::ATOMIC_LOAD_ADD_16: 1315 case ISD::ATOMIC_LOAD_SUB_16: 1316 case ISD::ATOMIC_LOAD_AND_16: 1317 case ISD::ATOMIC_LOAD_OR_16: 1318 case ISD::ATOMIC_LOAD_XOR_16: 1319 case ISD::ATOMIC_LOAD_NAND_16: 1320 case ISD::ATOMIC_LOAD_MIN_16: 1321 case ISD::ATOMIC_LOAD_MAX_16: 1322 case ISD::ATOMIC_LOAD_UMIN_16: 1323 case ISD::ATOMIC_LOAD_UMAX_16: 1324 case ISD::ATOMIC_SWAP_16: 1325 case ISD::ATOMIC_LOAD_ADD_32: 1326 case ISD::ATOMIC_LOAD_SUB_32: 1327 case ISD::ATOMIC_LOAD_AND_32: 1328 case ISD::ATOMIC_LOAD_OR_32: 1329 case ISD::ATOMIC_LOAD_XOR_32: 1330 case ISD::ATOMIC_LOAD_NAND_32: 1331 case ISD::ATOMIC_LOAD_MIN_32: 1332 case ISD::ATOMIC_LOAD_MAX_32: 1333 case ISD::ATOMIC_LOAD_UMIN_32: 1334 case ISD::ATOMIC_LOAD_UMAX_32: 1335 case ISD::ATOMIC_SWAP_32: 1336 case ISD::ATOMIC_LOAD_ADD_64: 1337 case ISD::ATOMIC_LOAD_SUB_64: 1338 case ISD::ATOMIC_LOAD_AND_64: 1339 case ISD::ATOMIC_LOAD_OR_64: 1340 case ISD::ATOMIC_LOAD_XOR_64: 1341 case ISD::ATOMIC_LOAD_NAND_64: 1342 case ISD::ATOMIC_LOAD_MIN_64: 1343 case ISD::ATOMIC_LOAD_MAX_64: 1344 case ISD::ATOMIC_LOAD_UMIN_64: 1345 case ISD::ATOMIC_LOAD_UMAX_64: 1346 case ISD::ATOMIC_SWAP_64: { 1347 unsigned int num_operands = 3; 1348 assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!"); 1349 SDValue Ops[3]; 1350 for (unsigned int x = 0; x < num_operands; ++x) 1351 Ops[x] = LegalizeOp(Node->getOperand(x)); 1352 Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands); 1353 1354 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 1355 default: assert(0 && "This action is not supported yet!"); 1356 case TargetLowering::Custom: 1357 Result = TLI.LowerOperation(Result, DAG); 1358 break; 1359 case TargetLowering::Legal: 1360 break; 1361 } 1362 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1363 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1364 return Result.getValue(Op.getResNo()); 1365 } 1366 case ISD::Constant: { 1367 ConstantSDNode *CN = cast<ConstantSDNode>(Node); 1368 unsigned opAction = 1369 TLI.getOperationAction(ISD::Constant, CN->getValueType(0)); 1370 1371 // We know we don't need to expand constants here, constants only have one 1372 // value and we check that it is fine above. 1373 1374 if (opAction == TargetLowering::Custom) { 1375 Tmp1 = TLI.LowerOperation(Result, DAG); 1376 if (Tmp1.getNode()) 1377 Result = Tmp1; 1378 } 1379 break; 1380 } 1381 case ISD::ConstantFP: { 1382 // Spill FP immediates to the constant pool if the target cannot directly 1383 // codegen them. Targets often have some immediate values that can be 1384 // efficiently generated into an FP register without a load. We explicitly 1385 // leave these constants as ConstantFP nodes for the target to deal with. 1386 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 1387 1388 switch (TLI.getOperationAction(ISD::ConstantFP, CFP->getValueType(0))) { 1389 default: assert(0 && "This action is not supported yet!"); 1390 case TargetLowering::Legal: 1391 break; 1392 case TargetLowering::Custom: 1393 Tmp3 = TLI.LowerOperation(Result, DAG); 1394 if (Tmp3.getNode()) { 1395 Result = Tmp3; 1396 break; 1397 } 1398 // FALLTHROUGH 1399 case TargetLowering::Expand: { 1400 // Check to see if this FP immediate is already legal. 1401 bool isLegal = false; 1402 for (TargetLowering::legal_fpimm_iterator I = TLI.legal_fpimm_begin(), 1403 E = TLI.legal_fpimm_end(); I != E; ++I) { 1404 if (CFP->isExactlyValue(*I)) { 1405 isLegal = true; 1406 break; 1407 } 1408 } 1409 // If this is a legal constant, turn it into a TargetConstantFP node. 1410 if (isLegal) 1411 break; 1412 Result = ExpandConstantFP(CFP, true, DAG, TLI); 1413 } 1414 } 1415 break; 1416 } 1417 case ISD::TokenFactor: 1418 if (Node->getNumOperands() == 2) { 1419 Tmp1 = LegalizeOp(Node->getOperand(0)); 1420 Tmp2 = LegalizeOp(Node->getOperand(1)); 1421 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1422 } else if (Node->getNumOperands() == 3) { 1423 Tmp1 = LegalizeOp(Node->getOperand(0)); 1424 Tmp2 = LegalizeOp(Node->getOperand(1)); 1425 Tmp3 = LegalizeOp(Node->getOperand(2)); 1426 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1427 } else { 1428 SmallVector<SDValue, 8> Ops; 1429 // Legalize the operands. 1430 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1431 Ops.push_back(LegalizeOp(Node->getOperand(i))); 1432 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1433 } 1434 break; 1435 1436 case ISD::FORMAL_ARGUMENTS: 1437 case ISD::CALL: 1438 // The only option for this is to custom lower it. 1439 Tmp3 = TLI.LowerOperation(Result.getValue(0), DAG); 1440 assert(Tmp3.getNode() && "Target didn't custom lower this node!"); 1441 // A call within a calling sequence must be legalized to something 1442 // other than the normal CALLSEQ_END. Violating this gets Legalize 1443 // into an infinite loop. 1444 assert ((!IsLegalizingCall || 1445 Node->getOpcode() != ISD::CALL || 1446 Tmp3.getNode()->getOpcode() != ISD::CALLSEQ_END) && 1447 "Nested CALLSEQ_START..CALLSEQ_END not supported."); 1448 1449 // The number of incoming and outgoing values should match; unless the final 1450 // outgoing value is a flag. 1451 assert((Tmp3.getNode()->getNumValues() == Result.getNode()->getNumValues() || 1452 (Tmp3.getNode()->getNumValues() == Result.getNode()->getNumValues() + 1 && 1453 Tmp3.getNode()->getValueType(Tmp3.getNode()->getNumValues() - 1) == 1454 MVT::Flag)) && 1455 "Lowering call/formal_arguments produced unexpected # results!"); 1456 1457 // Since CALL/FORMAL_ARGUMENTS nodes produce multiple values, make sure to 1458 // remember that we legalized all of them, so it doesn't get relegalized. 1459 for (unsigned i = 0, e = Tmp3.getNode()->getNumValues(); i != e; ++i) { 1460 if (Tmp3.getNode()->getValueType(i) == MVT::Flag) 1461 continue; 1462 Tmp1 = LegalizeOp(Tmp3.getValue(i)); 1463 if (Op.getResNo() == i) 1464 Tmp2 = Tmp1; 1465 AddLegalizedOperand(SDValue(Node, i), Tmp1); 1466 } 1467 return Tmp2; 1468 case ISD::EXTRACT_SUBREG: { 1469 Tmp1 = LegalizeOp(Node->getOperand(0)); 1470 ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 1471 assert(idx && "Operand must be a constant"); 1472 Tmp2 = DAG.getTargetConstant(idx->getAPIntValue(), idx->getValueType(0)); 1473 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1474 } 1475 break; 1476 case ISD::INSERT_SUBREG: { 1477 Tmp1 = LegalizeOp(Node->getOperand(0)); 1478 Tmp2 = LegalizeOp(Node->getOperand(1)); 1479 ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(2)); 1480 assert(idx && "Operand must be a constant"); 1481 Tmp3 = DAG.getTargetConstant(idx->getAPIntValue(), idx->getValueType(0)); 1482 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1483 } 1484 break; 1485 case ISD::BUILD_VECTOR: 1486 switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) { 1487 default: assert(0 && "This action is not supported yet!"); 1488 case TargetLowering::Custom: 1489 Tmp3 = TLI.LowerOperation(Result, DAG); 1490 if (Tmp3.getNode()) { 1491 Result = Tmp3; 1492 break; 1493 } 1494 // FALLTHROUGH 1495 case TargetLowering::Expand: 1496 Result = ExpandBUILD_VECTOR(Result.getNode()); 1497 break; 1498 } 1499 break; 1500 case ISD::INSERT_VECTOR_ELT: 1501 Tmp1 = LegalizeOp(Node->getOperand(0)); // InVec 1502 Tmp3 = LegalizeOp(Node->getOperand(2)); // InEltNo 1503 1504 // The type of the value to insert may not be legal, even though the vector 1505 // type is legal. Legalize/Promote accordingly. We do not handle Expand 1506 // here. 1507 switch (getTypeAction(Node->getOperand(1).getValueType())) { 1508 default: assert(0 && "Cannot expand insert element operand"); 1509 case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; 1510 case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; 1511 case Expand: 1512 // FIXME: An alternative would be to check to see if the target is not 1513 // going to custom lower this operation, we could bitcast to half elt 1514 // width and perform two inserts at that width, if that is legal. 1515 Tmp2 = Node->getOperand(1); 1516 break; 1517 } 1518 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1519 1520 switch (TLI.getOperationAction(ISD::INSERT_VECTOR_ELT, 1521 Node->getValueType(0))) { 1522 default: assert(0 && "This action is not supported yet!"); 1523 case TargetLowering::Legal: 1524 break; 1525 case TargetLowering::Custom: 1526 Tmp4 = TLI.LowerOperation(Result, DAG); 1527 if (Tmp4.getNode()) { 1528 Result = Tmp4; 1529 break; 1530 } 1531 // FALLTHROUGH 1532 case TargetLowering::Promote: 1533 // Fall thru for vector case 1534 case TargetLowering::Expand: { 1535 // If the insert index is a constant, codegen this as a scalar_to_vector, 1536 // then a shuffle that inserts it into the right position in the vector. 1537 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Tmp3)) { 1538 // SCALAR_TO_VECTOR requires that the type of the value being inserted 1539 // match the element type of the vector being created. 1540 if (Tmp2.getValueType() == 1541 Op.getValueType().getVectorElementType()) { 1542 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, 1543 Tmp1.getValueType(), Tmp2); 1544 1545 unsigned NumElts = Tmp1.getValueType().getVectorNumElements(); 1546 MVT ShufMaskVT = 1547 MVT::getIntVectorWithNumElements(NumElts); 1548 MVT ShufMaskEltVT = ShufMaskVT.getVectorElementType(); 1549 1550 // We generate a shuffle of InVec and ScVec, so the shuffle mask 1551 // should be 0,1,2,3,4,5... with the appropriate element replaced with 1552 // elt 0 of the RHS. 1553 SmallVector<SDValue, 8> ShufOps; 1554 for (unsigned i = 0; i != NumElts; ++i) { 1555 if (i != InsertPos->getZExtValue()) 1556 ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT)); 1557 else 1558 ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT)); 1559 } 1560 SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMaskVT, 1561 &ShufOps[0], ShufOps.size()); 1562 1563 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, Tmp1.getValueType(), 1564 Tmp1, ScVec, ShufMask); 1565 Result = LegalizeOp(Result); 1566 break; 1567 } 1568 } 1569 Result = PerformInsertVectorEltInMemory(Tmp1, Tmp2, Tmp3); 1570 break; 1571 } 1572 } 1573 break; 1574 case ISD::SCALAR_TO_VECTOR: 1575 if (!TLI.isTypeLegal(Node->getOperand(0).getValueType())) { 1576 Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node)); 1577 break; 1578 } 1579 1580 Tmp1 = LegalizeOp(Node->getOperand(0)); // InVal 1581 Result = DAG.UpdateNodeOperands(Result, Tmp1); 1582 switch (TLI.getOperationAction(ISD::SCALAR_TO_VECTOR, 1583 Node->getValueType(0))) { 1584 default: assert(0 && "This action is not supported yet!"); 1585 case TargetLowering::Legal: 1586 break; 1587 case TargetLowering::Custom: 1588 Tmp3 = TLI.LowerOperation(Result, DAG); 1589 if (Tmp3.getNode()) { 1590 Result = Tmp3; 1591 break; 1592 } 1593 // FALLTHROUGH 1594 case TargetLowering::Expand: 1595 Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node)); 1596 break; 1597 } 1598 break; 1599 case ISD::VECTOR_SHUFFLE: 1600 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input vectors, 1601 Tmp2 = LegalizeOp(Node->getOperand(1)); // but not the shuffle mask. 1602 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 1603 1604 // Allow targets to custom lower the SHUFFLEs they support. 1605 switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE,Result.getValueType())) { 1606 default: assert(0 && "Unknown operation action!"); 1607 case TargetLowering::Legal: 1608 assert(isShuffleLegal(Result.getValueType(), Node->getOperand(2)) && 1609 "vector shuffle should not be created if not legal!"); 1610 break; 1611 case TargetLowering::Custom: 1612 Tmp3 = TLI.LowerOperation(Result, DAG); 1613 if (Tmp3.getNode()) { 1614 Result = Tmp3; 1615 break; 1616 } 1617 // FALLTHROUGH 1618 case TargetLowering::Expand: { 1619 MVT VT = Node->getValueType(0); 1620 MVT EltVT = VT.getVectorElementType(); 1621 MVT PtrVT = TLI.getPointerTy(); 1622 SDValue Mask = Node->getOperand(2); 1623 unsigned NumElems = Mask.getNumOperands(); 1624 SmallVector<SDValue,8> Ops; 1625 for (unsigned i = 0; i != NumElems; ++i) { 1626 SDValue Arg = Mask.getOperand(i); 1627 if (Arg.getOpcode() == ISD::UNDEF) { 1628 Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 1629 } else { 1630 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 1631 unsigned Idx = cast<ConstantSDNode>(Arg)->getZExtValue(); 1632 if (Idx < NumElems) 1633 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp1, 1634 DAG.getConstant(Idx, PtrVT))); 1635 else 1636 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp2, 1637 DAG.getConstant(Idx - NumElems, PtrVT))); 1638 } 1639 } 1640 Result = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); 1641 break; 1642 } 1643 case TargetLowering::Promote: { 1644 // Change base type to a different vector type. 1645 MVT OVT = Node->getValueType(0); 1646 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 1647 1648 // Cast the two input vectors. 1649 Tmp1 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp1); 1650 Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2); 1651 1652 // Convert the shuffle mask to the right # elements. 1653 Tmp3 = SDValue(isShuffleLegal(OVT, Node->getOperand(2)), 0); 1654 assert(Tmp3.getNode() && "Shuffle not legal?"); 1655 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, NVT, Tmp1, Tmp2, Tmp3); 1656 Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result); 1657 break; 1658 } 1659 } 1660 break; 1661 1662 case ISD::EXTRACT_VECTOR_ELT: 1663 Tmp1 = Node->getOperand(0); 1664 Tmp2 = LegalizeOp(Node->getOperand(1)); 1665 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1666 Result = ExpandEXTRACT_VECTOR_ELT(Result); 1667 break; 1668 1669 case ISD::EXTRACT_SUBVECTOR: 1670 Tmp1 = Node->getOperand(0); 1671 Tmp2 = LegalizeOp(Node->getOperand(1)); 1672 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1673 Result = ExpandEXTRACT_SUBVECTOR(Result); 1674 break; 1675 1676 case ISD::CONCAT_VECTORS: { 1677 // Use extract/insert/build vector for now. We might try to be 1678 // more clever later. 1679 MVT PtrVT = TLI.getPointerTy(); 1680 SmallVector<SDValue, 8> Ops; 1681 unsigned NumOperands = Node->getNumOperands(); 1682 for (unsigned i=0; i < NumOperands; ++i) { 1683 SDValue SubOp = Node->getOperand(i); 1684 MVT VVT = SubOp.getNode()->getValueType(0); 1685 MVT EltVT = VVT.getVectorElementType(); 1686 unsigned NumSubElem = VVT.getVectorNumElements(); 1687 for (unsigned j=0; j < NumSubElem; ++j) { 1688 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, SubOp, 1689 DAG.getConstant(j, PtrVT))); 1690 } 1691 } 1692 return LegalizeOp(DAG.getNode(ISD::BUILD_VECTOR, Node->getValueType(0), 1693 &Ops[0], Ops.size())); 1694 } 1695 1696 case ISD::CALLSEQ_START: { 1697 SDNode *CallEnd = FindCallEndFromCallStart(Node); 1698 1699 // Recursively Legalize all of the inputs of the call end that do not lead 1700 // to this call start. This ensures that any libcalls that need be inserted 1701 // are inserted *before* the CALLSEQ_START. 1702 {SmallPtrSet<SDNode*, 32> NodesLeadingTo; 1703 for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i) 1704 LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).getNode(), Node, 1705 NodesLeadingTo); 1706 } 1707 1708 // Now that we legalized all of the inputs (which may have inserted 1709 // libcalls) create the new CALLSEQ_START node. 1710 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1711 1712 // Merge in the last call, to ensure that this call start after the last 1713 // call ended. 1714 if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken) { 1715 Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END); 1716 Tmp1 = LegalizeOp(Tmp1); 1717 } 1718 1719 // Do not try to legalize the target-specific arguments (#1+). 1720 if (Tmp1 != Node->getOperand(0)) { 1721 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1722 Ops[0] = Tmp1; 1723 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1724 } 1725 1726 // Remember that the CALLSEQ_START is legalized. 1727 AddLegalizedOperand(Op.getValue(0), Result); 1728 if (Node->getNumValues() == 2) // If this has a flag result, remember it. 1729 AddLegalizedOperand(Op.getValue(1), Result.getValue(1)); 1730 1731 // Now that the callseq_start and all of the non-call nodes above this call 1732 // sequence have been legalized, legalize the call itself. During this 1733 // process, no libcalls can/will be inserted, guaranteeing that no calls 1734 // can overlap. 1735 assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!"); 1736 // Note that we are selecting this call! 1737 LastCALLSEQ_END = SDValue(CallEnd, 0); 1738 IsLegalizingCall = true; 1739 1740 // Legalize the call, starting from the CALLSEQ_END. 1741 LegalizeOp(LastCALLSEQ_END); 1742 assert(!IsLegalizingCall && "CALLSEQ_END should have cleared this!"); 1743 return Result; 1744 } 1745 case ISD::CALLSEQ_END: 1746 // If the CALLSEQ_START node hasn't been legalized first, legalize it. This 1747 // will cause this node to be legalized as well as handling libcalls right. 1748 if (LastCALLSEQ_END.getNode() != Node) { 1749 LegalizeOp(SDValue(FindCallStartFromCallEnd(Node), 0)); 1750 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 1751 assert(I != LegalizedNodes.end() && 1752 "Legalizing the call start should have legalized this node!"); 1753 return I->second; 1754 } 1755 1756 // Otherwise, the call start has been legalized and everything is going 1757 // according to plan. Just legalize ourselves normally here. 1758 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1759 // Do not try to legalize the target-specific arguments (#1+), except for 1760 // an optional flag input. 1761 if (Node->getOperand(Node->getNumOperands()-1).getValueType() != MVT::Flag){ 1762 if (Tmp1 != Node->getOperand(0)) { 1763 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1764 Ops[0] = Tmp1; 1765 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1766 } 1767 } else { 1768 Tmp2 = LegalizeOp(Node->getOperand(Node->getNumOperands()-1)); 1769 if (Tmp1 != Node->getOperand(0) || 1770 Tmp2 != Node->getOperand(Node->getNumOperands()-1)) { 1771 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1772 Ops[0] = Tmp1; 1773 Ops.back() = Tmp2; 1774 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1775 } 1776 } 1777 assert(IsLegalizingCall && "Call sequence imbalance between start/end?"); 1778 // This finishes up call legalization. 1779 IsLegalizingCall = false; 1780 1781 // If the CALLSEQ_END node has a flag, remember that we legalized it. 1782 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1783 if (Node->getNumValues() == 2) 1784 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1785 return Result.getValue(Op.getResNo()); 1786 case ISD::DYNAMIC_STACKALLOC: { 1787 MVT VT = Node->getValueType(0); 1788 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1789 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the size. 1790 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the alignment. 1791 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1792 1793 Tmp1 = Result.getValue(0); 1794 Tmp2 = Result.getValue(1); 1795 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1796 default: assert(0 && "This action is not supported yet!"); 1797 case TargetLowering::Expand: { 1798 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1799 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1800 " not tell us which reg is the stack pointer!"); 1801 SDValue Chain = Tmp1.getOperand(0); 1802 1803 // Chain the dynamic stack allocation so that it doesn't modify the stack 1804 // pointer when other instructions are using the stack. 1805 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); 1806 1807 SDValue Size = Tmp2.getOperand(1); 1808 SDValue SP = DAG.getCopyFromReg(Chain, SPReg, VT); 1809 Chain = SP.getValue(1); 1810 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1811 unsigned StackAlign = 1812 TLI.getTargetMachine().getFrameInfo()->getStackAlignment(); 1813 if (Align > StackAlign) 1814 SP = DAG.getNode(ISD::AND, VT, SP, 1815 DAG.getConstant(-(uint64_t)Align, VT)); 1816 Tmp1 = DAG.getNode(ISD::SUB, VT, SP, Size); // Value 1817 Chain = DAG.getCopyToReg(Chain, SPReg, Tmp1); // Output chain 1818 1819 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), 1820 DAG.getIntPtrConstant(0, true), SDValue()); 1821 1822 Tmp1 = LegalizeOp(Tmp1); 1823 Tmp2 = LegalizeOp(Tmp2); 1824 break; 1825 } 1826 case TargetLowering::Custom: 1827 Tmp3 = TLI.LowerOperation(Tmp1, DAG); 1828 if (Tmp3.getNode()) { 1829 Tmp1 = LegalizeOp(Tmp3); 1830 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 1831 } 1832 break; 1833 case TargetLowering::Legal: 1834 break; 1835 } 1836 // Since this op produce two values, make sure to remember that we 1837 // legalized both of them. 1838 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 1839 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 1840 return Op.getResNo() ? Tmp2 : Tmp1; 1841 } 1842 case ISD::INLINEASM: { 1843 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1844 bool Changed = false; 1845 // Legalize all of the operands of the inline asm, in case they are nodes 1846 // that need to be expanded or something. Note we skip the asm string and 1847 // all of the TargetConstant flags. 1848 SDValue Op = LegalizeOp(Ops[0]); 1849 Changed = Op != Ops[0]; 1850 Ops[0] = Op; 1851 1852 bool HasInFlag = Ops.back().getValueType() == MVT::Flag; 1853 for (unsigned i = 2, e = Ops.size()-HasInFlag; i < e; ) { 1854 unsigned NumVals = cast<ConstantSDNode>(Ops[i])->getZExtValue() >> 3; 1855 for (++i; NumVals; ++i, --NumVals) { 1856 SDValue Op = LegalizeOp(Ops[i]); 1857 if (Op != Ops[i]) { 1858 Changed = true; 1859 Ops[i] = Op; 1860 } 1861 } 1862 } 1863 1864 if (HasInFlag) { 1865 Op = LegalizeOp(Ops.back()); 1866 Changed |= Op != Ops.back(); 1867 Ops.back() = Op; 1868 } 1869 1870 if (Changed) 1871 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1872 1873 // INLINE asm returns a chain and flag, make sure to add both to the map. 1874 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1875 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1876 return Result.getValue(Op.getResNo()); 1877 } 1878 case ISD::BR: 1879 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1880 // Ensure that libcalls are emitted before a branch. 1881 Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END); 1882 Tmp1 = LegalizeOp(Tmp1); 1883 LastCALLSEQ_END = DAG.getEntryNode(); 1884 1885 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1886 break; 1887 case ISD::BRIND: 1888 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1889 // Ensure that libcalls are emitted before a branch. 1890 Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END); 1891 Tmp1 = LegalizeOp(Tmp1); 1892 LastCALLSEQ_END = DAG.getEntryNode(); 1893 1894 switch (getTypeAction(Node->getOperand(1).getValueType())) { 1895 default: assert(0 && "Indirect target must be legal type (pointer)!"); 1896 case Legal: 1897 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition. 1898 break; 1899 } 1900 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1901 break; 1902 case ISD::BR_JT: 1903 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1904 // Ensure that libcalls are emitted before a branch. 1905 Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END); 1906 Tmp1 = LegalizeOp(Tmp1); 1907 LastCALLSEQ_END = DAG.getEntryNode(); 1908 1909 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the jumptable node. 1910 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 1911 1912 switch (TLI.getOperationAction(ISD::BR_JT, MVT::Other)) { 1913 default: assert(0 && "This action is not supported yet!"); 1914 case TargetLowering::Legal: break; 1915 case TargetLowering::Custom: 1916 Tmp1 = TLI.LowerOperation(Result, DAG); 1917 if (Tmp1.getNode()) Result = Tmp1; 1918 break; 1919 case TargetLowering::Expand: { 1920 SDValue Chain = Result.getOperand(0); 1921 SDValue Table = Result.getOperand(1); 1922 SDValue Index = Result.getOperand(2); 1923 1924 MVT PTy = TLI.getPointerTy(); 1925 MachineFunction &MF = DAG.getMachineFunction(); 1926 unsigned EntrySize = MF.getJumpTableInfo()->getEntrySize(); 1927 Index= DAG.getNode(ISD::MUL, PTy, Index, DAG.getConstant(EntrySize, PTy)); 1928 SDValue Addr = DAG.getNode(ISD::ADD, PTy, Index, Table); 1929 1930 SDValue LD; 1931 switch (EntrySize) { 1932 default: assert(0 && "Size of jump table not supported yet."); break; 1933 case 4: LD = DAG.getLoad(MVT::i32, Chain, Addr, 1934 PseudoSourceValue::getJumpTable(), 0); break; 1935 case 8: LD = DAG.getLoad(MVT::i64, Chain, Addr, 1936 PseudoSourceValue::getJumpTable(), 0); break; 1937 } 1938 1939 Addr = LD; 1940 if (TLI.getTargetMachine().getRelocationModel() == Reloc::PIC_) { 1941 // For PIC, the sequence is: 1942 // BRIND(load(Jumptable + index) + RelocBase) 1943 // RelocBase can be JumpTable, GOT or some sort of global base. 1944 if (PTy != MVT::i32) 1945 Addr = DAG.getNode(ISD::SIGN_EXTEND, PTy, Addr); 1946 Addr = DAG.getNode(ISD::ADD, PTy, Addr, 1947 TLI.getPICJumpTableRelocBase(Table, DAG)); 1948 } 1949 Result = DAG.getNode(ISD::BRIND, MVT::Other, LD.getValue(1), Addr); 1950 } 1951 } 1952 break; 1953 case ISD::BRCOND: 1954 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1955 // Ensure that libcalls are emitted before a return. 1956 Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END); 1957 Tmp1 = LegalizeOp(Tmp1); 1958 LastCALLSEQ_END = DAG.getEntryNode(); 1959 1960 switch (getTypeAction(Node->getOperand(1).getValueType())) { 1961 case Expand: assert(0 && "It's impossible to expand bools"); 1962 case Legal: 1963 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition. 1964 break; 1965 case Promote: { 1966 Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the condition. 1967 1968 // The top bits of the promoted condition are not necessarily zero, ensure 1969 // that the value is properly zero extended. 1970 unsigned BitWidth = Tmp2.getValueSizeInBits(); 1971 if (!DAG.MaskedValueIsZero(Tmp2, 1972 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 1973 Tmp2 = DAG.getZeroExtendInReg(Tmp2, MVT::i1); 1974 break; 1975 } 1976 } 1977 1978 // Basic block destination (Op#2) is always legal. 1979 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 1980 1981 switch (TLI.getOperationAction(ISD::BRCOND, MVT::Other)) { 1982 default: assert(0 && "This action is not supported yet!"); 1983 case TargetLowering::Legal: break; 1984 case TargetLowering::Custom: 1985 Tmp1 = TLI.LowerOperation(Result, DAG); 1986 if (Tmp1.getNode()) Result = Tmp1; 1987 break; 1988 case TargetLowering::Expand: 1989 // Expand brcond's setcc into its constituent parts and create a BR_CC 1990 // Node. 1991 if (Tmp2.getOpcode() == ISD::SETCC) { 1992 Result = DAG.getNode(ISD::BR_CC, MVT::Other, Tmp1, Tmp2.getOperand(2), 1993 Tmp2.getOperand(0), Tmp2.getOperand(1), 1994 Node->getOperand(2)); 1995 } else { 1996 Result = DAG.getNode(ISD::BR_CC, MVT::Other, Tmp1, 1997 DAG.getCondCode(ISD::SETNE), Tmp2, 1998 DAG.getConstant(0, Tmp2.getValueType()), 1999 Node->getOperand(2)); 2000 } 2001 break; 2002 } 2003 break; 2004 case ISD::BR_CC: 2005 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2006 // Ensure that libcalls are emitted before a branch. 2007 Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END); 2008 Tmp1 = LegalizeOp(Tmp1); 2009 Tmp2 = Node->getOperand(2); // LHS 2010 Tmp3 = Node->getOperand(3); // RHS 2011 Tmp4 = Node->getOperand(1); // CC 2012 2013 LegalizeSetCC(TLI.getSetCCResultType(Tmp2), Tmp2, Tmp3, Tmp4); 2014 LastCALLSEQ_END = DAG.getEntryNode(); 2015 2016 // If we didn't get both a LHS and RHS back from LegalizeSetCC, 2017 // the LHS is a legal SETCC itself. In this case, we need to compare 2018 // the result against zero to select between true and false values. 2019 if (Tmp3.getNode() == 0) { 2020 Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); 2021 Tmp4 = DAG.getCondCode(ISD::SETNE); 2022 } 2023 2024 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp4, Tmp2, Tmp3, 2025 Node->getOperand(4)); 2026 2027 switch (TLI.getOperationAction(ISD::BR_CC, Tmp3.getValueType())) { 2028 default: assert(0 && "Unexpected action for BR_CC!"); 2029 case TargetLowering::Legal: break; 2030 case TargetLowering::Custom: 2031 Tmp4 = TLI.LowerOperation(Result, DAG); 2032 if (Tmp4.getNode()) Result = Tmp4; 2033 break; 2034 } 2035 break; 2036 case ISD::LOAD: { 2037 LoadSDNode *LD = cast<LoadSDNode>(Node); 2038 Tmp1 = LegalizeOp(LD->getChain()); // Legalize the chain. 2039 Tmp2 = LegalizeOp(LD->getBasePtr()); // Legalize the base pointer. 2040 2041 ISD::LoadExtType ExtType = LD->getExtensionType(); 2042 if (ExtType == ISD::NON_EXTLOAD) { 2043 MVT VT = Node->getValueType(0); 2044 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset()); 2045 Tmp3 = Result.getValue(0); 2046 Tmp4 = Result.getValue(1); 2047 2048 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 2049 default: assert(0 && "This action is not supported yet!"); 2050 case TargetLowering::Legal: 2051 // If this is an unaligned load and the target doesn't support it, 2052 // expand it. 2053 if (!TLI.allowsUnalignedMemoryAccesses()) { 2054 unsigned ABIAlignment = TLI.getTargetData()-> 2055 getABITypeAlignment(LD->getMemoryVT().getTypeForMVT()); 2056 if (LD->getAlignment() < ABIAlignment){ 2057 Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()), DAG, 2058 TLI); 2059 Tmp3 = Result.getOperand(0); 2060 Tmp4 = Result.getOperand(1); 2061 Tmp3 = LegalizeOp(Tmp3); 2062 Tmp4 = LegalizeOp(Tmp4); 2063 } 2064 } 2065 break; 2066 case TargetLowering::Custom: 2067 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 2068 if (Tmp1.getNode()) { 2069 Tmp3 = LegalizeOp(Tmp1); 2070 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 2071 } 2072 break; 2073 case TargetLowering::Promote: { 2074 // Only promote a load of vector type to another. 2075 assert(VT.isVector() && "Cannot promote this load!"); 2076 // Change base type to a different vector type. 2077 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 2078 2079 Tmp1 = DAG.getLoad(NVT, Tmp1, Tmp2, LD->getSrcValue(), 2080 LD->getSrcValueOffset(), 2081 LD->isVolatile(), LD->getAlignment()); 2082 Tmp3 = LegalizeOp(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp1)); 2083 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 2084 break; 2085 } 2086 } 2087 // Since loads produce two values, make sure to remember that we 2088 // legalized both of them. 2089 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 2090 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 2091 return Op.getResNo() ? Tmp4 : Tmp3; 2092 } else { 2093 MVT SrcVT = LD->getMemoryVT(); 2094 unsigned SrcWidth = SrcVT.getSizeInBits(); 2095 int SVOffset = LD->getSrcValueOffset(); 2096 unsigned Alignment = LD->getAlignment(); 2097 bool isVolatile = LD->isVolatile(); 2098 2099 if (SrcWidth != SrcVT.getStoreSizeInBits() && 2100 // Some targets pretend to have an i1 loading operation, and actually 2101 // load an i8. This trick is correct for ZEXTLOAD because the top 7 2102 // bits are guaranteed to be zero; it helps the optimizers understand 2103 // that these bits are zero. It is also useful for EXTLOAD, since it 2104 // tells the optimizers that those bits are undefined. It would be 2105 // nice to have an effective generic way of getting these benefits... 2106 // Until such a way is found, don't insist on promoting i1 here. 2107 (SrcVT != MVT::i1 || 2108 TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { 2109 // Promote to a byte-sized load if not loading an integral number of 2110 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 2111 unsigned NewWidth = SrcVT.getStoreSizeInBits(); 2112 MVT NVT = MVT::getIntegerVT(NewWidth); 2113 SDValue Ch; 2114 2115 // The extra bits are guaranteed to be zero, since we stored them that 2116 // way. A zext load from NVT thus automatically gives zext from SrcVT. 2117 2118 ISD::LoadExtType NewExtType = 2119 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 2120 2121 Result = DAG.getExtLoad(NewExtType, Node->getValueType(0), 2122 Tmp1, Tmp2, LD->getSrcValue(), SVOffset, 2123 NVT, isVolatile, Alignment); 2124 2125 Ch = Result.getValue(1); // The chain. 2126 2127 if (ExtType == ISD::SEXTLOAD) 2128 // Having the top bits zero doesn't help when sign extending. 2129 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(), 2130 Result, DAG.getValueType(SrcVT)); 2131 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 2132 // All the top bits are guaranteed to be zero - inform the optimizers. 2133 Result = DAG.getNode(ISD::AssertZext, Result.getValueType(), Result, 2134 DAG.getValueType(SrcVT)); 2135 2136 Tmp1 = LegalizeOp(Result); 2137 Tmp2 = LegalizeOp(Ch); 2138 } else if (SrcWidth & (SrcWidth - 1)) { 2139 // If not loading a power-of-2 number of bits, expand as two loads. 2140 assert(SrcVT.isExtended() && !SrcVT.isVector() && 2141 "Unsupported extload!"); 2142 unsigned RoundWidth = 1 << Log2_32(SrcWidth); 2143 assert(RoundWidth < SrcWidth); 2144 unsigned ExtraWidth = SrcWidth - RoundWidth; 2145 assert(ExtraWidth < RoundWidth); 2146 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 2147 "Load size not an integral number of bytes!"); 2148 MVT RoundVT = MVT::getIntegerVT(RoundWidth); 2149 MVT ExtraVT = MVT::getIntegerVT(ExtraWidth); 2150 SDValue Lo, Hi, Ch; 2151 unsigned IncrementSize; 2152 2153 if (TLI.isLittleEndian()) { 2154 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 2155 // Load the bottom RoundWidth bits. 2156 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, Node->getValueType(0), Tmp1, Tmp2, 2157 LD->getSrcValue(), SVOffset, RoundVT, isVolatile, 2158 Alignment); 2159 2160 // Load the remaining ExtraWidth bits. 2161 IncrementSize = RoundWidth / 8; 2162 Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2, 2163 DAG.getIntPtrConstant(IncrementSize)); 2164 Hi = DAG.getExtLoad(ExtType, Node->getValueType(0), Tmp1, Tmp2, 2165 LD->getSrcValue(), SVOffset + IncrementSize, 2166 ExtraVT, isVolatile, 2167 MinAlign(Alignment, IncrementSize)); 2168 2169 // Build a factor node to remember that this load is independent of the 2170 // other one. 2171 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 2172 Hi.getValue(1)); 2173 2174 // Move the top bits to the right place. 2175 Hi = DAG.getNode(ISD::SHL, Hi.getValueType(), Hi, 2176 DAG.getConstant(RoundWidth, TLI.getShiftAmountTy())); 2177 2178 // Join the hi and lo parts. 2179 Result = DAG.getNode(ISD::OR, Node->getValueType(0), Lo, Hi); 2180 } else { 2181 // Big endian - avoid unaligned loads. 2182 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 2183 // Load the top RoundWidth bits. 2184 Hi = DAG.getExtLoad(ExtType, Node->getValueType(0), Tmp1, Tmp2, 2185 LD->getSrcValue(), SVOffset, RoundVT, isVolatile, 2186 Alignment); 2187 2188 // Load the remaining ExtraWidth bits. 2189 IncrementSize = RoundWidth / 8; 2190 Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2, 2191 DAG.getIntPtrConstant(IncrementSize)); 2192 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, Node->getValueType(0), Tmp1, Tmp2, 2193 LD->getSrcValue(), SVOffset + IncrementSize, 2194 ExtraVT, isVolatile, 2195 MinAlign(Alignment, IncrementSize)); 2196 2197 // Build a factor node to remember that this load is independent of the 2198 // other one. 2199 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 2200 Hi.getValue(1)); 2201 2202 // Move the top bits to the right place. 2203 Hi = DAG.getNode(ISD::SHL, Hi.getValueType(), Hi, 2204 DAG.getConstant(ExtraWidth, TLI.getShiftAmountTy())); 2205 2206 // Join the hi and lo parts. 2207 Result = DAG.getNode(ISD::OR, Node->getValueType(0), Lo, Hi); 2208 } 2209 2210 Tmp1 = LegalizeOp(Result); 2211 Tmp2 = LegalizeOp(Ch); 2212 } else { 2213 switch (TLI.getLoadExtAction(ExtType, SrcVT)) { 2214 default: assert(0 && "This action is not supported yet!"); 2215 case TargetLowering::Custom: 2216 isCustom = true; 2217 // FALLTHROUGH 2218 case TargetLowering::Legal: 2219 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset()); 2220 Tmp1 = Result.getValue(0); 2221 Tmp2 = Result.getValue(1); 2222 2223 if (isCustom) { 2224 Tmp3 = TLI.LowerOperation(Result, DAG); 2225 if (Tmp3.getNode()) { 2226 Tmp1 = LegalizeOp(Tmp3); 2227 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 2228 } 2229 } else { 2230 // If this is an unaligned load and the target doesn't support it, 2231 // expand it. 2232 if (!TLI.allowsUnalignedMemoryAccesses()) { 2233 unsigned ABIAlignment = TLI.getTargetData()-> 2234 getABITypeAlignment(LD->getMemoryVT().getTypeForMVT()); 2235 if (LD->getAlignment() < ABIAlignment){ 2236 Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()), DAG, 2237 TLI); 2238 Tmp1 = Result.getOperand(0); 2239 Tmp2 = Result.getOperand(1); 2240 Tmp1 = LegalizeOp(Tmp1); 2241 Tmp2 = LegalizeOp(Tmp2); 2242 } 2243 } 2244 } 2245 break; 2246 case TargetLowering::Expand: 2247 // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND 2248 if (SrcVT == MVT::f32 && Node->getValueType(0) == MVT::f64) { 2249 SDValue Load = DAG.getLoad(SrcVT, Tmp1, Tmp2, LD->getSrcValue(), 2250 LD->getSrcValueOffset(), 2251 LD->isVolatile(), LD->getAlignment()); 2252 Result = DAG.getNode(ISD::FP_EXTEND, Node->getValueType(0), Load); 2253 Tmp1 = LegalizeOp(Result); // Relegalize new nodes. 2254 Tmp2 = LegalizeOp(Load.getValue(1)); 2255 break; 2256 } 2257 assert(ExtType != ISD::EXTLOAD &&"EXTLOAD should always be supported!"); 2258 // Turn the unsupported load into an EXTLOAD followed by an explicit 2259 // zero/sign extend inreg. 2260 Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0), 2261 Tmp1, Tmp2, LD->getSrcValue(), 2262 LD->getSrcValueOffset(), SrcVT, 2263 LD->isVolatile(), LD->getAlignment()); 2264 SDValue ValRes; 2265 if (ExtType == ISD::SEXTLOAD) 2266 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(), 2267 Result, DAG.getValueType(SrcVT)); 2268 else 2269 ValRes = DAG.getZeroExtendInReg(Result, SrcVT); 2270 Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes. 2271 Tmp2 = LegalizeOp(Result.getValue(1)); // Relegalize new nodes. 2272 break; 2273 } 2274 } 2275 2276 // Since loads produce two values, make sure to remember that we legalized 2277 // both of them. 2278 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2279 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2280 return Op.getResNo() ? Tmp2 : Tmp1; 2281 } 2282 } 2283 case ISD::EXTRACT_ELEMENT: { 2284 MVT OpTy = Node->getOperand(0).getValueType(); 2285 switch (getTypeAction(OpTy)) { 2286 default: assert(0 && "EXTRACT_ELEMENT action for type unimplemented!"); 2287 case Legal: 2288 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 2289 // 1 -> Hi 2290 Result = DAG.getNode(ISD::SRL, OpTy, Node->getOperand(0), 2291 DAG.getConstant(OpTy.getSizeInBits()/2, 2292 TLI.getShiftAmountTy())); 2293 Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), Result); 2294 } else { 2295 // 0 -> Lo 2296 Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), 2297 Node->getOperand(0)); 2298 } 2299 break; 2300 case Expand: 2301 // Get both the low and high parts. 2302 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 2303 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) 2304 Result = Tmp2; // 1 -> Hi 2305 else 2306 Result = Tmp1; // 0 -> Lo 2307 break; 2308 } 2309 break; 2310 } 2311 2312 case ISD::CopyToReg: 2313 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2314 2315 assert(isTypeLegal(Node->getOperand(2).getValueType()) && 2316 "Register type must be legal!"); 2317 // Legalize the incoming value (must be a legal type). 2318 Tmp2 = LegalizeOp(Node->getOperand(2)); 2319 if (Node->getNumValues() == 1) { 2320 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2); 2321 } else { 2322 assert(Node->getNumValues() == 2 && "Unknown CopyToReg"); 2323 if (Node->getNumOperands() == 4) { 2324 Tmp3 = LegalizeOp(Node->getOperand(3)); 2325 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2, 2326 Tmp3); 2327 } else { 2328 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2); 2329 } 2330 2331 // Since this produces two values, make sure to remember that we legalized 2332 // both of them. 2333 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 2334 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 2335 return Result; 2336 } 2337 break; 2338 2339 case ISD::RET: 2340 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2341 2342 // Ensure that libcalls are emitted before a return. 2343 Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END); 2344 Tmp1 = LegalizeOp(Tmp1); 2345 LastCALLSEQ_END = DAG.getEntryNode(); 2346 2347 switch (Node->getNumOperands()) { 2348 case 3: // ret val 2349 Tmp2 = Node->getOperand(1); 2350 Tmp3 = Node->getOperand(2); // Signness 2351 switch (getTypeAction(Tmp2.getValueType())) { 2352 case Legal: 2353 Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3); 2354 break; 2355 case Expand: 2356 if (!Tmp2.getValueType().isVector()) { 2357 SDValue Lo, Hi; 2358 ExpandOp(Tmp2, Lo, Hi); 2359 2360 // Big endian systems want the hi reg first. 2361 if (TLI.isBigEndian()) 2362 std::swap(Lo, Hi); 2363 2364 if (Hi.getNode()) 2365 Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3); 2366 else 2367 Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3); 2368 Result = LegalizeOp(Result); 2369 } else { 2370 SDNode *InVal = Tmp2.getNode(); 2371 int InIx = Tmp2.getResNo(); 2372 unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements(); 2373 MVT EVT = InVal->getValueType(InIx).getVectorElementType(); 2374 2375 // Figure out if there is a simple type corresponding to this Vector 2376 // type. If so, convert to the vector type. 2377 MVT TVT = MVT::getVectorVT(EVT, NumElems); 2378 if (TLI.isTypeLegal(TVT)) { 2379 // Turn this into a return of the vector type. 2380 Tmp2 = LegalizeOp(Tmp2); 2381 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2382 } else if (NumElems == 1) { 2383 // Turn this into a return of the scalar type. 2384 Tmp2 = ScalarizeVectorOp(Tmp2); 2385 Tmp2 = LegalizeOp(Tmp2); 2386 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2387 2388 // FIXME: Returns of gcc generic vectors smaller than a legal type 2389 // should be returned in integer registers! 2390 2391 // The scalarized value type may not be legal, e.g. it might require 2392 // promotion or expansion. Relegalize the return. 2393 Result = LegalizeOp(Result); 2394 } else { 2395 // FIXME: Returns of gcc generic vectors larger than a legal vector 2396 // type should be returned by reference! 2397 SDValue Lo, Hi; 2398 SplitVectorOp(Tmp2, Lo, Hi); 2399 Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3); 2400 Result = LegalizeOp(Result); 2401 } 2402 } 2403 break; 2404 case Promote: 2405 Tmp2 = PromoteOp(Node->getOperand(1)); 2406 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2407 Result = LegalizeOp(Result); 2408 break; 2409 } 2410 break; 2411 case 1: // ret void 2412 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2413 break; 2414 default: { // ret <values> 2415 SmallVector<SDValue, 8> NewValues; 2416 NewValues.push_back(Tmp1); 2417 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) 2418 switch (getTypeAction(Node->getOperand(i).getValueType())) { 2419 case Legal: 2420 NewValues.push_back(LegalizeOp(Node->getOperand(i))); 2421 NewValues.push_back(Node->getOperand(i+1)); 2422 break; 2423 case Expand: { 2424 SDValue Lo, Hi; 2425 assert(!Node->getOperand(i).getValueType().isExtended() && 2426 "FIXME: TODO: implement returning non-legal vector types!"); 2427 ExpandOp(Node->getOperand(i), Lo, Hi); 2428 NewValues.push_back(Lo); 2429 NewValues.push_back(Node->getOperand(i+1)); 2430 if (Hi.getNode()) { 2431 NewValues.push_back(Hi); 2432 NewValues.push_back(Node->getOperand(i+1)); 2433 } 2434 break; 2435 } 2436 case Promote: 2437 assert(0 && "Can't promote multiple return value yet!"); 2438 } 2439 2440 if (NewValues.size() == Node->getNumOperands()) 2441 Result = DAG.UpdateNodeOperands(Result, &NewValues[0],NewValues.size()); 2442 else 2443 Result = DAG.getNode(ISD::RET, MVT::Other, 2444 &NewValues[0], NewValues.size()); 2445 break; 2446 } 2447 } 2448 2449 if (Result.getOpcode() == ISD::RET) { 2450 switch (TLI.getOperationAction(Result.getOpcode(), MVT::Other)) { 2451 default: assert(0 && "This action is not supported yet!"); 2452 case TargetLowering::Legal: break; 2453 case TargetLowering::Custom: 2454 Tmp1 = TLI.LowerOperation(Result, DAG); 2455 if (Tmp1.getNode()) Result = Tmp1; 2456 break; 2457 } 2458 } 2459 break; 2460 case ISD::STORE: { 2461 StoreSDNode *ST = cast<StoreSDNode>(Node); 2462 Tmp1 = LegalizeOp(ST->getChain()); // Legalize the chain. 2463 Tmp2 = LegalizeOp(ST->getBasePtr()); // Legalize the pointer. 2464 int SVOffset = ST->getSrcValueOffset(); 2465 unsigned Alignment = ST->getAlignment(); 2466 bool isVolatile = ST->isVolatile(); 2467 2468 if (!ST->isTruncatingStore()) { 2469 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 2470 // FIXME: We shouldn't do this for TargetConstantFP's. 2471 // FIXME: move this to the DAG Combiner! Note that we can't regress due 2472 // to phase ordering between legalized code and the dag combiner. This 2473 // probably means that we need to integrate dag combiner and legalizer 2474 // together. 2475 // We generally can't do this one for long doubles. 2476 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 2477 if (CFP->getValueType(0) == MVT::f32 && 2478 getTypeAction(MVT::i32) == Legal) { 2479 Tmp3 = DAG.getConstant(CFP->getValueAPF(). 2480 bitcastToAPInt().zextOrTrunc(32), 2481 MVT::i32); 2482 Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2483 SVOffset, isVolatile, Alignment); 2484 break; 2485 } else if (CFP->getValueType(0) == MVT::f64) { 2486 // If this target supports 64-bit registers, do a single 64-bit store. 2487 if (getTypeAction(MVT::i64) == Legal) { 2488 Tmp3 = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 2489 zextOrTrunc(64), MVT::i64); 2490 Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2491 SVOffset, isVolatile, Alignment); 2492 break; 2493 } else if (getTypeAction(MVT::i32) == Legal && !ST->isVolatile()) { 2494 // Otherwise, if the target supports 32-bit registers, use 2 32-bit 2495 // stores. If the target supports neither 32- nor 64-bits, this 2496 // xform is certainly not worth it. 2497 const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); 2498 SDValue Lo = DAG.getConstant(APInt(IntVal).trunc(32), MVT::i32); 2499 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); 2500 if (TLI.isBigEndian()) std::swap(Lo, Hi); 2501 2502 Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(), 2503 SVOffset, isVolatile, Alignment); 2504 Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2, 2505 DAG.getIntPtrConstant(4)); 2506 Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(), SVOffset+4, 2507 isVolatile, MinAlign(Alignment, 4U)); 2508 2509 Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 2510 break; 2511 } 2512 } 2513 } 2514 2515 switch (getTypeAction(ST->getMemoryVT())) { 2516 case Legal: { 2517 Tmp3 = LegalizeOp(ST->getValue()); 2518 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 2519 ST->getOffset()); 2520 2521 MVT VT = Tmp3.getValueType(); 2522 switch (TLI.getOperationAction(ISD::STORE, VT)) { 2523 default: assert(0 && "This action is not supported yet!"); 2524 case TargetLowering::Legal: 2525 // If this is an unaligned store and the target doesn't support it, 2526 // expand it. 2527 if (!TLI.allowsUnalignedMemoryAccesses()) { 2528 unsigned ABIAlignment = TLI.getTargetData()-> 2529 getABITypeAlignment(ST->getMemoryVT().getTypeForMVT()); 2530 if (ST->getAlignment() < ABIAlignment) 2531 Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()), DAG, 2532 TLI); 2533 } 2534 break; 2535 case TargetLowering::Custom: 2536 Tmp1 = TLI.LowerOperation(Result, DAG); 2537 if (Tmp1.getNode()) Result = Tmp1; 2538 break; 2539 case TargetLowering::Promote: 2540 assert(VT.isVector() && "Unknown legal promote case!"); 2541 Tmp3 = DAG.getNode(ISD::BIT_CONVERT, 2542 TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3); 2543 Result = DAG.getStore(Tmp1, Tmp3, Tmp2, 2544 ST->getSrcValue(), SVOffset, isVolatile, 2545 Alignment); 2546 break; 2547 } 2548 break; 2549 } 2550 case Promote: 2551 if (!ST->getMemoryVT().isVector()) { 2552 // Truncate the value and store the result. 2553 Tmp3 = PromoteOp(ST->getValue()); 2554 Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2555 SVOffset, ST->getMemoryVT(), 2556 isVolatile, Alignment); 2557 break; 2558 } 2559 // Fall thru to expand for vector 2560 case Expand: { 2561 unsigned IncrementSize = 0; 2562 SDValue Lo, Hi; 2563 2564 // If this is a vector type, then we have to calculate the increment as 2565 // the product of the element size in bytes, and the number of elements 2566 // in the high half of the vector. 2567 if (ST->getValue().getValueType().isVector()) { 2568 SDNode *InVal = ST->getValue().getNode(); 2569 int InIx = ST->getValue().getResNo(); 2570 MVT InVT = InVal->getValueType(InIx); 2571 unsigned NumElems = InVT.getVectorNumElements(); 2572 MVT EVT = InVT.getVectorElementType(); 2573 2574 // Figure out if there is a simple type corresponding to this Vector 2575 // type. If so, convert to the vector type. 2576 MVT TVT = MVT::getVectorVT(EVT, NumElems); 2577 if (TLI.isTypeLegal(TVT)) { 2578 // Turn this into a normal store of the vector type. 2579 Tmp3 = LegalizeOp(ST->getValue()); 2580 Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2581 SVOffset, isVolatile, Alignment); 2582 Result = LegalizeOp(Result); 2583 break; 2584 } else if (NumElems == 1) { 2585 // Turn this into a normal store of the scalar type. 2586 Tmp3 = ScalarizeVectorOp(ST->getValue()); 2587 Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2588 SVOffset, isVolatile, Alignment); 2589 // The scalarized value type may not be legal, e.g. it might require 2590 // promotion or expansion. Relegalize the scalar store. 2591 Result = LegalizeOp(Result); 2592 break; 2593 } else { 2594 // Check if we have widen this node with another value 2595 std::map<SDValue, SDValue>::iterator I = 2596 WidenNodes.find(ST->getValue()); 2597 if (I != WidenNodes.end()) { 2598 Result = StoreWidenVectorOp(ST, Tmp1, Tmp2); 2599 break; 2600 } 2601 else { 2602 SplitVectorOp(ST->getValue(), Lo, Hi); 2603 IncrementSize = Lo.getNode()->getValueType(0).getVectorNumElements() * 2604 EVT.getSizeInBits()/8; 2605 } 2606 } 2607 } else { 2608 ExpandOp(ST->getValue(), Lo, Hi); 2609 IncrementSize = Hi.getNode() ? Hi.getValueType().getSizeInBits()/8 : 0; 2610 2611 if (Hi.getNode() && TLI.isBigEndian()) 2612 std::swap(Lo, Hi); 2613 } 2614 2615 Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(), 2616 SVOffset, isVolatile, Alignment); 2617 2618 if (Hi.getNode() == NULL) { 2619 // Must be int <-> float one-to-one expansion. 2620 Result = Lo; 2621 break; 2622 } 2623 2624 Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2, 2625 DAG.getIntPtrConstant(IncrementSize)); 2626 assert(isTypeLegal(Tmp2.getValueType()) && 2627 "Pointers must be legal!"); 2628 SVOffset += IncrementSize; 2629 Alignment = MinAlign(Alignment, IncrementSize); 2630 Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(), 2631 SVOffset, isVolatile, Alignment); 2632 Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 2633 break; 2634 } // case Expand 2635 } 2636 } else { 2637 switch (getTypeAction(ST->getValue().getValueType())) { 2638 case Legal: 2639 Tmp3 = LegalizeOp(ST->getValue()); 2640 break; 2641 case Promote: 2642 if (!ST->getValue().getValueType().isVector()) { 2643 // We can promote the value, the truncstore will still take care of it. 2644 Tmp3 = PromoteOp(ST->getValue()); 2645 break; 2646 } 2647 // Vector case falls through to expand 2648 case Expand: 2649 // Just store the low part. This may become a non-trunc store, so make 2650 // sure to use getTruncStore, not UpdateNodeOperands below. 2651 ExpandOp(ST->getValue(), Tmp3, Tmp4); 2652 return DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2653 SVOffset, MVT::i8, isVolatile, Alignment); 2654 } 2655 2656 MVT StVT = ST->getMemoryVT(); 2657 unsigned StWidth = StVT.getSizeInBits(); 2658 2659 if (StWidth != StVT.getStoreSizeInBits()) { 2660 // Promote to a byte-sized store with upper bits zero if not 2661 // storing an integral number of bytes. For example, promote 2662 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 2663 MVT NVT = MVT::getIntegerVT(StVT.getStoreSizeInBits()); 2664 Tmp3 = DAG.getZeroExtendInReg(Tmp3, StVT); 2665 Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2666 SVOffset, NVT, isVolatile, Alignment); 2667 } else if (StWidth & (StWidth - 1)) { 2668 // If not storing a power-of-2 number of bits, expand as two stores. 2669 assert(StVT.isExtended() && !StVT.isVector() && 2670 "Unsupported truncstore!"); 2671 unsigned RoundWidth = 1 << Log2_32(StWidth); 2672 assert(RoundWidth < StWidth); 2673 unsigned ExtraWidth = StWidth - RoundWidth; 2674 assert(ExtraWidth < RoundWidth); 2675 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 2676 "Store size not an integral number of bytes!"); 2677 MVT RoundVT = MVT::getIntegerVT(RoundWidth); 2678 MVT ExtraVT = MVT::getIntegerVT(ExtraWidth); 2679 SDValue Lo, Hi; 2680 unsigned IncrementSize; 2681 2682 if (TLI.isLittleEndian()) { 2683 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 2684 // Store the bottom RoundWidth bits. 2685 Lo = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2686 SVOffset, RoundVT, 2687 isVolatile, Alignment); 2688 2689 // Store the remaining ExtraWidth bits. 2690 IncrementSize = RoundWidth / 8; 2691 Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2, 2692 DAG.getIntPtrConstant(IncrementSize)); 2693 Hi = DAG.getNode(ISD::SRL, Tmp3.getValueType(), Tmp3, 2694 DAG.getConstant(RoundWidth, TLI.getShiftAmountTy())); 2695 Hi = DAG.getTruncStore(Tmp1, Hi, Tmp2, ST->getSrcValue(), 2696 SVOffset + IncrementSize, ExtraVT, isVolatile, 2697 MinAlign(Alignment, IncrementSize)); 2698 } else { 2699 // Big endian - avoid unaligned stores. 2700 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 2701 // Store the top RoundWidth bits. 2702 Hi = DAG.getNode(ISD::SRL, Tmp3.getValueType(), Tmp3, 2703 DAG.getConstant(ExtraWidth, TLI.getShiftAmountTy())); 2704 Hi = DAG.getTruncStore(Tmp1, Hi, Tmp2, ST->getSrcValue(), SVOffset, 2705 RoundVT, isVolatile, Alignment); 2706 2707 // Store the remaining ExtraWidth bits. 2708 IncrementSize = RoundWidth / 8; 2709 Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2, 2710 DAG.getIntPtrConstant(IncrementSize)); 2711 Lo = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), 2712 SVOffset + IncrementSize, ExtraVT, isVolatile, 2713 MinAlign(Alignment, IncrementSize)); 2714 } 2715 2716 // The order of the stores doesn't matter. 2717 Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 2718 } else { 2719 if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() || 2720 Tmp2 != ST->getBasePtr()) 2721 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 2722 ST->getOffset()); 2723 2724 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { 2725 default: assert(0 && "This action is not supported yet!"); 2726 case TargetLowering::Legal: 2727 // If this is an unaligned store and the target doesn't support it, 2728 // expand it. 2729 if (!TLI.allowsUnalignedMemoryAccesses()) { 2730 unsigned ABIAlignment = TLI.getTargetData()-> 2731 getABITypeAlignment(ST->getMemoryVT().getTypeForMVT()); 2732 if (ST->getAlignment() < ABIAlignment) 2733 Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()), DAG, 2734 TLI); 2735 } 2736 break; 2737 case TargetLowering::Custom: 2738 Result = TLI.LowerOperation(Result, DAG); 2739 break; 2740 case Expand: 2741 // TRUNCSTORE:i16 i32 -> STORE i16 2742 assert(isTypeLegal(StVT) && "Do not know how to expand this store!"); 2743 Tmp3 = DAG.getNode(ISD::TRUNCATE, StVT, Tmp3); 2744 Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), SVOffset, 2745 isVolatile, Alignment); 2746 break; 2747 } 2748 } 2749 } 2750 break; 2751 } 2752 case ISD::PCMARKER: 2753 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2754 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 2755 break; 2756 case ISD::STACKSAVE: 2757 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2758 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2759 Tmp1 = Result.getValue(0); 2760 Tmp2 = Result.getValue(1); 2761 2762 switch (TLI.getOperationAction(ISD::STACKSAVE, MVT::Other)) { 2763 default: assert(0 && "This action is not supported yet!"); 2764 case TargetLowering::Legal: break; 2765 case TargetLowering::Custom: 2766 Tmp3 = TLI.LowerOperation(Result, DAG); 2767 if (Tmp3.getNode()) { 2768 Tmp1 = LegalizeOp(Tmp3); 2769 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 2770 } 2771 break; 2772 case TargetLowering::Expand: 2773 // Expand to CopyFromReg if the target set 2774 // StackPointerRegisterToSaveRestore. 2775 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 2776 Tmp1 = DAG.getCopyFromReg(Result.getOperand(0), SP, 2777 Node->getValueType(0)); 2778 Tmp2 = Tmp1.getValue(1); 2779 } else { 2780 Tmp1 = DAG.getNode(ISD::UNDEF, Node->getValueType(0)); 2781 Tmp2 = Node->getOperand(0); 2782 } 2783 break; 2784 } 2785 2786 // Since stacksave produce two values, make sure to remember that we 2787 // legalized both of them. 2788 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2789 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2790 return Op.getResNo() ? Tmp2 : Tmp1; 2791 2792 case ISD::STACKRESTORE: 2793 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2794 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 2795 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 2796 2797 switch (TLI.getOperationAction(ISD::STACKRESTORE, MVT::Other)) { 2798 default: assert(0 && "This action is not supported yet!"); 2799 case TargetLowering::Legal: break; 2800 case TargetLowering::Custom: 2801 Tmp1 = TLI.LowerOperation(Result, DAG); 2802 if (Tmp1.getNode()) Result = Tmp1; 2803 break; 2804 case TargetLowering::Expand: 2805 // Expand to CopyToReg if the target set 2806 // StackPointerRegisterToSaveRestore. 2807 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 2808 Result = DAG.getCopyToReg(Tmp1, SP, Tmp2); 2809 } else { 2810 Result = Tmp1; 2811 } 2812 break; 2813 } 2814 break; 2815 2816 case ISD::READCYCLECOUNTER: 2817 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain 2818 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2819 switch (TLI.getOperationAction(ISD::READCYCLECOUNTER, 2820 Node->getValueType(0))) { 2821 default: assert(0 && "This action is not supported yet!"); 2822 case TargetLowering::Legal: 2823 Tmp1 = Result.getValue(0); 2824 Tmp2 = Result.getValue(1); 2825 break; 2826 case TargetLowering::Custom: 2827 Result = TLI.LowerOperation(Result, DAG); 2828 Tmp1 = LegalizeOp(Result.getValue(0)); 2829 Tmp2 = LegalizeOp(Result.getValue(1)); 2830 break; 2831 } 2832 2833 // Since rdcc produce two values, make sure to remember that we legalized 2834 // both of them. 2835 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2836 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2837 return Result; 2838 2839 case ISD::SELECT: 2840 switch (getTypeAction(Node->getOperand(0).getValueType())) { 2841 case Expand: assert(0 && "It's impossible to expand bools"); 2842 case Legal: 2843 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition. 2844 break; 2845 case Promote: { 2846 assert(!Node->getOperand(0).getValueType().isVector() && "not possible"); 2847 Tmp1 = PromoteOp(Node->getOperand(0)); // Promote the condition. 2848 // Make sure the condition is either zero or one. 2849 unsigned BitWidth = Tmp1.getValueSizeInBits(); 2850 if (!DAG.MaskedValueIsZero(Tmp1, 2851 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 2852 Tmp1 = DAG.getZeroExtendInReg(Tmp1, MVT::i1); 2853 break; 2854 } 2855 } 2856 Tmp2 = LegalizeOp(Node->getOperand(1)); // TrueVal 2857 Tmp3 = LegalizeOp(Node->getOperand(2)); // FalseVal 2858 2859 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2860 2861 switch (TLI.getOperationAction(ISD::SELECT, Tmp2.getValueType())) { 2862 default: assert(0 && "This action is not supported yet!"); 2863 case TargetLowering::Legal: break; 2864 case TargetLowering::Custom: { 2865 Tmp1 = TLI.LowerOperation(Result, DAG); 2866 if (Tmp1.getNode()) Result = Tmp1; 2867 break; 2868 } 2869 case TargetLowering::Expand: 2870 if (Tmp1.getOpcode() == ISD::SETCC) { 2871 Result = DAG.getSelectCC(Tmp1.getOperand(0), Tmp1.getOperand(1), 2872 Tmp2, Tmp3, 2873 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 2874 } else { 2875 Result = DAG.getSelectCC(Tmp1, 2876 DAG.getConstant(0, Tmp1.getValueType()), 2877 Tmp2, Tmp3, ISD::SETNE); 2878 } 2879 break; 2880 case TargetLowering::Promote: { 2881 MVT NVT = 2882 TLI.getTypeToPromoteTo(ISD::SELECT, Tmp2.getValueType()); 2883 unsigned ExtOp, TruncOp; 2884 if (Tmp2.getValueType().isVector()) { 2885 ExtOp = ISD::BIT_CONVERT; 2886 TruncOp = ISD::BIT_CONVERT; 2887 } else if (Tmp2.getValueType().isInteger()) { 2888 ExtOp = ISD::ANY_EXTEND; 2889 TruncOp = ISD::TRUNCATE; 2890 } else { 2891 ExtOp = ISD::FP_EXTEND; 2892 TruncOp = ISD::FP_ROUND; 2893 } 2894 // Promote each of the values to the new type. 2895 Tmp2 = DAG.getNode(ExtOp, NVT, Tmp2); 2896 Tmp3 = DAG.getNode(ExtOp, NVT, Tmp3); 2897 // Perform the larger operation, then round down. 2898 Result = DAG.getNode(ISD::SELECT, NVT, Tmp1, Tmp2,Tmp3); 2899 if (TruncOp != ISD::FP_ROUND) 2900 Result = DAG.getNode(TruncOp, Node->getValueType(0), Result); 2901 else 2902 Result = DAG.getNode(TruncOp, Node->getValueType(0), Result, 2903 DAG.getIntPtrConstant(0)); 2904 break; 2905 } 2906 } 2907 break; 2908 case ISD::SELECT_CC: { 2909 Tmp1 = Node->getOperand(0); // LHS 2910 Tmp2 = Node->getOperand(1); // RHS 2911 Tmp3 = LegalizeOp(Node->getOperand(2)); // True 2912 Tmp4 = LegalizeOp(Node->getOperand(3)); // False 2913 SDValue CC = Node->getOperand(4); 2914 2915 LegalizeSetCC(TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2, CC); 2916 2917 // If we didn't get both a LHS and RHS back from LegalizeSetCC, 2918 // the LHS is a legal SETCC itself. In this case, we need to compare 2919 // the result against zero to select between true and false values. 2920 if (Tmp2.getNode() == 0) { 2921 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 2922 CC = DAG.getCondCode(ISD::SETNE); 2923 } 2924 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, CC); 2925 2926 // Everything is legal, see if we should expand this op or something. 2927 switch (TLI.getOperationAction(ISD::SELECT_CC, Tmp3.getValueType())) { 2928 default: assert(0 && "This action is not supported yet!"); 2929 case TargetLowering::Legal: break; 2930 case TargetLowering::Custom: 2931 Tmp1 = TLI.LowerOperation(Result, DAG); 2932 if (Tmp1.getNode()) Result = Tmp1; 2933 break; 2934 } 2935 break; 2936 } 2937 case ISD::SETCC: 2938 Tmp1 = Node->getOperand(0); 2939 Tmp2 = Node->getOperand(1); 2940 Tmp3 = Node->getOperand(2); 2941 LegalizeSetCC(Node->getValueType(0), Tmp1, Tmp2, Tmp3); 2942 2943 // If we had to Expand the SetCC operands into a SELECT node, then it may 2944 // not always be possible to return a true LHS & RHS. In this case, just 2945 // return the value we legalized, returned in the LHS 2946 if (Tmp2.getNode() == 0) { 2947 Result = Tmp1; 2948 break; 2949 } 2950 2951 switch (TLI.getOperationAction(ISD::SETCC, Tmp1.getValueType())) { 2952 default: assert(0 && "Cannot handle this action for SETCC yet!"); 2953 case TargetLowering::Custom: 2954 isCustom = true; 2955 // FALLTHROUGH. 2956 case TargetLowering::Legal: 2957 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2958 if (isCustom) { 2959 Tmp4 = TLI.LowerOperation(Result, DAG); 2960 if (Tmp4.getNode()) Result = Tmp4; 2961 } 2962 break; 2963 case TargetLowering::Promote: { 2964 // First step, figure out the appropriate operation to use. 2965 // Allow SETCC to not be supported for all legal data types 2966 // Mostly this targets FP 2967 MVT NewInTy = Node->getOperand(0).getValueType(); 2968 MVT OldVT = NewInTy; OldVT = OldVT; 2969 2970 // Scan for the appropriate larger type to use. 2971 while (1) { 2972 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT()+1); 2973 2974 assert(NewInTy.isInteger() == OldVT.isInteger() && 2975 "Fell off of the edge of the integer world"); 2976 assert(NewInTy.isFloatingPoint() == OldVT.isFloatingPoint() && 2977 "Fell off of the edge of the floating point world"); 2978 2979 // If the target supports SETCC of this type, use it. 2980 if (TLI.isOperationLegal(ISD::SETCC, NewInTy)) 2981 break; 2982 } 2983 if (NewInTy.isInteger()) 2984 assert(0 && "Cannot promote Legal Integer SETCC yet"); 2985 else { 2986 Tmp1 = DAG.getNode(ISD::FP_EXTEND, NewInTy, Tmp1); 2987 Tmp2 = DAG.getNode(ISD::FP_EXTEND, NewInTy, Tmp2); 2988 } 2989 Tmp1 = LegalizeOp(Tmp1); 2990 Tmp2 = LegalizeOp(Tmp2); 2991 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2992 Result = LegalizeOp(Result); 2993 break; 2994 } 2995 case TargetLowering::Expand: 2996 // Expand a setcc node into a select_cc of the same condition, lhs, and 2997 // rhs that selects between const 1 (true) and const 0 (false). 2998 MVT VT = Node->getValueType(0); 2999 Result = DAG.getNode(ISD::SELECT_CC, VT, Tmp1, Tmp2, 3000 DAG.getConstant(1, VT), DAG.getConstant(0, VT), 3001 Tmp3); 3002 break; 3003 } 3004 break; 3005 case ISD::VSETCC: { 3006 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3007 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3008 SDValue CC = Node->getOperand(2); 3009 3010 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, CC); 3011 3012 // Everything is legal, see if we should expand this op or something. 3013 switch (TLI.getOperationAction(ISD::VSETCC, Tmp1.getValueType())) { 3014 default: assert(0 && "This action is not supported yet!"); 3015 case TargetLowering::Legal: break; 3016 case TargetLowering::Custom: 3017 Tmp1 = TLI.LowerOperation(Result, DAG); 3018 if (Tmp1.getNode()) Result = Tmp1; 3019 break; 3020 } 3021 break; 3022 } 3023 3024 case ISD::SHL_PARTS: 3025 case ISD::SRA_PARTS: 3026 case ISD::SRL_PARTS: { 3027 SmallVector<SDValue, 8> Ops; 3028 bool Changed = false; 3029 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 3030 Ops.push_back(LegalizeOp(Node->getOperand(i))); 3031 Changed |= Ops.back() != Node->getOperand(i); 3032 } 3033 if (Changed) 3034 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 3035 3036 switch (TLI.getOperationAction(Node->getOpcode(), 3037 Node->getValueType(0))) { 3038 default: assert(0 && "This action is not supported yet!"); 3039 case TargetLowering::Legal: break; 3040 case TargetLowering::Custom: 3041 Tmp1 = TLI.LowerOperation(Result, DAG); 3042 if (Tmp1.getNode()) { 3043 SDValue Tmp2, RetVal(0, 0); 3044 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { 3045 Tmp2 = LegalizeOp(Tmp1.getValue(i)); 3046 AddLegalizedOperand(SDValue(Node, i), Tmp2); 3047 if (i == Op.getResNo()) 3048 RetVal = Tmp2; 3049 } 3050 assert(RetVal.getNode() && "Illegal result number"); 3051 return RetVal; 3052 } 3053 break; 3054 } 3055 3056 // Since these produce multiple values, make sure to remember that we 3057 // legalized all of them. 3058 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 3059 AddLegalizedOperand(SDValue(Node, i), Result.getValue(i)); 3060 return Result.getValue(Op.getResNo()); 3061 } 3062 3063 // Binary operators 3064 case ISD::ADD: 3065 case ISD::SUB: 3066 case ISD::MUL: 3067 case ISD::MULHS: 3068 case ISD::MULHU: 3069 case ISD::UDIV: 3070 case ISD::SDIV: 3071 case ISD::AND: 3072 case ISD::OR: 3073 case ISD::XOR: 3074 case ISD::SHL: 3075 case ISD::SRL: 3076 case ISD::SRA: 3077 case ISD::FADD: 3078 case ISD::FSUB: 3079 case ISD::FMUL: 3080 case ISD::FDIV: 3081 case ISD::FPOW: 3082 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3083 switch (getTypeAction(Node->getOperand(1).getValueType())) { 3084 case Expand: assert(0 && "Not possible"); 3085 case Legal: 3086 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS. 3087 break; 3088 case Promote: 3089 Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the RHS. 3090 break; 3091 } 3092 3093 if ((Node->getOpcode() == ISD::SHL || 3094 Node->getOpcode() == ISD::SRL || 3095 Node->getOpcode() == ISD::SRA) && 3096 !Node->getValueType(0).isVector()) { 3097 if (TLI.getShiftAmountTy().bitsLT(Tmp2.getValueType())) 3098 Tmp2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Tmp2); 3099 else if (TLI.getShiftAmountTy().bitsGT(Tmp2.getValueType())) 3100 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Tmp2); 3101 } 3102 3103 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3104 3105 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3106 default: assert(0 && "BinOp legalize operation not supported"); 3107 case TargetLowering::Legal: break; 3108 case TargetLowering::Custom: 3109 Tmp1 = TLI.LowerOperation(Result, DAG); 3110 if (Tmp1.getNode()) { 3111 Result = Tmp1; 3112 break; 3113 } 3114 // Fall through if the custom lower can't deal with the operation 3115 case TargetLowering::Expand: { 3116 MVT VT = Op.getValueType(); 3117 3118 // See if multiply or divide can be lowered using two-result operations. 3119 SDVTList VTs = DAG.getVTList(VT, VT); 3120 if (Node->getOpcode() == ISD::MUL) { 3121 // We just need the low half of the multiply; try both the signed 3122 // and unsigned forms. If the target supports both SMUL_LOHI and 3123 // UMUL_LOHI, form a preference by checking which forms of plain 3124 // MULH it supports. 3125 bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, VT); 3126 bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, VT); 3127 bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, VT); 3128 bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, VT); 3129 unsigned OpToUse = 0; 3130 if (HasSMUL_LOHI && !HasMULHS) { 3131 OpToUse = ISD::SMUL_LOHI; 3132 } else if (HasUMUL_LOHI && !HasMULHU) { 3133 OpToUse = ISD::UMUL_LOHI; 3134 } else if (HasSMUL_LOHI) { 3135 OpToUse = ISD::SMUL_LOHI; 3136 } else if (HasUMUL_LOHI) { 3137 OpToUse = ISD::UMUL_LOHI; 3138 } 3139 if (OpToUse) { 3140 Result = SDValue(DAG.getNode(OpToUse, VTs, Tmp1, Tmp2).getNode(), 0); 3141 break; 3142 } 3143 } 3144 if (Node->getOpcode() == ISD::MULHS && 3145 TLI.isOperationLegal(ISD::SMUL_LOHI, VT)) { 3146 Result = SDValue(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), 3147 1); 3148 break; 3149 } 3150 if (Node->getOpcode() == ISD::MULHU && 3151 TLI.isOperationLegal(ISD::UMUL_LOHI, VT)) { 3152 Result = SDValue(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), 3153 1); 3154 break; 3155 } 3156 if (Node->getOpcode() == ISD::SDIV && 3157 TLI.isOperationLegal(ISD::SDIVREM, VT)) { 3158 Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(), 3159 0); 3160 break; 3161 } 3162 if (Node->getOpcode() == ISD::UDIV && 3163 TLI.isOperationLegal(ISD::UDIVREM, VT)) { 3164 Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(), 3165 0); 3166 break; 3167 } 3168 3169 // Check to see if we have a libcall for this operator. 3170 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3171 bool isSigned = false; 3172 switch (Node->getOpcode()) { 3173 case ISD::UDIV: 3174 case ISD::SDIV: 3175 if (VT == MVT::i32) { 3176 LC = Node->getOpcode() == ISD::UDIV 3177 ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32; 3178 isSigned = Node->getOpcode() == ISD::SDIV; 3179 } 3180 break; 3181 case ISD::MUL: 3182 if (VT == MVT::i32) 3183 LC = RTLIB::MUL_I32; 3184 break; 3185 case ISD::FPOW: 3186 LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, 3187 RTLIB::POW_PPCF128); 3188 break; 3189 default: break; 3190 } 3191 if (LC != RTLIB::UNKNOWN_LIBCALL) { 3192 SDValue Dummy; 3193 Result = ExpandLibCall(LC, Node, isSigned, Dummy); 3194 break; 3195 } 3196 3197 assert(Node->getValueType(0).isVector() && 3198 "Cannot expand this binary operator!"); 3199 // Expand the operation into a bunch of nasty scalar code. 3200 Result = LegalizeOp(UnrollVectorOp(Op)); 3201 break; 3202 } 3203 case TargetLowering::Promote: { 3204 switch (Node->getOpcode()) { 3205 default: assert(0 && "Do not know how to promote this BinOp!"); 3206 case ISD::AND: 3207 case ISD::OR: 3208 case ISD::XOR: { 3209 MVT OVT = Node->getValueType(0); 3210 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3211 assert(OVT.isVector() && "Cannot promote this BinOp!"); 3212 // Bit convert each of the values to the new type. 3213 Tmp1 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp1); 3214 Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2); 3215 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2); 3216 // Bit convert the result back the original type. 3217 Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result); 3218 break; 3219 } 3220 } 3221 } 3222 } 3223 break; 3224 3225 case ISD::SMUL_LOHI: 3226 case ISD::UMUL_LOHI: 3227 case ISD::SDIVREM: 3228 case ISD::UDIVREM: 3229 // These nodes will only be produced by target-specific lowering, so 3230 // they shouldn't be here if they aren't legal. 3231 assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) && 3232 "This must be legal!"); 3233 3234 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3235 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3236 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3237 break; 3238 3239 case ISD::FCOPYSIGN: // FCOPYSIGN does not require LHS/RHS to match type! 3240 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3241 switch (getTypeAction(Node->getOperand(1).getValueType())) { 3242 case Expand: assert(0 && "Not possible"); 3243 case Legal: 3244 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS. 3245 break; 3246 case Promote: 3247 Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the RHS. 3248 break; 3249 } 3250 3251 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3252 3253 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3254 default: assert(0 && "Operation not supported"); 3255 case TargetLowering::Custom: 3256 Tmp1 = TLI.LowerOperation(Result, DAG); 3257 if (Tmp1.getNode()) Result = Tmp1; 3258 break; 3259 case TargetLowering::Legal: break; 3260 case TargetLowering::Expand: { 3261 // If this target supports fabs/fneg natively and select is cheap, 3262 // do this efficiently. 3263 if (!TLI.isSelectExpensive() && 3264 TLI.getOperationAction(ISD::FABS, Tmp1.getValueType()) == 3265 TargetLowering::Legal && 3266 TLI.getOperationAction(ISD::FNEG, Tmp1.getValueType()) == 3267 TargetLowering::Legal) { 3268 // Get the sign bit of the RHS. 3269 MVT IVT = 3270 Tmp2.getValueType() == MVT::f32 ? MVT::i32 : MVT::i64; 3271 SDValue SignBit = DAG.getNode(ISD::BIT_CONVERT, IVT, Tmp2); 3272 SignBit = DAG.getSetCC(TLI.getSetCCResultType(SignBit), 3273 SignBit, DAG.getConstant(0, IVT), ISD::SETLT); 3274 // Get the absolute value of the result. 3275 SDValue AbsVal = DAG.getNode(ISD::FABS, Tmp1.getValueType(), Tmp1); 3276 // Select between the nabs and abs value based on the sign bit of 3277 // the input. 3278 Result = DAG.getNode(ISD::SELECT, AbsVal.getValueType(), SignBit, 3279 DAG.getNode(ISD::FNEG, AbsVal.getValueType(), 3280 AbsVal), 3281 AbsVal); 3282 Result = LegalizeOp(Result); 3283 break; 3284 } 3285 3286 // Otherwise, do bitwise ops! 3287 MVT NVT = 3288 Node->getValueType(0) == MVT::f32 ? MVT::i32 : MVT::i64; 3289 Result = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI); 3290 Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), Result); 3291 Result = LegalizeOp(Result); 3292 break; 3293 } 3294 } 3295 break; 3296 3297 case ISD::ADDC: 3298 case ISD::SUBC: 3299 Tmp1 = LegalizeOp(Node->getOperand(0)); 3300 Tmp2 = LegalizeOp(Node->getOperand(1)); 3301 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3302 Tmp3 = Result.getValue(0); 3303 Tmp4 = Result.getValue(1); 3304 3305 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3306 default: assert(0 && "This action is not supported yet!"); 3307 case TargetLowering::Legal: 3308 break; 3309 case TargetLowering::Custom: 3310 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 3311 if (Tmp1.getNode() != NULL) { 3312 Tmp3 = LegalizeOp(Tmp1); 3313 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 3314 } 3315 break; 3316 } 3317 // Since this produces two values, make sure to remember that we legalized 3318 // both of them. 3319 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 3320 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 3321 return Op.getResNo() ? Tmp4 : Tmp3; 3322 3323 case ISD::ADDE: 3324 case ISD::SUBE: 3325 Tmp1 = LegalizeOp(Node->getOperand(0)); 3326 Tmp2 = LegalizeOp(Node->getOperand(1)); 3327 Tmp3 = LegalizeOp(Node->getOperand(2)); 3328 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 3329 Tmp3 = Result.getValue(0); 3330 Tmp4 = Result.getValue(1); 3331 3332 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3333 default: assert(0 && "This action is not supported yet!"); 3334 case TargetLowering::Legal: 3335 break; 3336 case TargetLowering::Custom: 3337 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 3338 if (Tmp1.getNode() != NULL) { 3339 Tmp3 = LegalizeOp(Tmp1); 3340 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 3341 } 3342 break; 3343 } 3344 // Since this produces two values, make sure to remember that we legalized 3345 // both of them. 3346 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 3347 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 3348 return Op.getResNo() ? Tmp4 : Tmp3; 3349 3350 case ISD::BUILD_PAIR: { 3351 MVT PairTy = Node->getValueType(0); 3352 // TODO: handle the case where the Lo and Hi operands are not of legal type 3353 Tmp1 = LegalizeOp(Node->getOperand(0)); // Lo 3354 Tmp2 = LegalizeOp(Node->getOperand(1)); // Hi 3355 switch (TLI.getOperationAction(ISD::BUILD_PAIR, PairTy)) { 3356 case TargetLowering::Promote: 3357 case TargetLowering::Custom: 3358 assert(0 && "Cannot promote/custom this yet!"); 3359 case TargetLowering::Legal: 3360 if (Tmp1 != Node->getOperand(0) || Tmp2 != Node->getOperand(1)) 3361 Result = DAG.getNode(ISD::BUILD_PAIR, PairTy, Tmp1, Tmp2); 3362 break; 3363 case TargetLowering::Expand: 3364 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, PairTy, Tmp1); 3365 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, PairTy, Tmp2); 3366 Tmp2 = DAG.getNode(ISD::SHL, PairTy, Tmp2, 3367 DAG.getConstant(PairTy.getSizeInBits()/2, 3368 TLI.getShiftAmountTy())); 3369 Result = DAG.getNode(ISD::OR, PairTy, Tmp1, Tmp2); 3370 break; 3371 } 3372 break; 3373 } 3374 3375 case ISD::UREM: 3376 case ISD::SREM: 3377 case ISD::FREM: 3378 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3379 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3380 3381 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3382 case TargetLowering::Promote: assert(0 && "Cannot promote this yet!"); 3383 case TargetLowering::Custom: 3384 isCustom = true; 3385 // FALLTHROUGH 3386 case TargetLowering::Legal: 3387 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3388 if (isCustom) { 3389 Tmp1 = TLI.LowerOperation(Result, DAG); 3390 if (Tmp1.getNode()) Result = Tmp1; 3391 } 3392 break; 3393 case TargetLowering::Expand: { 3394 unsigned DivOpc= (Node->getOpcode() == ISD::UREM) ? ISD::UDIV : ISD::SDIV; 3395 bool isSigned = DivOpc == ISD::SDIV; 3396 MVT VT = Node->getValueType(0); 3397 3398 // See if remainder can be lowered using two-result operations. 3399 SDVTList VTs = DAG.getVTList(VT, VT); 3400 if (Node->getOpcode() == ISD::SREM && 3401 TLI.isOperationLegal(ISD::SDIVREM, VT)) { 3402 Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(), 1); 3403 break; 3404 } 3405 if (Node->getOpcode() == ISD::UREM && 3406 TLI.isOperationLegal(ISD::UDIVREM, VT)) { 3407 Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(), 1); 3408 break; 3409 } 3410 3411 if (VT.isInteger()) { 3412 if (TLI.getOperationAction(DivOpc, VT) == 3413 TargetLowering::Legal) { 3414 // X % Y -> X-X/Y*Y 3415 Result = DAG.getNode(DivOpc, VT, Tmp1, Tmp2); 3416 Result = DAG.getNode(ISD::MUL, VT, Result, Tmp2); 3417 Result = DAG.getNode(ISD::SUB, VT, Tmp1, Result); 3418 } else if (VT.isVector()) { 3419 Result = LegalizeOp(UnrollVectorOp(Op)); 3420 } else { 3421 assert(VT == MVT::i32 && 3422 "Cannot expand this binary operator!"); 3423 RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM 3424 ? RTLIB::UREM_I32 : RTLIB::SREM_I32; 3425 SDValue Dummy; 3426 Result = ExpandLibCall(LC, Node, isSigned, Dummy); 3427 } 3428 } else { 3429 assert(VT.isFloatingPoint() && 3430 "remainder op must have integer or floating-point type"); 3431 if (VT.isVector()) { 3432 Result = LegalizeOp(UnrollVectorOp(Op)); 3433 } else { 3434 // Floating point mod -> fmod libcall. 3435 RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::REM_F32, RTLIB::REM_F64, 3436 RTLIB::REM_F80, RTLIB::REM_PPCF128); 3437 SDValue Dummy; 3438 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3439 } 3440 } 3441 break; 3442 } 3443 } 3444 break; 3445 case ISD::VAARG: { 3446 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3447 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3448 3449 MVT VT = Node->getValueType(0); 3450 switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) { 3451 default: assert(0 && "This action is not supported yet!"); 3452 case TargetLowering::Custom: 3453 isCustom = true; 3454 // FALLTHROUGH 3455 case TargetLowering::Legal: 3456 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3457 Result = Result.getValue(0); 3458 Tmp1 = Result.getValue(1); 3459 3460 if (isCustom) { 3461 Tmp2 = TLI.LowerOperation(Result, DAG); 3462 if (Tmp2.getNode()) { 3463 Result = LegalizeOp(Tmp2); 3464 Tmp1 = LegalizeOp(Tmp2.getValue(1)); 3465 } 3466 } 3467 break; 3468 case TargetLowering::Expand: { 3469 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 3470 SDValue VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0); 3471 // Increment the pointer, VAList, to the next vaarg 3472 Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, 3473 DAG.getConstant(TLI.getTargetData()->getABITypeSize(VT.getTypeForMVT()), 3474 TLI.getPointerTy())); 3475 // Store the incremented VAList to the legalized pointer 3476 Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, V, 0); 3477 // Load the actual argument out of the pointer VAList 3478 Result = DAG.getLoad(VT, Tmp3, VAList, NULL, 0); 3479 Tmp1 = LegalizeOp(Result.getValue(1)); 3480 Result = LegalizeOp(Result); 3481 break; 3482 } 3483 } 3484 // Since VAARG produces two values, make sure to remember that we 3485 // legalized both of them. 3486 AddLegalizedOperand(SDValue(Node, 0), Result); 3487 AddLegalizedOperand(SDValue(Node, 1), Tmp1); 3488 return Op.getResNo() ? Tmp1 : Result; 3489 } 3490 3491 case ISD::VACOPY: 3492 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3493 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the dest pointer. 3494 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the source pointer. 3495 3496 switch (TLI.getOperationAction(ISD::VACOPY, MVT::Other)) { 3497 default: assert(0 && "This action is not supported yet!"); 3498 case TargetLowering::Custom: 3499 isCustom = true; 3500 // FALLTHROUGH 3501 case TargetLowering::Legal: 3502 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, 3503 Node->getOperand(3), Node->getOperand(4)); 3504 if (isCustom) { 3505 Tmp1 = TLI.LowerOperation(Result, DAG); 3506 if (Tmp1.getNode()) Result = Tmp1; 3507 } 3508 break; 3509 case TargetLowering::Expand: 3510 // This defaults to loading a pointer from the input and storing it to the 3511 // output, returning the chain. 3512 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 3513 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 3514 Tmp4 = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp3, VS, 0); 3515 Result = DAG.getStore(Tmp4.getValue(1), Tmp4, Tmp2, VD, 0); 3516 break; 3517 } 3518 break; 3519 3520 case ISD::VAEND: 3521 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3522 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3523 3524 switch (TLI.getOperationAction(ISD::VAEND, MVT::Other)) { 3525 default: assert(0 && "This action is not supported yet!"); 3526 case TargetLowering::Custom: 3527 isCustom = true; 3528 // FALLTHROUGH 3529 case TargetLowering::Legal: 3530 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3531 if (isCustom) { 3532 Tmp1 = TLI.LowerOperation(Tmp1, DAG); 3533 if (Tmp1.getNode()) Result = Tmp1; 3534 } 3535 break; 3536 case TargetLowering::Expand: 3537 Result = Tmp1; // Default to a no-op, return the chain 3538 break; 3539 } 3540 break; 3541 3542 case ISD::VASTART: 3543 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3544 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3545 3546 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3547 3548 switch (TLI.getOperationAction(ISD::VASTART, MVT::Other)) { 3549 default: assert(0 && "This action is not supported yet!"); 3550 case TargetLowering::Legal: break; 3551 case TargetLowering::Custom: 3552 Tmp1 = TLI.LowerOperation(Result, DAG); 3553 if (Tmp1.getNode()) Result = Tmp1; 3554 break; 3555 } 3556 break; 3557 3558 case ISD::ROTL: 3559 case ISD::ROTR: 3560 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3561 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3562 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3563 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3564 default: 3565 assert(0 && "ROTL/ROTR legalize operation not supported"); 3566 break; 3567 case TargetLowering::Legal: 3568 break; 3569 case TargetLowering::Custom: 3570 Tmp1 = TLI.LowerOperation(Result, DAG); 3571 if (Tmp1.getNode()) Result = Tmp1; 3572 break; 3573 case TargetLowering::Promote: 3574 assert(0 && "Do not know how to promote ROTL/ROTR"); 3575 break; 3576 case TargetLowering::Expand: 3577 assert(0 && "Do not know how to expand ROTL/ROTR"); 3578 break; 3579 } 3580 break; 3581 3582 case ISD::BSWAP: 3583 Tmp1 = LegalizeOp(Node->getOperand(0)); // Op 3584 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3585 case TargetLowering::Custom: 3586 assert(0 && "Cannot custom legalize this yet!"); 3587 case TargetLowering::Legal: 3588 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3589 break; 3590 case TargetLowering::Promote: { 3591 MVT OVT = Tmp1.getValueType(); 3592 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3593 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 3594 3595 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1); 3596 Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1); 3597 Result = DAG.getNode(ISD::SRL, NVT, Tmp1, 3598 DAG.getConstant(DiffBits, TLI.getShiftAmountTy())); 3599 break; 3600 } 3601 case TargetLowering::Expand: 3602 Result = ExpandBSWAP(Tmp1); 3603 break; 3604 } 3605 break; 3606 3607 case ISD::CTPOP: 3608 case ISD::CTTZ: 3609 case ISD::CTLZ: 3610 Tmp1 = LegalizeOp(Node->getOperand(0)); // Op 3611 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3612 case TargetLowering::Custom: 3613 case TargetLowering::Legal: 3614 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3615 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 3616 TargetLowering::Custom) { 3617 Tmp1 = TLI.LowerOperation(Result, DAG); 3618 if (Tmp1.getNode()) { 3619 Result = Tmp1; 3620 } 3621 } 3622 break; 3623 case TargetLowering::Promote: { 3624 MVT OVT = Tmp1.getValueType(); 3625 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3626 3627 // Zero extend the argument. 3628 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1); 3629 // Perform the larger operation, then subtract if needed. 3630 Tmp1 = DAG.getNode(Node->getOpcode(), Node->getValueType(0), Tmp1); 3631 switch (Node->getOpcode()) { 3632 case ISD::CTPOP: 3633 Result = Tmp1; 3634 break; 3635 case ISD::CTTZ: 3636 //if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT) 3637 Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1, 3638 DAG.getConstant(NVT.getSizeInBits(), NVT), 3639 ISD::SETEQ); 3640 Result = DAG.getNode(ISD::SELECT, NVT, Tmp2, 3641 DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); 3642 break; 3643 case ISD::CTLZ: 3644 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 3645 Result = DAG.getNode(ISD::SUB, NVT, Tmp1, 3646 DAG.getConstant(NVT.getSizeInBits() - 3647 OVT.getSizeInBits(), NVT)); 3648 break; 3649 } 3650 break; 3651 } 3652 case TargetLowering::Expand: 3653 Result = ExpandBitCount(Node->getOpcode(), Tmp1); 3654 break; 3655 } 3656 break; 3657 3658 // Unary operators 3659 case ISD::FABS: 3660 case ISD::FNEG: 3661 case ISD::FSQRT: 3662 case ISD::FSIN: 3663 case ISD::FCOS: 3664 case ISD::FLOG: 3665 case ISD::FLOG2: 3666 case ISD::FLOG10: 3667 case ISD::FEXP: 3668 case ISD::FEXP2: 3669 case ISD::FTRUNC: 3670 case ISD::FFLOOR: 3671 case ISD::FCEIL: 3672 case ISD::FRINT: 3673 case ISD::FNEARBYINT: 3674 Tmp1 = LegalizeOp(Node->getOperand(0)); 3675 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3676 case TargetLowering::Promote: 3677 case TargetLowering::Custom: 3678 isCustom = true; 3679 // FALLTHROUGH 3680 case TargetLowering::Legal: 3681 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3682 if (isCustom) { 3683 Tmp1 = TLI.LowerOperation(Result, DAG); 3684 if (Tmp1.getNode()) Result = Tmp1; 3685 } 3686 break; 3687 case TargetLowering::Expand: 3688 switch (Node->getOpcode()) { 3689 default: assert(0 && "Unreachable!"); 3690 case ISD::FNEG: 3691 // Expand Y = FNEG(X) -> Y = SUB -0.0, X 3692 Tmp2 = DAG.getConstantFP(-0.0, Node->getValueType(0)); 3693 Result = DAG.getNode(ISD::FSUB, Node->getValueType(0), Tmp2, Tmp1); 3694 break; 3695 case ISD::FABS: { 3696 // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). 3697 MVT VT = Node->getValueType(0); 3698 Tmp2 = DAG.getConstantFP(0.0, VT); 3699 Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2, 3700 ISD::SETUGT); 3701 Tmp3 = DAG.getNode(ISD::FNEG, VT, Tmp1); 3702 Result = DAG.getNode(ISD::SELECT, VT, Tmp2, Tmp1, Tmp3); 3703 break; 3704 } 3705 case ISD::FSQRT: 3706 case ISD::FSIN: 3707 case ISD::FCOS: 3708 case ISD::FLOG: 3709 case ISD::FLOG2: 3710 case ISD::FLOG10: 3711 case ISD::FEXP: 3712 case ISD::FEXP2: 3713 case ISD::FTRUNC: 3714 case ISD::FFLOOR: 3715 case ISD::FCEIL: 3716 case ISD::FRINT: 3717 case ISD::FNEARBYINT: { 3718 MVT VT = Node->getValueType(0); 3719 3720 // Expand unsupported unary vector operators by unrolling them. 3721 if (VT.isVector()) { 3722 Result = LegalizeOp(UnrollVectorOp(Op)); 3723 break; 3724 } 3725 3726 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3727 switch(Node->getOpcode()) { 3728 case ISD::FSQRT: 3729 LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 3730 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128); 3731 break; 3732 case ISD::FSIN: 3733 LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64, 3734 RTLIB::SIN_F80, RTLIB::SIN_PPCF128); 3735 break; 3736 case ISD::FCOS: 3737 LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, 3738 RTLIB::COS_F80, RTLIB::COS_PPCF128); 3739 break; 3740 case ISD::FLOG: 3741 LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, 3742 RTLIB::LOG_F80, RTLIB::LOG_PPCF128); 3743 break; 3744 case ISD::FLOG2: 3745 LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 3746 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); 3747 break; 3748 case ISD::FLOG10: 3749 LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 3750 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); 3751 break; 3752 case ISD::FEXP: 3753 LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, 3754 RTLIB::EXP_F80, RTLIB::EXP_PPCF128); 3755 break; 3756 case ISD::FEXP2: 3757 LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 3758 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); 3759 break; 3760 case ISD::FTRUNC: 3761 LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 3762 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); 3763 break; 3764 case ISD::FFLOOR: 3765 LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 3766 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); 3767 break; 3768 case ISD::FCEIL: 3769 LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 3770 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); 3771 break; 3772 case ISD::FRINT: 3773 LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, 3774 RTLIB::RINT_F80, RTLIB::RINT_PPCF128); 3775 break; 3776 case ISD::FNEARBYINT: 3777 LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, 3778 RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); 3779 break; 3780 break; 3781 default: assert(0 && "Unreachable!"); 3782 } 3783 SDValue Dummy; 3784 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3785 break; 3786 } 3787 } 3788 break; 3789 } 3790 break; 3791 case ISD::FPOWI: { 3792 MVT VT = Node->getValueType(0); 3793 3794 // Expand unsupported unary vector operators by unrolling them. 3795 if (VT.isVector()) { 3796 Result = LegalizeOp(UnrollVectorOp(Op)); 3797 break; 3798 } 3799 3800 // We always lower FPOWI into a libcall. No target support for it yet. 3801 RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, 3802 RTLIB::POWI_F80, RTLIB::POWI_PPCF128); 3803 SDValue Dummy; 3804 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3805 break; 3806 } 3807 case ISD::BIT_CONVERT: 3808 if (!isTypeLegal(Node->getOperand(0).getValueType())) { 3809 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 3810 Node->getValueType(0)); 3811 } else if (Op.getOperand(0).getValueType().isVector()) { 3812 // The input has to be a vector type, we have to either scalarize it, pack 3813 // it, or convert it based on whether the input vector type is legal. 3814 SDNode *InVal = Node->getOperand(0).getNode(); 3815 int InIx = Node->getOperand(0).getResNo(); 3816 unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements(); 3817 MVT EVT = InVal->getValueType(InIx).getVectorElementType(); 3818 3819 // Figure out if there is a simple type corresponding to this Vector 3820 // type. If so, convert to the vector type. 3821 MVT TVT = MVT::getVectorVT(EVT, NumElems); 3822 if (TLI.isTypeLegal(TVT)) { 3823 // Turn this into a bit convert of the vector input. 3824 Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), 3825 LegalizeOp(Node->getOperand(0))); 3826 break; 3827 } else if (NumElems == 1) { 3828 // Turn this into a bit convert of the scalar input. 3829 Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), 3830 ScalarizeVectorOp(Node->getOperand(0))); 3831 break; 3832 } else { 3833 // FIXME: UNIMP! Store then reload 3834 assert(0 && "Cast from unsupported vector type not implemented yet!"); 3835 } 3836 } else { 3837 switch (TLI.getOperationAction(ISD::BIT_CONVERT, 3838 Node->getOperand(0).getValueType())) { 3839 default: assert(0 && "Unknown operation action!"); 3840 case TargetLowering::Expand: 3841 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 3842 Node->getValueType(0)); 3843 break; 3844 case TargetLowering::Legal: 3845 Tmp1 = LegalizeOp(Node->getOperand(0)); 3846 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3847 break; 3848 } 3849 } 3850 break; 3851 case ISD::CONVERT_RNDSAT: { 3852 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 3853 switch (CvtCode) { 3854 default: assert(0 && "Unknown cvt code!"); 3855 case ISD::CVT_SF: 3856 case ISD::CVT_UF: 3857 break; 3858 case ISD::CVT_FF: 3859 case ISD::CVT_FS: 3860 case ISD::CVT_FU: 3861 case ISD::CVT_SS: 3862 case ISD::CVT_SU: 3863 case ISD::CVT_US: 3864 case ISD::CVT_UU: { 3865 SDValue DTyOp = Node->getOperand(1); 3866 SDValue STyOp = Node->getOperand(2); 3867 SDValue RndOp = Node->getOperand(3); 3868 SDValue SatOp = Node->getOperand(4); 3869 switch (getTypeAction(Node->getOperand(0).getValueType())) { 3870 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 3871 case Legal: 3872 Tmp1 = LegalizeOp(Node->getOperand(0)); 3873 Result = DAG.UpdateNodeOperands(Result, Tmp1, DTyOp, STyOp, 3874 RndOp, SatOp); 3875 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 3876 TargetLowering::Custom) { 3877 Tmp1 = TLI.LowerOperation(Result, DAG); 3878 if (Tmp1.getNode()) Result = Tmp1; 3879 } 3880 break; 3881 case Promote: 3882 Result = PromoteOp(Node->getOperand(0)); 3883 // For FP, make Op1 a i32 3884 3885 Result = DAG.getConvertRndSat(Result.getValueType(), Result, 3886 DTyOp, STyOp, RndOp, SatOp, CvtCode); 3887 break; 3888 } 3889 break; 3890 } 3891 } // end switch CvtCode 3892 break; 3893 } 3894 // Conversion operators. The source and destination have different types. 3895 case ISD::SINT_TO_FP: 3896 case ISD::UINT_TO_FP: { 3897 bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; 3898 Result = LegalizeINT_TO_FP(Result, isSigned, 3899 Node->getValueType(0), Node->getOperand(0)); 3900 break; 3901 } 3902 case ISD::TRUNCATE: 3903 switch (getTypeAction(Node->getOperand(0).getValueType())) { 3904 case Legal: 3905 Tmp1 = LegalizeOp(Node->getOperand(0)); 3906 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3907 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 3908 TargetLowering::Custom) { 3909 Tmp1 = TLI.LowerOperation(Result, DAG); 3910 if (Tmp1.getNode()) Result = Tmp1; 3911 } 3912 break; 3913 case Expand: 3914 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 3915 3916 // Since the result is legal, we should just be able to truncate the low 3917 // part of the source. 3918 Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), Tmp1); 3919 break; 3920 case Promote: 3921 Result = PromoteOp(Node->getOperand(0)); 3922 Result = DAG.getNode(ISD::TRUNCATE, Op.getValueType(), Result); 3923 break; 3924 } 3925 break; 3926 3927 case ISD::FP_TO_SINT: 3928 case ISD::FP_TO_UINT: 3929 switch (getTypeAction(Node->getOperand(0).getValueType())) { 3930 case Legal: 3931 Tmp1 = LegalizeOp(Node->getOperand(0)); 3932 3933 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))){ 3934 default: assert(0 && "Unknown operation action!"); 3935 case TargetLowering::Custom: 3936 isCustom = true; 3937 // FALLTHROUGH 3938 case TargetLowering::Legal: 3939 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3940 if (isCustom) { 3941 Tmp1 = TLI.LowerOperation(Result, DAG); 3942 if (Tmp1.getNode()) Result = Tmp1; 3943 } 3944 break; 3945 case TargetLowering::Promote: 3946 Result = PromoteLegalFP_TO_INT(Tmp1, Node->getValueType(0), 3947 Node->getOpcode() == ISD::FP_TO_SINT); 3948 break; 3949 case TargetLowering::Expand: 3950 if (Node->getOpcode() == ISD::FP_TO_UINT) { 3951 SDValue True, False; 3952 MVT VT = Node->getOperand(0).getValueType(); 3953 MVT NVT = Node->getValueType(0); 3954 const uint64_t zero[] = {0, 0}; 3955 APFloat apf = APFloat(APInt(VT.getSizeInBits(), 2, zero)); 3956 APInt x = APInt::getSignBit(NVT.getSizeInBits()); 3957 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); 3958 Tmp2 = DAG.getConstantFP(apf, VT); 3959 Tmp3 = DAG.getSetCC(TLI.getSetCCResultType(Node->getOperand(0)), 3960 Node->getOperand(0), Tmp2, ISD::SETLT); 3961 True = DAG.getNode(ISD::FP_TO_SINT, NVT, Node->getOperand(0)); 3962 False = DAG.getNode(ISD::FP_TO_SINT, NVT, 3963 DAG.getNode(ISD::FSUB, VT, Node->getOperand(0), 3964 Tmp2)); 3965 False = DAG.getNode(ISD::XOR, NVT, False, 3966 DAG.getConstant(x, NVT)); 3967 Result = DAG.getNode(ISD::SELECT, NVT, Tmp3, True, False); 3968 break; 3969 } else { 3970 assert(0 && "Do not know how to expand FP_TO_SINT yet!"); 3971 } 3972 break; 3973 } 3974 break; 3975 case Expand: { 3976 MVT VT = Op.getValueType(); 3977 MVT OVT = Node->getOperand(0).getValueType(); 3978 // Convert ppcf128 to i32 3979 if (OVT == MVT::ppcf128 && VT == MVT::i32) { 3980 if (Node->getOpcode() == ISD::FP_TO_SINT) { 3981 Result = DAG.getNode(ISD::FP_ROUND_INREG, MVT::ppcf128, 3982 Node->getOperand(0), DAG.getValueType(MVT::f64)); 3983 Result = DAG.getNode(ISD::FP_ROUND, MVT::f64, Result, 3984 DAG.getIntPtrConstant(1)); 3985 Result = DAG.getNode(ISD::FP_TO_SINT, VT, Result); 3986 } else { 3987 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; 3988 APFloat apf = APFloat(APInt(128, 2, TwoE31)); 3989 Tmp2 = DAG.getConstantFP(apf, OVT); 3990 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X 3991 // FIXME: generated code sucks. 3992 Result = DAG.getNode(ISD::SELECT_CC, VT, Node->getOperand(0), Tmp2, 3993 DAG.getNode(ISD::ADD, MVT::i32, 3994 DAG.getNode(ISD::FP_TO_SINT, VT, 3995 DAG.getNode(ISD::FSUB, OVT, 3996 Node->getOperand(0), Tmp2)), 3997 DAG.getConstant(0x80000000, MVT::i32)), 3998 DAG.getNode(ISD::FP_TO_SINT, VT, 3999 Node->getOperand(0)), 4000 DAG.getCondCode(ISD::SETGE)); 4001 } 4002 break; 4003 } 4004 // Convert f32 / f64 to i32 / i64 / i128. 4005 RTLIB::Libcall LC = (Node->getOpcode() == ISD::FP_TO_SINT) ? 4006 RTLIB::getFPTOSINT(OVT, VT) : RTLIB::getFPTOUINT(OVT, VT); 4007 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpectd fp-to-int conversion!"); 4008 SDValue Dummy; 4009 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 4010 break; 4011 } 4012 case Promote: 4013 Tmp1 = PromoteOp(Node->getOperand(0)); 4014 Result = DAG.UpdateNodeOperands(Result, LegalizeOp(Tmp1)); 4015 Result = LegalizeOp(Result); 4016 break; 4017 } 4018 break; 4019 4020 case ISD::FP_EXTEND: { 4021 MVT DstVT = Op.getValueType(); 4022 MVT SrcVT = Op.getOperand(0).getValueType(); 4023 if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) { 4024 // The only other way we can lower this is to turn it into a STORE, 4025 // LOAD pair, targetting a temporary location (a stack slot). 4026 Result = EmitStackConvert(Node->getOperand(0), SrcVT, DstVT); 4027 break; 4028 } 4029 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4030 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4031 case Legal: 4032 Tmp1 = LegalizeOp(Node->getOperand(0)); 4033 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4034 break; 4035 case Promote: 4036 Tmp1 = PromoteOp(Node->getOperand(0)); 4037 Result = DAG.getNode(ISD::FP_EXTEND, Op.getValueType(), Tmp1); 4038 break; 4039 } 4040 break; 4041 } 4042 case ISD::FP_ROUND: { 4043 MVT DstVT = Op.getValueType(); 4044 MVT SrcVT = Op.getOperand(0).getValueType(); 4045 if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) { 4046 if (SrcVT == MVT::ppcf128) { 4047 SDValue Lo; 4048 ExpandOp(Node->getOperand(0), Lo, Result); 4049 // Round it the rest of the way (e.g. to f32) if needed. 4050 if (DstVT!=MVT::f64) 4051 Result = DAG.getNode(ISD::FP_ROUND, DstVT, Result, Op.getOperand(1)); 4052 break; 4053 } 4054 // The only other way we can lower this is to turn it into a STORE, 4055 // LOAD pair, targetting a temporary location (a stack slot). 4056 Result = EmitStackConvert(Node->getOperand(0), DstVT, DstVT); 4057 break; 4058 } 4059 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4060 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4061 case Legal: 4062 Tmp1 = LegalizeOp(Node->getOperand(0)); 4063 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 4064 break; 4065 case Promote: 4066 Tmp1 = PromoteOp(Node->getOperand(0)); 4067 Result = DAG.getNode(ISD::FP_ROUND, Op.getValueType(), Tmp1, 4068 Node->getOperand(1)); 4069 break; 4070 } 4071 break; 4072 } 4073 case ISD::ANY_EXTEND: 4074 case ISD::ZERO_EXTEND: 4075 case ISD::SIGN_EXTEND: 4076 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4077 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4078 case Legal: 4079 Tmp1 = LegalizeOp(Node->getOperand(0)); 4080 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4081 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 4082 TargetLowering::Custom) { 4083 Tmp1 = TLI.LowerOperation(Result, DAG); 4084 if (Tmp1.getNode()) Result = Tmp1; 4085 } 4086 break; 4087 case Promote: 4088 switch (Node->getOpcode()) { 4089 case ISD::ANY_EXTEND: 4090 Tmp1 = PromoteOp(Node->getOperand(0)); 4091 Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Tmp1); 4092 break; 4093 case ISD::ZERO_EXTEND: 4094 Result = PromoteOp(Node->getOperand(0)); 4095 Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Result); 4096 Result = DAG.getZeroExtendInReg(Result, 4097 Node->getOperand(0).getValueType()); 4098 break; 4099 case ISD::SIGN_EXTEND: 4100 Result = PromoteOp(Node->getOperand(0)); 4101 Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Result); 4102 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(), 4103 Result, 4104 DAG.getValueType(Node->getOperand(0).getValueType())); 4105 break; 4106 } 4107 } 4108 break; 4109 case ISD::FP_ROUND_INREG: 4110 case ISD::SIGN_EXTEND_INREG: { 4111 Tmp1 = LegalizeOp(Node->getOperand(0)); 4112 MVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 4113 4114 // If this operation is not supported, convert it to a shl/shr or load/store 4115 // pair. 4116 switch (TLI.getOperationAction(Node->getOpcode(), ExtraVT)) { 4117 default: assert(0 && "This action not supported for this op yet!"); 4118 case TargetLowering::Legal: 4119 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 4120 break; 4121 case TargetLowering::Expand: 4122 // If this is an integer extend and shifts are supported, do that. 4123 if (Node->getOpcode() == ISD::SIGN_EXTEND_INREG) { 4124 // NOTE: we could fall back on load/store here too for targets without 4125 // SAR. However, it is doubtful that any exist. 4126 unsigned BitsDiff = Node->getValueType(0).getSizeInBits() - 4127 ExtraVT.getSizeInBits(); 4128 SDValue ShiftCst = DAG.getConstant(BitsDiff, TLI.getShiftAmountTy()); 4129 Result = DAG.getNode(ISD::SHL, Node->getValueType(0), 4130 Node->getOperand(0), ShiftCst); 4131 Result = DAG.getNode(ISD::SRA, Node->getValueType(0), 4132 Result, ShiftCst); 4133 } else if (Node->getOpcode() == ISD::FP_ROUND_INREG) { 4134 // The only way we can lower this is to turn it into a TRUNCSTORE, 4135 // EXTLOAD pair, targetting a temporary location (a stack slot). 4136 4137 // NOTE: there is a choice here between constantly creating new stack 4138 // slots and always reusing the same one. We currently always create 4139 // new ones, as reuse may inhibit scheduling. 4140 Result = EmitStackConvert(Node->getOperand(0), ExtraVT, 4141 Node->getValueType(0)); 4142 } else { 4143 assert(0 && "Unknown op"); 4144 } 4145 break; 4146 } 4147 break; 4148 } 4149 case ISD::TRAMPOLINE: { 4150 SDValue Ops[6]; 4151 for (unsigned i = 0; i != 6; ++i) 4152 Ops[i] = LegalizeOp(Node->getOperand(i)); 4153 Result = DAG.UpdateNodeOperands(Result, Ops, 6); 4154 // The only option for this node is to custom lower it. 4155 Result = TLI.LowerOperation(Result, DAG); 4156 assert(Result.getNode() && "Should always custom lower!"); 4157 4158 // Since trampoline produces two values, make sure to remember that we 4159 // legalized both of them. 4160 Tmp1 = LegalizeOp(Result.getValue(1)); 4161 Result = LegalizeOp(Result); 4162 AddLegalizedOperand(SDValue(Node, 0), Result); 4163 AddLegalizedOperand(SDValue(Node, 1), Tmp1); 4164 return Op.getResNo() ? Tmp1 : Result; 4165 } 4166 case ISD::FLT_ROUNDS_: { 4167 MVT VT = Node->getValueType(0); 4168 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4169 default: assert(0 && "This action not supported for this op yet!"); 4170 case TargetLowering::Custom: 4171 Result = TLI.LowerOperation(Op, DAG); 4172 if (Result.getNode()) break; 4173 // Fall Thru 4174 case TargetLowering::Legal: 4175 // If this operation is not supported, lower it to constant 1 4176 Result = DAG.getConstant(1, VT); 4177 break; 4178 } 4179 break; 4180 } 4181 case ISD::TRAP: { 4182 MVT VT = Node->getValueType(0); 4183 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4184 default: assert(0 && "This action not supported for this op yet!"); 4185 case TargetLowering::Legal: 4186 Tmp1 = LegalizeOp(Node->getOperand(0)); 4187 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4188 break; 4189 case TargetLowering::Custom: 4190 Result = TLI.LowerOperation(Op, DAG); 4191 if (Result.getNode()) break; 4192 // Fall Thru 4193 case TargetLowering::Expand: 4194 // If this operation is not supported, lower it to 'abort()' call 4195 Tmp1 = LegalizeOp(Node->getOperand(0)); 4196 TargetLowering::ArgListTy Args; 4197 std::pair<SDValue,SDValue> CallResult = 4198 TLI.LowerCallTo(Tmp1, Type::VoidTy, 4199 false, false, false, false, CallingConv::C, false, 4200 DAG.getExternalSymbol("abort", TLI.getPointerTy()), 4201 Args, DAG); 4202 Result = CallResult.second; 4203 break; 4204 } 4205 break; 4206 } 4207 4208 case ISD::SADDO: { 4209 MVT VT = Node->getValueType(0); 4210 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4211 default: assert(0 && "This action not supported for this op yet!"); 4212 case TargetLowering::Custom: 4213 Result = TLI.LowerOperation(Op, DAG); 4214 if (Result.getNode()) break; 4215 // FALLTHROUGH 4216 case TargetLowering::Legal: { 4217 SDValue LHS = LegalizeOp(Node->getOperand(0)); 4218 SDValue RHS = LegalizeOp(Node->getOperand(1)); 4219 4220 SDValue Sum = DAG.getNode(ISD::ADD, LHS.getValueType(), LHS, RHS); 4221 MVT OType = Node->getValueType(1); 4222 4223 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 4224 4225 // LHSSign -> LHS >= 0 4226 // RHSSign -> RHS >= 0 4227 // SumSign -> Sum >= 0 4228 // 4229 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 4230 // 4231 SDValue LHSSign = DAG.getSetCC(OType, LHS, Zero, ISD::SETGE); 4232 SDValue RHSSign = DAG.getSetCC(OType, RHS, Zero, ISD::SETGE); 4233 SDValue SignsEq = DAG.getSetCC(OType, LHSSign, RHSSign, ISD::SETEQ); 4234 4235 SDValue SumSign = DAG.getSetCC(OType, Sum, Zero, ISD::SETGE); 4236 SDValue SumSignNE = DAG.getSetCC(OType, LHSSign, SumSign, ISD::SETNE); 4237 4238 SDValue Cmp = DAG.getNode(ISD::AND, OType, SignsEq, SumSignNE); 4239 4240 MVT ValueVTs[] = { LHS.getValueType(), OType }; 4241 SDValue Ops[] = { Sum, Cmp }; 4242 4243 Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(&ValueVTs[0], 2), 4244 &Ops[0], 2); 4245 SDNode *RNode = Result.getNode(); 4246 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), SDValue(RNode, 0)); 4247 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(RNode, 1)); 4248 break; 4249 } 4250 } 4251 4252 break; 4253 } 4254 case ISD::UADDO: { 4255 MVT VT = Node->getValueType(0); 4256 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4257 default: assert(0 && "This action not supported for this op yet!"); 4258 case TargetLowering::Custom: 4259 Result = TLI.LowerOperation(Op, DAG); 4260 if (Result.getNode()) break; 4261 // FALLTHROUGH 4262 case TargetLowering::Legal: { 4263 SDValue LHS = LegalizeOp(Node->getOperand(0)); 4264 SDValue RHS = LegalizeOp(Node->getOperand(1)); 4265 4266 SDValue Sum = DAG.getNode(ISD::ADD, LHS.getValueType(), LHS, RHS); 4267 MVT OType = Node->getValueType(1); 4268 SDValue Cmp = DAG.getSetCC(OType, Sum, LHS, ISD::SETULT); 4269 4270 MVT ValueVTs[] = { LHS.getValueType(), OType }; 4271 SDValue Ops[] = { Sum, Cmp }; 4272 4273 Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(&ValueVTs[0], 2), 4274 &Ops[0], 2); 4275 SDNode *RNode = Result.getNode(); 4276 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), SDValue(RNode, 0)); 4277 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(RNode, 1)); 4278 break; 4279 } 4280 } 4281 4282 break; 4283 } 4284 } 4285 4286 assert(Result.getValueType() == Op.getValueType() && 4287 "Bad legalization!"); 4288 4289 // Make sure that the generated code is itself legal. 4290 if (Result != Op) 4291 Result = LegalizeOp(Result); 4292 4293 // Note that LegalizeOp may be reentered even from single-use nodes, which 4294 // means that we always must cache transformed nodes. 4295 AddLegalizedOperand(Op, Result); 4296 return Result; 4297} 4298 4299/// PromoteOp - Given an operation that produces a value in an invalid type, 4300/// promote it to compute the value into a larger type. The produced value will 4301/// have the correct bits for the low portion of the register, but no guarantee 4302/// is made about the top bits: it may be zero, sign-extended, or garbage. 4303SDValue SelectionDAGLegalize::PromoteOp(SDValue Op) { 4304 MVT VT = Op.getValueType(); 4305 MVT NVT = TLI.getTypeToTransformTo(VT); 4306 assert(getTypeAction(VT) == Promote && 4307 "Caller should expand or legalize operands that are not promotable!"); 4308 assert(NVT.bitsGT(VT) && NVT.isInteger() == VT.isInteger() && 4309 "Cannot promote to smaller type!"); 4310 4311 SDValue Tmp1, Tmp2, Tmp3; 4312 SDValue Result; 4313 SDNode *Node = Op.getNode(); 4314 4315 DenseMap<SDValue, SDValue>::iterator I = PromotedNodes.find(Op); 4316 if (I != PromotedNodes.end()) return I->second; 4317 4318 switch (Node->getOpcode()) { 4319 case ISD::CopyFromReg: 4320 assert(0 && "CopyFromReg must be legal!"); 4321 default: 4322#ifndef NDEBUG 4323 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 4324#endif 4325 assert(0 && "Do not know how to promote this operator!"); 4326 abort(); 4327 case ISD::UNDEF: 4328 Result = DAG.getNode(ISD::UNDEF, NVT); 4329 break; 4330 case ISD::Constant: 4331 if (VT != MVT::i1) 4332 Result = DAG.getNode(ISD::SIGN_EXTEND, NVT, Op); 4333 else 4334 Result = DAG.getNode(ISD::ZERO_EXTEND, NVT, Op); 4335 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold zext?"); 4336 break; 4337 case ISD::ConstantFP: 4338 Result = DAG.getNode(ISD::FP_EXTEND, NVT, Op); 4339 assert(isa<ConstantFPSDNode>(Result) && "Didn't constant fold fp_extend?"); 4340 break; 4341 4342 case ISD::SETCC: 4343 assert(isTypeLegal(TLI.getSetCCResultType(Node->getOperand(0))) 4344 && "SetCC type is not legal??"); 4345 Result = DAG.getNode(ISD::SETCC, 4346 TLI.getSetCCResultType(Node->getOperand(0)), 4347 Node->getOperand(0), Node->getOperand(1), 4348 Node->getOperand(2)); 4349 break; 4350 4351 case ISD::TRUNCATE: 4352 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4353 case Legal: 4354 Result = LegalizeOp(Node->getOperand(0)); 4355 assert(Result.getValueType().bitsGE(NVT) && 4356 "This truncation doesn't make sense!"); 4357 if (Result.getValueType().bitsGT(NVT)) // Truncate to NVT instead of VT 4358 Result = DAG.getNode(ISD::TRUNCATE, NVT, Result); 4359 break; 4360 case Promote: 4361 // The truncation is not required, because we don't guarantee anything 4362 // about high bits anyway. 4363 Result = PromoteOp(Node->getOperand(0)); 4364 break; 4365 case Expand: 4366 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 4367 // Truncate the low part of the expanded value to the result type 4368 Result = DAG.getNode(ISD::TRUNCATE, NVT, Tmp1); 4369 } 4370 break; 4371 case ISD::SIGN_EXTEND: 4372 case ISD::ZERO_EXTEND: 4373 case ISD::ANY_EXTEND: 4374 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4375 case Expand: assert(0 && "BUG: Smaller reg should have been promoted!"); 4376 case Legal: 4377 // Input is legal? Just do extend all the way to the larger type. 4378 Result = DAG.getNode(Node->getOpcode(), NVT, Node->getOperand(0)); 4379 break; 4380 case Promote: 4381 // Promote the reg if it's smaller. 4382 Result = PromoteOp(Node->getOperand(0)); 4383 // The high bits are not guaranteed to be anything. Insert an extend. 4384 if (Node->getOpcode() == ISD::SIGN_EXTEND) 4385 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Result, 4386 DAG.getValueType(Node->getOperand(0).getValueType())); 4387 else if (Node->getOpcode() == ISD::ZERO_EXTEND) 4388 Result = DAG.getZeroExtendInReg(Result, 4389 Node->getOperand(0).getValueType()); 4390 break; 4391 } 4392 break; 4393 case ISD::CONVERT_RNDSAT: { 4394 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 4395 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 4396 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 4397 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 4398 "can only promote integers"); 4399 Result = DAG.getConvertRndSat(NVT, Node->getOperand(0), 4400 Node->getOperand(1), Node->getOperand(2), 4401 Node->getOperand(3), Node->getOperand(4), 4402 CvtCode); 4403 break; 4404 4405 } 4406 case ISD::BIT_CONVERT: 4407 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 4408 Node->getValueType(0)); 4409 Result = PromoteOp(Result); 4410 break; 4411 4412 case ISD::FP_EXTEND: 4413 assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); 4414 case ISD::FP_ROUND: 4415 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4416 case Expand: assert(0 && "BUG: Cannot expand FP regs!"); 4417 case Promote: assert(0 && "Unreachable with 2 FP types!"); 4418 case Legal: 4419 if (Node->getConstantOperandVal(1) == 0) { 4420 // Input is legal? Do an FP_ROUND_INREG. 4421 Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Node->getOperand(0), 4422 DAG.getValueType(VT)); 4423 } else { 4424 // Just remove the truncate, it isn't affecting the value. 4425 Result = DAG.getNode(ISD::FP_ROUND, NVT, Node->getOperand(0), 4426 Node->getOperand(1)); 4427 } 4428 break; 4429 } 4430 break; 4431 case ISD::SINT_TO_FP: 4432 case ISD::UINT_TO_FP: 4433 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4434 case Legal: 4435 // No extra round required here. 4436 Result = DAG.getNode(Node->getOpcode(), NVT, Node->getOperand(0)); 4437 break; 4438 4439 case Promote: 4440 Result = PromoteOp(Node->getOperand(0)); 4441 if (Node->getOpcode() == ISD::SINT_TO_FP) 4442 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(), 4443 Result, 4444 DAG.getValueType(Node->getOperand(0).getValueType())); 4445 else 4446 Result = DAG.getZeroExtendInReg(Result, 4447 Node->getOperand(0).getValueType()); 4448 // No extra round required here. 4449 Result = DAG.getNode(Node->getOpcode(), NVT, Result); 4450 break; 4451 case Expand: 4452 Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, NVT, 4453 Node->getOperand(0)); 4454 // Round if we cannot tolerate excess precision. 4455 if (NoExcessFPPrecision) 4456 Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result, 4457 DAG.getValueType(VT)); 4458 break; 4459 } 4460 break; 4461 4462 case ISD::SIGN_EXTEND_INREG: 4463 Result = PromoteOp(Node->getOperand(0)); 4464 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Result, 4465 Node->getOperand(1)); 4466 break; 4467 case ISD::FP_TO_SINT: 4468 case ISD::FP_TO_UINT: 4469 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4470 case Legal: 4471 case Expand: 4472 Tmp1 = Node->getOperand(0); 4473 break; 4474 case Promote: 4475 // The input result is prerounded, so we don't have to do anything 4476 // special. 4477 Tmp1 = PromoteOp(Node->getOperand(0)); 4478 break; 4479 } 4480 // If we're promoting a UINT to a larger size, check to see if the new node 4481 // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since 4482 // we can use that instead. This allows us to generate better code for 4483 // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not 4484 // legal, such as PowerPC. 4485 if (Node->getOpcode() == ISD::FP_TO_UINT && 4486 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 4487 (TLI.isOperationLegal(ISD::FP_TO_SINT, NVT) || 4488 TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)){ 4489 Result = DAG.getNode(ISD::FP_TO_SINT, NVT, Tmp1); 4490 } else { 4491 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1); 4492 } 4493 break; 4494 4495 case ISD::FABS: 4496 case ISD::FNEG: 4497 Tmp1 = PromoteOp(Node->getOperand(0)); 4498 assert(Tmp1.getValueType() == NVT); 4499 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1); 4500 // NOTE: we do not have to do any extra rounding here for 4501 // NoExcessFPPrecision, because we know the input will have the appropriate 4502 // precision, and these operations don't modify precision at all. 4503 break; 4504 4505 case ISD::FLOG: 4506 case ISD::FLOG2: 4507 case ISD::FLOG10: 4508 case ISD::FEXP: 4509 case ISD::FEXP2: 4510 case ISD::FSQRT: 4511 case ISD::FSIN: 4512 case ISD::FCOS: 4513 case ISD::FTRUNC: 4514 case ISD::FFLOOR: 4515 case ISD::FCEIL: 4516 case ISD::FRINT: 4517 case ISD::FNEARBYINT: 4518 Tmp1 = PromoteOp(Node->getOperand(0)); 4519 assert(Tmp1.getValueType() == NVT); 4520 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1); 4521 if (NoExcessFPPrecision) 4522 Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result, 4523 DAG.getValueType(VT)); 4524 break; 4525 4526 case ISD::FPOW: 4527 case ISD::FPOWI: { 4528 // Promote f32 pow(i) to f64 pow(i). Note that this could insert a libcall 4529 // directly as well, which may be better. 4530 Tmp1 = PromoteOp(Node->getOperand(0)); 4531 Tmp2 = Node->getOperand(1); 4532 if (Node->getOpcode() == ISD::FPOW) 4533 Tmp2 = PromoteOp(Tmp2); 4534 assert(Tmp1.getValueType() == NVT); 4535 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2); 4536 if (NoExcessFPPrecision) 4537 Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result, 4538 DAG.getValueType(VT)); 4539 break; 4540 } 4541 4542 case ISD::ATOMIC_CMP_SWAP_8: 4543 case ISD::ATOMIC_CMP_SWAP_16: 4544 case ISD::ATOMIC_CMP_SWAP_32: 4545 case ISD::ATOMIC_CMP_SWAP_64: { 4546 AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node); 4547 Tmp2 = PromoteOp(Node->getOperand(2)); 4548 Tmp3 = PromoteOp(Node->getOperand(3)); 4549 Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(), 4550 AtomNode->getBasePtr(), Tmp2, Tmp3, 4551 AtomNode->getSrcValue(), 4552 AtomNode->getAlignment()); 4553 // Remember that we legalized the chain. 4554 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4555 break; 4556 } 4557 case ISD::ATOMIC_LOAD_ADD_8: 4558 case ISD::ATOMIC_LOAD_SUB_8: 4559 case ISD::ATOMIC_LOAD_AND_8: 4560 case ISD::ATOMIC_LOAD_OR_8: 4561 case ISD::ATOMIC_LOAD_XOR_8: 4562 case ISD::ATOMIC_LOAD_NAND_8: 4563 case ISD::ATOMIC_LOAD_MIN_8: 4564 case ISD::ATOMIC_LOAD_MAX_8: 4565 case ISD::ATOMIC_LOAD_UMIN_8: 4566 case ISD::ATOMIC_LOAD_UMAX_8: 4567 case ISD::ATOMIC_SWAP_8: 4568 case ISD::ATOMIC_LOAD_ADD_16: 4569 case ISD::ATOMIC_LOAD_SUB_16: 4570 case ISD::ATOMIC_LOAD_AND_16: 4571 case ISD::ATOMIC_LOAD_OR_16: 4572 case ISD::ATOMIC_LOAD_XOR_16: 4573 case ISD::ATOMIC_LOAD_NAND_16: 4574 case ISD::ATOMIC_LOAD_MIN_16: 4575 case ISD::ATOMIC_LOAD_MAX_16: 4576 case ISD::ATOMIC_LOAD_UMIN_16: 4577 case ISD::ATOMIC_LOAD_UMAX_16: 4578 case ISD::ATOMIC_SWAP_16: 4579 case ISD::ATOMIC_LOAD_ADD_32: 4580 case ISD::ATOMIC_LOAD_SUB_32: 4581 case ISD::ATOMIC_LOAD_AND_32: 4582 case ISD::ATOMIC_LOAD_OR_32: 4583 case ISD::ATOMIC_LOAD_XOR_32: 4584 case ISD::ATOMIC_LOAD_NAND_32: 4585 case ISD::ATOMIC_LOAD_MIN_32: 4586 case ISD::ATOMIC_LOAD_MAX_32: 4587 case ISD::ATOMIC_LOAD_UMIN_32: 4588 case ISD::ATOMIC_LOAD_UMAX_32: 4589 case ISD::ATOMIC_SWAP_32: 4590 case ISD::ATOMIC_LOAD_ADD_64: 4591 case ISD::ATOMIC_LOAD_SUB_64: 4592 case ISD::ATOMIC_LOAD_AND_64: 4593 case ISD::ATOMIC_LOAD_OR_64: 4594 case ISD::ATOMIC_LOAD_XOR_64: 4595 case ISD::ATOMIC_LOAD_NAND_64: 4596 case ISD::ATOMIC_LOAD_MIN_64: 4597 case ISD::ATOMIC_LOAD_MAX_64: 4598 case ISD::ATOMIC_LOAD_UMIN_64: 4599 case ISD::ATOMIC_LOAD_UMAX_64: 4600 case ISD::ATOMIC_SWAP_64: { 4601 AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node); 4602 Tmp2 = PromoteOp(Node->getOperand(2)); 4603 Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(), 4604 AtomNode->getBasePtr(), Tmp2, 4605 AtomNode->getSrcValue(), 4606 AtomNode->getAlignment()); 4607 // Remember that we legalized the chain. 4608 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4609 break; 4610 } 4611 4612 case ISD::AND: 4613 case ISD::OR: 4614 case ISD::XOR: 4615 case ISD::ADD: 4616 case ISD::SUB: 4617 case ISD::MUL: 4618 // The input may have strange things in the top bits of the registers, but 4619 // these operations don't care. They may have weird bits going out, but 4620 // that too is okay if they are integer operations. 4621 Tmp1 = PromoteOp(Node->getOperand(0)); 4622 Tmp2 = PromoteOp(Node->getOperand(1)); 4623 assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT); 4624 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2); 4625 break; 4626 case ISD::FADD: 4627 case ISD::FSUB: 4628 case ISD::FMUL: 4629 Tmp1 = PromoteOp(Node->getOperand(0)); 4630 Tmp2 = PromoteOp(Node->getOperand(1)); 4631 assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT); 4632 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2); 4633 4634 // Floating point operations will give excess precision that we may not be 4635 // able to tolerate. If we DO allow excess precision, just leave it, 4636 // otherwise excise it. 4637 // FIXME: Why would we need to round FP ops more than integer ones? 4638 // Is Round(Add(Add(A,B),C)) != Round(Add(Round(Add(A,B)), C)) 4639 if (NoExcessFPPrecision) 4640 Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result, 4641 DAG.getValueType(VT)); 4642 break; 4643 4644 case ISD::SDIV: 4645 case ISD::SREM: 4646 // These operators require that their input be sign extended. 4647 Tmp1 = PromoteOp(Node->getOperand(0)); 4648 Tmp2 = PromoteOp(Node->getOperand(1)); 4649 if (NVT.isInteger()) { 4650 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1, 4651 DAG.getValueType(VT)); 4652 Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2, 4653 DAG.getValueType(VT)); 4654 } 4655 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2); 4656 4657 // Perform FP_ROUND: this is probably overly pessimistic. 4658 if (NVT.isFloatingPoint() && NoExcessFPPrecision) 4659 Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result, 4660 DAG.getValueType(VT)); 4661 break; 4662 case ISD::FDIV: 4663 case ISD::FREM: 4664 case ISD::FCOPYSIGN: 4665 // These operators require that their input be fp extended. 4666 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4667 case Expand: assert(0 && "not implemented"); 4668 case Legal: Tmp1 = LegalizeOp(Node->getOperand(0)); break; 4669 case Promote: Tmp1 = PromoteOp(Node->getOperand(0)); break; 4670 } 4671 switch (getTypeAction(Node->getOperand(1).getValueType())) { 4672 case Expand: assert(0 && "not implemented"); 4673 case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; 4674 case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; 4675 } 4676 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2); 4677 4678 // Perform FP_ROUND: this is probably overly pessimistic. 4679 if (NoExcessFPPrecision && Node->getOpcode() != ISD::FCOPYSIGN) 4680 Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result, 4681 DAG.getValueType(VT)); 4682 break; 4683 4684 case ISD::UDIV: 4685 case ISD::UREM: 4686 // These operators require that their input be zero extended. 4687 Tmp1 = PromoteOp(Node->getOperand(0)); 4688 Tmp2 = PromoteOp(Node->getOperand(1)); 4689 assert(NVT.isInteger() && "Operators don't apply to FP!"); 4690 Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT); 4691 Tmp2 = DAG.getZeroExtendInReg(Tmp2, VT); 4692 Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2); 4693 break; 4694 4695 case ISD::SHL: 4696 Tmp1 = PromoteOp(Node->getOperand(0)); 4697 Result = DAG.getNode(ISD::SHL, NVT, Tmp1, Node->getOperand(1)); 4698 break; 4699 case ISD::SRA: 4700 // The input value must be properly sign extended. 4701 Tmp1 = PromoteOp(Node->getOperand(0)); 4702 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1, 4703 DAG.getValueType(VT)); 4704 Result = DAG.getNode(ISD::SRA, NVT, Tmp1, Node->getOperand(1)); 4705 break; 4706 case ISD::SRL: 4707 // The input value must be properly zero extended. 4708 Tmp1 = PromoteOp(Node->getOperand(0)); 4709 Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT); 4710 Result = DAG.getNode(ISD::SRL, NVT, Tmp1, Node->getOperand(1)); 4711 break; 4712 4713 case ISD::VAARG: 4714 Tmp1 = Node->getOperand(0); // Get the chain. 4715 Tmp2 = Node->getOperand(1); // Get the pointer. 4716 if (TLI.getOperationAction(ISD::VAARG, VT) == TargetLowering::Custom) { 4717 Tmp3 = DAG.getVAArg(VT, Tmp1, Tmp2, Node->getOperand(2)); 4718 Result = TLI.LowerOperation(Tmp3, DAG); 4719 } else { 4720 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 4721 SDValue VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0); 4722 // Increment the pointer, VAList, to the next vaarg 4723 Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, 4724 DAG.getConstant(VT.getSizeInBits()/8, 4725 TLI.getPointerTy())); 4726 // Store the incremented VAList to the legalized pointer 4727 Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, V, 0); 4728 // Load the actual argument out of the pointer VAList 4729 Result = DAG.getExtLoad(ISD::EXTLOAD, NVT, Tmp3, VAList, NULL, 0, VT); 4730 } 4731 // Remember that we legalized the chain. 4732 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4733 break; 4734 4735 case ISD::LOAD: { 4736 LoadSDNode *LD = cast<LoadSDNode>(Node); 4737 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(Node) 4738 ? ISD::EXTLOAD : LD->getExtensionType(); 4739 Result = DAG.getExtLoad(ExtType, NVT, 4740 LD->getChain(), LD->getBasePtr(), 4741 LD->getSrcValue(), LD->getSrcValueOffset(), 4742 LD->getMemoryVT(), 4743 LD->isVolatile(), 4744 LD->getAlignment()); 4745 // Remember that we legalized the chain. 4746 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4747 break; 4748 } 4749 case ISD::SELECT: { 4750 Tmp2 = PromoteOp(Node->getOperand(1)); // Legalize the op0 4751 Tmp3 = PromoteOp(Node->getOperand(2)); // Legalize the op1 4752 4753 MVT VT2 = Tmp2.getValueType(); 4754 assert(VT2 == Tmp3.getValueType() 4755 && "PromoteOp SELECT: Operands 2 and 3 ValueTypes don't match"); 4756 // Ensure that the resulting node is at least the same size as the operands' 4757 // value types, because we cannot assume that TLI.getSetCCValueType() is 4758 // constant. 4759 Result = DAG.getNode(ISD::SELECT, VT2, Node->getOperand(0), Tmp2, Tmp3); 4760 break; 4761 } 4762 case ISD::SELECT_CC: 4763 Tmp2 = PromoteOp(Node->getOperand(2)); // True 4764 Tmp3 = PromoteOp(Node->getOperand(3)); // False 4765 Result = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0), 4766 Node->getOperand(1), Tmp2, Tmp3, Node->getOperand(4)); 4767 break; 4768 case ISD::BSWAP: 4769 Tmp1 = Node->getOperand(0); 4770 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1); 4771 Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1); 4772 Result = DAG.getNode(ISD::SRL, NVT, Tmp1, 4773 DAG.getConstant(NVT.getSizeInBits() - 4774 VT.getSizeInBits(), 4775 TLI.getShiftAmountTy())); 4776 break; 4777 case ISD::CTPOP: 4778 case ISD::CTTZ: 4779 case ISD::CTLZ: 4780 // Zero extend the argument 4781 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Node->getOperand(0)); 4782 // Perform the larger operation, then subtract if needed. 4783 Tmp1 = DAG.getNode(Node->getOpcode(), NVT, Tmp1); 4784 switch(Node->getOpcode()) { 4785 case ISD::CTPOP: 4786 Result = Tmp1; 4787 break; 4788 case ISD::CTTZ: 4789 // if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT) 4790 Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1, 4791 DAG.getConstant(NVT.getSizeInBits(), NVT), 4792 ISD::SETEQ); 4793 Result = DAG.getNode(ISD::SELECT, NVT, Tmp2, 4794 DAG.getConstant(VT.getSizeInBits(), NVT), Tmp1); 4795 break; 4796 case ISD::CTLZ: 4797 //Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 4798 Result = DAG.getNode(ISD::SUB, NVT, Tmp1, 4799 DAG.getConstant(NVT.getSizeInBits() - 4800 VT.getSizeInBits(), NVT)); 4801 break; 4802 } 4803 break; 4804 case ISD::EXTRACT_SUBVECTOR: 4805 Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op)); 4806 break; 4807 case ISD::EXTRACT_VECTOR_ELT: 4808 Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op)); 4809 break; 4810 } 4811 4812 assert(Result.getNode() && "Didn't set a result!"); 4813 4814 // Make sure the result is itself legal. 4815 Result = LegalizeOp(Result); 4816 4817 // Remember that we promoted this! 4818 AddPromotedOperand(Op, Result); 4819 return Result; 4820} 4821 4822/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into 4823/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic, 4824/// based on the vector type. The return type of this matches the element type 4825/// of the vector, which may not be legal for the target. 4826SDValue SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDValue Op) { 4827 // We know that operand #0 is the Vec vector. If the index is a constant 4828 // or if the invec is a supported hardware type, we can use it. Otherwise, 4829 // lower to a store then an indexed load. 4830 SDValue Vec = Op.getOperand(0); 4831 SDValue Idx = Op.getOperand(1); 4832 4833 MVT TVT = Vec.getValueType(); 4834 unsigned NumElems = TVT.getVectorNumElements(); 4835 4836 switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) { 4837 default: assert(0 && "This action is not supported yet!"); 4838 case TargetLowering::Custom: { 4839 Vec = LegalizeOp(Vec); 4840 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 4841 SDValue Tmp3 = TLI.LowerOperation(Op, DAG); 4842 if (Tmp3.getNode()) 4843 return Tmp3; 4844 break; 4845 } 4846 case TargetLowering::Legal: 4847 if (isTypeLegal(TVT)) { 4848 Vec = LegalizeOp(Vec); 4849 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 4850 return Op; 4851 } 4852 break; 4853 case TargetLowering::Promote: 4854 assert(TVT.isVector() && "not vector type"); 4855 // fall thru to expand since vectors are by default are promote 4856 case TargetLowering::Expand: 4857 break; 4858 } 4859 4860 if (NumElems == 1) { 4861 // This must be an access of the only element. Return it. 4862 Op = ScalarizeVectorOp(Vec); 4863 } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) { 4864 unsigned NumLoElts = 1 << Log2_32(NumElems-1); 4865 ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx); 4866 SDValue Lo, Hi; 4867 SplitVectorOp(Vec, Lo, Hi); 4868 if (CIdx->getZExtValue() < NumLoElts) { 4869 Vec = Lo; 4870 } else { 4871 Vec = Hi; 4872 Idx = DAG.getConstant(CIdx->getZExtValue() - NumLoElts, 4873 Idx.getValueType()); 4874 } 4875 4876 // It's now an extract from the appropriate high or low part. Recurse. 4877 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 4878 Op = ExpandEXTRACT_VECTOR_ELT(Op); 4879 } else { 4880 // Store the value to a temporary stack slot, then LOAD the scalar 4881 // element back out. 4882 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 4883 SDValue Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); 4884 4885 // Add the offset to the index. 4886 unsigned EltSize = Op.getValueType().getSizeInBits()/8; 4887 Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx, 4888 DAG.getConstant(EltSize, Idx.getValueType())); 4889 4890 if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 4891 Idx = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), Idx); 4892 else 4893 Idx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), Idx); 4894 4895 StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr); 4896 4897 Op = DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0); 4898 } 4899 return Op; 4900} 4901 4902/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation. For now 4903/// we assume the operation can be split if it is not already legal. 4904SDValue SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDValue Op) { 4905 // We know that operand #0 is the Vec vector. For now we assume the index 4906 // is a constant and that the extracted result is a supported hardware type. 4907 SDValue Vec = Op.getOperand(0); 4908 SDValue Idx = LegalizeOp(Op.getOperand(1)); 4909 4910 unsigned NumElems = Vec.getValueType().getVectorNumElements(); 4911 4912 if (NumElems == Op.getValueType().getVectorNumElements()) { 4913 // This must be an access of the desired vector length. Return it. 4914 return Vec; 4915 } 4916 4917 ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx); 4918 SDValue Lo, Hi; 4919 SplitVectorOp(Vec, Lo, Hi); 4920 if (CIdx->getZExtValue() < NumElems/2) { 4921 Vec = Lo; 4922 } else { 4923 Vec = Hi; 4924 Idx = DAG.getConstant(CIdx->getZExtValue() - NumElems/2, 4925 Idx.getValueType()); 4926 } 4927 4928 // It's now an extract from the appropriate high or low part. Recurse. 4929 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 4930 return ExpandEXTRACT_SUBVECTOR(Op); 4931} 4932 4933/// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC 4934/// with condition CC on the current target. This usually involves legalizing 4935/// or promoting the arguments. In the case where LHS and RHS must be expanded, 4936/// there may be no choice but to create a new SetCC node to represent the 4937/// legalized value of setcc lhs, rhs. In this case, the value is returned in 4938/// LHS, and the SDValue returned in RHS has a nil SDNode value. 4939void SelectionDAGLegalize::LegalizeSetCCOperands(SDValue &LHS, 4940 SDValue &RHS, 4941 SDValue &CC) { 4942 SDValue Tmp1, Tmp2, Tmp3, Result; 4943 4944 switch (getTypeAction(LHS.getValueType())) { 4945 case Legal: 4946 Tmp1 = LegalizeOp(LHS); // LHS 4947 Tmp2 = LegalizeOp(RHS); // RHS 4948 break; 4949 case Promote: 4950 Tmp1 = PromoteOp(LHS); // LHS 4951 Tmp2 = PromoteOp(RHS); // RHS 4952 4953 // If this is an FP compare, the operands have already been extended. 4954 if (LHS.getValueType().isInteger()) { 4955 MVT VT = LHS.getValueType(); 4956 MVT NVT = TLI.getTypeToTransformTo(VT); 4957 4958 // Otherwise, we have to insert explicit sign or zero extends. Note 4959 // that we could insert sign extends for ALL conditions, but zero extend 4960 // is cheaper on many machines (an AND instead of two shifts), so prefer 4961 // it. 4962 switch (cast<CondCodeSDNode>(CC)->get()) { 4963 default: assert(0 && "Unknown integer comparison!"); 4964 case ISD::SETEQ: 4965 case ISD::SETNE: 4966 case ISD::SETUGE: 4967 case ISD::SETUGT: 4968 case ISD::SETULE: 4969 case ISD::SETULT: 4970 // ALL of these operations will work if we either sign or zero extend 4971 // the operands (including the unsigned comparisons!). Zero extend is 4972 // usually a simpler/cheaper operation, so prefer it. 4973 Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT); 4974 Tmp2 = DAG.getZeroExtendInReg(Tmp2, VT); 4975 break; 4976 case ISD::SETGE: 4977 case ISD::SETGT: 4978 case ISD::SETLT: 4979 case ISD::SETLE: 4980 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1, 4981 DAG.getValueType(VT)); 4982 Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2, 4983 DAG.getValueType(VT)); 4984 Tmp1 = LegalizeOp(Tmp1); // Relegalize new nodes. 4985 Tmp2 = LegalizeOp(Tmp2); // Relegalize new nodes. 4986 break; 4987 } 4988 } 4989 break; 4990 case Expand: { 4991 MVT VT = LHS.getValueType(); 4992 if (VT == MVT::f32 || VT == MVT::f64) { 4993 // Expand into one or more soft-fp libcall(s). 4994 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL; 4995 switch (cast<CondCodeSDNode>(CC)->get()) { 4996 case ISD::SETEQ: 4997 case ISD::SETOEQ: 4998 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; 4999 break; 5000 case ISD::SETNE: 5001 case ISD::SETUNE: 5002 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : RTLIB::UNE_F64; 5003 break; 5004 case ISD::SETGE: 5005 case ISD::SETOGE: 5006 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; 5007 break; 5008 case ISD::SETLT: 5009 case ISD::SETOLT: 5010 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5011 break; 5012 case ISD::SETLE: 5013 case ISD::SETOLE: 5014 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; 5015 break; 5016 case ISD::SETGT: 5017 case ISD::SETOGT: 5018 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; 5019 break; 5020 case ISD::SETUO: 5021 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; 5022 break; 5023 case ISD::SETO: 5024 LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : RTLIB::O_F64; 5025 break; 5026 default: 5027 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; 5028 switch (cast<CondCodeSDNode>(CC)->get()) { 5029 case ISD::SETONE: 5030 // SETONE = SETOLT | SETOGT 5031 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5032 // Fallthrough 5033 case ISD::SETUGT: 5034 LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; 5035 break; 5036 case ISD::SETUGE: 5037 LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; 5038 break; 5039 case ISD::SETULT: 5040 LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5041 break; 5042 case ISD::SETULE: 5043 LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; 5044 break; 5045 case ISD::SETUEQ: 5046 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; 5047 break; 5048 default: assert(0 && "Unsupported FP setcc!"); 5049 } 5050 } 5051 5052 SDValue Dummy; 5053 SDValue Ops[2] = { LHS, RHS }; 5054 Tmp1 = ExpandLibCall(LC1, DAG.getMergeValues(Ops, 2).getNode(), 5055 false /*sign irrelevant*/, Dummy); 5056 Tmp2 = DAG.getConstant(0, MVT::i32); 5057 CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1)); 5058 if (LC2 != RTLIB::UNKNOWN_LIBCALL) { 5059 Tmp1 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2, 5060 CC); 5061 LHS = ExpandLibCall(LC2, DAG.getMergeValues(Ops, 2).getNode(), 5062 false /*sign irrelevant*/, Dummy); 5063 Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHS), LHS, Tmp2, 5064 DAG.getCondCode(TLI.getCmpLibcallCC(LC2))); 5065 Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2); 5066 Tmp2 = SDValue(); 5067 } 5068 LHS = LegalizeOp(Tmp1); 5069 RHS = Tmp2; 5070 return; 5071 } 5072 5073 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 5074 ExpandOp(LHS, LHSLo, LHSHi); 5075 ExpandOp(RHS, RHSLo, RHSHi); 5076 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 5077 5078 if (VT==MVT::ppcf128) { 5079 // FIXME: This generated code sucks. We want to generate 5080 // FCMPU crN, hi1, hi2 5081 // BNE crN, L: 5082 // FCMPU crN, lo1, lo2 5083 // The following can be improved, but not that much. 5084 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 5085 ISD::SETOEQ); 5086 Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, CCCode); 5087 Tmp3 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2); 5088 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 5089 ISD::SETUNE); 5090 Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, CCCode); 5091 Tmp1 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2); 5092 Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp3); 5093 Tmp2 = SDValue(); 5094 break; 5095 } 5096 5097 switch (CCCode) { 5098 case ISD::SETEQ: 5099 case ISD::SETNE: 5100 if (RHSLo == RHSHi) 5101 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) 5102 if (RHSCST->isAllOnesValue()) { 5103 // Comparison to -1. 5104 Tmp1 = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi); 5105 Tmp2 = RHSLo; 5106 break; 5107 } 5108 5109 Tmp1 = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo); 5110 Tmp2 = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi); 5111 Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2); 5112 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 5113 break; 5114 default: 5115 // If this is a comparison of the sign bit, just look at the top part. 5116 // X > -1, x < 0 5117 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(RHS)) 5118 if ((cast<CondCodeSDNode>(CC)->get() == ISD::SETLT && 5119 CST->isNullValue()) || // X < 0 5120 (cast<CondCodeSDNode>(CC)->get() == ISD::SETGT && 5121 CST->isAllOnesValue())) { // X > -1 5122 Tmp1 = LHSHi; 5123 Tmp2 = RHSHi; 5124 break; 5125 } 5126 5127 // FIXME: This generated code sucks. 5128 ISD::CondCode LowCC; 5129 switch (CCCode) { 5130 default: assert(0 && "Unknown integer setcc!"); 5131 case ISD::SETLT: 5132 case ISD::SETULT: LowCC = ISD::SETULT; break; 5133 case ISD::SETGT: 5134 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 5135 case ISD::SETLE: 5136 case ISD::SETULE: LowCC = ISD::SETULE; break; 5137 case ISD::SETGE: 5138 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 5139 } 5140 5141 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 5142 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 5143 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 5144 5145 // NOTE: on targets without efficient SELECT of bools, we can always use 5146 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 5147 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL); 5148 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, 5149 LowCC, false, DagCombineInfo); 5150 if (!Tmp1.getNode()) 5151 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC); 5152 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 5153 CCCode, false, DagCombineInfo); 5154 if (!Tmp2.getNode()) 5155 Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHSHi), LHSHi, 5156 RHSHi,CC); 5157 5158 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 5159 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 5160 if ((Tmp1C && Tmp1C->isNullValue()) || 5161 (Tmp2C && Tmp2C->isNullValue() && 5162 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 5163 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 5164 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 5165 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 5166 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 5167 // low part is known false, returns high part. 5168 // For LE / GE, if high part is known false, ignore the low part. 5169 // For LT / GT, if high part is known true, ignore the low part. 5170 Tmp1 = Tmp2; 5171 Tmp2 = SDValue(); 5172 } else { 5173 Result = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 5174 ISD::SETEQ, false, DagCombineInfo); 5175 if (!Result.getNode()) 5176 Result=DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 5177 ISD::SETEQ); 5178 Result = LegalizeOp(DAG.getNode(ISD::SELECT, Tmp1.getValueType(), 5179 Result, Tmp1, Tmp2)); 5180 Tmp1 = Result; 5181 Tmp2 = SDValue(); 5182 } 5183 } 5184 } 5185 } 5186 LHS = Tmp1; 5187 RHS = Tmp2; 5188} 5189 5190/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and 5191/// condition code CC on the current target. This routine assumes LHS and rHS 5192/// have already been legalized by LegalizeSetCCOperands. It expands SETCC with 5193/// illegal condition code into AND / OR of multiple SETCC values. 5194void SelectionDAGLegalize::LegalizeSetCCCondCode(MVT VT, 5195 SDValue &LHS, SDValue &RHS, 5196 SDValue &CC) { 5197 MVT OpVT = LHS.getValueType(); 5198 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 5199 switch (TLI.getCondCodeAction(CCCode, OpVT)) { 5200 default: assert(0 && "Unknown condition code action!"); 5201 case TargetLowering::Legal: 5202 // Nothing to do. 5203 break; 5204 case TargetLowering::Expand: { 5205 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 5206 unsigned Opc = 0; 5207 switch (CCCode) { 5208 default: assert(0 && "Don't know how to expand this condition!"); abort(); 5209 case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break; 5210 case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break; 5211 case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5212 case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break; 5213 case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5214 case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5215 case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5216 case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5217 case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5218 case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5219 case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5220 case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5221 // FIXME: Implement more expansions. 5222 } 5223 5224 SDValue SetCC1 = DAG.getSetCC(VT, LHS, RHS, CC1); 5225 SDValue SetCC2 = DAG.getSetCC(VT, LHS, RHS, CC2); 5226 LHS = DAG.getNode(Opc, VT, SetCC1, SetCC2); 5227 RHS = SDValue(); 5228 CC = SDValue(); 5229 break; 5230 } 5231 } 5232} 5233 5234/// EmitStackConvert - Emit a store/load combination to the stack. This stores 5235/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 5236/// a load from the stack slot to DestVT, extending it if needed. 5237/// The resultant code need not be legal. 5238SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, 5239 MVT SlotVT, 5240 MVT DestVT) { 5241 // Create the stack frame object. 5242 unsigned SrcAlign = TLI.getTargetData()->getPrefTypeAlignment( 5243 SrcOp.getValueType().getTypeForMVT()); 5244 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 5245 5246 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 5247 int SPFI = StackPtrFI->getIndex(); 5248 5249 unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); 5250 unsigned SlotSize = SlotVT.getSizeInBits(); 5251 unsigned DestSize = DestVT.getSizeInBits(); 5252 unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment( 5253 DestVT.getTypeForMVT()); 5254 5255 // Emit a store to the stack slot. Use a truncstore if the input value is 5256 // later than DestVT. 5257 SDValue Store; 5258 5259 if (SrcSize > SlotSize) 5260 Store = DAG.getTruncStore(DAG.getEntryNode(), SrcOp, FIPtr, 5261 PseudoSourceValue::getFixedStack(SPFI), 0, 5262 SlotVT, false, SrcAlign); 5263 else { 5264 assert(SrcSize == SlotSize && "Invalid store"); 5265 Store = DAG.getStore(DAG.getEntryNode(), SrcOp, FIPtr, 5266 PseudoSourceValue::getFixedStack(SPFI), 0, 5267 false, SrcAlign); 5268 } 5269 5270 // Result is a load from the stack slot. 5271 if (SlotSize == DestSize) 5272 return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0, false, DestAlign); 5273 5274 assert(SlotSize < DestSize && "Unknown extension!"); 5275 return DAG.getExtLoad(ISD::EXTLOAD, DestVT, Store, FIPtr, NULL, 0, SlotVT, 5276 false, DestAlign); 5277} 5278 5279SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 5280 // Create a vector sized/aligned stack slot, store the value to element #0, 5281 // then load the whole vector back out. 5282 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 5283 5284 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 5285 int SPFI = StackPtrFI->getIndex(); 5286 5287 SDValue Ch = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0), StackPtr, 5288 PseudoSourceValue::getFixedStack(SPFI), 0); 5289 return DAG.getLoad(Node->getValueType(0), Ch, StackPtr, 5290 PseudoSourceValue::getFixedStack(SPFI), 0); 5291} 5292 5293 5294/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't 5295/// support the operation, but do support the resultant vector type. 5296SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 5297 5298 // If the only non-undef value is the low element, turn this into a 5299 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 5300 unsigned NumElems = Node->getNumOperands(); 5301 bool isOnlyLowElement = true; 5302 SDValue SplatValue = Node->getOperand(0); 5303 5304 // FIXME: it would be far nicer to change this into map<SDValue,uint64_t> 5305 // and use a bitmask instead of a list of elements. 5306 std::map<SDValue, std::vector<unsigned> > Values; 5307 Values[SplatValue].push_back(0); 5308 bool isConstant = true; 5309 if (!isa<ConstantFPSDNode>(SplatValue) && !isa<ConstantSDNode>(SplatValue) && 5310 SplatValue.getOpcode() != ISD::UNDEF) 5311 isConstant = false; 5312 5313 for (unsigned i = 1; i < NumElems; ++i) { 5314 SDValue V = Node->getOperand(i); 5315 Values[V].push_back(i); 5316 if (V.getOpcode() != ISD::UNDEF) 5317 isOnlyLowElement = false; 5318 if (SplatValue != V) 5319 SplatValue = SDValue(0,0); 5320 5321 // If this isn't a constant element or an undef, we can't use a constant 5322 // pool load. 5323 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V) && 5324 V.getOpcode() != ISD::UNDEF) 5325 isConstant = false; 5326 } 5327 5328 if (isOnlyLowElement) { 5329 // If the low element is an undef too, then this whole things is an undef. 5330 if (Node->getOperand(0).getOpcode() == ISD::UNDEF) 5331 return DAG.getNode(ISD::UNDEF, Node->getValueType(0)); 5332 // Otherwise, turn this into a scalar_to_vector node. 5333 return DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), 5334 Node->getOperand(0)); 5335 } 5336 5337 // If all elements are constants, create a load from the constant pool. 5338 if (isConstant) { 5339 MVT VT = Node->getValueType(0); 5340 std::vector<Constant*> CV; 5341 for (unsigned i = 0, e = NumElems; i != e; ++i) { 5342 if (ConstantFPSDNode *V = 5343 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 5344 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 5345 } else if (ConstantSDNode *V = 5346 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 5347 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 5348 } else { 5349 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); 5350 const Type *OpNTy = 5351 Node->getOperand(0).getValueType().getTypeForMVT(); 5352 CV.push_back(UndefValue::get(OpNTy)); 5353 } 5354 } 5355 Constant *CP = ConstantVector::get(CV); 5356 SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); 5357 unsigned Alignment = 1 << cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 5358 return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, 5359 PseudoSourceValue::getConstantPool(), 0, 5360 false, Alignment); 5361 } 5362 5363 if (SplatValue.getNode()) { // Splat of one value? 5364 // Build the shuffle constant vector: <0, 0, 0, 0> 5365 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 5366 SDValue Zero = DAG.getConstant(0, MaskVT.getVectorElementType()); 5367 std::vector<SDValue> ZeroVec(NumElems, Zero); 5368 SDValue SplatMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 5369 &ZeroVec[0], ZeroVec.size()); 5370 5371 // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it. 5372 if (isShuffleLegal(Node->getValueType(0), SplatMask)) { 5373 // Get the splatted value into the low element of a vector register. 5374 SDValue LowValVec = 5375 DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), SplatValue); 5376 5377 // Return shuffle(LowValVec, undef, <0,0,0,0>) 5378 return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0), LowValVec, 5379 DAG.getNode(ISD::UNDEF, Node->getValueType(0)), 5380 SplatMask); 5381 } 5382 } 5383 5384 // If there are only two unique elements, we may be able to turn this into a 5385 // vector shuffle. 5386 if (Values.size() == 2) { 5387 // Get the two values in deterministic order. 5388 SDValue Val1 = Node->getOperand(1); 5389 SDValue Val2; 5390 std::map<SDValue, std::vector<unsigned> >::iterator MI = Values.begin(); 5391 if (MI->first != Val1) 5392 Val2 = MI->first; 5393 else 5394 Val2 = (++MI)->first; 5395 5396 // If Val1 is an undef, make sure end ends up as Val2, to ensure that our 5397 // vector shuffle has the undef vector on the RHS. 5398 if (Val1.getOpcode() == ISD::UNDEF) 5399 std::swap(Val1, Val2); 5400 5401 // Build the shuffle constant vector: e.g. <0, 4, 0, 4> 5402 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 5403 MVT MaskEltVT = MaskVT.getVectorElementType(); 5404 std::vector<SDValue> MaskVec(NumElems); 5405 5406 // Set elements of the shuffle mask for Val1. 5407 std::vector<unsigned> &Val1Elts = Values[Val1]; 5408 for (unsigned i = 0, e = Val1Elts.size(); i != e; ++i) 5409 MaskVec[Val1Elts[i]] = DAG.getConstant(0, MaskEltVT); 5410 5411 // Set elements of the shuffle mask for Val2. 5412 std::vector<unsigned> &Val2Elts = Values[Val2]; 5413 for (unsigned i = 0, e = Val2Elts.size(); i != e; ++i) 5414 if (Val2.getOpcode() != ISD::UNDEF) 5415 MaskVec[Val2Elts[i]] = DAG.getConstant(NumElems, MaskEltVT); 5416 else 5417 MaskVec[Val2Elts[i]] = DAG.getNode(ISD::UNDEF, MaskEltVT); 5418 5419 SDValue ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, 5420 &MaskVec[0], MaskVec.size()); 5421 5422 // If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it. 5423 if (TLI.isOperationLegal(ISD::SCALAR_TO_VECTOR, Node->getValueType(0)) && 5424 isShuffleLegal(Node->getValueType(0), ShuffleMask)) { 5425 Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), Val1); 5426 Val2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), Val2); 5427 SDValue Ops[] = { Val1, Val2, ShuffleMask }; 5428 5429 // Return shuffle(LoValVec, HiValVec, <0,1,0,1>) 5430 return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0), Ops, 3); 5431 } 5432 } 5433 5434 // Otherwise, we can't handle this case efficiently. Allocate a sufficiently 5435 // aligned object on the stack, store each element into it, then load 5436 // the result as a vector. 5437 MVT VT = Node->getValueType(0); 5438 // Create the stack frame object. 5439 SDValue FIPtr = DAG.CreateStackTemporary(VT); 5440 5441 // Emit a store of each element to the stack slot. 5442 SmallVector<SDValue, 8> Stores; 5443 unsigned TypeByteSize = Node->getOperand(0).getValueType().getSizeInBits()/8; 5444 // Store (in the right endianness) the elements to memory. 5445 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 5446 // Ignore undef elements. 5447 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; 5448 5449 unsigned Offset = TypeByteSize*i; 5450 5451 SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); 5452 Idx = DAG.getNode(ISD::ADD, FIPtr.getValueType(), FIPtr, Idx); 5453 5454 Stores.push_back(DAG.getStore(DAG.getEntryNode(), Node->getOperand(i), Idx, 5455 NULL, 0)); 5456 } 5457 5458 SDValue StoreChain; 5459 if (!Stores.empty()) // Not all undef elements? 5460 StoreChain = DAG.getNode(ISD::TokenFactor, MVT::Other, 5461 &Stores[0], Stores.size()); 5462 else 5463 StoreChain = DAG.getEntryNode(); 5464 5465 // Result is a load from the stack slot. 5466 return DAG.getLoad(VT, StoreChain, FIPtr, NULL, 0); 5467} 5468 5469void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp, 5470 SDValue Op, SDValue Amt, 5471 SDValue &Lo, SDValue &Hi) { 5472 // Expand the subcomponents. 5473 SDValue LHSL, LHSH; 5474 ExpandOp(Op, LHSL, LHSH); 5475 5476 SDValue Ops[] = { LHSL, LHSH, Amt }; 5477 MVT VT = LHSL.getValueType(); 5478 Lo = DAG.getNode(NodeOp, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3); 5479 Hi = Lo.getValue(1); 5480} 5481 5482 5483/// ExpandShift - Try to find a clever way to expand this shift operation out to 5484/// smaller elements. If we can't find a way that is more efficient than a 5485/// libcall on this target, return false. Otherwise, return true with the 5486/// low-parts expanded into Lo and Hi. 5487bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDValue Op,SDValue Amt, 5488 SDValue &Lo, SDValue &Hi) { 5489 assert((Opc == ISD::SHL || Opc == ISD::SRA || Opc == ISD::SRL) && 5490 "This is not a shift!"); 5491 5492 MVT NVT = TLI.getTypeToTransformTo(Op.getValueType()); 5493 SDValue ShAmt = LegalizeOp(Amt); 5494 MVT ShTy = ShAmt.getValueType(); 5495 unsigned ShBits = ShTy.getSizeInBits(); 5496 unsigned VTBits = Op.getValueType().getSizeInBits(); 5497 unsigned NVTBits = NVT.getSizeInBits(); 5498 5499 // Handle the case when Amt is an immediate. 5500 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Amt.getNode())) { 5501 unsigned Cst = CN->getZExtValue(); 5502 // Expand the incoming operand to be shifted, so that we have its parts 5503 SDValue InL, InH; 5504 ExpandOp(Op, InL, InH); 5505 switch(Opc) { 5506 case ISD::SHL: 5507 if (Cst > VTBits) { 5508 Lo = DAG.getConstant(0, NVT); 5509 Hi = DAG.getConstant(0, NVT); 5510 } else if (Cst > NVTBits) { 5511 Lo = DAG.getConstant(0, NVT); 5512 Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Cst-NVTBits,ShTy)); 5513 } else if (Cst == NVTBits) { 5514 Lo = DAG.getConstant(0, NVT); 5515 Hi = InL; 5516 } else { 5517 Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Cst, ShTy)); 5518 Hi = DAG.getNode(ISD::OR, NVT, 5519 DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(Cst, ShTy)), 5520 DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(NVTBits-Cst, ShTy))); 5521 } 5522 return true; 5523 case ISD::SRL: 5524 if (Cst > VTBits) { 5525 Lo = DAG.getConstant(0, NVT); 5526 Hi = DAG.getConstant(0, NVT); 5527 } else if (Cst > NVTBits) { 5528 Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Cst-NVTBits,ShTy)); 5529 Hi = DAG.getConstant(0, NVT); 5530 } else if (Cst == NVTBits) { 5531 Lo = InH; 5532 Hi = DAG.getConstant(0, NVT); 5533 } else { 5534 Lo = DAG.getNode(ISD::OR, NVT, 5535 DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(Cst, ShTy)), 5536 DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy))); 5537 Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Cst, ShTy)); 5538 } 5539 return true; 5540 case ISD::SRA: 5541 if (Cst > VTBits) { 5542 Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH, 5543 DAG.getConstant(NVTBits-1, ShTy)); 5544 } else if (Cst > NVTBits) { 5545 Lo = DAG.getNode(ISD::SRA, NVT, InH, 5546 DAG.getConstant(Cst-NVTBits, ShTy)); 5547 Hi = DAG.getNode(ISD::SRA, NVT, InH, 5548 DAG.getConstant(NVTBits-1, ShTy)); 5549 } else if (Cst == NVTBits) { 5550 Lo = InH; 5551 Hi = DAG.getNode(ISD::SRA, NVT, InH, 5552 DAG.getConstant(NVTBits-1, ShTy)); 5553 } else { 5554 Lo = DAG.getNode(ISD::OR, NVT, 5555 DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(Cst, ShTy)), 5556 DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy))); 5557 Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Cst, ShTy)); 5558 } 5559 return true; 5560 } 5561 } 5562 5563 // Okay, the shift amount isn't constant. However, if we can tell that it is 5564 // >= 32 or < 32, we can still simplify it, without knowing the actual value. 5565 APInt Mask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 5566 APInt KnownZero, KnownOne; 5567 DAG.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne); 5568 5569 // If we know that if any of the high bits of the shift amount are one, then 5570 // we can do this as a couple of simple shifts. 5571 if (KnownOne.intersects(Mask)) { 5572 // Mask out the high bit, which we know is set. 5573 Amt = DAG.getNode(ISD::AND, Amt.getValueType(), Amt, 5574 DAG.getConstant(~Mask, Amt.getValueType())); 5575 5576 // Expand the incoming operand to be shifted, so that we have its parts 5577 SDValue InL, InH; 5578 ExpandOp(Op, InL, InH); 5579 switch(Opc) { 5580 case ISD::SHL: 5581 Lo = DAG.getConstant(0, NVT); // Low part is zero. 5582 Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part. 5583 return true; 5584 case ISD::SRL: 5585 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 5586 Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part. 5587 return true; 5588 case ISD::SRA: 5589 Hi = DAG.getNode(ISD::SRA, NVT, InH, // Sign extend high part. 5590 DAG.getConstant(NVTBits-1, Amt.getValueType())); 5591 Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part. 5592 return true; 5593 } 5594 } 5595 5596 // If we know that the high bits of the shift amount are all zero, then we can 5597 // do this as a couple of simple shifts. 5598 if ((KnownZero & Mask) == Mask) { 5599 // Compute 32-amt. 5600 SDValue Amt2 = DAG.getNode(ISD::SUB, Amt.getValueType(), 5601 DAG.getConstant(NVTBits, Amt.getValueType()), 5602 Amt); 5603 5604 // Expand the incoming operand to be shifted, so that we have its parts 5605 SDValue InL, InH; 5606 ExpandOp(Op, InL, InH); 5607 switch(Opc) { 5608 case ISD::SHL: 5609 Lo = DAG.getNode(ISD::SHL, NVT, InL, Amt); 5610 Hi = DAG.getNode(ISD::OR, NVT, 5611 DAG.getNode(ISD::SHL, NVT, InH, Amt), 5612 DAG.getNode(ISD::SRL, NVT, InL, Amt2)); 5613 return true; 5614 case ISD::SRL: 5615 Hi = DAG.getNode(ISD::SRL, NVT, InH, Amt); 5616 Lo = DAG.getNode(ISD::OR, NVT, 5617 DAG.getNode(ISD::SRL, NVT, InL, Amt), 5618 DAG.getNode(ISD::SHL, NVT, InH, Amt2)); 5619 return true; 5620 case ISD::SRA: 5621 Hi = DAG.getNode(ISD::SRA, NVT, InH, Amt); 5622 Lo = DAG.getNode(ISD::OR, NVT, 5623 DAG.getNode(ISD::SRL, NVT, InL, Amt), 5624 DAG.getNode(ISD::SHL, NVT, InH, Amt2)); 5625 return true; 5626 } 5627 } 5628 5629 return false; 5630} 5631 5632 5633// ExpandLibCall - Expand a node into a call to a libcall. If the result value 5634// does not fit into a register, return the lo part and set the hi part to the 5635// by-reg argument. If it does fit into a single register, return the result 5636// and leave the Hi part unset. 5637SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 5638 bool isSigned, SDValue &Hi) { 5639 assert(!IsLegalizingCall && "Cannot overlap legalization of calls!"); 5640 // The input chain to this libcall is the entry node of the function. 5641 // Legalizing the call will automatically add the previous call to the 5642 // dependence. 5643 SDValue InChain = DAG.getEntryNode(); 5644 5645 TargetLowering::ArgListTy Args; 5646 TargetLowering::ArgListEntry Entry; 5647 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 5648 MVT ArgVT = Node->getOperand(i).getValueType(); 5649 const Type *ArgTy = ArgVT.getTypeForMVT(); 5650 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 5651 Entry.isSExt = isSigned; 5652 Entry.isZExt = !isSigned; 5653 Args.push_back(Entry); 5654 } 5655 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 5656 TLI.getPointerTy()); 5657 5658 // Splice the libcall in wherever FindInputOutputChains tells us to. 5659 const Type *RetTy = Node->getValueType(0).getTypeForMVT(); 5660 std::pair<SDValue,SDValue> CallInfo = 5661 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, 5662 CallingConv::C, false, Callee, Args, DAG); 5663 5664 // Legalize the call sequence, starting with the chain. This will advance 5665 // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that 5666 // was added by LowerCallTo (guaranteeing proper serialization of calls). 5667 LegalizeOp(CallInfo.second); 5668 SDValue Result; 5669 switch (getTypeAction(CallInfo.first.getValueType())) { 5670 default: assert(0 && "Unknown thing"); 5671 case Legal: 5672 Result = CallInfo.first; 5673 break; 5674 case Expand: 5675 ExpandOp(CallInfo.first, Result, Hi); 5676 break; 5677 } 5678 return Result; 5679} 5680 5681/// LegalizeINT_TO_FP - Legalize a [US]INT_TO_FP operation. 5682/// 5683SDValue SelectionDAGLegalize:: 5684LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op) { 5685 bool isCustom = false; 5686 SDValue Tmp1; 5687 switch (getTypeAction(Op.getValueType())) { 5688 case Legal: 5689 switch (TLI.getOperationAction(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, 5690 Op.getValueType())) { 5691 default: assert(0 && "Unknown operation action!"); 5692 case TargetLowering::Custom: 5693 isCustom = true; 5694 // FALLTHROUGH 5695 case TargetLowering::Legal: 5696 Tmp1 = LegalizeOp(Op); 5697 if (Result.getNode()) 5698 Result = DAG.UpdateNodeOperands(Result, Tmp1); 5699 else 5700 Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, 5701 DestTy, Tmp1); 5702 if (isCustom) { 5703 Tmp1 = TLI.LowerOperation(Result, DAG); 5704 if (Tmp1.getNode()) Result = Tmp1; 5705 } 5706 break; 5707 case TargetLowering::Expand: 5708 Result = ExpandLegalINT_TO_FP(isSigned, LegalizeOp(Op), DestTy); 5709 break; 5710 case TargetLowering::Promote: 5711 Result = PromoteLegalINT_TO_FP(LegalizeOp(Op), DestTy, isSigned); 5712 break; 5713 } 5714 break; 5715 case Expand: 5716 Result = ExpandIntToFP(isSigned, DestTy, Op); 5717 break; 5718 case Promote: 5719 Tmp1 = PromoteOp(Op); 5720 if (isSigned) { 5721 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp1.getValueType(), 5722 Tmp1, DAG.getValueType(Op.getValueType())); 5723 } else { 5724 Tmp1 = DAG.getZeroExtendInReg(Tmp1, 5725 Op.getValueType()); 5726 } 5727 if (Result.getNode()) 5728 Result = DAG.UpdateNodeOperands(Result, Tmp1); 5729 else 5730 Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, 5731 DestTy, Tmp1); 5732 Result = LegalizeOp(Result); // The 'op' is not necessarily legal! 5733 break; 5734 } 5735 return Result; 5736} 5737 5738/// ExpandIntToFP - Expand a [US]INT_TO_FP operation. 5739/// 5740SDValue SelectionDAGLegalize:: 5741ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source) { 5742 MVT SourceVT = Source.getValueType(); 5743 bool ExpandSource = getTypeAction(SourceVT) == Expand; 5744 5745 // Expand unsupported int-to-fp vector casts by unrolling them. 5746 if (DestTy.isVector()) { 5747 if (!ExpandSource) 5748 return LegalizeOp(UnrollVectorOp(Source)); 5749 MVT DestEltTy = DestTy.getVectorElementType(); 5750 if (DestTy.getVectorNumElements() == 1) { 5751 SDValue Scalar = ScalarizeVectorOp(Source); 5752 SDValue Result = LegalizeINT_TO_FP(SDValue(), isSigned, 5753 DestEltTy, Scalar); 5754 return DAG.getNode(ISD::BUILD_VECTOR, DestTy, Result); 5755 } 5756 SDValue Lo, Hi; 5757 SplitVectorOp(Source, Lo, Hi); 5758 MVT SplitDestTy = MVT::getVectorVT(DestEltTy, 5759 DestTy.getVectorNumElements() / 2); 5760 SDValue LoResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Lo); 5761 SDValue HiResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Hi); 5762 return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, DestTy, LoResult, 5763 HiResult)); 5764 } 5765 5766 // Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc. 5767 if (!isSigned && SourceVT != MVT::i32) { 5768 // The integer value loaded will be incorrectly if the 'sign bit' of the 5769 // incoming integer is set. To handle this, we dynamically test to see if 5770 // it is set, and, if so, add a fudge factor. 5771 SDValue Hi; 5772 if (ExpandSource) { 5773 SDValue Lo; 5774 ExpandOp(Source, Lo, Hi); 5775 Source = DAG.getNode(ISD::BUILD_PAIR, SourceVT, Lo, Hi); 5776 } else { 5777 // The comparison for the sign bit will use the entire operand. 5778 Hi = Source; 5779 } 5780 5781 // Check to see if the target has a custom way to lower this. If so, use 5782 // it. (Note we've already expanded the operand in this case.) 5783 switch (TLI.getOperationAction(ISD::UINT_TO_FP, SourceVT)) { 5784 default: assert(0 && "This action not implemented for this operation!"); 5785 case TargetLowering::Legal: 5786 case TargetLowering::Expand: 5787 break; // This case is handled below. 5788 case TargetLowering::Custom: { 5789 SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::UINT_TO_FP, DestTy, 5790 Source), DAG); 5791 if (NV.getNode()) 5792 return LegalizeOp(NV); 5793 break; // The target decided this was legal after all 5794 } 5795 } 5796 5797 // If this is unsigned, and not supported, first perform the conversion to 5798 // signed, then adjust the result if the sign bit is set. 5799 SDValue SignedConv = ExpandIntToFP(true, DestTy, Source); 5800 5801 SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi, 5802 DAG.getConstant(0, Hi.getValueType()), 5803 ISD::SETLT); 5804 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 5805 SDValue CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(), 5806 SignSet, Four, Zero); 5807 uint64_t FF = 0x5f800000ULL; 5808 if (TLI.isLittleEndian()) FF <<= 32; 5809 static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); 5810 5811 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 5812 unsigned Alignment = 1 << cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 5813 CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset); 5814 Alignment = std::min(Alignment, 4u); 5815 SDValue FudgeInReg; 5816 if (DestTy == MVT::f32) 5817 FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, 5818 PseudoSourceValue::getConstantPool(), 0, 5819 false, Alignment); 5820 else if (DestTy.bitsGT(MVT::f32)) 5821 // FIXME: Avoid the extend by construction the right constantpool? 5822 FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(), 5823 CPIdx, 5824 PseudoSourceValue::getConstantPool(), 0, 5825 MVT::f32, false, Alignment); 5826 else 5827 assert(0 && "Unexpected conversion"); 5828 5829 MVT SCVT = SignedConv.getValueType(); 5830 if (SCVT != DestTy) { 5831 // Destination type needs to be expanded as well. The FADD now we are 5832 // constructing will be expanded into a libcall. 5833 if (SCVT.getSizeInBits() != DestTy.getSizeInBits()) { 5834 assert(SCVT.getSizeInBits() * 2 == DestTy.getSizeInBits()); 5835 SignedConv = DAG.getNode(ISD::BUILD_PAIR, DestTy, 5836 SignedConv, SignedConv.getValue(1)); 5837 } 5838 SignedConv = DAG.getNode(ISD::BIT_CONVERT, DestTy, SignedConv); 5839 } 5840 return DAG.getNode(ISD::FADD, DestTy, SignedConv, FudgeInReg); 5841 } 5842 5843 // Check to see if the target has a custom way to lower this. If so, use it. 5844 switch (TLI.getOperationAction(ISD::SINT_TO_FP, SourceVT)) { 5845 default: assert(0 && "This action not implemented for this operation!"); 5846 case TargetLowering::Legal: 5847 case TargetLowering::Expand: 5848 break; // This case is handled below. 5849 case TargetLowering::Custom: { 5850 SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy, 5851 Source), DAG); 5852 if (NV.getNode()) 5853 return LegalizeOp(NV); 5854 break; // The target decided this was legal after all 5855 } 5856 } 5857 5858 // Expand the source, then glue it back together for the call. We must expand 5859 // the source in case it is shared (this pass of legalize must traverse it). 5860 if (ExpandSource) { 5861 SDValue SrcLo, SrcHi; 5862 ExpandOp(Source, SrcLo, SrcHi); 5863 Source = DAG.getNode(ISD::BUILD_PAIR, SourceVT, SrcLo, SrcHi); 5864 } 5865 5866 RTLIB::Libcall LC = isSigned ? 5867 RTLIB::getSINTTOFP(SourceVT, DestTy) : 5868 RTLIB::getUINTTOFP(SourceVT, DestTy); 5869 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unknown int value type"); 5870 5871 Source = DAG.getNode(ISD::SINT_TO_FP, DestTy, Source); 5872 SDValue HiPart; 5873 SDValue Result = ExpandLibCall(LC, Source.getNode(), isSigned, HiPart); 5874 if (Result.getValueType() != DestTy && HiPart.getNode()) 5875 Result = DAG.getNode(ISD::BUILD_PAIR, DestTy, Result, HiPart); 5876 return Result; 5877} 5878 5879/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a 5880/// INT_TO_FP operation of the specified operand when the target requests that 5881/// we expand it. At this point, we know that the result and operand types are 5882/// legal for the target. 5883SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, 5884 SDValue Op0, 5885 MVT DestVT) { 5886 if (Op0.getValueType() == MVT::i32) { 5887 // simple 32-bit [signed|unsigned] integer to float/double expansion 5888 5889 // Get the stack frame index of a 8 byte buffer. 5890 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 5891 5892 // word offset constant for Hi/Lo address computation 5893 SDValue WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy()); 5894 // set up Hi and Lo (into buffer) address based on endian 5895 SDValue Hi = StackSlot; 5896 SDValue Lo = DAG.getNode(ISD::ADD, TLI.getPointerTy(), StackSlot,WordOff); 5897 if (TLI.isLittleEndian()) 5898 std::swap(Hi, Lo); 5899 5900 // if signed map to unsigned space 5901 SDValue Op0Mapped; 5902 if (isSigned) { 5903 // constant used to invert sign bit (signed to unsigned mapping) 5904 SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); 5905 Op0Mapped = DAG.getNode(ISD::XOR, MVT::i32, Op0, SignBit); 5906 } else { 5907 Op0Mapped = Op0; 5908 } 5909 // store the lo of the constructed double - based on integer input 5910 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), 5911 Op0Mapped, Lo, NULL, 0); 5912 // initial hi portion of constructed double 5913 SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); 5914 // store the hi of the constructed double - biased exponent 5915 SDValue Store2=DAG.getStore(Store1, InitialHi, Hi, NULL, 0); 5916 // load the constructed double 5917 SDValue Load = DAG.getLoad(MVT::f64, Store2, StackSlot, NULL, 0); 5918 // FP constant to bias correct the final result 5919 SDValue Bias = DAG.getConstantFP(isSigned ? 5920 BitsToDouble(0x4330000080000000ULL) 5921 : BitsToDouble(0x4330000000000000ULL), 5922 MVT::f64); 5923 // subtract the bias 5924 SDValue Sub = DAG.getNode(ISD::FSUB, MVT::f64, Load, Bias); 5925 // final result 5926 SDValue Result; 5927 // handle final rounding 5928 if (DestVT == MVT::f64) { 5929 // do nothing 5930 Result = Sub; 5931 } else if (DestVT.bitsLT(MVT::f64)) { 5932 Result = DAG.getNode(ISD::FP_ROUND, DestVT, Sub, 5933 DAG.getIntPtrConstant(0)); 5934 } else if (DestVT.bitsGT(MVT::f64)) { 5935 Result = DAG.getNode(ISD::FP_EXTEND, DestVT, Sub); 5936 } 5937 return Result; 5938 } 5939 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 5940 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, DestVT, Op0); 5941 5942 SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Op0), Op0, 5943 DAG.getConstant(0, Op0.getValueType()), 5944 ISD::SETLT); 5945 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 5946 SDValue CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(), 5947 SignSet, Four, Zero); 5948 5949 // If the sign bit of the integer is set, the large number will be treated 5950 // as a negative number. To counteract this, the dynamic code adds an 5951 // offset depending on the data type. 5952 uint64_t FF; 5953 switch (Op0.getValueType().getSimpleVT()) { 5954 default: assert(0 && "Unsupported integer type!"); 5955 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 5956 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 5957 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 5958 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 5959 } 5960 if (TLI.isLittleEndian()) FF <<= 32; 5961 static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); 5962 5963 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 5964 unsigned Alignment = 1 << cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 5965 CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset); 5966 Alignment = std::min(Alignment, 4u); 5967 SDValue FudgeInReg; 5968 if (DestVT == MVT::f32) 5969 FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, 5970 PseudoSourceValue::getConstantPool(), 0, 5971 false, Alignment); 5972 else { 5973 FudgeInReg = 5974 LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, DestVT, 5975 DAG.getEntryNode(), CPIdx, 5976 PseudoSourceValue::getConstantPool(), 0, 5977 MVT::f32, false, Alignment)); 5978 } 5979 5980 return DAG.getNode(ISD::FADD, DestVT, Tmp1, FudgeInReg); 5981} 5982 5983/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a 5984/// *INT_TO_FP operation of the specified operand when the target requests that 5985/// we promote it. At this point, we know that the result and operand types are 5986/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 5987/// operation that takes a larger input. 5988SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, 5989 MVT DestVT, 5990 bool isSigned) { 5991 // First step, figure out the appropriate *INT_TO_FP operation to use. 5992 MVT NewInTy = LegalOp.getValueType(); 5993 5994 unsigned OpToUse = 0; 5995 5996 // Scan for the appropriate larger type to use. 5997 while (1) { 5998 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT()+1); 5999 assert(NewInTy.isInteger() && "Ran out of possibilities!"); 6000 6001 // If the target supports SINT_TO_FP of this type, use it. 6002 switch (TLI.getOperationAction(ISD::SINT_TO_FP, NewInTy)) { 6003 default: break; 6004 case TargetLowering::Legal: 6005 if (!TLI.isTypeLegal(NewInTy)) 6006 break; // Can't use this datatype. 6007 // FALL THROUGH. 6008 case TargetLowering::Custom: 6009 OpToUse = ISD::SINT_TO_FP; 6010 break; 6011 } 6012 if (OpToUse) break; 6013 if (isSigned) continue; 6014 6015 // If the target supports UINT_TO_FP of this type, use it. 6016 switch (TLI.getOperationAction(ISD::UINT_TO_FP, NewInTy)) { 6017 default: break; 6018 case TargetLowering::Legal: 6019 if (!TLI.isTypeLegal(NewInTy)) 6020 break; // Can't use this datatype. 6021 // FALL THROUGH. 6022 case TargetLowering::Custom: 6023 OpToUse = ISD::UINT_TO_FP; 6024 break; 6025 } 6026 if (OpToUse) break; 6027 6028 // Otherwise, try a larger type. 6029 } 6030 6031 // Okay, we found the operation and type to use. Zero extend our input to the 6032 // desired type then run the operation on it. 6033 return DAG.getNode(OpToUse, DestVT, 6034 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 6035 NewInTy, LegalOp)); 6036} 6037 6038/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a 6039/// FP_TO_*INT operation of the specified operand when the target requests that 6040/// we promote it. At this point, we know that the result and operand types are 6041/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 6042/// operation that returns a larger result. 6043SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, 6044 MVT DestVT, 6045 bool isSigned) { 6046 // First step, figure out the appropriate FP_TO*INT operation to use. 6047 MVT NewOutTy = DestVT; 6048 6049 unsigned OpToUse = 0; 6050 6051 // Scan for the appropriate larger type to use. 6052 while (1) { 6053 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT()+1); 6054 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 6055 6056 // If the target supports FP_TO_SINT returning this type, use it. 6057 switch (TLI.getOperationAction(ISD::FP_TO_SINT, NewOutTy)) { 6058 default: break; 6059 case TargetLowering::Legal: 6060 if (!TLI.isTypeLegal(NewOutTy)) 6061 break; // Can't use this datatype. 6062 // FALL THROUGH. 6063 case TargetLowering::Custom: 6064 OpToUse = ISD::FP_TO_SINT; 6065 break; 6066 } 6067 if (OpToUse) break; 6068 6069 // If the target supports FP_TO_UINT of this type, use it. 6070 switch (TLI.getOperationAction(ISD::FP_TO_UINT, NewOutTy)) { 6071 default: break; 6072 case TargetLowering::Legal: 6073 if (!TLI.isTypeLegal(NewOutTy)) 6074 break; // Can't use this datatype. 6075 // FALL THROUGH. 6076 case TargetLowering::Custom: 6077 OpToUse = ISD::FP_TO_UINT; 6078 break; 6079 } 6080 if (OpToUse) break; 6081 6082 // Otherwise, try a larger type. 6083 } 6084 6085 6086 // Okay, we found the operation and type to use. 6087 SDValue Operation = DAG.getNode(OpToUse, NewOutTy, LegalOp); 6088 6089 // If the operation produces an invalid type, it must be custom lowered. Use 6090 // the target lowering hooks to expand it. Just keep the low part of the 6091 // expanded operation, we know that we're truncating anyway. 6092 if (getTypeAction(NewOutTy) == Expand) { 6093 SmallVector<SDValue, 2> Results; 6094 TLI.ReplaceNodeResults(Operation.getNode(), Results, DAG); 6095 assert(Results.size() == 1 && "Incorrect FP_TO_XINT lowering!"); 6096 Operation = Results[0]; 6097 } 6098 6099 // Truncate the result of the extended FP_TO_*INT operation to the desired 6100 // size. 6101 return DAG.getNode(ISD::TRUNCATE, DestVT, Operation); 6102} 6103 6104/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. 6105/// 6106SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op) { 6107 MVT VT = Op.getValueType(); 6108 MVT SHVT = TLI.getShiftAmountTy(); 6109 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 6110 switch (VT.getSimpleVT()) { 6111 default: assert(0 && "Unhandled Expand type in BSWAP!"); abort(); 6112 case MVT::i16: 6113 Tmp2 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT)); 6114 Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT)); 6115 return DAG.getNode(ISD::OR, VT, Tmp1, Tmp2); 6116 case MVT::i32: 6117 Tmp4 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(24, SHVT)); 6118 Tmp3 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT)); 6119 Tmp2 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT)); 6120 Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(24, SHVT)); 6121 Tmp3 = DAG.getNode(ISD::AND, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); 6122 Tmp2 = DAG.getNode(ISD::AND, VT, Tmp2, DAG.getConstant(0xFF00, VT)); 6123 Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp3); 6124 Tmp2 = DAG.getNode(ISD::OR, VT, Tmp2, Tmp1); 6125 return DAG.getNode(ISD::OR, VT, Tmp4, Tmp2); 6126 case MVT::i64: 6127 Tmp8 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(56, SHVT)); 6128 Tmp7 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(40, SHVT)); 6129 Tmp6 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(24, SHVT)); 6130 Tmp5 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT)); 6131 Tmp4 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT)); 6132 Tmp3 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(24, SHVT)); 6133 Tmp2 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(40, SHVT)); 6134 Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(56, SHVT)); 6135 Tmp7 = DAG.getNode(ISD::AND, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); 6136 Tmp6 = DAG.getNode(ISD::AND, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); 6137 Tmp5 = DAG.getNode(ISD::AND, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); 6138 Tmp4 = DAG.getNode(ISD::AND, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); 6139 Tmp3 = DAG.getNode(ISD::AND, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); 6140 Tmp2 = DAG.getNode(ISD::AND, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); 6141 Tmp8 = DAG.getNode(ISD::OR, VT, Tmp8, Tmp7); 6142 Tmp6 = DAG.getNode(ISD::OR, VT, Tmp6, Tmp5); 6143 Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp3); 6144 Tmp2 = DAG.getNode(ISD::OR, VT, Tmp2, Tmp1); 6145 Tmp8 = DAG.getNode(ISD::OR, VT, Tmp8, Tmp6); 6146 Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp2); 6147 return DAG.getNode(ISD::OR, VT, Tmp8, Tmp4); 6148 } 6149} 6150 6151/// ExpandBitCount - Expand the specified bitcount instruction into operations. 6152/// 6153SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op) { 6154 switch (Opc) { 6155 default: assert(0 && "Cannot expand this yet!"); 6156 case ISD::CTPOP: { 6157 static const uint64_t mask[6] = { 6158 0x5555555555555555ULL, 0x3333333333333333ULL, 6159 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL, 6160 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL 6161 }; 6162 MVT VT = Op.getValueType(); 6163 MVT ShVT = TLI.getShiftAmountTy(); 6164 unsigned len = VT.getSizeInBits(); 6165 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 6166 //x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8]) 6167 SDValue Tmp2 = DAG.getConstant(mask[i], VT); 6168 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 6169 Op = DAG.getNode(ISD::ADD, VT, DAG.getNode(ISD::AND, VT, Op, Tmp2), 6170 DAG.getNode(ISD::AND, VT, 6171 DAG.getNode(ISD::SRL, VT, Op, Tmp3),Tmp2)); 6172 } 6173 return Op; 6174 } 6175 case ISD::CTLZ: { 6176 // for now, we do this: 6177 // x = x | (x >> 1); 6178 // x = x | (x >> 2); 6179 // ... 6180 // x = x | (x >>16); 6181 // x = x | (x >>32); // for 64-bit input 6182 // return popcount(~x); 6183 // 6184 // but see also: http://www.hackersdelight.org/HDcode/nlz.cc 6185 MVT VT = Op.getValueType(); 6186 MVT ShVT = TLI.getShiftAmountTy(); 6187 unsigned len = VT.getSizeInBits(); 6188 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 6189 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 6190 Op = DAG.getNode(ISD::OR, VT, Op, DAG.getNode(ISD::SRL, VT, Op, Tmp3)); 6191 } 6192 Op = DAG.getNode(ISD::XOR, VT, Op, DAG.getConstant(~0ULL, VT)); 6193 return DAG.getNode(ISD::CTPOP, VT, Op); 6194 } 6195 case ISD::CTTZ: { 6196 // for now, we use: { return popcount(~x & (x - 1)); } 6197 // unless the target has ctlz but not ctpop, in which case we use: 6198 // { return 32 - nlz(~x & (x-1)); } 6199 // see also http://www.hackersdelight.org/HDcode/ntz.cc 6200 MVT VT = Op.getValueType(); 6201 SDValue Tmp2 = DAG.getConstant(~0ULL, VT); 6202 SDValue Tmp3 = DAG.getNode(ISD::AND, VT, 6203 DAG.getNode(ISD::XOR, VT, Op, Tmp2), 6204 DAG.getNode(ISD::SUB, VT, Op, DAG.getConstant(1, VT))); 6205 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. 6206 if (!TLI.isOperationLegal(ISD::CTPOP, VT) && 6207 TLI.isOperationLegal(ISD::CTLZ, VT)) 6208 return DAG.getNode(ISD::SUB, VT, 6209 DAG.getConstant(VT.getSizeInBits(), VT), 6210 DAG.getNode(ISD::CTLZ, VT, Tmp3)); 6211 return DAG.getNode(ISD::CTPOP, VT, Tmp3); 6212 } 6213 } 6214} 6215 6216/// ExpandOp - Expand the specified SDValue into its two component pieces 6217/// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, the 6218/// LegalizedNodes map is filled in for any results that are not expanded, the 6219/// ExpandedNodes map is filled in for any results that are expanded, and the 6220/// Lo/Hi values are returned. 6221void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ 6222 MVT VT = Op.getValueType(); 6223 MVT NVT = TLI.getTypeToTransformTo(VT); 6224 SDNode *Node = Op.getNode(); 6225 assert(getTypeAction(VT) == Expand && "Not an expanded type!"); 6226 assert(((NVT.isInteger() && NVT.bitsLT(VT)) || VT.isFloatingPoint() || 6227 VT.isVector()) && "Cannot expand to FP value or to larger int value!"); 6228 6229 // See if we already expanded it. 6230 DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator I 6231 = ExpandedNodes.find(Op); 6232 if (I != ExpandedNodes.end()) { 6233 Lo = I->second.first; 6234 Hi = I->second.second; 6235 return; 6236 } 6237 6238 switch (Node->getOpcode()) { 6239 case ISD::CopyFromReg: 6240 assert(0 && "CopyFromReg must be legal!"); 6241 case ISD::FP_ROUND_INREG: 6242 if (VT == MVT::ppcf128 && 6243 TLI.getOperationAction(ISD::FP_ROUND_INREG, VT) == 6244 TargetLowering::Custom) { 6245 SDValue SrcLo, SrcHi, Src; 6246 ExpandOp(Op.getOperand(0), SrcLo, SrcHi); 6247 Src = DAG.getNode(ISD::BUILD_PAIR, VT, SrcLo, SrcHi); 6248 SDValue Result = TLI.LowerOperation( 6249 DAG.getNode(ISD::FP_ROUND_INREG, VT, Src, Op.getOperand(1)), DAG); 6250 assert(Result.getNode()->getOpcode() == ISD::BUILD_PAIR); 6251 Lo = Result.getNode()->getOperand(0); 6252 Hi = Result.getNode()->getOperand(1); 6253 break; 6254 } 6255 // fall through 6256 default: 6257#ifndef NDEBUG 6258 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 6259#endif 6260 assert(0 && "Do not know how to expand this operator!"); 6261 abort(); 6262 case ISD::EXTRACT_ELEMENT: 6263 ExpandOp(Node->getOperand(0), Lo, Hi); 6264 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) 6265 return ExpandOp(Hi, Lo, Hi); 6266 return ExpandOp(Lo, Lo, Hi); 6267 case ISD::EXTRACT_VECTOR_ELT: 6268 // ExpandEXTRACT_VECTOR_ELT tolerates invalid result types. 6269 Lo = ExpandEXTRACT_VECTOR_ELT(Op); 6270 return ExpandOp(Lo, Lo, Hi); 6271 case ISD::UNDEF: 6272 Lo = DAG.getNode(ISD::UNDEF, NVT); 6273 Hi = DAG.getNode(ISD::UNDEF, NVT); 6274 break; 6275 case ISD::Constant: { 6276 unsigned NVTBits = NVT.getSizeInBits(); 6277 const APInt &Cst = cast<ConstantSDNode>(Node)->getAPIntValue(); 6278 Lo = DAG.getConstant(APInt(Cst).trunc(NVTBits), NVT); 6279 Hi = DAG.getConstant(Cst.lshr(NVTBits).trunc(NVTBits), NVT); 6280 break; 6281 } 6282 case ISD::ConstantFP: { 6283 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 6284 if (CFP->getValueType(0) == MVT::ppcf128) { 6285 APInt api = CFP->getValueAPF().bitcastToAPInt(); 6286 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])), 6287 MVT::f64); 6288 Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])), 6289 MVT::f64); 6290 break; 6291 } 6292 Lo = ExpandConstantFP(CFP, false, DAG, TLI); 6293 if (getTypeAction(Lo.getValueType()) == Expand) 6294 ExpandOp(Lo, Lo, Hi); 6295 break; 6296 } 6297 case ISD::BUILD_PAIR: 6298 // Return the operands. 6299 Lo = Node->getOperand(0); 6300 Hi = Node->getOperand(1); 6301 break; 6302 6303 case ISD::MERGE_VALUES: 6304 if (Node->getNumValues() == 1) { 6305 ExpandOp(Op.getOperand(0), Lo, Hi); 6306 break; 6307 } 6308 // FIXME: For now only expand i64,chain = MERGE_VALUES (x, y) 6309 assert(Op.getResNo() == 0 && Node->getNumValues() == 2 && 6310 Op.getValue(1).getValueType() == MVT::Other && 6311 "unhandled MERGE_VALUES"); 6312 ExpandOp(Op.getOperand(0), Lo, Hi); 6313 // Remember that we legalized the chain. 6314 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Op.getOperand(1))); 6315 break; 6316 6317 case ISD::SIGN_EXTEND_INREG: 6318 ExpandOp(Node->getOperand(0), Lo, Hi); 6319 // sext_inreg the low part if needed. 6320 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Lo, Node->getOperand(1)); 6321 6322 // The high part gets the sign extension from the lo-part. This handles 6323 // things like sextinreg V:i64 from i8. 6324 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 6325 DAG.getConstant(NVT.getSizeInBits()-1, 6326 TLI.getShiftAmountTy())); 6327 break; 6328 6329 case ISD::BSWAP: { 6330 ExpandOp(Node->getOperand(0), Lo, Hi); 6331 SDValue TempLo = DAG.getNode(ISD::BSWAP, NVT, Hi); 6332 Hi = DAG.getNode(ISD::BSWAP, NVT, Lo); 6333 Lo = TempLo; 6334 break; 6335 } 6336 6337 case ISD::CTPOP: 6338 ExpandOp(Node->getOperand(0), Lo, Hi); 6339 Lo = DAG.getNode(ISD::ADD, NVT, // ctpop(HL) -> ctpop(H)+ctpop(L) 6340 DAG.getNode(ISD::CTPOP, NVT, Lo), 6341 DAG.getNode(ISD::CTPOP, NVT, Hi)); 6342 Hi = DAG.getConstant(0, NVT); 6343 break; 6344 6345 case ISD::CTLZ: { 6346 // ctlz (HL) -> ctlz(H) != 32 ? ctlz(H) : (ctlz(L)+32) 6347 ExpandOp(Node->getOperand(0), Lo, Hi); 6348 SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); 6349 SDValue HLZ = DAG.getNode(ISD::CTLZ, NVT, Hi); 6350 SDValue TopNotZero = DAG.getSetCC(TLI.getSetCCResultType(HLZ), HLZ, BitsC, 6351 ISD::SETNE); 6352 SDValue LowPart = DAG.getNode(ISD::CTLZ, NVT, Lo); 6353 LowPart = DAG.getNode(ISD::ADD, NVT, LowPart, BitsC); 6354 6355 Lo = DAG.getNode(ISD::SELECT, NVT, TopNotZero, HLZ, LowPart); 6356 Hi = DAG.getConstant(0, NVT); 6357 break; 6358 } 6359 6360 case ISD::CTTZ: { 6361 // cttz (HL) -> cttz(L) != 32 ? cttz(L) : (cttz(H)+32) 6362 ExpandOp(Node->getOperand(0), Lo, Hi); 6363 SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); 6364 SDValue LTZ = DAG.getNode(ISD::CTTZ, NVT, Lo); 6365 SDValue BotNotZero = DAG.getSetCC(TLI.getSetCCResultType(LTZ), LTZ, BitsC, 6366 ISD::SETNE); 6367 SDValue HiPart = DAG.getNode(ISD::CTTZ, NVT, Hi); 6368 HiPart = DAG.getNode(ISD::ADD, NVT, HiPart, BitsC); 6369 6370 Lo = DAG.getNode(ISD::SELECT, NVT, BotNotZero, LTZ, HiPart); 6371 Hi = DAG.getConstant(0, NVT); 6372 break; 6373 } 6374 6375 case ISD::VAARG: { 6376 SDValue Ch = Node->getOperand(0); // Legalize the chain. 6377 SDValue Ptr = Node->getOperand(1); // Legalize the pointer. 6378 Lo = DAG.getVAArg(NVT, Ch, Ptr, Node->getOperand(2)); 6379 Hi = DAG.getVAArg(NVT, Lo.getValue(1), Ptr, Node->getOperand(2)); 6380 6381 // Remember that we legalized the chain. 6382 Hi = LegalizeOp(Hi); 6383 AddLegalizedOperand(Op.getValue(1), Hi.getValue(1)); 6384 if (TLI.isBigEndian()) 6385 std::swap(Lo, Hi); 6386 break; 6387 } 6388 6389 case ISD::LOAD: { 6390 LoadSDNode *LD = cast<LoadSDNode>(Node); 6391 SDValue Ch = LD->getChain(); // Legalize the chain. 6392 SDValue Ptr = LD->getBasePtr(); // Legalize the pointer. 6393 ISD::LoadExtType ExtType = LD->getExtensionType(); 6394 const Value *SV = LD->getSrcValue(); 6395 int SVOffset = LD->getSrcValueOffset(); 6396 unsigned Alignment = LD->getAlignment(); 6397 bool isVolatile = LD->isVolatile(); 6398 6399 if (ExtType == ISD::NON_EXTLOAD) { 6400 Lo = DAG.getLoad(NVT, Ch, Ptr, SV, SVOffset, 6401 isVolatile, Alignment); 6402 if (VT == MVT::f32 || VT == MVT::f64) { 6403 // f32->i32 or f64->i64 one to one expansion. 6404 // Remember that we legalized the chain. 6405 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); 6406 // Recursively expand the new load. 6407 if (getTypeAction(NVT) == Expand) 6408 ExpandOp(Lo, Lo, Hi); 6409 break; 6410 } 6411 6412 // Increment the pointer to the other half. 6413 unsigned IncrementSize = Lo.getValueType().getSizeInBits()/8; 6414 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 6415 DAG.getIntPtrConstant(IncrementSize)); 6416 SVOffset += IncrementSize; 6417 Alignment = MinAlign(Alignment, IncrementSize); 6418 Hi = DAG.getLoad(NVT, Ch, Ptr, SV, SVOffset, 6419 isVolatile, Alignment); 6420 6421 // Build a factor node to remember that this load is independent of the 6422 // other one. 6423 SDValue TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 6424 Hi.getValue(1)); 6425 6426 // Remember that we legalized the chain. 6427 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); 6428 if (TLI.isBigEndian()) 6429 std::swap(Lo, Hi); 6430 } else { 6431 MVT EVT = LD->getMemoryVT(); 6432 6433 if ((VT == MVT::f64 && EVT == MVT::f32) || 6434 (VT == MVT::ppcf128 && (EVT==MVT::f64 || EVT==MVT::f32))) { 6435 // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND 6436 SDValue Load = DAG.getLoad(EVT, Ch, Ptr, SV, 6437 SVOffset, isVolatile, Alignment); 6438 // Remember that we legalized the chain. 6439 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Load.getValue(1))); 6440 ExpandOp(DAG.getNode(ISD::FP_EXTEND, VT, Load), Lo, Hi); 6441 break; 6442 } 6443 6444 if (EVT == NVT) 6445 Lo = DAG.getLoad(NVT, Ch, Ptr, SV, 6446 SVOffset, isVolatile, Alignment); 6447 else 6448 Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, SV, 6449 SVOffset, EVT, isVolatile, 6450 Alignment); 6451 6452 // Remember that we legalized the chain. 6453 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); 6454 6455 if (ExtType == ISD::SEXTLOAD) { 6456 // The high part is obtained by SRA'ing all but one of the bits of the 6457 // lo part. 6458 unsigned LoSize = Lo.getValueType().getSizeInBits(); 6459 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 6460 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 6461 } else if (ExtType == ISD::ZEXTLOAD) { 6462 // The high part is just a zero. 6463 Hi = DAG.getConstant(0, NVT); 6464 } else /* if (ExtType == ISD::EXTLOAD) */ { 6465 // The high part is undefined. 6466 Hi = DAG.getNode(ISD::UNDEF, NVT); 6467 } 6468 } 6469 break; 6470 } 6471 case ISD::AND: 6472 case ISD::OR: 6473 case ISD::XOR: { // Simple logical operators -> two trivial pieces. 6474 SDValue LL, LH, RL, RH; 6475 ExpandOp(Node->getOperand(0), LL, LH); 6476 ExpandOp(Node->getOperand(1), RL, RH); 6477 Lo = DAG.getNode(Node->getOpcode(), NVT, LL, RL); 6478 Hi = DAG.getNode(Node->getOpcode(), NVT, LH, RH); 6479 break; 6480 } 6481 case ISD::SELECT: { 6482 SDValue LL, LH, RL, RH; 6483 ExpandOp(Node->getOperand(1), LL, LH); 6484 ExpandOp(Node->getOperand(2), RL, RH); 6485 if (getTypeAction(NVT) == Expand) 6486 NVT = TLI.getTypeToExpandTo(NVT); 6487 Lo = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), LL, RL); 6488 if (VT != MVT::f32) 6489 Hi = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), LH, RH); 6490 break; 6491 } 6492 case ISD::SELECT_CC: { 6493 SDValue TL, TH, FL, FH; 6494 ExpandOp(Node->getOperand(2), TL, TH); 6495 ExpandOp(Node->getOperand(3), FL, FH); 6496 if (getTypeAction(NVT) == Expand) 6497 NVT = TLI.getTypeToExpandTo(NVT); 6498 Lo = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0), 6499 Node->getOperand(1), TL, FL, Node->getOperand(4)); 6500 if (VT != MVT::f32) 6501 Hi = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0), 6502 Node->getOperand(1), TH, FH, Node->getOperand(4)); 6503 break; 6504 } 6505 case ISD::ANY_EXTEND: 6506 // The low part is any extension of the input (which degenerates to a copy). 6507 Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Node->getOperand(0)); 6508 // The high part is undefined. 6509 Hi = DAG.getNode(ISD::UNDEF, NVT); 6510 break; 6511 case ISD::SIGN_EXTEND: { 6512 // The low part is just a sign extension of the input (which degenerates to 6513 // a copy). 6514 Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, Node->getOperand(0)); 6515 6516 // The high part is obtained by SRA'ing all but one of the bits of the lo 6517 // part. 6518 unsigned LoSize = Lo.getValueType().getSizeInBits(); 6519 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 6520 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 6521 break; 6522 } 6523 case ISD::ZERO_EXTEND: 6524 // The low part is just a zero extension of the input (which degenerates to 6525 // a copy). 6526 Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, Node->getOperand(0)); 6527 6528 // The high part is just a zero. 6529 Hi = DAG.getConstant(0, NVT); 6530 break; 6531 6532 case ISD::TRUNCATE: { 6533 // The input value must be larger than this value. Expand *it*. 6534 SDValue NewLo; 6535 ExpandOp(Node->getOperand(0), NewLo, Hi); 6536 6537 // The low part is now either the right size, or it is closer. If not the 6538 // right size, make an illegal truncate so we recursively expand it. 6539 if (NewLo.getValueType() != Node->getValueType(0)) 6540 NewLo = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), NewLo); 6541 ExpandOp(NewLo, Lo, Hi); 6542 break; 6543 } 6544 6545 case ISD::BIT_CONVERT: { 6546 SDValue Tmp; 6547 if (TLI.getOperationAction(ISD::BIT_CONVERT, VT) == TargetLowering::Custom){ 6548 // If the target wants to, allow it to lower this itself. 6549 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6550 case Expand: assert(0 && "cannot expand FP!"); 6551 case Legal: Tmp = LegalizeOp(Node->getOperand(0)); break; 6552 case Promote: Tmp = PromoteOp (Node->getOperand(0)); break; 6553 } 6554 Tmp = TLI.LowerOperation(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp), DAG); 6555 } 6556 6557 // f32 / f64 must be expanded to i32 / i64. 6558 if (VT == MVT::f32 || VT == MVT::f64) { 6559 Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0)); 6560 if (getTypeAction(NVT) == Expand) 6561 ExpandOp(Lo, Lo, Hi); 6562 break; 6563 } 6564 6565 // If source operand will be expanded to the same type as VT, i.e. 6566 // i64 <- f64, i32 <- f32, expand the source operand instead. 6567 MVT VT0 = Node->getOperand(0).getValueType(); 6568 if (getTypeAction(VT0) == Expand && TLI.getTypeToTransformTo(VT0) == VT) { 6569 ExpandOp(Node->getOperand(0), Lo, Hi); 6570 break; 6571 } 6572 6573 // Turn this into a load/store pair by default. 6574 if (Tmp.getNode() == 0) 6575 Tmp = EmitStackConvert(Node->getOperand(0), VT, VT); 6576 6577 ExpandOp(Tmp, Lo, Hi); 6578 break; 6579 } 6580 6581 case ISD::READCYCLECOUNTER: { 6582 assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) == 6583 TargetLowering::Custom && 6584 "Must custom expand ReadCycleCounter"); 6585 SDValue Tmp = TLI.LowerOperation(Op, DAG); 6586 assert(Tmp.getNode() && "Node must be custom expanded!"); 6587 ExpandOp(Tmp.getValue(0), Lo, Hi); 6588 AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. 6589 LegalizeOp(Tmp.getValue(1))); 6590 break; 6591 } 6592 6593 case ISD::ATOMIC_CMP_SWAP_64: { 6594 // This operation does not need a loop. 6595 SDValue Tmp = TLI.LowerOperation(Op, DAG); 6596 assert(Tmp.getNode() && "Node must be custom expanded!"); 6597 ExpandOp(Tmp.getValue(0), Lo, Hi); 6598 AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. 6599 LegalizeOp(Tmp.getValue(1))); 6600 break; 6601 } 6602 6603 case ISD::ATOMIC_LOAD_ADD_64: 6604 case ISD::ATOMIC_LOAD_SUB_64: 6605 case ISD::ATOMIC_LOAD_AND_64: 6606 case ISD::ATOMIC_LOAD_OR_64: 6607 case ISD::ATOMIC_LOAD_XOR_64: 6608 case ISD::ATOMIC_LOAD_NAND_64: 6609 case ISD::ATOMIC_SWAP_64: { 6610 // These operations require a loop to be generated. We can't do that yet, 6611 // so substitute a target-dependent pseudo and expand that later. 6612 SDValue In2Lo, In2Hi, In2; 6613 ExpandOp(Op.getOperand(2), In2Lo, In2Hi); 6614 In2 = DAG.getNode(ISD::BUILD_PAIR, VT, In2Lo, In2Hi); 6615 AtomicSDNode* Anode = cast<AtomicSDNode>(Node); 6616 SDValue Replace = 6617 DAG.getAtomic(Op.getOpcode(), Op.getOperand(0), Op.getOperand(1), In2, 6618 Anode->getSrcValue(), Anode->getAlignment()); 6619 SDValue Result = TLI.LowerOperation(Replace, DAG); 6620 ExpandOp(Result.getValue(0), Lo, Hi); 6621 // Remember that we legalized the chain. 6622 AddLegalizedOperand(SDValue(Node,1), LegalizeOp(Result.getValue(1))); 6623 break; 6624 } 6625 6626 // These operators cannot be expanded directly, emit them as calls to 6627 // library functions. 6628 case ISD::FP_TO_SINT: { 6629 if (TLI.getOperationAction(ISD::FP_TO_SINT, VT) == TargetLowering::Custom) { 6630 SDValue Op; 6631 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6632 case Expand: assert(0 && "cannot expand FP!"); 6633 case Legal: Op = LegalizeOp(Node->getOperand(0)); break; 6634 case Promote: Op = PromoteOp (Node->getOperand(0)); break; 6635 } 6636 6637 Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_SINT, VT, Op), DAG); 6638 6639 // Now that the custom expander is done, expand the result, which is still 6640 // VT. 6641 if (Op.getNode()) { 6642 ExpandOp(Op, Lo, Hi); 6643 break; 6644 } 6645 } 6646 6647 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Node->getOperand(0).getValueType(), 6648 VT); 6649 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected uint-to-fp conversion!"); 6650 Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); 6651 break; 6652 } 6653 6654 case ISD::FP_TO_UINT: { 6655 if (TLI.getOperationAction(ISD::FP_TO_UINT, VT) == TargetLowering::Custom) { 6656 SDValue Op; 6657 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6658 case Expand: assert(0 && "cannot expand FP!"); 6659 case Legal: Op = LegalizeOp(Node->getOperand(0)); break; 6660 case Promote: Op = PromoteOp (Node->getOperand(0)); break; 6661 } 6662 6663 Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_UINT, VT, Op), DAG); 6664 6665 // Now that the custom expander is done, expand the result. 6666 if (Op.getNode()) { 6667 ExpandOp(Op, Lo, Hi); 6668 break; 6669 } 6670 } 6671 6672 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Node->getOperand(0).getValueType(), 6673 VT); 6674 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 6675 Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); 6676 break; 6677 } 6678 6679 case ISD::SHL: { 6680 // If the target wants custom lowering, do so. 6681 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6682 if (TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Custom) { 6683 SDValue Op = DAG.getNode(ISD::SHL, VT, Node->getOperand(0), ShiftAmt); 6684 Op = TLI.LowerOperation(Op, DAG); 6685 if (Op.getNode()) { 6686 // Now that the custom expander is done, expand the result, which is 6687 // still VT. 6688 ExpandOp(Op, Lo, Hi); 6689 break; 6690 } 6691 } 6692 6693 // If ADDC/ADDE are supported and if the shift amount is a constant 1, emit 6694 // this X << 1 as X+X. 6695 if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(ShiftAmt)) { 6696 if (ShAmt->getAPIntValue() == 1 && TLI.isOperationLegal(ISD::ADDC, NVT) && 6697 TLI.isOperationLegal(ISD::ADDE, NVT)) { 6698 SDValue LoOps[2], HiOps[3]; 6699 ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]); 6700 SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag); 6701 LoOps[1] = LoOps[0]; 6702 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 6703 6704 HiOps[1] = HiOps[0]; 6705 HiOps[2] = Lo.getValue(1); 6706 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 6707 break; 6708 } 6709 } 6710 6711 // If we can emit an efficient shift operation, do so now. 6712 if (ExpandShift(ISD::SHL, Node->getOperand(0), ShiftAmt, Lo, Hi)) 6713 break; 6714 6715 // If this target supports SHL_PARTS, use it. 6716 TargetLowering::LegalizeAction Action = 6717 TLI.getOperationAction(ISD::SHL_PARTS, NVT); 6718 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 6719 Action == TargetLowering::Custom) { 6720 ExpandShiftParts(ISD::SHL_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi); 6721 break; 6722 } 6723 6724 // Otherwise, emit a libcall. 6725 Lo = ExpandLibCall(RTLIB::SHL_I64, Node, false/*left shift=unsigned*/, Hi); 6726 break; 6727 } 6728 6729 case ISD::SRA: { 6730 // If the target wants custom lowering, do so. 6731 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6732 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Custom) { 6733 SDValue Op = DAG.getNode(ISD::SRA, VT, Node->getOperand(0), ShiftAmt); 6734 Op = TLI.LowerOperation(Op, DAG); 6735 if (Op.getNode()) { 6736 // Now that the custom expander is done, expand the result, which is 6737 // still VT. 6738 ExpandOp(Op, Lo, Hi); 6739 break; 6740 } 6741 } 6742 6743 // If we can emit an efficient shift operation, do so now. 6744 if (ExpandShift(ISD::SRA, Node->getOperand(0), ShiftAmt, Lo, Hi)) 6745 break; 6746 6747 // If this target supports SRA_PARTS, use it. 6748 TargetLowering::LegalizeAction Action = 6749 TLI.getOperationAction(ISD::SRA_PARTS, NVT); 6750 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 6751 Action == TargetLowering::Custom) { 6752 ExpandShiftParts(ISD::SRA_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi); 6753 break; 6754 } 6755 6756 // Otherwise, emit a libcall. 6757 Lo = ExpandLibCall(RTLIB::SRA_I64, Node, true/*ashr is signed*/, Hi); 6758 break; 6759 } 6760 6761 case ISD::SRL: { 6762 // If the target wants custom lowering, do so. 6763 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6764 if (TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Custom) { 6765 SDValue Op = DAG.getNode(ISD::SRL, VT, Node->getOperand(0), ShiftAmt); 6766 Op = TLI.LowerOperation(Op, DAG); 6767 if (Op.getNode()) { 6768 // Now that the custom expander is done, expand the result, which is 6769 // still VT. 6770 ExpandOp(Op, Lo, Hi); 6771 break; 6772 } 6773 } 6774 6775 // If we can emit an efficient shift operation, do so now. 6776 if (ExpandShift(ISD::SRL, Node->getOperand(0), ShiftAmt, Lo, Hi)) 6777 break; 6778 6779 // If this target supports SRL_PARTS, use it. 6780 TargetLowering::LegalizeAction Action = 6781 TLI.getOperationAction(ISD::SRL_PARTS, NVT); 6782 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 6783 Action == TargetLowering::Custom) { 6784 ExpandShiftParts(ISD::SRL_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi); 6785 break; 6786 } 6787 6788 // Otherwise, emit a libcall. 6789 Lo = ExpandLibCall(RTLIB::SRL_I64, Node, false/*lshr is unsigned*/, Hi); 6790 break; 6791 } 6792 6793 case ISD::ADD: 6794 case ISD::SUB: { 6795 // If the target wants to custom expand this, let them. 6796 if (TLI.getOperationAction(Node->getOpcode(), VT) == 6797 TargetLowering::Custom) { 6798 SDValue Result = TLI.LowerOperation(Op, DAG); 6799 if (Result.getNode()) { 6800 ExpandOp(Result, Lo, Hi); 6801 break; 6802 } 6803 } 6804 // Expand the subcomponents. 6805 SDValue LHSL, LHSH, RHSL, RHSH; 6806 ExpandOp(Node->getOperand(0), LHSL, LHSH); 6807 ExpandOp(Node->getOperand(1), RHSL, RHSH); 6808 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 6809 SDValue LoOps[2], HiOps[3]; 6810 LoOps[0] = LHSL; 6811 LoOps[1] = RHSL; 6812 HiOps[0] = LHSH; 6813 HiOps[1] = RHSH; 6814 6815 //cascaded check to see if any smaller size has a a carry flag. 6816 unsigned OpV = Node->getOpcode() == ISD::ADD ? ISD::ADDC : ISD::SUBC; 6817 bool hasCarry = false; 6818 for (unsigned BitSize = NVT.getSizeInBits(); BitSize != 0; BitSize /= 2) { 6819 MVT AVT = MVT::getIntegerVT(BitSize); 6820 if (TLI.isOperationLegal(OpV, AVT)) { 6821 hasCarry = true; 6822 break; 6823 } 6824 } 6825 6826 if(hasCarry) { 6827 if (Node->getOpcode() == ISD::ADD) { 6828 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 6829 HiOps[2] = Lo.getValue(1); 6830 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 6831 } else { 6832 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); 6833 HiOps[2] = Lo.getValue(1); 6834 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); 6835 } 6836 break; 6837 } else { 6838 if (Node->getOpcode() == ISD::ADD) { 6839 Lo = DAG.getNode(ISD::ADD, VTList, LoOps, 2); 6840 Hi = DAG.getNode(ISD::ADD, VTList, HiOps, 2); 6841 SDValue Cmp1 = DAG.getSetCC(TLI.getSetCCResultType(Lo), 6842 Lo, LoOps[0], ISD::SETULT); 6843 SDValue Carry1 = DAG.getNode(ISD::SELECT, NVT, Cmp1, 6844 DAG.getConstant(1, NVT), 6845 DAG.getConstant(0, NVT)); 6846 SDValue Cmp2 = DAG.getSetCC(TLI.getSetCCResultType(Lo), 6847 Lo, LoOps[1], ISD::SETULT); 6848 SDValue Carry2 = DAG.getNode(ISD::SELECT, NVT, Cmp2, 6849 DAG.getConstant(1, NVT), 6850 Carry1); 6851 Hi = DAG.getNode(ISD::ADD, NVT, Hi, Carry2); 6852 } else { 6853 Lo = DAG.getNode(ISD::SUB, VTList, LoOps, 2); 6854 Hi = DAG.getNode(ISD::SUB, VTList, HiOps, 2); 6855 SDValue Cmp = DAG.getSetCC(NVT, LoOps[0], LoOps[1], ISD::SETULT); 6856 SDValue Borrow = DAG.getNode(ISD::SELECT, NVT, Cmp, 6857 DAG.getConstant(1, NVT), 6858 DAG.getConstant(0, NVT)); 6859 Hi = DAG.getNode(ISD::SUB, NVT, Hi, Borrow); 6860 } 6861 break; 6862 } 6863 } 6864 6865 case ISD::ADDC: 6866 case ISD::SUBC: { 6867 // Expand the subcomponents. 6868 SDValue LHSL, LHSH, RHSL, RHSH; 6869 ExpandOp(Node->getOperand(0), LHSL, LHSH); 6870 ExpandOp(Node->getOperand(1), RHSL, RHSH); 6871 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 6872 SDValue LoOps[2] = { LHSL, RHSL }; 6873 SDValue HiOps[3] = { LHSH, RHSH }; 6874 6875 if (Node->getOpcode() == ISD::ADDC) { 6876 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 6877 HiOps[2] = Lo.getValue(1); 6878 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 6879 } else { 6880 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); 6881 HiOps[2] = Lo.getValue(1); 6882 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); 6883 } 6884 // Remember that we legalized the flag. 6885 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1))); 6886 break; 6887 } 6888 case ISD::ADDE: 6889 case ISD::SUBE: { 6890 // Expand the subcomponents. 6891 SDValue LHSL, LHSH, RHSL, RHSH; 6892 ExpandOp(Node->getOperand(0), LHSL, LHSH); 6893 ExpandOp(Node->getOperand(1), RHSL, RHSH); 6894 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 6895 SDValue LoOps[3] = { LHSL, RHSL, Node->getOperand(2) }; 6896 SDValue HiOps[3] = { LHSH, RHSH }; 6897 6898 Lo = DAG.getNode(Node->getOpcode(), VTList, LoOps, 3); 6899 HiOps[2] = Lo.getValue(1); 6900 Hi = DAG.getNode(Node->getOpcode(), VTList, HiOps, 3); 6901 6902 // Remember that we legalized the flag. 6903 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1))); 6904 break; 6905 } 6906 case ISD::MUL: { 6907 // If the target wants to custom expand this, let them. 6908 if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) { 6909 SDValue New = TLI.LowerOperation(Op, DAG); 6910 if (New.getNode()) { 6911 ExpandOp(New, Lo, Hi); 6912 break; 6913 } 6914 } 6915 6916 bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT); 6917 bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT); 6918 bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT); 6919 bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT); 6920 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 6921 SDValue LL, LH, RL, RH; 6922 ExpandOp(Node->getOperand(0), LL, LH); 6923 ExpandOp(Node->getOperand(1), RL, RH); 6924 unsigned OuterBitSize = Op.getValueSizeInBits(); 6925 unsigned InnerBitSize = RH.getValueSizeInBits(); 6926 unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0)); 6927 unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1)); 6928 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 6929 if (DAG.MaskedValueIsZero(Node->getOperand(0), HighMask) && 6930 DAG.MaskedValueIsZero(Node->getOperand(1), HighMask)) { 6931 // The inputs are both zero-extended. 6932 if (HasUMUL_LOHI) { 6933 // We can emit a umul_lohi. 6934 Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); 6935 Hi = SDValue(Lo.getNode(), 1); 6936 break; 6937 } 6938 if (HasMULHU) { 6939 // We can emit a mulhu+mul. 6940 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 6941 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); 6942 break; 6943 } 6944 } 6945 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 6946 // The input values are both sign-extended. 6947 if (HasSMUL_LOHI) { 6948 // We can emit a smul_lohi. 6949 Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); 6950 Hi = SDValue(Lo.getNode(), 1); 6951 break; 6952 } 6953 if (HasMULHS) { 6954 // We can emit a mulhs+mul. 6955 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 6956 Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL); 6957 break; 6958 } 6959 } 6960 if (HasUMUL_LOHI) { 6961 // Lo,Hi = umul LHS, RHS. 6962 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, 6963 DAG.getVTList(NVT, NVT), LL, RL); 6964 Lo = UMulLOHI; 6965 Hi = UMulLOHI.getValue(1); 6966 RH = DAG.getNode(ISD::MUL, NVT, LL, RH); 6967 LH = DAG.getNode(ISD::MUL, NVT, LH, RL); 6968 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); 6969 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH); 6970 break; 6971 } 6972 if (HasMULHU) { 6973 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 6974 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); 6975 RH = DAG.getNode(ISD::MUL, NVT, LL, RH); 6976 LH = DAG.getNode(ISD::MUL, NVT, LH, RL); 6977 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); 6978 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH); 6979 break; 6980 } 6981 } 6982 6983 // If nothing else, we can make a libcall. 6984 Lo = ExpandLibCall(RTLIB::MUL_I64, Node, false/*sign irrelevant*/, Hi); 6985 break; 6986 } 6987 case ISD::SDIV: 6988 Lo = ExpandLibCall(RTLIB::SDIV_I64, Node, true, Hi); 6989 break; 6990 case ISD::UDIV: 6991 Lo = ExpandLibCall(RTLIB::UDIV_I64, Node, true, Hi); 6992 break; 6993 case ISD::SREM: 6994 Lo = ExpandLibCall(RTLIB::SREM_I64, Node, true, Hi); 6995 break; 6996 case ISD::UREM: 6997 Lo = ExpandLibCall(RTLIB::UREM_I64, Node, true, Hi); 6998 break; 6999 7000 case ISD::FADD: 7001 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::ADD_F32, 7002 RTLIB::ADD_F64, 7003 RTLIB::ADD_F80, 7004 RTLIB::ADD_PPCF128), 7005 Node, false, Hi); 7006 break; 7007 case ISD::FSUB: 7008 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::SUB_F32, 7009 RTLIB::SUB_F64, 7010 RTLIB::SUB_F80, 7011 RTLIB::SUB_PPCF128), 7012 Node, false, Hi); 7013 break; 7014 case ISD::FMUL: 7015 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::MUL_F32, 7016 RTLIB::MUL_F64, 7017 RTLIB::MUL_F80, 7018 RTLIB::MUL_PPCF128), 7019 Node, false, Hi); 7020 break; 7021 case ISD::FDIV: 7022 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::DIV_F32, 7023 RTLIB::DIV_F64, 7024 RTLIB::DIV_F80, 7025 RTLIB::DIV_PPCF128), 7026 Node, false, Hi); 7027 break; 7028 case ISD::FP_EXTEND: { 7029 if (VT == MVT::ppcf128) { 7030 assert(Node->getOperand(0).getValueType()==MVT::f32 || 7031 Node->getOperand(0).getValueType()==MVT::f64); 7032 const uint64_t zero = 0; 7033 if (Node->getOperand(0).getValueType()==MVT::f32) 7034 Hi = DAG.getNode(ISD::FP_EXTEND, MVT::f64, Node->getOperand(0)); 7035 else 7036 Hi = Node->getOperand(0); 7037 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7038 break; 7039 } 7040 RTLIB::Libcall LC = RTLIB::getFPEXT(Node->getOperand(0).getValueType(), VT); 7041 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 7042 Lo = ExpandLibCall(LC, Node, true, Hi); 7043 break; 7044 } 7045 case ISD::FP_ROUND: { 7046 RTLIB::Libcall LC = RTLIB::getFPROUND(Node->getOperand(0).getValueType(), 7047 VT); 7048 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 7049 Lo = ExpandLibCall(LC, Node, true, Hi); 7050 break; 7051 } 7052 case ISD::FSQRT: 7053 case ISD::FSIN: 7054 case ISD::FCOS: 7055 case ISD::FLOG: 7056 case ISD::FLOG2: 7057 case ISD::FLOG10: 7058 case ISD::FEXP: 7059 case ISD::FEXP2: 7060 case ISD::FTRUNC: 7061 case ISD::FFLOOR: 7062 case ISD::FCEIL: 7063 case ISD::FRINT: 7064 case ISD::FNEARBYINT: 7065 case ISD::FPOW: 7066 case ISD::FPOWI: { 7067 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 7068 switch(Node->getOpcode()) { 7069 case ISD::FSQRT: 7070 LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 7071 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128); 7072 break; 7073 case ISD::FSIN: 7074 LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64, 7075 RTLIB::SIN_F80, RTLIB::SIN_PPCF128); 7076 break; 7077 case ISD::FCOS: 7078 LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, 7079 RTLIB::COS_F80, RTLIB::COS_PPCF128); 7080 break; 7081 case ISD::FLOG: 7082 LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, 7083 RTLIB::LOG_F80, RTLIB::LOG_PPCF128); 7084 break; 7085 case ISD::FLOG2: 7086 LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 7087 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); 7088 break; 7089 case ISD::FLOG10: 7090 LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 7091 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); 7092 break; 7093 case ISD::FEXP: 7094 LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, 7095 RTLIB::EXP_F80, RTLIB::EXP_PPCF128); 7096 break; 7097 case ISD::FEXP2: 7098 LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 7099 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); 7100 break; 7101 case ISD::FTRUNC: 7102 LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 7103 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); 7104 break; 7105 case ISD::FFLOOR: 7106 LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 7107 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); 7108 break; 7109 case ISD::FCEIL: 7110 LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 7111 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); 7112 break; 7113 case ISD::FRINT: 7114 LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, 7115 RTLIB::RINT_F80, RTLIB::RINT_PPCF128); 7116 break; 7117 case ISD::FNEARBYINT: 7118 LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, 7119 RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); 7120 break; 7121 case ISD::FPOW: 7122 LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, 7123 RTLIB::POW_PPCF128); 7124 break; 7125 case ISD::FPOWI: 7126 LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, RTLIB::POWI_F80, 7127 RTLIB::POWI_PPCF128); 7128 break; 7129 default: assert(0 && "Unreachable!"); 7130 } 7131 Lo = ExpandLibCall(LC, Node, false, Hi); 7132 break; 7133 } 7134 case ISD::FABS: { 7135 if (VT == MVT::ppcf128) { 7136 SDValue Tmp; 7137 ExpandOp(Node->getOperand(0), Lo, Tmp); 7138 Hi = DAG.getNode(ISD::FABS, NVT, Tmp); 7139 // lo = hi==fabs(hi) ? lo : -lo; 7140 Lo = DAG.getNode(ISD::SELECT_CC, NVT, Hi, Tmp, 7141 Lo, DAG.getNode(ISD::FNEG, NVT, Lo), 7142 DAG.getCondCode(ISD::SETEQ)); 7143 break; 7144 } 7145 SDValue Mask = (VT == MVT::f64) 7146 ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) 7147 : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); 7148 Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask); 7149 Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0)); 7150 Lo = DAG.getNode(ISD::AND, NVT, Lo, Mask); 7151 if (getTypeAction(NVT) == Expand) 7152 ExpandOp(Lo, Lo, Hi); 7153 break; 7154 } 7155 case ISD::FNEG: { 7156 if (VT == MVT::ppcf128) { 7157 ExpandOp(Node->getOperand(0), Lo, Hi); 7158 Lo = DAG.getNode(ISD::FNEG, MVT::f64, Lo); 7159 Hi = DAG.getNode(ISD::FNEG, MVT::f64, Hi); 7160 break; 7161 } 7162 SDValue Mask = (VT == MVT::f64) 7163 ? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT) 7164 : DAG.getConstantFP(BitsToFloat(1U << 31), VT); 7165 Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask); 7166 Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0)); 7167 Lo = DAG.getNode(ISD::XOR, NVT, Lo, Mask); 7168 if (getTypeAction(NVT) == Expand) 7169 ExpandOp(Lo, Lo, Hi); 7170 break; 7171 } 7172 case ISD::FCOPYSIGN: { 7173 Lo = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI); 7174 if (getTypeAction(NVT) == Expand) 7175 ExpandOp(Lo, Lo, Hi); 7176 break; 7177 } 7178 case ISD::SINT_TO_FP: 7179 case ISD::UINT_TO_FP: { 7180 bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; 7181 MVT SrcVT = Node->getOperand(0).getValueType(); 7182 7183 // Promote the operand if needed. Do this before checking for 7184 // ppcf128 so conversions of i16 and i8 work. 7185 if (getTypeAction(SrcVT) == Promote) { 7186 SDValue Tmp = PromoteOp(Node->getOperand(0)); 7187 Tmp = isSigned 7188 ? DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp.getValueType(), Tmp, 7189 DAG.getValueType(SrcVT)) 7190 : DAG.getZeroExtendInReg(Tmp, SrcVT); 7191 Node = DAG.UpdateNodeOperands(Op, Tmp).getNode(); 7192 SrcVT = Node->getOperand(0).getValueType(); 7193 } 7194 7195 if (VT == MVT::ppcf128 && SrcVT == MVT::i32) { 7196 static const uint64_t zero = 0; 7197 if (isSigned) { 7198 Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, MVT::f64, 7199 Node->getOperand(0))); 7200 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7201 } else { 7202 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 7203 Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, MVT::f64, 7204 Node->getOperand(0))); 7205 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7206 Hi = DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi); 7207 // X>=0 ? {(f64)x, 0} : {(f64)x, 0} + 2^32 7208 ExpandOp(DAG.getNode(ISD::SELECT_CC, MVT::ppcf128, Node->getOperand(0), 7209 DAG.getConstant(0, MVT::i32), 7210 DAG.getNode(ISD::FADD, MVT::ppcf128, Hi, 7211 DAG.getConstantFP( 7212 APFloat(APInt(128, 2, TwoE32)), 7213 MVT::ppcf128)), 7214 Hi, 7215 DAG.getCondCode(ISD::SETLT)), 7216 Lo, Hi); 7217 } 7218 break; 7219 } 7220 if (VT == MVT::ppcf128 && SrcVT == MVT::i64 && !isSigned) { 7221 // si64->ppcf128 done by libcall, below 7222 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 7223 ExpandOp(DAG.getNode(ISD::SINT_TO_FP, MVT::ppcf128, Node->getOperand(0)), 7224 Lo, Hi); 7225 Hi = DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi); 7226 // x>=0 ? (ppcf128)(i64)x : (ppcf128)(i64)x + 2^64 7227 ExpandOp(DAG.getNode(ISD::SELECT_CC, MVT::ppcf128, Node->getOperand(0), 7228 DAG.getConstant(0, MVT::i64), 7229 DAG.getNode(ISD::FADD, MVT::ppcf128, Hi, 7230 DAG.getConstantFP( 7231 APFloat(APInt(128, 2, TwoE64)), 7232 MVT::ppcf128)), 7233 Hi, 7234 DAG.getCondCode(ISD::SETLT)), 7235 Lo, Hi); 7236 break; 7237 } 7238 7239 Lo = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, VT, 7240 Node->getOperand(0)); 7241 if (getTypeAction(Lo.getValueType()) == Expand) 7242 // float to i32 etc. can be 'expanded' to a single node. 7243 ExpandOp(Lo, Lo, Hi); 7244 break; 7245 } 7246 } 7247 7248 // Make sure the resultant values have been legalized themselves, unless this 7249 // is a type that requires multi-step expansion. 7250 if (getTypeAction(NVT) != Expand && NVT != MVT::isVoid) { 7251 Lo = LegalizeOp(Lo); 7252 if (Hi.getNode()) 7253 // Don't legalize the high part if it is expanded to a single node. 7254 Hi = LegalizeOp(Hi); 7255 } 7256 7257 // Remember in a map if the values will be reused later. 7258 bool isNew = 7259 ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; 7260 assert(isNew && "Value already expanded?!?"); 7261 isNew = isNew; 7262} 7263 7264/// SplitVectorOp - Given an operand of vector type, break it down into 7265/// two smaller values, still of vector type. 7266void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo, 7267 SDValue &Hi) { 7268 assert(Op.getValueType().isVector() && "Cannot split non-vector type!"); 7269 SDNode *Node = Op.getNode(); 7270 unsigned NumElements = Op.getValueType().getVectorNumElements(); 7271 assert(NumElements > 1 && "Cannot split a single element vector!"); 7272 7273 MVT NewEltVT = Op.getValueType().getVectorElementType(); 7274 7275 unsigned NewNumElts_Lo = 1 << Log2_32(NumElements-1); 7276 unsigned NewNumElts_Hi = NumElements - NewNumElts_Lo; 7277 7278 MVT NewVT_Lo = MVT::getVectorVT(NewEltVT, NewNumElts_Lo); 7279 MVT NewVT_Hi = MVT::getVectorVT(NewEltVT, NewNumElts_Hi); 7280 7281 // See if we already split it. 7282 std::map<SDValue, std::pair<SDValue, SDValue> >::iterator I 7283 = SplitNodes.find(Op); 7284 if (I != SplitNodes.end()) { 7285 Lo = I->second.first; 7286 Hi = I->second.second; 7287 return; 7288 } 7289 7290 switch (Node->getOpcode()) { 7291 default: 7292#ifndef NDEBUG 7293 Node->dump(&DAG); 7294#endif 7295 assert(0 && "Unhandled operation in SplitVectorOp!"); 7296 case ISD::UNDEF: 7297 Lo = DAG.getNode(ISD::UNDEF, NewVT_Lo); 7298 Hi = DAG.getNode(ISD::UNDEF, NewVT_Hi); 7299 break; 7300 case ISD::BUILD_PAIR: 7301 Lo = Node->getOperand(0); 7302 Hi = Node->getOperand(1); 7303 break; 7304 case ISD::INSERT_VECTOR_ELT: { 7305 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Node->getOperand(2))) { 7306 SplitVectorOp(Node->getOperand(0), Lo, Hi); 7307 unsigned Index = Idx->getZExtValue(); 7308 SDValue ScalarOp = Node->getOperand(1); 7309 if (Index < NewNumElts_Lo) 7310 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT_Lo, Lo, ScalarOp, 7311 DAG.getIntPtrConstant(Index)); 7312 else 7313 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT_Hi, Hi, ScalarOp, 7314 DAG.getIntPtrConstant(Index - NewNumElts_Lo)); 7315 break; 7316 } 7317 SDValue Tmp = PerformInsertVectorEltInMemory(Node->getOperand(0), 7318 Node->getOperand(1), 7319 Node->getOperand(2)); 7320 SplitVectorOp(Tmp, Lo, Hi); 7321 break; 7322 } 7323 case ISD::VECTOR_SHUFFLE: { 7324 // Build the low part. 7325 SDValue Mask = Node->getOperand(2); 7326 SmallVector<SDValue, 8> Ops; 7327 MVT PtrVT = TLI.getPointerTy(); 7328 7329 // Insert all of the elements from the input that are needed. We use 7330 // buildvector of extractelement here because the input vectors will have 7331 // to be legalized, so this makes the code simpler. 7332 for (unsigned i = 0; i != NewNumElts_Lo; ++i) { 7333 SDValue IdxNode = Mask.getOperand(i); 7334 if (IdxNode.getOpcode() == ISD::UNDEF) { 7335 Ops.push_back(DAG.getNode(ISD::UNDEF, NewEltVT)); 7336 continue; 7337 } 7338 unsigned Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue(); 7339 SDValue InVec = Node->getOperand(0); 7340 if (Idx >= NumElements) { 7341 InVec = Node->getOperand(1); 7342 Idx -= NumElements; 7343 } 7344 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewEltVT, InVec, 7345 DAG.getConstant(Idx, PtrVT))); 7346 } 7347 Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Lo, &Ops[0], Ops.size()); 7348 Ops.clear(); 7349 7350 for (unsigned i = NewNumElts_Lo; i != NumElements; ++i) { 7351 SDValue IdxNode = Mask.getOperand(i); 7352 if (IdxNode.getOpcode() == ISD::UNDEF) { 7353 Ops.push_back(DAG.getNode(ISD::UNDEF, NewEltVT)); 7354 continue; 7355 } 7356 unsigned Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue(); 7357 SDValue InVec = Node->getOperand(0); 7358 if (Idx >= NumElements) { 7359 InVec = Node->getOperand(1); 7360 Idx -= NumElements; 7361 } 7362 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewEltVT, InVec, 7363 DAG.getConstant(Idx, PtrVT))); 7364 } 7365 Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Hi, &Ops[0], Ops.size()); 7366 break; 7367 } 7368 case ISD::BUILD_VECTOR: { 7369 SmallVector<SDValue, 8> LoOps(Node->op_begin(), 7370 Node->op_begin()+NewNumElts_Lo); 7371 Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Lo, &LoOps[0], LoOps.size()); 7372 7373 SmallVector<SDValue, 8> HiOps(Node->op_begin()+NewNumElts_Lo, 7374 Node->op_end()); 7375 Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Hi, &HiOps[0], HiOps.size()); 7376 break; 7377 } 7378 case ISD::CONCAT_VECTORS: { 7379 // FIXME: Handle non-power-of-two vectors? 7380 unsigned NewNumSubvectors = Node->getNumOperands() / 2; 7381 if (NewNumSubvectors == 1) { 7382 Lo = Node->getOperand(0); 7383 Hi = Node->getOperand(1); 7384 } else { 7385 SmallVector<SDValue, 8> LoOps(Node->op_begin(), 7386 Node->op_begin()+NewNumSubvectors); 7387 Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Lo, &LoOps[0], LoOps.size()); 7388 7389 SmallVector<SDValue, 8> HiOps(Node->op_begin()+NewNumSubvectors, 7390 Node->op_end()); 7391 Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Hi, &HiOps[0], HiOps.size()); 7392 } 7393 break; 7394 } 7395 case ISD::EXTRACT_SUBVECTOR: { 7396 SDValue Vec = Op.getOperand(0); 7397 SDValue Idx = Op.getOperand(1); 7398 MVT IdxVT = Idx.getValueType(); 7399 7400 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Lo, Vec, Idx); 7401 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 7402 if (CIdx) { 7403 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Hi, Vec, 7404 DAG.getConstant(CIdx->getZExtValue() + NewNumElts_Lo, 7405 IdxVT)); 7406 } else { 7407 Idx = DAG.getNode(ISD::ADD, IdxVT, Idx, 7408 DAG.getConstant(NewNumElts_Lo, IdxVT)); 7409 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Hi, Vec, Idx); 7410 } 7411 break; 7412 } 7413 case ISD::SELECT: { 7414 SDValue Cond = Node->getOperand(0); 7415 7416 SDValue LL, LH, RL, RH; 7417 SplitVectorOp(Node->getOperand(1), LL, LH); 7418 SplitVectorOp(Node->getOperand(2), RL, RH); 7419 7420 if (Cond.getValueType().isVector()) { 7421 // Handle a vector merge. 7422 SDValue CL, CH; 7423 SplitVectorOp(Cond, CL, CH); 7424 Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, CL, LL, RL); 7425 Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, CH, LH, RH); 7426 } else { 7427 // Handle a simple select with vector operands. 7428 Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, Cond, LL, RL); 7429 Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, Cond, LH, RH); 7430 } 7431 break; 7432 } 7433 case ISD::SELECT_CC: { 7434 SDValue CondLHS = Node->getOperand(0); 7435 SDValue CondRHS = Node->getOperand(1); 7436 SDValue CondCode = Node->getOperand(4); 7437 7438 SDValue LL, LH, RL, RH; 7439 SplitVectorOp(Node->getOperand(2), LL, LH); 7440 SplitVectorOp(Node->getOperand(3), RL, RH); 7441 7442 // Handle a simple select with vector operands. 7443 Lo = DAG.getNode(ISD::SELECT_CC, NewVT_Lo, CondLHS, CondRHS, 7444 LL, RL, CondCode); 7445 Hi = DAG.getNode(ISD::SELECT_CC, NewVT_Hi, CondLHS, CondRHS, 7446 LH, RH, CondCode); 7447 break; 7448 } 7449 case ISD::VSETCC: { 7450 SDValue LL, LH, RL, RH; 7451 SplitVectorOp(Node->getOperand(0), LL, LH); 7452 SplitVectorOp(Node->getOperand(1), RL, RH); 7453 Lo = DAG.getNode(ISD::VSETCC, NewVT_Lo, LL, RL, Node->getOperand(2)); 7454 Hi = DAG.getNode(ISD::VSETCC, NewVT_Hi, LH, RH, Node->getOperand(2)); 7455 break; 7456 } 7457 case ISD::ADD: 7458 case ISD::SUB: 7459 case ISD::MUL: 7460 case ISD::FADD: 7461 case ISD::FSUB: 7462 case ISD::FMUL: 7463 case ISD::SDIV: 7464 case ISD::UDIV: 7465 case ISD::FDIV: 7466 case ISD::FPOW: 7467 case ISD::AND: 7468 case ISD::OR: 7469 case ISD::XOR: 7470 case ISD::UREM: 7471 case ISD::SREM: 7472 case ISD::FREM: { 7473 SDValue LL, LH, RL, RH; 7474 SplitVectorOp(Node->getOperand(0), LL, LH); 7475 SplitVectorOp(Node->getOperand(1), RL, RH); 7476 7477 Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, LL, RL); 7478 Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, LH, RH); 7479 break; 7480 } 7481 case ISD::FP_ROUND: 7482 case ISD::FPOWI: { 7483 SDValue L, H; 7484 SplitVectorOp(Node->getOperand(0), L, H); 7485 7486 Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, L, Node->getOperand(1)); 7487 Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, H, Node->getOperand(1)); 7488 break; 7489 } 7490 case ISD::CTTZ: 7491 case ISD::CTLZ: 7492 case ISD::CTPOP: 7493 case ISD::FNEG: 7494 case ISD::FABS: 7495 case ISD::FSQRT: 7496 case ISD::FSIN: 7497 case ISD::FCOS: 7498 case ISD::FLOG: 7499 case ISD::FLOG2: 7500 case ISD::FLOG10: 7501 case ISD::FEXP: 7502 case ISD::FEXP2: 7503 case ISD::FP_TO_SINT: 7504 case ISD::FP_TO_UINT: 7505 case ISD::SINT_TO_FP: 7506 case ISD::UINT_TO_FP: 7507 case ISD::TRUNCATE: 7508 case ISD::ANY_EXTEND: 7509 case ISD::SIGN_EXTEND: 7510 case ISD::ZERO_EXTEND: 7511 case ISD::FP_EXTEND: { 7512 SDValue L, H; 7513 SplitVectorOp(Node->getOperand(0), L, H); 7514 7515 Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, L); 7516 Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, H); 7517 break; 7518 } 7519 case ISD::CONVERT_RNDSAT: { 7520 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 7521 SDValue L, H; 7522 SplitVectorOp(Node->getOperand(0), L, H); 7523 SDValue DTyOpL = DAG.getValueType(NewVT_Lo); 7524 SDValue DTyOpH = DAG.getValueType(NewVT_Hi); 7525 SDValue STyOpL = DAG.getValueType(L.getValueType()); 7526 SDValue STyOpH = DAG.getValueType(H.getValueType()); 7527 7528 SDValue RndOp = Node->getOperand(3); 7529 SDValue SatOp = Node->getOperand(4); 7530 7531 Lo = DAG.getConvertRndSat(NewVT_Lo, L, DTyOpL, STyOpL, 7532 RndOp, SatOp, CvtCode); 7533 Hi = DAG.getConvertRndSat(NewVT_Hi, H, DTyOpH, STyOpH, 7534 RndOp, SatOp, CvtCode); 7535 break; 7536 } 7537 case ISD::LOAD: { 7538 LoadSDNode *LD = cast<LoadSDNode>(Node); 7539 SDValue Ch = LD->getChain(); 7540 SDValue Ptr = LD->getBasePtr(); 7541 ISD::LoadExtType ExtType = LD->getExtensionType(); 7542 const Value *SV = LD->getSrcValue(); 7543 int SVOffset = LD->getSrcValueOffset(); 7544 MVT MemoryVT = LD->getMemoryVT(); 7545 unsigned Alignment = LD->getAlignment(); 7546 bool isVolatile = LD->isVolatile(); 7547 7548 assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); 7549 SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType()); 7550 7551 MVT MemNewEltVT = MemoryVT.getVectorElementType(); 7552 MVT MemNewVT_Lo = MVT::getVectorVT(MemNewEltVT, NewNumElts_Lo); 7553 MVT MemNewVT_Hi = MVT::getVectorVT(MemNewEltVT, NewNumElts_Hi); 7554 7555 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, 7556 NewVT_Lo, Ch, Ptr, Offset, 7557 SV, SVOffset, MemNewVT_Lo, isVolatile, Alignment); 7558 unsigned IncrementSize = NewNumElts_Lo * MemNewEltVT.getSizeInBits()/8; 7559 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 7560 DAG.getIntPtrConstant(IncrementSize)); 7561 SVOffset += IncrementSize; 7562 Alignment = MinAlign(Alignment, IncrementSize); 7563 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, 7564 NewVT_Hi, Ch, Ptr, Offset, 7565 SV, SVOffset, MemNewVT_Hi, isVolatile, Alignment); 7566 7567 // Build a factor node to remember that this load is independent of the 7568 // other one. 7569 SDValue TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 7570 Hi.getValue(1)); 7571 7572 // Remember that we legalized the chain. 7573 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); 7574 break; 7575 } 7576 case ISD::BIT_CONVERT: { 7577 // We know the result is a vector. The input may be either a vector or a 7578 // scalar value. 7579 SDValue InOp = Node->getOperand(0); 7580 if (!InOp.getValueType().isVector() || 7581 InOp.getValueType().getVectorNumElements() == 1) { 7582 // The input is a scalar or single-element vector. 7583 // Lower to a store/load so that it can be split. 7584 // FIXME: this could be improved probably. 7585 unsigned LdAlign = TLI.getTargetData()->getPrefTypeAlignment( 7586 Op.getValueType().getTypeForMVT()); 7587 SDValue Ptr = DAG.CreateStackTemporary(InOp.getValueType(), LdAlign); 7588 int FI = cast<FrameIndexSDNode>(Ptr.getNode())->getIndex(); 7589 7590 SDValue St = DAG.getStore(DAG.getEntryNode(), 7591 InOp, Ptr, 7592 PseudoSourceValue::getFixedStack(FI), 0); 7593 InOp = DAG.getLoad(Op.getValueType(), St, Ptr, 7594 PseudoSourceValue::getFixedStack(FI), 0); 7595 } 7596 // Split the vector and convert each of the pieces now. 7597 SplitVectorOp(InOp, Lo, Hi); 7598 Lo = DAG.getNode(ISD::BIT_CONVERT, NewVT_Lo, Lo); 7599 Hi = DAG.getNode(ISD::BIT_CONVERT, NewVT_Hi, Hi); 7600 break; 7601 } 7602 } 7603 7604 // Remember in a map if the values will be reused later. 7605 bool isNew = 7606 SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; 7607 assert(isNew && "Value already split?!?"); 7608 isNew = isNew; 7609} 7610 7611 7612/// ScalarizeVectorOp - Given an operand of single-element vector type 7613/// (e.g. v1f32), convert it into the equivalent operation that returns a 7614/// scalar (e.g. f32) value. 7615SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) { 7616 assert(Op.getValueType().isVector() && "Bad ScalarizeVectorOp invocation!"); 7617 SDNode *Node = Op.getNode(); 7618 MVT NewVT = Op.getValueType().getVectorElementType(); 7619 assert(Op.getValueType().getVectorNumElements() == 1); 7620 7621 // See if we already scalarized it. 7622 std::map<SDValue, SDValue>::iterator I = ScalarizedNodes.find(Op); 7623 if (I != ScalarizedNodes.end()) return I->second; 7624 7625 SDValue Result; 7626 switch (Node->getOpcode()) { 7627 default: 7628#ifndef NDEBUG 7629 Node->dump(&DAG); cerr << "\n"; 7630#endif 7631 assert(0 && "Unknown vector operation in ScalarizeVectorOp!"); 7632 case ISD::ADD: 7633 case ISD::FADD: 7634 case ISD::SUB: 7635 case ISD::FSUB: 7636 case ISD::MUL: 7637 case ISD::FMUL: 7638 case ISD::SDIV: 7639 case ISD::UDIV: 7640 case ISD::FDIV: 7641 case ISD::SREM: 7642 case ISD::UREM: 7643 case ISD::FREM: 7644 case ISD::FPOW: 7645 case ISD::AND: 7646 case ISD::OR: 7647 case ISD::XOR: 7648 Result = DAG.getNode(Node->getOpcode(), 7649 NewVT, 7650 ScalarizeVectorOp(Node->getOperand(0)), 7651 ScalarizeVectorOp(Node->getOperand(1))); 7652 break; 7653 case ISD::FNEG: 7654 case ISD::FABS: 7655 case ISD::FSQRT: 7656 case ISD::FSIN: 7657 case ISD::FCOS: 7658 case ISD::FLOG: 7659 case ISD::FLOG2: 7660 case ISD::FLOG10: 7661 case ISD::FEXP: 7662 case ISD::FEXP2: 7663 case ISD::FP_TO_SINT: 7664 case ISD::FP_TO_UINT: 7665 case ISD::SINT_TO_FP: 7666 case ISD::UINT_TO_FP: 7667 case ISD::SIGN_EXTEND: 7668 case ISD::ZERO_EXTEND: 7669 case ISD::ANY_EXTEND: 7670 case ISD::TRUNCATE: 7671 case ISD::FP_EXTEND: 7672 Result = DAG.getNode(Node->getOpcode(), 7673 NewVT, 7674 ScalarizeVectorOp(Node->getOperand(0))); 7675 break; 7676 case ISD::CONVERT_RNDSAT: { 7677 SDValue Op0 = ScalarizeVectorOp(Node->getOperand(0)); 7678 Result = DAG.getConvertRndSat(NewVT, Op0, 7679 DAG.getValueType(NewVT), 7680 DAG.getValueType(Op0.getValueType()), 7681 Node->getOperand(3), 7682 Node->getOperand(4), 7683 cast<CvtRndSatSDNode>(Node)->getCvtCode()); 7684 break; 7685 } 7686 case ISD::FPOWI: 7687 case ISD::FP_ROUND: 7688 Result = DAG.getNode(Node->getOpcode(), 7689 NewVT, 7690 ScalarizeVectorOp(Node->getOperand(0)), 7691 Node->getOperand(1)); 7692 break; 7693 case ISD::LOAD: { 7694 LoadSDNode *LD = cast<LoadSDNode>(Node); 7695 SDValue Ch = LegalizeOp(LD->getChain()); // Legalize the chain. 7696 SDValue Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer. 7697 ISD::LoadExtType ExtType = LD->getExtensionType(); 7698 const Value *SV = LD->getSrcValue(); 7699 int SVOffset = LD->getSrcValueOffset(); 7700 MVT MemoryVT = LD->getMemoryVT(); 7701 unsigned Alignment = LD->getAlignment(); 7702 bool isVolatile = LD->isVolatile(); 7703 7704 assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); 7705 SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType()); 7706 7707 Result = DAG.getLoad(ISD::UNINDEXED, ExtType, 7708 NewVT, Ch, Ptr, Offset, SV, SVOffset, 7709 MemoryVT.getVectorElementType(), 7710 isVolatile, Alignment); 7711 7712 // Remember that we legalized the chain. 7713 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 7714 break; 7715 } 7716 case ISD::BUILD_VECTOR: 7717 Result = Node->getOperand(0); 7718 break; 7719 case ISD::INSERT_VECTOR_ELT: 7720 // Returning the inserted scalar element. 7721 Result = Node->getOperand(1); 7722 break; 7723 case ISD::CONCAT_VECTORS: 7724 assert(Node->getOperand(0).getValueType() == NewVT && 7725 "Concat of non-legal vectors not yet supported!"); 7726 Result = Node->getOperand(0); 7727 break; 7728 case ISD::VECTOR_SHUFFLE: { 7729 // Figure out if the scalar is the LHS or RHS and return it. 7730 SDValue EltNum = Node->getOperand(2).getOperand(0); 7731 if (cast<ConstantSDNode>(EltNum)->getZExtValue()) 7732 Result = ScalarizeVectorOp(Node->getOperand(1)); 7733 else 7734 Result = ScalarizeVectorOp(Node->getOperand(0)); 7735 break; 7736 } 7737 case ISD::EXTRACT_SUBVECTOR: 7738 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, Node->getOperand(0), 7739 Node->getOperand(1)); 7740 break; 7741 case ISD::BIT_CONVERT: { 7742 SDValue Op0 = Op.getOperand(0); 7743 if (Op0.getValueType().getVectorNumElements() == 1) 7744 Op0 = ScalarizeVectorOp(Op0); 7745 Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op0); 7746 break; 7747 } 7748 case ISD::SELECT: 7749 Result = DAG.getNode(ISD::SELECT, NewVT, Op.getOperand(0), 7750 ScalarizeVectorOp(Op.getOperand(1)), 7751 ScalarizeVectorOp(Op.getOperand(2))); 7752 break; 7753 case ISD::SELECT_CC: 7754 Result = DAG.getNode(ISD::SELECT_CC, NewVT, Node->getOperand(0), 7755 Node->getOperand(1), 7756 ScalarizeVectorOp(Op.getOperand(2)), 7757 ScalarizeVectorOp(Op.getOperand(3)), 7758 Node->getOperand(4)); 7759 break; 7760 case ISD::VSETCC: { 7761 SDValue Op0 = ScalarizeVectorOp(Op.getOperand(0)); 7762 SDValue Op1 = ScalarizeVectorOp(Op.getOperand(1)); 7763 Result = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Op0), Op0, Op1, 7764 Op.getOperand(2)); 7765 Result = DAG.getNode(ISD::SELECT, NewVT, Result, 7766 DAG.getConstant(-1ULL, NewVT), 7767 DAG.getConstant(0ULL, NewVT)); 7768 break; 7769 } 7770 } 7771 7772 if (TLI.isTypeLegal(NewVT)) 7773 Result = LegalizeOp(Result); 7774 bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second; 7775 assert(isNew && "Value already scalarized?"); 7776 isNew = isNew; 7777 return Result; 7778} 7779 7780 7781SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) { 7782 std::map<SDValue, SDValue>::iterator I = WidenNodes.find(Op); 7783 if (I != WidenNodes.end()) return I->second; 7784 7785 MVT VT = Op.getValueType(); 7786 assert(VT.isVector() && "Cannot widen non-vector type!"); 7787 7788 SDValue Result; 7789 SDNode *Node = Op.getNode(); 7790 MVT EVT = VT.getVectorElementType(); 7791 7792 unsigned NumElts = VT.getVectorNumElements(); 7793 unsigned NewNumElts = WidenVT.getVectorNumElements(); 7794 assert(NewNumElts > NumElts && "Cannot widen to smaller type!"); 7795 assert(NewNumElts < 17); 7796 7797 // When widen is called, it is assumed that it is more efficient to use a 7798 // wide type. The default action is to widen to operation to a wider legal 7799 // vector type and then do the operation if it is legal by calling LegalizeOp 7800 // again. If there is no vector equivalent, we will unroll the operation, do 7801 // it, and rebuild the vector. If most of the operations are vectorizible to 7802 // the legal type, the resulting code will be more efficient. If this is not 7803 // the case, the resulting code will preform badly as we end up generating 7804 // code to pack/unpack the results. It is the function that calls widen 7805 // that is responsible for seeing this doesn't happen. 7806 switch (Node->getOpcode()) { 7807 default: 7808#ifndef NDEBUG 7809 Node->dump(&DAG); 7810#endif 7811 assert(0 && "Unexpected operation in WidenVectorOp!"); 7812 break; 7813 case ISD::CopyFromReg: 7814 assert(0 && "CopyFromReg doesn't need widening!"); 7815 case ISD::Constant: 7816 case ISD::ConstantFP: 7817 // To build a vector of these elements, clients should call BuildVector 7818 // and with each element instead of creating a node with a vector type 7819 assert(0 && "Unexpected operation in WidenVectorOp!"); 7820 case ISD::VAARG: 7821 // Variable Arguments with vector types doesn't make any sense to me 7822 assert(0 && "Unexpected operation in WidenVectorOp!"); 7823 break; 7824 case ISD::UNDEF: 7825 Result = DAG.getNode(ISD::UNDEF, WidenVT); 7826 break; 7827 case ISD::BUILD_VECTOR: { 7828 // Build a vector with undefined for the new nodes 7829 SDValueVector NewOps(Node->op_begin(), Node->op_end()); 7830 for (unsigned i = NumElts; i < NewNumElts; ++i) { 7831 NewOps.push_back(DAG.getNode(ISD::UNDEF,EVT)); 7832 } 7833 Result = DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &NewOps[0], NewOps.size()); 7834 break; 7835 } 7836 case ISD::INSERT_VECTOR_ELT: { 7837 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 7838 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, WidenVT, Tmp1, 7839 Node->getOperand(1), Node->getOperand(2)); 7840 break; 7841 } 7842 case ISD::VECTOR_SHUFFLE: { 7843 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 7844 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); 7845 // VECTOR_SHUFFLE 3rd operand must be a constant build vector that is 7846 // used as permutation array. We build the vector here instead of widening 7847 // because we don't want to legalize and have it turned to something else. 7848 SDValue PermOp = Node->getOperand(2); 7849 SDValueVector NewOps; 7850 MVT PVT = PermOp.getValueType().getVectorElementType(); 7851 for (unsigned i = 0; i < NumElts; ++i) { 7852 if (PermOp.getOperand(i).getOpcode() == ISD::UNDEF) { 7853 NewOps.push_back(PermOp.getOperand(i)); 7854 } else { 7855 unsigned Idx = 7856 cast<ConstantSDNode>(PermOp.getOperand(i))->getZExtValue(); 7857 if (Idx < NumElts) { 7858 NewOps.push_back(PermOp.getOperand(i)); 7859 } 7860 else { 7861 NewOps.push_back(DAG.getConstant(Idx + NewNumElts - NumElts, 7862 PermOp.getOperand(i).getValueType())); 7863 } 7864 } 7865 } 7866 for (unsigned i = NumElts; i < NewNumElts; ++i) { 7867 NewOps.push_back(DAG.getNode(ISD::UNDEF,PVT)); 7868 } 7869 7870 SDValue Tmp3 = DAG.getNode(ISD::BUILD_VECTOR, 7871 MVT::getVectorVT(PVT, NewOps.size()), 7872 &NewOps[0], NewOps.size()); 7873 7874 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, WidenVT, Tmp1, Tmp2, Tmp3); 7875 break; 7876 } 7877 case ISD::LOAD: { 7878 // If the load widen returns true, we can use a single load for the 7879 // vector. Otherwise, it is returning a token factor for multiple 7880 // loads. 7881 SDValue TFOp; 7882 if (LoadWidenVectorOp(Result, TFOp, Op, WidenVT)) 7883 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(1))); 7884 else 7885 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(0))); 7886 break; 7887 } 7888 7889 case ISD::BIT_CONVERT: { 7890 SDValue Tmp1 = Node->getOperand(0); 7891 // Converts between two different types so we need to determine 7892 // the correct widen type for the input operand. 7893 MVT TVT = Tmp1.getValueType(); 7894 assert(TVT.isVector() && "can not widen non vector type"); 7895 MVT TEVT = TVT.getVectorElementType(); 7896 assert(WidenVT.getSizeInBits() % EVT.getSizeInBits() == 0 && 7897 "can not widen bit bit convert that are not multiple of element type"); 7898 MVT TWidenVT = MVT::getVectorVT(TEVT, 7899 WidenVT.getSizeInBits()/EVT.getSizeInBits()); 7900 Tmp1 = WidenVectorOp(Tmp1, TWidenVT); 7901 assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits()); 7902 Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); 7903 7904 TargetLowering::LegalizeAction action = 7905 TLI.getOperationAction(Node->getOpcode(), WidenVT); 7906 switch (action) { 7907 default: assert(0 && "action not supported"); 7908 case TargetLowering::Legal: 7909 break; 7910 case TargetLowering::Promote: 7911 // We defer the promotion to when we legalize the op 7912 break; 7913 case TargetLowering::Expand: 7914 // Expand the operation into a bunch of nasty scalar code. 7915 Result = LegalizeOp(UnrollVectorOp(Result)); 7916 break; 7917 } 7918 break; 7919 } 7920 7921 case ISD::SINT_TO_FP: 7922 case ISD::UINT_TO_FP: 7923 case ISD::FP_TO_SINT: 7924 case ISD::FP_TO_UINT: { 7925 SDValue Tmp1 = Node->getOperand(0); 7926 // Converts between two different types so we need to determine 7927 // the correct widen type for the input operand. 7928 MVT TVT = Tmp1.getValueType(); 7929 assert(TVT.isVector() && "can not widen non vector type"); 7930 MVT TEVT = TVT.getVectorElementType(); 7931 MVT TWidenVT = MVT::getVectorVT(TEVT, NewNumElts); 7932 Tmp1 = WidenVectorOp(Tmp1, TWidenVT); 7933 assert(Tmp1.getValueType().getVectorNumElements() == NewNumElts); 7934 Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); 7935 break; 7936 } 7937 7938 case ISD::FP_EXTEND: 7939 assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); 7940 case ISD::TRUNCATE: 7941 case ISD::SIGN_EXTEND: 7942 case ISD::ZERO_EXTEND: 7943 case ISD::ANY_EXTEND: 7944 case ISD::FP_ROUND: 7945 case ISD::SIGN_EXTEND_INREG: 7946 case ISD::FABS: 7947 case ISD::FNEG: 7948 case ISD::FSQRT: 7949 case ISD::FSIN: 7950 case ISD::FCOS: 7951 case ISD::CTPOP: 7952 case ISD::CTTZ: 7953 case ISD::CTLZ: { 7954 // Unary op widening 7955 SDValue Tmp1; 7956 Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 7957 assert(Tmp1.getValueType() == WidenVT); 7958 Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); 7959 break; 7960 } 7961 case ISD::CONVERT_RNDSAT: { 7962 SDValue RndOp = Node->getOperand(3); 7963 SDValue SatOp = Node->getOperand(4); 7964 SDValue SrcOp = Node->getOperand(0); 7965 7966 // Converts between two different types so we need to determine 7967 // the correct widen type for the input operand. 7968 MVT SVT = SrcOp.getValueType(); 7969 assert(SVT.isVector() && "can not widen non vector type"); 7970 MVT SEVT = SVT.getVectorElementType(); 7971 MVT SWidenVT = MVT::getVectorVT(SEVT, NewNumElts); 7972 7973 SrcOp = WidenVectorOp(SrcOp, SWidenVT); 7974 assert(SrcOp.getValueType() == WidenVT); 7975 SDValue DTyOp = DAG.getValueType(WidenVT); 7976 SDValue STyOp = DAG.getValueType(SrcOp.getValueType()); 7977 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 7978 7979 Result = DAG.getConvertRndSat(WidenVT, SrcOp, DTyOp, STyOp, 7980 RndOp, SatOp, CvtCode); 7981 break; 7982 } 7983 case ISD::FPOW: 7984 case ISD::FPOWI: 7985 case ISD::ADD: 7986 case ISD::SUB: 7987 case ISD::MUL: 7988 case ISD::MULHS: 7989 case ISD::MULHU: 7990 case ISD::AND: 7991 case ISD::OR: 7992 case ISD::XOR: 7993 case ISD::FADD: 7994 case ISD::FSUB: 7995 case ISD::FMUL: 7996 case ISD::SDIV: 7997 case ISD::SREM: 7998 case ISD::FDIV: 7999 case ISD::FREM: 8000 case ISD::FCOPYSIGN: 8001 case ISD::UDIV: 8002 case ISD::UREM: 8003 case ISD::BSWAP: { 8004 // Binary op widening 8005 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8006 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); 8007 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); 8008 Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2); 8009 break; 8010 } 8011 8012 case ISD::SHL: 8013 case ISD::SRA: 8014 case ISD::SRL: { 8015 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8016 assert(Tmp1.getValueType() == WidenVT); 8017 SDValue ShOp = Node->getOperand(1); 8018 MVT ShVT = ShOp.getValueType(); 8019 MVT NewShVT = MVT::getVectorVT(ShVT.getVectorElementType(), 8020 WidenVT.getVectorNumElements()); 8021 ShOp = WidenVectorOp(ShOp, NewShVT); 8022 assert(ShOp.getValueType() == NewShVT); 8023 Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, ShOp); 8024 break; 8025 } 8026 8027 case ISD::EXTRACT_VECTOR_ELT: { 8028 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8029 assert(Tmp1.getValueType() == WidenVT); 8030 Result = DAG.getNode(Node->getOpcode(), EVT, Tmp1, Node->getOperand(1)); 8031 break; 8032 } 8033 case ISD::CONCAT_VECTORS: { 8034 // We concurrently support only widen on a multiple of the incoming vector. 8035 // We could widen on a multiple of the incoming operand if necessary. 8036 unsigned NumConcat = NewNumElts / NumElts; 8037 assert(NewNumElts % NumElts == 0 && "Can widen only a multiple of vector"); 8038 SDValue UndefVal = DAG.getNode(ISD::UNDEF, VT); 8039 SmallVector<SDValue, 8> MOps; 8040 MOps.push_back(Op); 8041 for (unsigned i = 1; i != NumConcat; ++i) { 8042 MOps.push_back(UndefVal); 8043 } 8044 Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT, 8045 &MOps[0], MOps.size())); 8046 break; 8047 } 8048 case ISD::EXTRACT_SUBVECTOR: { 8049 SDValue Tmp1 = Node->getOperand(0); 8050 SDValue Idx = Node->getOperand(1); 8051 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 8052 if (CIdx && CIdx->getZExtValue() == 0) { 8053 // Since we are access the start of the vector, the incoming 8054 // vector type might be the proper. 8055 MVT Tmp1VT = Tmp1.getValueType(); 8056 if (Tmp1VT == WidenVT) 8057 return Tmp1; 8058 else { 8059 unsigned Tmp1VTNumElts = Tmp1VT.getVectorNumElements(); 8060 if (Tmp1VTNumElts < NewNumElts) 8061 Result = WidenVectorOp(Tmp1, WidenVT); 8062 else 8063 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, WidenVT, Tmp1, Idx); 8064 } 8065 } else if (NewNumElts % NumElts == 0) { 8066 // Widen the extracted subvector. 8067 unsigned NumConcat = NewNumElts / NumElts; 8068 SDValue UndefVal = DAG.getNode(ISD::UNDEF, VT); 8069 SmallVector<SDValue, 8> MOps; 8070 MOps.push_back(Op); 8071 for (unsigned i = 1; i != NumConcat; ++i) { 8072 MOps.push_back(UndefVal); 8073 } 8074 Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT, 8075 &MOps[0], MOps.size())); 8076 } else { 8077 assert(0 && "can not widen extract subvector"); 8078 // This could be implemented using insert and build vector but I would 8079 // like to see when this happens. 8080 } 8081 break; 8082 } 8083 8084 case ISD::SELECT: { 8085 // Determine new condition widen type and widen 8086 SDValue Cond1 = Node->getOperand(0); 8087 MVT CondVT = Cond1.getValueType(); 8088 assert(CondVT.isVector() && "can not widen non vector type"); 8089 MVT CondEVT = CondVT.getVectorElementType(); 8090 MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); 8091 Cond1 = WidenVectorOp(Cond1, CondWidenVT); 8092 assert(Cond1.getValueType() == CondWidenVT && "Condition not widen"); 8093 8094 SDValue Tmp1 = WidenVectorOp(Node->getOperand(1), WidenVT); 8095 SDValue Tmp2 = WidenVectorOp(Node->getOperand(2), WidenVT); 8096 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); 8097 Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Tmp1, Tmp2); 8098 break; 8099 } 8100 8101 case ISD::SELECT_CC: { 8102 // Determine new condition widen type and widen 8103 SDValue Cond1 = Node->getOperand(0); 8104 SDValue Cond2 = Node->getOperand(1); 8105 MVT CondVT = Cond1.getValueType(); 8106 assert(CondVT.isVector() && "can not widen non vector type"); 8107 assert(CondVT == Cond2.getValueType() && "mismatch lhs/rhs"); 8108 MVT CondEVT = CondVT.getVectorElementType(); 8109 MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); 8110 Cond1 = WidenVectorOp(Cond1, CondWidenVT); 8111 Cond2 = WidenVectorOp(Cond2, CondWidenVT); 8112 assert(Cond1.getValueType() == CondWidenVT && 8113 Cond2.getValueType() == CondWidenVT && "condition not widen"); 8114 8115 SDValue Tmp1 = WidenVectorOp(Node->getOperand(2), WidenVT); 8116 SDValue Tmp2 = WidenVectorOp(Node->getOperand(3), WidenVT); 8117 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT && 8118 "operands not widen"); 8119 Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Cond2, Tmp1, 8120 Tmp2, Node->getOperand(4)); 8121 break; 8122 } 8123 case ISD::VSETCC: { 8124 // Determine widen for the operand 8125 SDValue Tmp1 = Node->getOperand(0); 8126 MVT TmpVT = Tmp1.getValueType(); 8127 assert(TmpVT.isVector() && "can not widen non vector type"); 8128 MVT TmpEVT = TmpVT.getVectorElementType(); 8129 MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts); 8130 Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT); 8131 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT); 8132 Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2, 8133 Node->getOperand(2)); 8134 break; 8135 } 8136 case ISD::ATOMIC_CMP_SWAP_8: 8137 case ISD::ATOMIC_CMP_SWAP_16: 8138 case ISD::ATOMIC_CMP_SWAP_32: 8139 case ISD::ATOMIC_CMP_SWAP_64: 8140 case ISD::ATOMIC_LOAD_ADD_8: 8141 case ISD::ATOMIC_LOAD_SUB_8: 8142 case ISD::ATOMIC_LOAD_AND_8: 8143 case ISD::ATOMIC_LOAD_OR_8: 8144 case ISD::ATOMIC_LOAD_XOR_8: 8145 case ISD::ATOMIC_LOAD_NAND_8: 8146 case ISD::ATOMIC_LOAD_MIN_8: 8147 case ISD::ATOMIC_LOAD_MAX_8: 8148 case ISD::ATOMIC_LOAD_UMIN_8: 8149 case ISD::ATOMIC_LOAD_UMAX_8: 8150 case ISD::ATOMIC_SWAP_8: 8151 case ISD::ATOMIC_LOAD_ADD_16: 8152 case ISD::ATOMIC_LOAD_SUB_16: 8153 case ISD::ATOMIC_LOAD_AND_16: 8154 case ISD::ATOMIC_LOAD_OR_16: 8155 case ISD::ATOMIC_LOAD_XOR_16: 8156 case ISD::ATOMIC_LOAD_NAND_16: 8157 case ISD::ATOMIC_LOAD_MIN_16: 8158 case ISD::ATOMIC_LOAD_MAX_16: 8159 case ISD::ATOMIC_LOAD_UMIN_16: 8160 case ISD::ATOMIC_LOAD_UMAX_16: 8161 case ISD::ATOMIC_SWAP_16: 8162 case ISD::ATOMIC_LOAD_ADD_32: 8163 case ISD::ATOMIC_LOAD_SUB_32: 8164 case ISD::ATOMIC_LOAD_AND_32: 8165 case ISD::ATOMIC_LOAD_OR_32: 8166 case ISD::ATOMIC_LOAD_XOR_32: 8167 case ISD::ATOMIC_LOAD_NAND_32: 8168 case ISD::ATOMIC_LOAD_MIN_32: 8169 case ISD::ATOMIC_LOAD_MAX_32: 8170 case ISD::ATOMIC_LOAD_UMIN_32: 8171 case ISD::ATOMIC_LOAD_UMAX_32: 8172 case ISD::ATOMIC_SWAP_32: 8173 case ISD::ATOMIC_LOAD_ADD_64: 8174 case ISD::ATOMIC_LOAD_SUB_64: 8175 case ISD::ATOMIC_LOAD_AND_64: 8176 case ISD::ATOMIC_LOAD_OR_64: 8177 case ISD::ATOMIC_LOAD_XOR_64: 8178 case ISD::ATOMIC_LOAD_NAND_64: 8179 case ISD::ATOMIC_LOAD_MIN_64: 8180 case ISD::ATOMIC_LOAD_MAX_64: 8181 case ISD::ATOMIC_LOAD_UMIN_64: 8182 case ISD::ATOMIC_LOAD_UMAX_64: 8183 case ISD::ATOMIC_SWAP_64: { 8184 // For now, we assume that using vectors for these operations don't make 8185 // much sense so we just split it. We return an empty result 8186 SDValue X, Y; 8187 SplitVectorOp(Op, X, Y); 8188 return Result; 8189 break; 8190 } 8191 8192 } // end switch (Node->getOpcode()) 8193 8194 assert(Result.getNode() && "Didn't set a result!"); 8195 if (Result != Op) 8196 Result = LegalizeOp(Result); 8197 8198 AddWidenedOperand(Op, Result); 8199 return Result; 8200} 8201 8202// Utility function to find a legal vector type and its associated element 8203// type from a preferred width and whose vector type must be the same size 8204// as the VVT. 8205// TLI: Target lowering used to determine legal types 8206// Width: Preferred width of element type 8207// VVT: Vector value type whose size we must match. 8208// Returns VecEVT and EVT - the vector type and its associated element type 8209static void FindWidenVecType(TargetLowering &TLI, unsigned Width, MVT VVT, 8210 MVT& EVT, MVT& VecEVT) { 8211 // We start with the preferred width, make it a power of 2 and see if 8212 // we can find a vector type of that width. If not, we reduce it by 8213 // another power of 2. If we have widen the type, a vector of bytes should 8214 // always be legal. 8215 assert(TLI.isTypeLegal(VVT)); 8216 unsigned EWidth = Width + 1; 8217 do { 8218 assert(EWidth > 0); 8219 EWidth = (1 << Log2_32(EWidth-1)); 8220 EVT = MVT::getIntegerVT(EWidth); 8221 unsigned NumEVT = VVT.getSizeInBits()/EWidth; 8222 VecEVT = MVT::getVectorVT(EVT, NumEVT); 8223 } while (!TLI.isTypeLegal(VecEVT) || 8224 VVT.getSizeInBits() != VecEVT.getSizeInBits()); 8225} 8226 8227SDValue SelectionDAGLegalize::genWidenVectorLoads(SDValueVector& LdChain, 8228 SDValue Chain, 8229 SDValue BasePtr, 8230 const Value *SV, 8231 int SVOffset, 8232 unsigned Alignment, 8233 bool isVolatile, 8234 unsigned LdWidth, 8235 MVT ResType) { 8236 // We assume that we have good rules to handle loading power of two loads so 8237 // we break down the operations to power of 2 loads. The strategy is to 8238 // load the largest power of 2 that we can easily transform to a legal vector 8239 // and then insert into that vector, and the cast the result into the legal 8240 // vector that we want. This avoids unnecessary stack converts. 8241 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 8242 // the load is nonvolatile, we an use a wider load for the value. 8243 // Find a vector length we can load a large chunk 8244 MVT EVT, VecEVT; 8245 unsigned EVTWidth; 8246 FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); 8247 EVTWidth = EVT.getSizeInBits(); 8248 8249 SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV, SVOffset, 8250 isVolatile, Alignment); 8251 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecEVT, LdOp); 8252 LdChain.push_back(LdOp.getValue(1)); 8253 8254 // Check if we can load the element with one instruction 8255 if (LdWidth == EVTWidth) { 8256 return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp); 8257 } 8258 8259 // The vector element order is endianness dependent. 8260 unsigned Idx = 1; 8261 LdWidth -= EVTWidth; 8262 unsigned Offset = 0; 8263 8264 while (LdWidth > 0) { 8265 unsigned Increment = EVTWidth / 8; 8266 Offset += Increment; 8267 BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr, 8268 DAG.getIntPtrConstant(Increment)); 8269 8270 if (LdWidth < EVTWidth) { 8271 // Our current type we are using is too large, use a smaller size by 8272 // using a smaller power of 2 8273 unsigned oEVTWidth = EVTWidth; 8274 FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); 8275 EVTWidth = EVT.getSizeInBits(); 8276 // Readjust position and vector position based on new load type 8277 Idx = Idx * (oEVTWidth/EVTWidth); 8278 VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp); 8279 } 8280 8281 SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV, 8282 SVOffset+Offset, isVolatile, 8283 MinAlign(Alignment, Offset)); 8284 LdChain.push_back(LdOp.getValue(1)); 8285 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, VecEVT, VecOp, LdOp, 8286 DAG.getIntPtrConstant(Idx++)); 8287 8288 LdWidth -= EVTWidth; 8289 } 8290 8291 return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp); 8292} 8293 8294bool SelectionDAGLegalize::LoadWidenVectorOp(SDValue& Result, 8295 SDValue& TFOp, 8296 SDValue Op, 8297 MVT NVT) { 8298 // TODO: Add support for ConcatVec and the ability to load many vector 8299 // types (e.g., v4i8). This will not work when a vector register 8300 // to memory mapping is strange (e.g., vector elements are not 8301 // stored in some sequential order). 8302 8303 // It must be true that the widen vector type is bigger than where 8304 // we need to load from. 8305 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 8306 MVT LdVT = LD->getMemoryVT(); 8307 assert(LdVT.isVector() && NVT.isVector()); 8308 assert(LdVT.getVectorElementType() == NVT.getVectorElementType()); 8309 8310 // Load information 8311 SDValue Chain = LD->getChain(); 8312 SDValue BasePtr = LD->getBasePtr(); 8313 int SVOffset = LD->getSrcValueOffset(); 8314 unsigned Alignment = LD->getAlignment(); 8315 bool isVolatile = LD->isVolatile(); 8316 const Value *SV = LD->getSrcValue(); 8317 unsigned int LdWidth = LdVT.getSizeInBits(); 8318 8319 // Load value as a large register 8320 SDValueVector LdChain; 8321 Result = genWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 8322 Alignment, isVolatile, LdWidth, NVT); 8323 8324 if (LdChain.size() == 1) { 8325 TFOp = LdChain[0]; 8326 return true; 8327 } 8328 else { 8329 TFOp=DAG.getNode(ISD::TokenFactor, MVT::Other, &LdChain[0], LdChain.size()); 8330 return false; 8331 } 8332} 8333 8334 8335void SelectionDAGLegalize::genWidenVectorStores(SDValueVector& StChain, 8336 SDValue Chain, 8337 SDValue BasePtr, 8338 const Value *SV, 8339 int SVOffset, 8340 unsigned Alignment, 8341 bool isVolatile, 8342 SDValue ValOp, 8343 unsigned StWidth) { 8344 // Breaks the stores into a series of power of 2 width stores. For any 8345 // width, we convert the vector to the vector of element size that we 8346 // want to store. This avoids requiring a stack convert. 8347 8348 // Find a width of the element type we can store with 8349 MVT VVT = ValOp.getValueType(); 8350 MVT EVT, VecEVT; 8351 unsigned EVTWidth; 8352 FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); 8353 EVTWidth = EVT.getSizeInBits(); 8354 8355 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, ValOp); 8356 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp, 8357 DAG.getIntPtrConstant(0)); 8358 SDValue StOp = DAG.getStore(Chain, EOp, BasePtr, SV, SVOffset, 8359 isVolatile, Alignment); 8360 StChain.push_back(StOp); 8361 8362 // Check if we are done 8363 if (StWidth == EVTWidth) { 8364 return; 8365 } 8366 8367 unsigned Idx = 1; 8368 StWidth -= EVTWidth; 8369 unsigned Offset = 0; 8370 8371 while (StWidth > 0) { 8372 unsigned Increment = EVTWidth / 8; 8373 Offset += Increment; 8374 BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr, 8375 DAG.getIntPtrConstant(Increment)); 8376 8377 if (StWidth < EVTWidth) { 8378 // Our current type we are using is too large, use a smaller size by 8379 // using a smaller power of 2 8380 unsigned oEVTWidth = EVTWidth; 8381 FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); 8382 EVTWidth = EVT.getSizeInBits(); 8383 // Readjust position and vector position based on new load type 8384 Idx = Idx * (oEVTWidth/EVTWidth); 8385 VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp); 8386 } 8387 8388 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp, 8389 DAG.getIntPtrConstant(Idx++)); 8390 StChain.push_back(DAG.getStore(Chain, EOp, BasePtr, SV, 8391 SVOffset + Offset, isVolatile, 8392 MinAlign(Alignment, Offset))); 8393 StWidth -= EVTWidth; 8394 } 8395} 8396 8397 8398SDValue SelectionDAGLegalize::StoreWidenVectorOp(StoreSDNode *ST, 8399 SDValue Chain, 8400 SDValue BasePtr) { 8401 // TODO: It might be cleaner if we can use SplitVector and have more legal 8402 // vector types that can be stored into memory (e.g., v4xi8 can 8403 // be stored as a word). This will not work when a vector register 8404 // to memory mapping is strange (e.g., vector elements are not 8405 // stored in some sequential order). 8406 8407 MVT StVT = ST->getMemoryVT(); 8408 SDValue ValOp = ST->getValue(); 8409 8410 // Check if we have widen this node with another value 8411 std::map<SDValue, SDValue>::iterator I = WidenNodes.find(ValOp); 8412 if (I != WidenNodes.end()) 8413 ValOp = I->second; 8414 8415 MVT VVT = ValOp.getValueType(); 8416 8417 // It must be true that we the widen vector type is bigger than where 8418 // we need to store. 8419 assert(StVT.isVector() && VVT.isVector()); 8420 assert(StVT.getSizeInBits() < VVT.getSizeInBits()); 8421 assert(StVT.getVectorElementType() == VVT.getVectorElementType()); 8422 8423 // Store value 8424 SDValueVector StChain; 8425 genWidenVectorStores(StChain, Chain, BasePtr, ST->getSrcValue(), 8426 ST->getSrcValueOffset(), ST->getAlignment(), 8427 ST->isVolatile(), ValOp, StVT.getSizeInBits()); 8428 if (StChain.size() == 1) 8429 return StChain[0]; 8430 else 8431 return DAG.getNode(ISD::TokenFactor, MVT::Other,&StChain[0],StChain.size()); 8432} 8433 8434 8435// SelectionDAG::Legalize - This is the entry point for the file. 8436// 8437void SelectionDAG::Legalize() { 8438 /// run - This is the main entry point to this class. 8439 /// 8440 SelectionDAGLegalize(*this).LegalizeDAG(); 8441} 8442 8443