LegalizeDAG.cpp revision 33c960f523f2308482d5b2816af46a7ec90a6d3d
1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SelectionDAG::Legalize method. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "llvm/CodeGen/MachineFunction.h" 16#include "llvm/CodeGen/MachineFrameInfo.h" 17#include "llvm/CodeGen/MachineJumpTableInfo.h" 18#include "llvm/CodeGen/MachineModuleInfo.h" 19#include "llvm/CodeGen/DwarfWriter.h" 20#include "llvm/Analysis/DebugInfo.h" 21#include "llvm/CodeGen/PseudoSourceValue.h" 22#include "llvm/Target/TargetFrameInfo.h" 23#include "llvm/Target/TargetLowering.h" 24#include "llvm/Target/TargetData.h" 25#include "llvm/Target/TargetMachine.h" 26#include "llvm/Target/TargetOptions.h" 27#include "llvm/Target/TargetSubtarget.h" 28#include "llvm/CallingConv.h" 29#include "llvm/Constants.h" 30#include "llvm/DerivedTypes.h" 31#include "llvm/GlobalVariable.h" 32#include "llvm/Support/CommandLine.h" 33#include "llvm/Support/Compiler.h" 34#include "llvm/Support/MathExtras.h" 35#include "llvm/ADT/DenseMap.h" 36#include "llvm/ADT/SmallVector.h" 37#include "llvm/ADT/SmallPtrSet.h" 38#include <map> 39using namespace llvm; 40 41//===----------------------------------------------------------------------===// 42/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and 43/// hacks on it until the target machine can handle it. This involves 44/// eliminating value sizes the machine cannot handle (promoting small sizes to 45/// large sizes or splitting up large values into small values) as well as 46/// eliminating operations the machine cannot handle. 47/// 48/// This code also does a small amount of optimization and recognition of idioms 49/// as part of its processing. For example, if a target does not support a 50/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 51/// will attempt merge setcc and brc instructions into brcc's. 52/// 53namespace { 54class VISIBILITY_HIDDEN SelectionDAGLegalize { 55 TargetLowering &TLI; 56 SelectionDAG &DAG; 57 bool TypesNeedLegalizing; 58 59 // Libcall insertion helpers. 60 61 /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been 62 /// legalized. We use this to ensure that calls are properly serialized 63 /// against each other, including inserted libcalls. 64 SDValue LastCALLSEQ_END; 65 66 /// IsLegalizingCall - This member is used *only* for purposes of providing 67 /// helpful assertions that a libcall isn't created while another call is 68 /// being legalized (which could lead to non-serialized call sequences). 69 bool IsLegalizingCall; 70 71 /// IsLegalizingCallArguments - This member is used only for the purpose 72 /// of providing assert to check for LegalizeTypes because legalizing an 73 /// operation might introduce call nodes that might need type legalization. 74 bool IsLegalizingCallArgs; 75 76 enum LegalizeAction { 77 Legal, // The target natively supports this operation. 78 Promote, // This operation should be executed in a larger type. 79 Expand // Try to expand this to other ops, otherwise use a libcall. 80 }; 81 82 /// ValueTypeActions - This is a bitvector that contains two bits for each 83 /// value type, where the two bits correspond to the LegalizeAction enum. 84 /// This can be queried with "getTypeAction(VT)". 85 TargetLowering::ValueTypeActionImpl ValueTypeActions; 86 87 /// LegalizedNodes - For nodes that are of legal width, and that have more 88 /// than one use, this map indicates what regularized operand to use. This 89 /// allows us to avoid legalizing the same thing more than once. 90 DenseMap<SDValue, SDValue> LegalizedNodes; 91 92 /// PromotedNodes - For nodes that are below legal width, and that have more 93 /// than one use, this map indicates what promoted value to use. This allows 94 /// us to avoid promoting the same thing more than once. 95 DenseMap<SDValue, SDValue> PromotedNodes; 96 97 /// ExpandedNodes - For nodes that need to be expanded this map indicates 98 /// which operands are the expanded version of the input. This allows 99 /// us to avoid expanding the same node more than once. 100 DenseMap<SDValue, std::pair<SDValue, SDValue> > ExpandedNodes; 101 102 /// SplitNodes - For vector nodes that need to be split, this map indicates 103 /// which operands are the split version of the input. This allows us 104 /// to avoid splitting the same node more than once. 105 std::map<SDValue, std::pair<SDValue, SDValue> > SplitNodes; 106 107 /// ScalarizedNodes - For nodes that need to be converted from vector types to 108 /// scalar types, this contains the mapping of ones we have already 109 /// processed to the result. 110 std::map<SDValue, SDValue> ScalarizedNodes; 111 112 /// WidenNodes - For nodes that need to be widened from one vector type to 113 /// another, this contains the mapping of those that we have already widen. 114 /// This allows us to avoid widening more than once. 115 std::map<SDValue, SDValue> WidenNodes; 116 117 void AddLegalizedOperand(SDValue From, SDValue To) { 118 LegalizedNodes.insert(std::make_pair(From, To)); 119 // If someone requests legalization of the new node, return itself. 120 if (From != To) 121 LegalizedNodes.insert(std::make_pair(To, To)); 122 } 123 void AddPromotedOperand(SDValue From, SDValue To) { 124 bool isNew = PromotedNodes.insert(std::make_pair(From, To)).second; 125 assert(isNew && "Got into the map somehow?"); 126 isNew = isNew; 127 // If someone requests legalization of the new node, return itself. 128 LegalizedNodes.insert(std::make_pair(To, To)); 129 } 130 void AddWidenedOperand(SDValue From, SDValue To) { 131 bool isNew = WidenNodes.insert(std::make_pair(From, To)).second; 132 assert(isNew && "Got into the map somehow?"); 133 isNew = isNew; 134 // If someone requests legalization of the new node, return itself. 135 LegalizedNodes.insert(std::make_pair(To, To)); 136 } 137 138public: 139 explicit SelectionDAGLegalize(SelectionDAG &DAG, bool TypesNeedLegalizing); 140 141 /// getTypeAction - Return how we should legalize values of this type, either 142 /// it is already legal or we need to expand it into multiple registers of 143 /// smaller integer type, or we need to promote it to a larger type. 144 LegalizeAction getTypeAction(MVT VT) const { 145 return (LegalizeAction)ValueTypeActions.getTypeAction(VT); 146 } 147 148 /// isTypeLegal - Return true if this type is legal on this target. 149 /// 150 bool isTypeLegal(MVT VT) const { 151 return getTypeAction(VT) == Legal; 152 } 153 154 void LegalizeDAG(); 155 156private: 157 /// HandleOp - Legalize, Promote, or Expand the specified operand as 158 /// appropriate for its type. 159 void HandleOp(SDValue Op); 160 161 /// LegalizeOp - We know that the specified value has a legal type. 162 /// Recursively ensure that the operands have legal types, then return the 163 /// result. 164 SDValue LegalizeOp(SDValue O); 165 166 /// UnrollVectorOp - We know that the given vector has a legal type, however 167 /// the operation it performs is not legal and is an operation that we have 168 /// no way of lowering. "Unroll" the vector, splitting out the scalars and 169 /// operating on each element individually. 170 SDValue UnrollVectorOp(SDValue O); 171 172 /// PerformInsertVectorEltInMemory - Some target cannot handle a variable 173 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 174 /// is necessary to spill the vector being inserted into to memory, perform 175 /// the insert there, and then read the result back. 176 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, 177 SDValue Idx, DebugLoc dl); 178 179 /// PromoteOp - Given an operation that produces a value in an invalid type, 180 /// promote it to compute the value into a larger type. The produced value 181 /// will have the correct bits for the low portion of the register, but no 182 /// guarantee is made about the top bits: it may be zero, sign-extended, or 183 /// garbage. 184 SDValue PromoteOp(SDValue O); 185 186 /// ExpandOp - Expand the specified SDValue into its two component pieces 187 /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, 188 /// the LegalizedNodes map is filled in for any results that are not expanded, 189 /// the ExpandedNodes map is filled in for any results that are expanded, and 190 /// the Lo/Hi values are returned. This applies to integer types and Vector 191 /// types. 192 void ExpandOp(SDValue O, SDValue &Lo, SDValue &Hi); 193 194 /// WidenVectorOp - Widen a vector operation to a wider type given by WidenVT 195 /// (e.g., v3i32 to v4i32). The produced value will have the correct value 196 /// for the existing elements but no guarantee is made about the new elements 197 /// at the end of the vector: it may be zero, ones, or garbage. This is useful 198 /// when we have an instruction operating on an illegal vector type and we 199 /// want to widen it to do the computation on a legal wider vector type. 200 SDValue WidenVectorOp(SDValue Op, MVT WidenVT); 201 202 /// SplitVectorOp - Given an operand of vector type, break it down into 203 /// two smaller values. 204 void SplitVectorOp(SDValue O, SDValue &Lo, SDValue &Hi); 205 206 /// ScalarizeVectorOp - Given an operand of single-element vector type 207 /// (e.g. v1f32), convert it into the equivalent operation that returns a 208 /// scalar (e.g. f32) value. 209 SDValue ScalarizeVectorOp(SDValue O); 210 211 /// Useful 16 element vector type that is used to pass operands for widening. 212 typedef SmallVector<SDValue, 16> SDValueVector; 213 214 /// LoadWidenVectorOp - Load a vector for a wider type. Returns true if 215 /// the LdChain contains a single load and false if it contains a token 216 /// factor for multiple loads. It takes 217 /// Result: location to return the result 218 /// LdChain: location to return the load chain 219 /// Op: load operation to widen 220 /// NVT: widen vector result type we want for the load 221 bool LoadWidenVectorOp(SDValue& Result, SDValue& LdChain, 222 SDValue Op, MVT NVT); 223 224 /// Helper genWidenVectorLoads - Helper function to generate a set of 225 /// loads to load a vector with a resulting wider type. It takes 226 /// LdChain: list of chains for the load we have generated 227 /// Chain: incoming chain for the ld vector 228 /// BasePtr: base pointer to load from 229 /// SV: memory disambiguation source value 230 /// SVOffset: memory disambiugation offset 231 /// Alignment: alignment of the memory 232 /// isVolatile: volatile load 233 /// LdWidth: width of memory that we want to load 234 /// ResType: the wider result result type for the resulting loaded vector 235 SDValue genWidenVectorLoads(SDValueVector& LdChain, SDValue Chain, 236 SDValue BasePtr, const Value *SV, 237 int SVOffset, unsigned Alignment, 238 bool isVolatile, unsigned LdWidth, 239 MVT ResType, DebugLoc dl); 240 241 /// StoreWidenVectorOp - Stores a widen vector into non widen memory 242 /// location. It takes 243 /// ST: store node that we want to replace 244 /// Chain: incoming store chain 245 /// BasePtr: base address of where we want to store into 246 SDValue StoreWidenVectorOp(StoreSDNode *ST, SDValue Chain, 247 SDValue BasePtr); 248 249 /// Helper genWidenVectorStores - Helper function to generate a set of 250 /// stores to store a widen vector into non widen memory 251 // It takes 252 // StChain: list of chains for the stores we have generated 253 // Chain: incoming chain for the ld vector 254 // BasePtr: base pointer to load from 255 // SV: memory disambiguation source value 256 // SVOffset: memory disambiugation offset 257 // Alignment: alignment of the memory 258 // isVolatile: volatile lod 259 // ValOp: value to store 260 // StWidth: width of memory that we want to store 261 void genWidenVectorStores(SDValueVector& StChain, SDValue Chain, 262 SDValue BasePtr, const Value *SV, 263 int SVOffset, unsigned Alignment, 264 bool isVolatile, SDValue ValOp, 265 unsigned StWidth, DebugLoc dl); 266 267 /// isShuffleLegal - Return non-null if a vector shuffle is legal with the 268 /// specified mask and type. Targets can specify exactly which masks they 269 /// support and the code generator is tasked with not creating illegal masks. 270 /// 271 /// Note that this will also return true for shuffles that are promoted to a 272 /// different type. 273 /// 274 /// If this is a legal shuffle, this method returns the (possibly promoted) 275 /// build_vector Mask. If it's not a legal shuffle, it returns null. 276 SDNode *isShuffleLegal(MVT VT, SDValue Mask) const; 277 278 bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, 279 SmallPtrSet<SDNode*, 32> &NodesLeadingTo); 280 281 void LegalizeSetCCOperands(SDValue &LHS, SDValue &RHS, SDValue &CC, 282 DebugLoc dl); 283 void LegalizeSetCCCondCode(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 284 DebugLoc dl); 285 void LegalizeSetCC(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 286 DebugLoc dl) { 287 LegalizeSetCCOperands(LHS, RHS, CC, dl); 288 LegalizeSetCCCondCode(VT, LHS, RHS, CC, dl); 289 } 290 291 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned, 292 SDValue &Hi); 293 SDValue ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source, DebugLoc dl); 294 295 SDValue EmitStackConvert(SDValue SrcOp, MVT SlotVT, MVT DestVT, DebugLoc dl); 296 SDValue ExpandBUILD_VECTOR(SDNode *Node); 297 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 298 SDValue LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, 299 SDValue Op, DebugLoc dl); 300 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, MVT DestVT, 301 DebugLoc dl); 302 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, MVT DestVT, bool isSigned, 303 DebugLoc dl); 304 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, MVT DestVT, bool isSigned, 305 DebugLoc dl); 306 307 SDValue ExpandBSWAP(SDValue Op, DebugLoc dl); 308 SDValue ExpandBitCount(unsigned Opc, SDValue Op, DebugLoc dl); 309 bool ExpandShift(unsigned Opc, SDValue Op, SDValue Amt, 310 SDValue &Lo, SDValue &Hi, DebugLoc dl); 311 void ExpandShiftParts(unsigned NodeOp, SDValue Op, SDValue Amt, 312 SDValue &Lo, SDValue &Hi, DebugLoc dl); 313 314 SDValue ExpandEXTRACT_SUBVECTOR(SDValue Op); 315 SDValue ExpandEXTRACT_VECTOR_ELT(SDValue Op); 316}; 317} 318 319/// isVectorShuffleLegal - Return true if a vector shuffle is legal with the 320/// specified mask and type. Targets can specify exactly which masks they 321/// support and the code generator is tasked with not creating illegal masks. 322/// 323/// Note that this will also return true for shuffles that are promoted to a 324/// different type. 325SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDValue Mask) const { 326 switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) { 327 default: return 0; 328 case TargetLowering::Legal: 329 case TargetLowering::Custom: 330 break; 331 case TargetLowering::Promote: { 332 // If this is promoted to a different type, convert the shuffle mask and 333 // ask if it is legal in the promoted type! 334 MVT NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT); 335 MVT EltVT = NVT.getVectorElementType(); 336 337 // If we changed # elements, change the shuffle mask. 338 unsigned NumEltsGrowth = 339 NVT.getVectorNumElements() / VT.getVectorNumElements(); 340 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 341 if (NumEltsGrowth > 1) { 342 // Renumber the elements. 343 SmallVector<SDValue, 8> Ops; 344 for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) { 345 SDValue InOp = Mask.getOperand(i); 346 for (unsigned j = 0; j != NumEltsGrowth; ++j) { 347 if (InOp.getOpcode() == ISD::UNDEF) 348 Ops.push_back(DAG.getNode(ISD::UNDEF, 349 InOp.getNode()->getDebugLoc(), EltVT)); 350 else { 351 unsigned InEltNo = cast<ConstantSDNode>(InOp)->getZExtValue(); 352 Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, EltVT)); 353 } 354 } 355 } 356 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getNode()->getDebugLoc(), 357 NVT, &Ops[0], Ops.size()); 358 } 359 VT = NVT; 360 break; 361 } 362 } 363 return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.getNode() : 0; 364} 365 366SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag, bool types) 367 : TLI(dag.getTargetLoweringInfo()), DAG(dag), TypesNeedLegalizing(types), 368 ValueTypeActions(TLI.getValueTypeActions()) { 369 assert(MVT::LAST_VALUETYPE <= 32 && 370 "Too many value types for ValueTypeActions to hold!"); 371} 372 373void SelectionDAGLegalize::LegalizeDAG() { 374 LastCALLSEQ_END = DAG.getEntryNode(); 375 IsLegalizingCall = false; 376 IsLegalizingCallArgs = false; 377 378 // The legalize process is inherently a bottom-up recursive process (users 379 // legalize their uses before themselves). Given infinite stack space, we 380 // could just start legalizing on the root and traverse the whole graph. In 381 // practice however, this causes us to run out of stack space on large basic 382 // blocks. To avoid this problem, compute an ordering of the nodes where each 383 // node is only legalized after all of its operands are legalized. 384 DAG.AssignTopologicalOrder(); 385 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 386 E = prior(DAG.allnodes_end()); I != next(E); ++I) 387 HandleOp(SDValue(I, 0)); 388 389 // Finally, it's possible the root changed. Get the new root. 390 SDValue OldRoot = DAG.getRoot(); 391 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?"); 392 DAG.setRoot(LegalizedNodes[OldRoot]); 393 394 ExpandedNodes.clear(); 395 LegalizedNodes.clear(); 396 PromotedNodes.clear(); 397 SplitNodes.clear(); 398 ScalarizedNodes.clear(); 399 WidenNodes.clear(); 400 401 // Remove dead nodes now. 402 DAG.RemoveDeadNodes(); 403} 404 405 406/// FindCallEndFromCallStart - Given a chained node that is part of a call 407/// sequence, find the CALLSEQ_END node that terminates the call sequence. 408static SDNode *FindCallEndFromCallStart(SDNode *Node) { 409 if (Node->getOpcode() == ISD::CALLSEQ_END) 410 return Node; 411 if (Node->use_empty()) 412 return 0; // No CallSeqEnd 413 414 // The chain is usually at the end. 415 SDValue TheChain(Node, Node->getNumValues()-1); 416 if (TheChain.getValueType() != MVT::Other) { 417 // Sometimes it's at the beginning. 418 TheChain = SDValue(Node, 0); 419 if (TheChain.getValueType() != MVT::Other) { 420 // Otherwise, hunt for it. 421 for (unsigned i = 1, e = Node->getNumValues(); i != e; ++i) 422 if (Node->getValueType(i) == MVT::Other) { 423 TheChain = SDValue(Node, i); 424 break; 425 } 426 427 // Otherwise, we walked into a node without a chain. 428 if (TheChain.getValueType() != MVT::Other) 429 return 0; 430 } 431 } 432 433 for (SDNode::use_iterator UI = Node->use_begin(), 434 E = Node->use_end(); UI != E; ++UI) { 435 436 // Make sure to only follow users of our token chain. 437 SDNode *User = *UI; 438 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) 439 if (User->getOperand(i) == TheChain) 440 if (SDNode *Result = FindCallEndFromCallStart(User)) 441 return Result; 442 } 443 return 0; 444} 445 446/// FindCallStartFromCallEnd - Given a chained node that is part of a call 447/// sequence, find the CALLSEQ_START node that initiates the call sequence. 448static SDNode *FindCallStartFromCallEnd(SDNode *Node) { 449 assert(Node && "Didn't find callseq_start for a call??"); 450 if (Node->getOpcode() == ISD::CALLSEQ_START) return Node; 451 452 assert(Node->getOperand(0).getValueType() == MVT::Other && 453 "Node doesn't have a token chain argument!"); 454 return FindCallStartFromCallEnd(Node->getOperand(0).getNode()); 455} 456 457/// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to 458/// see if any uses can reach Dest. If no dest operands can get to dest, 459/// legalize them, legalize ourself, and return false, otherwise, return true. 460/// 461/// Keep track of the nodes we fine that actually do lead to Dest in 462/// NodesLeadingTo. This avoids retraversing them exponential number of times. 463/// 464bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, 465 SmallPtrSet<SDNode*, 32> &NodesLeadingTo) { 466 if (N == Dest) return true; // N certainly leads to Dest :) 467 468 // If we've already processed this node and it does lead to Dest, there is no 469 // need to reprocess it. 470 if (NodesLeadingTo.count(N)) return true; 471 472 // If the first result of this node has been already legalized, then it cannot 473 // reach N. 474 switch (getTypeAction(N->getValueType(0))) { 475 case Legal: 476 if (LegalizedNodes.count(SDValue(N, 0))) return false; 477 break; 478 case Promote: 479 if (PromotedNodes.count(SDValue(N, 0))) return false; 480 break; 481 case Expand: 482 if (ExpandedNodes.count(SDValue(N, 0))) return false; 483 break; 484 } 485 486 // Okay, this node has not already been legalized. Check and legalize all 487 // operands. If none lead to Dest, then we can legalize this node. 488 bool OperandsLeadToDest = false; 489 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 490 OperandsLeadToDest |= // If an operand leads to Dest, so do we. 491 LegalizeAllNodesNotLeadingTo(N->getOperand(i).getNode(), Dest, NodesLeadingTo); 492 493 if (OperandsLeadToDest) { 494 NodesLeadingTo.insert(N); 495 return true; 496 } 497 498 // Okay, this node looks safe, legalize it and return false. 499 HandleOp(SDValue(N, 0)); 500 return false; 501} 502 503/// HandleOp - Legalize, Promote, Widen, or Expand the specified operand as 504/// appropriate for its type. 505void SelectionDAGLegalize::HandleOp(SDValue Op) { 506 MVT VT = Op.getValueType(); 507 // If the type legalizer was run then we should never see any illegal result 508 // types here except for target constants (the type legalizer does not touch 509 // those) or for build vector used as a mask for a vector shuffle. 510 // FIXME: We can removed the BUILD_VECTOR case when we fix PR2957. 511 assert((TypesNeedLegalizing || getTypeAction(VT) == Legal || 512 IsLegalizingCallArgs || Op.getOpcode() == ISD::TargetConstant || 513 Op.getOpcode() == ISD::BUILD_VECTOR) && 514 "Illegal type introduced after type legalization?"); 515 switch (getTypeAction(VT)) { 516 default: assert(0 && "Bad type action!"); 517 case Legal: (void)LegalizeOp(Op); break; 518 case Promote: 519 if (!VT.isVector()) { 520 (void)PromoteOp(Op); 521 break; 522 } 523 else { 524 // See if we can widen otherwise use Expand to either scalarize or split 525 MVT WidenVT = TLI.getWidenVectorType(VT); 526 if (WidenVT != MVT::Other) { 527 (void) WidenVectorOp(Op, WidenVT); 528 break; 529 } 530 // else fall thru to expand since we can't widen the vector 531 } 532 case Expand: 533 if (!VT.isVector()) { 534 // If this is an illegal scalar, expand it into its two component 535 // pieces. 536 SDValue X, Y; 537 if (Op.getOpcode() == ISD::TargetConstant) 538 break; // Allow illegal target nodes. 539 ExpandOp(Op, X, Y); 540 } else if (VT.getVectorNumElements() == 1) { 541 // If this is an illegal single element vector, convert it to a 542 // scalar operation. 543 (void)ScalarizeVectorOp(Op); 544 } else { 545 // This is an illegal multiple element vector. 546 // Split it in half and legalize both parts. 547 SDValue X, Y; 548 SplitVectorOp(Op, X, Y); 549 } 550 break; 551 } 552} 553 554/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or 555/// a load from the constant pool. 556static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, 557 SelectionDAG &DAG, const TargetLowering &TLI) { 558 bool Extend = false; 559 DebugLoc dl = CFP->getDebugLoc(); 560 561 // If a FP immediate is precise when represented as a float and if the 562 // target can do an extending load from float to double, we put it into 563 // the constant pool as a float, even if it's is statically typed as a 564 // double. This shrinks FP constants and canonicalizes them for targets where 565 // an FP extending load is the same cost as a normal load (such as on the x87 566 // fp stack or PPC FP unit). 567 MVT VT = CFP->getValueType(0); 568 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 569 if (!UseCP) { 570 if (VT!=MVT::f64 && VT!=MVT::f32) 571 assert(0 && "Invalid type expansion"); 572 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), 573 (VT == MVT::f64) ? MVT::i64 : MVT::i32); 574 } 575 576 MVT OrigVT = VT; 577 MVT SVT = VT; 578 while (SVT != MVT::f32) { 579 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT() - 1); 580 if (CFP->isValueValidForType(SVT, CFP->getValueAPF()) && 581 // Only do this if the target has a native EXTLOAD instruction from 582 // smaller type. 583 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && 584 TLI.ShouldShrinkFPConstant(OrigVT)) { 585 const Type *SType = SVT.getTypeForMVT(); 586 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 587 VT = SVT; 588 Extend = true; 589 } 590 } 591 592 SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); 593 unsigned Alignment = 1 << cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 594 if (Extend) 595 return DAG.getExtLoad(ISD::EXTLOAD, dl, 596 OrigVT, DAG.getEntryNode(), 597 CPIdx, PseudoSourceValue::getConstantPool(), 598 0, VT, false, Alignment); 599 return DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx, 600 PseudoSourceValue::getConstantPool(), 0, false, Alignment); 601} 602 603 604/// ExpandFCOPYSIGNToBitwiseOps - Expands fcopysign to a series of bitwise 605/// operations. 606static 607SDValue ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, 608 SelectionDAG &DAG, 609 const TargetLowering &TLI) { 610 DebugLoc dl = Node->getDebugLoc(); 611 MVT VT = Node->getValueType(0); 612 MVT SrcVT = Node->getOperand(1).getValueType(); 613 assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) && 614 "fcopysign expansion only supported for f32 and f64"); 615 MVT SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32; 616 617 // First get the sign bit of second operand. 618 SDValue Mask1 = (SrcVT == MVT::f64) 619 ? DAG.getConstantFP(BitsToDouble(1ULL << 63), SrcVT) 620 : DAG.getConstantFP(BitsToFloat(1U << 31), SrcVT); 621 Mask1 = DAG.getNode(ISD::BIT_CONVERT, dl, SrcNVT, Mask1); 622 SDValue SignBit= DAG.getNode(ISD::BIT_CONVERT, dl, SrcNVT, 623 Node->getOperand(1)); 624 SignBit = DAG.getNode(ISD::AND, dl, SrcNVT, SignBit, Mask1); 625 // Shift right or sign-extend it if the two operands have different types. 626 int SizeDiff = SrcNVT.getSizeInBits() - NVT.getSizeInBits(); 627 if (SizeDiff > 0) { 628 SignBit = DAG.getNode(ISD::SRL, dl, SrcNVT, SignBit, 629 DAG.getConstant(SizeDiff, TLI.getShiftAmountTy())); 630 SignBit = DAG.getNode(ISD::TRUNCATE, dl, NVT, SignBit); 631 } else if (SizeDiff < 0) { 632 SignBit = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, SignBit); 633 SignBit = DAG.getNode(ISD::SHL, dl, NVT, SignBit, 634 DAG.getConstant(-SizeDiff, TLI.getShiftAmountTy())); 635 } 636 637 // Clear the sign bit of first operand. 638 SDValue Mask2 = (VT == MVT::f64) 639 ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) 640 : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); 641 Mask2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask2); 642 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); 643 Result = DAG.getNode(ISD::AND, dl, NVT, Result, Mask2); 644 645 // Or the value with the sign bit. 646 Result = DAG.getNode(ISD::OR, dl, NVT, Result, SignBit); 647 return Result; 648} 649 650/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. 651static 652SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, 653 const TargetLowering &TLI) { 654 SDValue Chain = ST->getChain(); 655 SDValue Ptr = ST->getBasePtr(); 656 SDValue Val = ST->getValue(); 657 MVT VT = Val.getValueType(); 658 int Alignment = ST->getAlignment(); 659 int SVOffset = ST->getSrcValueOffset(); 660 DebugLoc dl = ST->getDebugLoc(); 661 if (ST->getMemoryVT().isFloatingPoint() || 662 ST->getMemoryVT().isVector()) { 663 MVT intVT = MVT::getIntegerVT(VT.getSizeInBits()); 664 if (TLI.isTypeLegal(intVT)) { 665 // Expand to a bitconvert of the value to the integer type of the 666 // same size, then a (misaligned) int store. 667 // FIXME: Does not handle truncating floating point stores! 668 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, dl, intVT, Val); 669 return DAG.getStore(Chain, dl, Result, Ptr, ST->getSrcValue(), 670 SVOffset, ST->isVolatile(), Alignment); 671 } else { 672 // Do a (aligned) store to a stack slot, then copy from the stack slot 673 // to the final destination using (unaligned) integer loads and stores. 674 MVT StoredVT = ST->getMemoryVT(); 675 MVT RegVT = 676 TLI.getRegisterType(MVT::getIntegerVT(StoredVT.getSizeInBits())); 677 unsigned StoredBytes = StoredVT.getSizeInBits() / 8; 678 unsigned RegBytes = RegVT.getSizeInBits() / 8; 679 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; 680 681 // Make sure the stack slot is also aligned for the register type. 682 SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); 683 684 // Perform the original store, only redirected to the stack slot. 685 SDValue Store = DAG.getTruncStore(Chain, dl, 686 Val, StackPtr, NULL, 0,StoredVT); 687 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 688 SmallVector<SDValue, 8> Stores; 689 unsigned Offset = 0; 690 691 // Do all but one copies using the full register width. 692 for (unsigned i = 1; i < NumRegs; i++) { 693 // Load one integer register's worth from the stack slot. 694 SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr, NULL, 0); 695 // Store it to the final location. Remember the store. 696 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, 697 ST->getSrcValue(), SVOffset + Offset, 698 ST->isVolatile(), 699 MinAlign(ST->getAlignment(), Offset))); 700 // Increment the pointers. 701 Offset += RegBytes; 702 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 703 Increment); 704 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 705 } 706 707 // The last store may be partial. Do a truncating store. On big-endian 708 // machines this requires an extending load from the stack slot to ensure 709 // that the bits are in the right place. 710 MVT MemVT = MVT::getIntegerVT(8 * (StoredBytes - Offset)); 711 712 // Load from the stack slot. 713 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, 714 NULL, 0, MemVT); 715 716 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, 717 ST->getSrcValue(), SVOffset + Offset, 718 MemVT, ST->isVolatile(), 719 MinAlign(ST->getAlignment(), Offset))); 720 // The order of the stores doesn't matter - say it with a TokenFactor. 721 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 722 Stores.size()); 723 } 724 } 725 assert(ST->getMemoryVT().isInteger() && 726 !ST->getMemoryVT().isVector() && 727 "Unaligned store of unknown type."); 728 // Get the half-size VT 729 MVT NewStoredVT = 730 (MVT::SimpleValueType)(ST->getMemoryVT().getSimpleVT() - 1); 731 int NumBits = NewStoredVT.getSizeInBits(); 732 int IncrementSize = NumBits / 8; 733 734 // Divide the stored value in two parts. 735 SDValue ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); 736 SDValue Lo = Val; 737 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); 738 739 // Store the two parts 740 SDValue Store1, Store2; 741 Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr, 742 ST->getSrcValue(), SVOffset, NewStoredVT, 743 ST->isVolatile(), Alignment); 744 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 745 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 746 Alignment = MinAlign(Alignment, IncrementSize); 747 Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr, 748 ST->getSrcValue(), SVOffset + IncrementSize, 749 NewStoredVT, ST->isVolatile(), Alignment); 750 751 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); 752} 753 754/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. 755static 756SDValue ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, 757 const TargetLowering &TLI) { 758 int SVOffset = LD->getSrcValueOffset(); 759 SDValue Chain = LD->getChain(); 760 SDValue Ptr = LD->getBasePtr(); 761 MVT VT = LD->getValueType(0); 762 MVT LoadedVT = LD->getMemoryVT(); 763 DebugLoc dl = LD->getDebugLoc(); 764 if (VT.isFloatingPoint() || VT.isVector()) { 765 MVT intVT = MVT::getIntegerVT(LoadedVT.getSizeInBits()); 766 if (TLI.isTypeLegal(intVT)) { 767 // Expand to a (misaligned) integer load of the same size, 768 // then bitconvert to floating point or vector. 769 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, LD->getSrcValue(), 770 SVOffset, LD->isVolatile(), 771 LD->getAlignment()); 772 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, dl, LoadedVT, newLoad); 773 if (VT.isFloatingPoint() && LoadedVT != VT) 774 Result = DAG.getNode(ISD::FP_EXTEND, dl, VT, Result); 775 776 SDValue Ops[] = { Result, Chain }; 777 return DAG.getMergeValues(Ops, 2, dl); 778 } else { 779 // Copy the value to a (aligned) stack slot using (unaligned) integer 780 // loads and stores, then do a (aligned) load from the stack slot. 781 MVT RegVT = TLI.getRegisterType(intVT); 782 unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; 783 unsigned RegBytes = RegVT.getSizeInBits() / 8; 784 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; 785 786 // Make sure the stack slot is also aligned for the register type. 787 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT); 788 789 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 790 SmallVector<SDValue, 8> Stores; 791 SDValue StackPtr = StackBase; 792 unsigned Offset = 0; 793 794 // Do all but one copies using the full register width. 795 for (unsigned i = 1; i < NumRegs; i++) { 796 // Load one integer register's worth from the original location. 797 SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr, LD->getSrcValue(), 798 SVOffset + Offset, LD->isVolatile(), 799 MinAlign(LD->getAlignment(), Offset)); 800 // Follow the load with a store to the stack slot. Remember the store. 801 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr, 802 NULL, 0)); 803 // Increment the pointers. 804 Offset += RegBytes; 805 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 806 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 807 Increment); 808 } 809 810 // The last copy may be partial. Do an extending load. 811 MVT MemVT = MVT::getIntegerVT(8 * (LoadedBytes - Offset)); 812 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, 813 LD->getSrcValue(), SVOffset + Offset, 814 MemVT, LD->isVolatile(), 815 MinAlign(LD->getAlignment(), Offset)); 816 // Follow the load with a store to the stack slot. Remember the store. 817 // On big-endian machines this requires a truncating store to ensure 818 // that the bits end up in the right place. 819 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr, 820 NULL, 0, MemVT)); 821 822 // The order of the stores doesn't matter - say it with a TokenFactor. 823 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 824 Stores.size()); 825 826 // Finally, perform the original load only redirected to the stack slot. 827 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, 828 NULL, 0, LoadedVT); 829 830 // Callers expect a MERGE_VALUES node. 831 SDValue Ops[] = { Load, TF }; 832 return DAG.getMergeValues(Ops, 2, dl); 833 } 834 } 835 assert(LoadedVT.isInteger() && !LoadedVT.isVector() && 836 "Unaligned load of unsupported type."); 837 838 // Compute the new VT that is half the size of the old one. This is an 839 // integer MVT. 840 unsigned NumBits = LoadedVT.getSizeInBits(); 841 MVT NewLoadedVT; 842 NewLoadedVT = MVT::getIntegerVT(NumBits/2); 843 NumBits >>= 1; 844 845 unsigned Alignment = LD->getAlignment(); 846 unsigned IncrementSize = NumBits / 8; 847 ISD::LoadExtType HiExtType = LD->getExtensionType(); 848 849 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. 850 if (HiExtType == ISD::NON_EXTLOAD) 851 HiExtType = ISD::ZEXTLOAD; 852 853 // Load the value in two parts 854 SDValue Lo, Hi; 855 if (TLI.isLittleEndian()) { 856 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getSrcValue(), 857 SVOffset, NewLoadedVT, LD->isVolatile(), Alignment); 858 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 859 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 860 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getSrcValue(), 861 SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(), 862 MinAlign(Alignment, IncrementSize)); 863 } else { 864 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getSrcValue(), 865 SVOffset, NewLoadedVT,LD->isVolatile(), Alignment); 866 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 867 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 868 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getSrcValue(), 869 SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(), 870 MinAlign(Alignment, IncrementSize)); 871 } 872 873 // aggregate the two parts 874 SDValue ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); 875 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); 876 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); 877 878 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 879 Hi.getValue(1)); 880 881 SDValue Ops[] = { Result, TF }; 882 return DAG.getMergeValues(Ops, 2, dl); 883} 884 885/// UnrollVectorOp - We know that the given vector has a legal type, however 886/// the operation it performs is not legal and is an operation that we have 887/// no way of lowering. "Unroll" the vector, splitting out the scalars and 888/// operating on each element individually. 889SDValue SelectionDAGLegalize::UnrollVectorOp(SDValue Op) { 890 MVT VT = Op.getValueType(); 891 assert(isTypeLegal(VT) && 892 "Caller should expand or promote operands that are not legal!"); 893 assert(Op.getNode()->getNumValues() == 1 && 894 "Can't unroll a vector with multiple results!"); 895 unsigned NE = VT.getVectorNumElements(); 896 MVT EltVT = VT.getVectorElementType(); 897 DebugLoc dl = Op.getNode()->getDebugLoc(); 898 899 SmallVector<SDValue, 8> Scalars; 900 SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 901 for (unsigned i = 0; i != NE; ++i) { 902 for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 903 SDValue Operand = Op.getOperand(j); 904 MVT OperandVT = Operand.getValueType(); 905 if (OperandVT.isVector()) { 906 // A vector operand; extract a single element. 907 MVT OperandEltVT = OperandVT.getVectorElementType(); 908 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 909 OperandEltVT, 910 Operand, 911 DAG.getConstant(i, MVT::i32)); 912 } else { 913 // A scalar operand; just use it as is. 914 Operands[j] = Operand; 915 } 916 } 917 918 switch (Op.getOpcode()) { 919 default: 920 Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, 921 &Operands[0], Operands.size())); 922 break; 923 case ISD::SHL: 924 case ISD::SRA: 925 case ISD::SRL: 926 case ISD::ROTL: 927 case ISD::ROTR: 928 Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, Operands[0], 929 DAG.getShiftAmountOperand(Operands[1]))); 930 break; 931 } 932 } 933 934 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Scalars[0], Scalars.size()); 935} 936 937/// GetFPLibCall - Return the right libcall for the given floating point type. 938static RTLIB::Libcall GetFPLibCall(MVT VT, 939 RTLIB::Libcall Call_F32, 940 RTLIB::Libcall Call_F64, 941 RTLIB::Libcall Call_F80, 942 RTLIB::Libcall Call_PPCF128) { 943 return 944 VT == MVT::f32 ? Call_F32 : 945 VT == MVT::f64 ? Call_F64 : 946 VT == MVT::f80 ? Call_F80 : 947 VT == MVT::ppcf128 ? Call_PPCF128 : 948 RTLIB::UNKNOWN_LIBCALL; 949} 950 951/// PerformInsertVectorEltInMemory - Some target cannot handle a variable 952/// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 953/// is necessary to spill the vector being inserted into to memory, perform 954/// the insert there, and then read the result back. 955SDValue SelectionDAGLegalize:: 956PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 957 DebugLoc dl) { 958 SDValue Tmp1 = Vec; 959 SDValue Tmp2 = Val; 960 SDValue Tmp3 = Idx; 961 962 // If the target doesn't support this, we have to spill the input vector 963 // to a temporary stack slot, update the element, then reload it. This is 964 // badness. We could also load the value into a vector register (either 965 // with a "move to register" or "extload into register" instruction, then 966 // permute it into place, if the idx is a constant and if the idx is 967 // supported by the target. 968 MVT VT = Tmp1.getValueType(); 969 MVT EltVT = VT.getVectorElementType(); 970 MVT IdxVT = Tmp3.getValueType(); 971 MVT PtrVT = TLI.getPointerTy(); 972 SDValue StackPtr = DAG.CreateStackTemporary(VT); 973 974 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 975 976 // Store the vector. 977 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr, 978 PseudoSourceValue::getFixedStack(SPFI), 0); 979 980 // Truncate or zero extend offset to target pointer type. 981 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 982 Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3); 983 // Add the offset to the index. 984 unsigned EltSize = EltVT.getSizeInBits()/8; 985 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); 986 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); 987 // Store the scalar value. 988 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, 989 PseudoSourceValue::getFixedStack(SPFI), 0, EltVT); 990 // Load the updated vector. 991 return DAG.getLoad(VT, dl, Ch, StackPtr, 992 PseudoSourceValue::getFixedStack(SPFI), 0); 993} 994 995 996/// LegalizeOp - We know that the specified value has a legal type, and 997/// that its operands are legal. Now ensure that the operation itself 998/// is legal, recursively ensuring that the operands' operations remain 999/// legal. 1000SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { 1001 if (Op.getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. 1002 return Op; 1003 1004 assert(isTypeLegal(Op.getValueType()) && 1005 "Caller should expand or promote operands that are not legal!"); 1006 SDNode *Node = Op.getNode(); 1007 DebugLoc dl = Node->getDebugLoc(); 1008 1009 // If this operation defines any values that cannot be represented in a 1010 // register on this target, make sure to expand or promote them. 1011 if (Node->getNumValues() > 1) { 1012 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1013 if (getTypeAction(Node->getValueType(i)) != Legal) { 1014 HandleOp(Op.getValue(i)); 1015 assert(LegalizedNodes.count(Op) && 1016 "Handling didn't add legal operands!"); 1017 return LegalizedNodes[Op]; 1018 } 1019 } 1020 1021 // Note that LegalizeOp may be reentered even from single-use nodes, which 1022 // means that we always must cache transformed nodes. 1023 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 1024 if (I != LegalizedNodes.end()) return I->second; 1025 1026 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 1027 SDValue Result = Op; 1028 bool isCustom = false; 1029 1030 switch (Node->getOpcode()) { 1031 case ISD::FrameIndex: 1032 case ISD::EntryToken: 1033 case ISD::Register: 1034 case ISD::BasicBlock: 1035 case ISD::TargetFrameIndex: 1036 case ISD::TargetJumpTable: 1037 case ISD::TargetConstant: 1038 case ISD::TargetConstantFP: 1039 case ISD::TargetConstantPool: 1040 case ISD::TargetGlobalAddress: 1041 case ISD::TargetGlobalTLSAddress: 1042 case ISD::TargetExternalSymbol: 1043 case ISD::VALUETYPE: 1044 case ISD::SRCVALUE: 1045 case ISD::MEMOPERAND: 1046 case ISD::CONDCODE: 1047 case ISD::ARG_FLAGS: 1048 // Primitives must all be legal. 1049 assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) && 1050 "This must be legal!"); 1051 break; 1052 default: 1053 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 1054 // If this is a target node, legalize it by legalizing the operands then 1055 // passing it through. 1056 SmallVector<SDValue, 8> Ops; 1057 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1058 Ops.push_back(LegalizeOp(Node->getOperand(i))); 1059 1060 Result = DAG.UpdateNodeOperands(Result.getValue(0), &Ops[0], Ops.size()); 1061 1062 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1063 AddLegalizedOperand(Op.getValue(i), Result.getValue(i)); 1064 return Result.getValue(Op.getResNo()); 1065 } 1066 // Otherwise this is an unhandled builtin node. splat. 1067#ifndef NDEBUG 1068 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 1069#endif 1070 assert(0 && "Do not know how to legalize this operator!"); 1071 abort(); 1072 case ISD::GLOBAL_OFFSET_TABLE: 1073 case ISD::GlobalAddress: 1074 case ISD::GlobalTLSAddress: 1075 case ISD::ExternalSymbol: 1076 case ISD::ConstantPool: 1077 case ISD::JumpTable: // Nothing to do. 1078 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 1079 default: assert(0 && "This action is not supported yet!"); 1080 case TargetLowering::Custom: 1081 Tmp1 = TLI.LowerOperation(Op, DAG); 1082 if (Tmp1.getNode()) Result = Tmp1; 1083 // FALLTHROUGH if the target doesn't want to lower this op after all. 1084 case TargetLowering::Legal: 1085 break; 1086 } 1087 break; 1088 case ISD::FRAMEADDR: 1089 case ISD::RETURNADDR: 1090 // The only option for these nodes is to custom lower them. If the target 1091 // does not custom lower them, then return zero. 1092 Tmp1 = TLI.LowerOperation(Op, DAG); 1093 if (Tmp1.getNode()) 1094 Result = Tmp1; 1095 else 1096 Result = DAG.getConstant(0, TLI.getPointerTy()); 1097 break; 1098 case ISD::FRAME_TO_ARGS_OFFSET: { 1099 MVT VT = Node->getValueType(0); 1100 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1101 default: assert(0 && "This action is not supported yet!"); 1102 case TargetLowering::Custom: 1103 Result = TLI.LowerOperation(Op, DAG); 1104 if (Result.getNode()) break; 1105 // Fall Thru 1106 case TargetLowering::Legal: 1107 Result = DAG.getConstant(0, VT); 1108 break; 1109 } 1110 } 1111 break; 1112 case ISD::EXCEPTIONADDR: { 1113 Tmp1 = LegalizeOp(Node->getOperand(0)); 1114 MVT VT = Node->getValueType(0); 1115 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1116 default: assert(0 && "This action is not supported yet!"); 1117 case TargetLowering::Expand: { 1118 unsigned Reg = TLI.getExceptionAddressRegister(); 1119 Result = DAG.getCopyFromReg(Tmp1, dl, Reg, VT); 1120 } 1121 break; 1122 case TargetLowering::Custom: 1123 Result = TLI.LowerOperation(Op, DAG); 1124 if (Result.getNode()) break; 1125 // Fall Thru 1126 case TargetLowering::Legal: { 1127 SDValue Ops[] = { DAG.getConstant(0, VT), Tmp1 }; 1128 Result = DAG.getMergeValues(Ops, 2, dl); 1129 break; 1130 } 1131 } 1132 } 1133 if (Result.getNode()->getNumValues() == 1) break; 1134 1135 assert(Result.getNode()->getNumValues() == 2 && 1136 "Cannot return more than two values!"); 1137 1138 // Since we produced two values, make sure to remember that we 1139 // legalized both of them. 1140 Tmp1 = LegalizeOp(Result); 1141 Tmp2 = LegalizeOp(Result.getValue(1)); 1142 AddLegalizedOperand(Op.getValue(0), Tmp1); 1143 AddLegalizedOperand(Op.getValue(1), Tmp2); 1144 return Op.getResNo() ? Tmp2 : Tmp1; 1145 case ISD::EHSELECTION: { 1146 Tmp1 = LegalizeOp(Node->getOperand(0)); 1147 Tmp2 = LegalizeOp(Node->getOperand(1)); 1148 MVT VT = Node->getValueType(0); 1149 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1150 default: assert(0 && "This action is not supported yet!"); 1151 case TargetLowering::Expand: { 1152 unsigned Reg = TLI.getExceptionSelectorRegister(); 1153 Result = DAG.getCopyFromReg(Tmp2, dl, Reg, VT); 1154 } 1155 break; 1156 case TargetLowering::Custom: 1157 Result = TLI.LowerOperation(Op, DAG); 1158 if (Result.getNode()) break; 1159 // Fall Thru 1160 case TargetLowering::Legal: { 1161 SDValue Ops[] = { DAG.getConstant(0, VT), Tmp2 }; 1162 Result = DAG.getMergeValues(Ops, 2, dl); 1163 break; 1164 } 1165 } 1166 } 1167 if (Result.getNode()->getNumValues() == 1) break; 1168 1169 assert(Result.getNode()->getNumValues() == 2 && 1170 "Cannot return more than two values!"); 1171 1172 // Since we produced two values, make sure to remember that we 1173 // legalized both of them. 1174 Tmp1 = LegalizeOp(Result); 1175 Tmp2 = LegalizeOp(Result.getValue(1)); 1176 AddLegalizedOperand(Op.getValue(0), Tmp1); 1177 AddLegalizedOperand(Op.getValue(1), Tmp2); 1178 return Op.getResNo() ? Tmp2 : Tmp1; 1179 case ISD::EH_RETURN: { 1180 MVT VT = Node->getValueType(0); 1181 // The only "good" option for this node is to custom lower it. 1182 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1183 default: assert(0 && "This action is not supported at all!"); 1184 case TargetLowering::Custom: 1185 Result = TLI.LowerOperation(Op, DAG); 1186 if (Result.getNode()) break; 1187 // Fall Thru 1188 case TargetLowering::Legal: 1189 // Target does not know, how to lower this, lower to noop 1190 Result = LegalizeOp(Node->getOperand(0)); 1191 break; 1192 } 1193 } 1194 break; 1195 case ISD::AssertSext: 1196 case ISD::AssertZext: 1197 Tmp1 = LegalizeOp(Node->getOperand(0)); 1198 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1199 break; 1200 case ISD::MERGE_VALUES: 1201 // Legalize eliminates MERGE_VALUES nodes. 1202 Result = Node->getOperand(Op.getResNo()); 1203 break; 1204 case ISD::CopyFromReg: 1205 Tmp1 = LegalizeOp(Node->getOperand(0)); 1206 Result = Op.getValue(0); 1207 if (Node->getNumValues() == 2) { 1208 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1209 } else { 1210 assert(Node->getNumValues() == 3 && "Invalid copyfromreg!"); 1211 if (Node->getNumOperands() == 3) { 1212 Tmp2 = LegalizeOp(Node->getOperand(2)); 1213 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2); 1214 } else { 1215 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1216 } 1217 AddLegalizedOperand(Op.getValue(2), Result.getValue(2)); 1218 } 1219 // Since CopyFromReg produces two values, make sure to remember that we 1220 // legalized both of them. 1221 AddLegalizedOperand(Op.getValue(0), Result); 1222 AddLegalizedOperand(Op.getValue(1), Result.getValue(1)); 1223 return Result.getValue(Op.getResNo()); 1224 case ISD::UNDEF: { 1225 MVT VT = Op.getValueType(); 1226 switch (TLI.getOperationAction(ISD::UNDEF, VT)) { 1227 default: assert(0 && "This action is not supported yet!"); 1228 case TargetLowering::Expand: 1229 if (VT.isInteger()) 1230 Result = DAG.getConstant(0, VT); 1231 else if (VT.isFloatingPoint()) 1232 Result = DAG.getConstantFP(APFloat(APInt(VT.getSizeInBits(), 0)), 1233 VT); 1234 else 1235 assert(0 && "Unknown value type!"); 1236 break; 1237 case TargetLowering::Legal: 1238 break; 1239 } 1240 break; 1241 } 1242 1243 case ISD::INTRINSIC_W_CHAIN: 1244 case ISD::INTRINSIC_WO_CHAIN: 1245 case ISD::INTRINSIC_VOID: { 1246 SmallVector<SDValue, 8> Ops; 1247 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1248 Ops.push_back(LegalizeOp(Node->getOperand(i))); 1249 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1250 1251 // Allow the target to custom lower its intrinsics if it wants to. 1252 if (TLI.getOperationAction(Node->getOpcode(), MVT::Other) == 1253 TargetLowering::Custom) { 1254 Tmp3 = TLI.LowerOperation(Result, DAG); 1255 if (Tmp3.getNode()) Result = Tmp3; 1256 } 1257 1258 if (Result.getNode()->getNumValues() == 1) break; 1259 1260 // Must have return value and chain result. 1261 assert(Result.getNode()->getNumValues() == 2 && 1262 "Cannot return more than two values!"); 1263 1264 // Since loads produce two values, make sure to remember that we 1265 // legalized both of them. 1266 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1267 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1268 return Result.getValue(Op.getResNo()); 1269 } 1270 1271 case ISD::DBG_STOPPOINT: 1272 assert(Node->getNumOperands() == 1 && "Invalid DBG_STOPPOINT node!"); 1273 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input chain. 1274 1275 switch (TLI.getOperationAction(ISD::DBG_STOPPOINT, MVT::Other)) { 1276 case TargetLowering::Promote: 1277 default: assert(0 && "This action is not supported yet!"); 1278 case TargetLowering::Expand: { 1279 DwarfWriter *DW = DAG.getDwarfWriter(); 1280 bool useDEBUG_LOC = TLI.isOperationLegalOrCustom(ISD::DEBUG_LOC, 1281 MVT::Other); 1282 bool useLABEL = TLI.isOperationLegalOrCustom(ISD::DBG_LABEL, MVT::Other); 1283 1284 const DbgStopPointSDNode *DSP = cast<DbgStopPointSDNode>(Node); 1285 GlobalVariable *CU_GV = cast<GlobalVariable>(DSP->getCompileUnit()); 1286 if (DW && (useDEBUG_LOC || useLABEL) && !CU_GV->isDeclaration()) { 1287 DICompileUnit CU(cast<GlobalVariable>(DSP->getCompileUnit())); 1288 unsigned SrcFile = DW->RecordSource(CU.getDirectory(), 1289 CU.getFilename()); 1290 1291 unsigned Line = DSP->getLine(); 1292 unsigned Col = DSP->getColumn(); 1293 1294 // A bit self-referential to have DebugLoc on Debug_Loc nodes, but 1295 // it won't hurt anything. 1296 if (useDEBUG_LOC) { 1297 SDValue Ops[] = { Tmp1, DAG.getConstant(Line, MVT::i32), 1298 DAG.getConstant(Col, MVT::i32), 1299 DAG.getConstant(SrcFile, MVT::i32) }; 1300 Result = DAG.getNode(ISD::DEBUG_LOC, dl, MVT::Other, Ops, 4); 1301 } else { 1302 unsigned ID = DW->RecordSourceLine(Line, Col, SrcFile); 1303 Result = DAG.getLabel(ISD::DBG_LABEL, dl, Tmp1, ID); 1304 } 1305 } else { 1306 Result = Tmp1; // chain 1307 } 1308 break; 1309 } 1310 case TargetLowering::Legal: { 1311 LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); 1312 if (Action == Legal && Tmp1 == Node->getOperand(0)) 1313 break; 1314 1315 SmallVector<SDValue, 8> Ops; 1316 Ops.push_back(Tmp1); 1317 if (Action == Legal) { 1318 Ops.push_back(Node->getOperand(1)); // line # must be legal. 1319 Ops.push_back(Node->getOperand(2)); // col # must be legal. 1320 } else { 1321 // Otherwise promote them. 1322 Ops.push_back(PromoteOp(Node->getOperand(1))); 1323 Ops.push_back(PromoteOp(Node->getOperand(2))); 1324 } 1325 Ops.push_back(Node->getOperand(3)); // filename must be legal. 1326 Ops.push_back(Node->getOperand(4)); // working dir # must be legal. 1327 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1328 break; 1329 } 1330 } 1331 break; 1332 1333 case ISD::DECLARE: 1334 assert(Node->getNumOperands() == 3 && "Invalid DECLARE node!"); 1335 switch (TLI.getOperationAction(ISD::DECLARE, MVT::Other)) { 1336 default: assert(0 && "This action is not supported yet!"); 1337 case TargetLowering::Legal: 1338 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1339 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the address. 1340 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the variable. 1341 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1342 break; 1343 case TargetLowering::Expand: 1344 Result = LegalizeOp(Node->getOperand(0)); 1345 break; 1346 } 1347 break; 1348 1349 case ISD::DEBUG_LOC: 1350 assert(Node->getNumOperands() == 4 && "Invalid DEBUG_LOC node!"); 1351 switch (TLI.getOperationAction(ISD::DEBUG_LOC, MVT::Other)) { 1352 default: assert(0 && "This action is not supported yet!"); 1353 case TargetLowering::Legal: { 1354 LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); 1355 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1356 if (Action == Legal && Tmp1 == Node->getOperand(0)) 1357 break; 1358 if (Action == Legal) { 1359 Tmp2 = Node->getOperand(1); 1360 Tmp3 = Node->getOperand(2); 1361 Tmp4 = Node->getOperand(3); 1362 } else { 1363 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the line #. 1364 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the col #. 1365 Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize the source file id. 1366 } 1367 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4); 1368 break; 1369 } 1370 } 1371 break; 1372 1373 case ISD::DBG_LABEL: 1374 case ISD::EH_LABEL: 1375 assert(Node->getNumOperands() == 1 && "Invalid LABEL node!"); 1376 switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) { 1377 default: assert(0 && "This action is not supported yet!"); 1378 case TargetLowering::Legal: 1379 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1380 Result = DAG.UpdateNodeOperands(Result, Tmp1); 1381 break; 1382 case TargetLowering::Expand: 1383 Result = LegalizeOp(Node->getOperand(0)); 1384 break; 1385 } 1386 break; 1387 1388 case ISD::PREFETCH: 1389 assert(Node->getNumOperands() == 4 && "Invalid Prefetch node!"); 1390 switch (TLI.getOperationAction(ISD::PREFETCH, MVT::Other)) { 1391 default: assert(0 && "This action is not supported yet!"); 1392 case TargetLowering::Legal: 1393 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1394 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the address. 1395 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the rw specifier. 1396 Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize locality specifier. 1397 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4); 1398 break; 1399 case TargetLowering::Expand: 1400 // It's a noop. 1401 Result = LegalizeOp(Node->getOperand(0)); 1402 break; 1403 } 1404 break; 1405 1406 case ISD::MEMBARRIER: { 1407 assert(Node->getNumOperands() == 6 && "Invalid MemBarrier node!"); 1408 switch (TLI.getOperationAction(ISD::MEMBARRIER, MVT::Other)) { 1409 default: assert(0 && "This action is not supported yet!"); 1410 case TargetLowering::Legal: { 1411 SDValue Ops[6]; 1412 Ops[0] = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1413 for (int x = 1; x < 6; ++x) { 1414 Ops[x] = Node->getOperand(x); 1415 if (!isTypeLegal(Ops[x].getValueType())) 1416 Ops[x] = PromoteOp(Ops[x]); 1417 } 1418 Result = DAG.UpdateNodeOperands(Result, &Ops[0], 6); 1419 break; 1420 } 1421 case TargetLowering::Expand: 1422 //There is no libgcc call for this op 1423 Result = Node->getOperand(0); // Noop 1424 break; 1425 } 1426 break; 1427 } 1428 1429 case ISD::ATOMIC_CMP_SWAP: { 1430 unsigned int num_operands = 4; 1431 assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!"); 1432 SDValue Ops[4]; 1433 for (unsigned int x = 0; x < num_operands; ++x) 1434 Ops[x] = LegalizeOp(Node->getOperand(x)); 1435 Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands); 1436 1437 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 1438 default: assert(0 && "This action is not supported yet!"); 1439 case TargetLowering::Custom: 1440 Result = TLI.LowerOperation(Result, DAG); 1441 break; 1442 case TargetLowering::Legal: 1443 break; 1444 } 1445 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1446 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1447 return Result.getValue(Op.getResNo()); 1448 } 1449 case ISD::ATOMIC_LOAD_ADD: 1450 case ISD::ATOMIC_LOAD_SUB: 1451 case ISD::ATOMIC_LOAD_AND: 1452 case ISD::ATOMIC_LOAD_OR: 1453 case ISD::ATOMIC_LOAD_XOR: 1454 case ISD::ATOMIC_LOAD_NAND: 1455 case ISD::ATOMIC_LOAD_MIN: 1456 case ISD::ATOMIC_LOAD_MAX: 1457 case ISD::ATOMIC_LOAD_UMIN: 1458 case ISD::ATOMIC_LOAD_UMAX: 1459 case ISD::ATOMIC_SWAP: { 1460 unsigned int num_operands = 3; 1461 assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!"); 1462 SDValue Ops[3]; 1463 for (unsigned int x = 0; x < num_operands; ++x) 1464 Ops[x] = LegalizeOp(Node->getOperand(x)); 1465 Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands); 1466 1467 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 1468 default: assert(0 && "This action is not supported yet!"); 1469 case TargetLowering::Custom: 1470 Result = TLI.LowerOperation(Result, DAG); 1471 break; 1472 case TargetLowering::Legal: 1473 break; 1474 } 1475 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1476 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1477 return Result.getValue(Op.getResNo()); 1478 } 1479 case ISD::Constant: { 1480 ConstantSDNode *CN = cast<ConstantSDNode>(Node); 1481 unsigned opAction = 1482 TLI.getOperationAction(ISD::Constant, CN->getValueType(0)); 1483 1484 // We know we don't need to expand constants here, constants only have one 1485 // value and we check that it is fine above. 1486 1487 if (opAction == TargetLowering::Custom) { 1488 Tmp1 = TLI.LowerOperation(Result, DAG); 1489 if (Tmp1.getNode()) 1490 Result = Tmp1; 1491 } 1492 break; 1493 } 1494 case ISD::ConstantFP: { 1495 // Spill FP immediates to the constant pool if the target cannot directly 1496 // codegen them. Targets often have some immediate values that can be 1497 // efficiently generated into an FP register without a load. We explicitly 1498 // leave these constants as ConstantFP nodes for the target to deal with. 1499 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 1500 1501 switch (TLI.getOperationAction(ISD::ConstantFP, CFP->getValueType(0))) { 1502 default: assert(0 && "This action is not supported yet!"); 1503 case TargetLowering::Legal: 1504 break; 1505 case TargetLowering::Custom: 1506 Tmp3 = TLI.LowerOperation(Result, DAG); 1507 if (Tmp3.getNode()) { 1508 Result = Tmp3; 1509 break; 1510 } 1511 // FALLTHROUGH 1512 case TargetLowering::Expand: { 1513 // Check to see if this FP immediate is already legal. 1514 bool isLegal = false; 1515 for (TargetLowering::legal_fpimm_iterator I = TLI.legal_fpimm_begin(), 1516 E = TLI.legal_fpimm_end(); I != E; ++I) { 1517 if (CFP->isExactlyValue(*I)) { 1518 isLegal = true; 1519 break; 1520 } 1521 } 1522 // If this is a legal constant, turn it into a TargetConstantFP node. 1523 if (isLegal) 1524 break; 1525 Result = ExpandConstantFP(CFP, true, DAG, TLI); 1526 } 1527 } 1528 break; 1529 } 1530 case ISD::TokenFactor: 1531 if (Node->getNumOperands() == 2) { 1532 Tmp1 = LegalizeOp(Node->getOperand(0)); 1533 Tmp2 = LegalizeOp(Node->getOperand(1)); 1534 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1535 } else if (Node->getNumOperands() == 3) { 1536 Tmp1 = LegalizeOp(Node->getOperand(0)); 1537 Tmp2 = LegalizeOp(Node->getOperand(1)); 1538 Tmp3 = LegalizeOp(Node->getOperand(2)); 1539 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1540 } else { 1541 SmallVector<SDValue, 8> Ops; 1542 // Legalize the operands. 1543 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1544 Ops.push_back(LegalizeOp(Node->getOperand(i))); 1545 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1546 } 1547 break; 1548 1549 case ISD::FORMAL_ARGUMENTS: 1550 case ISD::CALL: 1551 // The only option for this is to custom lower it. 1552 Tmp3 = TLI.LowerOperation(Result.getValue(0), DAG); 1553 assert(Tmp3.getNode() && "Target didn't custom lower this node!"); 1554 // A call within a calling sequence must be legalized to something 1555 // other than the normal CALLSEQ_END. Violating this gets Legalize 1556 // into an infinite loop. 1557 assert ((!IsLegalizingCall || 1558 Node->getOpcode() != ISD::CALL || 1559 Tmp3.getNode()->getOpcode() != ISD::CALLSEQ_END) && 1560 "Nested CALLSEQ_START..CALLSEQ_END not supported."); 1561 1562 // The number of incoming and outgoing values should match; unless the final 1563 // outgoing value is a flag. 1564 assert((Tmp3.getNode()->getNumValues() == Result.getNode()->getNumValues() || 1565 (Tmp3.getNode()->getNumValues() == Result.getNode()->getNumValues() + 1 && 1566 Tmp3.getNode()->getValueType(Tmp3.getNode()->getNumValues() - 1) == 1567 MVT::Flag)) && 1568 "Lowering call/formal_arguments produced unexpected # results!"); 1569 1570 // Since CALL/FORMAL_ARGUMENTS nodes produce multiple values, make sure to 1571 // remember that we legalized all of them, so it doesn't get relegalized. 1572 for (unsigned i = 0, e = Tmp3.getNode()->getNumValues(); i != e; ++i) { 1573 if (Tmp3.getNode()->getValueType(i) == MVT::Flag) 1574 continue; 1575 Tmp1 = LegalizeOp(Tmp3.getValue(i)); 1576 if (Op.getResNo() == i) 1577 Tmp2 = Tmp1; 1578 AddLegalizedOperand(SDValue(Node, i), Tmp1); 1579 } 1580 return Tmp2; 1581 case ISD::EXTRACT_SUBREG: { 1582 Tmp1 = LegalizeOp(Node->getOperand(0)); 1583 ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 1584 assert(idx && "Operand must be a constant"); 1585 Tmp2 = DAG.getTargetConstant(idx->getAPIntValue(), idx->getValueType(0)); 1586 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1587 } 1588 break; 1589 case ISD::INSERT_SUBREG: { 1590 Tmp1 = LegalizeOp(Node->getOperand(0)); 1591 Tmp2 = LegalizeOp(Node->getOperand(1)); 1592 ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(2)); 1593 assert(idx && "Operand must be a constant"); 1594 Tmp3 = DAG.getTargetConstant(idx->getAPIntValue(), idx->getValueType(0)); 1595 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1596 } 1597 break; 1598 case ISD::BUILD_VECTOR: 1599 switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) { 1600 default: assert(0 && "This action is not supported yet!"); 1601 case TargetLowering::Custom: 1602 Tmp3 = TLI.LowerOperation(Result, DAG); 1603 if (Tmp3.getNode()) { 1604 Result = Tmp3; 1605 break; 1606 } 1607 // FALLTHROUGH 1608 case TargetLowering::Expand: 1609 Result = ExpandBUILD_VECTOR(Result.getNode()); 1610 break; 1611 } 1612 break; 1613 case ISD::INSERT_VECTOR_ELT: 1614 Tmp1 = LegalizeOp(Node->getOperand(0)); // InVec 1615 Tmp3 = LegalizeOp(Node->getOperand(2)); // InEltNo 1616 1617 // The type of the value to insert may not be legal, even though the vector 1618 // type is legal. Legalize/Promote accordingly. We do not handle Expand 1619 // here. 1620 switch (getTypeAction(Node->getOperand(1).getValueType())) { 1621 default: assert(0 && "Cannot expand insert element operand"); 1622 case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; 1623 case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; 1624 case Expand: 1625 // FIXME: An alternative would be to check to see if the target is not 1626 // going to custom lower this operation, we could bitcast to half elt 1627 // width and perform two inserts at that width, if that is legal. 1628 Tmp2 = Node->getOperand(1); 1629 break; 1630 } 1631 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1632 1633 switch (TLI.getOperationAction(ISD::INSERT_VECTOR_ELT, 1634 Node->getValueType(0))) { 1635 default: assert(0 && "This action is not supported yet!"); 1636 case TargetLowering::Legal: 1637 break; 1638 case TargetLowering::Custom: 1639 Tmp4 = TLI.LowerOperation(Result, DAG); 1640 if (Tmp4.getNode()) { 1641 Result = Tmp4; 1642 break; 1643 } 1644 // FALLTHROUGH 1645 case TargetLowering::Promote: 1646 // Fall thru for vector case 1647 case TargetLowering::Expand: { 1648 // If the insert index is a constant, codegen this as a scalar_to_vector, 1649 // then a shuffle that inserts it into the right position in the vector. 1650 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Tmp3)) { 1651 // SCALAR_TO_VECTOR requires that the type of the value being inserted 1652 // match the element type of the vector being created. 1653 if (Tmp2.getValueType() == 1654 Op.getValueType().getVectorElementType()) { 1655 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 1656 Tmp1.getValueType(), Tmp2); 1657 1658 unsigned NumElts = Tmp1.getValueType().getVectorNumElements(); 1659 MVT ShufMaskVT = 1660 MVT::getIntVectorWithNumElements(NumElts); 1661 MVT ShufMaskEltVT = ShufMaskVT.getVectorElementType(); 1662 1663 // We generate a shuffle of InVec and ScVec, so the shuffle mask 1664 // should be 0,1,2,3,4,5... with the appropriate element replaced with 1665 // elt 0 of the RHS. 1666 SmallVector<SDValue, 8> ShufOps; 1667 for (unsigned i = 0; i != NumElts; ++i) { 1668 if (i != InsertPos->getZExtValue()) 1669 ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT)); 1670 else 1671 ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT)); 1672 } 1673 SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, dl, ShufMaskVT, 1674 &ShufOps[0], ShufOps.size()); 1675 1676 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, Tmp1.getValueType(), 1677 Tmp1, ScVec, ShufMask); 1678 Result = LegalizeOp(Result); 1679 break; 1680 } 1681 } 1682 Result = PerformInsertVectorEltInMemory(Tmp1, Tmp2, Tmp3, dl); 1683 break; 1684 } 1685 } 1686 break; 1687 case ISD::SCALAR_TO_VECTOR: 1688 if (!TLI.isTypeLegal(Node->getOperand(0).getValueType())) { 1689 Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node)); 1690 break; 1691 } 1692 1693 Tmp1 = LegalizeOp(Node->getOperand(0)); // InVal 1694 Result = DAG.UpdateNodeOperands(Result, Tmp1); 1695 switch (TLI.getOperationAction(ISD::SCALAR_TO_VECTOR, 1696 Node->getValueType(0))) { 1697 default: assert(0 && "This action is not supported yet!"); 1698 case TargetLowering::Legal: 1699 break; 1700 case TargetLowering::Custom: 1701 Tmp3 = TLI.LowerOperation(Result, DAG); 1702 if (Tmp3.getNode()) { 1703 Result = Tmp3; 1704 break; 1705 } 1706 // FALLTHROUGH 1707 case TargetLowering::Expand: 1708 Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node)); 1709 break; 1710 } 1711 break; 1712 case ISD::VECTOR_SHUFFLE: 1713 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input vectors, 1714 Tmp2 = LegalizeOp(Node->getOperand(1)); // but not the shuffle mask. 1715 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 1716 1717 // Allow targets to custom lower the SHUFFLEs they support. 1718 switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE,Result.getValueType())) { 1719 default: assert(0 && "Unknown operation action!"); 1720 case TargetLowering::Legal: 1721 assert(isShuffleLegal(Result.getValueType(), Node->getOperand(2)) && 1722 "vector shuffle should not be created if not legal!"); 1723 break; 1724 case TargetLowering::Custom: 1725 Tmp3 = TLI.LowerOperation(Result, DAG); 1726 if (Tmp3.getNode()) { 1727 Result = Tmp3; 1728 break; 1729 } 1730 // FALLTHROUGH 1731 case TargetLowering::Expand: { 1732 MVT VT = Node->getValueType(0); 1733 MVT EltVT = VT.getVectorElementType(); 1734 MVT PtrVT = TLI.getPointerTy(); 1735 SDValue Mask = Node->getOperand(2); 1736 unsigned NumElems = Mask.getNumOperands(); 1737 SmallVector<SDValue,8> Ops; 1738 for (unsigned i = 0; i != NumElems; ++i) { 1739 SDValue Arg = Mask.getOperand(i); 1740 if (Arg.getOpcode() == ISD::UNDEF) { 1741 Ops.push_back(DAG.getNode(ISD::UNDEF, dl, EltVT)); 1742 } else { 1743 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); 1744 unsigned Idx = cast<ConstantSDNode>(Arg)->getZExtValue(); 1745 if (Idx < NumElems) 1746 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp1, 1747 DAG.getConstant(Idx, PtrVT))); 1748 else 1749 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp2, 1750 DAG.getConstant(Idx - NumElems, PtrVT))); 1751 } 1752 } 1753 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size()); 1754 break; 1755 } 1756 case TargetLowering::Promote: { 1757 // Change base type to a different vector type. 1758 MVT OVT = Node->getValueType(0); 1759 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 1760 1761 // Cast the two input vectors. 1762 Tmp1 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp1); 1763 Tmp2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp2); 1764 1765 // Convert the shuffle mask to the right # elements. 1766 Tmp3 = SDValue(isShuffleLegal(OVT, Node->getOperand(2)), 0); 1767 assert(Tmp3.getNode() && "Shuffle not legal?"); 1768 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, NVT, Tmp1, Tmp2, Tmp3); 1769 Result = DAG.getNode(ISD::BIT_CONVERT, dl, OVT, Result); 1770 break; 1771 } 1772 } 1773 break; 1774 1775 case ISD::EXTRACT_VECTOR_ELT: 1776 Tmp1 = Node->getOperand(0); 1777 Tmp2 = LegalizeOp(Node->getOperand(1)); 1778 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1779 Result = ExpandEXTRACT_VECTOR_ELT(Result); 1780 break; 1781 1782 case ISD::EXTRACT_SUBVECTOR: 1783 Tmp1 = Node->getOperand(0); 1784 Tmp2 = LegalizeOp(Node->getOperand(1)); 1785 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1786 Result = ExpandEXTRACT_SUBVECTOR(Result); 1787 break; 1788 1789 case ISD::CONCAT_VECTORS: { 1790 // Use extract/insert/build vector for now. We might try to be 1791 // more clever later. 1792 MVT PtrVT = TLI.getPointerTy(); 1793 SmallVector<SDValue, 8> Ops; 1794 unsigned NumOperands = Node->getNumOperands(); 1795 for (unsigned i=0; i < NumOperands; ++i) { 1796 SDValue SubOp = Node->getOperand(i); 1797 MVT VVT = SubOp.getNode()->getValueType(0); 1798 MVT EltVT = VVT.getVectorElementType(); 1799 unsigned NumSubElem = VVT.getVectorNumElements(); 1800 for (unsigned j=0; j < NumSubElem; ++j) { 1801 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp, 1802 DAG.getConstant(j, PtrVT))); 1803 } 1804 } 1805 return LegalizeOp(DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), 1806 &Ops[0], Ops.size())); 1807 } 1808 1809 case ISD::CALLSEQ_START: { 1810 SDNode *CallEnd = FindCallEndFromCallStart(Node); 1811 1812 // Recursively Legalize all of the inputs of the call end that do not lead 1813 // to this call start. This ensures that any libcalls that need be inserted 1814 // are inserted *before* the CALLSEQ_START. 1815 IsLegalizingCallArgs = true; 1816 {SmallPtrSet<SDNode*, 32> NodesLeadingTo; 1817 for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i) 1818 LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).getNode(), Node, 1819 NodesLeadingTo); 1820 } 1821 IsLegalizingCallArgs = false; 1822 1823 // Now that we legalized all of the inputs (which may have inserted 1824 // libcalls) create the new CALLSEQ_START node. 1825 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1826 1827 // Merge in the last call, to ensure that this call start after the last 1828 // call ended. 1829 if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken) { 1830 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1831 Tmp1, LastCALLSEQ_END); 1832 Tmp1 = LegalizeOp(Tmp1); 1833 } 1834 1835 // Do not try to legalize the target-specific arguments (#1+). 1836 if (Tmp1 != Node->getOperand(0)) { 1837 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1838 Ops[0] = Tmp1; 1839 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1840 } 1841 1842 // Remember that the CALLSEQ_START is legalized. 1843 AddLegalizedOperand(Op.getValue(0), Result); 1844 if (Node->getNumValues() == 2) // If this has a flag result, remember it. 1845 AddLegalizedOperand(Op.getValue(1), Result.getValue(1)); 1846 1847 // Now that the callseq_start and all of the non-call nodes above this call 1848 // sequence have been legalized, legalize the call itself. During this 1849 // process, no libcalls can/will be inserted, guaranteeing that no calls 1850 // can overlap. 1851 assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!"); 1852 // Note that we are selecting this call! 1853 LastCALLSEQ_END = SDValue(CallEnd, 0); 1854 IsLegalizingCall = true; 1855 1856 // Legalize the call, starting from the CALLSEQ_END. 1857 LegalizeOp(LastCALLSEQ_END); 1858 assert(!IsLegalizingCall && "CALLSEQ_END should have cleared this!"); 1859 return Result; 1860 } 1861 case ISD::CALLSEQ_END: 1862 // If the CALLSEQ_START node hasn't been legalized first, legalize it. This 1863 // will cause this node to be legalized as well as handling libcalls right. 1864 if (LastCALLSEQ_END.getNode() != Node) { 1865 LegalizeOp(SDValue(FindCallStartFromCallEnd(Node), 0)); 1866 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 1867 assert(I != LegalizedNodes.end() && 1868 "Legalizing the call start should have legalized this node!"); 1869 return I->second; 1870 } 1871 1872 // Otherwise, the call start has been legalized and everything is going 1873 // according to plan. Just legalize ourselves normally here. 1874 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1875 // Do not try to legalize the target-specific arguments (#1+), except for 1876 // an optional flag input. 1877 if (Node->getOperand(Node->getNumOperands()-1).getValueType() != MVT::Flag){ 1878 if (Tmp1 != Node->getOperand(0)) { 1879 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1880 Ops[0] = Tmp1; 1881 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1882 } 1883 } else { 1884 Tmp2 = LegalizeOp(Node->getOperand(Node->getNumOperands()-1)); 1885 if (Tmp1 != Node->getOperand(0) || 1886 Tmp2 != Node->getOperand(Node->getNumOperands()-1)) { 1887 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1888 Ops[0] = Tmp1; 1889 Ops.back() = Tmp2; 1890 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1891 } 1892 } 1893 assert(IsLegalizingCall && "Call sequence imbalance between start/end?"); 1894 // This finishes up call legalization. 1895 IsLegalizingCall = false; 1896 1897 // If the CALLSEQ_END node has a flag, remember that we legalized it. 1898 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1899 if (Node->getNumValues() == 2) 1900 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1901 return Result.getValue(Op.getResNo()); 1902 case ISD::DYNAMIC_STACKALLOC: { 1903 MVT VT = Node->getValueType(0); 1904 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1905 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the size. 1906 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the alignment. 1907 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1908 1909 Tmp1 = Result.getValue(0); 1910 Tmp2 = Result.getValue(1); 1911 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1912 default: assert(0 && "This action is not supported yet!"); 1913 case TargetLowering::Expand: { 1914 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1915 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1916 " not tell us which reg is the stack pointer!"); 1917 SDValue Chain = Tmp1.getOperand(0); 1918 1919 // Chain the dynamic stack allocation so that it doesn't modify the stack 1920 // pointer when other instructions are using the stack. 1921 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); 1922 1923 SDValue Size = Tmp2.getOperand(1); 1924 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1925 Chain = SP.getValue(1); 1926 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1927 unsigned StackAlign = 1928 TLI.getTargetMachine().getFrameInfo()->getStackAlignment(); 1929 if (Align > StackAlign) 1930 SP = DAG.getNode(ISD::AND, dl, VT, SP, 1931 DAG.getConstant(-(uint64_t)Align, VT)); 1932 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value 1933 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1934 1935 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), 1936 DAG.getIntPtrConstant(0, true), SDValue()); 1937 1938 Tmp1 = LegalizeOp(Tmp1); 1939 Tmp2 = LegalizeOp(Tmp2); 1940 break; 1941 } 1942 case TargetLowering::Custom: 1943 Tmp3 = TLI.LowerOperation(Tmp1, DAG); 1944 if (Tmp3.getNode()) { 1945 Tmp1 = LegalizeOp(Tmp3); 1946 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 1947 } 1948 break; 1949 case TargetLowering::Legal: 1950 break; 1951 } 1952 // Since this op produce two values, make sure to remember that we 1953 // legalized both of them. 1954 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 1955 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 1956 return Op.getResNo() ? Tmp2 : Tmp1; 1957 } 1958 case ISD::INLINEASM: { 1959 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1960 bool Changed = false; 1961 // Legalize all of the operands of the inline asm, in case they are nodes 1962 // that need to be expanded or something. Note we skip the asm string and 1963 // all of the TargetConstant flags. 1964 SDValue Op = LegalizeOp(Ops[0]); 1965 Changed = Op != Ops[0]; 1966 Ops[0] = Op; 1967 1968 bool HasInFlag = Ops.back().getValueType() == MVT::Flag; 1969 for (unsigned i = 2, e = Ops.size()-HasInFlag; i < e; ) { 1970 unsigned NumVals = cast<ConstantSDNode>(Ops[i])->getZExtValue() >> 3; 1971 for (++i; NumVals; ++i, --NumVals) { 1972 SDValue Op = LegalizeOp(Ops[i]); 1973 if (Op != Ops[i]) { 1974 Changed = true; 1975 Ops[i] = Op; 1976 } 1977 } 1978 } 1979 1980 if (HasInFlag) { 1981 Op = LegalizeOp(Ops.back()); 1982 Changed |= Op != Ops.back(); 1983 Ops.back() = Op; 1984 } 1985 1986 if (Changed) 1987 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1988 1989 // INLINE asm returns a chain and flag, make sure to add both to the map. 1990 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1991 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1992 return Result.getValue(Op.getResNo()); 1993 } 1994 case ISD::BR: 1995 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1996 // Ensure that libcalls are emitted before a branch. 1997 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 1998 Tmp1 = LegalizeOp(Tmp1); 1999 LastCALLSEQ_END = DAG.getEntryNode(); 2000 2001 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 2002 break; 2003 case ISD::BRIND: 2004 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2005 // Ensure that libcalls are emitted before a branch. 2006 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 2007 Tmp1 = LegalizeOp(Tmp1); 2008 LastCALLSEQ_END = DAG.getEntryNode(); 2009 2010 switch (getTypeAction(Node->getOperand(1).getValueType())) { 2011 default: assert(0 && "Indirect target must be legal type (pointer)!"); 2012 case Legal: 2013 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition. 2014 break; 2015 } 2016 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 2017 break; 2018 case ISD::BR_JT: 2019 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2020 // Ensure that libcalls are emitted before a branch. 2021 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 2022 Tmp1 = LegalizeOp(Tmp1); 2023 LastCALLSEQ_END = DAG.getEntryNode(); 2024 2025 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the jumptable node. 2026 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 2027 2028 switch (TLI.getOperationAction(ISD::BR_JT, MVT::Other)) { 2029 default: assert(0 && "This action is not supported yet!"); 2030 case TargetLowering::Legal: break; 2031 case TargetLowering::Custom: 2032 Tmp1 = TLI.LowerOperation(Result, DAG); 2033 if (Tmp1.getNode()) Result = Tmp1; 2034 break; 2035 case TargetLowering::Expand: { 2036 SDValue Chain = Result.getOperand(0); 2037 SDValue Table = Result.getOperand(1); 2038 SDValue Index = Result.getOperand(2); 2039 2040 MVT PTy = TLI.getPointerTy(); 2041 MachineFunction &MF = DAG.getMachineFunction(); 2042 unsigned EntrySize = MF.getJumpTableInfo()->getEntrySize(); 2043 Index= DAG.getNode(ISD::MUL, dl, PTy, 2044 Index, DAG.getConstant(EntrySize, PTy)); 2045 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table); 2046 2047 MVT MemVT = MVT::getIntegerVT(EntrySize * 8); 2048 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, 2049 PseudoSourceValue::getJumpTable(), 0, MemVT); 2050 Addr = LD; 2051 if (TLI.getTargetMachine().getRelocationModel() == Reloc::PIC_) { 2052 // For PIC, the sequence is: 2053 // BRIND(load(Jumptable + index) + RelocBase) 2054 // RelocBase can be JumpTable, GOT or some sort of global base. 2055 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 2056 TLI.getPICJumpTableRelocBase(Table, DAG)); 2057 } 2058 Result = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); 2059 } 2060 } 2061 break; 2062 case ISD::BRCOND: 2063 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2064 // Ensure that libcalls are emitted before a return. 2065 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 2066 Tmp1 = LegalizeOp(Tmp1); 2067 LastCALLSEQ_END = DAG.getEntryNode(); 2068 2069 switch (getTypeAction(Node->getOperand(1).getValueType())) { 2070 case Expand: assert(0 && "It's impossible to expand bools"); 2071 case Legal: 2072 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition. 2073 break; 2074 case Promote: { 2075 Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the condition. 2076 2077 // The top bits of the promoted condition are not necessarily zero, ensure 2078 // that the value is properly zero extended. 2079 unsigned BitWidth = Tmp2.getValueSizeInBits(); 2080 if (!DAG.MaskedValueIsZero(Tmp2, 2081 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 2082 Tmp2 = DAG.getZeroExtendInReg(Tmp2, dl, MVT::i1); 2083 break; 2084 } 2085 } 2086 2087 // Basic block destination (Op#2) is always legal. 2088 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 2089 2090 switch (TLI.getOperationAction(ISD::BRCOND, MVT::Other)) { 2091 default: assert(0 && "This action is not supported yet!"); 2092 case TargetLowering::Legal: break; 2093 case TargetLowering::Custom: 2094 Tmp1 = TLI.LowerOperation(Result, DAG); 2095 if (Tmp1.getNode()) Result = Tmp1; 2096 break; 2097 case TargetLowering::Expand: 2098 // Expand brcond's setcc into its constituent parts and create a BR_CC 2099 // Node. 2100 if (Tmp2.getOpcode() == ISD::SETCC) { 2101 Result = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 2102 Tmp1, Tmp2.getOperand(2), 2103 Tmp2.getOperand(0), Tmp2.getOperand(1), 2104 Node->getOperand(2)); 2105 } else { 2106 Result = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 2107 DAG.getCondCode(ISD::SETNE), Tmp2, 2108 DAG.getConstant(0, Tmp2.getValueType()), 2109 Node->getOperand(2)); 2110 } 2111 break; 2112 } 2113 break; 2114 case ISD::BR_CC: 2115 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2116 // Ensure that libcalls are emitted before a branch. 2117 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 2118 Tmp1 = LegalizeOp(Tmp1); 2119 Tmp2 = Node->getOperand(2); // LHS 2120 Tmp3 = Node->getOperand(3); // RHS 2121 Tmp4 = Node->getOperand(1); // CC 2122 2123 LegalizeSetCC(TLI.getSetCCResultType(Tmp2.getValueType()), 2124 Tmp2, Tmp3, Tmp4, dl); 2125 LastCALLSEQ_END = DAG.getEntryNode(); 2126 2127 // If we didn't get both a LHS and RHS back from LegalizeSetCC, 2128 // the LHS is a legal SETCC itself. In this case, we need to compare 2129 // the result against zero to select between true and false values. 2130 if (Tmp3.getNode() == 0) { 2131 Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); 2132 Tmp4 = DAG.getCondCode(ISD::SETNE); 2133 } 2134 2135 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp4, Tmp2, Tmp3, 2136 Node->getOperand(4)); 2137 2138 switch (TLI.getOperationAction(ISD::BR_CC, Tmp3.getValueType())) { 2139 default: assert(0 && "Unexpected action for BR_CC!"); 2140 case TargetLowering::Legal: break; 2141 case TargetLowering::Custom: 2142 Tmp4 = TLI.LowerOperation(Result, DAG); 2143 if (Tmp4.getNode()) Result = Tmp4; 2144 break; 2145 } 2146 break; 2147 case ISD::LOAD: { 2148 LoadSDNode *LD = cast<LoadSDNode>(Node); 2149 Tmp1 = LegalizeOp(LD->getChain()); // Legalize the chain. 2150 Tmp2 = LegalizeOp(LD->getBasePtr()); // Legalize the base pointer. 2151 2152 ISD::LoadExtType ExtType = LD->getExtensionType(); 2153 if (ExtType == ISD::NON_EXTLOAD) { 2154 MVT VT = Node->getValueType(0); 2155 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset()); 2156 Tmp3 = Result.getValue(0); 2157 Tmp4 = Result.getValue(1); 2158 2159 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 2160 default: assert(0 && "This action is not supported yet!"); 2161 case TargetLowering::Legal: 2162 // If this is an unaligned load and the target doesn't support it, 2163 // expand it. 2164 if (!TLI.allowsUnalignedMemoryAccesses()) { 2165 unsigned ABIAlignment = TLI.getTargetData()-> 2166 getABITypeAlignment(LD->getMemoryVT().getTypeForMVT()); 2167 if (LD->getAlignment() < ABIAlignment){ 2168 Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()), DAG, 2169 TLI); 2170 Tmp3 = Result.getOperand(0); 2171 Tmp4 = Result.getOperand(1); 2172 Tmp3 = LegalizeOp(Tmp3); 2173 Tmp4 = LegalizeOp(Tmp4); 2174 } 2175 } 2176 break; 2177 case TargetLowering::Custom: 2178 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 2179 if (Tmp1.getNode()) { 2180 Tmp3 = LegalizeOp(Tmp1); 2181 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 2182 } 2183 break; 2184 case TargetLowering::Promote: { 2185 // Only promote a load of vector type to another. 2186 assert(VT.isVector() && "Cannot promote this load!"); 2187 // Change base type to a different vector type. 2188 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 2189 2190 Tmp1 = DAG.getLoad(NVT, dl, Tmp1, Tmp2, LD->getSrcValue(), 2191 LD->getSrcValueOffset(), 2192 LD->isVolatile(), LD->getAlignment()); 2193 Tmp3 = LegalizeOp(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp1)); 2194 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 2195 break; 2196 } 2197 } 2198 // Since loads produce two values, make sure to remember that we 2199 // legalized both of them. 2200 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 2201 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 2202 return Op.getResNo() ? Tmp4 : Tmp3; 2203 } else { 2204 MVT SrcVT = LD->getMemoryVT(); 2205 unsigned SrcWidth = SrcVT.getSizeInBits(); 2206 int SVOffset = LD->getSrcValueOffset(); 2207 unsigned Alignment = LD->getAlignment(); 2208 bool isVolatile = LD->isVolatile(); 2209 2210 if (SrcWidth != SrcVT.getStoreSizeInBits() && 2211 // Some targets pretend to have an i1 loading operation, and actually 2212 // load an i8. This trick is correct for ZEXTLOAD because the top 7 2213 // bits are guaranteed to be zero; it helps the optimizers understand 2214 // that these bits are zero. It is also useful for EXTLOAD, since it 2215 // tells the optimizers that those bits are undefined. It would be 2216 // nice to have an effective generic way of getting these benefits... 2217 // Until such a way is found, don't insist on promoting i1 here. 2218 (SrcVT != MVT::i1 || 2219 TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { 2220 // Promote to a byte-sized load if not loading an integral number of 2221 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 2222 unsigned NewWidth = SrcVT.getStoreSizeInBits(); 2223 MVT NVT = MVT::getIntegerVT(NewWidth); 2224 SDValue Ch; 2225 2226 // The extra bits are guaranteed to be zero, since we stored them that 2227 // way. A zext load from NVT thus automatically gives zext from SrcVT. 2228 2229 ISD::LoadExtType NewExtType = 2230 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 2231 2232 Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), 2233 Tmp1, Tmp2, LD->getSrcValue(), SVOffset, 2234 NVT, isVolatile, Alignment); 2235 2236 Ch = Result.getValue(1); // The chain. 2237 2238 if (ExtType == ISD::SEXTLOAD) 2239 // Having the top bits zero doesn't help when sign extending. 2240 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 2241 Result.getValueType(), 2242 Result, DAG.getValueType(SrcVT)); 2243 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 2244 // All the top bits are guaranteed to be zero - inform the optimizers. 2245 Result = DAG.getNode(ISD::AssertZext, dl, 2246 Result.getValueType(), Result, 2247 DAG.getValueType(SrcVT)); 2248 2249 Tmp1 = LegalizeOp(Result); 2250 Tmp2 = LegalizeOp(Ch); 2251 } else if (SrcWidth & (SrcWidth - 1)) { 2252 // If not loading a power-of-2 number of bits, expand as two loads. 2253 assert(SrcVT.isExtended() && !SrcVT.isVector() && 2254 "Unsupported extload!"); 2255 unsigned RoundWidth = 1 << Log2_32(SrcWidth); 2256 assert(RoundWidth < SrcWidth); 2257 unsigned ExtraWidth = SrcWidth - RoundWidth; 2258 assert(ExtraWidth < RoundWidth); 2259 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 2260 "Load size not an integral number of bytes!"); 2261 MVT RoundVT = MVT::getIntegerVT(RoundWidth); 2262 MVT ExtraVT = MVT::getIntegerVT(ExtraWidth); 2263 SDValue Lo, Hi, Ch; 2264 unsigned IncrementSize; 2265 2266 if (TLI.isLittleEndian()) { 2267 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 2268 // Load the bottom RoundWidth bits. 2269 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, 2270 Node->getValueType(0), Tmp1, Tmp2, 2271 LD->getSrcValue(), SVOffset, RoundVT, isVolatile, 2272 Alignment); 2273 2274 // Load the remaining ExtraWidth bits. 2275 IncrementSize = RoundWidth / 8; 2276 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2277 DAG.getIntPtrConstant(IncrementSize)); 2278 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Tmp1, Tmp2, 2279 LD->getSrcValue(), SVOffset + IncrementSize, 2280 ExtraVT, isVolatile, 2281 MinAlign(Alignment, IncrementSize)); 2282 2283 // Build a factor node to remember that this load is independent of the 2284 // other one. 2285 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2286 Hi.getValue(1)); 2287 2288 // Move the top bits to the right place. 2289 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 2290 DAG.getConstant(RoundWidth, TLI.getShiftAmountTy())); 2291 2292 // Join the hi and lo parts. 2293 Result = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 2294 } else { 2295 // Big endian - avoid unaligned loads. 2296 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 2297 // Load the top RoundWidth bits. 2298 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Tmp1, Tmp2, 2299 LD->getSrcValue(), SVOffset, RoundVT, isVolatile, 2300 Alignment); 2301 2302 // Load the remaining ExtraWidth bits. 2303 IncrementSize = RoundWidth / 8; 2304 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2305 DAG.getIntPtrConstant(IncrementSize)); 2306 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, 2307 Node->getValueType(0), Tmp1, Tmp2, 2308 LD->getSrcValue(), SVOffset + IncrementSize, 2309 ExtraVT, isVolatile, 2310 MinAlign(Alignment, IncrementSize)); 2311 2312 // Build a factor node to remember that this load is independent of the 2313 // other one. 2314 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2315 Hi.getValue(1)); 2316 2317 // Move the top bits to the right place. 2318 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 2319 DAG.getConstant(ExtraWidth, TLI.getShiftAmountTy())); 2320 2321 // Join the hi and lo parts. 2322 Result = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 2323 } 2324 2325 Tmp1 = LegalizeOp(Result); 2326 Tmp2 = LegalizeOp(Ch); 2327 } else { 2328 switch (TLI.getLoadExtAction(ExtType, SrcVT)) { 2329 default: assert(0 && "This action is not supported yet!"); 2330 case TargetLowering::Custom: 2331 isCustom = true; 2332 // FALLTHROUGH 2333 case TargetLowering::Legal: 2334 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset()); 2335 Tmp1 = Result.getValue(0); 2336 Tmp2 = Result.getValue(1); 2337 2338 if (isCustom) { 2339 Tmp3 = TLI.LowerOperation(Result, DAG); 2340 if (Tmp3.getNode()) { 2341 Tmp1 = LegalizeOp(Tmp3); 2342 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 2343 } 2344 } else { 2345 // If this is an unaligned load and the target doesn't support it, 2346 // expand it. 2347 if (!TLI.allowsUnalignedMemoryAccesses()) { 2348 unsigned ABIAlignment = TLI.getTargetData()-> 2349 getABITypeAlignment(LD->getMemoryVT().getTypeForMVT()); 2350 if (LD->getAlignment() < ABIAlignment){ 2351 Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()), DAG, 2352 TLI); 2353 Tmp1 = Result.getOperand(0); 2354 Tmp2 = Result.getOperand(1); 2355 Tmp1 = LegalizeOp(Tmp1); 2356 Tmp2 = LegalizeOp(Tmp2); 2357 } 2358 } 2359 } 2360 break; 2361 case TargetLowering::Expand: 2362 // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND 2363 if (SrcVT == MVT::f32 && Node->getValueType(0) == MVT::f64) { 2364 SDValue Load = DAG.getLoad(SrcVT, dl, Tmp1, Tmp2, LD->getSrcValue(), 2365 LD->getSrcValueOffset(), 2366 LD->isVolatile(), LD->getAlignment()); 2367 Result = DAG.getNode(ISD::FP_EXTEND, dl, 2368 Node->getValueType(0), Load); 2369 Tmp1 = LegalizeOp(Result); // Relegalize new nodes. 2370 Tmp2 = LegalizeOp(Load.getValue(1)); 2371 break; 2372 } 2373 assert(ExtType != ISD::EXTLOAD &&"EXTLOAD should always be supported!"); 2374 // Turn the unsupported load into an EXTLOAD followed by an explicit 2375 // zero/sign extend inreg. 2376 Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0), 2377 Tmp1, Tmp2, LD->getSrcValue(), 2378 LD->getSrcValueOffset(), SrcVT, 2379 LD->isVolatile(), LD->getAlignment()); 2380 SDValue ValRes; 2381 if (ExtType == ISD::SEXTLOAD) 2382 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 2383 Result.getValueType(), 2384 Result, DAG.getValueType(SrcVT)); 2385 else 2386 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT); 2387 Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes. 2388 Tmp2 = LegalizeOp(Result.getValue(1)); // Relegalize new nodes. 2389 break; 2390 } 2391 } 2392 2393 // Since loads produce two values, make sure to remember that we legalized 2394 // both of them. 2395 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2396 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2397 return Op.getResNo() ? Tmp2 : Tmp1; 2398 } 2399 } 2400 case ISD::EXTRACT_ELEMENT: { 2401 MVT OpTy = Node->getOperand(0).getValueType(); 2402 switch (getTypeAction(OpTy)) { 2403 default: assert(0 && "EXTRACT_ELEMENT action for type unimplemented!"); 2404 case Legal: 2405 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 2406 // 1 -> Hi 2407 Result = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 2408 DAG.getConstant(OpTy.getSizeInBits()/2, 2409 TLI.getShiftAmountTy())); 2410 Result = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); 2411 } else { 2412 // 0 -> Lo 2413 Result = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 2414 Node->getOperand(0)); 2415 } 2416 break; 2417 case Expand: 2418 // Get both the low and high parts. 2419 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 2420 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) 2421 Result = Tmp2; // 1 -> Hi 2422 else 2423 Result = Tmp1; // 0 -> Lo 2424 break; 2425 } 2426 break; 2427 } 2428 2429 case ISD::CopyToReg: 2430 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2431 2432 assert(isTypeLegal(Node->getOperand(2).getValueType()) && 2433 "Register type must be legal!"); 2434 // Legalize the incoming value (must be a legal type). 2435 Tmp2 = LegalizeOp(Node->getOperand(2)); 2436 if (Node->getNumValues() == 1) { 2437 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2); 2438 } else { 2439 assert(Node->getNumValues() == 2 && "Unknown CopyToReg"); 2440 if (Node->getNumOperands() == 4) { 2441 Tmp3 = LegalizeOp(Node->getOperand(3)); 2442 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2, 2443 Tmp3); 2444 } else { 2445 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2); 2446 } 2447 2448 // Since this produces two values, make sure to remember that we legalized 2449 // both of them. 2450 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 2451 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 2452 return Result; 2453 } 2454 break; 2455 2456 case ISD::RET: 2457 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2458 2459 // Ensure that libcalls are emitted before a return. 2460 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 2461 Tmp1 = LegalizeOp(Tmp1); 2462 LastCALLSEQ_END = DAG.getEntryNode(); 2463 2464 switch (Node->getNumOperands()) { 2465 case 3: // ret val 2466 Tmp2 = Node->getOperand(1); 2467 Tmp3 = Node->getOperand(2); // Signness 2468 switch (getTypeAction(Tmp2.getValueType())) { 2469 case Legal: 2470 Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3); 2471 break; 2472 case Expand: 2473 if (!Tmp2.getValueType().isVector()) { 2474 SDValue Lo, Hi; 2475 ExpandOp(Tmp2, Lo, Hi); 2476 2477 // Big endian systems want the hi reg first. 2478 if (TLI.isBigEndian()) 2479 std::swap(Lo, Hi); 2480 2481 if (Hi.getNode()) 2482 Result = DAG.getNode(ISD::RET, dl, MVT::Other, 2483 Tmp1, Lo, Tmp3, Hi,Tmp3); 2484 else 2485 Result = DAG.getNode(ISD::RET, dl, MVT::Other, Tmp1, Lo, Tmp3); 2486 Result = LegalizeOp(Result); 2487 } else { 2488 SDNode *InVal = Tmp2.getNode(); 2489 int InIx = Tmp2.getResNo(); 2490 unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements(); 2491 MVT EVT = InVal->getValueType(InIx).getVectorElementType(); 2492 2493 // Figure out if there is a simple type corresponding to this Vector 2494 // type. If so, convert to the vector type. 2495 MVT TVT = MVT::getVectorVT(EVT, NumElems); 2496 if (TLI.isTypeLegal(TVT)) { 2497 // Turn this into a return of the vector type. 2498 Tmp2 = LegalizeOp(Tmp2); 2499 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2500 } else if (NumElems == 1) { 2501 // Turn this into a return of the scalar type. 2502 Tmp2 = ScalarizeVectorOp(Tmp2); 2503 Tmp2 = LegalizeOp(Tmp2); 2504 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2505 2506 // FIXME: Returns of gcc generic vectors smaller than a legal type 2507 // should be returned in integer registers! 2508 2509 // The scalarized value type may not be legal, e.g. it might require 2510 // promotion or expansion. Relegalize the return. 2511 Result = LegalizeOp(Result); 2512 } else { 2513 // FIXME: Returns of gcc generic vectors larger than a legal vector 2514 // type should be returned by reference! 2515 SDValue Lo, Hi; 2516 SplitVectorOp(Tmp2, Lo, Hi); 2517 Result = DAG.getNode(ISD::RET, dl, MVT::Other, 2518 Tmp1, Lo, Tmp3, Hi,Tmp3); 2519 Result = LegalizeOp(Result); 2520 } 2521 } 2522 break; 2523 case Promote: 2524 Tmp2 = PromoteOp(Node->getOperand(1)); 2525 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2526 Result = LegalizeOp(Result); 2527 break; 2528 } 2529 break; 2530 case 1: // ret void 2531 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2532 break; 2533 default: { // ret <values> 2534 SmallVector<SDValue, 8> NewValues; 2535 NewValues.push_back(Tmp1); 2536 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) 2537 switch (getTypeAction(Node->getOperand(i).getValueType())) { 2538 case Legal: 2539 NewValues.push_back(LegalizeOp(Node->getOperand(i))); 2540 NewValues.push_back(Node->getOperand(i+1)); 2541 break; 2542 case Expand: { 2543 SDValue Lo, Hi; 2544 assert(!Node->getOperand(i).getValueType().isExtended() && 2545 "FIXME: TODO: implement returning non-legal vector types!"); 2546 ExpandOp(Node->getOperand(i), Lo, Hi); 2547 NewValues.push_back(Lo); 2548 NewValues.push_back(Node->getOperand(i+1)); 2549 if (Hi.getNode()) { 2550 NewValues.push_back(Hi); 2551 NewValues.push_back(Node->getOperand(i+1)); 2552 } 2553 break; 2554 } 2555 case Promote: 2556 assert(0 && "Can't promote multiple return value yet!"); 2557 } 2558 2559 if (NewValues.size() == Node->getNumOperands()) 2560 Result = DAG.UpdateNodeOperands(Result, &NewValues[0],NewValues.size()); 2561 else 2562 Result = DAG.getNode(ISD::RET, dl, MVT::Other, 2563 &NewValues[0], NewValues.size()); 2564 break; 2565 } 2566 } 2567 2568 if (Result.getOpcode() == ISD::RET) { 2569 switch (TLI.getOperationAction(Result.getOpcode(), MVT::Other)) { 2570 default: assert(0 && "This action is not supported yet!"); 2571 case TargetLowering::Legal: break; 2572 case TargetLowering::Custom: 2573 Tmp1 = TLI.LowerOperation(Result, DAG); 2574 if (Tmp1.getNode()) Result = Tmp1; 2575 break; 2576 } 2577 } 2578 break; 2579 case ISD::STORE: { 2580 StoreSDNode *ST = cast<StoreSDNode>(Node); 2581 Tmp1 = LegalizeOp(ST->getChain()); // Legalize the chain. 2582 Tmp2 = LegalizeOp(ST->getBasePtr()); // Legalize the pointer. 2583 int SVOffset = ST->getSrcValueOffset(); 2584 unsigned Alignment = ST->getAlignment(); 2585 bool isVolatile = ST->isVolatile(); 2586 2587 if (!ST->isTruncatingStore()) { 2588 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 2589 // FIXME: We shouldn't do this for TargetConstantFP's. 2590 // FIXME: move this to the DAG Combiner! Note that we can't regress due 2591 // to phase ordering between legalized code and the dag combiner. This 2592 // probably means that we need to integrate dag combiner and legalizer 2593 // together. 2594 // We generally can't do this one for long doubles. 2595 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 2596 if (CFP->getValueType(0) == MVT::f32 && 2597 getTypeAction(MVT::i32) == Legal) { 2598 Tmp3 = DAG.getConstant(CFP->getValueAPF(). 2599 bitcastToAPInt().zextOrTrunc(32), 2600 MVT::i32); 2601 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2602 SVOffset, isVolatile, Alignment); 2603 break; 2604 } else if (CFP->getValueType(0) == MVT::f64) { 2605 // If this target supports 64-bit registers, do a single 64-bit store. 2606 if (getTypeAction(MVT::i64) == Legal) { 2607 Tmp3 = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 2608 zextOrTrunc(64), MVT::i64); 2609 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2610 SVOffset, isVolatile, Alignment); 2611 break; 2612 } else if (getTypeAction(MVT::i32) == Legal && !ST->isVolatile()) { 2613 // Otherwise, if the target supports 32-bit registers, use 2 32-bit 2614 // stores. If the target supports neither 32- nor 64-bits, this 2615 // xform is certainly not worth it. 2616 const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); 2617 SDValue Lo = DAG.getConstant(APInt(IntVal).trunc(32), MVT::i32); 2618 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); 2619 if (TLI.isBigEndian()) std::swap(Lo, Hi); 2620 2621 Lo = DAG.getStore(Tmp1, dl, Lo, Tmp2, ST->getSrcValue(), 2622 SVOffset, isVolatile, Alignment); 2623 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2624 DAG.getIntPtrConstant(4)); 2625 Hi = DAG.getStore(Tmp1, dl, Hi, Tmp2, ST->getSrcValue(), SVOffset+4, 2626 isVolatile, MinAlign(Alignment, 4U)); 2627 2628 Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2629 break; 2630 } 2631 } 2632 } 2633 2634 switch (getTypeAction(ST->getMemoryVT())) { 2635 case Legal: { 2636 Tmp3 = LegalizeOp(ST->getValue()); 2637 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 2638 ST->getOffset()); 2639 2640 MVT VT = Tmp3.getValueType(); 2641 switch (TLI.getOperationAction(ISD::STORE, VT)) { 2642 default: assert(0 && "This action is not supported yet!"); 2643 case TargetLowering::Legal: 2644 // If this is an unaligned store and the target doesn't support it, 2645 // expand it. 2646 if (!TLI.allowsUnalignedMemoryAccesses()) { 2647 unsigned ABIAlignment = TLI.getTargetData()-> 2648 getABITypeAlignment(ST->getMemoryVT().getTypeForMVT()); 2649 if (ST->getAlignment() < ABIAlignment) 2650 Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()), DAG, 2651 TLI); 2652 } 2653 break; 2654 case TargetLowering::Custom: 2655 Tmp1 = TLI.LowerOperation(Result, DAG); 2656 if (Tmp1.getNode()) Result = Tmp1; 2657 break; 2658 case TargetLowering::Promote: 2659 assert(VT.isVector() && "Unknown legal promote case!"); 2660 Tmp3 = DAG.getNode(ISD::BIT_CONVERT, dl, 2661 TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3); 2662 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, 2663 ST->getSrcValue(), SVOffset, isVolatile, 2664 Alignment); 2665 break; 2666 } 2667 break; 2668 } 2669 case Promote: 2670 if (!ST->getMemoryVT().isVector()) { 2671 // Truncate the value and store the result. 2672 Tmp3 = PromoteOp(ST->getValue()); 2673 Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2674 SVOffset, ST->getMemoryVT(), 2675 isVolatile, Alignment); 2676 break; 2677 } 2678 // Fall thru to expand for vector 2679 case Expand: { 2680 unsigned IncrementSize = 0; 2681 SDValue Lo, Hi; 2682 2683 // If this is a vector type, then we have to calculate the increment as 2684 // the product of the element size in bytes, and the number of elements 2685 // in the high half of the vector. 2686 if (ST->getValue().getValueType().isVector()) { 2687 SDNode *InVal = ST->getValue().getNode(); 2688 int InIx = ST->getValue().getResNo(); 2689 MVT InVT = InVal->getValueType(InIx); 2690 unsigned NumElems = InVT.getVectorNumElements(); 2691 MVT EVT = InVT.getVectorElementType(); 2692 2693 // Figure out if there is a simple type corresponding to this Vector 2694 // type. If so, convert to the vector type. 2695 MVT TVT = MVT::getVectorVT(EVT, NumElems); 2696 if (TLI.isTypeLegal(TVT)) { 2697 // Turn this into a normal store of the vector type. 2698 Tmp3 = LegalizeOp(ST->getValue()); 2699 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2700 SVOffset, isVolatile, Alignment); 2701 Result = LegalizeOp(Result); 2702 break; 2703 } else if (NumElems == 1) { 2704 // Turn this into a normal store of the scalar type. 2705 Tmp3 = ScalarizeVectorOp(ST->getValue()); 2706 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2707 SVOffset, isVolatile, Alignment); 2708 // The scalarized value type may not be legal, e.g. it might require 2709 // promotion or expansion. Relegalize the scalar store. 2710 Result = LegalizeOp(Result); 2711 break; 2712 } else { 2713 // Check if we have widen this node with another value 2714 std::map<SDValue, SDValue>::iterator I = 2715 WidenNodes.find(ST->getValue()); 2716 if (I != WidenNodes.end()) { 2717 Result = StoreWidenVectorOp(ST, Tmp1, Tmp2); 2718 break; 2719 } 2720 else { 2721 SplitVectorOp(ST->getValue(), Lo, Hi); 2722 IncrementSize = Lo.getNode()->getValueType(0).getVectorNumElements() * 2723 EVT.getSizeInBits()/8; 2724 } 2725 } 2726 } else { 2727 ExpandOp(ST->getValue(), Lo, Hi); 2728 IncrementSize = Hi.getNode() ? Hi.getValueType().getSizeInBits()/8 : 0; 2729 2730 if (Hi.getNode() && TLI.isBigEndian()) 2731 std::swap(Lo, Hi); 2732 } 2733 2734 Lo = DAG.getStore(Tmp1, dl, Lo, Tmp2, ST->getSrcValue(), 2735 SVOffset, isVolatile, Alignment); 2736 2737 if (Hi.getNode() == NULL) { 2738 // Must be int <-> float one-to-one expansion. 2739 Result = Lo; 2740 break; 2741 } 2742 2743 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2744 DAG.getIntPtrConstant(IncrementSize)); 2745 assert(isTypeLegal(Tmp2.getValueType()) && 2746 "Pointers must be legal!"); 2747 SVOffset += IncrementSize; 2748 Alignment = MinAlign(Alignment, IncrementSize); 2749 Hi = DAG.getStore(Tmp1, dl, Hi, Tmp2, ST->getSrcValue(), 2750 SVOffset, isVolatile, Alignment); 2751 Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2752 break; 2753 } // case Expand 2754 } 2755 } else { 2756 switch (getTypeAction(ST->getValue().getValueType())) { 2757 case Legal: 2758 Tmp3 = LegalizeOp(ST->getValue()); 2759 break; 2760 case Promote: 2761 if (!ST->getValue().getValueType().isVector()) { 2762 // We can promote the value, the truncstore will still take care of it. 2763 Tmp3 = PromoteOp(ST->getValue()); 2764 break; 2765 } 2766 // Vector case falls through to expand 2767 case Expand: 2768 // Just store the low part. This may become a non-trunc store, so make 2769 // sure to use getTruncStore, not UpdateNodeOperands below. 2770 ExpandOp(ST->getValue(), Tmp3, Tmp4); 2771 return DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2772 SVOffset, MVT::i8, isVolatile, Alignment); 2773 } 2774 2775 MVT StVT = ST->getMemoryVT(); 2776 unsigned StWidth = StVT.getSizeInBits(); 2777 2778 if (StWidth != StVT.getStoreSizeInBits()) { 2779 // Promote to a byte-sized store with upper bits zero if not 2780 // storing an integral number of bytes. For example, promote 2781 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 2782 MVT NVT = MVT::getIntegerVT(StVT.getStoreSizeInBits()); 2783 Tmp3 = DAG.getZeroExtendInReg(Tmp3, dl, StVT); 2784 Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2785 SVOffset, NVT, isVolatile, Alignment); 2786 } else if (StWidth & (StWidth - 1)) { 2787 // If not storing a power-of-2 number of bits, expand as two stores. 2788 assert(StVT.isExtended() && !StVT.isVector() && 2789 "Unsupported truncstore!"); 2790 unsigned RoundWidth = 1 << Log2_32(StWidth); 2791 assert(RoundWidth < StWidth); 2792 unsigned ExtraWidth = StWidth - RoundWidth; 2793 assert(ExtraWidth < RoundWidth); 2794 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 2795 "Store size not an integral number of bytes!"); 2796 MVT RoundVT = MVT::getIntegerVT(RoundWidth); 2797 MVT ExtraVT = MVT::getIntegerVT(ExtraWidth); 2798 SDValue Lo, Hi; 2799 unsigned IncrementSize; 2800 2801 if (TLI.isLittleEndian()) { 2802 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 2803 // Store the bottom RoundWidth bits. 2804 Lo = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2805 SVOffset, RoundVT, 2806 isVolatile, Alignment); 2807 2808 // Store the remaining ExtraWidth bits. 2809 IncrementSize = RoundWidth / 8; 2810 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2811 DAG.getIntPtrConstant(IncrementSize)); 2812 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3, 2813 DAG.getConstant(RoundWidth, TLI.getShiftAmountTy())); 2814 Hi = DAG.getTruncStore(Tmp1, dl, Hi, Tmp2, ST->getSrcValue(), 2815 SVOffset + IncrementSize, ExtraVT, isVolatile, 2816 MinAlign(Alignment, IncrementSize)); 2817 } else { 2818 // Big endian - avoid unaligned stores. 2819 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 2820 // Store the top RoundWidth bits. 2821 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3, 2822 DAG.getConstant(ExtraWidth, TLI.getShiftAmountTy())); 2823 Hi = DAG.getTruncStore(Tmp1, dl, Hi, Tmp2, ST->getSrcValue(), 2824 SVOffset, RoundVT, isVolatile, Alignment); 2825 2826 // Store the remaining ExtraWidth bits. 2827 IncrementSize = RoundWidth / 8; 2828 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2829 DAG.getIntPtrConstant(IncrementSize)); 2830 Lo = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2831 SVOffset + IncrementSize, ExtraVT, isVolatile, 2832 MinAlign(Alignment, IncrementSize)); 2833 } 2834 2835 // The order of the stores doesn't matter. 2836 Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2837 } else { 2838 if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() || 2839 Tmp2 != ST->getBasePtr()) 2840 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 2841 ST->getOffset()); 2842 2843 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { 2844 default: assert(0 && "This action is not supported yet!"); 2845 case TargetLowering::Legal: 2846 // If this is an unaligned store and the target doesn't support it, 2847 // expand it. 2848 if (!TLI.allowsUnalignedMemoryAccesses()) { 2849 unsigned ABIAlignment = TLI.getTargetData()-> 2850 getABITypeAlignment(ST->getMemoryVT().getTypeForMVT()); 2851 if (ST->getAlignment() < ABIAlignment) 2852 Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()), DAG, 2853 TLI); 2854 } 2855 break; 2856 case TargetLowering::Custom: 2857 Result = TLI.LowerOperation(Result, DAG); 2858 break; 2859 case Expand: 2860 // TRUNCSTORE:i16 i32 -> STORE i16 2861 assert(isTypeLegal(StVT) && "Do not know how to expand this store!"); 2862 Tmp3 = DAG.getNode(ISD::TRUNCATE, dl, StVT, Tmp3); 2863 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2864 SVOffset, isVolatile, Alignment); 2865 break; 2866 } 2867 } 2868 } 2869 break; 2870 } 2871 case ISD::PCMARKER: 2872 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2873 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 2874 break; 2875 case ISD::STACKSAVE: 2876 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2877 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2878 Tmp1 = Result.getValue(0); 2879 Tmp2 = Result.getValue(1); 2880 2881 switch (TLI.getOperationAction(ISD::STACKSAVE, MVT::Other)) { 2882 default: assert(0 && "This action is not supported yet!"); 2883 case TargetLowering::Legal: break; 2884 case TargetLowering::Custom: 2885 Tmp3 = TLI.LowerOperation(Result, DAG); 2886 if (Tmp3.getNode()) { 2887 Tmp1 = LegalizeOp(Tmp3); 2888 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 2889 } 2890 break; 2891 case TargetLowering::Expand: 2892 // Expand to CopyFromReg if the target set 2893 // StackPointerRegisterToSaveRestore. 2894 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 2895 Tmp1 = DAG.getCopyFromReg(Result.getOperand(0), dl, SP, 2896 Node->getValueType(0)); 2897 Tmp2 = Tmp1.getValue(1); 2898 } else { 2899 Tmp1 = DAG.getNode(ISD::UNDEF, dl, Node->getValueType(0)); 2900 Tmp2 = Node->getOperand(0); 2901 } 2902 break; 2903 } 2904 2905 // Since stacksave produce two values, make sure to remember that we 2906 // legalized both of them. 2907 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2908 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2909 return Op.getResNo() ? Tmp2 : Tmp1; 2910 2911 case ISD::STACKRESTORE: 2912 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2913 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 2914 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 2915 2916 switch (TLI.getOperationAction(ISD::STACKRESTORE, MVT::Other)) { 2917 default: assert(0 && "This action is not supported yet!"); 2918 case TargetLowering::Legal: break; 2919 case TargetLowering::Custom: 2920 Tmp1 = TLI.LowerOperation(Result, DAG); 2921 if (Tmp1.getNode()) Result = Tmp1; 2922 break; 2923 case TargetLowering::Expand: 2924 // Expand to CopyToReg if the target set 2925 // StackPointerRegisterToSaveRestore. 2926 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 2927 Result = DAG.getCopyToReg(Tmp1, dl, SP, Tmp2); 2928 } else { 2929 Result = Tmp1; 2930 } 2931 break; 2932 } 2933 break; 2934 2935 case ISD::READCYCLECOUNTER: 2936 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain 2937 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2938 switch (TLI.getOperationAction(ISD::READCYCLECOUNTER, 2939 Node->getValueType(0))) { 2940 default: assert(0 && "This action is not supported yet!"); 2941 case TargetLowering::Legal: 2942 Tmp1 = Result.getValue(0); 2943 Tmp2 = Result.getValue(1); 2944 break; 2945 case TargetLowering::Custom: 2946 Result = TLI.LowerOperation(Result, DAG); 2947 Tmp1 = LegalizeOp(Result.getValue(0)); 2948 Tmp2 = LegalizeOp(Result.getValue(1)); 2949 break; 2950 } 2951 2952 // Since rdcc produce two values, make sure to remember that we legalized 2953 // both of them. 2954 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2955 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2956 return Result; 2957 2958 case ISD::SELECT: 2959 switch (getTypeAction(Node->getOperand(0).getValueType())) { 2960 case Expand: assert(0 && "It's impossible to expand bools"); 2961 case Legal: 2962 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition. 2963 break; 2964 case Promote: { 2965 assert(!Node->getOperand(0).getValueType().isVector() && "not possible"); 2966 Tmp1 = PromoteOp(Node->getOperand(0)); // Promote the condition. 2967 // Make sure the condition is either zero or one. 2968 unsigned BitWidth = Tmp1.getValueSizeInBits(); 2969 if (!DAG.MaskedValueIsZero(Tmp1, 2970 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 2971 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, MVT::i1); 2972 break; 2973 } 2974 } 2975 Tmp2 = LegalizeOp(Node->getOperand(1)); // TrueVal 2976 Tmp3 = LegalizeOp(Node->getOperand(2)); // FalseVal 2977 2978 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2979 2980 switch (TLI.getOperationAction(ISD::SELECT, Tmp2.getValueType())) { 2981 default: assert(0 && "This action is not supported yet!"); 2982 case TargetLowering::Legal: break; 2983 case TargetLowering::Custom: { 2984 Tmp1 = TLI.LowerOperation(Result, DAG); 2985 if (Tmp1.getNode()) Result = Tmp1; 2986 break; 2987 } 2988 case TargetLowering::Expand: 2989 if (Tmp1.getOpcode() == ISD::SETCC) { 2990 Result = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 2991 Tmp2, Tmp3, 2992 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 2993 } else { 2994 Result = DAG.getSelectCC(dl, Tmp1, 2995 DAG.getConstant(0, Tmp1.getValueType()), 2996 Tmp2, Tmp3, ISD::SETNE); 2997 } 2998 break; 2999 case TargetLowering::Promote: { 3000 MVT NVT = 3001 TLI.getTypeToPromoteTo(ISD::SELECT, Tmp2.getValueType()); 3002 unsigned ExtOp, TruncOp; 3003 if (Tmp2.getValueType().isVector()) { 3004 ExtOp = ISD::BIT_CONVERT; 3005 TruncOp = ISD::BIT_CONVERT; 3006 } else if (Tmp2.getValueType().isInteger()) { 3007 ExtOp = ISD::ANY_EXTEND; 3008 TruncOp = ISD::TRUNCATE; 3009 } else { 3010 ExtOp = ISD::FP_EXTEND; 3011 TruncOp = ISD::FP_ROUND; 3012 } 3013 // Promote each of the values to the new type. 3014 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Tmp2); 3015 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Tmp3); 3016 // Perform the larger operation, then round down. 3017 Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2,Tmp3); 3018 if (TruncOp != ISD::FP_ROUND) 3019 Result = DAG.getNode(TruncOp, dl, Node->getValueType(0), Result); 3020 else 3021 Result = DAG.getNode(TruncOp, dl, Node->getValueType(0), Result, 3022 DAG.getIntPtrConstant(0)); 3023 break; 3024 } 3025 } 3026 break; 3027 case ISD::SELECT_CC: { 3028 Tmp1 = Node->getOperand(0); // LHS 3029 Tmp2 = Node->getOperand(1); // RHS 3030 Tmp3 = LegalizeOp(Node->getOperand(2)); // True 3031 Tmp4 = LegalizeOp(Node->getOperand(3)); // False 3032 SDValue CC = Node->getOperand(4); 3033 3034 LegalizeSetCC(TLI.getSetCCResultType(Tmp1.getValueType()), 3035 Tmp1, Tmp2, CC, dl); 3036 3037 // If we didn't get both a LHS and RHS back from LegalizeSetCC, 3038 // the LHS is a legal SETCC itself. In this case, we need to compare 3039 // the result against zero to select between true and false values. 3040 if (Tmp2.getNode() == 0) { 3041 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 3042 CC = DAG.getCondCode(ISD::SETNE); 3043 } 3044 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, CC); 3045 3046 // Everything is legal, see if we should expand this op or something. 3047 switch (TLI.getOperationAction(ISD::SELECT_CC, Tmp3.getValueType())) { 3048 default: assert(0 && "This action is not supported yet!"); 3049 case TargetLowering::Legal: break; 3050 case TargetLowering::Custom: 3051 Tmp1 = TLI.LowerOperation(Result, DAG); 3052 if (Tmp1.getNode()) Result = Tmp1; 3053 break; 3054 } 3055 break; 3056 } 3057 case ISD::SETCC: 3058 Tmp1 = Node->getOperand(0); 3059 Tmp2 = Node->getOperand(1); 3060 Tmp3 = Node->getOperand(2); 3061 LegalizeSetCC(Node->getValueType(0), Tmp1, Tmp2, Tmp3, dl); 3062 3063 // If we had to Expand the SetCC operands into a SELECT node, then it may 3064 // not always be possible to return a true LHS & RHS. In this case, just 3065 // return the value we legalized, returned in the LHS 3066 if (Tmp2.getNode() == 0) { 3067 Result = Tmp1; 3068 break; 3069 } 3070 3071 switch (TLI.getOperationAction(ISD::SETCC, Tmp1.getValueType())) { 3072 default: assert(0 && "Cannot handle this action for SETCC yet!"); 3073 case TargetLowering::Custom: 3074 isCustom = true; 3075 // FALLTHROUGH. 3076 case TargetLowering::Legal: 3077 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 3078 if (isCustom) { 3079 Tmp4 = TLI.LowerOperation(Result, DAG); 3080 if (Tmp4.getNode()) Result = Tmp4; 3081 } 3082 break; 3083 case TargetLowering::Promote: { 3084 // First step, figure out the appropriate operation to use. 3085 // Allow SETCC to not be supported for all legal data types 3086 // Mostly this targets FP 3087 MVT NewInTy = Node->getOperand(0).getValueType(); 3088 MVT OldVT = NewInTy; OldVT = OldVT; 3089 3090 // Scan for the appropriate larger type to use. 3091 while (1) { 3092 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT()+1); 3093 3094 assert(NewInTy.isInteger() == OldVT.isInteger() && 3095 "Fell off of the edge of the integer world"); 3096 assert(NewInTy.isFloatingPoint() == OldVT.isFloatingPoint() && 3097 "Fell off of the edge of the floating point world"); 3098 3099 // If the target supports SETCC of this type, use it. 3100 if (TLI.isOperationLegalOrCustom(ISD::SETCC, NewInTy)) 3101 break; 3102 } 3103 if (NewInTy.isInteger()) 3104 assert(0 && "Cannot promote Legal Integer SETCC yet"); 3105 else { 3106 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewInTy, Tmp1); 3107 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NewInTy, Tmp2); 3108 } 3109 Tmp1 = LegalizeOp(Tmp1); 3110 Tmp2 = LegalizeOp(Tmp2); 3111 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 3112 Result = LegalizeOp(Result); 3113 break; 3114 } 3115 case TargetLowering::Expand: 3116 // Expand a setcc node into a select_cc of the same condition, lhs, and 3117 // rhs that selects between const 1 (true) and const 0 (false). 3118 MVT VT = Node->getValueType(0); 3119 Result = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3120 DAG.getConstant(1, VT), DAG.getConstant(0, VT), 3121 Tmp3); 3122 break; 3123 } 3124 break; 3125 case ISD::VSETCC: { 3126 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3127 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3128 SDValue CC = Node->getOperand(2); 3129 3130 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, CC); 3131 3132 // Everything is legal, see if we should expand this op or something. 3133 switch (TLI.getOperationAction(ISD::VSETCC, Tmp1.getValueType())) { 3134 default: assert(0 && "This action is not supported yet!"); 3135 case TargetLowering::Legal: break; 3136 case TargetLowering::Custom: 3137 Tmp1 = TLI.LowerOperation(Result, DAG); 3138 if (Tmp1.getNode()) Result = Tmp1; 3139 break; 3140 case TargetLowering::Expand: { 3141 // Unroll into a nasty set of scalar code for now. 3142 MVT VT = Node->getValueType(0); 3143 unsigned NumElems = VT.getVectorNumElements(); 3144 MVT EltVT = VT.getVectorElementType(); 3145 MVT TmpEltVT = Tmp1.getValueType().getVectorElementType(); 3146 SmallVector<SDValue, 8> Ops(NumElems); 3147 for (unsigned i = 0; i < NumElems; ++i) { 3148 SDValue In1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, 3149 Tmp1, DAG.getIntPtrConstant(i)); 3150 Ops[i] = DAG.getNode(ISD::SETCC, dl, TLI.getSetCCResultType(TmpEltVT), 3151 In1, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3152 TmpEltVT, Tmp2, 3153 DAG.getIntPtrConstant(i)), 3154 CC); 3155 Ops[i] = DAG.getNode(ISD::SELECT, dl, EltVT, Ops[i], DAG.getConstant( 3156 APInt::getAllOnesValue(EltVT.getSizeInBits()), 3157 EltVT), DAG.getConstant(0, EltVT)); 3158 } 3159 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElems); 3160 break; 3161 } 3162 } 3163 break; 3164 } 3165 3166 case ISD::SHL_PARTS: 3167 case ISD::SRA_PARTS: 3168 case ISD::SRL_PARTS: { 3169 SmallVector<SDValue, 8> Ops; 3170 bool Changed = false; 3171 unsigned N = Node->getNumOperands(); 3172 for (unsigned i = 0; i + 1 < N; ++i) { 3173 Ops.push_back(LegalizeOp(Node->getOperand(i))); 3174 Changed |= Ops.back() != Node->getOperand(i); 3175 } 3176 Ops.push_back(LegalizeOp(DAG.getShiftAmountOperand(Node->getOperand(N-1)))); 3177 Changed |= Ops.back() != Node->getOperand(N-1); 3178 if (Changed) 3179 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 3180 3181 switch (TLI.getOperationAction(Node->getOpcode(), 3182 Node->getValueType(0))) { 3183 default: assert(0 && "This action is not supported yet!"); 3184 case TargetLowering::Legal: break; 3185 case TargetLowering::Custom: 3186 Tmp1 = TLI.LowerOperation(Result, DAG); 3187 if (Tmp1.getNode()) { 3188 SDValue Tmp2, RetVal(0, 0); 3189 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { 3190 Tmp2 = LegalizeOp(Tmp1.getValue(i)); 3191 AddLegalizedOperand(SDValue(Node, i), Tmp2); 3192 if (i == Op.getResNo()) 3193 RetVal = Tmp2; 3194 } 3195 assert(RetVal.getNode() && "Illegal result number"); 3196 return RetVal; 3197 } 3198 break; 3199 } 3200 3201 // Since these produce multiple values, make sure to remember that we 3202 // legalized all of them. 3203 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 3204 AddLegalizedOperand(SDValue(Node, i), Result.getValue(i)); 3205 return Result.getValue(Op.getResNo()); 3206 } 3207 3208 // Binary operators 3209 case ISD::ADD: 3210 case ISD::SUB: 3211 case ISD::MUL: 3212 case ISD::MULHS: 3213 case ISD::MULHU: 3214 case ISD::UDIV: 3215 case ISD::SDIV: 3216 case ISD::AND: 3217 case ISD::OR: 3218 case ISD::XOR: 3219 case ISD::SHL: 3220 case ISD::SRL: 3221 case ISD::SRA: 3222 case ISD::FADD: 3223 case ISD::FSUB: 3224 case ISD::FMUL: 3225 case ISD::FDIV: 3226 case ISD::FPOW: 3227 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3228 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3229 3230 if ((Node->getOpcode() == ISD::SHL || 3231 Node->getOpcode() == ISD::SRL || 3232 Node->getOpcode() == ISD::SRA) && 3233 !Node->getValueType(0).isVector()) 3234 Tmp2 = DAG.getShiftAmountOperand(Tmp2); 3235 3236 switch (getTypeAction(Tmp2.getValueType())) { 3237 case Expand: assert(0 && "Not possible"); 3238 case Legal: 3239 Tmp2 = LegalizeOp(Tmp2); // Legalize the RHS. 3240 break; 3241 case Promote: 3242 Tmp2 = PromoteOp(Tmp2); // Promote the RHS. 3243 break; 3244 } 3245 3246 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3247 3248 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3249 default: assert(0 && "BinOp legalize operation not supported"); 3250 case TargetLowering::Legal: break; 3251 case TargetLowering::Custom: 3252 Tmp1 = TLI.LowerOperation(Result, DAG); 3253 if (Tmp1.getNode()) { 3254 Result = Tmp1; 3255 break; 3256 } 3257 // Fall through if the custom lower can't deal with the operation 3258 case TargetLowering::Expand: { 3259 MVT VT = Op.getValueType(); 3260 3261 // See if multiply or divide can be lowered using two-result operations. 3262 SDVTList VTs = DAG.getVTList(VT, VT); 3263 if (Node->getOpcode() == ISD::MUL) { 3264 // We just need the low half of the multiply; try both the signed 3265 // and unsigned forms. If the target supports both SMUL_LOHI and 3266 // UMUL_LOHI, form a preference by checking which forms of plain 3267 // MULH it supports. 3268 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3269 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3270 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3271 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3272 unsigned OpToUse = 0; 3273 if (HasSMUL_LOHI && !HasMULHS) { 3274 OpToUse = ISD::SMUL_LOHI; 3275 } else if (HasUMUL_LOHI && !HasMULHU) { 3276 OpToUse = ISD::UMUL_LOHI; 3277 } else if (HasSMUL_LOHI) { 3278 OpToUse = ISD::SMUL_LOHI; 3279 } else if (HasUMUL_LOHI) { 3280 OpToUse = ISD::UMUL_LOHI; 3281 } 3282 if (OpToUse) { 3283 Result = SDValue(DAG.getNode(OpToUse, dl, VTs, Tmp1, Tmp2).getNode(), 3284 0); 3285 break; 3286 } 3287 } 3288 if (Node->getOpcode() == ISD::MULHS && 3289 TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) { 3290 Result = SDValue(DAG.getNode(ISD::SMUL_LOHI, dl, 3291 VTs, Tmp1, Tmp2).getNode(), 3292 1); 3293 break; 3294 } 3295 if (Node->getOpcode() == ISD::MULHU && 3296 TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT)) { 3297 Result = SDValue(DAG.getNode(ISD::UMUL_LOHI, dl, 3298 VTs, Tmp1, Tmp2).getNode(), 3299 1); 3300 break; 3301 } 3302 if (Node->getOpcode() == ISD::SDIV && 3303 TLI.isOperationLegalOrCustom(ISD::SDIVREM, VT)) { 3304 Result = SDValue(DAG.getNode(ISD::SDIVREM, dl, 3305 VTs, Tmp1, Tmp2).getNode(), 3306 0); 3307 break; 3308 } 3309 if (Node->getOpcode() == ISD::UDIV && 3310 TLI.isOperationLegalOrCustom(ISD::UDIVREM, VT)) { 3311 Result = SDValue(DAG.getNode(ISD::UDIVREM, dl, 3312 VTs, Tmp1, Tmp2).getNode(), 3313 0); 3314 break; 3315 } 3316 3317 // Check to see if we have a libcall for this operator. 3318 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3319 bool isSigned = false; 3320 switch (Node->getOpcode()) { 3321 case ISD::UDIV: 3322 case ISD::SDIV: 3323 if (VT == MVT::i32) { 3324 LC = Node->getOpcode() == ISD::UDIV 3325 ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32; 3326 isSigned = Node->getOpcode() == ISD::SDIV; 3327 } 3328 break; 3329 case ISD::MUL: 3330 if (VT == MVT::i32) 3331 LC = RTLIB::MUL_I32; 3332 else if (VT == MVT::i64) 3333 LC = RTLIB::MUL_I64; 3334 break; 3335 case ISD::FPOW: 3336 LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, 3337 RTLIB::POW_PPCF128); 3338 break; 3339 case ISD::FDIV: 3340 LC = GetFPLibCall(VT, RTLIB::DIV_F32, RTLIB::DIV_F64, RTLIB::DIV_F80, 3341 RTLIB::DIV_PPCF128); 3342 break; 3343 default: break; 3344 } 3345 if (LC != RTLIB::UNKNOWN_LIBCALL) { 3346 SDValue Dummy; 3347 Result = ExpandLibCall(LC, Node, isSigned, Dummy); 3348 break; 3349 } 3350 3351 assert(Node->getValueType(0).isVector() && 3352 "Cannot expand this binary operator!"); 3353 // Expand the operation into a bunch of nasty scalar code. 3354 Result = LegalizeOp(UnrollVectorOp(Op)); 3355 break; 3356 } 3357 case TargetLowering::Promote: { 3358 switch (Node->getOpcode()) { 3359 default: assert(0 && "Do not know how to promote this BinOp!"); 3360 case ISD::AND: 3361 case ISD::OR: 3362 case ISD::XOR: { 3363 MVT OVT = Node->getValueType(0); 3364 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3365 assert(OVT.isVector() && "Cannot promote this BinOp!"); 3366 // Bit convert each of the values to the new type. 3367 Tmp1 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp1); 3368 Tmp2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp2); 3369 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3370 // Bit convert the result back the original type. 3371 Result = DAG.getNode(ISD::BIT_CONVERT, dl, OVT, Result); 3372 break; 3373 } 3374 } 3375 } 3376 } 3377 break; 3378 3379 case ISD::SMUL_LOHI: 3380 case ISD::UMUL_LOHI: 3381 case ISD::SDIVREM: 3382 case ISD::UDIVREM: 3383 // These nodes will only be produced by target-specific lowering, so 3384 // they shouldn't be here if they aren't legal. 3385 assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) && 3386 "This must be legal!"); 3387 3388 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3389 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3390 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3391 break; 3392 3393 case ISD::FCOPYSIGN: // FCOPYSIGN does not require LHS/RHS to match type! 3394 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3395 switch (getTypeAction(Node->getOperand(1).getValueType())) { 3396 case Expand: assert(0 && "Not possible"); 3397 case Legal: 3398 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS. 3399 break; 3400 case Promote: 3401 Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the RHS. 3402 break; 3403 } 3404 3405 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3406 3407 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3408 default: assert(0 && "Operation not supported"); 3409 case TargetLowering::Custom: 3410 Tmp1 = TLI.LowerOperation(Result, DAG); 3411 if (Tmp1.getNode()) Result = Tmp1; 3412 break; 3413 case TargetLowering::Legal: break; 3414 case TargetLowering::Expand: { 3415 // If this target supports fabs/fneg natively and select is cheap, 3416 // do this efficiently. 3417 if (!TLI.isSelectExpensive() && 3418 TLI.getOperationAction(ISD::FABS, Tmp1.getValueType()) == 3419 TargetLowering::Legal && 3420 TLI.getOperationAction(ISD::FNEG, Tmp1.getValueType()) == 3421 TargetLowering::Legal) { 3422 // Get the sign bit of the RHS. 3423 MVT IVT = 3424 Tmp2.getValueType() == MVT::f32 ? MVT::i32 : MVT::i64; 3425 SDValue SignBit = DAG.getNode(ISD::BIT_CONVERT, dl, IVT, Tmp2); 3426 SignBit = DAG.getSetCC(dl, TLI.getSetCCResultType(IVT), 3427 SignBit, DAG.getConstant(0, IVT), ISD::SETLT); 3428 // Get the absolute value of the result. 3429 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); 3430 // Select between the nabs and abs value based on the sign bit of 3431 // the input. 3432 Result = DAG.getNode(ISD::SELECT, dl, AbsVal.getValueType(), SignBit, 3433 DAG.getNode(ISD::FNEG, AbsVal.getValueType(), 3434 AbsVal), 3435 AbsVal); 3436 Result = LegalizeOp(Result); 3437 break; 3438 } 3439 3440 // Otherwise, do bitwise ops! 3441 MVT NVT = 3442 Node->getValueType(0) == MVT::f32 ? MVT::i32 : MVT::i64; 3443 Result = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI); 3444 Result = DAG.getNode(ISD::BIT_CONVERT, dl, Node->getValueType(0), Result); 3445 Result = LegalizeOp(Result); 3446 break; 3447 } 3448 } 3449 break; 3450 3451 case ISD::ADDC: 3452 case ISD::SUBC: 3453 Tmp1 = LegalizeOp(Node->getOperand(0)); 3454 Tmp2 = LegalizeOp(Node->getOperand(1)); 3455 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3456 Tmp3 = Result.getValue(0); 3457 Tmp4 = Result.getValue(1); 3458 3459 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3460 default: assert(0 && "This action is not supported yet!"); 3461 case TargetLowering::Legal: 3462 break; 3463 case TargetLowering::Custom: 3464 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 3465 if (Tmp1.getNode() != NULL) { 3466 Tmp3 = LegalizeOp(Tmp1); 3467 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 3468 } 3469 break; 3470 } 3471 // Since this produces two values, make sure to remember that we legalized 3472 // both of them. 3473 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 3474 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 3475 return Op.getResNo() ? Tmp4 : Tmp3; 3476 3477 case ISD::ADDE: 3478 case ISD::SUBE: 3479 Tmp1 = LegalizeOp(Node->getOperand(0)); 3480 Tmp2 = LegalizeOp(Node->getOperand(1)); 3481 Tmp3 = LegalizeOp(Node->getOperand(2)); 3482 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 3483 Tmp3 = Result.getValue(0); 3484 Tmp4 = Result.getValue(1); 3485 3486 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3487 default: assert(0 && "This action is not supported yet!"); 3488 case TargetLowering::Legal: 3489 break; 3490 case TargetLowering::Custom: 3491 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 3492 if (Tmp1.getNode() != NULL) { 3493 Tmp3 = LegalizeOp(Tmp1); 3494 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 3495 } 3496 break; 3497 } 3498 // Since this produces two values, make sure to remember that we legalized 3499 // both of them. 3500 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 3501 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 3502 return Op.getResNo() ? Tmp4 : Tmp3; 3503 3504 case ISD::BUILD_PAIR: { 3505 MVT PairTy = Node->getValueType(0); 3506 // TODO: handle the case where the Lo and Hi operands are not of legal type 3507 Tmp1 = LegalizeOp(Node->getOperand(0)); // Lo 3508 Tmp2 = LegalizeOp(Node->getOperand(1)); // Hi 3509 switch (TLI.getOperationAction(ISD::BUILD_PAIR, PairTy)) { 3510 case TargetLowering::Promote: 3511 case TargetLowering::Custom: 3512 assert(0 && "Cannot promote/custom this yet!"); 3513 case TargetLowering::Legal: 3514 if (Tmp1 != Node->getOperand(0) || Tmp2 != Node->getOperand(1)) 3515 Result = DAG.getNode(ISD::BUILD_PAIR, dl, PairTy, Tmp1, Tmp2); 3516 break; 3517 case TargetLowering::Expand: 3518 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Tmp1); 3519 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Tmp2); 3520 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, 3521 DAG.getConstant(PairTy.getSizeInBits()/2, 3522 TLI.getShiftAmountTy())); 3523 Result = DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2); 3524 break; 3525 } 3526 break; 3527 } 3528 3529 case ISD::UREM: 3530 case ISD::SREM: 3531 case ISD::FREM: 3532 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3533 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3534 3535 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3536 case TargetLowering::Promote: assert(0 && "Cannot promote this yet!"); 3537 case TargetLowering::Custom: 3538 isCustom = true; 3539 // FALLTHROUGH 3540 case TargetLowering::Legal: 3541 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3542 if (isCustom) { 3543 Tmp1 = TLI.LowerOperation(Result, DAG); 3544 if (Tmp1.getNode()) Result = Tmp1; 3545 } 3546 break; 3547 case TargetLowering::Expand: { 3548 unsigned DivOpc= (Node->getOpcode() == ISD::UREM) ? ISD::UDIV : ISD::SDIV; 3549 bool isSigned = DivOpc == ISD::SDIV; 3550 MVT VT = Node->getValueType(0); 3551 3552 // See if remainder can be lowered using two-result operations. 3553 SDVTList VTs = DAG.getVTList(VT, VT); 3554 if (Node->getOpcode() == ISD::SREM && 3555 TLI.isOperationLegalOrCustom(ISD::SDIVREM, VT)) { 3556 Result = SDValue(DAG.getNode(ISD::SDIVREM, dl, 3557 VTs, Tmp1, Tmp2).getNode(), 1); 3558 break; 3559 } 3560 if (Node->getOpcode() == ISD::UREM && 3561 TLI.isOperationLegalOrCustom(ISD::UDIVREM, VT)) { 3562 Result = SDValue(DAG.getNode(ISD::UDIVREM, dl, 3563 VTs, Tmp1, Tmp2).getNode(), 1); 3564 break; 3565 } 3566 3567 if (VT.isInteger()) { 3568 if (TLI.getOperationAction(DivOpc, VT) == 3569 TargetLowering::Legal) { 3570 // X % Y -> X-X/Y*Y 3571 Result = DAG.getNode(DivOpc, dl, VT, Tmp1, Tmp2); 3572 Result = DAG.getNode(ISD::MUL, dl, VT, Result, Tmp2); 3573 Result = DAG.getNode(ISD::SUB, dl, VT, Tmp1, Result); 3574 } else if (VT.isVector()) { 3575 Result = LegalizeOp(UnrollVectorOp(Op)); 3576 } else { 3577 assert(VT == MVT::i32 && 3578 "Cannot expand this binary operator!"); 3579 RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM 3580 ? RTLIB::UREM_I32 : RTLIB::SREM_I32; 3581 SDValue Dummy; 3582 Result = ExpandLibCall(LC, Node, isSigned, Dummy); 3583 } 3584 } else { 3585 assert(VT.isFloatingPoint() && 3586 "remainder op must have integer or floating-point type"); 3587 if (VT.isVector()) { 3588 Result = LegalizeOp(UnrollVectorOp(Op)); 3589 } else { 3590 // Floating point mod -> fmod libcall. 3591 RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::REM_F32, RTLIB::REM_F64, 3592 RTLIB::REM_F80, RTLIB::REM_PPCF128); 3593 SDValue Dummy; 3594 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3595 } 3596 } 3597 break; 3598 } 3599 } 3600 break; 3601 case ISD::VAARG: { 3602 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3603 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3604 3605 MVT VT = Node->getValueType(0); 3606 switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) { 3607 default: assert(0 && "This action is not supported yet!"); 3608 case TargetLowering::Custom: 3609 isCustom = true; 3610 // FALLTHROUGH 3611 case TargetLowering::Legal: 3612 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3613 Result = Result.getValue(0); 3614 Tmp1 = Result.getValue(1); 3615 3616 if (isCustom) { 3617 Tmp2 = TLI.LowerOperation(Result, DAG); 3618 if (Tmp2.getNode()) { 3619 Result = LegalizeOp(Tmp2); 3620 Tmp1 = LegalizeOp(Tmp2.getValue(1)); 3621 } 3622 } 3623 break; 3624 case TargetLowering::Expand: { 3625 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 3626 SDValue VAList = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, V, 0); 3627 // Increment the pointer, VAList, to the next vaarg 3628 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 3629 DAG.getConstant(TLI.getTargetData()-> 3630 getTypePaddedSize(VT.getTypeForMVT()), 3631 TLI.getPointerTy())); 3632 // Store the incremented VAList to the legalized pointer 3633 Tmp3 = DAG.getStore(VAList.getValue(1), dl, Tmp3, Tmp2, V, 0); 3634 // Load the actual argument out of the pointer VAList 3635 Result = DAG.getLoad(VT, dl, Tmp3, VAList, NULL, 0); 3636 Tmp1 = LegalizeOp(Result.getValue(1)); 3637 Result = LegalizeOp(Result); 3638 break; 3639 } 3640 } 3641 // Since VAARG produces two values, make sure to remember that we 3642 // legalized both of them. 3643 AddLegalizedOperand(SDValue(Node, 0), Result); 3644 AddLegalizedOperand(SDValue(Node, 1), Tmp1); 3645 return Op.getResNo() ? Tmp1 : Result; 3646 } 3647 3648 case ISD::VACOPY: 3649 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3650 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the dest pointer. 3651 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the source pointer. 3652 3653 switch (TLI.getOperationAction(ISD::VACOPY, MVT::Other)) { 3654 default: assert(0 && "This action is not supported yet!"); 3655 case TargetLowering::Custom: 3656 isCustom = true; 3657 // FALLTHROUGH 3658 case TargetLowering::Legal: 3659 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, 3660 Node->getOperand(3), Node->getOperand(4)); 3661 if (isCustom) { 3662 Tmp1 = TLI.LowerOperation(Result, DAG); 3663 if (Tmp1.getNode()) Result = Tmp1; 3664 } 3665 break; 3666 case TargetLowering::Expand: 3667 // This defaults to loading a pointer from the input and storing it to the 3668 // output, returning the chain. 3669 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 3670 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 3671 Tmp4 = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp3, VS, 0); 3672 Result = DAG.getStore(Tmp4.getValue(1), dl, Tmp4, Tmp2, VD, 0); 3673 break; 3674 } 3675 break; 3676 3677 case ISD::VAEND: 3678 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3679 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3680 3681 switch (TLI.getOperationAction(ISD::VAEND, MVT::Other)) { 3682 default: assert(0 && "This action is not supported yet!"); 3683 case TargetLowering::Custom: 3684 isCustom = true; 3685 // FALLTHROUGH 3686 case TargetLowering::Legal: 3687 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3688 if (isCustom) { 3689 Tmp1 = TLI.LowerOperation(Tmp1, DAG); 3690 if (Tmp1.getNode()) Result = Tmp1; 3691 } 3692 break; 3693 case TargetLowering::Expand: 3694 Result = Tmp1; // Default to a no-op, return the chain 3695 break; 3696 } 3697 break; 3698 3699 case ISD::VASTART: 3700 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3701 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3702 3703 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3704 3705 switch (TLI.getOperationAction(ISD::VASTART, MVT::Other)) { 3706 default: assert(0 && "This action is not supported yet!"); 3707 case TargetLowering::Legal: break; 3708 case TargetLowering::Custom: 3709 Tmp1 = TLI.LowerOperation(Result, DAG); 3710 if (Tmp1.getNode()) Result = Tmp1; 3711 break; 3712 } 3713 break; 3714 3715 case ISD::ROTL: 3716 case ISD::ROTR: 3717 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3718 Tmp2 = LegalizeOp(DAG.getShiftAmountOperand(Node->getOperand(1))); // RHS 3719 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3720 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3721 default: 3722 assert(0 && "ROTL/ROTR legalize operation not supported"); 3723 break; 3724 case TargetLowering::Legal: 3725 break; 3726 case TargetLowering::Custom: 3727 Tmp1 = TLI.LowerOperation(Result, DAG); 3728 if (Tmp1.getNode()) Result = Tmp1; 3729 break; 3730 case TargetLowering::Promote: 3731 assert(0 && "Do not know how to promote ROTL/ROTR"); 3732 break; 3733 case TargetLowering::Expand: 3734 assert(0 && "Do not know how to expand ROTL/ROTR"); 3735 break; 3736 } 3737 break; 3738 3739 case ISD::BSWAP: 3740 Tmp1 = LegalizeOp(Node->getOperand(0)); // Op 3741 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3742 case TargetLowering::Custom: 3743 assert(0 && "Cannot custom legalize this yet!"); 3744 case TargetLowering::Legal: 3745 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3746 break; 3747 case TargetLowering::Promote: { 3748 MVT OVT = Tmp1.getValueType(); 3749 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3750 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 3751 3752 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1); 3753 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 3754 Result = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 3755 DAG.getConstant(DiffBits, TLI.getShiftAmountTy())); 3756 break; 3757 } 3758 case TargetLowering::Expand: 3759 Result = ExpandBSWAP(Tmp1, dl); 3760 break; 3761 } 3762 break; 3763 3764 case ISD::CTPOP: 3765 case ISD::CTTZ: 3766 case ISD::CTLZ: 3767 Tmp1 = LegalizeOp(Node->getOperand(0)); // Op 3768 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3769 case TargetLowering::Custom: 3770 case TargetLowering::Legal: 3771 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3772 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 3773 TargetLowering::Custom) { 3774 Tmp1 = TLI.LowerOperation(Result, DAG); 3775 if (Tmp1.getNode()) { 3776 Result = Tmp1; 3777 } 3778 } 3779 break; 3780 case TargetLowering::Promote: { 3781 MVT OVT = Tmp1.getValueType(); 3782 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3783 3784 // Zero extend the argument. 3785 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1); 3786 // Perform the larger operation, then subtract if needed. 3787 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1); 3788 switch (Node->getOpcode()) { 3789 case ISD::CTPOP: 3790 Result = Tmp1; 3791 break; 3792 case ISD::CTTZ: 3793 //if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT) 3794 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), 3795 Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), 3796 ISD::SETEQ); 3797 Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, 3798 DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); 3799 break; 3800 case ISD::CTLZ: 3801 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 3802 Result = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 3803 DAG.getConstant(NVT.getSizeInBits() - 3804 OVT.getSizeInBits(), NVT)); 3805 break; 3806 } 3807 break; 3808 } 3809 case TargetLowering::Expand: 3810 Result = ExpandBitCount(Node->getOpcode(), Tmp1, dl); 3811 break; 3812 } 3813 break; 3814 3815 // Unary operators 3816 case ISD::FABS: 3817 case ISD::FNEG: 3818 case ISD::FSQRT: 3819 case ISD::FSIN: 3820 case ISD::FCOS: 3821 case ISD::FLOG: 3822 case ISD::FLOG2: 3823 case ISD::FLOG10: 3824 case ISD::FEXP: 3825 case ISD::FEXP2: 3826 case ISD::FTRUNC: 3827 case ISD::FFLOOR: 3828 case ISD::FCEIL: 3829 case ISD::FRINT: 3830 case ISD::FNEARBYINT: 3831 Tmp1 = LegalizeOp(Node->getOperand(0)); 3832 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3833 case TargetLowering::Promote: 3834 case TargetLowering::Custom: 3835 isCustom = true; 3836 // FALLTHROUGH 3837 case TargetLowering::Legal: 3838 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3839 if (isCustom) { 3840 Tmp1 = TLI.LowerOperation(Result, DAG); 3841 if (Tmp1.getNode()) Result = Tmp1; 3842 } 3843 break; 3844 case TargetLowering::Expand: 3845 switch (Node->getOpcode()) { 3846 default: assert(0 && "Unreachable!"); 3847 case ISD::FNEG: 3848 // Expand Y = FNEG(X) -> Y = SUB -0.0, X 3849 Tmp2 = DAG.getConstantFP(-0.0, Node->getValueType(0)); 3850 Result = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp2, Tmp1); 3851 break; 3852 case ISD::FABS: { 3853 // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). 3854 MVT VT = Node->getValueType(0); 3855 Tmp2 = DAG.getConstantFP(0.0, VT); 3856 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), 3857 Tmp1, Tmp2, ISD::SETUGT); 3858 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); 3859 Result = DAG.getNode(ISD::SELECT, dl, VT, Tmp2, Tmp1, Tmp3); 3860 break; 3861 } 3862 case ISD::FSQRT: 3863 case ISD::FSIN: 3864 case ISD::FCOS: 3865 case ISD::FLOG: 3866 case ISD::FLOG2: 3867 case ISD::FLOG10: 3868 case ISD::FEXP: 3869 case ISD::FEXP2: 3870 case ISD::FTRUNC: 3871 case ISD::FFLOOR: 3872 case ISD::FCEIL: 3873 case ISD::FRINT: 3874 case ISD::FNEARBYINT: { 3875 MVT VT = Node->getValueType(0); 3876 3877 // Expand unsupported unary vector operators by unrolling them. 3878 if (VT.isVector()) { 3879 Result = LegalizeOp(UnrollVectorOp(Op)); 3880 break; 3881 } 3882 3883 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3884 switch(Node->getOpcode()) { 3885 case ISD::FSQRT: 3886 LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 3887 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128); 3888 break; 3889 case ISD::FSIN: 3890 LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64, 3891 RTLIB::SIN_F80, RTLIB::SIN_PPCF128); 3892 break; 3893 case ISD::FCOS: 3894 LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, 3895 RTLIB::COS_F80, RTLIB::COS_PPCF128); 3896 break; 3897 case ISD::FLOG: 3898 LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, 3899 RTLIB::LOG_F80, RTLIB::LOG_PPCF128); 3900 break; 3901 case ISD::FLOG2: 3902 LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 3903 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); 3904 break; 3905 case ISD::FLOG10: 3906 LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 3907 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); 3908 break; 3909 case ISD::FEXP: 3910 LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, 3911 RTLIB::EXP_F80, RTLIB::EXP_PPCF128); 3912 break; 3913 case ISD::FEXP2: 3914 LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 3915 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); 3916 break; 3917 case ISD::FTRUNC: 3918 LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 3919 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); 3920 break; 3921 case ISD::FFLOOR: 3922 LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 3923 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); 3924 break; 3925 case ISD::FCEIL: 3926 LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 3927 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); 3928 break; 3929 case ISD::FRINT: 3930 LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, 3931 RTLIB::RINT_F80, RTLIB::RINT_PPCF128); 3932 break; 3933 case ISD::FNEARBYINT: 3934 LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, 3935 RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); 3936 break; 3937 break; 3938 default: assert(0 && "Unreachable!"); 3939 } 3940 SDValue Dummy; 3941 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3942 break; 3943 } 3944 } 3945 break; 3946 } 3947 break; 3948 case ISD::FPOWI: { 3949 MVT VT = Node->getValueType(0); 3950 3951 // Expand unsupported unary vector operators by unrolling them. 3952 if (VT.isVector()) { 3953 Result = LegalizeOp(UnrollVectorOp(Op)); 3954 break; 3955 } 3956 3957 // We always lower FPOWI into a libcall. No target support for it yet. 3958 RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, 3959 RTLIB::POWI_F80, RTLIB::POWI_PPCF128); 3960 SDValue Dummy; 3961 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3962 break; 3963 } 3964 case ISD::BIT_CONVERT: 3965 if (!isTypeLegal(Node->getOperand(0).getValueType())) { 3966 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 3967 Node->getValueType(0), dl); 3968 } else if (Op.getOperand(0).getValueType().isVector()) { 3969 // The input has to be a vector type, we have to either scalarize it, pack 3970 // it, or convert it based on whether the input vector type is legal. 3971 SDNode *InVal = Node->getOperand(0).getNode(); 3972 int InIx = Node->getOperand(0).getResNo(); 3973 unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements(); 3974 MVT EVT = InVal->getValueType(InIx).getVectorElementType(); 3975 3976 // Figure out if there is a simple type corresponding to this Vector 3977 // type. If so, convert to the vector type. 3978 MVT TVT = MVT::getVectorVT(EVT, NumElems); 3979 if (TLI.isTypeLegal(TVT)) { 3980 // Turn this into a bit convert of the vector input. 3981 Result = DAG.getNode(ISD::BIT_CONVERT, dl, Node->getValueType(0), 3982 LegalizeOp(Node->getOperand(0))); 3983 break; 3984 } else if (NumElems == 1) { 3985 // Turn this into a bit convert of the scalar input. 3986 Result = DAG.getNode(ISD::BIT_CONVERT, dl, Node->getValueType(0), 3987 ScalarizeVectorOp(Node->getOperand(0))); 3988 break; 3989 } else { 3990 // FIXME: UNIMP! Store then reload 3991 assert(0 && "Cast from unsupported vector type not implemented yet!"); 3992 } 3993 } else { 3994 switch (TLI.getOperationAction(ISD::BIT_CONVERT, 3995 Node->getOperand(0).getValueType())) { 3996 default: assert(0 && "Unknown operation action!"); 3997 case TargetLowering::Expand: 3998 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 3999 Node->getValueType(0), dl); 4000 break; 4001 case TargetLowering::Legal: 4002 Tmp1 = LegalizeOp(Node->getOperand(0)); 4003 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4004 break; 4005 } 4006 } 4007 break; 4008 case ISD::CONVERT_RNDSAT: { 4009 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 4010 switch (CvtCode) { 4011 default: assert(0 && "Unknown cvt code!"); 4012 case ISD::CVT_SF: 4013 case ISD::CVT_UF: 4014 case ISD::CVT_FF: 4015 break; 4016 case ISD::CVT_FS: 4017 case ISD::CVT_FU: 4018 case ISD::CVT_SS: 4019 case ISD::CVT_SU: 4020 case ISD::CVT_US: 4021 case ISD::CVT_UU: { 4022 SDValue DTyOp = Node->getOperand(1); 4023 SDValue STyOp = Node->getOperand(2); 4024 SDValue RndOp = Node->getOperand(3); 4025 SDValue SatOp = Node->getOperand(4); 4026 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4027 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4028 case Legal: 4029 Tmp1 = LegalizeOp(Node->getOperand(0)); 4030 Result = DAG.UpdateNodeOperands(Result, Tmp1, DTyOp, STyOp, 4031 RndOp, SatOp); 4032 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 4033 TargetLowering::Custom) { 4034 Tmp1 = TLI.LowerOperation(Result, DAG); 4035 if (Tmp1.getNode()) Result = Tmp1; 4036 } 4037 break; 4038 case Promote: 4039 Result = PromoteOp(Node->getOperand(0)); 4040 // For FP, make Op1 a i32 4041 4042 Result = DAG.getConvertRndSat(Op.getValueType(), dl, Result, 4043 DTyOp, STyOp, RndOp, SatOp, CvtCode); 4044 break; 4045 } 4046 break; 4047 } 4048 } // end switch CvtCode 4049 break; 4050 } 4051 // Conversion operators. The source and destination have different types. 4052 case ISD::SINT_TO_FP: 4053 case ISD::UINT_TO_FP: { 4054 bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; 4055 Result = LegalizeINT_TO_FP(Result, isSigned, 4056 Node->getValueType(0), Node->getOperand(0), dl); 4057 break; 4058 } 4059 case ISD::TRUNCATE: 4060 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4061 case Legal: 4062 Tmp1 = LegalizeOp(Node->getOperand(0)); 4063 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 4064 default: assert(0 && "Unknown TRUNCATE legalization operation action!"); 4065 case TargetLowering::Custom: 4066 isCustom = true; 4067 // FALLTHROUGH 4068 case TargetLowering::Legal: 4069 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4070 if (isCustom) { 4071 Tmp1 = TLI.LowerOperation(Result, DAG); 4072 if (Tmp1.getNode()) Result = Tmp1; 4073 } 4074 break; 4075 case TargetLowering::Expand: 4076 assert(Result.getValueType().isVector() && "must be vector type"); 4077 // Unroll the truncate. We should do better. 4078 Result = LegalizeOp(UnrollVectorOp(Result)); 4079 } 4080 break; 4081 case Expand: 4082 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 4083 4084 // Since the result is legal, we should just be able to truncate the low 4085 // part of the source. 4086 Result = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 4087 break; 4088 case Promote: 4089 Result = PromoteOp(Node->getOperand(0)); 4090 Result = DAG.getNode(ISD::TRUNCATE, dl, Op.getValueType(), Result); 4091 break; 4092 } 4093 break; 4094 4095 case ISD::FP_TO_SINT: 4096 case ISD::FP_TO_UINT: 4097 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4098 case Legal: 4099 Tmp1 = LegalizeOp(Node->getOperand(0)); 4100 4101 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))){ 4102 default: assert(0 && "Unknown operation action!"); 4103 case TargetLowering::Custom: 4104 isCustom = true; 4105 // FALLTHROUGH 4106 case TargetLowering::Legal: 4107 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4108 if (isCustom) { 4109 Tmp1 = TLI.LowerOperation(Result, DAG); 4110 if (Tmp1.getNode()) Result = Tmp1; 4111 } 4112 break; 4113 case TargetLowering::Promote: 4114 Result = PromoteLegalFP_TO_INT(Tmp1, Node->getValueType(0), 4115 Node->getOpcode() == ISD::FP_TO_SINT, 4116 dl); 4117 break; 4118 case TargetLowering::Expand: 4119 if (Node->getOpcode() == ISD::FP_TO_UINT) { 4120 SDValue True, False; 4121 MVT VT = Node->getOperand(0).getValueType(); 4122 MVT NVT = Node->getValueType(0); 4123 const uint64_t zero[] = {0, 0}; 4124 APFloat apf = APFloat(APInt(VT.getSizeInBits(), 2, zero)); 4125 APInt x = APInt::getSignBit(NVT.getSizeInBits()); 4126 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); 4127 Tmp2 = DAG.getConstantFP(apf, VT); 4128 Tmp3 = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), 4129 Node->getOperand(0), 4130 Tmp2, ISD::SETLT); 4131 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); 4132 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, 4133 DAG.getNode(ISD::FSUB, dl, VT, 4134 Node->getOperand(0), Tmp2)); 4135 False = DAG.getNode(ISD::XOR, dl, NVT, False, 4136 DAG.getConstant(x, NVT)); 4137 Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp3, True, False); 4138 break; 4139 } else { 4140 assert(0 && "Do not know how to expand FP_TO_SINT yet!"); 4141 } 4142 break; 4143 } 4144 break; 4145 case Expand: { 4146 MVT VT = Op.getValueType(); 4147 MVT OVT = Node->getOperand(0).getValueType(); 4148 // Convert ppcf128 to i32 4149 if (OVT == MVT::ppcf128 && VT == MVT::i32) { 4150 if (Node->getOpcode() == ISD::FP_TO_SINT) { 4151 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, MVT::ppcf128, 4152 Node->getOperand(0), DAG.getValueType(MVT::f64)); 4153 Result = DAG.getNode(ISD::FP_ROUND, dl, MVT::f64, Result, 4154 DAG.getIntPtrConstant(1)); 4155 Result = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Result); 4156 } else { 4157 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; 4158 APFloat apf = APFloat(APInt(128, 2, TwoE31)); 4159 Tmp2 = DAG.getConstantFP(apf, OVT); 4160 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X 4161 // FIXME: generated code sucks. 4162 Result = DAG.getNode(ISD::SELECT_CC, dl, VT, Node->getOperand(0), 4163 Tmp2, 4164 DAG.getNode(ISD::ADD, dl, MVT::i32, 4165 DAG.getNode(ISD::FP_TO_SINT, dl, VT, 4166 DAG.getNode(ISD::FSUB, dl, OVT, 4167 Node->getOperand(0), Tmp2)), 4168 DAG.getConstant(0x80000000, MVT::i32)), 4169 DAG.getNode(ISD::FP_TO_SINT, dl, VT, 4170 Node->getOperand(0)), 4171 DAG.getCondCode(ISD::SETGE)); 4172 } 4173 break; 4174 } 4175 // Convert f32 / f64 to i32 / i64 / i128. 4176 RTLIB::Libcall LC = (Node->getOpcode() == ISD::FP_TO_SINT) ? 4177 RTLIB::getFPTOSINT(OVT, VT) : RTLIB::getFPTOUINT(OVT, VT); 4178 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpectd fp-to-int conversion!"); 4179 SDValue Dummy; 4180 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 4181 break; 4182 } 4183 case Promote: 4184 Tmp1 = PromoteOp(Node->getOperand(0)); 4185 Result = DAG.UpdateNodeOperands(Result, LegalizeOp(Tmp1)); 4186 Result = LegalizeOp(Result); 4187 break; 4188 } 4189 break; 4190 4191 case ISD::FP_EXTEND: { 4192 MVT DstVT = Op.getValueType(); 4193 MVT SrcVT = Op.getOperand(0).getValueType(); 4194 if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) { 4195 // The only other way we can lower this is to turn it into a STORE, 4196 // LOAD pair, targetting a temporary location (a stack slot). 4197 Result = EmitStackConvert(Node->getOperand(0), SrcVT, DstVT, dl); 4198 break; 4199 } 4200 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4201 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4202 case Legal: 4203 Tmp1 = LegalizeOp(Node->getOperand(0)); 4204 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4205 break; 4206 case Promote: 4207 Tmp1 = PromoteOp(Node->getOperand(0)); 4208 Result = DAG.getNode(ISD::FP_EXTEND, dl, Op.getValueType(), Tmp1); 4209 break; 4210 } 4211 break; 4212 } 4213 case ISD::FP_ROUND: { 4214 MVT DstVT = Op.getValueType(); 4215 MVT SrcVT = Op.getOperand(0).getValueType(); 4216 if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) { 4217 if (SrcVT == MVT::ppcf128) { 4218 SDValue Lo; 4219 ExpandOp(Node->getOperand(0), Lo, Result); 4220 // Round it the rest of the way (e.g. to f32) if needed. 4221 if (DstVT!=MVT::f64) 4222 Result = DAG.getNode(ISD::FP_ROUND, dl, 4223 DstVT, Result, Op.getOperand(1)); 4224 break; 4225 } 4226 // The only other way we can lower this is to turn it into a STORE, 4227 // LOAD pair, targetting a temporary location (a stack slot). 4228 Result = EmitStackConvert(Node->getOperand(0), DstVT, DstVT, dl); 4229 break; 4230 } 4231 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4232 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4233 case Legal: 4234 Tmp1 = LegalizeOp(Node->getOperand(0)); 4235 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 4236 break; 4237 case Promote: 4238 Tmp1 = PromoteOp(Node->getOperand(0)); 4239 Result = DAG.getNode(ISD::FP_ROUND, dl, Op.getValueType(), Tmp1, 4240 Node->getOperand(1)); 4241 break; 4242 } 4243 break; 4244 } 4245 case ISD::ANY_EXTEND: 4246 case ISD::ZERO_EXTEND: 4247 case ISD::SIGN_EXTEND: 4248 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4249 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4250 case Legal: 4251 Tmp1 = LegalizeOp(Node->getOperand(0)); 4252 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4253 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 4254 TargetLowering::Custom) { 4255 Tmp1 = TLI.LowerOperation(Result, DAG); 4256 if (Tmp1.getNode()) Result = Tmp1; 4257 } 4258 break; 4259 case Promote: 4260 switch (Node->getOpcode()) { 4261 case ISD::ANY_EXTEND: 4262 Tmp1 = PromoteOp(Node->getOperand(0)); 4263 Result = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), Tmp1); 4264 break; 4265 case ISD::ZERO_EXTEND: 4266 Result = PromoteOp(Node->getOperand(0)); 4267 Result = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), Result); 4268 Result = DAG.getZeroExtendInReg(Result, dl, 4269 Node->getOperand(0).getValueType()); 4270 break; 4271 case ISD::SIGN_EXTEND: 4272 Result = PromoteOp(Node->getOperand(0)); 4273 Result = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), Result); 4274 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Result.getValueType(), 4275 Result, 4276 DAG.getValueType(Node->getOperand(0).getValueType())); 4277 break; 4278 } 4279 } 4280 break; 4281 case ISD::FP_ROUND_INREG: 4282 case ISD::SIGN_EXTEND_INREG: { 4283 Tmp1 = LegalizeOp(Node->getOperand(0)); 4284 MVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 4285 4286 // If this operation is not supported, convert it to a shl/shr or load/store 4287 // pair. 4288 switch (TLI.getOperationAction(Node->getOpcode(), ExtraVT)) { 4289 default: assert(0 && "This action not supported for this op yet!"); 4290 case TargetLowering::Legal: 4291 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 4292 break; 4293 case TargetLowering::Expand: 4294 // If this is an integer extend and shifts are supported, do that. 4295 if (Node->getOpcode() == ISD::SIGN_EXTEND_INREG) { 4296 // NOTE: we could fall back on load/store here too for targets without 4297 // SAR. However, it is doubtful that any exist. 4298 unsigned BitsDiff = Node->getValueType(0).getSizeInBits() - 4299 ExtraVT.getSizeInBits(); 4300 SDValue ShiftCst = DAG.getConstant(BitsDiff, TLI.getShiftAmountTy()); 4301 Result = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 4302 Node->getOperand(0), ShiftCst); 4303 Result = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), 4304 Result, ShiftCst); 4305 } else if (Node->getOpcode() == ISD::FP_ROUND_INREG) { 4306 // The only way we can lower this is to turn it into a TRUNCSTORE, 4307 // EXTLOAD pair, targetting a temporary location (a stack slot). 4308 4309 // NOTE: there is a choice here between constantly creating new stack 4310 // slots and always reusing the same one. We currently always create 4311 // new ones, as reuse may inhibit scheduling. 4312 Result = EmitStackConvert(Node->getOperand(0), ExtraVT, 4313 Node->getValueType(0), dl); 4314 } else { 4315 assert(0 && "Unknown op"); 4316 } 4317 break; 4318 } 4319 break; 4320 } 4321 case ISD::TRAMPOLINE: { 4322 SDValue Ops[6]; 4323 for (unsigned i = 0; i != 6; ++i) 4324 Ops[i] = LegalizeOp(Node->getOperand(i)); 4325 Result = DAG.UpdateNodeOperands(Result, Ops, 6); 4326 // The only option for this node is to custom lower it. 4327 Result = TLI.LowerOperation(Result, DAG); 4328 assert(Result.getNode() && "Should always custom lower!"); 4329 4330 // Since trampoline produces two values, make sure to remember that we 4331 // legalized both of them. 4332 Tmp1 = LegalizeOp(Result.getValue(1)); 4333 Result = LegalizeOp(Result); 4334 AddLegalizedOperand(SDValue(Node, 0), Result); 4335 AddLegalizedOperand(SDValue(Node, 1), Tmp1); 4336 return Op.getResNo() ? Tmp1 : Result; 4337 } 4338 case ISD::FLT_ROUNDS_: { 4339 MVT VT = Node->getValueType(0); 4340 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4341 default: assert(0 && "This action not supported for this op yet!"); 4342 case TargetLowering::Custom: 4343 Result = TLI.LowerOperation(Op, DAG); 4344 if (Result.getNode()) break; 4345 // Fall Thru 4346 case TargetLowering::Legal: 4347 // If this operation is not supported, lower it to constant 1 4348 Result = DAG.getConstant(1, VT); 4349 break; 4350 } 4351 break; 4352 } 4353 case ISD::TRAP: { 4354 MVT VT = Node->getValueType(0); 4355 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4356 default: assert(0 && "This action not supported for this op yet!"); 4357 case TargetLowering::Legal: 4358 Tmp1 = LegalizeOp(Node->getOperand(0)); 4359 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4360 break; 4361 case TargetLowering::Custom: 4362 Result = TLI.LowerOperation(Op, DAG); 4363 if (Result.getNode()) break; 4364 // Fall Thru 4365 case TargetLowering::Expand: 4366 // If this operation is not supported, lower it to 'abort()' call 4367 Tmp1 = LegalizeOp(Node->getOperand(0)); 4368 TargetLowering::ArgListTy Args; 4369 std::pair<SDValue,SDValue> CallResult = 4370 TLI.LowerCallTo(Tmp1, Type::VoidTy, 4371 false, false, false, false, CallingConv::C, false, 4372 DAG.getExternalSymbol("abort", TLI.getPointerTy()), 4373 Args, DAG, dl); 4374 Result = CallResult.second; 4375 break; 4376 } 4377 break; 4378 } 4379 4380 case ISD::SADDO: 4381 case ISD::SSUBO: { 4382 MVT VT = Node->getValueType(0); 4383 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4384 default: assert(0 && "This action not supported for this op yet!"); 4385 case TargetLowering::Custom: 4386 Result = TLI.LowerOperation(Op, DAG); 4387 if (Result.getNode()) break; 4388 // FALLTHROUGH 4389 case TargetLowering::Legal: { 4390 SDValue LHS = LegalizeOp(Node->getOperand(0)); 4391 SDValue RHS = LegalizeOp(Node->getOperand(1)); 4392 4393 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 4394 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 4395 LHS, RHS); 4396 MVT OType = Node->getValueType(1); 4397 4398 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 4399 4400 // LHSSign -> LHS >= 0 4401 // RHSSign -> RHS >= 0 4402 // SumSign -> Sum >= 0 4403 // 4404 // Add: 4405 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 4406 // Sub: 4407 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 4408 // 4409 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 4410 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 4411 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 4412 Node->getOpcode() == ISD::SADDO ? 4413 ISD::SETEQ : ISD::SETNE); 4414 4415 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 4416 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 4417 4418 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 4419 4420 MVT ValueVTs[] = { LHS.getValueType(), OType }; 4421 SDValue Ops[] = { Sum, Cmp }; 4422 4423 Result = DAG.getNode(ISD::MERGE_VALUES, dl, 4424 DAG.getVTList(&ValueVTs[0], 2), 4425 &Ops[0], 2); 4426 SDNode *RNode = Result.getNode(); 4427 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), SDValue(RNode, 0)); 4428 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(RNode, 1)); 4429 break; 4430 } 4431 } 4432 4433 break; 4434 } 4435 case ISD::UADDO: 4436 case ISD::USUBO: { 4437 MVT VT = Node->getValueType(0); 4438 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4439 default: assert(0 && "This action not supported for this op yet!"); 4440 case TargetLowering::Custom: 4441 Result = TLI.LowerOperation(Op, DAG); 4442 if (Result.getNode()) break; 4443 // FALLTHROUGH 4444 case TargetLowering::Legal: { 4445 SDValue LHS = LegalizeOp(Node->getOperand(0)); 4446 SDValue RHS = LegalizeOp(Node->getOperand(1)); 4447 4448 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? 4449 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 4450 LHS, RHS); 4451 MVT OType = Node->getValueType(1); 4452 SDValue Cmp = DAG.getSetCC(dl, OType, Sum, LHS, 4453 Node->getOpcode () == ISD::UADDO ? 4454 ISD::SETULT : ISD::SETUGT); 4455 4456 MVT ValueVTs[] = { LHS.getValueType(), OType }; 4457 SDValue Ops[] = { Sum, Cmp }; 4458 4459 Result = DAG.getNode(ISD::MERGE_VALUES, dl, 4460 DAG.getVTList(&ValueVTs[0], 2), 4461 &Ops[0], 2); 4462 SDNode *RNode = Result.getNode(); 4463 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), SDValue(RNode, 0)); 4464 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(RNode, 1)); 4465 break; 4466 } 4467 } 4468 4469 break; 4470 } 4471 case ISD::SMULO: 4472 case ISD::UMULO: { 4473 MVT VT = Node->getValueType(0); 4474 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4475 default: assert(0 && "This action is not supported at all!"); 4476 case TargetLowering::Custom: 4477 Result = TLI.LowerOperation(Op, DAG); 4478 if (Result.getNode()) break; 4479 // Fall Thru 4480 case TargetLowering::Legal: 4481 // FIXME: According to Hacker's Delight, this can be implemented in 4482 // target independent lowering, but it would be inefficient, since it 4483 // requires a division + a branch. 4484 assert(0 && "Target independent lowering is not supported for SMULO/UMULO!"); 4485 break; 4486 } 4487 break; 4488 } 4489 4490 } 4491 4492 assert(Result.getValueType() == Op.getValueType() && 4493 "Bad legalization!"); 4494 4495 // Make sure that the generated code is itself legal. 4496 if (Result != Op) 4497 Result = LegalizeOp(Result); 4498 4499 // Note that LegalizeOp may be reentered even from single-use nodes, which 4500 // means that we always must cache transformed nodes. 4501 AddLegalizedOperand(Op, Result); 4502 return Result; 4503} 4504 4505/// PromoteOp - Given an operation that produces a value in an invalid type, 4506/// promote it to compute the value into a larger type. The produced value will 4507/// have the correct bits for the low portion of the register, but no guarantee 4508/// is made about the top bits: it may be zero, sign-extended, or garbage. 4509SDValue SelectionDAGLegalize::PromoteOp(SDValue Op) { 4510 MVT VT = Op.getValueType(); 4511 MVT NVT = TLI.getTypeToTransformTo(VT); 4512 assert(getTypeAction(VT) == Promote && 4513 "Caller should expand or legalize operands that are not promotable!"); 4514 assert(NVT.bitsGT(VT) && NVT.isInteger() == VT.isInteger() && 4515 "Cannot promote to smaller type!"); 4516 4517 SDValue Tmp1, Tmp2, Tmp3; 4518 SDValue Result; 4519 SDNode *Node = Op.getNode(); 4520 DebugLoc dl = Node->getDebugLoc(); 4521 4522 DenseMap<SDValue, SDValue>::iterator I = PromotedNodes.find(Op); 4523 if (I != PromotedNodes.end()) return I->second; 4524 4525 switch (Node->getOpcode()) { 4526 case ISD::CopyFromReg: 4527 assert(0 && "CopyFromReg must be legal!"); 4528 default: 4529#ifndef NDEBUG 4530 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 4531#endif 4532 assert(0 && "Do not know how to promote this operator!"); 4533 abort(); 4534 case ISD::UNDEF: 4535 Result = DAG.getNode(ISD::UNDEF, dl, NVT); 4536 break; 4537 case ISD::Constant: 4538 if (VT != MVT::i1) 4539 Result = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, Op); 4540 else 4541 Result = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Op); 4542 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold zext?"); 4543 break; 4544 case ISD::ConstantFP: 4545 Result = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Op); 4546 assert(isa<ConstantFPSDNode>(Result) && "Didn't constant fold fp_extend?"); 4547 break; 4548 4549 case ISD::SETCC: { 4550 MVT VT0 = Node->getOperand(0).getValueType(); 4551 assert(isTypeLegal(TLI.getSetCCResultType(VT0)) 4552 && "SetCC type is not legal??"); 4553 Result = DAG.getNode(ISD::SETCC, dl, TLI.getSetCCResultType(VT0), 4554 Node->getOperand(0), Node->getOperand(1), 4555 Node->getOperand(2)); 4556 break; 4557 } 4558 case ISD::TRUNCATE: 4559 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4560 case Legal: 4561 Result = LegalizeOp(Node->getOperand(0)); 4562 assert(Result.getValueType().bitsGE(NVT) && 4563 "This truncation doesn't make sense!"); 4564 if (Result.getValueType().bitsGT(NVT)) // Truncate to NVT instead of VT 4565 Result = DAG.getNode(ISD::TRUNCATE, dl, NVT, Result); 4566 break; 4567 case Promote: 4568 // The truncation is not required, because we don't guarantee anything 4569 // about high bits anyway. 4570 Result = PromoteOp(Node->getOperand(0)); 4571 break; 4572 case Expand: 4573 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 4574 // Truncate the low part of the expanded value to the result type 4575 Result = DAG.getNode(ISD::TRUNCATE, dl, NVT, Tmp1); 4576 } 4577 break; 4578 case ISD::SIGN_EXTEND: 4579 case ISD::ZERO_EXTEND: 4580 case ISD::ANY_EXTEND: 4581 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4582 case Expand: assert(0 && "BUG: Smaller reg should have been promoted!"); 4583 case Legal: 4584 // Input is legal? Just do extend all the way to the larger type. 4585 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Node->getOperand(0)); 4586 break; 4587 case Promote: 4588 // Promote the reg if it's smaller. 4589 Result = PromoteOp(Node->getOperand(0)); 4590 // The high bits are not guaranteed to be anything. Insert an extend. 4591 if (Node->getOpcode() == ISD::SIGN_EXTEND) 4592 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Result, 4593 DAG.getValueType(Node->getOperand(0).getValueType())); 4594 else if (Node->getOpcode() == ISD::ZERO_EXTEND) 4595 Result = DAG.getZeroExtendInReg(Result, dl, 4596 Node->getOperand(0).getValueType()); 4597 break; 4598 } 4599 break; 4600 case ISD::CONVERT_RNDSAT: { 4601 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 4602 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 4603 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 4604 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 4605 "can only promote integers"); 4606 Result = DAG.getConvertRndSat(NVT, dl, Node->getOperand(0), 4607 Node->getOperand(1), Node->getOperand(2), 4608 Node->getOperand(3), Node->getOperand(4), 4609 CvtCode); 4610 break; 4611 4612 } 4613 case ISD::BIT_CONVERT: 4614 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 4615 Node->getValueType(0), dl); 4616 Result = PromoteOp(Result); 4617 break; 4618 4619 case ISD::FP_EXTEND: 4620 assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); 4621 case ISD::FP_ROUND: 4622 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4623 case Expand: assert(0 && "BUG: Cannot expand FP regs!"); 4624 case Promote: assert(0 && "Unreachable with 2 FP types!"); 4625 case Legal: 4626 if (Node->getConstantOperandVal(1) == 0) { 4627 // Input is legal? Do an FP_ROUND_INREG. 4628 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Node->getOperand(0), 4629 DAG.getValueType(VT)); 4630 } else { 4631 // Just remove the truncate, it isn't affecting the value. 4632 Result = DAG.getNode(ISD::FP_ROUND, dl, NVT, Node->getOperand(0), 4633 Node->getOperand(1)); 4634 } 4635 break; 4636 } 4637 break; 4638 case ISD::SINT_TO_FP: 4639 case ISD::UINT_TO_FP: 4640 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4641 case Legal: 4642 // No extra round required here. 4643 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Node->getOperand(0)); 4644 break; 4645 4646 case Promote: 4647 Result = PromoteOp(Node->getOperand(0)); 4648 if (Node->getOpcode() == ISD::SINT_TO_FP) 4649 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Result.getValueType(), 4650 Result, 4651 DAG.getValueType(Node->getOperand(0).getValueType())); 4652 else 4653 Result = DAG.getZeroExtendInReg(Result, dl, 4654 Node->getOperand(0).getValueType()); 4655 // No extra round required here. 4656 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Result); 4657 break; 4658 case Expand: 4659 Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, NVT, 4660 Node->getOperand(0), dl); 4661 // Round if we cannot tolerate excess precision. 4662 if (NoExcessFPPrecision) 4663 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4664 DAG.getValueType(VT)); 4665 break; 4666 } 4667 break; 4668 4669 case ISD::SIGN_EXTEND_INREG: 4670 Result = PromoteOp(Node->getOperand(0)); 4671 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Result, 4672 Node->getOperand(1)); 4673 break; 4674 case ISD::FP_TO_SINT: 4675 case ISD::FP_TO_UINT: 4676 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4677 case Legal: 4678 case Expand: 4679 Tmp1 = Node->getOperand(0); 4680 break; 4681 case Promote: 4682 // The input result is prerounded, so we don't have to do anything 4683 // special. 4684 Tmp1 = PromoteOp(Node->getOperand(0)); 4685 break; 4686 } 4687 // If we're promoting a UINT to a larger size, check to see if the new node 4688 // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since 4689 // we can use that instead. This allows us to generate better code for 4690 // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not 4691 // legal, such as PowerPC. 4692 if (Node->getOpcode() == ISD::FP_TO_UINT && 4693 !TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NVT) && 4694 (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT) || 4695 TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)){ 4696 Result = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Tmp1); 4697 } else { 4698 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4699 } 4700 break; 4701 4702 case ISD::FABS: 4703 case ISD::FNEG: 4704 Tmp1 = PromoteOp(Node->getOperand(0)); 4705 assert(Tmp1.getValueType() == NVT); 4706 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4707 // NOTE: we do not have to do any extra rounding here for 4708 // NoExcessFPPrecision, because we know the input will have the appropriate 4709 // precision, and these operations don't modify precision at all. 4710 break; 4711 4712 case ISD::FLOG: 4713 case ISD::FLOG2: 4714 case ISD::FLOG10: 4715 case ISD::FEXP: 4716 case ISD::FEXP2: 4717 case ISD::FSQRT: 4718 case ISD::FSIN: 4719 case ISD::FCOS: 4720 case ISD::FTRUNC: 4721 case ISD::FFLOOR: 4722 case ISD::FCEIL: 4723 case ISD::FRINT: 4724 case ISD::FNEARBYINT: 4725 Tmp1 = PromoteOp(Node->getOperand(0)); 4726 assert(Tmp1.getValueType() == NVT); 4727 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4728 if (NoExcessFPPrecision) 4729 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4730 DAG.getValueType(VT)); 4731 break; 4732 4733 case ISD::FPOW: 4734 case ISD::FPOWI: { 4735 // Promote f32 pow(i) to f64 pow(i). Note that this could insert a libcall 4736 // directly as well, which may be better. 4737 Tmp1 = PromoteOp(Node->getOperand(0)); 4738 Tmp2 = Node->getOperand(1); 4739 if (Node->getOpcode() == ISD::FPOW) 4740 Tmp2 = PromoteOp(Tmp2); 4741 assert(Tmp1.getValueType() == NVT); 4742 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4743 if (NoExcessFPPrecision) 4744 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4745 DAG.getValueType(VT)); 4746 break; 4747 } 4748 4749 case ISD::ATOMIC_CMP_SWAP: { 4750 AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node); 4751 Tmp2 = PromoteOp(Node->getOperand(2)); 4752 Tmp3 = PromoteOp(Node->getOperand(3)); 4753 Result = DAG.getAtomic(Node->getOpcode(), dl, AtomNode->getMemoryVT(), 4754 AtomNode->getChain(), 4755 AtomNode->getBasePtr(), Tmp2, Tmp3, 4756 AtomNode->getSrcValue(), 4757 AtomNode->getAlignment()); 4758 // Remember that we legalized the chain. 4759 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4760 break; 4761 } 4762 case ISD::ATOMIC_LOAD_ADD: 4763 case ISD::ATOMIC_LOAD_SUB: 4764 case ISD::ATOMIC_LOAD_AND: 4765 case ISD::ATOMIC_LOAD_OR: 4766 case ISD::ATOMIC_LOAD_XOR: 4767 case ISD::ATOMIC_LOAD_NAND: 4768 case ISD::ATOMIC_LOAD_MIN: 4769 case ISD::ATOMIC_LOAD_MAX: 4770 case ISD::ATOMIC_LOAD_UMIN: 4771 case ISD::ATOMIC_LOAD_UMAX: 4772 case ISD::ATOMIC_SWAP: { 4773 AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node); 4774 Tmp2 = PromoteOp(Node->getOperand(2)); 4775 Result = DAG.getAtomic(Node->getOpcode(), dl, AtomNode->getMemoryVT(), 4776 AtomNode->getChain(), 4777 AtomNode->getBasePtr(), Tmp2, 4778 AtomNode->getSrcValue(), 4779 AtomNode->getAlignment()); 4780 // Remember that we legalized the chain. 4781 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4782 break; 4783 } 4784 4785 case ISD::AND: 4786 case ISD::OR: 4787 case ISD::XOR: 4788 case ISD::ADD: 4789 case ISD::SUB: 4790 case ISD::MUL: 4791 // The input may have strange things in the top bits of the registers, but 4792 // these operations don't care. They may have weird bits going out, but 4793 // that too is okay if they are integer operations. 4794 Tmp1 = PromoteOp(Node->getOperand(0)); 4795 Tmp2 = PromoteOp(Node->getOperand(1)); 4796 assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT); 4797 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4798 break; 4799 case ISD::FADD: 4800 case ISD::FSUB: 4801 case ISD::FMUL: 4802 Tmp1 = PromoteOp(Node->getOperand(0)); 4803 Tmp2 = PromoteOp(Node->getOperand(1)); 4804 assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT); 4805 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4806 4807 // Floating point operations will give excess precision that we may not be 4808 // able to tolerate. If we DO allow excess precision, just leave it, 4809 // otherwise excise it. 4810 // FIXME: Why would we need to round FP ops more than integer ones? 4811 // Is Round(Add(Add(A,B),C)) != Round(Add(Round(Add(A,B)), C)) 4812 if (NoExcessFPPrecision) 4813 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4814 DAG.getValueType(VT)); 4815 break; 4816 4817 case ISD::SDIV: 4818 case ISD::SREM: 4819 // These operators require that their input be sign extended. 4820 Tmp1 = PromoteOp(Node->getOperand(0)); 4821 Tmp2 = PromoteOp(Node->getOperand(1)); 4822 if (NVT.isInteger()) { 4823 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, 4824 DAG.getValueType(VT)); 4825 Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp2, 4826 DAG.getValueType(VT)); 4827 } 4828 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4829 4830 // Perform FP_ROUND: this is probably overly pessimistic. 4831 if (NVT.isFloatingPoint() && NoExcessFPPrecision) 4832 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4833 DAG.getValueType(VT)); 4834 break; 4835 case ISD::FDIV: 4836 case ISD::FREM: 4837 case ISD::FCOPYSIGN: 4838 // These operators require that their input be fp extended. 4839 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4840 case Expand: assert(0 && "not implemented"); 4841 case Legal: Tmp1 = LegalizeOp(Node->getOperand(0)); break; 4842 case Promote: Tmp1 = PromoteOp(Node->getOperand(0)); break; 4843 } 4844 switch (getTypeAction(Node->getOperand(1).getValueType())) { 4845 case Expand: assert(0 && "not implemented"); 4846 case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; 4847 case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; 4848 } 4849 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4850 4851 // Perform FP_ROUND: this is probably overly pessimistic. 4852 if (NoExcessFPPrecision && Node->getOpcode() != ISD::FCOPYSIGN) 4853 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4854 DAG.getValueType(VT)); 4855 break; 4856 4857 case ISD::UDIV: 4858 case ISD::UREM: 4859 // These operators require that their input be zero extended. 4860 Tmp1 = PromoteOp(Node->getOperand(0)); 4861 Tmp2 = PromoteOp(Node->getOperand(1)); 4862 assert(NVT.isInteger() && "Operators don't apply to FP!"); 4863 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); 4864 Tmp2 = DAG.getZeroExtendInReg(Tmp2, dl, VT); 4865 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4866 break; 4867 4868 case ISD::SHL: 4869 Tmp1 = PromoteOp(Node->getOperand(0)); 4870 Result = DAG.getNode(ISD::SHL, dl, NVT, Tmp1, Node->getOperand(1)); 4871 break; 4872 case ISD::SRA: 4873 // The input value must be properly sign extended. 4874 Tmp1 = PromoteOp(Node->getOperand(0)); 4875 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, 4876 DAG.getValueType(VT)); 4877 Result = DAG.getNode(ISD::SRA, dl, NVT, Tmp1, Node->getOperand(1)); 4878 break; 4879 case ISD::SRL: 4880 // The input value must be properly zero extended. 4881 Tmp1 = PromoteOp(Node->getOperand(0)); 4882 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); 4883 Result = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, Node->getOperand(1)); 4884 break; 4885 4886 case ISD::VAARG: 4887 Tmp1 = Node->getOperand(0); // Get the chain. 4888 Tmp2 = Node->getOperand(1); // Get the pointer. 4889 if (TLI.getOperationAction(ISD::VAARG, VT) == TargetLowering::Custom) { 4890 Tmp3 = DAG.getVAArg(VT, dl, Tmp1, Tmp2, Node->getOperand(2)); 4891 Result = TLI.LowerOperation(Tmp3, DAG); 4892 } else { 4893 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 4894 SDValue VAList = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, V, 0); 4895 // Increment the pointer, VAList, to the next vaarg 4896 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 4897 DAG.getConstant(VT.getSizeInBits()/8, 4898 TLI.getPointerTy())); 4899 // Store the incremented VAList to the legalized pointer 4900 Tmp3 = DAG.getStore(VAList.getValue(1), dl, Tmp3, Tmp2, V, 0); 4901 // Load the actual argument out of the pointer VAList 4902 Result = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Tmp3, VAList, NULL, 0, VT); 4903 } 4904 // Remember that we legalized the chain. 4905 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4906 break; 4907 4908 case ISD::LOAD: { 4909 LoadSDNode *LD = cast<LoadSDNode>(Node); 4910 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(Node) 4911 ? ISD::EXTLOAD : LD->getExtensionType(); 4912 Result = DAG.getExtLoad(ExtType, dl, NVT, 4913 LD->getChain(), LD->getBasePtr(), 4914 LD->getSrcValue(), LD->getSrcValueOffset(), 4915 LD->getMemoryVT(), 4916 LD->isVolatile(), 4917 LD->getAlignment()); 4918 // Remember that we legalized the chain. 4919 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4920 break; 4921 } 4922 case ISD::SELECT: { 4923 Tmp2 = PromoteOp(Node->getOperand(1)); // Legalize the op0 4924 Tmp3 = PromoteOp(Node->getOperand(2)); // Legalize the op1 4925 4926 MVT VT2 = Tmp2.getValueType(); 4927 assert(VT2 == Tmp3.getValueType() 4928 && "PromoteOp SELECT: Operands 2 and 3 ValueTypes don't match"); 4929 // Ensure that the resulting node is at least the same size as the operands' 4930 // value types, because we cannot assume that TLI.getSetCCValueType() is 4931 // constant. 4932 Result = DAG.getNode(ISD::SELECT, dl, VT2, Node->getOperand(0), Tmp2, Tmp3); 4933 break; 4934 } 4935 case ISD::SELECT_CC: 4936 Tmp2 = PromoteOp(Node->getOperand(2)); // True 4937 Tmp3 = PromoteOp(Node->getOperand(3)); // False 4938 Result = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), 4939 Node->getOperand(1), Tmp2, Tmp3, Node->getOperand(4)); 4940 break; 4941 case ISD::BSWAP: 4942 Tmp1 = Node->getOperand(0); 4943 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1); 4944 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 4945 Result = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 4946 DAG.getConstant(NVT.getSizeInBits() - 4947 VT.getSizeInBits(), 4948 TLI.getShiftAmountTy())); 4949 break; 4950 case ISD::CTPOP: 4951 case ISD::CTTZ: 4952 case ISD::CTLZ: 4953 // Zero extend the argument 4954 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 4955 // Perform the larger operation, then subtract if needed. 4956 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4957 switch(Node->getOpcode()) { 4958 case ISD::CTPOP: 4959 Result = Tmp1; 4960 break; 4961 case ISD::CTTZ: 4962 // if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT) 4963 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), Tmp1, 4964 DAG.getConstant(NVT.getSizeInBits(), NVT), 4965 ISD::SETEQ); 4966 Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, 4967 DAG.getConstant(VT.getSizeInBits(), NVT), Tmp1); 4968 break; 4969 case ISD::CTLZ: 4970 //Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 4971 Result = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 4972 DAG.getConstant(NVT.getSizeInBits() - 4973 VT.getSizeInBits(), NVT)); 4974 break; 4975 } 4976 break; 4977 case ISD::EXTRACT_SUBVECTOR: 4978 Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op)); 4979 break; 4980 case ISD::EXTRACT_VECTOR_ELT: 4981 Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op)); 4982 break; 4983 } 4984 4985 assert(Result.getNode() && "Didn't set a result!"); 4986 4987 // Make sure the result is itself legal. 4988 Result = LegalizeOp(Result); 4989 4990 // Remember that we promoted this! 4991 AddPromotedOperand(Op, Result); 4992 return Result; 4993} 4994 4995/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into 4996/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic, 4997/// based on the vector type. The return type of this matches the element type 4998/// of the vector, which may not be legal for the target. 4999SDValue SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDValue Op) { 5000 // We know that operand #0 is the Vec vector. If the index is a constant 5001 // or if the invec is a supported hardware type, we can use it. Otherwise, 5002 // lower to a store then an indexed load. 5003 SDValue Vec = Op.getOperand(0); 5004 SDValue Idx = Op.getOperand(1); 5005 DebugLoc dl = Op.getNode()->getDebugLoc(); 5006 5007 MVT TVT = Vec.getValueType(); 5008 unsigned NumElems = TVT.getVectorNumElements(); 5009 5010 switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) { 5011 default: assert(0 && "This action is not supported yet!"); 5012 case TargetLowering::Custom: { 5013 Vec = LegalizeOp(Vec); 5014 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 5015 SDValue Tmp3 = TLI.LowerOperation(Op, DAG); 5016 if (Tmp3.getNode()) 5017 return Tmp3; 5018 break; 5019 } 5020 case TargetLowering::Legal: 5021 if (isTypeLegal(TVT)) { 5022 Vec = LegalizeOp(Vec); 5023 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 5024 return Op; 5025 } 5026 break; 5027 case TargetLowering::Promote: 5028 assert(TVT.isVector() && "not vector type"); 5029 // fall thru to expand since vectors are by default are promote 5030 case TargetLowering::Expand: 5031 break; 5032 } 5033 5034 if (NumElems == 1) { 5035 // This must be an access of the only element. Return it. 5036 Op = ScalarizeVectorOp(Vec); 5037 } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) { 5038 unsigned NumLoElts = 1 << Log2_32(NumElems-1); 5039 ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx); 5040 SDValue Lo, Hi; 5041 SplitVectorOp(Vec, Lo, Hi); 5042 if (CIdx->getZExtValue() < NumLoElts) { 5043 Vec = Lo; 5044 } else { 5045 Vec = Hi; 5046 Idx = DAG.getConstant(CIdx->getZExtValue() - NumLoElts, 5047 Idx.getValueType()); 5048 } 5049 5050 // It's now an extract from the appropriate high or low part. Recurse. 5051 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 5052 Op = ExpandEXTRACT_VECTOR_ELT(Op); 5053 } else { 5054 // Store the value to a temporary stack slot, then LOAD the scalar 5055 // element back out. 5056 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 5057 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 5058 5059 // Add the offset to the index. 5060 unsigned EltSize = Op.getValueType().getSizeInBits()/8; 5061 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 5062 DAG.getConstant(EltSize, Idx.getValueType())); 5063 5064 if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 5065 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx); 5066 else 5067 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 5068 5069 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); 5070 5071 Op = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, NULL, 0); 5072 } 5073 return Op; 5074} 5075 5076/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation. For now 5077/// we assume the operation can be split if it is not already legal. 5078SDValue SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDValue Op) { 5079 // We know that operand #0 is the Vec vector. For now we assume the index 5080 // is a constant and that the extracted result is a supported hardware type. 5081 SDValue Vec = Op.getOperand(0); 5082 SDValue Idx = LegalizeOp(Op.getOperand(1)); 5083 5084 unsigned NumElems = Vec.getValueType().getVectorNumElements(); 5085 5086 if (NumElems == Op.getValueType().getVectorNumElements()) { 5087 // This must be an access of the desired vector length. Return it. 5088 return Vec; 5089 } 5090 5091 ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx); 5092 SDValue Lo, Hi; 5093 SplitVectorOp(Vec, Lo, Hi); 5094 if (CIdx->getZExtValue() < NumElems/2) { 5095 Vec = Lo; 5096 } else { 5097 Vec = Hi; 5098 Idx = DAG.getConstant(CIdx->getZExtValue() - NumElems/2, 5099 Idx.getValueType()); 5100 } 5101 5102 // It's now an extract from the appropriate high or low part. Recurse. 5103 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 5104 return ExpandEXTRACT_SUBVECTOR(Op); 5105} 5106 5107/// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC 5108/// with condition CC on the current target. This usually involves legalizing 5109/// or promoting the arguments. In the case where LHS and RHS must be expanded, 5110/// there may be no choice but to create a new SetCC node to represent the 5111/// legalized value of setcc lhs, rhs. In this case, the value is returned in 5112/// LHS, and the SDValue returned in RHS has a nil SDNode value. 5113void SelectionDAGLegalize::LegalizeSetCCOperands(SDValue &LHS, 5114 SDValue &RHS, 5115 SDValue &CC, 5116 DebugLoc dl) { 5117 SDValue Tmp1, Tmp2, Tmp3, Result; 5118 5119 switch (getTypeAction(LHS.getValueType())) { 5120 case Legal: 5121 Tmp1 = LegalizeOp(LHS); // LHS 5122 Tmp2 = LegalizeOp(RHS); // RHS 5123 break; 5124 case Promote: 5125 Tmp1 = PromoteOp(LHS); // LHS 5126 Tmp2 = PromoteOp(RHS); // RHS 5127 5128 // If this is an FP compare, the operands have already been extended. 5129 if (LHS.getValueType().isInteger()) { 5130 MVT VT = LHS.getValueType(); 5131 MVT NVT = TLI.getTypeToTransformTo(VT); 5132 5133 // Otherwise, we have to insert explicit sign or zero extends. Note 5134 // that we could insert sign extends for ALL conditions, but zero extend 5135 // is cheaper on many machines (an AND instead of two shifts), so prefer 5136 // it. 5137 switch (cast<CondCodeSDNode>(CC)->get()) { 5138 default: assert(0 && "Unknown integer comparison!"); 5139 case ISD::SETEQ: 5140 case ISD::SETNE: 5141 case ISD::SETUGE: 5142 case ISD::SETUGT: 5143 case ISD::SETULE: 5144 case ISD::SETULT: 5145 // ALL of these operations will work if we either sign or zero extend 5146 // the operands (including the unsigned comparisons!). Zero extend is 5147 // usually a simpler/cheaper operation, so prefer it. 5148 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); 5149 Tmp2 = DAG.getZeroExtendInReg(Tmp2, dl, VT); 5150 break; 5151 case ISD::SETGE: 5152 case ISD::SETGT: 5153 case ISD::SETLT: 5154 case ISD::SETLE: 5155 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, 5156 DAG.getValueType(VT)); 5157 Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp2, 5158 DAG.getValueType(VT)); 5159 Tmp1 = LegalizeOp(Tmp1); // Relegalize new nodes. 5160 Tmp2 = LegalizeOp(Tmp2); // Relegalize new nodes. 5161 break; 5162 } 5163 } 5164 break; 5165 case Expand: { 5166 MVT VT = LHS.getValueType(); 5167 if (VT == MVT::f32 || VT == MVT::f64) { 5168 // Expand into one or more soft-fp libcall(s). 5169 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL; 5170 switch (cast<CondCodeSDNode>(CC)->get()) { 5171 case ISD::SETEQ: 5172 case ISD::SETOEQ: 5173 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; 5174 break; 5175 case ISD::SETNE: 5176 case ISD::SETUNE: 5177 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : RTLIB::UNE_F64; 5178 break; 5179 case ISD::SETGE: 5180 case ISD::SETOGE: 5181 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; 5182 break; 5183 case ISD::SETLT: 5184 case ISD::SETOLT: 5185 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5186 break; 5187 case ISD::SETLE: 5188 case ISD::SETOLE: 5189 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; 5190 break; 5191 case ISD::SETGT: 5192 case ISD::SETOGT: 5193 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; 5194 break; 5195 case ISD::SETUO: 5196 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; 5197 break; 5198 case ISD::SETO: 5199 LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : RTLIB::O_F64; 5200 break; 5201 default: 5202 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; 5203 switch (cast<CondCodeSDNode>(CC)->get()) { 5204 case ISD::SETONE: 5205 // SETONE = SETOLT | SETOGT 5206 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5207 // Fallthrough 5208 case ISD::SETUGT: 5209 LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; 5210 break; 5211 case ISD::SETUGE: 5212 LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; 5213 break; 5214 case ISD::SETULT: 5215 LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5216 break; 5217 case ISD::SETULE: 5218 LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; 5219 break; 5220 case ISD::SETUEQ: 5221 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; 5222 break; 5223 default: assert(0 && "Unsupported FP setcc!"); 5224 } 5225 } 5226 5227 SDValue Dummy; 5228 SDValue Ops[2] = { LHS, RHS }; 5229 Tmp1 = ExpandLibCall(LC1, DAG.getMergeValues(Ops, 2, dl).getNode(), 5230 false /*sign irrelevant*/, Dummy); 5231 Tmp2 = DAG.getConstant(0, MVT::i32); 5232 CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1)); 5233 if (LC2 != RTLIB::UNKNOWN_LIBCALL) { 5234 Tmp1 = DAG.getNode(ISD::SETCC, dl, 5235 TLI.getSetCCResultType(Tmp1.getValueType()), 5236 Tmp1, Tmp2, CC); 5237 LHS = ExpandLibCall(LC2, DAG.getMergeValues(Ops, 2, dl).getNode(), 5238 false /*sign irrelevant*/, Dummy); 5239 Tmp2 = DAG.getNode(ISD::SETCC, dl, 5240 TLI.getSetCCResultType(LHS.getValueType()), LHS, 5241 Tmp2, DAG.getCondCode(TLI.getCmpLibcallCC(LC2))); 5242 Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp2); 5243 Tmp2 = SDValue(); 5244 } 5245 LHS = LegalizeOp(Tmp1); 5246 RHS = Tmp2; 5247 return; 5248 } 5249 5250 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 5251 ExpandOp(LHS, LHSLo, LHSHi); 5252 ExpandOp(RHS, RHSLo, RHSHi); 5253 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 5254 5255 if (VT==MVT::ppcf128) { 5256 // FIXME: This generated code sucks. We want to generate 5257 // FCMPU crN, hi1, hi2 5258 // BNE crN, L: 5259 // FCMPU crN, lo1, lo2 5260 // The following can be improved, but not that much. 5261 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 5262 LHSHi, RHSHi, ISD::SETOEQ); 5263 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), 5264 LHSLo, RHSLo, CCCode); 5265 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 5266 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 5267 LHSHi, RHSHi, ISD::SETUNE); 5268 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 5269 LHSHi, RHSHi, CCCode); 5270 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 5271 Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 5272 Tmp2 = SDValue(); 5273 break; 5274 } 5275 5276 switch (CCCode) { 5277 case ISD::SETEQ: 5278 case ISD::SETNE: 5279 if (RHSLo == RHSHi) 5280 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) 5281 if (RHSCST->isAllOnesValue()) { 5282 // Comparison to -1. 5283 Tmp1 = DAG.getNode(ISD::AND, dl,LHSLo.getValueType(), LHSLo, LHSHi); 5284 Tmp2 = RHSLo; 5285 break; 5286 } 5287 5288 Tmp1 = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 5289 Tmp2 = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 5290 Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp2); 5291 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 5292 break; 5293 default: 5294 // If this is a comparison of the sign bit, just look at the top part. 5295 // X > -1, x < 0 5296 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(RHS)) 5297 if ((cast<CondCodeSDNode>(CC)->get() == ISD::SETLT && 5298 CST->isNullValue()) || // X < 0 5299 (cast<CondCodeSDNode>(CC)->get() == ISD::SETGT && 5300 CST->isAllOnesValue())) { // X > -1 5301 Tmp1 = LHSHi; 5302 Tmp2 = RHSHi; 5303 break; 5304 } 5305 5306 // FIXME: This generated code sucks. 5307 ISD::CondCode LowCC; 5308 switch (CCCode) { 5309 default: assert(0 && "Unknown integer setcc!"); 5310 case ISD::SETLT: 5311 case ISD::SETULT: LowCC = ISD::SETULT; break; 5312 case ISD::SETGT: 5313 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 5314 case ISD::SETLE: 5315 case ISD::SETULE: LowCC = ISD::SETULE; break; 5316 case ISD::SETGE: 5317 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 5318 } 5319 5320 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 5321 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 5322 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 5323 5324 // NOTE: on targets without efficient SELECT of bools, we can always use 5325 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 5326 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL); 5327 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), 5328 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); 5329 if (!Tmp1.getNode()) 5330 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), 5331 LHSLo, RHSLo, LowCC); 5332 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 5333 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); 5334 if (!Tmp2.getNode()) 5335 Tmp2 = DAG.getNode(ISD::SETCC, dl, 5336 TLI.getSetCCResultType(LHSHi.getValueType()), 5337 LHSHi, RHSHi,CC); 5338 5339 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 5340 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 5341 if ((Tmp1C && Tmp1C->isNullValue()) || 5342 (Tmp2C && Tmp2C->isNullValue() && 5343 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 5344 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 5345 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 5346 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 5347 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 5348 // low part is known false, returns high part. 5349 // For LE / GE, if high part is known false, ignore the low part. 5350 // For LT / GT, if high part is known true, ignore the low part. 5351 Tmp1 = Tmp2; 5352 Tmp2 = SDValue(); 5353 } else { 5354 Result = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 5355 LHSHi, RHSHi, ISD::SETEQ, false, 5356 DagCombineInfo, dl); 5357 if (!Result.getNode()) 5358 Result=DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 5359 LHSHi, RHSHi, ISD::SETEQ); 5360 Result = LegalizeOp(DAG.getNode(ISD::SELECT, dl, Tmp1.getValueType(), 5361 Result, Tmp1, Tmp2)); 5362 Tmp1 = Result; 5363 Tmp2 = SDValue(); 5364 } 5365 } 5366 } 5367 } 5368 LHS = Tmp1; 5369 RHS = Tmp2; 5370} 5371 5372/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and 5373/// condition code CC on the current target. This routine assumes LHS and rHS 5374/// have already been legalized by LegalizeSetCCOperands. It expands SETCC with 5375/// illegal condition code into AND / OR of multiple SETCC values. 5376void SelectionDAGLegalize::LegalizeSetCCCondCode(MVT VT, 5377 SDValue &LHS, SDValue &RHS, 5378 SDValue &CC, 5379 DebugLoc dl) { 5380 MVT OpVT = LHS.getValueType(); 5381 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 5382 switch (TLI.getCondCodeAction(CCCode, OpVT)) { 5383 default: assert(0 && "Unknown condition code action!"); 5384 case TargetLowering::Legal: 5385 // Nothing to do. 5386 break; 5387 case TargetLowering::Expand: { 5388 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 5389 unsigned Opc = 0; 5390 switch (CCCode) { 5391 default: assert(0 && "Don't know how to expand this condition!"); abort(); 5392 case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break; 5393 case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break; 5394 case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5395 case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break; 5396 case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5397 case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5398 case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5399 case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5400 case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5401 case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5402 case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5403 case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5404 // FIXME: Implement more expansions. 5405 } 5406 5407 SDValue SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); 5408 SDValue SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); 5409 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 5410 RHS = SDValue(); 5411 CC = SDValue(); 5412 break; 5413 } 5414 } 5415} 5416 5417/// EmitStackConvert - Emit a store/load combination to the stack. This stores 5418/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 5419/// a load from the stack slot to DestVT, extending it if needed. 5420/// The resultant code need not be legal. 5421SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, 5422 MVT SlotVT, 5423 MVT DestVT, 5424 DebugLoc dl) { 5425 // Create the stack frame object. 5426 unsigned SrcAlign = TLI.getTargetData()->getPrefTypeAlignment( 5427 SrcOp.getValueType().getTypeForMVT()); 5428 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 5429 5430 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 5431 int SPFI = StackPtrFI->getIndex(); 5432 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 5433 5434 unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); 5435 unsigned SlotSize = SlotVT.getSizeInBits(); 5436 unsigned DestSize = DestVT.getSizeInBits(); 5437 unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment( 5438 DestVT.getTypeForMVT()); 5439 5440 // Emit a store to the stack slot. Use a truncstore if the input value is 5441 // later than DestVT. 5442 SDValue Store; 5443 5444 if (SrcSize > SlotSize) 5445 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 5446 SV, 0, SlotVT, false, SrcAlign); 5447 else { 5448 assert(SrcSize == SlotSize && "Invalid store"); 5449 Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 5450 SV, 0, false, SrcAlign); 5451 } 5452 5453 // Result is a load from the stack slot. 5454 if (SlotSize == DestSize) 5455 return DAG.getLoad(DestVT, dl, Store, FIPtr, SV, 0, false, DestAlign); 5456 5457 assert(SlotSize < DestSize && "Unknown extension!"); 5458 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, SV, 0, SlotVT, 5459 false, DestAlign); 5460} 5461 5462SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 5463 DebugLoc dl = Node->getDebugLoc(); 5464 // Create a vector sized/aligned stack slot, store the value to element #0, 5465 // then load the whole vector back out. 5466 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 5467 5468 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 5469 int SPFI = StackPtrFI->getIndex(); 5470 5471 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(0), 5472 StackPtr, 5473 PseudoSourceValue::getFixedStack(SPFI), 0); 5474 return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr, 5475 PseudoSourceValue::getFixedStack(SPFI), 0); 5476} 5477 5478 5479/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't 5480/// support the operation, but do support the resultant vector type. 5481SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 5482 5483 // If the only non-undef value is the low element, turn this into a 5484 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 5485 unsigned NumElems = Node->getNumOperands(); 5486 bool isOnlyLowElement = true; 5487 SDValue SplatValue = Node->getOperand(0); 5488 DebugLoc dl = Node->getDebugLoc(); 5489 5490 // FIXME: it would be far nicer to change this into map<SDValue,uint64_t> 5491 // and use a bitmask instead of a list of elements. 5492 std::map<SDValue, std::vector<unsigned> > Values; 5493 Values[SplatValue].push_back(0); 5494 bool isConstant = true; 5495 if (!isa<ConstantFPSDNode>(SplatValue) && !isa<ConstantSDNode>(SplatValue) && 5496 SplatValue.getOpcode() != ISD::UNDEF) 5497 isConstant = false; 5498 5499 for (unsigned i = 1; i < NumElems; ++i) { 5500 SDValue V = Node->getOperand(i); 5501 Values[V].push_back(i); 5502 if (V.getOpcode() != ISD::UNDEF) 5503 isOnlyLowElement = false; 5504 if (SplatValue != V) 5505 SplatValue = SDValue(0,0); 5506 5507 // If this isn't a constant element or an undef, we can't use a constant 5508 // pool load. 5509 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V) && 5510 V.getOpcode() != ISD::UNDEF) 5511 isConstant = false; 5512 } 5513 5514 if (isOnlyLowElement) { 5515 // If the low element is an undef too, then this whole things is an undef. 5516 if (Node->getOperand(0).getOpcode() == ISD::UNDEF) 5517 return DAG.getNode(ISD::UNDEF, dl, Node->getValueType(0)); 5518 // Otherwise, turn this into a scalar_to_vector node. 5519 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, Node->getValueType(0), 5520 Node->getOperand(0)); 5521 } 5522 5523 // If all elements are constants, create a load from the constant pool. 5524 if (isConstant) { 5525 MVT VT = Node->getValueType(0); 5526 std::vector<Constant*> CV; 5527 for (unsigned i = 0, e = NumElems; i != e; ++i) { 5528 if (ConstantFPSDNode *V = 5529 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 5530 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 5531 } else if (ConstantSDNode *V = 5532 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 5533 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 5534 } else { 5535 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); 5536 const Type *OpNTy = 5537 Node->getOperand(0).getValueType().getTypeForMVT(); 5538 CV.push_back(UndefValue::get(OpNTy)); 5539 } 5540 } 5541 Constant *CP = ConstantVector::get(CV); 5542 SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); 5543 unsigned Alignment = 1 << cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 5544 return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 5545 PseudoSourceValue::getConstantPool(), 0, 5546 false, Alignment); 5547 } 5548 5549 if (SplatValue.getNode()) { // Splat of one value? 5550 // Build the shuffle constant vector: <0, 0, 0, 0> 5551 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 5552 SDValue Zero = DAG.getConstant(0, MaskVT.getVectorElementType()); 5553 std::vector<SDValue> ZeroVec(NumElems, Zero); 5554 SDValue SplatMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT, 5555 &ZeroVec[0], ZeroVec.size()); 5556 5557 // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it. 5558 if (isShuffleLegal(Node->getValueType(0), SplatMask)) { 5559 // Get the splatted value into the low element of a vector register. 5560 SDValue LowValVec = 5561 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 5562 Node->getValueType(0), SplatValue); 5563 5564 // Return shuffle(LowValVec, undef, <0,0,0,0>) 5565 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, 5566 Node->getValueType(0), LowValVec, 5567 DAG.getNode(ISD::UNDEF, Node->getValueType(0)), 5568 SplatMask); 5569 } 5570 } 5571 5572 // If there are only two unique elements, we may be able to turn this into a 5573 // vector shuffle. 5574 if (Values.size() == 2) { 5575 // Get the two values in deterministic order. 5576 SDValue Val1 = Node->getOperand(1); 5577 SDValue Val2; 5578 std::map<SDValue, std::vector<unsigned> >::iterator MI = Values.begin(); 5579 if (MI->first != Val1) 5580 Val2 = MI->first; 5581 else 5582 Val2 = (++MI)->first; 5583 5584 // If Val1 is an undef, make sure end ends up as Val2, to ensure that our 5585 // vector shuffle has the undef vector on the RHS. 5586 if (Val1.getOpcode() == ISD::UNDEF) 5587 std::swap(Val1, Val2); 5588 5589 // Build the shuffle constant vector: e.g. <0, 4, 0, 4> 5590 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); 5591 MVT MaskEltVT = MaskVT.getVectorElementType(); 5592 std::vector<SDValue> MaskVec(NumElems); 5593 5594 // Set elements of the shuffle mask for Val1. 5595 std::vector<unsigned> &Val1Elts = Values[Val1]; 5596 for (unsigned i = 0, e = Val1Elts.size(); i != e; ++i) 5597 MaskVec[Val1Elts[i]] = DAG.getConstant(0, MaskEltVT); 5598 5599 // Set elements of the shuffle mask for Val2. 5600 std::vector<unsigned> &Val2Elts = Values[Val2]; 5601 for (unsigned i = 0, e = Val2Elts.size(); i != e; ++i) 5602 if (Val2.getOpcode() != ISD::UNDEF) 5603 MaskVec[Val2Elts[i]] = DAG.getConstant(NumElems, MaskEltVT); 5604 else 5605 MaskVec[Val2Elts[i]] = DAG.getNode(ISD::UNDEF, dl, MaskEltVT); 5606 5607 SDValue ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT, 5608 &MaskVec[0], MaskVec.size()); 5609 5610 // If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it. 5611 if (TLI.isOperationLegalOrCustom(ISD::SCALAR_TO_VECTOR, 5612 Node->getValueType(0)) && 5613 isShuffleLegal(Node->getValueType(0), ShuffleMask)) { 5614 Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,Node->getValueType(0), Val1); 5615 Val2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,Node->getValueType(0), Val2); 5616 SDValue Ops[] = { Val1, Val2, ShuffleMask }; 5617 5618 // Return shuffle(LoValVec, HiValVec, <0,1,0,1>) 5619 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl,Node->getValueType(0), Ops, 3); 5620 } 5621 } 5622 5623 // Otherwise, we can't handle this case efficiently. Allocate a sufficiently 5624 // aligned object on the stack, store each element into it, then load 5625 // the result as a vector. 5626 MVT VT = Node->getValueType(0); 5627 // Create the stack frame object. 5628 SDValue FIPtr = DAG.CreateStackTemporary(VT); 5629 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 5630 const Value *SV = PseudoSourceValue::getFixedStack(FI); 5631 5632 // Emit a store of each element to the stack slot. 5633 SmallVector<SDValue, 8> Stores; 5634 unsigned TypeByteSize = Node->getOperand(0).getValueType().getSizeInBits()/8; 5635 // Store (in the right endianness) the elements to memory. 5636 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 5637 // Ignore undef elements. 5638 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; 5639 5640 unsigned Offset = TypeByteSize*i; 5641 5642 SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); 5643 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); 5644 5645 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), 5646 Idx, SV, Offset)); 5647 } 5648 5649 SDValue StoreChain; 5650 if (!Stores.empty()) // Not all undef elements? 5651 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 5652 &Stores[0], Stores.size()); 5653 else 5654 StoreChain = DAG.getEntryNode(); 5655 5656 // Result is a load from the stack slot. 5657 return DAG.getLoad(VT, dl, StoreChain, FIPtr, SV, 0); 5658} 5659 5660void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp, 5661 SDValue Op, SDValue Amt, 5662 SDValue &Lo, SDValue &Hi, 5663 DebugLoc dl) { 5664 // Expand the subcomponents. 5665 SDValue LHSL, LHSH; 5666 ExpandOp(Op, LHSL, LHSH); 5667 5668 SDValue Ops[] = { LHSL, LHSH, Amt }; 5669 MVT VT = LHSL.getValueType(); 5670 Lo = DAG.getNode(NodeOp, dl, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3); 5671 Hi = Lo.getValue(1); 5672} 5673 5674 5675/// ExpandShift - Try to find a clever way to expand this shift operation out to 5676/// smaller elements. If we can't find a way that is more efficient than a 5677/// libcall on this target, return false. Otherwise, return true with the 5678/// low-parts expanded into Lo and Hi. 5679bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDValue Op,SDValue Amt, 5680 SDValue &Lo, SDValue &Hi, 5681 DebugLoc dl) { 5682 assert((Opc == ISD::SHL || Opc == ISD::SRA || Opc == ISD::SRL) && 5683 "This is not a shift!"); 5684 5685 MVT NVT = TLI.getTypeToTransformTo(Op.getValueType()); 5686 SDValue ShAmt = LegalizeOp(Amt); 5687 MVT ShTy = ShAmt.getValueType(); 5688 unsigned ShBits = ShTy.getSizeInBits(); 5689 unsigned VTBits = Op.getValueType().getSizeInBits(); 5690 unsigned NVTBits = NVT.getSizeInBits(); 5691 5692 // Handle the case when Amt is an immediate. 5693 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Amt.getNode())) { 5694 unsigned Cst = CN->getZExtValue(); 5695 // Expand the incoming operand to be shifted, so that we have its parts 5696 SDValue InL, InH; 5697 ExpandOp(Op, InL, InH); 5698 switch(Opc) { 5699 case ISD::SHL: 5700 if (Cst > VTBits) { 5701 Lo = DAG.getConstant(0, NVT); 5702 Hi = DAG.getConstant(0, NVT); 5703 } else if (Cst > NVTBits) { 5704 Lo = DAG.getConstant(0, NVT); 5705 Hi = DAG.getNode(ISD::SHL, dl, 5706 NVT, InL, DAG.getConstant(Cst-NVTBits,ShTy)); 5707 } else if (Cst == NVTBits) { 5708 Lo = DAG.getConstant(0, NVT); 5709 Hi = InL; 5710 } else { 5711 Lo = DAG.getNode(ISD::SHL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)); 5712 Hi = DAG.getNode(ISD::OR, dl, NVT, 5713 DAG.getNode(ISD::SHL, dl, NVT, InH, DAG.getConstant(Cst, ShTy)), 5714 DAG.getNode(ISD::SRL, dl, NVT, InL, 5715 DAG.getConstant(NVTBits-Cst, ShTy))); 5716 } 5717 return true; 5718 case ISD::SRL: 5719 if (Cst > VTBits) { 5720 Lo = DAG.getConstant(0, NVT); 5721 Hi = DAG.getConstant(0, NVT); 5722 } else if (Cst > NVTBits) { 5723 Lo = DAG.getNode(ISD::SRL, dl, NVT, 5724 InH, DAG.getConstant(Cst-NVTBits,ShTy)); 5725 Hi = DAG.getConstant(0, NVT); 5726 } else if (Cst == NVTBits) { 5727 Lo = InH; 5728 Hi = DAG.getConstant(0, NVT); 5729 } else { 5730 Lo = DAG.getNode(ISD::OR, dl, NVT, 5731 DAG.getNode(ISD::SRL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)), 5732 DAG.getNode(ISD::SHL, dl, NVT, InH, 5733 DAG.getConstant(NVTBits-Cst, ShTy))); 5734 Hi = DAG.getNode(ISD::SRL, dl, NVT, InH, DAG.getConstant(Cst, ShTy)); 5735 } 5736 return true; 5737 case ISD::SRA: 5738 if (Cst > VTBits) { 5739 Hi = Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, 5740 DAG.getConstant(NVTBits-1, ShTy)); 5741 } else if (Cst > NVTBits) { 5742 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, 5743 DAG.getConstant(Cst-NVTBits, ShTy)); 5744 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, 5745 DAG.getConstant(NVTBits-1, ShTy)); 5746 } else if (Cst == NVTBits) { 5747 Lo = InH; 5748 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, 5749 DAG.getConstant(NVTBits-1, ShTy)); 5750 } else { 5751 Lo = DAG.getNode(ISD::OR, dl, NVT, 5752 DAG.getNode(ISD::SRL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)), 5753 DAG.getNode(ISD::SHL, dl, 5754 NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy))); 5755 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, DAG.getConstant(Cst, ShTy)); 5756 } 5757 return true; 5758 } 5759 } 5760 5761 // Okay, the shift amount isn't constant. However, if we can tell that it is 5762 // >= 32 or < 32, we can still simplify it, without knowing the actual value. 5763 APInt Mask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 5764 APInt KnownZero, KnownOne; 5765 DAG.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne); 5766 5767 // If we know that if any of the high bits of the shift amount are one, then 5768 // we can do this as a couple of simple shifts. 5769 if (KnownOne.intersects(Mask)) { 5770 // Mask out the high bit, which we know is set. 5771 Amt = DAG.getNode(ISD::AND, dl, Amt.getValueType(), Amt, 5772 DAG.getConstant(~Mask, Amt.getValueType())); 5773 5774 // Expand the incoming operand to be shifted, so that we have its parts 5775 SDValue InL, InH; 5776 ExpandOp(Op, InL, InH); 5777 switch(Opc) { 5778 case ISD::SHL: 5779 Lo = DAG.getConstant(0, NVT); // Low part is zero. 5780 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 5781 return true; 5782 case ISD::SRL: 5783 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 5784 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 5785 return true; 5786 case ISD::SRA: 5787 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 5788 DAG.getConstant(NVTBits-1, Amt.getValueType())); 5789 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 5790 return true; 5791 } 5792 } 5793 5794 // If we know that the high bits of the shift amount are all zero, then we can 5795 // do this as a couple of simple shifts. 5796 if ((KnownZero & Mask) == Mask) { 5797 // Compute 32-amt. 5798 SDValue Amt2 = DAG.getNode(ISD::SUB, dl, Amt.getValueType(), 5799 DAG.getConstant(NVTBits, Amt.getValueType()), 5800 Amt); 5801 5802 // Expand the incoming operand to be shifted, so that we have its parts 5803 SDValue InL, InH; 5804 ExpandOp(Op, InL, InH); 5805 switch(Opc) { 5806 case ISD::SHL: 5807 Lo = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 5808 Hi = DAG.getNode(ISD::OR, dl, NVT, 5809 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 5810 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt2)); 5811 return true; 5812 case ISD::SRL: 5813 Hi = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 5814 Lo = DAG.getNode(ISD::OR, dl, NVT, 5815 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 5816 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt2)); 5817 return true; 5818 case ISD::SRA: 5819 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 5820 Lo = DAG.getNode(ISD::OR, dl, NVT, 5821 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 5822 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt2)); 5823 return true; 5824 } 5825 } 5826 5827 return false; 5828} 5829 5830 5831// ExpandLibCall - Expand a node into a call to a libcall. If the result value 5832// does not fit into a register, return the lo part and set the hi part to the 5833// by-reg argument. If it does fit into a single register, return the result 5834// and leave the Hi part unset. 5835SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 5836 bool isSigned, SDValue &Hi) { 5837 assert(!IsLegalizingCall && "Cannot overlap legalization of calls!"); 5838 // The input chain to this libcall is the entry node of the function. 5839 // Legalizing the call will automatically add the previous call to the 5840 // dependence. 5841 SDValue InChain = DAG.getEntryNode(); 5842 5843 TargetLowering::ArgListTy Args; 5844 TargetLowering::ArgListEntry Entry; 5845 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 5846 MVT ArgVT = Node->getOperand(i).getValueType(); 5847 const Type *ArgTy = ArgVT.getTypeForMVT(); 5848 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 5849 Entry.isSExt = isSigned; 5850 Entry.isZExt = !isSigned; 5851 Args.push_back(Entry); 5852 } 5853 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 5854 TLI.getPointerTy()); 5855 5856 // Splice the libcall in wherever FindInputOutputChains tells us to. 5857 const Type *RetTy = Node->getValueType(0).getTypeForMVT(); 5858 std::pair<SDValue,SDValue> CallInfo = 5859 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, 5860 CallingConv::C, false, Callee, Args, DAG, 5861 Node->getDebugLoc()); 5862 5863 // Legalize the call sequence, starting with the chain. This will advance 5864 // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that 5865 // was added by LowerCallTo (guaranteeing proper serialization of calls). 5866 LegalizeOp(CallInfo.second); 5867 SDValue Result; 5868 switch (getTypeAction(CallInfo.first.getValueType())) { 5869 default: assert(0 && "Unknown thing"); 5870 case Legal: 5871 Result = CallInfo.first; 5872 break; 5873 case Expand: 5874 ExpandOp(CallInfo.first, Result, Hi); 5875 break; 5876 } 5877 return Result; 5878} 5879 5880/// LegalizeINT_TO_FP - Legalize a [US]INT_TO_FP operation. 5881/// 5882SDValue SelectionDAGLegalize:: 5883LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op, 5884 DebugLoc dl) { 5885 bool isCustom = false; 5886 SDValue Tmp1; 5887 switch (getTypeAction(Op.getValueType())) { 5888 case Legal: 5889 switch (TLI.getOperationAction(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, 5890 Op.getValueType())) { 5891 default: assert(0 && "Unknown operation action!"); 5892 case TargetLowering::Custom: 5893 isCustom = true; 5894 // FALLTHROUGH 5895 case TargetLowering::Legal: 5896 Tmp1 = LegalizeOp(Op); 5897 if (Result.getNode()) 5898 Result = DAG.UpdateNodeOperands(Result, Tmp1); 5899 else 5900 Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, dl, 5901 DestTy, Tmp1); 5902 if (isCustom) { 5903 Tmp1 = TLI.LowerOperation(Result, DAG); 5904 if (Tmp1.getNode()) Result = Tmp1; 5905 } 5906 break; 5907 case TargetLowering::Expand: 5908 Result = ExpandLegalINT_TO_FP(isSigned, LegalizeOp(Op), DestTy, dl); 5909 break; 5910 case TargetLowering::Promote: 5911 Result = PromoteLegalINT_TO_FP(LegalizeOp(Op), DestTy, isSigned, dl); 5912 break; 5913 } 5914 break; 5915 case Expand: 5916 Result = ExpandIntToFP(isSigned, DestTy, Op, dl) ; 5917 break; 5918 case Promote: 5919 Tmp1 = PromoteOp(Op); 5920 if (isSigned) { 5921 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Tmp1.getValueType(), 5922 Tmp1, DAG.getValueType(Op.getValueType())); 5923 } else { 5924 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, 5925 Op.getValueType()); 5926 } 5927 if (Result.getNode()) 5928 Result = DAG.UpdateNodeOperands(Result, Tmp1); 5929 else 5930 Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, dl, 5931 DestTy, Tmp1); 5932 Result = LegalizeOp(Result); // The 'op' is not necessarily legal! 5933 break; 5934 } 5935 return Result; 5936} 5937 5938/// ExpandIntToFP - Expand a [US]INT_TO_FP operation. 5939/// 5940SDValue SelectionDAGLegalize:: 5941ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source, DebugLoc dl) { 5942 MVT SourceVT = Source.getValueType(); 5943 bool ExpandSource = getTypeAction(SourceVT) == Expand; 5944 5945 // Expand unsupported int-to-fp vector casts by unrolling them. 5946 if (DestTy.isVector()) { 5947 if (!ExpandSource) 5948 return LegalizeOp(UnrollVectorOp(Source)); 5949 MVT DestEltTy = DestTy.getVectorElementType(); 5950 if (DestTy.getVectorNumElements() == 1) { 5951 SDValue Scalar = ScalarizeVectorOp(Source); 5952 SDValue Result = LegalizeINT_TO_FP(SDValue(), isSigned, 5953 DestEltTy, Scalar, dl); 5954 return DAG.getNode(ISD::BUILD_VECTOR, dl, DestTy, Result); 5955 } 5956 SDValue Lo, Hi; 5957 SplitVectorOp(Source, Lo, Hi); 5958 MVT SplitDestTy = MVT::getVectorVT(DestEltTy, 5959 DestTy.getVectorNumElements() / 2); 5960 SDValue LoResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, 5961 Lo, dl); 5962 SDValue HiResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, 5963 Hi, dl); 5964 return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, DestTy, LoResult, 5965 HiResult)); 5966 } 5967 5968 // Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc. 5969 if (!isSigned && SourceVT != MVT::i32) { 5970 // The integer value loaded will be incorrectly if the 'sign bit' of the 5971 // incoming integer is set. To handle this, we dynamically test to see if 5972 // it is set, and, if so, add a fudge factor. 5973 SDValue Hi; 5974 if (ExpandSource) { 5975 SDValue Lo; 5976 ExpandOp(Source, Lo, Hi); 5977 Source = DAG.getNode(ISD::BUILD_PAIR, dl, SourceVT, Lo, Hi); 5978 } else { 5979 // The comparison for the sign bit will use the entire operand. 5980 Hi = Source; 5981 } 5982 5983 // Check to see if the target has a custom way to lower this. If so, use 5984 // it. (Note we've already expanded the operand in this case.) 5985 switch (TLI.getOperationAction(ISD::UINT_TO_FP, SourceVT)) { 5986 default: assert(0 && "This action not implemented for this operation!"); 5987 case TargetLowering::Legal: 5988 case TargetLowering::Expand: 5989 break; // This case is handled below. 5990 case TargetLowering::Custom: { 5991 SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::UINT_TO_FP, DestTy, 5992 Source), DAG); 5993 if (NV.getNode()) 5994 return LegalizeOp(NV); 5995 break; // The target decided this was legal after all 5996 } 5997 } 5998 5999 // If this is unsigned, and not supported, first perform the conversion to 6000 // signed, then adjust the result if the sign bit is set. 6001 SDValue SignedConv = ExpandIntToFP(true, DestTy, Source, dl); 6002 6003 SDValue SignSet = DAG.getSetCC(dl, 6004 TLI.getSetCCResultType(Hi.getValueType()), 6005 Hi, DAG.getConstant(0, Hi.getValueType()), 6006 ISD::SETLT); 6007 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 6008 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), 6009 SignSet, Four, Zero); 6010 uint64_t FF = 0x5f800000ULL; 6011 if (TLI.isLittleEndian()) FF <<= 32; 6012 static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); 6013 6014 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 6015 unsigned Alignment = 1 << cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 6016 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset); 6017 Alignment = std::min(Alignment, 4u); 6018 SDValue FudgeInReg; 6019 if (DestTy == MVT::f32) 6020 FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 6021 PseudoSourceValue::getConstantPool(), 0, 6022 false, Alignment); 6023 else if (DestTy.bitsGT(MVT::f32)) 6024 // FIXME: Avoid the extend by construction the right constantpool? 6025 FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, dl, DestTy, DAG.getEntryNode(), 6026 CPIdx, 6027 PseudoSourceValue::getConstantPool(), 0, 6028 MVT::f32, false, Alignment); 6029 else 6030 assert(0 && "Unexpected conversion"); 6031 6032 MVT SCVT = SignedConv.getValueType(); 6033 if (SCVT != DestTy) { 6034 // Destination type needs to be expanded as well. The FADD now we are 6035 // constructing will be expanded into a libcall. 6036 if (SCVT.getSizeInBits() != DestTy.getSizeInBits()) { 6037 assert(SCVT.getSizeInBits() * 2 == DestTy.getSizeInBits()); 6038 SignedConv = DAG.getNode(ISD::BUILD_PAIR, dl, DestTy, 6039 SignedConv, SignedConv.getValue(1)); 6040 } 6041 SignedConv = DAG.getNode(ISD::BIT_CONVERT, dl, DestTy, SignedConv); 6042 } 6043 return DAG.getNode(ISD::FADD, dl, DestTy, SignedConv, FudgeInReg); 6044 } 6045 6046 // Check to see if the target has a custom way to lower this. If so, use it. 6047 switch (TLI.getOperationAction(ISD::SINT_TO_FP, SourceVT)) { 6048 default: assert(0 && "This action not implemented for this operation!"); 6049 case TargetLowering::Legal: 6050 case TargetLowering::Expand: 6051 break; // This case is handled below. 6052 case TargetLowering::Custom: { 6053 SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, dl, DestTy, 6054 Source), DAG); 6055 if (NV.getNode()) 6056 return LegalizeOp(NV); 6057 break; // The target decided this was legal after all 6058 } 6059 } 6060 6061 // Expand the source, then glue it back together for the call. We must expand 6062 // the source in case it is shared (this pass of legalize must traverse it). 6063 if (ExpandSource) { 6064 SDValue SrcLo, SrcHi; 6065 ExpandOp(Source, SrcLo, SrcHi); 6066 Source = DAG.getNode(ISD::BUILD_PAIR, dl, SourceVT, SrcLo, SrcHi); 6067 } 6068 6069 RTLIB::Libcall LC = isSigned ? 6070 RTLIB::getSINTTOFP(SourceVT, DestTy) : 6071 RTLIB::getUINTTOFP(SourceVT, DestTy); 6072 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unknown int value type"); 6073 6074 Source = DAG.getNode(ISD::SINT_TO_FP, dl, DestTy, Source); 6075 SDValue HiPart; 6076 SDValue Result = ExpandLibCall(LC, Source.getNode(), isSigned, HiPart); 6077 if (Result.getValueType() != DestTy && HiPart.getNode()) 6078 Result = DAG.getNode(ISD::BUILD_PAIR, dl, DestTy, Result, HiPart); 6079 return Result; 6080} 6081 6082/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a 6083/// INT_TO_FP operation of the specified operand when the target requests that 6084/// we expand it. At this point, we know that the result and operand types are 6085/// legal for the target. 6086SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, 6087 SDValue Op0, 6088 MVT DestVT, 6089 DebugLoc dl) { 6090 if (Op0.getValueType() == MVT::i32) { 6091 // simple 32-bit [signed|unsigned] integer to float/double expansion 6092 6093 // Get the stack frame index of a 8 byte buffer. 6094 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 6095 6096 // word offset constant for Hi/Lo address computation 6097 SDValue WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy()); 6098 // set up Hi and Lo (into buffer) address based on endian 6099 SDValue Hi = StackSlot; 6100 SDValue Lo = DAG.getNode(ISD::ADD, dl, 6101 TLI.getPointerTy(), StackSlot,WordOff); 6102 if (TLI.isLittleEndian()) 6103 std::swap(Hi, Lo); 6104 6105 // if signed map to unsigned space 6106 SDValue Op0Mapped; 6107 if (isSigned) { 6108 // constant used to invert sign bit (signed to unsigned mapping) 6109 SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); 6110 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); 6111 } else { 6112 Op0Mapped = Op0; 6113 } 6114 // store the lo of the constructed double - based on integer input 6115 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, 6116 Op0Mapped, Lo, NULL, 0); 6117 // initial hi portion of constructed double 6118 SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); 6119 // store the hi of the constructed double - biased exponent 6120 SDValue Store2=DAG.getStore(Store1, dl, InitialHi, Hi, NULL, 0); 6121 // load the constructed double 6122 SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot, NULL, 0); 6123 // FP constant to bias correct the final result 6124 SDValue Bias = DAG.getConstantFP(isSigned ? 6125 BitsToDouble(0x4330000080000000ULL) 6126 : BitsToDouble(0x4330000000000000ULL), 6127 MVT::f64); 6128 // subtract the bias 6129 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 6130 // final result 6131 SDValue Result; 6132 // handle final rounding 6133 if (DestVT == MVT::f64) { 6134 // do nothing 6135 Result = Sub; 6136 } else if (DestVT.bitsLT(MVT::f64)) { 6137 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 6138 DAG.getIntPtrConstant(0)); 6139 } else if (DestVT.bitsGT(MVT::f64)) { 6140 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 6141 } 6142 return Result; 6143 } 6144 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 6145 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 6146 6147 SDValue SignSet = DAG.getSetCC(dl, TLI.getSetCCResultType(Op0.getValueType()), 6148 Op0, DAG.getConstant(0, Op0.getValueType()), 6149 ISD::SETLT); 6150 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 6151 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), 6152 SignSet, Four, Zero); 6153 6154 // If the sign bit of the integer is set, the large number will be treated 6155 // as a negative number. To counteract this, the dynamic code adds an 6156 // offset depending on the data type. 6157 uint64_t FF; 6158 switch (Op0.getValueType().getSimpleVT()) { 6159 default: assert(0 && "Unsupported integer type!"); 6160 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 6161 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 6162 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 6163 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 6164 } 6165 if (TLI.isLittleEndian()) FF <<= 32; 6166 static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); 6167 6168 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 6169 unsigned Alignment = 1 << cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 6170 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset); 6171 Alignment = std::min(Alignment, 4u); 6172 SDValue FudgeInReg; 6173 if (DestVT == MVT::f32) 6174 FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 6175 PseudoSourceValue::getConstantPool(), 0, 6176 false, Alignment); 6177 else { 6178 FudgeInReg = 6179 LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, 6180 DAG.getEntryNode(), CPIdx, 6181 PseudoSourceValue::getConstantPool(), 0, 6182 MVT::f32, false, Alignment)); 6183 } 6184 6185 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 6186} 6187 6188/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a 6189/// *INT_TO_FP operation of the specified operand when the target requests that 6190/// we promote it. At this point, we know that the result and operand types are 6191/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 6192/// operation that takes a larger input. 6193SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, 6194 MVT DestVT, 6195 bool isSigned, 6196 DebugLoc dl) { 6197 // First step, figure out the appropriate *INT_TO_FP operation to use. 6198 MVT NewInTy = LegalOp.getValueType(); 6199 6200 unsigned OpToUse = 0; 6201 6202 // Scan for the appropriate larger type to use. 6203 while (1) { 6204 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT()+1); 6205 assert(NewInTy.isInteger() && "Ran out of possibilities!"); 6206 6207 // If the target supports SINT_TO_FP of this type, use it. 6208 switch (TLI.getOperationAction(ISD::SINT_TO_FP, NewInTy)) { 6209 default: break; 6210 case TargetLowering::Legal: 6211 if (!TLI.isTypeLegal(NewInTy)) 6212 break; // Can't use this datatype. 6213 // FALL THROUGH. 6214 case TargetLowering::Custom: 6215 OpToUse = ISD::SINT_TO_FP; 6216 break; 6217 } 6218 if (OpToUse) break; 6219 if (isSigned) continue; 6220 6221 // If the target supports UINT_TO_FP of this type, use it. 6222 switch (TLI.getOperationAction(ISD::UINT_TO_FP, NewInTy)) { 6223 default: break; 6224 case TargetLowering::Legal: 6225 if (!TLI.isTypeLegal(NewInTy)) 6226 break; // Can't use this datatype. 6227 // FALL THROUGH. 6228 case TargetLowering::Custom: 6229 OpToUse = ISD::UINT_TO_FP; 6230 break; 6231 } 6232 if (OpToUse) break; 6233 6234 // Otherwise, try a larger type. 6235 } 6236 6237 // Okay, we found the operation and type to use. Zero extend our input to the 6238 // desired type then run the operation on it. 6239 return DAG.getNode(OpToUse, dl, DestVT, 6240 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 6241 dl, NewInTy, LegalOp)); 6242} 6243 6244/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a 6245/// FP_TO_*INT operation of the specified operand when the target requests that 6246/// we promote it. At this point, we know that the result and operand types are 6247/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 6248/// operation that returns a larger result. 6249SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, 6250 MVT DestVT, 6251 bool isSigned, 6252 DebugLoc dl) { 6253 // First step, figure out the appropriate FP_TO*INT operation to use. 6254 MVT NewOutTy = DestVT; 6255 6256 unsigned OpToUse = 0; 6257 6258 // Scan for the appropriate larger type to use. 6259 while (1) { 6260 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT()+1); 6261 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 6262 6263 // If the target supports FP_TO_SINT returning this type, use it. 6264 switch (TLI.getOperationAction(ISD::FP_TO_SINT, NewOutTy)) { 6265 default: break; 6266 case TargetLowering::Legal: 6267 if (!TLI.isTypeLegal(NewOutTy)) 6268 break; // Can't use this datatype. 6269 // FALL THROUGH. 6270 case TargetLowering::Custom: 6271 OpToUse = ISD::FP_TO_SINT; 6272 break; 6273 } 6274 if (OpToUse) break; 6275 6276 // If the target supports FP_TO_UINT of this type, use it. 6277 switch (TLI.getOperationAction(ISD::FP_TO_UINT, NewOutTy)) { 6278 default: break; 6279 case TargetLowering::Legal: 6280 if (!TLI.isTypeLegal(NewOutTy)) 6281 break; // Can't use this datatype. 6282 // FALL THROUGH. 6283 case TargetLowering::Custom: 6284 OpToUse = ISD::FP_TO_UINT; 6285 break; 6286 } 6287 if (OpToUse) break; 6288 6289 // Otherwise, try a larger type. 6290 } 6291 6292 6293 // Okay, we found the operation and type to use. 6294 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 6295 6296 // If the operation produces an invalid type, it must be custom lowered. Use 6297 // the target lowering hooks to expand it. Just keep the low part of the 6298 // expanded operation, we know that we're truncating anyway. 6299 if (getTypeAction(NewOutTy) == Expand) { 6300 SmallVector<SDValue, 2> Results; 6301 TLI.ReplaceNodeResults(Operation.getNode(), Results, DAG); 6302 assert(Results.size() == 1 && "Incorrect FP_TO_XINT lowering!"); 6303 Operation = Results[0]; 6304 } 6305 6306 // Truncate the result of the extended FP_TO_*INT operation to the desired 6307 // size. 6308 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 6309} 6310 6311/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. 6312/// 6313SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) { 6314 MVT VT = Op.getValueType(); 6315 MVT SHVT = TLI.getShiftAmountTy(); 6316 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 6317 switch (VT.getSimpleVT()) { 6318 default: assert(0 && "Unhandled Expand type in BSWAP!"); abort(); 6319 case MVT::i16: 6320 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6321 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6322 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 6323 case MVT::i32: 6324 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 6325 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6326 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6327 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 6328 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); 6329 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); 6330 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 6331 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 6332 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 6333 case MVT::i64: 6334 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); 6335 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); 6336 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 6337 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6338 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6339 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 6340 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); 6341 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); 6342 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); 6343 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); 6344 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); 6345 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); 6346 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); 6347 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); 6348 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 6349 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 6350 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 6351 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 6352 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 6353 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 6354 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 6355 } 6356} 6357 6358/// ExpandBitCount - Expand the specified bitcount instruction into operations. 6359/// 6360SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, 6361 DebugLoc dl) { 6362 switch (Opc) { 6363 default: assert(0 && "Cannot expand this yet!"); 6364 case ISD::CTPOP: { 6365 static const uint64_t mask[6] = { 6366 0x5555555555555555ULL, 0x3333333333333333ULL, 6367 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL, 6368 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL 6369 }; 6370 MVT VT = Op.getValueType(); 6371 MVT ShVT = TLI.getShiftAmountTy(); 6372 unsigned len = VT.getSizeInBits(); 6373 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 6374 //x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8]) 6375 unsigned EltSize = VT.isVector() ? 6376 VT.getVectorElementType().getSizeInBits() : len; 6377 SDValue Tmp2 = DAG.getConstant(APInt(EltSize, mask[i]), VT); 6378 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 6379 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, VT, Op, Tmp2), 6380 DAG.getNode(ISD::AND, dl, VT, 6381 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3), 6382 Tmp2)); 6383 } 6384 return Op; 6385 } 6386 case ISD::CTLZ: { 6387 // for now, we do this: 6388 // x = x | (x >> 1); 6389 // x = x | (x >> 2); 6390 // ... 6391 // x = x | (x >>16); 6392 // x = x | (x >>32); // for 64-bit input 6393 // return popcount(~x); 6394 // 6395 // but see also: http://www.hackersdelight.org/HDcode/nlz.cc 6396 MVT VT = Op.getValueType(); 6397 MVT ShVT = TLI.getShiftAmountTy(); 6398 unsigned len = VT.getSizeInBits(); 6399 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 6400 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 6401 Op = DAG.getNode(ISD::OR, dl, VT, Op, 6402 DAG.getNode(ISD::SRL, VT, Op, Tmp3)); 6403 } 6404 Op = DAG.getNOT(dl, Op, VT); 6405 return DAG.getNode(ISD::CTPOP, dl, VT, Op); 6406 } 6407 case ISD::CTTZ: { 6408 // for now, we use: { return popcount(~x & (x - 1)); } 6409 // unless the target has ctlz but not ctpop, in which case we use: 6410 // { return 32 - nlz(~x & (x-1)); } 6411 // see also http://www.hackersdelight.org/HDcode/ntz.cc 6412 MVT VT = Op.getValueType(); 6413 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, 6414 DAG.getNOT(dl, Op, VT), 6415 DAG.getNode(ISD::SUB, dl, VT, Op, 6416 DAG.getConstant(1, VT))); 6417 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. 6418 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && 6419 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) 6420 return DAG.getNode(ISD::SUB, dl, VT, 6421 DAG.getConstant(VT.getSizeInBits(), VT), 6422 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); 6423 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); 6424 } 6425 } 6426} 6427 6428/// ExpandOp - Expand the specified SDValue into its two component pieces 6429/// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, the 6430/// LegalizedNodes map is filled in for any results that are not expanded, the 6431/// ExpandedNodes map is filled in for any results that are expanded, and the 6432/// Lo/Hi values are returned. 6433void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ 6434 MVT VT = Op.getValueType(); 6435 MVT NVT = TLI.getTypeToTransformTo(VT); 6436 SDNode *Node = Op.getNode(); 6437 DebugLoc dl = Node->getDebugLoc(); 6438 assert(getTypeAction(VT) == Expand && "Not an expanded type!"); 6439 assert(((NVT.isInteger() && NVT.bitsLT(VT)) || VT.isFloatingPoint() || 6440 VT.isVector()) && "Cannot expand to FP value or to larger int value!"); 6441 6442 // See if we already expanded it. 6443 DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator I 6444 = ExpandedNodes.find(Op); 6445 if (I != ExpandedNodes.end()) { 6446 Lo = I->second.first; 6447 Hi = I->second.second; 6448 return; 6449 } 6450 6451 switch (Node->getOpcode()) { 6452 case ISD::CopyFromReg: 6453 assert(0 && "CopyFromReg must be legal!"); 6454 case ISD::FP_ROUND_INREG: 6455 if (VT == MVT::ppcf128 && 6456 TLI.getOperationAction(ISD::FP_ROUND_INREG, VT) == 6457 TargetLowering::Custom) { 6458 SDValue SrcLo, SrcHi, Src; 6459 ExpandOp(Op.getOperand(0), SrcLo, SrcHi); 6460 Src = DAG.getNode(ISD::BUILD_PAIR, dl, VT, SrcLo, SrcHi); 6461 SDValue Result = TLI.LowerOperation( 6462 DAG.getNode(ISD::FP_ROUND_INREG, dl, VT, Src, Op.getOperand(1)), DAG); 6463 assert(Result.getNode()->getOpcode() == ISD::BUILD_PAIR); 6464 Lo = Result.getNode()->getOperand(0); 6465 Hi = Result.getNode()->getOperand(1); 6466 break; 6467 } 6468 // fall through 6469 default: 6470#ifndef NDEBUG 6471 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 6472#endif 6473 assert(0 && "Do not know how to expand this operator!"); 6474 abort(); 6475 case ISD::EXTRACT_ELEMENT: 6476 ExpandOp(Node->getOperand(0), Lo, Hi); 6477 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) 6478 return ExpandOp(Hi, Lo, Hi); 6479 return ExpandOp(Lo, Lo, Hi); 6480 case ISD::EXTRACT_VECTOR_ELT: 6481 // ExpandEXTRACT_VECTOR_ELT tolerates invalid result types. 6482 Lo = ExpandEXTRACT_VECTOR_ELT(Op); 6483 return ExpandOp(Lo, Lo, Hi); 6484 case ISD::UNDEF: 6485 Lo = DAG.getNode(ISD::UNDEF, dl, NVT); 6486 Hi = DAG.getNode(ISD::UNDEF, dl, NVT); 6487 break; 6488 case ISD::Constant: { 6489 unsigned NVTBits = NVT.getSizeInBits(); 6490 const APInt &Cst = cast<ConstantSDNode>(Node)->getAPIntValue(); 6491 Lo = DAG.getConstant(APInt(Cst).trunc(NVTBits), NVT); 6492 Hi = DAG.getConstant(Cst.lshr(NVTBits).trunc(NVTBits), NVT); 6493 break; 6494 } 6495 case ISD::ConstantFP: { 6496 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 6497 if (CFP->getValueType(0) == MVT::ppcf128) { 6498 APInt api = CFP->getValueAPF().bitcastToAPInt(); 6499 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])), 6500 MVT::f64); 6501 Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])), 6502 MVT::f64); 6503 break; 6504 } 6505 Lo = ExpandConstantFP(CFP, false, DAG, TLI); 6506 if (getTypeAction(Lo.getValueType()) == Expand) 6507 ExpandOp(Lo, Lo, Hi); 6508 break; 6509 } 6510 case ISD::BUILD_PAIR: 6511 // Return the operands. 6512 Lo = Node->getOperand(0); 6513 Hi = Node->getOperand(1); 6514 break; 6515 6516 case ISD::MERGE_VALUES: 6517 if (Node->getNumValues() == 1) { 6518 ExpandOp(Op.getOperand(0), Lo, Hi); 6519 break; 6520 } 6521 // FIXME: For now only expand i64,chain = MERGE_VALUES (x, y) 6522 assert(Op.getResNo() == 0 && Node->getNumValues() == 2 && 6523 Op.getValue(1).getValueType() == MVT::Other && 6524 "unhandled MERGE_VALUES"); 6525 ExpandOp(Op.getOperand(0), Lo, Hi); 6526 // Remember that we legalized the chain. 6527 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Op.getOperand(1))); 6528 break; 6529 6530 case ISD::SIGN_EXTEND_INREG: 6531 ExpandOp(Node->getOperand(0), Lo, Hi); 6532 // sext_inreg the low part if needed. 6533 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Lo, Node->getOperand(1)); 6534 6535 // The high part gets the sign extension from the lo-part. This handles 6536 // things like sextinreg V:i64 from i8. 6537 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 6538 DAG.getConstant(NVT.getSizeInBits()-1, 6539 TLI.getShiftAmountTy())); 6540 break; 6541 6542 case ISD::BSWAP: { 6543 ExpandOp(Node->getOperand(0), Lo, Hi); 6544 SDValue TempLo = DAG.getNode(ISD::BSWAP, dl, NVT, Hi); 6545 Hi = DAG.getNode(ISD::BSWAP, dl, NVT, Lo); 6546 Lo = TempLo; 6547 break; 6548 } 6549 6550 case ISD::CTPOP: 6551 ExpandOp(Node->getOperand(0), Lo, Hi); 6552 Lo = DAG.getNode(ISD::ADD, dl, NVT, // ctpop(HL) -> ctpop(H)+ctpop(L) 6553 DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 6554 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 6555 Hi = DAG.getConstant(0, NVT); 6556 break; 6557 6558 case ISD::CTLZ: { 6559 // ctlz (HL) -> ctlz(H) != 32 ? ctlz(H) : (ctlz(L)+32) 6560 ExpandOp(Node->getOperand(0), Lo, Hi); 6561 SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); 6562 SDValue HLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Hi); 6563 SDValue TopNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), HLZ, 6564 BitsC, ISD::SETNE); 6565 SDValue LowPart = DAG.getNode(ISD::CTLZ, dl, NVT, Lo); 6566 LowPart = DAG.getNode(ISD::ADD, dl, NVT, LowPart, BitsC); 6567 6568 Lo = DAG.getNode(ISD::SELECT, dl, NVT, TopNotZero, HLZ, LowPart); 6569 Hi = DAG.getConstant(0, NVT); 6570 break; 6571 } 6572 6573 case ISD::CTTZ: { 6574 // cttz (HL) -> cttz(L) != 32 ? cttz(L) : (cttz(H)+32) 6575 ExpandOp(Node->getOperand(0), Lo, Hi); 6576 SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); 6577 SDValue LTZ = DAG.getNode(ISD::CTTZ, dl, NVT, Lo); 6578 SDValue BotNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), LTZ, 6579 BitsC, ISD::SETNE); 6580 SDValue HiPart = DAG.getNode(ISD::CTTZ, dl, NVT, Hi); 6581 HiPart = DAG.getNode(ISD::ADD, dl, NVT, HiPart, BitsC); 6582 6583 Lo = DAG.getNode(ISD::SELECT, dl, NVT, BotNotZero, LTZ, HiPart); 6584 Hi = DAG.getConstant(0, NVT); 6585 break; 6586 } 6587 6588 case ISD::VAARG: { 6589 SDValue Ch = Node->getOperand(0); // Legalize the chain. 6590 SDValue Ptr = Node->getOperand(1); // Legalize the pointer. 6591 Lo = DAG.getVAArg(NVT, dl, Ch, Ptr, Node->getOperand(2)); 6592 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, Node->getOperand(2)); 6593 6594 // Remember that we legalized the chain. 6595 Hi = LegalizeOp(Hi); 6596 AddLegalizedOperand(Op.getValue(1), Hi.getValue(1)); 6597 if (TLI.isBigEndian()) 6598 std::swap(Lo, Hi); 6599 break; 6600 } 6601 6602 case ISD::LOAD: { 6603 LoadSDNode *LD = cast<LoadSDNode>(Node); 6604 SDValue Ch = LD->getChain(); // Legalize the chain. 6605 SDValue Ptr = LD->getBasePtr(); // Legalize the pointer. 6606 ISD::LoadExtType ExtType = LD->getExtensionType(); 6607 const Value *SV = LD->getSrcValue(); 6608 int SVOffset = LD->getSrcValueOffset(); 6609 unsigned Alignment = LD->getAlignment(); 6610 bool isVolatile = LD->isVolatile(); 6611 6612 if (ExtType == ISD::NON_EXTLOAD) { 6613 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, SV, SVOffset, 6614 isVolatile, Alignment); 6615 if (VT == MVT::f32 || VT == MVT::f64) { 6616 // f32->i32 or f64->i64 one to one expansion. 6617 // Remember that we legalized the chain. 6618 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); 6619 // Recursively expand the new load. 6620 if (getTypeAction(NVT) == Expand) 6621 ExpandOp(Lo, Lo, Hi); 6622 break; 6623 } 6624 6625 // Increment the pointer to the other half. 6626 unsigned IncrementSize = Lo.getValueType().getSizeInBits()/8; 6627 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 6628 DAG.getIntPtrConstant(IncrementSize)); 6629 SVOffset += IncrementSize; 6630 Alignment = MinAlign(Alignment, IncrementSize); 6631 Hi = DAG.getLoad(NVT, dl, Ch, Ptr, SV, SVOffset, 6632 isVolatile, Alignment); 6633 6634 // Build a factor node to remember that this load is independent of the 6635 // other one. 6636 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 6637 Hi.getValue(1)); 6638 6639 // Remember that we legalized the chain. 6640 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); 6641 if (TLI.isBigEndian()) 6642 std::swap(Lo, Hi); 6643 } else { 6644 MVT EVT = LD->getMemoryVT(); 6645 6646 if ((VT == MVT::f64 && EVT == MVT::f32) || 6647 (VT == MVT::ppcf128 && (EVT==MVT::f64 || EVT==MVT::f32))) { 6648 // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND 6649 SDValue Load = DAG.getLoad(EVT, dl, Ch, Ptr, SV, 6650 SVOffset, isVolatile, Alignment); 6651 // Remember that we legalized the chain. 6652 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Load.getValue(1))); 6653 ExpandOp(DAG.getNode(ISD::FP_EXTEND, dl, VT, Load), Lo, Hi); 6654 break; 6655 } 6656 6657 if (EVT == NVT) 6658 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, SV, 6659 SVOffset, isVolatile, Alignment); 6660 else 6661 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, SV, 6662 SVOffset, EVT, isVolatile, 6663 Alignment); 6664 6665 // Remember that we legalized the chain. 6666 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); 6667 6668 if (ExtType == ISD::SEXTLOAD) { 6669 // The high part is obtained by SRA'ing all but one of the bits of the 6670 // lo part. 6671 unsigned LoSize = Lo.getValueType().getSizeInBits(); 6672 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 6673 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 6674 } else if (ExtType == ISD::ZEXTLOAD) { 6675 // The high part is just a zero. 6676 Hi = DAG.getConstant(0, NVT); 6677 } else /* if (ExtType == ISD::EXTLOAD) */ { 6678 // The high part is undefined. 6679 Hi = DAG.getNode(ISD::UNDEF, dl, NVT); 6680 } 6681 } 6682 break; 6683 } 6684 case ISD::AND: 6685 case ISD::OR: 6686 case ISD::XOR: { // Simple logical operators -> two trivial pieces. 6687 SDValue LL, LH, RL, RH; 6688 ExpandOp(Node->getOperand(0), LL, LH); 6689 ExpandOp(Node->getOperand(1), RL, RH); 6690 Lo = DAG.getNode(Node->getOpcode(), dl, NVT, LL, RL); 6691 Hi = DAG.getNode(Node->getOpcode(), dl, NVT, LH, RH); 6692 break; 6693 } 6694 case ISD::SELECT: { 6695 SDValue LL, LH, RL, RH; 6696 ExpandOp(Node->getOperand(1), LL, LH); 6697 ExpandOp(Node->getOperand(2), RL, RH); 6698 if (getTypeAction(NVT) == Expand) 6699 NVT = TLI.getTypeToExpandTo(NVT); 6700 Lo = DAG.getNode(ISD::SELECT, dl, NVT, Node->getOperand(0), LL, RL); 6701 if (VT != MVT::f32) 6702 Hi = DAG.getNode(ISD::SELECT, dl, NVT, Node->getOperand(0), LH, RH); 6703 break; 6704 } 6705 case ISD::SELECT_CC: { 6706 SDValue TL, TH, FL, FH; 6707 ExpandOp(Node->getOperand(2), TL, TH); 6708 ExpandOp(Node->getOperand(3), FL, FH); 6709 if (getTypeAction(NVT) == Expand) 6710 NVT = TLI.getTypeToExpandTo(NVT); 6711 Lo = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), 6712 Node->getOperand(1), TL, FL, Node->getOperand(4)); 6713 if (VT != MVT::f32) 6714 Hi = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), 6715 Node->getOperand(1), TH, FH, Node->getOperand(4)); 6716 break; 6717 } 6718 case ISD::ANY_EXTEND: 6719 // The low part is any extension of the input (which degenerates to a copy). 6720 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); 6721 // The high part is undefined. 6722 Hi = DAG.getNode(ISD::UNDEF, dl, NVT); 6723 break; 6724 case ISD::SIGN_EXTEND: { 6725 // The low part is just a sign extension of the input (which degenerates to 6726 // a copy). 6727 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, Node->getOperand(0)); 6728 6729 // The high part is obtained by SRA'ing all but one of the bits of the lo 6730 // part. 6731 unsigned LoSize = Lo.getValueType().getSizeInBits(); 6732 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 6733 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 6734 break; 6735 } 6736 case ISD::ZERO_EXTEND: 6737 // The low part is just a zero extension of the input (which degenerates to 6738 // a copy). 6739 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 6740 6741 // The high part is just a zero. 6742 Hi = DAG.getConstant(0, NVT); 6743 break; 6744 6745 case ISD::TRUNCATE: { 6746 // The input value must be larger than this value. Expand *it*. 6747 SDValue NewLo; 6748 ExpandOp(Node->getOperand(0), NewLo, Hi); 6749 6750 // The low part is now either the right size, or it is closer. If not the 6751 // right size, make an illegal truncate so we recursively expand it. 6752 if (NewLo.getValueType() != Node->getValueType(0)) 6753 NewLo = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), NewLo); 6754 ExpandOp(NewLo, Lo, Hi); 6755 break; 6756 } 6757 6758 case ISD::BIT_CONVERT: { 6759 SDValue Tmp; 6760 if (TLI.getOperationAction(ISD::BIT_CONVERT, VT) == TargetLowering::Custom){ 6761 // If the target wants to, allow it to lower this itself. 6762 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6763 case Expand: assert(0 && "cannot expand FP!"); 6764 case Legal: Tmp = LegalizeOp(Node->getOperand(0)); break; 6765 case Promote: Tmp = PromoteOp (Node->getOperand(0)); break; 6766 } 6767 Tmp = TLI.LowerOperation(DAG.getNode(ISD::BIT_CONVERT, dl, VT, Tmp), DAG); 6768 } 6769 6770 // f32 / f64 must be expanded to i32 / i64. 6771 if (VT == MVT::f32 || VT == MVT::f64) { 6772 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); 6773 if (getTypeAction(NVT) == Expand) 6774 ExpandOp(Lo, Lo, Hi); 6775 break; 6776 } 6777 6778 // If source operand will be expanded to the same type as VT, i.e. 6779 // i64 <- f64, i32 <- f32, expand the source operand instead. 6780 MVT VT0 = Node->getOperand(0).getValueType(); 6781 if (getTypeAction(VT0) == Expand && TLI.getTypeToTransformTo(VT0) == VT) { 6782 ExpandOp(Node->getOperand(0), Lo, Hi); 6783 break; 6784 } 6785 6786 // Turn this into a load/store pair by default. 6787 if (Tmp.getNode() == 0) 6788 Tmp = EmitStackConvert(Node->getOperand(0), VT, VT, dl); 6789 6790 ExpandOp(Tmp, Lo, Hi); 6791 break; 6792 } 6793 6794 case ISD::READCYCLECOUNTER: { 6795 assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) == 6796 TargetLowering::Custom && 6797 "Must custom expand ReadCycleCounter"); 6798 SDValue Tmp = TLI.LowerOperation(Op, DAG); 6799 assert(Tmp.getNode() && "Node must be custom expanded!"); 6800 ExpandOp(Tmp.getValue(0), Lo, Hi); 6801 AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. 6802 LegalizeOp(Tmp.getValue(1))); 6803 break; 6804 } 6805 6806 case ISD::ATOMIC_CMP_SWAP: { 6807 // This operation does not need a loop. 6808 SDValue Tmp = TLI.LowerOperation(Op, DAG); 6809 assert(Tmp.getNode() && "Node must be custom expanded!"); 6810 ExpandOp(Tmp.getValue(0), Lo, Hi); 6811 AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. 6812 LegalizeOp(Tmp.getValue(1))); 6813 break; 6814 } 6815 6816 case ISD::ATOMIC_LOAD_ADD: 6817 case ISD::ATOMIC_LOAD_SUB: 6818 case ISD::ATOMIC_LOAD_AND: 6819 case ISD::ATOMIC_LOAD_OR: 6820 case ISD::ATOMIC_LOAD_XOR: 6821 case ISD::ATOMIC_LOAD_NAND: 6822 case ISD::ATOMIC_SWAP: { 6823 // These operations require a loop to be generated. We can't do that yet, 6824 // so substitute a target-dependent pseudo and expand that later. 6825 SDValue In2Lo, In2Hi, In2; 6826 ExpandOp(Op.getOperand(2), In2Lo, In2Hi); 6827 In2 = DAG.getNode(ISD::BUILD_PAIR, dl, VT, In2Lo, In2Hi); 6828 AtomicSDNode* Anode = cast<AtomicSDNode>(Node); 6829 SDValue Replace = 6830 DAG.getAtomic(Op.getOpcode(), dl, Anode->getMemoryVT(), 6831 Op.getOperand(0), Op.getOperand(1), In2, 6832 Anode->getSrcValue(), Anode->getAlignment()); 6833 SDValue Result = TLI.LowerOperation(Replace, DAG); 6834 ExpandOp(Result.getValue(0), Lo, Hi); 6835 // Remember that we legalized the chain. 6836 AddLegalizedOperand(SDValue(Node,1), LegalizeOp(Result.getValue(1))); 6837 break; 6838 } 6839 6840 // These operators cannot be expanded directly, emit them as calls to 6841 // library functions. 6842 case ISD::FP_TO_SINT: { 6843 if (TLI.getOperationAction(ISD::FP_TO_SINT, VT) == TargetLowering::Custom) { 6844 SDValue Op; 6845 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6846 case Expand: assert(0 && "cannot expand FP!"); 6847 case Legal: Op = LegalizeOp(Node->getOperand(0)); break; 6848 case Promote: Op = PromoteOp (Node->getOperand(0)); break; 6849 } 6850 6851 Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op), DAG); 6852 6853 // Now that the custom expander is done, expand the result, which is still 6854 // VT. 6855 if (Op.getNode()) { 6856 ExpandOp(Op, Lo, Hi); 6857 break; 6858 } 6859 } 6860 6861 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Node->getOperand(0).getValueType(), 6862 VT); 6863 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected uint-to-fp conversion!"); 6864 Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); 6865 break; 6866 } 6867 6868 case ISD::FP_TO_UINT: { 6869 if (TLI.getOperationAction(ISD::FP_TO_UINT, VT) == TargetLowering::Custom) { 6870 SDValue Op; 6871 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6872 case Expand: assert(0 && "cannot expand FP!"); 6873 case Legal: Op = LegalizeOp(Node->getOperand(0)); break; 6874 case Promote: Op = PromoteOp (Node->getOperand(0)); break; 6875 } 6876 6877 Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op), DAG); 6878 6879 // Now that the custom expander is done, expand the result. 6880 if (Op.getNode()) { 6881 ExpandOp(Op, Lo, Hi); 6882 break; 6883 } 6884 } 6885 6886 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Node->getOperand(0).getValueType(), 6887 VT); 6888 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 6889 Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); 6890 break; 6891 } 6892 6893 case ISD::SHL: { 6894 // If the target wants custom lowering, do so. 6895 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6896 if (TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Custom) { 6897 SDValue Op = DAG.getNode(ISD::SHL, dl, VT, Node->getOperand(0), ShiftAmt); 6898 Op = TLI.LowerOperation(Op, DAG); 6899 if (Op.getNode()) { 6900 // Now that the custom expander is done, expand the result, which is 6901 // still VT. 6902 ExpandOp(Op, Lo, Hi); 6903 break; 6904 } 6905 } 6906 6907 // If ADDC/ADDE are supported and if the shift amount is a constant 1, emit 6908 // this X << 1 as X+X. 6909 if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(ShiftAmt)) { 6910 if (ShAmt->getAPIntValue() == 1 && 6911 TLI.isOperationLegalOrCustom(ISD::ADDC, NVT) && 6912 TLI.isOperationLegalOrCustom(ISD::ADDE, NVT)) { 6913 SDValue LoOps[2], HiOps[3]; 6914 ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]); 6915 SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag); 6916 LoOps[1] = LoOps[0]; 6917 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 6918 6919 HiOps[1] = HiOps[0]; 6920 HiOps[2] = Lo.getValue(1); 6921 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 6922 break; 6923 } 6924 } 6925 6926 // If we can emit an efficient shift operation, do so now. 6927 if (ExpandShift(ISD::SHL, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) 6928 break; 6929 6930 // If this target supports SHL_PARTS, use it. 6931 TargetLowering::LegalizeAction Action = 6932 TLI.getOperationAction(ISD::SHL_PARTS, NVT); 6933 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 6934 Action == TargetLowering::Custom) { 6935 ExpandShiftParts(ISD::SHL_PARTS, Node->getOperand(0), 6936 ShiftAmt, Lo, Hi, dl); 6937 break; 6938 } 6939 6940 // Otherwise, emit a libcall. 6941 Lo = ExpandLibCall(RTLIB::SHL_I64, Node, false/*left shift=unsigned*/, Hi); 6942 break; 6943 } 6944 6945 case ISD::SRA: { 6946 // If the target wants custom lowering, do so. 6947 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6948 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Custom) { 6949 SDValue Op = DAG.getNode(ISD::SRA, VT, Node->getOperand(0), ShiftAmt); 6950 Op = TLI.LowerOperation(Op, DAG); 6951 if (Op.getNode()) { 6952 // Now that the custom expander is done, expand the result, which is 6953 // still VT. 6954 ExpandOp(Op, Lo, Hi); 6955 break; 6956 } 6957 } 6958 6959 // If we can emit an efficient shift operation, do so now. 6960 if (ExpandShift(ISD::SRA, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) 6961 break; 6962 6963 // If this target supports SRA_PARTS, use it. 6964 TargetLowering::LegalizeAction Action = 6965 TLI.getOperationAction(ISD::SRA_PARTS, NVT); 6966 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 6967 Action == TargetLowering::Custom) { 6968 ExpandShiftParts(ISD::SRA_PARTS, Node->getOperand(0), 6969 ShiftAmt, Lo, Hi, dl); 6970 break; 6971 } 6972 6973 // Otherwise, emit a libcall. 6974 Lo = ExpandLibCall(RTLIB::SRA_I64, Node, true/*ashr is signed*/, Hi); 6975 break; 6976 } 6977 6978 case ISD::SRL: { 6979 // If the target wants custom lowering, do so. 6980 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6981 if (TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Custom) { 6982 SDValue Op = DAG.getNode(ISD::SRL, dl, VT, Node->getOperand(0), ShiftAmt); 6983 Op = TLI.LowerOperation(Op, DAG); 6984 if (Op.getNode()) { 6985 // Now that the custom expander is done, expand the result, which is 6986 // still VT. 6987 ExpandOp(Op, Lo, Hi); 6988 break; 6989 } 6990 } 6991 6992 // If we can emit an efficient shift operation, do so now. 6993 if (ExpandShift(ISD::SRL, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) 6994 break; 6995 6996 // If this target supports SRL_PARTS, use it. 6997 TargetLowering::LegalizeAction Action = 6998 TLI.getOperationAction(ISD::SRL_PARTS, NVT); 6999 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 7000 Action == TargetLowering::Custom) { 7001 ExpandShiftParts(ISD::SRL_PARTS, 7002 Node->getOperand(0), ShiftAmt, Lo, Hi, dl); 7003 break; 7004 } 7005 7006 // Otherwise, emit a libcall. 7007 Lo = ExpandLibCall(RTLIB::SRL_I64, Node, false/*lshr is unsigned*/, Hi); 7008 break; 7009 } 7010 7011 case ISD::ADD: 7012 case ISD::SUB: { 7013 // If the target wants to custom expand this, let them. 7014 if (TLI.getOperationAction(Node->getOpcode(), VT) == 7015 TargetLowering::Custom) { 7016 SDValue Result = TLI.LowerOperation(Op, DAG); 7017 if (Result.getNode()) { 7018 ExpandOp(Result, Lo, Hi); 7019 break; 7020 } 7021 } 7022 // Expand the subcomponents. 7023 SDValue LHSL, LHSH, RHSL, RHSH; 7024 ExpandOp(Node->getOperand(0), LHSL, LHSH); 7025 ExpandOp(Node->getOperand(1), RHSL, RHSH); 7026 SDValue LoOps[2], HiOps[3]; 7027 LoOps[0] = LHSL; 7028 LoOps[1] = RHSL; 7029 HiOps[0] = LHSH; 7030 HiOps[1] = RHSH; 7031 7032 //cascaded check to see if any smaller size has a a carry flag. 7033 unsigned OpV = Node->getOpcode() == ISD::ADD ? ISD::ADDC : ISD::SUBC; 7034 bool hasCarry = false; 7035 for (unsigned BitSize = NVT.getSizeInBits(); BitSize != 0; BitSize /= 2) { 7036 MVT AVT = MVT::getIntegerVT(BitSize); 7037 if (TLI.isOperationLegalOrCustom(OpV, AVT)) { 7038 hasCarry = true; 7039 break; 7040 } 7041 } 7042 7043 if(hasCarry) { 7044 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 7045 if (Node->getOpcode() == ISD::ADD) { 7046 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 7047 HiOps[2] = Lo.getValue(1); 7048 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 7049 } else { 7050 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 7051 HiOps[2] = Lo.getValue(1); 7052 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 7053 } 7054 break; 7055 } else { 7056 if (Node->getOpcode() == ISD::ADD) { 7057 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps, 2); 7058 Hi = DAG.getNode(ISD::ADD, dl, NVT, HiOps, 2); 7059 SDValue Cmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), 7060 Lo, LoOps[0], ISD::SETULT); 7061 SDValue Carry1 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp1, 7062 DAG.getConstant(1, NVT), 7063 DAG.getConstant(0, NVT)); 7064 SDValue Cmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), 7065 Lo, LoOps[1], ISD::SETULT); 7066 SDValue Carry2 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp2, 7067 DAG.getConstant(1, NVT), 7068 Carry1); 7069 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 7070 } else { 7071 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps, 2); 7072 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps, 2); 7073 SDValue Cmp = DAG.getSetCC(dl, NVT, LoOps[0], LoOps[1], ISD::SETULT); 7074 SDValue Borrow = DAG.getNode(ISD::SELECT, dl, NVT, Cmp, 7075 DAG.getConstant(1, NVT), 7076 DAG.getConstant(0, NVT)); 7077 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 7078 } 7079 break; 7080 } 7081 } 7082 7083 case ISD::ADDC: 7084 case ISD::SUBC: { 7085 // Expand the subcomponents. 7086 SDValue LHSL, LHSH, RHSL, RHSH; 7087 ExpandOp(Node->getOperand(0), LHSL, LHSH); 7088 ExpandOp(Node->getOperand(1), RHSL, RHSH); 7089 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 7090 SDValue LoOps[2] = { LHSL, RHSL }; 7091 SDValue HiOps[3] = { LHSH, RHSH }; 7092 7093 if (Node->getOpcode() == ISD::ADDC) { 7094 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 7095 HiOps[2] = Lo.getValue(1); 7096 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 7097 } else { 7098 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 7099 HiOps[2] = Lo.getValue(1); 7100 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 7101 } 7102 // Remember that we legalized the flag. 7103 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1))); 7104 break; 7105 } 7106 case ISD::ADDE: 7107 case ISD::SUBE: { 7108 // Expand the subcomponents. 7109 SDValue LHSL, LHSH, RHSL, RHSH; 7110 ExpandOp(Node->getOperand(0), LHSL, LHSH); 7111 ExpandOp(Node->getOperand(1), RHSL, RHSH); 7112 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 7113 SDValue LoOps[3] = { LHSL, RHSL, Node->getOperand(2) }; 7114 SDValue HiOps[3] = { LHSH, RHSH }; 7115 7116 Lo = DAG.getNode(Node->getOpcode(), dl, VTList, LoOps, 3); 7117 HiOps[2] = Lo.getValue(1); 7118 Hi = DAG.getNode(Node->getOpcode(), dl, VTList, HiOps, 3); 7119 7120 // Remember that we legalized the flag. 7121 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1))); 7122 break; 7123 } 7124 case ISD::MUL: { 7125 // If the target wants to custom expand this, let them. 7126 if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) { 7127 SDValue New = TLI.LowerOperation(Op, DAG); 7128 if (New.getNode()) { 7129 ExpandOp(New, Lo, Hi); 7130 break; 7131 } 7132 } 7133 7134 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT); 7135 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT); 7136 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT); 7137 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT); 7138 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 7139 SDValue LL, LH, RL, RH; 7140 ExpandOp(Node->getOperand(0), LL, LH); 7141 ExpandOp(Node->getOperand(1), RL, RH); 7142 unsigned OuterBitSize = Op.getValueSizeInBits(); 7143 unsigned InnerBitSize = RH.getValueSizeInBits(); 7144 unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0)); 7145 unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1)); 7146 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 7147 if (DAG.MaskedValueIsZero(Node->getOperand(0), HighMask) && 7148 DAG.MaskedValueIsZero(Node->getOperand(1), HighMask)) { 7149 // The inputs are both zero-extended. 7150 if (HasUMUL_LOHI) { 7151 // We can emit a umul_lohi. 7152 Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 7153 Hi = SDValue(Lo.getNode(), 1); 7154 break; 7155 } 7156 if (HasMULHU) { 7157 // We can emit a mulhu+mul. 7158 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 7159 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 7160 break; 7161 } 7162 } 7163 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 7164 // The input values are both sign-extended. 7165 if (HasSMUL_LOHI) { 7166 // We can emit a smul_lohi. 7167 Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 7168 Hi = SDValue(Lo.getNode(), 1); 7169 break; 7170 } 7171 if (HasMULHS) { 7172 // We can emit a mulhs+mul. 7173 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 7174 Hi = DAG.getNode(ISD::MULHS, dl, NVT, LL, RL); 7175 break; 7176 } 7177 } 7178 if (HasUMUL_LOHI) { 7179 // Lo,Hi = umul LHS, RHS. 7180 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl, 7181 DAG.getVTList(NVT, NVT), LL, RL); 7182 Lo = UMulLOHI; 7183 Hi = UMulLOHI.getValue(1); 7184 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 7185 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 7186 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 7187 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 7188 break; 7189 } 7190 if (HasMULHU) { 7191 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 7192 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 7193 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 7194 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 7195 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 7196 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 7197 break; 7198 } 7199 } 7200 7201 // If nothing else, we can make a libcall. 7202 Lo = ExpandLibCall(RTLIB::MUL_I64, Node, false/*sign irrelevant*/, Hi); 7203 break; 7204 } 7205 case ISD::SDIV: 7206 Lo = ExpandLibCall(RTLIB::SDIV_I64, Node, true, Hi); 7207 break; 7208 case ISD::UDIV: 7209 Lo = ExpandLibCall(RTLIB::UDIV_I64, Node, true, Hi); 7210 break; 7211 case ISD::SREM: 7212 Lo = ExpandLibCall(RTLIB::SREM_I64, Node, true, Hi); 7213 break; 7214 case ISD::UREM: 7215 Lo = ExpandLibCall(RTLIB::UREM_I64, Node, true, Hi); 7216 break; 7217 7218 case ISD::FADD: 7219 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::ADD_F32, 7220 RTLIB::ADD_F64, 7221 RTLIB::ADD_F80, 7222 RTLIB::ADD_PPCF128), 7223 Node, false, Hi); 7224 break; 7225 case ISD::FSUB: 7226 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::SUB_F32, 7227 RTLIB::SUB_F64, 7228 RTLIB::SUB_F80, 7229 RTLIB::SUB_PPCF128), 7230 Node, false, Hi); 7231 break; 7232 case ISD::FMUL: 7233 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::MUL_F32, 7234 RTLIB::MUL_F64, 7235 RTLIB::MUL_F80, 7236 RTLIB::MUL_PPCF128), 7237 Node, false, Hi); 7238 break; 7239 case ISD::FDIV: 7240 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::DIV_F32, 7241 RTLIB::DIV_F64, 7242 RTLIB::DIV_F80, 7243 RTLIB::DIV_PPCF128), 7244 Node, false, Hi); 7245 break; 7246 case ISD::FP_EXTEND: { 7247 if (VT == MVT::ppcf128) { 7248 assert(Node->getOperand(0).getValueType()==MVT::f32 || 7249 Node->getOperand(0).getValueType()==MVT::f64); 7250 const uint64_t zero = 0; 7251 if (Node->getOperand(0).getValueType()==MVT::f32) 7252 Hi = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Node->getOperand(0)); 7253 else 7254 Hi = Node->getOperand(0); 7255 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7256 break; 7257 } 7258 RTLIB::Libcall LC = RTLIB::getFPEXT(Node->getOperand(0).getValueType(), VT); 7259 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 7260 Lo = ExpandLibCall(LC, Node, true, Hi); 7261 break; 7262 } 7263 case ISD::FP_ROUND: { 7264 RTLIB::Libcall LC = RTLIB::getFPROUND(Node->getOperand(0).getValueType(), 7265 VT); 7266 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 7267 Lo = ExpandLibCall(LC, Node, true, Hi); 7268 break; 7269 } 7270 case ISD::FSQRT: 7271 case ISD::FSIN: 7272 case ISD::FCOS: 7273 case ISD::FLOG: 7274 case ISD::FLOG2: 7275 case ISD::FLOG10: 7276 case ISD::FEXP: 7277 case ISD::FEXP2: 7278 case ISD::FTRUNC: 7279 case ISD::FFLOOR: 7280 case ISD::FCEIL: 7281 case ISD::FRINT: 7282 case ISD::FNEARBYINT: 7283 case ISD::FPOW: 7284 case ISD::FPOWI: { 7285 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 7286 switch(Node->getOpcode()) { 7287 case ISD::FSQRT: 7288 LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 7289 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128); 7290 break; 7291 case ISD::FSIN: 7292 LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64, 7293 RTLIB::SIN_F80, RTLIB::SIN_PPCF128); 7294 break; 7295 case ISD::FCOS: 7296 LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, 7297 RTLIB::COS_F80, RTLIB::COS_PPCF128); 7298 break; 7299 case ISD::FLOG: 7300 LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, 7301 RTLIB::LOG_F80, RTLIB::LOG_PPCF128); 7302 break; 7303 case ISD::FLOG2: 7304 LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 7305 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); 7306 break; 7307 case ISD::FLOG10: 7308 LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 7309 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); 7310 break; 7311 case ISD::FEXP: 7312 LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, 7313 RTLIB::EXP_F80, RTLIB::EXP_PPCF128); 7314 break; 7315 case ISD::FEXP2: 7316 LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 7317 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); 7318 break; 7319 case ISD::FTRUNC: 7320 LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 7321 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); 7322 break; 7323 case ISD::FFLOOR: 7324 LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 7325 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); 7326 break; 7327 case ISD::FCEIL: 7328 LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 7329 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); 7330 break; 7331 case ISD::FRINT: 7332 LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, 7333 RTLIB::RINT_F80, RTLIB::RINT_PPCF128); 7334 break; 7335 case ISD::FNEARBYINT: 7336 LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, 7337 RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); 7338 break; 7339 case ISD::FPOW: 7340 LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, 7341 RTLIB::POW_PPCF128); 7342 break; 7343 case ISD::FPOWI: 7344 LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, RTLIB::POWI_F80, 7345 RTLIB::POWI_PPCF128); 7346 break; 7347 default: assert(0 && "Unreachable!"); 7348 } 7349 Lo = ExpandLibCall(LC, Node, false, Hi); 7350 break; 7351 } 7352 case ISD::FABS: { 7353 if (VT == MVT::ppcf128) { 7354 SDValue Tmp; 7355 ExpandOp(Node->getOperand(0), Lo, Tmp); 7356 Hi = DAG.getNode(ISD::FABS, dl, NVT, Tmp); 7357 // lo = hi==fabs(hi) ? lo : -lo; 7358 Lo = DAG.getNode(ISD::SELECT_CC, dl, NVT, Hi, Tmp, 7359 Lo, DAG.getNode(ISD::FNEG, dl, NVT, Lo), 7360 DAG.getCondCode(ISD::SETEQ)); 7361 break; 7362 } 7363 SDValue Mask = (VT == MVT::f64) 7364 ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) 7365 : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); 7366 Mask = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask); 7367 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); 7368 Lo = DAG.getNode(ISD::AND, dl, NVT, Lo, Mask); 7369 if (getTypeAction(NVT) == Expand) 7370 ExpandOp(Lo, Lo, Hi); 7371 break; 7372 } 7373 case ISD::FNEG: { 7374 if (VT == MVT::ppcf128) { 7375 ExpandOp(Node->getOperand(0), Lo, Hi); 7376 Lo = DAG.getNode(ISD::FNEG, dl, MVT::f64, Lo); 7377 Hi = DAG.getNode(ISD::FNEG, dl, MVT::f64, Hi); 7378 break; 7379 } 7380 SDValue Mask = (VT == MVT::f64) 7381 ? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT) 7382 : DAG.getConstantFP(BitsToFloat(1U << 31), VT); 7383 Mask = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask); 7384 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); 7385 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Mask); 7386 if (getTypeAction(NVT) == Expand) 7387 ExpandOp(Lo, Lo, Hi); 7388 break; 7389 } 7390 case ISD::FCOPYSIGN: { 7391 Lo = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI); 7392 if (getTypeAction(NVT) == Expand) 7393 ExpandOp(Lo, Lo, Hi); 7394 break; 7395 } 7396 case ISD::SINT_TO_FP: 7397 case ISD::UINT_TO_FP: { 7398 bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; 7399 MVT SrcVT = Node->getOperand(0).getValueType(); 7400 7401 // Promote the operand if needed. Do this before checking for 7402 // ppcf128 so conversions of i16 and i8 work. 7403 if (getTypeAction(SrcVT) == Promote) { 7404 SDValue Tmp = PromoteOp(Node->getOperand(0)); 7405 Tmp = isSigned 7406 ? DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Tmp.getValueType(), Tmp, 7407 DAG.getValueType(SrcVT)) 7408 : DAG.getZeroExtendInReg(Tmp, dl, SrcVT); 7409 Node = DAG.UpdateNodeOperands(Op, Tmp).getNode(); 7410 SrcVT = Node->getOperand(0).getValueType(); 7411 } 7412 7413 if (VT == MVT::ppcf128 && SrcVT == MVT::i32) { 7414 static const uint64_t zero = 0; 7415 if (isSigned) { 7416 Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f64, 7417 Node->getOperand(0))); 7418 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7419 } else { 7420 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 7421 Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f64, 7422 Node->getOperand(0))); 7423 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7424 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 7425 // X>=0 ? {(f64)x, 0} : {(f64)x, 0} + 2^32 7426 ExpandOp(DAG.getNode(ISD::SELECT_CC, dl, 7427 MVT::ppcf128, Node->getOperand(0), 7428 DAG.getConstant(0, MVT::i32), 7429 DAG.getNode(ISD::FADD, dl, MVT::ppcf128, Hi, 7430 DAG.getConstantFP( 7431 APFloat(APInt(128, 2, TwoE32)), 7432 MVT::ppcf128)), 7433 Hi, 7434 DAG.getCondCode(ISD::SETLT)), 7435 Lo, Hi); 7436 } 7437 break; 7438 } 7439 if (VT == MVT::ppcf128 && SrcVT == MVT::i64 && !isSigned) { 7440 // si64->ppcf128 done by libcall, below 7441 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 7442 ExpandOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::ppcf128, 7443 Node->getOperand(0)), Lo, Hi); 7444 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 7445 // x>=0 ? (ppcf128)(i64)x : (ppcf128)(i64)x + 2^64 7446 ExpandOp(DAG.getNode(ISD::SELECT_CC, dl, MVT::ppcf128, 7447 Node->getOperand(0), 7448 DAG.getConstant(0, MVT::i64), 7449 DAG.getNode(ISD::FADD, dl, MVT::ppcf128, Hi, 7450 DAG.getConstantFP( 7451 APFloat(APInt(128, 2, TwoE64)), 7452 MVT::ppcf128)), 7453 Hi, 7454 DAG.getCondCode(ISD::SETLT)), 7455 Lo, Hi); 7456 break; 7457 } 7458 7459 Lo = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, VT, 7460 Node->getOperand(0), dl); 7461 if (getTypeAction(Lo.getValueType()) == Expand) 7462 // float to i32 etc. can be 'expanded' to a single node. 7463 ExpandOp(Lo, Lo, Hi); 7464 break; 7465 } 7466 } 7467 7468 // Make sure the resultant values have been legalized themselves, unless this 7469 // is a type that requires multi-step expansion. 7470 if (getTypeAction(NVT) != Expand && NVT != MVT::isVoid) { 7471 Lo = LegalizeOp(Lo); 7472 if (Hi.getNode()) 7473 // Don't legalize the high part if it is expanded to a single node. 7474 Hi = LegalizeOp(Hi); 7475 } 7476 7477 // Remember in a map if the values will be reused later. 7478 bool isNew = 7479 ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; 7480 assert(isNew && "Value already expanded?!?"); 7481 isNew = isNew; 7482} 7483 7484/// SplitVectorOp - Given an operand of vector type, break it down into 7485/// two smaller values, still of vector type. 7486void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo, 7487 SDValue &Hi) { 7488 assert(Op.getValueType().isVector() && "Cannot split non-vector type!"); 7489 SDNode *Node = Op.getNode(); 7490 DebugLoc dl = Node->getDebugLoc(); 7491 unsigned NumElements = Op.getValueType().getVectorNumElements(); 7492 assert(NumElements > 1 && "Cannot split a single element vector!"); 7493 7494 MVT NewEltVT = Op.getValueType().getVectorElementType(); 7495 7496 unsigned NewNumElts_Lo = 1 << Log2_32(NumElements-1); 7497 unsigned NewNumElts_Hi = NumElements - NewNumElts_Lo; 7498 7499 MVT NewVT_Lo = MVT::getVectorVT(NewEltVT, NewNumElts_Lo); 7500 MVT NewVT_Hi = MVT::getVectorVT(NewEltVT, NewNumElts_Hi); 7501 7502 // See if we already split it. 7503 std::map<SDValue, std::pair<SDValue, SDValue> >::iterator I 7504 = SplitNodes.find(Op); 7505 if (I != SplitNodes.end()) { 7506 Lo = I->second.first; 7507 Hi = I->second.second; 7508 return; 7509 } 7510 7511 switch (Node->getOpcode()) { 7512 default: 7513#ifndef NDEBUG 7514 Node->dump(&DAG); 7515#endif 7516 assert(0 && "Unhandled operation in SplitVectorOp!"); 7517 case ISD::UNDEF: 7518 Lo = DAG.getNode(ISD::UNDEF, dl, NewVT_Lo); 7519 Hi = DAG.getNode(ISD::UNDEF, dl, NewVT_Hi); 7520 break; 7521 case ISD::BUILD_PAIR: 7522 Lo = Node->getOperand(0); 7523 Hi = Node->getOperand(1); 7524 break; 7525 case ISD::INSERT_VECTOR_ELT: { 7526 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Node->getOperand(2))) { 7527 SplitVectorOp(Node->getOperand(0), Lo, Hi); 7528 unsigned Index = Idx->getZExtValue(); 7529 SDValue ScalarOp = Node->getOperand(1); 7530 if (Index < NewNumElts_Lo) 7531 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVT_Lo, Lo, ScalarOp, 7532 DAG.getIntPtrConstant(Index)); 7533 else 7534 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVT_Hi, Hi, ScalarOp, 7535 DAG.getIntPtrConstant(Index - NewNumElts_Lo)); 7536 break; 7537 } 7538 SDValue Tmp = PerformInsertVectorEltInMemory(Node->getOperand(0), 7539 Node->getOperand(1), 7540 Node->getOperand(2), dl); 7541 SplitVectorOp(Tmp, Lo, Hi); 7542 break; 7543 } 7544 case ISD::VECTOR_SHUFFLE: { 7545 // Build the low part. 7546 SDValue Mask = Node->getOperand(2); 7547 SmallVector<SDValue, 8> Ops; 7548 MVT PtrVT = TLI.getPointerTy(); 7549 7550 // Insert all of the elements from the input that are needed. We use 7551 // buildvector of extractelement here because the input vectors will have 7552 // to be legalized, so this makes the code simpler. 7553 for (unsigned i = 0; i != NewNumElts_Lo; ++i) { 7554 SDValue IdxNode = Mask.getOperand(i); 7555 if (IdxNode.getOpcode() == ISD::UNDEF) { 7556 Ops.push_back(DAG.getNode(ISD::UNDEF, dl, NewEltVT)); 7557 continue; 7558 } 7559 unsigned Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue(); 7560 SDValue InVec = Node->getOperand(0); 7561 if (Idx >= NumElements) { 7562 InVec = Node->getOperand(1); 7563 Idx -= NumElements; 7564 } 7565 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, InVec, 7566 DAG.getConstant(Idx, PtrVT))); 7567 } 7568 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Lo, &Ops[0], Ops.size()); 7569 Ops.clear(); 7570 7571 for (unsigned i = NewNumElts_Lo; i != NumElements; ++i) { 7572 SDValue IdxNode = Mask.getOperand(i); 7573 if (IdxNode.getOpcode() == ISD::UNDEF) { 7574 Ops.push_back(DAG.getNode(ISD::UNDEF, dl, NewEltVT)); 7575 continue; 7576 } 7577 unsigned Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue(); 7578 SDValue InVec = Node->getOperand(0); 7579 if (Idx >= NumElements) { 7580 InVec = Node->getOperand(1); 7581 Idx -= NumElements; 7582 } 7583 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, InVec, 7584 DAG.getConstant(Idx, PtrVT))); 7585 } 7586 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Hi, &Ops[0], Ops.size()); 7587 break; 7588 } 7589 case ISD::BUILD_VECTOR: { 7590 SmallVector<SDValue, 8> LoOps(Node->op_begin(), 7591 Node->op_begin()+NewNumElts_Lo); 7592 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Lo, &LoOps[0], LoOps.size()); 7593 7594 SmallVector<SDValue, 8> HiOps(Node->op_begin()+NewNumElts_Lo, 7595 Node->op_end()); 7596 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Hi, &HiOps[0], HiOps.size()); 7597 break; 7598 } 7599 case ISD::CONCAT_VECTORS: { 7600 // FIXME: Handle non-power-of-two vectors? 7601 unsigned NewNumSubvectors = Node->getNumOperands() / 2; 7602 if (NewNumSubvectors == 1) { 7603 Lo = Node->getOperand(0); 7604 Hi = Node->getOperand(1); 7605 } else { 7606 SmallVector<SDValue, 8> LoOps(Node->op_begin(), 7607 Node->op_begin()+NewNumSubvectors); 7608 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT_Lo, 7609 &LoOps[0], LoOps.size()); 7610 7611 SmallVector<SDValue, 8> HiOps(Node->op_begin()+NewNumSubvectors, 7612 Node->op_end()); 7613 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT_Hi, 7614 &HiOps[0], HiOps.size()); 7615 } 7616 break; 7617 } 7618 case ISD::EXTRACT_SUBVECTOR: { 7619 SDValue Vec = Op.getOperand(0); 7620 SDValue Idx = Op.getOperand(1); 7621 MVT IdxVT = Idx.getValueType(); 7622 7623 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Lo, Vec, Idx); 7624 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 7625 if (CIdx) { 7626 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Hi, Vec, 7627 DAG.getConstant(CIdx->getZExtValue() + NewNumElts_Lo, 7628 IdxVT)); 7629 } else { 7630 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 7631 DAG.getConstant(NewNumElts_Lo, IdxVT)); 7632 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Hi, Vec, Idx); 7633 } 7634 break; 7635 } 7636 case ISD::SELECT: { 7637 SDValue Cond = Node->getOperand(0); 7638 7639 SDValue LL, LH, RL, RH; 7640 SplitVectorOp(Node->getOperand(1), LL, LH); 7641 SplitVectorOp(Node->getOperand(2), RL, RH); 7642 7643 if (Cond.getValueType().isVector()) { 7644 // Handle a vector merge. 7645 SDValue CL, CH; 7646 SplitVectorOp(Cond, CL, CH); 7647 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, CL, LL, RL); 7648 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, CH, LH, RH); 7649 } else { 7650 // Handle a simple select with vector operands. 7651 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, Cond, LL, RL); 7652 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, Cond, LH, RH); 7653 } 7654 break; 7655 } 7656 case ISD::SELECT_CC: { 7657 SDValue CondLHS = Node->getOperand(0); 7658 SDValue CondRHS = Node->getOperand(1); 7659 SDValue CondCode = Node->getOperand(4); 7660 7661 SDValue LL, LH, RL, RH; 7662 SplitVectorOp(Node->getOperand(2), LL, LH); 7663 SplitVectorOp(Node->getOperand(3), RL, RH); 7664 7665 // Handle a simple select with vector operands. 7666 Lo = DAG.getNode(ISD::SELECT_CC, dl, NewVT_Lo, CondLHS, CondRHS, 7667 LL, RL, CondCode); 7668 Hi = DAG.getNode(ISD::SELECT_CC, dl, NewVT_Hi, CondLHS, CondRHS, 7669 LH, RH, CondCode); 7670 break; 7671 } 7672 case ISD::VSETCC: { 7673 SDValue LL, LH, RL, RH; 7674 SplitVectorOp(Node->getOperand(0), LL, LH); 7675 SplitVectorOp(Node->getOperand(1), RL, RH); 7676 Lo = DAG.getNode(ISD::VSETCC, dl, NewVT_Lo, LL, RL, Node->getOperand(2)); 7677 Hi = DAG.getNode(ISD::VSETCC, dl, NewVT_Hi, LH, RH, Node->getOperand(2)); 7678 break; 7679 } 7680 case ISD::ADD: 7681 case ISD::SUB: 7682 case ISD::MUL: 7683 case ISD::FADD: 7684 case ISD::FSUB: 7685 case ISD::FMUL: 7686 case ISD::SDIV: 7687 case ISD::UDIV: 7688 case ISD::FDIV: 7689 case ISD::FPOW: 7690 case ISD::AND: 7691 case ISD::OR: 7692 case ISD::XOR: 7693 case ISD::UREM: 7694 case ISD::SREM: 7695 case ISD::FREM: 7696 case ISD::SHL: 7697 case ISD::SRA: 7698 case ISD::SRL: { 7699 SDValue LL, LH, RL, RH; 7700 SplitVectorOp(Node->getOperand(0), LL, LH); 7701 SplitVectorOp(Node->getOperand(1), RL, RH); 7702 7703 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, LL, RL); 7704 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, LH, RH); 7705 break; 7706 } 7707 case ISD::FP_ROUND: 7708 case ISD::FPOWI: { 7709 SDValue L, H; 7710 SplitVectorOp(Node->getOperand(0), L, H); 7711 7712 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, L, Node->getOperand(1)); 7713 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, H, Node->getOperand(1)); 7714 break; 7715 } 7716 case ISD::CTTZ: 7717 case ISD::CTLZ: 7718 case ISD::CTPOP: 7719 case ISD::FNEG: 7720 case ISD::FABS: 7721 case ISD::FSQRT: 7722 case ISD::FSIN: 7723 case ISD::FCOS: 7724 case ISD::FLOG: 7725 case ISD::FLOG2: 7726 case ISD::FLOG10: 7727 case ISD::FEXP: 7728 case ISD::FEXP2: 7729 case ISD::FP_TO_SINT: 7730 case ISD::FP_TO_UINT: 7731 case ISD::SINT_TO_FP: 7732 case ISD::UINT_TO_FP: 7733 case ISD::TRUNCATE: 7734 case ISD::ANY_EXTEND: 7735 case ISD::SIGN_EXTEND: 7736 case ISD::ZERO_EXTEND: 7737 case ISD::FP_EXTEND: { 7738 SDValue L, H; 7739 SplitVectorOp(Node->getOperand(0), L, H); 7740 7741 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, L); 7742 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, H); 7743 break; 7744 } 7745 case ISD::CONVERT_RNDSAT: { 7746 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 7747 SDValue L, H; 7748 SplitVectorOp(Node->getOperand(0), L, H); 7749 SDValue DTyOpL = DAG.getValueType(NewVT_Lo); 7750 SDValue DTyOpH = DAG.getValueType(NewVT_Hi); 7751 SDValue STyOpL = DAG.getValueType(L.getValueType()); 7752 SDValue STyOpH = DAG.getValueType(H.getValueType()); 7753 7754 SDValue RndOp = Node->getOperand(3); 7755 SDValue SatOp = Node->getOperand(4); 7756 7757 Lo = DAG.getConvertRndSat(NewVT_Lo, dl, L, DTyOpL, STyOpL, 7758 RndOp, SatOp, CvtCode); 7759 Hi = DAG.getConvertRndSat(NewVT_Hi, dl, H, DTyOpH, STyOpH, 7760 RndOp, SatOp, CvtCode); 7761 break; 7762 } 7763 case ISD::LOAD: { 7764 LoadSDNode *LD = cast<LoadSDNode>(Node); 7765 SDValue Ch = LD->getChain(); 7766 SDValue Ptr = LD->getBasePtr(); 7767 ISD::LoadExtType ExtType = LD->getExtensionType(); 7768 const Value *SV = LD->getSrcValue(); 7769 int SVOffset = LD->getSrcValueOffset(); 7770 MVT MemoryVT = LD->getMemoryVT(); 7771 unsigned Alignment = LD->getAlignment(); 7772 bool isVolatile = LD->isVolatile(); 7773 7774 assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); 7775 SDValue Offset = DAG.getNode(ISD::UNDEF, dl, Ptr.getValueType()); 7776 7777 MVT MemNewEltVT = MemoryVT.getVectorElementType(); 7778 MVT MemNewVT_Lo = MVT::getVectorVT(MemNewEltVT, NewNumElts_Lo); 7779 MVT MemNewVT_Hi = MVT::getVectorVT(MemNewEltVT, NewNumElts_Hi); 7780 7781 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, 7782 NewVT_Lo, Ch, Ptr, Offset, 7783 SV, SVOffset, MemNewVT_Lo, isVolatile, Alignment); 7784 unsigned IncrementSize = NewNumElts_Lo * MemNewEltVT.getSizeInBits()/8; 7785 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 7786 DAG.getIntPtrConstant(IncrementSize)); 7787 SVOffset += IncrementSize; 7788 Alignment = MinAlign(Alignment, IncrementSize); 7789 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, 7790 NewVT_Hi, Ch, Ptr, Offset, 7791 SV, SVOffset, MemNewVT_Hi, isVolatile, Alignment); 7792 7793 // Build a factor node to remember that this load is independent of the 7794 // other one. 7795 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 7796 Hi.getValue(1)); 7797 7798 // Remember that we legalized the chain. 7799 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); 7800 break; 7801 } 7802 case ISD::BIT_CONVERT: { 7803 // We know the result is a vector. The input may be either a vector or a 7804 // scalar value. 7805 SDValue InOp = Node->getOperand(0); 7806 if (!InOp.getValueType().isVector() || 7807 InOp.getValueType().getVectorNumElements() == 1) { 7808 // The input is a scalar or single-element vector. 7809 // Lower to a store/load so that it can be split. 7810 // FIXME: this could be improved probably. 7811 unsigned LdAlign = TLI.getTargetData()->getPrefTypeAlignment( 7812 Op.getValueType().getTypeForMVT()); 7813 SDValue Ptr = DAG.CreateStackTemporary(InOp.getValueType(), LdAlign); 7814 int FI = cast<FrameIndexSDNode>(Ptr.getNode())->getIndex(); 7815 7816 SDValue St = DAG.getStore(DAG.getEntryNode(), dl, 7817 InOp, Ptr, 7818 PseudoSourceValue::getFixedStack(FI), 0); 7819 InOp = DAG.getLoad(Op.getValueType(), dl, St, Ptr, 7820 PseudoSourceValue::getFixedStack(FI), 0); 7821 } 7822 // Split the vector and convert each of the pieces now. 7823 SplitVectorOp(InOp, Lo, Hi); 7824 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT_Lo, Lo); 7825 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT_Hi, Hi); 7826 break; 7827 } 7828 } 7829 7830 // Remember in a map if the values will be reused later. 7831 bool isNew = 7832 SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; 7833 assert(isNew && "Value already split?!?"); 7834 isNew = isNew; 7835} 7836 7837 7838/// ScalarizeVectorOp - Given an operand of single-element vector type 7839/// (e.g. v1f32), convert it into the equivalent operation that returns a 7840/// scalar (e.g. f32) value. 7841SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) { 7842 assert(Op.getValueType().isVector() && "Bad ScalarizeVectorOp invocation!"); 7843 SDNode *Node = Op.getNode(); 7844 DebugLoc dl = Node->getDebugLoc(); 7845 MVT NewVT = Op.getValueType().getVectorElementType(); 7846 assert(Op.getValueType().getVectorNumElements() == 1); 7847 7848 // See if we already scalarized it. 7849 std::map<SDValue, SDValue>::iterator I = ScalarizedNodes.find(Op); 7850 if (I != ScalarizedNodes.end()) return I->second; 7851 7852 SDValue Result; 7853 switch (Node->getOpcode()) { 7854 default: 7855#ifndef NDEBUG 7856 Node->dump(&DAG); cerr << "\n"; 7857#endif 7858 assert(0 && "Unknown vector operation in ScalarizeVectorOp!"); 7859 case ISD::ADD: 7860 case ISD::FADD: 7861 case ISD::SUB: 7862 case ISD::FSUB: 7863 case ISD::MUL: 7864 case ISD::FMUL: 7865 case ISD::SDIV: 7866 case ISD::UDIV: 7867 case ISD::FDIV: 7868 case ISD::SREM: 7869 case ISD::UREM: 7870 case ISD::FREM: 7871 case ISD::FPOW: 7872 case ISD::AND: 7873 case ISD::OR: 7874 case ISD::XOR: 7875 Result = DAG.getNode(Node->getOpcode(), dl, 7876 NewVT, 7877 ScalarizeVectorOp(Node->getOperand(0)), 7878 ScalarizeVectorOp(Node->getOperand(1))); 7879 break; 7880 case ISD::FNEG: 7881 case ISD::FABS: 7882 case ISD::FSQRT: 7883 case ISD::FSIN: 7884 case ISD::FCOS: 7885 case ISD::FLOG: 7886 case ISD::FLOG2: 7887 case ISD::FLOG10: 7888 case ISD::FEXP: 7889 case ISD::FEXP2: 7890 case ISD::FP_TO_SINT: 7891 case ISD::FP_TO_UINT: 7892 case ISD::SINT_TO_FP: 7893 case ISD::UINT_TO_FP: 7894 case ISD::SIGN_EXTEND: 7895 case ISD::ZERO_EXTEND: 7896 case ISD::ANY_EXTEND: 7897 case ISD::TRUNCATE: 7898 case ISD::FP_EXTEND: 7899 Result = DAG.getNode(Node->getOpcode(), dl, 7900 NewVT, 7901 ScalarizeVectorOp(Node->getOperand(0))); 7902 break; 7903 case ISD::CONVERT_RNDSAT: { 7904 SDValue Op0 = ScalarizeVectorOp(Node->getOperand(0)); 7905 Result = DAG.getConvertRndSat(NewVT, dl, Op0, 7906 DAG.getValueType(NewVT), 7907 DAG.getValueType(Op0.getValueType()), 7908 Node->getOperand(3), 7909 Node->getOperand(4), 7910 cast<CvtRndSatSDNode>(Node)->getCvtCode()); 7911 break; 7912 } 7913 case ISD::FPOWI: 7914 case ISD::FP_ROUND: 7915 Result = DAG.getNode(Node->getOpcode(), dl, 7916 NewVT, 7917 ScalarizeVectorOp(Node->getOperand(0)), 7918 Node->getOperand(1)); 7919 break; 7920 case ISD::LOAD: { 7921 LoadSDNode *LD = cast<LoadSDNode>(Node); 7922 SDValue Ch = LegalizeOp(LD->getChain()); // Legalize the chain. 7923 SDValue Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer. 7924 ISD::LoadExtType ExtType = LD->getExtensionType(); 7925 const Value *SV = LD->getSrcValue(); 7926 int SVOffset = LD->getSrcValueOffset(); 7927 MVT MemoryVT = LD->getMemoryVT(); 7928 unsigned Alignment = LD->getAlignment(); 7929 bool isVolatile = LD->isVolatile(); 7930 7931 assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); 7932 SDValue Offset = DAG.getNode(ISD::UNDEF, dl, Ptr.getValueType()); 7933 7934 Result = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, 7935 NewVT, Ch, Ptr, Offset, SV, SVOffset, 7936 MemoryVT.getVectorElementType(), 7937 isVolatile, Alignment); 7938 7939 // Remember that we legalized the chain. 7940 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 7941 break; 7942 } 7943 case ISD::BUILD_VECTOR: 7944 Result = Node->getOperand(0); 7945 break; 7946 case ISD::INSERT_VECTOR_ELT: 7947 // Returning the inserted scalar element. 7948 Result = Node->getOperand(1); 7949 break; 7950 case ISD::CONCAT_VECTORS: 7951 assert(Node->getOperand(0).getValueType() == NewVT && 7952 "Concat of non-legal vectors not yet supported!"); 7953 Result = Node->getOperand(0); 7954 break; 7955 case ISD::VECTOR_SHUFFLE: { 7956 // Figure out if the scalar is the LHS or RHS and return it. 7957 SDValue EltNum = Node->getOperand(2).getOperand(0); 7958 if (cast<ConstantSDNode>(EltNum)->getZExtValue()) 7959 Result = ScalarizeVectorOp(Node->getOperand(1)); 7960 else 7961 Result = ScalarizeVectorOp(Node->getOperand(0)); 7962 break; 7963 } 7964 case ISD::EXTRACT_SUBVECTOR: 7965 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, 7966 Node->getOperand(0), Node->getOperand(1)); 7967 break; 7968 case ISD::BIT_CONVERT: { 7969 SDValue Op0 = Op.getOperand(0); 7970 if (Op0.getValueType().getVectorNumElements() == 1) 7971 Op0 = ScalarizeVectorOp(Op0); 7972 Result = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, Op0); 7973 break; 7974 } 7975 case ISD::SELECT: 7976 Result = DAG.getNode(ISD::SELECT, dl, NewVT, Op.getOperand(0), 7977 ScalarizeVectorOp(Op.getOperand(1)), 7978 ScalarizeVectorOp(Op.getOperand(2))); 7979 break; 7980 case ISD::SELECT_CC: 7981 Result = DAG.getNode(ISD::SELECT_CC, dl, NewVT, Node->getOperand(0), 7982 Node->getOperand(1), 7983 ScalarizeVectorOp(Op.getOperand(2)), 7984 ScalarizeVectorOp(Op.getOperand(3)), 7985 Node->getOperand(4)); 7986 break; 7987 case ISD::VSETCC: { 7988 SDValue Op0 = ScalarizeVectorOp(Op.getOperand(0)); 7989 SDValue Op1 = ScalarizeVectorOp(Op.getOperand(1)); 7990 Result = DAG.getNode(ISD::SETCC, dl, 7991 TLI.getSetCCResultType(Op0.getValueType()), 7992 Op0, Op1, Op.getOperand(2)); 7993 Result = DAG.getNode(ISD::SELECT, dl, NewVT, Result, 7994 DAG.getConstant(-1ULL, NewVT), 7995 DAG.getConstant(0ULL, NewVT)); 7996 break; 7997 } 7998 } 7999 8000 if (TLI.isTypeLegal(NewVT)) 8001 Result = LegalizeOp(Result); 8002 bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second; 8003 assert(isNew && "Value already scalarized?"); 8004 isNew = isNew; 8005 return Result; 8006} 8007 8008 8009SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) { 8010 std::map<SDValue, SDValue>::iterator I = WidenNodes.find(Op); 8011 if (I != WidenNodes.end()) return I->second; 8012 8013 MVT VT = Op.getValueType(); 8014 assert(VT.isVector() && "Cannot widen non-vector type!"); 8015 8016 SDValue Result; 8017 SDNode *Node = Op.getNode(); 8018 DebugLoc dl = Node->getDebugLoc(); 8019 MVT EVT = VT.getVectorElementType(); 8020 8021 unsigned NumElts = VT.getVectorNumElements(); 8022 unsigned NewNumElts = WidenVT.getVectorNumElements(); 8023 assert(NewNumElts > NumElts && "Cannot widen to smaller type!"); 8024 assert(NewNumElts < 17); 8025 8026 // When widen is called, it is assumed that it is more efficient to use a 8027 // wide type. The default action is to widen to operation to a wider legal 8028 // vector type and then do the operation if it is legal by calling LegalizeOp 8029 // again. If there is no vector equivalent, we will unroll the operation, do 8030 // it, and rebuild the vector. If most of the operations are vectorizible to 8031 // the legal type, the resulting code will be more efficient. If this is not 8032 // the case, the resulting code will preform badly as we end up generating 8033 // code to pack/unpack the results. It is the function that calls widen 8034 // that is responsible for seeing this doesn't happen. 8035 switch (Node->getOpcode()) { 8036 default: 8037#ifndef NDEBUG 8038 Node->dump(&DAG); 8039#endif 8040 assert(0 && "Unexpected operation in WidenVectorOp!"); 8041 break; 8042 case ISD::CopyFromReg: 8043 assert(0 && "CopyFromReg doesn't need widening!"); 8044 case ISD::Constant: 8045 case ISD::ConstantFP: 8046 // To build a vector of these elements, clients should call BuildVector 8047 // and with each element instead of creating a node with a vector type 8048 assert(0 && "Unexpected operation in WidenVectorOp!"); 8049 case ISD::VAARG: 8050 // Variable Arguments with vector types doesn't make any sense to me 8051 assert(0 && "Unexpected operation in WidenVectorOp!"); 8052 break; 8053 case ISD::UNDEF: 8054 Result = DAG.getNode(ISD::UNDEF, dl, WidenVT); 8055 break; 8056 case ISD::BUILD_VECTOR: { 8057 // Build a vector with undefined for the new nodes 8058 SDValueVector NewOps(Node->op_begin(), Node->op_end()); 8059 for (unsigned i = NumElts; i < NewNumElts; ++i) { 8060 NewOps.push_back(DAG.getNode(ISD::UNDEF, dl, EVT)); 8061 } 8062 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, 8063 &NewOps[0], NewOps.size()); 8064 break; 8065 } 8066 case ISD::INSERT_VECTOR_ELT: { 8067 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8068 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WidenVT, Tmp1, 8069 Node->getOperand(1), Node->getOperand(2)); 8070 break; 8071 } 8072 case ISD::VECTOR_SHUFFLE: { 8073 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8074 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); 8075 // VECTOR_SHUFFLE 3rd operand must be a constant build vector that is 8076 // used as permutation array. We build the vector here instead of widening 8077 // because we don't want to legalize and have it turned to something else. 8078 SDValue PermOp = Node->getOperand(2); 8079 SDValueVector NewOps; 8080 MVT PVT = PermOp.getValueType().getVectorElementType(); 8081 for (unsigned i = 0; i < NumElts; ++i) { 8082 if (PermOp.getOperand(i).getOpcode() == ISD::UNDEF) { 8083 NewOps.push_back(PermOp.getOperand(i)); 8084 } else { 8085 unsigned Idx = 8086 cast<ConstantSDNode>(PermOp.getOperand(i))->getZExtValue(); 8087 if (Idx < NumElts) { 8088 NewOps.push_back(PermOp.getOperand(i)); 8089 } 8090 else { 8091 NewOps.push_back(DAG.getConstant(Idx + NewNumElts - NumElts, 8092 PermOp.getOperand(i).getValueType())); 8093 } 8094 } 8095 } 8096 for (unsigned i = NumElts; i < NewNumElts; ++i) { 8097 NewOps.push_back(DAG.getNode(ISD::UNDEF, dl, PVT)); 8098 } 8099 8100 SDValue Tmp3 = DAG.getNode(ISD::BUILD_VECTOR, dl, 8101 MVT::getVectorVT(PVT, NewOps.size()), 8102 &NewOps[0], NewOps.size()); 8103 8104 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, WidenVT, Tmp1, Tmp2, Tmp3); 8105 break; 8106 } 8107 case ISD::LOAD: { 8108 // If the load widen returns true, we can use a single load for the 8109 // vector. Otherwise, it is returning a token factor for multiple 8110 // loads. 8111 SDValue TFOp; 8112 if (LoadWidenVectorOp(Result, TFOp, Op, WidenVT)) 8113 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(1))); 8114 else 8115 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(0))); 8116 break; 8117 } 8118 8119 case ISD::BIT_CONVERT: { 8120 SDValue Tmp1 = Node->getOperand(0); 8121 // Converts between two different types so we need to determine 8122 // the correct widen type for the input operand. 8123 MVT InVT = Tmp1.getValueType(); 8124 unsigned WidenSize = WidenVT.getSizeInBits(); 8125 if (InVT.isVector()) { 8126 MVT InEltVT = InVT.getVectorElementType(); 8127 unsigned InEltSize = InEltVT.getSizeInBits(); 8128 assert(WidenSize % InEltSize == 0 && 8129 "can not widen bit convert that are not multiple of element type"); 8130 MVT NewInWidenVT = MVT::getVectorVT(InEltVT, WidenSize / InEltSize); 8131 Tmp1 = WidenVectorOp(Tmp1, NewInWidenVT); 8132 assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits()); 8133 Result = DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, Tmp1); 8134 } else { 8135 // If the result size is a multiple of the input size, widen the input 8136 // and then convert. 8137 unsigned InSize = InVT.getSizeInBits(); 8138 assert(WidenSize % InSize == 0 && 8139 "can not widen bit convert that are not multiple of element type"); 8140 unsigned NewNumElts = WidenSize / InSize; 8141 SmallVector<SDValue, 16> Ops(NewNumElts); 8142 SDValue UndefVal = DAG.getNode(ISD::UNDEF, dl, InVT); 8143 Ops[0] = Tmp1; 8144 for (unsigned i = 1; i < NewNumElts; ++i) 8145 Ops[i] = UndefVal; 8146 8147 MVT NewInVT = MVT::getVectorVT(InVT, NewNumElts); 8148 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, &Ops[0], NewNumElts); 8149 Result = DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, Result); 8150 } 8151 break; 8152 } 8153 8154 case ISD::SINT_TO_FP: 8155 case ISD::UINT_TO_FP: 8156 case ISD::FP_TO_SINT: 8157 case ISD::FP_TO_UINT: 8158 case ISD::FP_ROUND: { 8159 SDValue Tmp1 = Node->getOperand(0); 8160 // Converts between two different types so we need to determine 8161 // the correct widen type for the input operand. 8162 MVT TVT = Tmp1.getValueType(); 8163 assert(TVT.isVector() && "can not widen non vector type"); 8164 MVT TEVT = TVT.getVectorElementType(); 8165 MVT TWidenVT = MVT::getVectorVT(TEVT, NewNumElts); 8166 Tmp1 = WidenVectorOp(Tmp1, TWidenVT); 8167 assert(Tmp1.getValueType().getVectorNumElements() == NewNumElts); 8168 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1); 8169 break; 8170 } 8171 8172 case ISD::FP_EXTEND: 8173 assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); 8174 case ISD::TRUNCATE: 8175 case ISD::SIGN_EXTEND: 8176 case ISD::ZERO_EXTEND: 8177 case ISD::ANY_EXTEND: 8178 case ISD::SIGN_EXTEND_INREG: 8179 case ISD::FABS: 8180 case ISD::FNEG: 8181 case ISD::FSQRT: 8182 case ISD::FSIN: 8183 case ISD::FCOS: 8184 case ISD::CTPOP: 8185 case ISD::CTTZ: 8186 case ISD::CTLZ: { 8187 // Unary op widening 8188 SDValue Tmp1; 8189 Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8190 assert(Tmp1.getValueType() == WidenVT); 8191 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1); 8192 break; 8193 } 8194 case ISD::CONVERT_RNDSAT: { 8195 SDValue RndOp = Node->getOperand(3); 8196 SDValue SatOp = Node->getOperand(4); 8197 SDValue SrcOp = Node->getOperand(0); 8198 8199 // Converts between two different types so we need to determine 8200 // the correct widen type for the input operand. 8201 MVT SVT = SrcOp.getValueType(); 8202 assert(SVT.isVector() && "can not widen non vector type"); 8203 MVT SEVT = SVT.getVectorElementType(); 8204 MVT SWidenVT = MVT::getVectorVT(SEVT, NewNumElts); 8205 8206 SrcOp = WidenVectorOp(SrcOp, SWidenVT); 8207 assert(SrcOp.getValueType() == WidenVT); 8208 SDValue DTyOp = DAG.getValueType(WidenVT); 8209 SDValue STyOp = DAG.getValueType(SrcOp.getValueType()); 8210 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 8211 8212 Result = DAG.getConvertRndSat(WidenVT, dl, SrcOp, DTyOp, STyOp, 8213 RndOp, SatOp, CvtCode); 8214 break; 8215 } 8216 case ISD::FPOW: 8217 case ISD::FPOWI: 8218 case ISD::ADD: 8219 case ISD::SUB: 8220 case ISD::MUL: 8221 case ISD::MULHS: 8222 case ISD::MULHU: 8223 case ISD::AND: 8224 case ISD::OR: 8225 case ISD::XOR: 8226 case ISD::FADD: 8227 case ISD::FSUB: 8228 case ISD::FMUL: 8229 case ISD::SDIV: 8230 case ISD::SREM: 8231 case ISD::FDIV: 8232 case ISD::FREM: 8233 case ISD::FCOPYSIGN: 8234 case ISD::UDIV: 8235 case ISD::UREM: 8236 case ISD::BSWAP: { 8237 // Binary op widening 8238 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8239 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); 8240 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); 8241 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, Tmp2); 8242 break; 8243 } 8244 8245 case ISD::SHL: 8246 case ISD::SRA: 8247 case ISD::SRL: { 8248 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8249 assert(Tmp1.getValueType() == WidenVT); 8250 SDValue ShOp = Node->getOperand(1); 8251 MVT ShVT = ShOp.getValueType(); 8252 MVT NewShVT = MVT::getVectorVT(ShVT.getVectorElementType(), 8253 WidenVT.getVectorNumElements()); 8254 ShOp = WidenVectorOp(ShOp, NewShVT); 8255 assert(ShOp.getValueType() == NewShVT); 8256 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, ShOp); 8257 break; 8258 } 8259 8260 case ISD::EXTRACT_VECTOR_ELT: { 8261 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8262 assert(Tmp1.getValueType() == WidenVT); 8263 Result = DAG.getNode(Node->getOpcode(), dl, EVT, Tmp1, Node->getOperand(1)); 8264 break; 8265 } 8266 case ISD::CONCAT_VECTORS: { 8267 // We concurrently support only widen on a multiple of the incoming vector. 8268 // We could widen on a multiple of the incoming operand if necessary. 8269 unsigned NumConcat = NewNumElts / NumElts; 8270 assert(NewNumElts % NumElts == 0 && "Can widen only a multiple of vector"); 8271 SDValue UndefVal = DAG.getNode(ISD::UNDEF, dl, VT); 8272 SmallVector<SDValue, 8> MOps; 8273 MOps.push_back(Op); 8274 for (unsigned i = 1; i != NumConcat; ++i) { 8275 MOps.push_back(UndefVal); 8276 } 8277 Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 8278 &MOps[0], MOps.size())); 8279 break; 8280 } 8281 case ISD::EXTRACT_SUBVECTOR: { 8282 SDValue Tmp1 = Node->getOperand(0); 8283 SDValue Idx = Node->getOperand(1); 8284 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 8285 if (CIdx && CIdx->getZExtValue() == 0) { 8286 // Since we are access the start of the vector, the incoming 8287 // vector type might be the proper. 8288 MVT Tmp1VT = Tmp1.getValueType(); 8289 if (Tmp1VT == WidenVT) 8290 return Tmp1; 8291 else { 8292 unsigned Tmp1VTNumElts = Tmp1VT.getVectorNumElements(); 8293 if (Tmp1VTNumElts < NewNumElts) 8294 Result = WidenVectorOp(Tmp1, WidenVT); 8295 else 8296 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, Tmp1, Idx); 8297 } 8298 } else if (NewNumElts % NumElts == 0) { 8299 // Widen the extracted subvector. 8300 unsigned NumConcat = NewNumElts / NumElts; 8301 SDValue UndefVal = DAG.getNode(ISD::UNDEF, dl, VT); 8302 SmallVector<SDValue, 8> MOps; 8303 MOps.push_back(Op); 8304 for (unsigned i = 1; i != NumConcat; ++i) { 8305 MOps.push_back(UndefVal); 8306 } 8307 Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 8308 &MOps[0], MOps.size())); 8309 } else { 8310 assert(0 && "can not widen extract subvector"); 8311 // This could be implemented using insert and build vector but I would 8312 // like to see when this happens. 8313 } 8314 break; 8315 } 8316 8317 case ISD::SELECT: { 8318 // Determine new condition widen type and widen 8319 SDValue Cond1 = Node->getOperand(0); 8320 MVT CondVT = Cond1.getValueType(); 8321 assert(CondVT.isVector() && "can not widen non vector type"); 8322 MVT CondEVT = CondVT.getVectorElementType(); 8323 MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); 8324 Cond1 = WidenVectorOp(Cond1, CondWidenVT); 8325 assert(Cond1.getValueType() == CondWidenVT && "Condition not widen"); 8326 8327 SDValue Tmp1 = WidenVectorOp(Node->getOperand(1), WidenVT); 8328 SDValue Tmp2 = WidenVectorOp(Node->getOperand(2), WidenVT); 8329 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); 8330 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Cond1, Tmp1, Tmp2); 8331 break; 8332 } 8333 8334 case ISD::SELECT_CC: { 8335 // Determine new condition widen type and widen 8336 SDValue Cond1 = Node->getOperand(0); 8337 SDValue Cond2 = Node->getOperand(1); 8338 MVT CondVT = Cond1.getValueType(); 8339 assert(CondVT.isVector() && "can not widen non vector type"); 8340 assert(CondVT == Cond2.getValueType() && "mismatch lhs/rhs"); 8341 MVT CondEVT = CondVT.getVectorElementType(); 8342 MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); 8343 Cond1 = WidenVectorOp(Cond1, CondWidenVT); 8344 Cond2 = WidenVectorOp(Cond2, CondWidenVT); 8345 assert(Cond1.getValueType() == CondWidenVT && 8346 Cond2.getValueType() == CondWidenVT && "condition not widen"); 8347 8348 SDValue Tmp1 = WidenVectorOp(Node->getOperand(2), WidenVT); 8349 SDValue Tmp2 = WidenVectorOp(Node->getOperand(3), WidenVT); 8350 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT && 8351 "operands not widen"); 8352 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Cond1, Cond2, Tmp1, 8353 Tmp2, Node->getOperand(4)); 8354 break; 8355 } 8356 case ISD::VSETCC: { 8357 // Determine widen for the operand 8358 SDValue Tmp1 = Node->getOperand(0); 8359 MVT TmpVT = Tmp1.getValueType(); 8360 assert(TmpVT.isVector() && "can not widen non vector type"); 8361 MVT TmpEVT = TmpVT.getVectorElementType(); 8362 MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts); 8363 Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT); 8364 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT); 8365 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, Tmp2, 8366 Node->getOperand(2)); 8367 break; 8368 } 8369 case ISD::ATOMIC_CMP_SWAP: 8370 case ISD::ATOMIC_LOAD_ADD: 8371 case ISD::ATOMIC_LOAD_SUB: 8372 case ISD::ATOMIC_LOAD_AND: 8373 case ISD::ATOMIC_LOAD_OR: 8374 case ISD::ATOMIC_LOAD_XOR: 8375 case ISD::ATOMIC_LOAD_NAND: 8376 case ISD::ATOMIC_LOAD_MIN: 8377 case ISD::ATOMIC_LOAD_MAX: 8378 case ISD::ATOMIC_LOAD_UMIN: 8379 case ISD::ATOMIC_LOAD_UMAX: 8380 case ISD::ATOMIC_SWAP: { 8381 // For now, we assume that using vectors for these operations don't make 8382 // much sense so we just split it. We return an empty result 8383 SDValue X, Y; 8384 SplitVectorOp(Op, X, Y); 8385 return Result; 8386 break; 8387 } 8388 8389 } // end switch (Node->getOpcode()) 8390 8391 assert(Result.getNode() && "Didn't set a result!"); 8392 if (Result != Op) 8393 Result = LegalizeOp(Result); 8394 8395 AddWidenedOperand(Op, Result); 8396 return Result; 8397} 8398 8399// Utility function to find a legal vector type and its associated element 8400// type from a preferred width and whose vector type must be the same size 8401// as the VVT. 8402// TLI: Target lowering used to determine legal types 8403// Width: Preferred width of element type 8404// VVT: Vector value type whose size we must match. 8405// Returns VecEVT and EVT - the vector type and its associated element type 8406static void FindWidenVecType(const TargetLowering &TLI, unsigned Width, MVT VVT, 8407 MVT& EVT, MVT& VecEVT) { 8408 // We start with the preferred width, make it a power of 2 and see if 8409 // we can find a vector type of that width. If not, we reduce it by 8410 // another power of 2. If we have widen the type, a vector of bytes should 8411 // always be legal. 8412 assert(TLI.isTypeLegal(VVT)); 8413 unsigned EWidth = Width + 1; 8414 do { 8415 assert(EWidth > 0); 8416 EWidth = (1 << Log2_32(EWidth-1)); 8417 EVT = MVT::getIntegerVT(EWidth); 8418 unsigned NumEVT = VVT.getSizeInBits()/EWidth; 8419 VecEVT = MVT::getVectorVT(EVT, NumEVT); 8420 } while (!TLI.isTypeLegal(VecEVT) || 8421 VVT.getSizeInBits() != VecEVT.getSizeInBits()); 8422} 8423 8424SDValue SelectionDAGLegalize::genWidenVectorLoads(SDValueVector& LdChain, 8425 SDValue Chain, 8426 SDValue BasePtr, 8427 const Value *SV, 8428 int SVOffset, 8429 unsigned Alignment, 8430 bool isVolatile, 8431 unsigned LdWidth, 8432 MVT ResType, 8433 DebugLoc dl) { 8434 // We assume that we have good rules to handle loading power of two loads so 8435 // we break down the operations to power of 2 loads. The strategy is to 8436 // load the largest power of 2 that we can easily transform to a legal vector 8437 // and then insert into that vector, and the cast the result into the legal 8438 // vector that we want. This avoids unnecessary stack converts. 8439 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 8440 // the load is nonvolatile, we an use a wider load for the value. 8441 // Find a vector length we can load a large chunk 8442 MVT EVT, VecEVT; 8443 unsigned EVTWidth; 8444 FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); 8445 EVTWidth = EVT.getSizeInBits(); 8446 8447 SDValue LdOp = DAG.getLoad(EVT, dl, Chain, BasePtr, SV, SVOffset, 8448 isVolatile, Alignment); 8449 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecEVT, LdOp); 8450 LdChain.push_back(LdOp.getValue(1)); 8451 8452 // Check if we can load the element with one instruction 8453 if (LdWidth == EVTWidth) { 8454 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 8455 } 8456 8457 // The vector element order is endianness dependent. 8458 unsigned Idx = 1; 8459 LdWidth -= EVTWidth; 8460 unsigned Offset = 0; 8461 8462 while (LdWidth > 0) { 8463 unsigned Increment = EVTWidth / 8; 8464 Offset += Increment; 8465 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 8466 DAG.getIntPtrConstant(Increment)); 8467 8468 if (LdWidth < EVTWidth) { 8469 // Our current type we are using is too large, use a smaller size by 8470 // using a smaller power of 2 8471 unsigned oEVTWidth = EVTWidth; 8472 FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); 8473 EVTWidth = EVT.getSizeInBits(); 8474 // Readjust position and vector position based on new load type 8475 Idx = Idx * (oEVTWidth/EVTWidth); 8476 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, VecOp); 8477 } 8478 8479 SDValue LdOp = DAG.getLoad(EVT, dl, Chain, BasePtr, SV, 8480 SVOffset+Offset, isVolatile, 8481 MinAlign(Alignment, Offset)); 8482 LdChain.push_back(LdOp.getValue(1)); 8483 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecEVT, VecOp, LdOp, 8484 DAG.getIntPtrConstant(Idx++)); 8485 8486 LdWidth -= EVTWidth; 8487 } 8488 8489 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 8490} 8491 8492bool SelectionDAGLegalize::LoadWidenVectorOp(SDValue& Result, 8493 SDValue& TFOp, 8494 SDValue Op, 8495 MVT NVT) { 8496 // TODO: Add support for ConcatVec and the ability to load many vector 8497 // types (e.g., v4i8). This will not work when a vector register 8498 // to memory mapping is strange (e.g., vector elements are not 8499 // stored in some sequential order). 8500 8501 // It must be true that the widen vector type is bigger than where 8502 // we need to load from. 8503 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 8504 MVT LdVT = LD->getMemoryVT(); 8505 DebugLoc dl = LD->getDebugLoc(); 8506 assert(LdVT.isVector() && NVT.isVector()); 8507 assert(LdVT.getVectorElementType() == NVT.getVectorElementType()); 8508 8509 // Load information 8510 SDValue Chain = LD->getChain(); 8511 SDValue BasePtr = LD->getBasePtr(); 8512 int SVOffset = LD->getSrcValueOffset(); 8513 unsigned Alignment = LD->getAlignment(); 8514 bool isVolatile = LD->isVolatile(); 8515 const Value *SV = LD->getSrcValue(); 8516 unsigned int LdWidth = LdVT.getSizeInBits(); 8517 8518 // Load value as a large register 8519 SDValueVector LdChain; 8520 Result = genWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 8521 Alignment, isVolatile, LdWidth, NVT, dl); 8522 8523 if (LdChain.size() == 1) { 8524 TFOp = LdChain[0]; 8525 return true; 8526 } 8527 else { 8528 TFOp=DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 8529 &LdChain[0], LdChain.size()); 8530 return false; 8531 } 8532} 8533 8534 8535void SelectionDAGLegalize::genWidenVectorStores(SDValueVector& StChain, 8536 SDValue Chain, 8537 SDValue BasePtr, 8538 const Value *SV, 8539 int SVOffset, 8540 unsigned Alignment, 8541 bool isVolatile, 8542 SDValue ValOp, 8543 unsigned StWidth, 8544 DebugLoc dl) { 8545 // Breaks the stores into a series of power of 2 width stores. For any 8546 // width, we convert the vector to the vector of element size that we 8547 // want to store. This avoids requiring a stack convert. 8548 8549 // Find a width of the element type we can store with 8550 MVT VVT = ValOp.getValueType(); 8551 MVT EVT, VecEVT; 8552 unsigned EVTWidth; 8553 FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); 8554 EVTWidth = EVT.getSizeInBits(); 8555 8556 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, ValOp); 8557 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EVT, VecOp, 8558 DAG.getIntPtrConstant(0)); 8559 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 8560 isVolatile, Alignment); 8561 StChain.push_back(StOp); 8562 8563 // Check if we are done 8564 if (StWidth == EVTWidth) { 8565 return; 8566 } 8567 8568 unsigned Idx = 1; 8569 StWidth -= EVTWidth; 8570 unsigned Offset = 0; 8571 8572 while (StWidth > 0) { 8573 unsigned Increment = EVTWidth / 8; 8574 Offset += Increment; 8575 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 8576 DAG.getIntPtrConstant(Increment)); 8577 8578 if (StWidth < EVTWidth) { 8579 // Our current type we are using is too large, use a smaller size by 8580 // using a smaller power of 2 8581 unsigned oEVTWidth = EVTWidth; 8582 FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); 8583 EVTWidth = EVT.getSizeInBits(); 8584 // Readjust position and vector position based on new load type 8585 Idx = Idx * (oEVTWidth/EVTWidth); 8586 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, VecOp); 8587 } 8588 8589 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EVT, VecOp, 8590 DAG.getIntPtrConstant(Idx++)); 8591 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 8592 SVOffset + Offset, isVolatile, 8593 MinAlign(Alignment, Offset))); 8594 StWidth -= EVTWidth; 8595 } 8596} 8597 8598 8599SDValue SelectionDAGLegalize::StoreWidenVectorOp(StoreSDNode *ST, 8600 SDValue Chain, 8601 SDValue BasePtr) { 8602 // TODO: It might be cleaner if we can use SplitVector and have more legal 8603 // vector types that can be stored into memory (e.g., v4xi8 can 8604 // be stored as a word). This will not work when a vector register 8605 // to memory mapping is strange (e.g., vector elements are not 8606 // stored in some sequential order). 8607 8608 MVT StVT = ST->getMemoryVT(); 8609 SDValue ValOp = ST->getValue(); 8610 DebugLoc dl = ST->getDebugLoc(); 8611 8612 // Check if we have widen this node with another value 8613 std::map<SDValue, SDValue>::iterator I = WidenNodes.find(ValOp); 8614 if (I != WidenNodes.end()) 8615 ValOp = I->second; 8616 8617 MVT VVT = ValOp.getValueType(); 8618 8619 // It must be true that we the widen vector type is bigger than where 8620 // we need to store. 8621 assert(StVT.isVector() && VVT.isVector()); 8622 assert(StVT.bitsLT(VVT)); 8623 assert(StVT.getVectorElementType() == VVT.getVectorElementType()); 8624 8625 // Store value 8626 SDValueVector StChain; 8627 genWidenVectorStores(StChain, Chain, BasePtr, ST->getSrcValue(), 8628 ST->getSrcValueOffset(), ST->getAlignment(), 8629 ST->isVolatile(), ValOp, StVT.getSizeInBits(), dl); 8630 if (StChain.size() == 1) 8631 return StChain[0]; 8632 else 8633 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 8634 &StChain[0], StChain.size()); 8635} 8636 8637 8638// SelectionDAG::Legalize - This is the entry point for the file. 8639// 8640void SelectionDAG::Legalize(bool TypesNeedLegalizing) { 8641 /// run - This is the main entry point to this class. 8642 /// 8643 SelectionDAGLegalize(*this, TypesNeedLegalizing).LegalizeDAG(); 8644} 8645 8646