LegalizeDAG.cpp revision ec8eee2d3a402dd001daae3865bc5c10b41c323e
1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SelectionDAG::Legalize method. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "llvm/CodeGen/MachineFunction.h" 16#include "llvm/CodeGen/MachineFrameInfo.h" 17#include "llvm/CodeGen/MachineJumpTableInfo.h" 18#include "llvm/CodeGen/MachineModuleInfo.h" 19#include "llvm/CodeGen/DwarfWriter.h" 20#include "llvm/Analysis/DebugInfo.h" 21#include "llvm/CodeGen/PseudoSourceValue.h" 22#include "llvm/Target/TargetFrameInfo.h" 23#include "llvm/Target/TargetLowering.h" 24#include "llvm/Target/TargetData.h" 25#include "llvm/Target/TargetMachine.h" 26#include "llvm/Target/TargetOptions.h" 27#include "llvm/Target/TargetSubtarget.h" 28#include "llvm/CallingConv.h" 29#include "llvm/Constants.h" 30#include "llvm/DerivedTypes.h" 31#include "llvm/Function.h" 32#include "llvm/GlobalVariable.h" 33#include "llvm/Support/CommandLine.h" 34#include "llvm/Support/Compiler.h" 35#include "llvm/Support/MathExtras.h" 36#include "llvm/ADT/DenseMap.h" 37#include "llvm/ADT/SmallVector.h" 38#include "llvm/ADT/SmallPtrSet.h" 39#include <map> 40using namespace llvm; 41 42//===----------------------------------------------------------------------===// 43/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and 44/// hacks on it until the target machine can handle it. This involves 45/// eliminating value sizes the machine cannot handle (promoting small sizes to 46/// large sizes or splitting up large values into small values) as well as 47/// eliminating operations the machine cannot handle. 48/// 49/// This code also does a small amount of optimization and recognition of idioms 50/// as part of its processing. For example, if a target does not support a 51/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 52/// will attempt merge setcc and brc instructions into brcc's. 53/// 54namespace { 55class VISIBILITY_HIDDEN SelectionDAGLegalize { 56 TargetLowering &TLI; 57 SelectionDAG &DAG; 58 unsigned OptLevel; 59 bool TypesNeedLegalizing; 60 61 // Libcall insertion helpers. 62 63 /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been 64 /// legalized. We use this to ensure that calls are properly serialized 65 /// against each other, including inserted libcalls. 66 SDValue LastCALLSEQ_END; 67 68 /// IsLegalizingCall - This member is used *only* for purposes of providing 69 /// helpful assertions that a libcall isn't created while another call is 70 /// being legalized (which could lead to non-serialized call sequences). 71 bool IsLegalizingCall; 72 73 /// IsLegalizingCallArguments - This member is used only for the purpose 74 /// of providing assert to check for LegalizeTypes because legalizing an 75 /// operation might introduce call nodes that might need type legalization. 76 bool IsLegalizingCallArgs; 77 78 enum LegalizeAction { 79 Legal, // The target natively supports this operation. 80 Promote, // This operation should be executed in a larger type. 81 Expand // Try to expand this to other ops, otherwise use a libcall. 82 }; 83 84 /// ValueTypeActions - This is a bitvector that contains two bits for each 85 /// value type, where the two bits correspond to the LegalizeAction enum. 86 /// This can be queried with "getTypeAction(VT)". 87 TargetLowering::ValueTypeActionImpl ValueTypeActions; 88 89 /// LegalizedNodes - For nodes that are of legal width, and that have more 90 /// than one use, this map indicates what regularized operand to use. This 91 /// allows us to avoid legalizing the same thing more than once. 92 DenseMap<SDValue, SDValue> LegalizedNodes; 93 94 /// PromotedNodes - For nodes that are below legal width, and that have more 95 /// than one use, this map indicates what promoted value to use. This allows 96 /// us to avoid promoting the same thing more than once. 97 DenseMap<SDValue, SDValue> PromotedNodes; 98 99 /// ExpandedNodes - For nodes that need to be expanded this map indicates 100 /// which operands are the expanded version of the input. This allows 101 /// us to avoid expanding the same node more than once. 102 DenseMap<SDValue, std::pair<SDValue, SDValue> > ExpandedNodes; 103 104 /// SplitNodes - For vector nodes that need to be split, this map indicates 105 /// which operands are the split version of the input. This allows us 106 /// to avoid splitting the same node more than once. 107 std::map<SDValue, std::pair<SDValue, SDValue> > SplitNodes; 108 109 /// ScalarizedNodes - For nodes that need to be converted from vector types to 110 /// scalar types, this contains the mapping of ones we have already 111 /// processed to the result. 112 std::map<SDValue, SDValue> ScalarizedNodes; 113 114 /// WidenNodes - For nodes that need to be widened from one vector type to 115 /// another, this contains the mapping of those that we have already widen. 116 /// This allows us to avoid widening more than once. 117 std::map<SDValue, SDValue> WidenNodes; 118 119 void AddLegalizedOperand(SDValue From, SDValue To) { 120 LegalizedNodes.insert(std::make_pair(From, To)); 121 // If someone requests legalization of the new node, return itself. 122 if (From != To) 123 LegalizedNodes.insert(std::make_pair(To, To)); 124 } 125 void AddPromotedOperand(SDValue From, SDValue To) { 126 bool isNew = PromotedNodes.insert(std::make_pair(From, To)).second; 127 assert(isNew && "Got into the map somehow?"); 128 isNew = isNew; 129 // If someone requests legalization of the new node, return itself. 130 LegalizedNodes.insert(std::make_pair(To, To)); 131 } 132 void AddWidenedOperand(SDValue From, SDValue To) { 133 bool isNew = WidenNodes.insert(std::make_pair(From, To)).second; 134 assert(isNew && "Got into the map somehow?"); 135 isNew = isNew; 136 // If someone requests legalization of the new node, return itself. 137 LegalizedNodes.insert(std::make_pair(To, To)); 138 } 139 140public: 141 explicit SelectionDAGLegalize(SelectionDAG &DAG, bool TypesNeedLegalizing, 142 unsigned ol); 143 144 /// getTypeAction - Return how we should legalize values of this type, either 145 /// it is already legal or we need to expand it into multiple registers of 146 /// smaller integer type, or we need to promote it to a larger type. 147 LegalizeAction getTypeAction(MVT VT) const { 148 return (LegalizeAction)ValueTypeActions.getTypeAction(VT); 149 } 150 151 /// isTypeLegal - Return true if this type is legal on this target. 152 /// 153 bool isTypeLegal(MVT VT) const { 154 return getTypeAction(VT) == Legal; 155 } 156 157 void LegalizeDAG(); 158 159private: 160 /// HandleOp - Legalize, Promote, or Expand the specified operand as 161 /// appropriate for its type. 162 void HandleOp(SDValue Op); 163 164 /// LegalizeOp - We know that the specified value has a legal type. 165 /// Recursively ensure that the operands have legal types, then return the 166 /// result. 167 SDValue LegalizeOp(SDValue O); 168 169 /// UnrollVectorOp - We know that the given vector has a legal type, however 170 /// the operation it performs is not legal and is an operation that we have 171 /// no way of lowering. "Unroll" the vector, splitting out the scalars and 172 /// operating on each element individually. 173 SDValue UnrollVectorOp(SDValue O); 174 175 /// PerformInsertVectorEltInMemory - Some target cannot handle a variable 176 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 177 /// is necessary to spill the vector being inserted into to memory, perform 178 /// the insert there, and then read the result back. 179 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, 180 SDValue Idx, DebugLoc dl); 181 182 /// PromoteOp - Given an operation that produces a value in an invalid type, 183 /// promote it to compute the value into a larger type. The produced value 184 /// will have the correct bits for the low portion of the register, but no 185 /// guarantee is made about the top bits: it may be zero, sign-extended, or 186 /// garbage. 187 SDValue PromoteOp(SDValue O); 188 189 /// ExpandOp - Expand the specified SDValue into its two component pieces 190 /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, 191 /// the LegalizedNodes map is filled in for any results that are not expanded, 192 /// the ExpandedNodes map is filled in for any results that are expanded, and 193 /// the Lo/Hi values are returned. This applies to integer types and Vector 194 /// types. 195 void ExpandOp(SDValue O, SDValue &Lo, SDValue &Hi); 196 197 /// WidenVectorOp - Widen a vector operation to a wider type given by WidenVT 198 /// (e.g., v3i32 to v4i32). The produced value will have the correct value 199 /// for the existing elements but no guarantee is made about the new elements 200 /// at the end of the vector: it may be zero, ones, or garbage. This is useful 201 /// when we have an instruction operating on an illegal vector type and we 202 /// want to widen it to do the computation on a legal wider vector type. 203 SDValue WidenVectorOp(SDValue Op, MVT WidenVT); 204 205 /// SplitVectorOp - Given an operand of vector type, break it down into 206 /// two smaller values. 207 void SplitVectorOp(SDValue O, SDValue &Lo, SDValue &Hi); 208 209 /// ScalarizeVectorOp - Given an operand of single-element vector type 210 /// (e.g. v1f32), convert it into the equivalent operation that returns a 211 /// scalar (e.g. f32) value. 212 SDValue ScalarizeVectorOp(SDValue O); 213 214 /// Useful 16 element vector type that is used to pass operands for widening. 215 typedef SmallVector<SDValue, 16> SDValueVector; 216 217 /// LoadWidenVectorOp - Load a vector for a wider type. Returns true if 218 /// the LdChain contains a single load and false if it contains a token 219 /// factor for multiple loads. It takes 220 /// Result: location to return the result 221 /// LdChain: location to return the load chain 222 /// Op: load operation to widen 223 /// NVT: widen vector result type we want for the load 224 bool LoadWidenVectorOp(SDValue& Result, SDValue& LdChain, 225 SDValue Op, MVT NVT); 226 227 /// Helper genWidenVectorLoads - Helper function to generate a set of 228 /// loads to load a vector with a resulting wider type. It takes 229 /// LdChain: list of chains for the load we have generated 230 /// Chain: incoming chain for the ld vector 231 /// BasePtr: base pointer to load from 232 /// SV: memory disambiguation source value 233 /// SVOffset: memory disambiugation offset 234 /// Alignment: alignment of the memory 235 /// isVolatile: volatile load 236 /// LdWidth: width of memory that we want to load 237 /// ResType: the wider result result type for the resulting loaded vector 238 SDValue genWidenVectorLoads(SDValueVector& LdChain, SDValue Chain, 239 SDValue BasePtr, const Value *SV, 240 int SVOffset, unsigned Alignment, 241 bool isVolatile, unsigned LdWidth, 242 MVT ResType, DebugLoc dl); 243 244 /// StoreWidenVectorOp - Stores a widen vector into non widen memory 245 /// location. It takes 246 /// ST: store node that we want to replace 247 /// Chain: incoming store chain 248 /// BasePtr: base address of where we want to store into 249 SDValue StoreWidenVectorOp(StoreSDNode *ST, SDValue Chain, 250 SDValue BasePtr); 251 252 /// Helper genWidenVectorStores - Helper function to generate a set of 253 /// stores to store a widen vector into non widen memory 254 // It takes 255 // StChain: list of chains for the stores we have generated 256 // Chain: incoming chain for the ld vector 257 // BasePtr: base pointer to load from 258 // SV: memory disambiguation source value 259 // SVOffset: memory disambiugation offset 260 // Alignment: alignment of the memory 261 // isVolatile: volatile lod 262 // ValOp: value to store 263 // StWidth: width of memory that we want to store 264 void genWidenVectorStores(SDValueVector& StChain, SDValue Chain, 265 SDValue BasePtr, const Value *SV, 266 int SVOffset, unsigned Alignment, 267 bool isVolatile, SDValue ValOp, 268 unsigned StWidth, DebugLoc dl); 269 270 /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 271 /// performs the same shuffe in terms of order or result bytes, but on a type 272 /// whose vector element type is narrower than the original shuffle type. 273 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 274 SDValue ShuffleWithNarrowerEltType(MVT NVT, MVT VT, DebugLoc dl, 275 SDValue N1, SDValue N2, 276 SmallVectorImpl<int> &Mask) const; 277 278 bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, 279 SmallPtrSet<SDNode*, 32> &NodesLeadingTo); 280 281 void LegalizeSetCCOperands(SDValue &LHS, SDValue &RHS, SDValue &CC, 282 DebugLoc dl); 283 void LegalizeSetCCCondCode(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 284 DebugLoc dl); 285 void LegalizeSetCC(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 286 DebugLoc dl) { 287 LegalizeSetCCOperands(LHS, RHS, CC, dl); 288 LegalizeSetCCCondCode(VT, LHS, RHS, CC, dl); 289 } 290 291 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned, 292 SDValue &Hi); 293 SDValue ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source, DebugLoc dl); 294 295 SDValue EmitStackConvert(SDValue SrcOp, MVT SlotVT, MVT DestVT, DebugLoc dl); 296 SDValue ExpandBUILD_VECTOR(SDNode *Node); 297 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 298 SDValue LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, 299 SDValue Op, DebugLoc dl); 300 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, MVT DestVT, 301 DebugLoc dl); 302 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, MVT DestVT, bool isSigned, 303 DebugLoc dl); 304 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, MVT DestVT, bool isSigned, 305 DebugLoc dl); 306 307 SDValue ExpandBSWAP(SDValue Op, DebugLoc dl); 308 SDValue ExpandBitCount(unsigned Opc, SDValue Op, DebugLoc dl); 309 bool ExpandShift(unsigned Opc, SDValue Op, SDValue Amt, 310 SDValue &Lo, SDValue &Hi, DebugLoc dl); 311 void ExpandShiftParts(unsigned NodeOp, SDValue Op, SDValue Amt, 312 SDValue &Lo, SDValue &Hi, DebugLoc dl); 313 314 SDValue ExpandEXTRACT_SUBVECTOR(SDValue Op); 315 SDValue ExpandEXTRACT_VECTOR_ELT(SDValue Op); 316}; 317} 318 319/// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 320/// performs the same shuffe in terms of order or result bytes, but on a type 321/// whose vector element type is narrower than the original shuffle type. 322/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 323SDValue 324SelectionDAGLegalize::ShuffleWithNarrowerEltType(MVT NVT, MVT VT, DebugLoc dl, 325 SDValue N1, SDValue N2, 326 SmallVectorImpl<int> &Mask) const { 327 MVT EltVT = NVT.getVectorElementType(); 328 unsigned NumMaskElts = VT.getVectorNumElements(); 329 unsigned NumDestElts = NVT.getVectorNumElements(); 330 unsigned NumEltsGrowth = NumDestElts / NumMaskElts; 331 332 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 333 334 if (NumEltsGrowth == 1) 335 return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]); 336 337 SmallVector<int, 8> NewMask; 338 for (unsigned i = 0; i != NumMaskElts; ++i) { 339 int Idx = Mask[i]; 340 for (unsigned j = 0; j != NumEltsGrowth; ++j) { 341 if (Idx < 0) 342 NewMask.push_back(-1); 343 else 344 NewMask.push_back(Idx * NumEltsGrowth + j); 345 } 346 } 347 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); 348 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); 349 return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); 350} 351 352SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag, 353 bool types, unsigned ol) 354 : TLI(dag.getTargetLoweringInfo()), DAG(dag), OptLevel(ol), 355 TypesNeedLegalizing(types), ValueTypeActions(TLI.getValueTypeActions()) { 356 assert(MVT::LAST_VALUETYPE <= 32 && 357 "Too many value types for ValueTypeActions to hold!"); 358} 359 360void SelectionDAGLegalize::LegalizeDAG() { 361 LastCALLSEQ_END = DAG.getEntryNode(); 362 IsLegalizingCall = false; 363 IsLegalizingCallArgs = false; 364 365 // The legalize process is inherently a bottom-up recursive process (users 366 // legalize their uses before themselves). Given infinite stack space, we 367 // could just start legalizing on the root and traverse the whole graph. In 368 // practice however, this causes us to run out of stack space on large basic 369 // blocks. To avoid this problem, compute an ordering of the nodes where each 370 // node is only legalized after all of its operands are legalized. 371 DAG.AssignTopologicalOrder(); 372 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 373 E = prior(DAG.allnodes_end()); I != next(E); ++I) 374 HandleOp(SDValue(I, 0)); 375 376 // Finally, it's possible the root changed. Get the new root. 377 SDValue OldRoot = DAG.getRoot(); 378 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?"); 379 DAG.setRoot(LegalizedNodes[OldRoot]); 380 381 ExpandedNodes.clear(); 382 LegalizedNodes.clear(); 383 PromotedNodes.clear(); 384 SplitNodes.clear(); 385 ScalarizedNodes.clear(); 386 WidenNodes.clear(); 387 388 // Remove dead nodes now. 389 DAG.RemoveDeadNodes(); 390} 391 392 393/// FindCallEndFromCallStart - Given a chained node that is part of a call 394/// sequence, find the CALLSEQ_END node that terminates the call sequence. 395static SDNode *FindCallEndFromCallStart(SDNode *Node) { 396 if (Node->getOpcode() == ISD::CALLSEQ_END) 397 return Node; 398 if (Node->use_empty()) 399 return 0; // No CallSeqEnd 400 401 // The chain is usually at the end. 402 SDValue TheChain(Node, Node->getNumValues()-1); 403 if (TheChain.getValueType() != MVT::Other) { 404 // Sometimes it's at the beginning. 405 TheChain = SDValue(Node, 0); 406 if (TheChain.getValueType() != MVT::Other) { 407 // Otherwise, hunt for it. 408 for (unsigned i = 1, e = Node->getNumValues(); i != e; ++i) 409 if (Node->getValueType(i) == MVT::Other) { 410 TheChain = SDValue(Node, i); 411 break; 412 } 413 414 // Otherwise, we walked into a node without a chain. 415 if (TheChain.getValueType() != MVT::Other) 416 return 0; 417 } 418 } 419 420 for (SDNode::use_iterator UI = Node->use_begin(), 421 E = Node->use_end(); UI != E; ++UI) { 422 423 // Make sure to only follow users of our token chain. 424 SDNode *User = *UI; 425 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) 426 if (User->getOperand(i) == TheChain) 427 if (SDNode *Result = FindCallEndFromCallStart(User)) 428 return Result; 429 } 430 return 0; 431} 432 433/// FindCallStartFromCallEnd - Given a chained node that is part of a call 434/// sequence, find the CALLSEQ_START node that initiates the call sequence. 435static SDNode *FindCallStartFromCallEnd(SDNode *Node) { 436 assert(Node && "Didn't find callseq_start for a call??"); 437 if (Node->getOpcode() == ISD::CALLSEQ_START) return Node; 438 439 assert(Node->getOperand(0).getValueType() == MVT::Other && 440 "Node doesn't have a token chain argument!"); 441 return FindCallStartFromCallEnd(Node->getOperand(0).getNode()); 442} 443 444/// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to 445/// see if any uses can reach Dest. If no dest operands can get to dest, 446/// legalize them, legalize ourself, and return false, otherwise, return true. 447/// 448/// Keep track of the nodes we fine that actually do lead to Dest in 449/// NodesLeadingTo. This avoids retraversing them exponential number of times. 450/// 451bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, 452 SmallPtrSet<SDNode*, 32> &NodesLeadingTo) { 453 if (N == Dest) return true; // N certainly leads to Dest :) 454 455 // If we've already processed this node and it does lead to Dest, there is no 456 // need to reprocess it. 457 if (NodesLeadingTo.count(N)) return true; 458 459 // If the first result of this node has been already legalized, then it cannot 460 // reach N. 461 switch (getTypeAction(N->getValueType(0))) { 462 case Legal: 463 if (LegalizedNodes.count(SDValue(N, 0))) return false; 464 break; 465 case Promote: 466 if (PromotedNodes.count(SDValue(N, 0))) return false; 467 break; 468 case Expand: 469 if (ExpandedNodes.count(SDValue(N, 0))) return false; 470 break; 471 } 472 473 // Okay, this node has not already been legalized. Check and legalize all 474 // operands. If none lead to Dest, then we can legalize this node. 475 bool OperandsLeadToDest = false; 476 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 477 OperandsLeadToDest |= // If an operand leads to Dest, so do we. 478 LegalizeAllNodesNotLeadingTo(N->getOperand(i).getNode(), Dest, NodesLeadingTo); 479 480 if (OperandsLeadToDest) { 481 NodesLeadingTo.insert(N); 482 return true; 483 } 484 485 // Okay, this node looks safe, legalize it and return false. 486 HandleOp(SDValue(N, 0)); 487 return false; 488} 489 490/// HandleOp - Legalize, Promote, Widen, or Expand the specified operand as 491/// appropriate for its type. 492void SelectionDAGLegalize::HandleOp(SDValue Op) { 493 MVT VT = Op.getValueType(); 494 // If the type legalizer was run then we should never see any illegal result 495 // types here except for target constants (the type legalizer does not touch 496 // those) or for build vector used as a mask for a vector shuffle. 497 assert((TypesNeedLegalizing || getTypeAction(VT) == Legal || 498 IsLegalizingCallArgs || Op.getOpcode() == ISD::TargetConstant) && 499 "Illegal type introduced after type legalization?"); 500 switch (getTypeAction(VT)) { 501 default: assert(0 && "Bad type action!"); 502 case Legal: (void)LegalizeOp(Op); break; 503 case Promote: 504 if (!VT.isVector()) { 505 (void)PromoteOp(Op); 506 break; 507 } 508 else { 509 // See if we can widen otherwise use Expand to either scalarize or split 510 MVT WidenVT = TLI.getWidenVectorType(VT); 511 if (WidenVT != MVT::Other) { 512 (void) WidenVectorOp(Op, WidenVT); 513 break; 514 } 515 // else fall thru to expand since we can't widen the vector 516 } 517 case Expand: 518 if (!VT.isVector()) { 519 // If this is an illegal scalar, expand it into its two component 520 // pieces. 521 SDValue X, Y; 522 if (Op.getOpcode() == ISD::TargetConstant) 523 break; // Allow illegal target nodes. 524 ExpandOp(Op, X, Y); 525 } else if (VT.getVectorNumElements() == 1) { 526 // If this is an illegal single element vector, convert it to a 527 // scalar operation. 528 (void)ScalarizeVectorOp(Op); 529 } else { 530 // This is an illegal multiple element vector. 531 // Split it in half and legalize both parts. 532 SDValue X, Y; 533 SplitVectorOp(Op, X, Y); 534 } 535 break; 536 } 537} 538 539/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or 540/// a load from the constant pool. 541static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, 542 SelectionDAG &DAG, const TargetLowering &TLI) { 543 bool Extend = false; 544 DebugLoc dl = CFP->getDebugLoc(); 545 546 // If a FP immediate is precise when represented as a float and if the 547 // target can do an extending load from float to double, we put it into 548 // the constant pool as a float, even if it's is statically typed as a 549 // double. This shrinks FP constants and canonicalizes them for targets where 550 // an FP extending load is the same cost as a normal load (such as on the x87 551 // fp stack or PPC FP unit). 552 MVT VT = CFP->getValueType(0); 553 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 554 if (!UseCP) { 555 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); 556 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), 557 (VT == MVT::f64) ? MVT::i64 : MVT::i32); 558 } 559 560 MVT OrigVT = VT; 561 MVT SVT = VT; 562 while (SVT != MVT::f32) { 563 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT() - 1); 564 if (CFP->isValueValidForType(SVT, CFP->getValueAPF()) && 565 // Only do this if the target has a native EXTLOAD instruction from 566 // smaller type. 567 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && 568 TLI.ShouldShrinkFPConstant(OrigVT)) { 569 const Type *SType = SVT.getTypeForMVT(); 570 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 571 VT = SVT; 572 Extend = true; 573 } 574 } 575 576 SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); 577 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 578 if (Extend) 579 return DAG.getExtLoad(ISD::EXTLOAD, dl, 580 OrigVT, DAG.getEntryNode(), 581 CPIdx, PseudoSourceValue::getConstantPool(), 582 0, VT, false, Alignment); 583 return DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx, 584 PseudoSourceValue::getConstantPool(), 0, false, Alignment); 585} 586 587 588/// ExpandFCOPYSIGNToBitwiseOps - Expands fcopysign to a series of bitwise 589/// operations. 590static 591SDValue ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, 592 SelectionDAG &DAG, 593 const TargetLowering &TLI) { 594 DebugLoc dl = Node->getDebugLoc(); 595 MVT VT = Node->getValueType(0); 596 MVT SrcVT = Node->getOperand(1).getValueType(); 597 assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) && 598 "fcopysign expansion only supported for f32 and f64"); 599 MVT SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32; 600 601 // First get the sign bit of second operand. 602 SDValue Mask1 = (SrcVT == MVT::f64) 603 ? DAG.getConstantFP(BitsToDouble(1ULL << 63), SrcVT) 604 : DAG.getConstantFP(BitsToFloat(1U << 31), SrcVT); 605 Mask1 = DAG.getNode(ISD::BIT_CONVERT, dl, SrcNVT, Mask1); 606 SDValue SignBit= DAG.getNode(ISD::BIT_CONVERT, dl, SrcNVT, 607 Node->getOperand(1)); 608 SignBit = DAG.getNode(ISD::AND, dl, SrcNVT, SignBit, Mask1); 609 // Shift right or sign-extend it if the two operands have different types. 610 int SizeDiff = SrcNVT.getSizeInBits() - NVT.getSizeInBits(); 611 if (SizeDiff > 0) { 612 SignBit = DAG.getNode(ISD::SRL, dl, SrcNVT, SignBit, 613 DAG.getConstant(SizeDiff, TLI.getShiftAmountTy())); 614 SignBit = DAG.getNode(ISD::TRUNCATE, dl, NVT, SignBit); 615 } else if (SizeDiff < 0) { 616 SignBit = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, SignBit); 617 SignBit = DAG.getNode(ISD::SHL, dl, NVT, SignBit, 618 DAG.getConstant(-SizeDiff, TLI.getShiftAmountTy())); 619 } 620 621 // Clear the sign bit of first operand. 622 SDValue Mask2 = (VT == MVT::f64) 623 ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) 624 : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); 625 Mask2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask2); 626 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); 627 Result = DAG.getNode(ISD::AND, dl, NVT, Result, Mask2); 628 629 // Or the value with the sign bit. 630 Result = DAG.getNode(ISD::OR, dl, NVT, Result, SignBit); 631 return Result; 632} 633 634/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. 635static 636SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, 637 const TargetLowering &TLI) { 638 SDValue Chain = ST->getChain(); 639 SDValue Ptr = ST->getBasePtr(); 640 SDValue Val = ST->getValue(); 641 MVT VT = Val.getValueType(); 642 int Alignment = ST->getAlignment(); 643 int SVOffset = ST->getSrcValueOffset(); 644 DebugLoc dl = ST->getDebugLoc(); 645 if (ST->getMemoryVT().isFloatingPoint() || 646 ST->getMemoryVT().isVector()) { 647 MVT intVT = MVT::getIntegerVT(VT.getSizeInBits()); 648 if (TLI.isTypeLegal(intVT)) { 649 // Expand to a bitconvert of the value to the integer type of the 650 // same size, then a (misaligned) int store. 651 // FIXME: Does not handle truncating floating point stores! 652 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, dl, intVT, Val); 653 return DAG.getStore(Chain, dl, Result, Ptr, ST->getSrcValue(), 654 SVOffset, ST->isVolatile(), Alignment); 655 } else { 656 // Do a (aligned) store to a stack slot, then copy from the stack slot 657 // to the final destination using (unaligned) integer loads and stores. 658 MVT StoredVT = ST->getMemoryVT(); 659 MVT RegVT = 660 TLI.getRegisterType(MVT::getIntegerVT(StoredVT.getSizeInBits())); 661 unsigned StoredBytes = StoredVT.getSizeInBits() / 8; 662 unsigned RegBytes = RegVT.getSizeInBits() / 8; 663 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; 664 665 // Make sure the stack slot is also aligned for the register type. 666 SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); 667 668 // Perform the original store, only redirected to the stack slot. 669 SDValue Store = DAG.getTruncStore(Chain, dl, 670 Val, StackPtr, NULL, 0, StoredVT); 671 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 672 SmallVector<SDValue, 8> Stores; 673 unsigned Offset = 0; 674 675 // Do all but one copies using the full register width. 676 for (unsigned i = 1; i < NumRegs; i++) { 677 // Load one integer register's worth from the stack slot. 678 SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr, NULL, 0); 679 // Store it to the final location. Remember the store. 680 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, 681 ST->getSrcValue(), SVOffset + Offset, 682 ST->isVolatile(), 683 MinAlign(ST->getAlignment(), Offset))); 684 // Increment the pointers. 685 Offset += RegBytes; 686 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 687 Increment); 688 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 689 } 690 691 // The last store may be partial. Do a truncating store. On big-endian 692 // machines this requires an extending load from the stack slot to ensure 693 // that the bits are in the right place. 694 MVT MemVT = MVT::getIntegerVT(8 * (StoredBytes - Offset)); 695 696 // Load from the stack slot. 697 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, 698 NULL, 0, MemVT); 699 700 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, 701 ST->getSrcValue(), SVOffset + Offset, 702 MemVT, ST->isVolatile(), 703 MinAlign(ST->getAlignment(), Offset))); 704 // The order of the stores doesn't matter - say it with a TokenFactor. 705 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 706 Stores.size()); 707 } 708 } 709 assert(ST->getMemoryVT().isInteger() && 710 !ST->getMemoryVT().isVector() && 711 "Unaligned store of unknown type."); 712 // Get the half-size VT 713 MVT NewStoredVT = 714 (MVT::SimpleValueType)(ST->getMemoryVT().getSimpleVT() - 1); 715 int NumBits = NewStoredVT.getSizeInBits(); 716 int IncrementSize = NumBits / 8; 717 718 // Divide the stored value in two parts. 719 SDValue ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); 720 SDValue Lo = Val; 721 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); 722 723 // Store the two parts 724 SDValue Store1, Store2; 725 Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr, 726 ST->getSrcValue(), SVOffset, NewStoredVT, 727 ST->isVolatile(), Alignment); 728 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 729 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 730 Alignment = MinAlign(Alignment, IncrementSize); 731 Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr, 732 ST->getSrcValue(), SVOffset + IncrementSize, 733 NewStoredVT, ST->isVolatile(), Alignment); 734 735 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); 736} 737 738/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. 739static 740SDValue ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, 741 const TargetLowering &TLI) { 742 int SVOffset = LD->getSrcValueOffset(); 743 SDValue Chain = LD->getChain(); 744 SDValue Ptr = LD->getBasePtr(); 745 MVT VT = LD->getValueType(0); 746 MVT LoadedVT = LD->getMemoryVT(); 747 DebugLoc dl = LD->getDebugLoc(); 748 if (VT.isFloatingPoint() || VT.isVector()) { 749 MVT intVT = MVT::getIntegerVT(LoadedVT.getSizeInBits()); 750 if (TLI.isTypeLegal(intVT)) { 751 // Expand to a (misaligned) integer load of the same size, 752 // then bitconvert to floating point or vector. 753 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, LD->getSrcValue(), 754 SVOffset, LD->isVolatile(), 755 LD->getAlignment()); 756 SDValue Result = DAG.getNode(ISD::BIT_CONVERT, dl, LoadedVT, newLoad); 757 if (VT.isFloatingPoint() && LoadedVT != VT) 758 Result = DAG.getNode(ISD::FP_EXTEND, dl, VT, Result); 759 760 SDValue Ops[] = { Result, Chain }; 761 return DAG.getMergeValues(Ops, 2, dl); 762 } else { 763 // Copy the value to a (aligned) stack slot using (unaligned) integer 764 // loads and stores, then do a (aligned) load from the stack slot. 765 MVT RegVT = TLI.getRegisterType(intVT); 766 unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; 767 unsigned RegBytes = RegVT.getSizeInBits() / 8; 768 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; 769 770 // Make sure the stack slot is also aligned for the register type. 771 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT); 772 773 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 774 SmallVector<SDValue, 8> Stores; 775 SDValue StackPtr = StackBase; 776 unsigned Offset = 0; 777 778 // Do all but one copies using the full register width. 779 for (unsigned i = 1; i < NumRegs; i++) { 780 // Load one integer register's worth from the original location. 781 SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr, LD->getSrcValue(), 782 SVOffset + Offset, LD->isVolatile(), 783 MinAlign(LD->getAlignment(), Offset)); 784 // Follow the load with a store to the stack slot. Remember the store. 785 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr, 786 NULL, 0)); 787 // Increment the pointers. 788 Offset += RegBytes; 789 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 790 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 791 Increment); 792 } 793 794 // The last copy may be partial. Do an extending load. 795 MVT MemVT = MVT::getIntegerVT(8 * (LoadedBytes - Offset)); 796 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, 797 LD->getSrcValue(), SVOffset + Offset, 798 MemVT, LD->isVolatile(), 799 MinAlign(LD->getAlignment(), Offset)); 800 // Follow the load with a store to the stack slot. Remember the store. 801 // On big-endian machines this requires a truncating store to ensure 802 // that the bits end up in the right place. 803 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr, 804 NULL, 0, MemVT)); 805 806 // The order of the stores doesn't matter - say it with a TokenFactor. 807 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 808 Stores.size()); 809 810 // Finally, perform the original load only redirected to the stack slot. 811 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, 812 NULL, 0, LoadedVT); 813 814 // Callers expect a MERGE_VALUES node. 815 SDValue Ops[] = { Load, TF }; 816 return DAG.getMergeValues(Ops, 2, dl); 817 } 818 } 819 assert(LoadedVT.isInteger() && !LoadedVT.isVector() && 820 "Unaligned load of unsupported type."); 821 822 // Compute the new VT that is half the size of the old one. This is an 823 // integer MVT. 824 unsigned NumBits = LoadedVT.getSizeInBits(); 825 MVT NewLoadedVT; 826 NewLoadedVT = MVT::getIntegerVT(NumBits/2); 827 NumBits >>= 1; 828 829 unsigned Alignment = LD->getAlignment(); 830 unsigned IncrementSize = NumBits / 8; 831 ISD::LoadExtType HiExtType = LD->getExtensionType(); 832 833 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. 834 if (HiExtType == ISD::NON_EXTLOAD) 835 HiExtType = ISD::ZEXTLOAD; 836 837 // Load the value in two parts 838 SDValue Lo, Hi; 839 if (TLI.isLittleEndian()) { 840 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getSrcValue(), 841 SVOffset, NewLoadedVT, LD->isVolatile(), Alignment); 842 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 843 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 844 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getSrcValue(), 845 SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(), 846 MinAlign(Alignment, IncrementSize)); 847 } else { 848 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getSrcValue(), 849 SVOffset, NewLoadedVT, LD->isVolatile(), Alignment); 850 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 851 DAG.getConstant(IncrementSize, TLI.getPointerTy())); 852 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getSrcValue(), 853 SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(), 854 MinAlign(Alignment, IncrementSize)); 855 } 856 857 // aggregate the two parts 858 SDValue ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); 859 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); 860 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); 861 862 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 863 Hi.getValue(1)); 864 865 SDValue Ops[] = { Result, TF }; 866 return DAG.getMergeValues(Ops, 2, dl); 867} 868 869/// UnrollVectorOp - We know that the given vector has a legal type, however 870/// the operation it performs is not legal and is an operation that we have 871/// no way of lowering. "Unroll" the vector, splitting out the scalars and 872/// operating on each element individually. 873SDValue SelectionDAGLegalize::UnrollVectorOp(SDValue Op) { 874 MVT VT = Op.getValueType(); 875 assert(isTypeLegal(VT) && 876 "Caller should expand or promote operands that are not legal!"); 877 assert(Op.getNode()->getNumValues() == 1 && 878 "Can't unroll a vector with multiple results!"); 879 unsigned NE = VT.getVectorNumElements(); 880 MVT EltVT = VT.getVectorElementType(); 881 DebugLoc dl = Op.getDebugLoc(); 882 883 SmallVector<SDValue, 8> Scalars; 884 SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 885 for (unsigned i = 0; i != NE; ++i) { 886 for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 887 SDValue Operand = Op.getOperand(j); 888 MVT OperandVT = Operand.getValueType(); 889 if (OperandVT.isVector()) { 890 // A vector operand; extract a single element. 891 MVT OperandEltVT = OperandVT.getVectorElementType(); 892 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 893 OperandEltVT, 894 Operand, 895 DAG.getConstant(i, MVT::i32)); 896 } else { 897 // A scalar operand; just use it as is. 898 Operands[j] = Operand; 899 } 900 } 901 902 switch (Op.getOpcode()) { 903 default: 904 Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, 905 &Operands[0], Operands.size())); 906 break; 907 case ISD::SHL: 908 case ISD::SRA: 909 case ISD::SRL: 910 case ISD::ROTL: 911 case ISD::ROTR: 912 Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, Operands[0], 913 DAG.getShiftAmountOperand(Operands[1]))); 914 break; 915 } 916 } 917 918 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Scalars[0], Scalars.size()); 919} 920 921/// GetFPLibCall - Return the right libcall for the given floating point type. 922static RTLIB::Libcall GetFPLibCall(MVT VT, 923 RTLIB::Libcall Call_F32, 924 RTLIB::Libcall Call_F64, 925 RTLIB::Libcall Call_F80, 926 RTLIB::Libcall Call_PPCF128) { 927 return 928 VT == MVT::f32 ? Call_F32 : 929 VT == MVT::f64 ? Call_F64 : 930 VT == MVT::f80 ? Call_F80 : 931 VT == MVT::ppcf128 ? Call_PPCF128 : 932 RTLIB::UNKNOWN_LIBCALL; 933} 934 935/// PerformInsertVectorEltInMemory - Some target cannot handle a variable 936/// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 937/// is necessary to spill the vector being inserted into to memory, perform 938/// the insert there, and then read the result back. 939SDValue SelectionDAGLegalize:: 940PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 941 DebugLoc dl) { 942 SDValue Tmp1 = Vec; 943 SDValue Tmp2 = Val; 944 SDValue Tmp3 = Idx; 945 946 // If the target doesn't support this, we have to spill the input vector 947 // to a temporary stack slot, update the element, then reload it. This is 948 // badness. We could also load the value into a vector register (either 949 // with a "move to register" or "extload into register" instruction, then 950 // permute it into place, if the idx is a constant and if the idx is 951 // supported by the target. 952 MVT VT = Tmp1.getValueType(); 953 MVT EltVT = VT.getVectorElementType(); 954 MVT IdxVT = Tmp3.getValueType(); 955 MVT PtrVT = TLI.getPointerTy(); 956 SDValue StackPtr = DAG.CreateStackTemporary(VT); 957 958 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 959 960 // Store the vector. 961 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr, 962 PseudoSourceValue::getFixedStack(SPFI), 0); 963 964 // Truncate or zero extend offset to target pointer type. 965 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 966 Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3); 967 // Add the offset to the index. 968 unsigned EltSize = EltVT.getSizeInBits()/8; 969 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); 970 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); 971 // Store the scalar value. 972 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, 973 PseudoSourceValue::getFixedStack(SPFI), 0, EltVT); 974 // Load the updated vector. 975 return DAG.getLoad(VT, dl, Ch, StackPtr, 976 PseudoSourceValue::getFixedStack(SPFI), 0); 977} 978 979 980/// LegalizeOp - We know that the specified value has a legal type, and 981/// that its operands are legal. Now ensure that the operation itself 982/// is legal, recursively ensuring that the operands' operations remain 983/// legal. 984SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { 985 if (Op.getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. 986 return Op; 987 988 assert(isTypeLegal(Op.getValueType()) && 989 "Caller should expand or promote operands that are not legal!"); 990 SDNode *Node = Op.getNode(); 991 DebugLoc dl = Node->getDebugLoc(); 992 993 // If this operation defines any values that cannot be represented in a 994 // register on this target, make sure to expand or promote them. 995 if (Node->getNumValues() > 1) { 996 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 997 if (getTypeAction(Node->getValueType(i)) != Legal) { 998 HandleOp(Op.getValue(i)); 999 assert(LegalizedNodes.count(Op) && 1000 "Handling didn't add legal operands!"); 1001 return LegalizedNodes[Op]; 1002 } 1003 } 1004 1005 // Note that LegalizeOp may be reentered even from single-use nodes, which 1006 // means that we always must cache transformed nodes. 1007 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 1008 if (I != LegalizedNodes.end()) return I->second; 1009 1010 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 1011 SDValue Result = Op; 1012 bool isCustom = false; 1013 1014 switch (Node->getOpcode()) { 1015 case ISD::FrameIndex: 1016 case ISD::EntryToken: 1017 case ISD::Register: 1018 case ISD::BasicBlock: 1019 case ISD::TargetFrameIndex: 1020 case ISD::TargetJumpTable: 1021 case ISD::TargetConstant: 1022 case ISD::TargetConstantFP: 1023 case ISD::TargetConstantPool: 1024 case ISD::TargetGlobalAddress: 1025 case ISD::TargetGlobalTLSAddress: 1026 case ISD::TargetExternalSymbol: 1027 case ISD::VALUETYPE: 1028 case ISD::SRCVALUE: 1029 case ISD::MEMOPERAND: 1030 case ISD::CONDCODE: 1031 case ISD::ARG_FLAGS: 1032 // Primitives must all be legal. 1033 assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) && 1034 "This must be legal!"); 1035 break; 1036 default: 1037 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 1038 // If this is a target node, legalize it by legalizing the operands then 1039 // passing it through. 1040 SmallVector<SDValue, 8> Ops; 1041 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1042 Ops.push_back(LegalizeOp(Node->getOperand(i))); 1043 1044 Result = DAG.UpdateNodeOperands(Result.getValue(0), &Ops[0], Ops.size()); 1045 1046 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1047 AddLegalizedOperand(Op.getValue(i), Result.getValue(i)); 1048 return Result.getValue(Op.getResNo()); 1049 } 1050 // Otherwise this is an unhandled builtin node. splat. 1051#ifndef NDEBUG 1052 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 1053#endif 1054 assert(0 && "Do not know how to legalize this operator!"); 1055 abort(); 1056 case ISD::GLOBAL_OFFSET_TABLE: 1057 case ISD::GlobalAddress: 1058 case ISD::GlobalTLSAddress: 1059 case ISD::ExternalSymbol: 1060 case ISD::ConstantPool: 1061 case ISD::JumpTable: // Nothing to do. 1062 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 1063 default: assert(0 && "This action is not supported yet!"); 1064 case TargetLowering::Custom: 1065 Tmp1 = TLI.LowerOperation(Op, DAG); 1066 if (Tmp1.getNode()) Result = Tmp1; 1067 // FALLTHROUGH if the target doesn't want to lower this op after all. 1068 case TargetLowering::Legal: 1069 break; 1070 } 1071 break; 1072 case ISD::FRAMEADDR: 1073 case ISD::RETURNADDR: 1074 // The only option for these nodes is to custom lower them. If the target 1075 // does not custom lower them, then return zero. 1076 Tmp1 = TLI.LowerOperation(Op, DAG); 1077 if (Tmp1.getNode()) 1078 Result = Tmp1; 1079 else 1080 Result = DAG.getConstant(0, TLI.getPointerTy()); 1081 break; 1082 case ISD::FRAME_TO_ARGS_OFFSET: { 1083 MVT VT = Node->getValueType(0); 1084 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1085 default: assert(0 && "This action is not supported yet!"); 1086 case TargetLowering::Custom: 1087 Result = TLI.LowerOperation(Op, DAG); 1088 if (Result.getNode()) break; 1089 // Fall Thru 1090 case TargetLowering::Legal: 1091 Result = DAG.getConstant(0, VT); 1092 break; 1093 } 1094 } 1095 break; 1096 case ISD::EXCEPTIONADDR: { 1097 Tmp1 = LegalizeOp(Node->getOperand(0)); 1098 MVT VT = Node->getValueType(0); 1099 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1100 default: assert(0 && "This action is not supported yet!"); 1101 case TargetLowering::Expand: { 1102 unsigned Reg = TLI.getExceptionAddressRegister(); 1103 Result = DAG.getCopyFromReg(Tmp1, dl, Reg, VT); 1104 } 1105 break; 1106 case TargetLowering::Custom: 1107 Result = TLI.LowerOperation(Op, DAG); 1108 if (Result.getNode()) break; 1109 // Fall Thru 1110 case TargetLowering::Legal: { 1111 SDValue Ops[] = { DAG.getConstant(0, VT), Tmp1 }; 1112 Result = DAG.getMergeValues(Ops, 2, dl); 1113 break; 1114 } 1115 } 1116 } 1117 if (Result.getNode()->getNumValues() == 1) break; 1118 1119 assert(Result.getNode()->getNumValues() == 2 && 1120 "Cannot return more than two values!"); 1121 1122 // Since we produced two values, make sure to remember that we 1123 // legalized both of them. 1124 Tmp1 = LegalizeOp(Result); 1125 Tmp2 = LegalizeOp(Result.getValue(1)); 1126 AddLegalizedOperand(Op.getValue(0), Tmp1); 1127 AddLegalizedOperand(Op.getValue(1), Tmp2); 1128 return Op.getResNo() ? Tmp2 : Tmp1; 1129 case ISD::EHSELECTION: { 1130 Tmp1 = LegalizeOp(Node->getOperand(0)); 1131 Tmp2 = LegalizeOp(Node->getOperand(1)); 1132 MVT VT = Node->getValueType(0); 1133 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1134 default: assert(0 && "This action is not supported yet!"); 1135 case TargetLowering::Expand: { 1136 unsigned Reg = TLI.getExceptionSelectorRegister(); 1137 Result = DAG.getCopyFromReg(Tmp2, dl, Reg, VT); 1138 } 1139 break; 1140 case TargetLowering::Custom: 1141 Result = TLI.LowerOperation(Op, DAG); 1142 if (Result.getNode()) break; 1143 // Fall Thru 1144 case TargetLowering::Legal: { 1145 SDValue Ops[] = { DAG.getConstant(0, VT), Tmp2 }; 1146 Result = DAG.getMergeValues(Ops, 2, dl); 1147 break; 1148 } 1149 } 1150 } 1151 if (Result.getNode()->getNumValues() == 1) break; 1152 1153 assert(Result.getNode()->getNumValues() == 2 && 1154 "Cannot return more than two values!"); 1155 1156 // Since we produced two values, make sure to remember that we 1157 // legalized both of them. 1158 Tmp1 = LegalizeOp(Result); 1159 Tmp2 = LegalizeOp(Result.getValue(1)); 1160 AddLegalizedOperand(Op.getValue(0), Tmp1); 1161 AddLegalizedOperand(Op.getValue(1), Tmp2); 1162 return Op.getResNo() ? Tmp2 : Tmp1; 1163 case ISD::EH_RETURN: { 1164 MVT VT = Node->getValueType(0); 1165 // The only "good" option for this node is to custom lower it. 1166 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1167 default: assert(0 && "This action is not supported at all!"); 1168 case TargetLowering::Custom: 1169 Result = TLI.LowerOperation(Op, DAG); 1170 if (Result.getNode()) break; 1171 // Fall Thru 1172 case TargetLowering::Legal: 1173 // Target does not know, how to lower this, lower to noop 1174 Result = LegalizeOp(Node->getOperand(0)); 1175 break; 1176 } 1177 } 1178 break; 1179 case ISD::AssertSext: 1180 case ISD::AssertZext: 1181 Tmp1 = LegalizeOp(Node->getOperand(0)); 1182 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1183 break; 1184 case ISD::MERGE_VALUES: 1185 // Legalize eliminates MERGE_VALUES nodes. 1186 Result = Node->getOperand(Op.getResNo()); 1187 break; 1188 case ISD::CopyFromReg: 1189 Tmp1 = LegalizeOp(Node->getOperand(0)); 1190 Result = Op.getValue(0); 1191 if (Node->getNumValues() == 2) { 1192 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1193 } else { 1194 assert(Node->getNumValues() == 3 && "Invalid copyfromreg!"); 1195 if (Node->getNumOperands() == 3) { 1196 Tmp2 = LegalizeOp(Node->getOperand(2)); 1197 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2); 1198 } else { 1199 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1200 } 1201 AddLegalizedOperand(Op.getValue(2), Result.getValue(2)); 1202 } 1203 // Since CopyFromReg produces two values, make sure to remember that we 1204 // legalized both of them. 1205 AddLegalizedOperand(Op.getValue(0), Result); 1206 AddLegalizedOperand(Op.getValue(1), Result.getValue(1)); 1207 return Result.getValue(Op.getResNo()); 1208 case ISD::UNDEF: { 1209 MVT VT = Op.getValueType(); 1210 switch (TLI.getOperationAction(ISD::UNDEF, VT)) { 1211 default: assert(0 && "This action is not supported yet!"); 1212 case TargetLowering::Expand: 1213 if (VT.isInteger()) 1214 Result = DAG.getConstant(0, VT); 1215 else if (VT.isFloatingPoint()) 1216 Result = DAG.getConstantFP(APFloat(APInt(VT.getSizeInBits(), 0)), 1217 VT); 1218 else 1219 assert(0 && "Unknown value type!"); 1220 break; 1221 case TargetLowering::Legal: 1222 break; 1223 } 1224 break; 1225 } 1226 1227 case ISD::INTRINSIC_W_CHAIN: 1228 case ISD::INTRINSIC_WO_CHAIN: 1229 case ISD::INTRINSIC_VOID: { 1230 SmallVector<SDValue, 8> Ops; 1231 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1232 Ops.push_back(LegalizeOp(Node->getOperand(i))); 1233 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1234 1235 // Allow the target to custom lower its intrinsics if it wants to. 1236 if (TLI.getOperationAction(Node->getOpcode(), MVT::Other) == 1237 TargetLowering::Custom) { 1238 Tmp3 = TLI.LowerOperation(Result, DAG); 1239 if (Tmp3.getNode()) Result = Tmp3; 1240 } 1241 1242 if (Result.getNode()->getNumValues() == 1) break; 1243 1244 // Must have return value and chain result. 1245 assert(Result.getNode()->getNumValues() == 2 && 1246 "Cannot return more than two values!"); 1247 1248 // Since loads produce two values, make sure to remember that we 1249 // legalized both of them. 1250 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1251 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1252 return Result.getValue(Op.getResNo()); 1253 } 1254 1255 case ISD::DBG_STOPPOINT: 1256 assert(Node->getNumOperands() == 1 && "Invalid DBG_STOPPOINT node!"); 1257 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input chain. 1258 1259 switch (TLI.getOperationAction(ISD::DBG_STOPPOINT, MVT::Other)) { 1260 case TargetLowering::Promote: 1261 default: assert(0 && "This action is not supported yet!"); 1262 case TargetLowering::Expand: { 1263 DwarfWriter *DW = DAG.getDwarfWriter(); 1264 bool useDEBUG_LOC = TLI.isOperationLegalOrCustom(ISD::DEBUG_LOC, 1265 MVT::Other); 1266 bool useLABEL = TLI.isOperationLegalOrCustom(ISD::DBG_LABEL, MVT::Other); 1267 1268 const DbgStopPointSDNode *DSP = cast<DbgStopPointSDNode>(Node); 1269 GlobalVariable *CU_GV = cast<GlobalVariable>(DSP->getCompileUnit()); 1270 if (DW && (useDEBUG_LOC || useLABEL) && !CU_GV->isDeclaration()) { 1271 DICompileUnit CU(cast<GlobalVariable>(DSP->getCompileUnit())); 1272 std::string Dir, FN; 1273 unsigned SrcFile = DW->getOrCreateSourceID(CU.getDirectory(Dir), 1274 CU.getFilename(FN)); 1275 1276 unsigned Line = DSP->getLine(); 1277 unsigned Col = DSP->getColumn(); 1278 1279 if (OptLevel == 0) { 1280 // A bit self-referential to have DebugLoc on Debug_Loc nodes, but it 1281 // won't hurt anything. 1282 if (useDEBUG_LOC) { 1283 SDValue Ops[] = { Tmp1, DAG.getConstant(Line, MVT::i32), 1284 DAG.getConstant(Col, MVT::i32), 1285 DAG.getConstant(SrcFile, MVT::i32) }; 1286 Result = DAG.getNode(ISD::DEBUG_LOC, dl, MVT::Other, Ops, 4); 1287 } else { 1288 unsigned ID = DW->RecordSourceLine(Line, Col, SrcFile); 1289 Result = DAG.getLabel(ISD::DBG_LABEL, dl, Tmp1, ID); 1290 } 1291 } else { 1292 Result = Tmp1; // chain 1293 } 1294 } else { 1295 Result = Tmp1; // chain 1296 } 1297 break; 1298 } 1299 case TargetLowering::Custom: 1300 Result = TLI.LowerOperation(Op, DAG); 1301 if (Result.getNode()) 1302 break; 1303 case TargetLowering::Legal: { 1304 LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); 1305 if (Action == Legal && Tmp1 == Node->getOperand(0)) 1306 break; 1307 1308 SmallVector<SDValue, 8> Ops; 1309 Ops.push_back(Tmp1); 1310 if (Action == Legal) { 1311 Ops.push_back(Node->getOperand(1)); // line # must be legal. 1312 Ops.push_back(Node->getOperand(2)); // col # must be legal. 1313 } else { 1314 // Otherwise promote them. 1315 Ops.push_back(PromoteOp(Node->getOperand(1))); 1316 Ops.push_back(PromoteOp(Node->getOperand(2))); 1317 } 1318 Ops.push_back(Node->getOperand(3)); // filename must be legal. 1319 Ops.push_back(Node->getOperand(4)); // working dir # must be legal. 1320 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1321 break; 1322 } 1323 } 1324 break; 1325 1326 case ISD::DECLARE: 1327 assert(Node->getNumOperands() == 3 && "Invalid DECLARE node!"); 1328 switch (TLI.getOperationAction(ISD::DECLARE, MVT::Other)) { 1329 default: assert(0 && "This action is not supported yet!"); 1330 case TargetLowering::Legal: 1331 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1332 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the address. 1333 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the variable. 1334 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1335 break; 1336 case TargetLowering::Expand: 1337 Result = LegalizeOp(Node->getOperand(0)); 1338 break; 1339 } 1340 break; 1341 1342 case ISD::DEBUG_LOC: 1343 assert(Node->getNumOperands() == 4 && "Invalid DEBUG_LOC node!"); 1344 switch (TLI.getOperationAction(ISD::DEBUG_LOC, MVT::Other)) { 1345 default: assert(0 && "This action is not supported yet!"); 1346 case TargetLowering::Legal: { 1347 LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); 1348 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1349 if (Action == Legal && Tmp1 == Node->getOperand(0)) 1350 break; 1351 if (Action == Legal) { 1352 Tmp2 = Node->getOperand(1); 1353 Tmp3 = Node->getOperand(2); 1354 Tmp4 = Node->getOperand(3); 1355 } else { 1356 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the line #. 1357 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the col #. 1358 Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize the source file id. 1359 } 1360 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4); 1361 break; 1362 } 1363 } 1364 break; 1365 1366 case ISD::DBG_LABEL: 1367 case ISD::EH_LABEL: 1368 assert(Node->getNumOperands() == 1 && "Invalid LABEL node!"); 1369 switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) { 1370 default: assert(0 && "This action is not supported yet!"); 1371 case TargetLowering::Legal: 1372 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1373 Result = DAG.UpdateNodeOperands(Result, Tmp1); 1374 break; 1375 case TargetLowering::Expand: 1376 Result = LegalizeOp(Node->getOperand(0)); 1377 break; 1378 } 1379 break; 1380 1381 case ISD::PREFETCH: 1382 assert(Node->getNumOperands() == 4 && "Invalid Prefetch node!"); 1383 switch (TLI.getOperationAction(ISD::PREFETCH, MVT::Other)) { 1384 default: assert(0 && "This action is not supported yet!"); 1385 case TargetLowering::Legal: 1386 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1387 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the address. 1388 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the rw specifier. 1389 Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize locality specifier. 1390 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4); 1391 break; 1392 case TargetLowering::Expand: 1393 // It's a noop. 1394 Result = LegalizeOp(Node->getOperand(0)); 1395 break; 1396 } 1397 break; 1398 1399 case ISD::MEMBARRIER: { 1400 assert(Node->getNumOperands() == 6 && "Invalid MemBarrier node!"); 1401 switch (TLI.getOperationAction(ISD::MEMBARRIER, MVT::Other)) { 1402 default: assert(0 && "This action is not supported yet!"); 1403 case TargetLowering::Legal: { 1404 SDValue Ops[6]; 1405 Ops[0] = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1406 for (int x = 1; x < 6; ++x) { 1407 Ops[x] = Node->getOperand(x); 1408 if (!isTypeLegal(Ops[x].getValueType())) 1409 Ops[x] = PromoteOp(Ops[x]); 1410 } 1411 Result = DAG.UpdateNodeOperands(Result, &Ops[0], 6); 1412 break; 1413 } 1414 case TargetLowering::Expand: 1415 //There is no libgcc call for this op 1416 Result = Node->getOperand(0); // Noop 1417 break; 1418 } 1419 break; 1420 } 1421 1422 case ISD::ATOMIC_CMP_SWAP: { 1423 unsigned int num_operands = 4; 1424 assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!"); 1425 SDValue Ops[4]; 1426 for (unsigned int x = 0; x < num_operands; ++x) 1427 Ops[x] = LegalizeOp(Node->getOperand(x)); 1428 Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands); 1429 1430 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 1431 default: assert(0 && "This action is not supported yet!"); 1432 case TargetLowering::Custom: 1433 Result = TLI.LowerOperation(Result, DAG); 1434 break; 1435 case TargetLowering::Legal: 1436 break; 1437 } 1438 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1439 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1440 return Result.getValue(Op.getResNo()); 1441 } 1442 case ISD::ATOMIC_LOAD_ADD: 1443 case ISD::ATOMIC_LOAD_SUB: 1444 case ISD::ATOMIC_LOAD_AND: 1445 case ISD::ATOMIC_LOAD_OR: 1446 case ISD::ATOMIC_LOAD_XOR: 1447 case ISD::ATOMIC_LOAD_NAND: 1448 case ISD::ATOMIC_LOAD_MIN: 1449 case ISD::ATOMIC_LOAD_MAX: 1450 case ISD::ATOMIC_LOAD_UMIN: 1451 case ISD::ATOMIC_LOAD_UMAX: 1452 case ISD::ATOMIC_SWAP: { 1453 unsigned int num_operands = 3; 1454 assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!"); 1455 SDValue Ops[3]; 1456 for (unsigned int x = 0; x < num_operands; ++x) 1457 Ops[x] = LegalizeOp(Node->getOperand(x)); 1458 Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands); 1459 1460 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 1461 default: assert(0 && "This action is not supported yet!"); 1462 case TargetLowering::Custom: 1463 Result = TLI.LowerOperation(Result, DAG); 1464 break; 1465 case TargetLowering::Legal: 1466 break; 1467 } 1468 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1469 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1470 return Result.getValue(Op.getResNo()); 1471 } 1472 case ISD::Constant: { 1473 ConstantSDNode *CN = cast<ConstantSDNode>(Node); 1474 unsigned opAction = 1475 TLI.getOperationAction(ISD::Constant, CN->getValueType(0)); 1476 1477 // We know we don't need to expand constants here, constants only have one 1478 // value and we check that it is fine above. 1479 1480 if (opAction == TargetLowering::Custom) { 1481 Tmp1 = TLI.LowerOperation(Result, DAG); 1482 if (Tmp1.getNode()) 1483 Result = Tmp1; 1484 } 1485 break; 1486 } 1487 case ISD::ConstantFP: { 1488 // Spill FP immediates to the constant pool if the target cannot directly 1489 // codegen them. Targets often have some immediate values that can be 1490 // efficiently generated into an FP register without a load. We explicitly 1491 // leave these constants as ConstantFP nodes for the target to deal with. 1492 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 1493 1494 switch (TLI.getOperationAction(ISD::ConstantFP, CFP->getValueType(0))) { 1495 default: assert(0 && "This action is not supported yet!"); 1496 case TargetLowering::Legal: 1497 break; 1498 case TargetLowering::Custom: 1499 Tmp3 = TLI.LowerOperation(Result, DAG); 1500 if (Tmp3.getNode()) { 1501 Result = Tmp3; 1502 break; 1503 } 1504 // FALLTHROUGH 1505 case TargetLowering::Expand: { 1506 // Check to see if this FP immediate is already legal. 1507 bool isLegal = false; 1508 for (TargetLowering::legal_fpimm_iterator I = TLI.legal_fpimm_begin(), 1509 E = TLI.legal_fpimm_end(); I != E; ++I) { 1510 if (CFP->isExactlyValue(*I)) { 1511 isLegal = true; 1512 break; 1513 } 1514 } 1515 // If this is a legal constant, turn it into a TargetConstantFP node. 1516 if (isLegal) 1517 break; 1518 Result = ExpandConstantFP(CFP, true, DAG, TLI); 1519 } 1520 } 1521 break; 1522 } 1523 case ISD::TokenFactor: 1524 if (Node->getNumOperands() == 2) { 1525 Tmp1 = LegalizeOp(Node->getOperand(0)); 1526 Tmp2 = LegalizeOp(Node->getOperand(1)); 1527 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1528 } else if (Node->getNumOperands() == 3) { 1529 Tmp1 = LegalizeOp(Node->getOperand(0)); 1530 Tmp2 = LegalizeOp(Node->getOperand(1)); 1531 Tmp3 = LegalizeOp(Node->getOperand(2)); 1532 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1533 } else { 1534 SmallVector<SDValue, 8> Ops; 1535 // Legalize the operands. 1536 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1537 Ops.push_back(LegalizeOp(Node->getOperand(i))); 1538 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1539 } 1540 break; 1541 1542 case ISD::FORMAL_ARGUMENTS: 1543 case ISD::CALL: 1544 // The only option for this is to custom lower it. 1545 Tmp3 = TLI.LowerOperation(Result.getValue(0), DAG); 1546 assert(Tmp3.getNode() && "Target didn't custom lower this node!"); 1547 // A call within a calling sequence must be legalized to something 1548 // other than the normal CALLSEQ_END. Violating this gets Legalize 1549 // into an infinite loop. 1550 assert ((!IsLegalizingCall || 1551 Node->getOpcode() != ISD::CALL || 1552 Tmp3.getNode()->getOpcode() != ISD::CALLSEQ_END) && 1553 "Nested CALLSEQ_START..CALLSEQ_END not supported."); 1554 1555 // The number of incoming and outgoing values should match; unless the final 1556 // outgoing value is a flag. 1557 assert((Tmp3.getNode()->getNumValues() == Result.getNode()->getNumValues() || 1558 (Tmp3.getNode()->getNumValues() == Result.getNode()->getNumValues() + 1 && 1559 Tmp3.getNode()->getValueType(Tmp3.getNode()->getNumValues() - 1) == 1560 MVT::Flag)) && 1561 "Lowering call/formal_arguments produced unexpected # results!"); 1562 1563 // Since CALL/FORMAL_ARGUMENTS nodes produce multiple values, make sure to 1564 // remember that we legalized all of them, so it doesn't get relegalized. 1565 for (unsigned i = 0, e = Tmp3.getNode()->getNumValues(); i != e; ++i) { 1566 if (Tmp3.getNode()->getValueType(i) == MVT::Flag) 1567 continue; 1568 Tmp1 = LegalizeOp(Tmp3.getValue(i)); 1569 if (Op.getResNo() == i) 1570 Tmp2 = Tmp1; 1571 AddLegalizedOperand(SDValue(Node, i), Tmp1); 1572 } 1573 return Tmp2; 1574 case ISD::BUILD_VECTOR: 1575 switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) { 1576 default: assert(0 && "This action is not supported yet!"); 1577 case TargetLowering::Custom: 1578 Tmp3 = TLI.LowerOperation(Result, DAG); 1579 if (Tmp3.getNode()) { 1580 Result = Tmp3; 1581 break; 1582 } 1583 // FALLTHROUGH 1584 case TargetLowering::Expand: 1585 Result = ExpandBUILD_VECTOR(Result.getNode()); 1586 break; 1587 } 1588 break; 1589 case ISD::INSERT_VECTOR_ELT: 1590 Tmp1 = LegalizeOp(Node->getOperand(0)); // InVec 1591 Tmp3 = LegalizeOp(Node->getOperand(2)); // InEltNo 1592 1593 // The type of the value to insert may not be legal, even though the vector 1594 // type is legal. Legalize/Promote accordingly. We do not handle Expand 1595 // here. 1596 switch (getTypeAction(Node->getOperand(1).getValueType())) { 1597 default: assert(0 && "Cannot expand insert element operand"); 1598 case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; 1599 case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; 1600 case Expand: 1601 // FIXME: An alternative would be to check to see if the target is not 1602 // going to custom lower this operation, we could bitcast to half elt 1603 // width and perform two inserts at that width, if that is legal. 1604 Tmp2 = Node->getOperand(1); 1605 break; 1606 } 1607 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1608 1609 switch (TLI.getOperationAction(ISD::INSERT_VECTOR_ELT, 1610 Node->getValueType(0))) { 1611 default: assert(0 && "This action is not supported yet!"); 1612 case TargetLowering::Legal: 1613 break; 1614 case TargetLowering::Custom: 1615 Tmp4 = TLI.LowerOperation(Result, DAG); 1616 if (Tmp4.getNode()) { 1617 Result = Tmp4; 1618 break; 1619 } 1620 // FALLTHROUGH 1621 case TargetLowering::Promote: 1622 // Fall thru for vector case 1623 case TargetLowering::Expand: { 1624 // If the insert index is a constant, codegen this as a scalar_to_vector, 1625 // then a shuffle that inserts it into the right position in the vector. 1626 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Tmp3)) { 1627 // SCALAR_TO_VECTOR requires that the type of the value being inserted 1628 // match the element type of the vector being created, except for 1629 // integers in which case the inserted value can be over width. 1630 MVT EltVT = Op.getValueType().getVectorElementType(); 1631 if (Tmp2.getValueType() == EltVT || 1632 (EltVT.isInteger() && Tmp2.getValueType().bitsGE(EltVT))) { 1633 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 1634 Tmp1.getValueType(), Tmp2); 1635 1636 unsigned NumElts = Tmp1.getValueType().getVectorNumElements(); 1637 // We generate a shuffle of InVec and ScVec, so the shuffle mask 1638 // should be 0,1,2,3,4,5... with the appropriate element replaced with 1639 // elt 0 of the RHS. 1640 SmallVector<int, 8> ShufOps; 1641 for (unsigned i = 0; i != NumElts; ++i) 1642 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); 1643 1644 Result = DAG.getVectorShuffle(Tmp1.getValueType(), dl, Tmp1, ScVec, 1645 &ShufOps[0]); 1646 Result = LegalizeOp(Result); 1647 break; 1648 } 1649 } 1650 Result = PerformInsertVectorEltInMemory(Tmp1, Tmp2, Tmp3, dl); 1651 break; 1652 } 1653 } 1654 break; 1655 case ISD::SCALAR_TO_VECTOR: 1656 if (!TLI.isTypeLegal(Node->getOperand(0).getValueType())) { 1657 Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node)); 1658 break; 1659 } 1660 1661 Tmp1 = LegalizeOp(Node->getOperand(0)); // InVal 1662 Result = DAG.UpdateNodeOperands(Result, Tmp1); 1663 switch (TLI.getOperationAction(ISD::SCALAR_TO_VECTOR, 1664 Node->getValueType(0))) { 1665 default: assert(0 && "This action is not supported yet!"); 1666 case TargetLowering::Legal: 1667 break; 1668 case TargetLowering::Custom: 1669 Tmp3 = TLI.LowerOperation(Result, DAG); 1670 if (Tmp3.getNode()) { 1671 Result = Tmp3; 1672 break; 1673 } 1674 // FALLTHROUGH 1675 case TargetLowering::Expand: 1676 Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node)); 1677 break; 1678 } 1679 break; 1680 case ISD::VECTOR_SHUFFLE: { 1681 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input vectors, 1682 Tmp2 = LegalizeOp(Node->getOperand(1)); // but not the shuffle mask. 1683 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1684 MVT VT = Result.getValueType(); 1685 1686 // Copy the Mask to a local SmallVector for use with isShuffleMaskLegal. 1687 SmallVector<int, 8> Mask; 1688 cast<ShuffleVectorSDNode>(Result)->getMask(Mask); 1689 1690 // Allow targets to custom lower the SHUFFLEs they support. 1691 switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) { 1692 default: assert(0 && "Unknown operation action!"); 1693 case TargetLowering::Legal: 1694 assert(TLI.isShuffleMaskLegal(Mask, VT) && 1695 "vector shuffle should not be created if not legal!"); 1696 break; 1697 case TargetLowering::Custom: 1698 Tmp3 = TLI.LowerOperation(Result, DAG); 1699 if (Tmp3.getNode()) { 1700 Result = Tmp3; 1701 break; 1702 } 1703 // FALLTHROUGH 1704 case TargetLowering::Expand: { 1705 MVT EltVT = VT.getVectorElementType(); 1706 unsigned NumElems = VT.getVectorNumElements(); 1707 SmallVector<SDValue, 8> Ops; 1708 for (unsigned i = 0; i != NumElems; ++i) { 1709 if (Mask[i] < 0) { 1710 Ops.push_back(DAG.getUNDEF(EltVT)); 1711 continue; 1712 } 1713 unsigned Idx = Mask[i]; 1714 if (Idx < NumElems) 1715 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp1, 1716 DAG.getIntPtrConstant(Idx))); 1717 else 1718 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp2, 1719 DAG.getIntPtrConstant(Idx - NumElems))); 1720 } 1721 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size()); 1722 break; 1723 } 1724 case TargetLowering::Promote: { 1725 // Change base type to a different vector type. 1726 MVT OVT = Node->getValueType(0); 1727 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 1728 1729 // Cast the two input vectors. 1730 Tmp1 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp1); 1731 Tmp2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp2); 1732 1733 // Convert the shuffle mask to the right # elements. 1734 Result = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); 1735 Result = DAG.getNode(ISD::BIT_CONVERT, dl, OVT, Result); 1736 break; 1737 } 1738 } 1739 break; 1740 } 1741 case ISD::EXTRACT_VECTOR_ELT: 1742 Tmp1 = Node->getOperand(0); 1743 Tmp2 = LegalizeOp(Node->getOperand(1)); 1744 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1745 Result = ExpandEXTRACT_VECTOR_ELT(Result); 1746 break; 1747 1748 case ISD::EXTRACT_SUBVECTOR: 1749 Tmp1 = Node->getOperand(0); 1750 Tmp2 = LegalizeOp(Node->getOperand(1)); 1751 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1752 Result = ExpandEXTRACT_SUBVECTOR(Result); 1753 break; 1754 1755 case ISD::CONCAT_VECTORS: { 1756 // Use extract/insert/build vector for now. We might try to be 1757 // more clever later. 1758 MVT PtrVT = TLI.getPointerTy(); 1759 SmallVector<SDValue, 8> Ops; 1760 unsigned NumOperands = Node->getNumOperands(); 1761 for (unsigned i=0; i < NumOperands; ++i) { 1762 SDValue SubOp = Node->getOperand(i); 1763 MVT VVT = SubOp.getNode()->getValueType(0); 1764 MVT EltVT = VVT.getVectorElementType(); 1765 unsigned NumSubElem = VVT.getVectorNumElements(); 1766 for (unsigned j=0; j < NumSubElem; ++j) { 1767 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp, 1768 DAG.getConstant(j, PtrVT))); 1769 } 1770 } 1771 return LegalizeOp(DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), 1772 &Ops[0], Ops.size())); 1773 } 1774 1775 case ISD::CALLSEQ_START: { 1776 SDNode *CallEnd = FindCallEndFromCallStart(Node); 1777 1778 // Recursively Legalize all of the inputs of the call end that do not lead 1779 // to this call start. This ensures that any libcalls that need be inserted 1780 // are inserted *before* the CALLSEQ_START. 1781 IsLegalizingCallArgs = true; 1782 {SmallPtrSet<SDNode*, 32> NodesLeadingTo; 1783 for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i) 1784 LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).getNode(), Node, 1785 NodesLeadingTo); 1786 } 1787 IsLegalizingCallArgs = false; 1788 1789 // Now that we legalized all of the inputs (which may have inserted 1790 // libcalls) create the new CALLSEQ_START node. 1791 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1792 1793 // Merge in the last call, to ensure that this call start after the last 1794 // call ended. 1795 if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken) { 1796 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1797 Tmp1, LastCALLSEQ_END); 1798 Tmp1 = LegalizeOp(Tmp1); 1799 } 1800 1801 // Do not try to legalize the target-specific arguments (#1+). 1802 if (Tmp1 != Node->getOperand(0)) { 1803 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1804 Ops[0] = Tmp1; 1805 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1806 } 1807 1808 // Remember that the CALLSEQ_START is legalized. 1809 AddLegalizedOperand(Op.getValue(0), Result); 1810 if (Node->getNumValues() == 2) // If this has a flag result, remember it. 1811 AddLegalizedOperand(Op.getValue(1), Result.getValue(1)); 1812 1813 // Now that the callseq_start and all of the non-call nodes above this call 1814 // sequence have been legalized, legalize the call itself. During this 1815 // process, no libcalls can/will be inserted, guaranteeing that no calls 1816 // can overlap. 1817 assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!"); 1818 // Note that we are selecting this call! 1819 LastCALLSEQ_END = SDValue(CallEnd, 0); 1820 IsLegalizingCall = true; 1821 1822 // Legalize the call, starting from the CALLSEQ_END. 1823 LegalizeOp(LastCALLSEQ_END); 1824 assert(!IsLegalizingCall && "CALLSEQ_END should have cleared this!"); 1825 return Result; 1826 } 1827 case ISD::CALLSEQ_END: 1828 // If the CALLSEQ_START node hasn't been legalized first, legalize it. This 1829 // will cause this node to be legalized as well as handling libcalls right. 1830 if (LastCALLSEQ_END.getNode() != Node) { 1831 LegalizeOp(SDValue(FindCallStartFromCallEnd(Node), 0)); 1832 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 1833 assert(I != LegalizedNodes.end() && 1834 "Legalizing the call start should have legalized this node!"); 1835 return I->second; 1836 } 1837 1838 // Otherwise, the call start has been legalized and everything is going 1839 // according to plan. Just legalize ourselves normally here. 1840 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1841 // Do not try to legalize the target-specific arguments (#1+), except for 1842 // an optional flag input. 1843 if (Node->getOperand(Node->getNumOperands()-1).getValueType() != MVT::Flag){ 1844 if (Tmp1 != Node->getOperand(0)) { 1845 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1846 Ops[0] = Tmp1; 1847 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1848 } 1849 } else { 1850 Tmp2 = LegalizeOp(Node->getOperand(Node->getNumOperands()-1)); 1851 if (Tmp1 != Node->getOperand(0) || 1852 Tmp2 != Node->getOperand(Node->getNumOperands()-1)) { 1853 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1854 Ops[0] = Tmp1; 1855 Ops.back() = Tmp2; 1856 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1857 } 1858 } 1859 assert(IsLegalizingCall && "Call sequence imbalance between start/end?"); 1860 // This finishes up call legalization. 1861 IsLegalizingCall = false; 1862 1863 // If the CALLSEQ_END node has a flag, remember that we legalized it. 1864 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1865 if (Node->getNumValues() == 2) 1866 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1867 return Result.getValue(Op.getResNo()); 1868 case ISD::DYNAMIC_STACKALLOC: { 1869 MVT VT = Node->getValueType(0); 1870 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1871 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the size. 1872 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the alignment. 1873 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 1874 1875 Tmp1 = Result.getValue(0); 1876 Tmp2 = Result.getValue(1); 1877 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 1878 default: assert(0 && "This action is not supported yet!"); 1879 case TargetLowering::Expand: { 1880 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1881 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1882 " not tell us which reg is the stack pointer!"); 1883 SDValue Chain = Tmp1.getOperand(0); 1884 1885 // Chain the dynamic stack allocation so that it doesn't modify the stack 1886 // pointer when other instructions are using the stack. 1887 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); 1888 1889 SDValue Size = Tmp2.getOperand(1); 1890 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1891 Chain = SP.getValue(1); 1892 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1893 unsigned StackAlign = 1894 TLI.getTargetMachine().getFrameInfo()->getStackAlignment(); 1895 if (Align > StackAlign) 1896 SP = DAG.getNode(ISD::AND, dl, VT, SP, 1897 DAG.getConstant(-(uint64_t)Align, VT)); 1898 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value 1899 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1900 1901 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), 1902 DAG.getIntPtrConstant(0, true), SDValue()); 1903 1904 Tmp1 = LegalizeOp(Tmp1); 1905 Tmp2 = LegalizeOp(Tmp2); 1906 break; 1907 } 1908 case TargetLowering::Custom: 1909 Tmp3 = TLI.LowerOperation(Tmp1, DAG); 1910 if (Tmp3.getNode()) { 1911 Tmp1 = LegalizeOp(Tmp3); 1912 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 1913 } 1914 break; 1915 case TargetLowering::Legal: 1916 break; 1917 } 1918 // Since this op produce two values, make sure to remember that we 1919 // legalized both of them. 1920 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 1921 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 1922 return Op.getResNo() ? Tmp2 : Tmp1; 1923 } 1924 case ISD::INLINEASM: { 1925 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end()); 1926 bool Changed = false; 1927 // Legalize all of the operands of the inline asm, in case they are nodes 1928 // that need to be expanded or something. Note we skip the asm string and 1929 // all of the TargetConstant flags. 1930 SDValue Op = LegalizeOp(Ops[0]); 1931 Changed = Op != Ops[0]; 1932 Ops[0] = Op; 1933 1934 bool HasInFlag = Ops.back().getValueType() == MVT::Flag; 1935 for (unsigned i = 2, e = Ops.size()-HasInFlag; i < e; ) { 1936 unsigned NumVals = InlineAsm:: 1937 getNumOperandRegisters(cast<ConstantSDNode>(Ops[i])->getZExtValue()); 1938 for (++i; NumVals; ++i, --NumVals) { 1939 SDValue Op = LegalizeOp(Ops[i]); 1940 if (Op != Ops[i]) { 1941 Changed = true; 1942 Ops[i] = Op; 1943 } 1944 } 1945 } 1946 1947 if (HasInFlag) { 1948 Op = LegalizeOp(Ops.back()); 1949 Changed |= Op != Ops.back(); 1950 Ops.back() = Op; 1951 } 1952 1953 if (Changed) 1954 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 1955 1956 // INLINE asm returns a chain and flag, make sure to add both to the map. 1957 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 1958 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 1959 return Result.getValue(Op.getResNo()); 1960 } 1961 case ISD::BR: 1962 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1963 // Ensure that libcalls are emitted before a branch. 1964 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 1965 Tmp1 = LegalizeOp(Tmp1); 1966 LastCALLSEQ_END = DAG.getEntryNode(); 1967 1968 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 1969 break; 1970 case ISD::BRIND: 1971 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1972 // Ensure that libcalls are emitted before a branch. 1973 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 1974 Tmp1 = LegalizeOp(Tmp1); 1975 LastCALLSEQ_END = DAG.getEntryNode(); 1976 1977 switch (getTypeAction(Node->getOperand(1).getValueType())) { 1978 default: assert(0 && "Indirect target must be legal type (pointer)!"); 1979 case Legal: 1980 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition. 1981 break; 1982 } 1983 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 1984 break; 1985 case ISD::BR_JT: 1986 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 1987 // Ensure that libcalls are emitted before a branch. 1988 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 1989 Tmp1 = LegalizeOp(Tmp1); 1990 LastCALLSEQ_END = DAG.getEntryNode(); 1991 1992 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the jumptable node. 1993 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 1994 1995 switch (TLI.getOperationAction(ISD::BR_JT, MVT::Other)) { 1996 default: assert(0 && "This action is not supported yet!"); 1997 case TargetLowering::Legal: break; 1998 case TargetLowering::Custom: 1999 Tmp1 = TLI.LowerOperation(Result, DAG); 2000 if (Tmp1.getNode()) Result = Tmp1; 2001 break; 2002 case TargetLowering::Expand: { 2003 SDValue Chain = Result.getOperand(0); 2004 SDValue Table = Result.getOperand(1); 2005 SDValue Index = Result.getOperand(2); 2006 2007 MVT PTy = TLI.getPointerTy(); 2008 MachineFunction &MF = DAG.getMachineFunction(); 2009 unsigned EntrySize = MF.getJumpTableInfo()->getEntrySize(); 2010 Index= DAG.getNode(ISD::MUL, dl, PTy, 2011 Index, DAG.getConstant(EntrySize, PTy)); 2012 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table); 2013 2014 MVT MemVT = MVT::getIntegerVT(EntrySize * 8); 2015 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, 2016 PseudoSourceValue::getJumpTable(), 0, MemVT); 2017 Addr = LD; 2018 if (TLI.getTargetMachine().getRelocationModel() == Reloc::PIC_) { 2019 // For PIC, the sequence is: 2020 // BRIND(load(Jumptable + index) + RelocBase) 2021 // RelocBase can be JumpTable, GOT or some sort of global base. 2022 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 2023 TLI.getPICJumpTableRelocBase(Table, DAG)); 2024 } 2025 Result = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); 2026 } 2027 } 2028 break; 2029 case ISD::BRCOND: 2030 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2031 // Ensure that libcalls are emitted before a return. 2032 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 2033 Tmp1 = LegalizeOp(Tmp1); 2034 LastCALLSEQ_END = DAG.getEntryNode(); 2035 2036 switch (getTypeAction(Node->getOperand(1).getValueType())) { 2037 case Expand: assert(0 && "It's impossible to expand bools"); 2038 case Legal: 2039 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition. 2040 break; 2041 case Promote: { 2042 Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the condition. 2043 2044 // The top bits of the promoted condition are not necessarily zero, ensure 2045 // that the value is properly zero extended. 2046 unsigned BitWidth = Tmp2.getValueSizeInBits(); 2047 if (!DAG.MaskedValueIsZero(Tmp2, 2048 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 2049 Tmp2 = DAG.getZeroExtendInReg(Tmp2, dl, MVT::i1); 2050 break; 2051 } 2052 } 2053 2054 // Basic block destination (Op#2) is always legal. 2055 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 2056 2057 switch (TLI.getOperationAction(ISD::BRCOND, MVT::Other)) { 2058 default: assert(0 && "This action is not supported yet!"); 2059 case TargetLowering::Legal: break; 2060 case TargetLowering::Custom: 2061 Tmp1 = TLI.LowerOperation(Result, DAG); 2062 if (Tmp1.getNode()) Result = Tmp1; 2063 break; 2064 case TargetLowering::Expand: 2065 // Expand brcond's setcc into its constituent parts and create a BR_CC 2066 // Node. 2067 if (Tmp2.getOpcode() == ISD::SETCC) { 2068 Result = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 2069 Tmp1, Tmp2.getOperand(2), 2070 Tmp2.getOperand(0), Tmp2.getOperand(1), 2071 Node->getOperand(2)); 2072 } else { 2073 Result = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 2074 DAG.getCondCode(ISD::SETNE), Tmp2, 2075 DAG.getConstant(0, Tmp2.getValueType()), 2076 Node->getOperand(2)); 2077 } 2078 break; 2079 } 2080 break; 2081 case ISD::BR_CC: 2082 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2083 // Ensure that libcalls are emitted before a branch. 2084 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 2085 Tmp1 = LegalizeOp(Tmp1); 2086 Tmp2 = Node->getOperand(2); // LHS 2087 Tmp3 = Node->getOperand(3); // RHS 2088 Tmp4 = Node->getOperand(1); // CC 2089 2090 LegalizeSetCC(TLI.getSetCCResultType(Tmp2.getValueType()), 2091 Tmp2, Tmp3, Tmp4, dl); 2092 LastCALLSEQ_END = DAG.getEntryNode(); 2093 2094 // If we didn't get both a LHS and RHS back from LegalizeSetCC, 2095 // the LHS is a legal SETCC itself. In this case, we need to compare 2096 // the result against zero to select between true and false values. 2097 if (Tmp3.getNode() == 0) { 2098 Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); 2099 Tmp4 = DAG.getCondCode(ISD::SETNE); 2100 } 2101 2102 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp4, Tmp2, Tmp3, 2103 Node->getOperand(4)); 2104 2105 switch (TLI.getOperationAction(ISD::BR_CC, Tmp3.getValueType())) { 2106 default: assert(0 && "Unexpected action for BR_CC!"); 2107 case TargetLowering::Legal: break; 2108 case TargetLowering::Custom: 2109 Tmp4 = TLI.LowerOperation(Result, DAG); 2110 if (Tmp4.getNode()) Result = Tmp4; 2111 break; 2112 } 2113 break; 2114 case ISD::LOAD: { 2115 LoadSDNode *LD = cast<LoadSDNode>(Node); 2116 Tmp1 = LegalizeOp(LD->getChain()); // Legalize the chain. 2117 Tmp2 = LegalizeOp(LD->getBasePtr()); // Legalize the base pointer. 2118 2119 ISD::LoadExtType ExtType = LD->getExtensionType(); 2120 if (ExtType == ISD::NON_EXTLOAD) { 2121 MVT VT = Node->getValueType(0); 2122 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset()); 2123 Tmp3 = Result.getValue(0); 2124 Tmp4 = Result.getValue(1); 2125 2126 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 2127 default: assert(0 && "This action is not supported yet!"); 2128 case TargetLowering::Legal: 2129 // If this is an unaligned load and the target doesn't support it, 2130 // expand it. 2131 if (!TLI.allowsUnalignedMemoryAccesses()) { 2132 unsigned ABIAlignment = TLI.getTargetData()-> 2133 getABITypeAlignment(LD->getMemoryVT().getTypeForMVT()); 2134 if (LD->getAlignment() < ABIAlignment){ 2135 Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()), DAG, 2136 TLI); 2137 Tmp3 = Result.getOperand(0); 2138 Tmp4 = Result.getOperand(1); 2139 Tmp3 = LegalizeOp(Tmp3); 2140 Tmp4 = LegalizeOp(Tmp4); 2141 } 2142 } 2143 break; 2144 case TargetLowering::Custom: 2145 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 2146 if (Tmp1.getNode()) { 2147 Tmp3 = LegalizeOp(Tmp1); 2148 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 2149 } 2150 break; 2151 case TargetLowering::Promote: { 2152 // Only promote a load of vector type to another. 2153 assert(VT.isVector() && "Cannot promote this load!"); 2154 // Change base type to a different vector type. 2155 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 2156 2157 Tmp1 = DAG.getLoad(NVT, dl, Tmp1, Tmp2, LD->getSrcValue(), 2158 LD->getSrcValueOffset(), 2159 LD->isVolatile(), LD->getAlignment()); 2160 Tmp3 = LegalizeOp(DAG.getNode(ISD::BIT_CONVERT, dl, VT, Tmp1)); 2161 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 2162 break; 2163 } 2164 } 2165 // Since loads produce two values, make sure to remember that we 2166 // legalized both of them. 2167 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 2168 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 2169 return Op.getResNo() ? Tmp4 : Tmp3; 2170 } else { 2171 MVT SrcVT = LD->getMemoryVT(); 2172 unsigned SrcWidth = SrcVT.getSizeInBits(); 2173 int SVOffset = LD->getSrcValueOffset(); 2174 unsigned Alignment = LD->getAlignment(); 2175 bool isVolatile = LD->isVolatile(); 2176 2177 if (SrcWidth != SrcVT.getStoreSizeInBits() && 2178 // Some targets pretend to have an i1 loading operation, and actually 2179 // load an i8. This trick is correct for ZEXTLOAD because the top 7 2180 // bits are guaranteed to be zero; it helps the optimizers understand 2181 // that these bits are zero. It is also useful for EXTLOAD, since it 2182 // tells the optimizers that those bits are undefined. It would be 2183 // nice to have an effective generic way of getting these benefits... 2184 // Until such a way is found, don't insist on promoting i1 here. 2185 (SrcVT != MVT::i1 || 2186 TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { 2187 // Promote to a byte-sized load if not loading an integral number of 2188 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 2189 unsigned NewWidth = SrcVT.getStoreSizeInBits(); 2190 MVT NVT = MVT::getIntegerVT(NewWidth); 2191 SDValue Ch; 2192 2193 // The extra bits are guaranteed to be zero, since we stored them that 2194 // way. A zext load from NVT thus automatically gives zext from SrcVT. 2195 2196 ISD::LoadExtType NewExtType = 2197 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 2198 2199 Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), 2200 Tmp1, Tmp2, LD->getSrcValue(), SVOffset, 2201 NVT, isVolatile, Alignment); 2202 2203 Ch = Result.getValue(1); // The chain. 2204 2205 if (ExtType == ISD::SEXTLOAD) 2206 // Having the top bits zero doesn't help when sign extending. 2207 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 2208 Result.getValueType(), 2209 Result, DAG.getValueType(SrcVT)); 2210 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 2211 // All the top bits are guaranteed to be zero - inform the optimizers. 2212 Result = DAG.getNode(ISD::AssertZext, dl, 2213 Result.getValueType(), Result, 2214 DAG.getValueType(SrcVT)); 2215 2216 Tmp1 = LegalizeOp(Result); 2217 Tmp2 = LegalizeOp(Ch); 2218 } else if (SrcWidth & (SrcWidth - 1)) { 2219 // If not loading a power-of-2 number of bits, expand as two loads. 2220 assert(SrcVT.isExtended() && !SrcVT.isVector() && 2221 "Unsupported extload!"); 2222 unsigned RoundWidth = 1 << Log2_32(SrcWidth); 2223 assert(RoundWidth < SrcWidth); 2224 unsigned ExtraWidth = SrcWidth - RoundWidth; 2225 assert(ExtraWidth < RoundWidth); 2226 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 2227 "Load size not an integral number of bytes!"); 2228 MVT RoundVT = MVT::getIntegerVT(RoundWidth); 2229 MVT ExtraVT = MVT::getIntegerVT(ExtraWidth); 2230 SDValue Lo, Hi, Ch; 2231 unsigned IncrementSize; 2232 2233 if (TLI.isLittleEndian()) { 2234 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 2235 // Load the bottom RoundWidth bits. 2236 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, 2237 Node->getValueType(0), Tmp1, Tmp2, 2238 LD->getSrcValue(), SVOffset, RoundVT, isVolatile, 2239 Alignment); 2240 2241 // Load the remaining ExtraWidth bits. 2242 IncrementSize = RoundWidth / 8; 2243 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2244 DAG.getIntPtrConstant(IncrementSize)); 2245 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Tmp1, Tmp2, 2246 LD->getSrcValue(), SVOffset + IncrementSize, 2247 ExtraVT, isVolatile, 2248 MinAlign(Alignment, IncrementSize)); 2249 2250 // Build a factor node to remember that this load is independent of the 2251 // other one. 2252 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2253 Hi.getValue(1)); 2254 2255 // Move the top bits to the right place. 2256 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 2257 DAG.getConstant(RoundWidth, TLI.getShiftAmountTy())); 2258 2259 // Join the hi and lo parts. 2260 Result = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 2261 } else { 2262 // Big endian - avoid unaligned loads. 2263 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 2264 // Load the top RoundWidth bits. 2265 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Tmp1, Tmp2, 2266 LD->getSrcValue(), SVOffset, RoundVT, isVolatile, 2267 Alignment); 2268 2269 // Load the remaining ExtraWidth bits. 2270 IncrementSize = RoundWidth / 8; 2271 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2272 DAG.getIntPtrConstant(IncrementSize)); 2273 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, 2274 Node->getValueType(0), Tmp1, Tmp2, 2275 LD->getSrcValue(), SVOffset + IncrementSize, 2276 ExtraVT, isVolatile, 2277 MinAlign(Alignment, IncrementSize)); 2278 2279 // Build a factor node to remember that this load is independent of the 2280 // other one. 2281 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2282 Hi.getValue(1)); 2283 2284 // Move the top bits to the right place. 2285 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 2286 DAG.getConstant(ExtraWidth, TLI.getShiftAmountTy())); 2287 2288 // Join the hi and lo parts. 2289 Result = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 2290 } 2291 2292 Tmp1 = LegalizeOp(Result); 2293 Tmp2 = LegalizeOp(Ch); 2294 } else { 2295 switch (TLI.getLoadExtAction(ExtType, SrcVT)) { 2296 default: assert(0 && "This action is not supported yet!"); 2297 case TargetLowering::Custom: 2298 isCustom = true; 2299 // FALLTHROUGH 2300 case TargetLowering::Legal: 2301 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset()); 2302 Tmp1 = Result.getValue(0); 2303 Tmp2 = Result.getValue(1); 2304 2305 if (isCustom) { 2306 Tmp3 = TLI.LowerOperation(Result, DAG); 2307 if (Tmp3.getNode()) { 2308 Tmp1 = LegalizeOp(Tmp3); 2309 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 2310 } 2311 } else { 2312 // If this is an unaligned load and the target doesn't support it, 2313 // expand it. 2314 if (!TLI.allowsUnalignedMemoryAccesses()) { 2315 unsigned ABIAlignment = TLI.getTargetData()-> 2316 getABITypeAlignment(LD->getMemoryVT().getTypeForMVT()); 2317 if (LD->getAlignment() < ABIAlignment){ 2318 Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()), DAG, 2319 TLI); 2320 Tmp1 = Result.getOperand(0); 2321 Tmp2 = Result.getOperand(1); 2322 Tmp1 = LegalizeOp(Tmp1); 2323 Tmp2 = LegalizeOp(Tmp2); 2324 } 2325 } 2326 } 2327 break; 2328 case TargetLowering::Expand: 2329 // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND 2330 if (SrcVT == MVT::f32 && Node->getValueType(0) == MVT::f64) { 2331 SDValue Load = DAG.getLoad(SrcVT, dl, Tmp1, Tmp2, LD->getSrcValue(), 2332 LD->getSrcValueOffset(), 2333 LD->isVolatile(), LD->getAlignment()); 2334 Result = DAG.getNode(ISD::FP_EXTEND, dl, 2335 Node->getValueType(0), Load); 2336 Tmp1 = LegalizeOp(Result); // Relegalize new nodes. 2337 Tmp2 = LegalizeOp(Load.getValue(1)); 2338 break; 2339 } 2340 assert(ExtType != ISD::EXTLOAD &&"EXTLOAD should always be supported!"); 2341 // Turn the unsupported load into an EXTLOAD followed by an explicit 2342 // zero/sign extend inreg. 2343 Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0), 2344 Tmp1, Tmp2, LD->getSrcValue(), 2345 LD->getSrcValueOffset(), SrcVT, 2346 LD->isVolatile(), LD->getAlignment()); 2347 SDValue ValRes; 2348 if (ExtType == ISD::SEXTLOAD) 2349 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 2350 Result.getValueType(), 2351 Result, DAG.getValueType(SrcVT)); 2352 else 2353 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT); 2354 Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes. 2355 Tmp2 = LegalizeOp(Result.getValue(1)); // Relegalize new nodes. 2356 break; 2357 } 2358 } 2359 2360 // Since loads produce two values, make sure to remember that we legalized 2361 // both of them. 2362 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2363 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2364 return Op.getResNo() ? Tmp2 : Tmp1; 2365 } 2366 } 2367 case ISD::EXTRACT_ELEMENT: { 2368 MVT OpTy = Node->getOperand(0).getValueType(); 2369 switch (getTypeAction(OpTy)) { 2370 default: assert(0 && "EXTRACT_ELEMENT action for type unimplemented!"); 2371 case Legal: 2372 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 2373 // 1 -> Hi 2374 Result = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 2375 DAG.getConstant(OpTy.getSizeInBits()/2, 2376 TLI.getShiftAmountTy())); 2377 Result = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); 2378 } else { 2379 // 0 -> Lo 2380 Result = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 2381 Node->getOperand(0)); 2382 } 2383 break; 2384 case Expand: 2385 // Get both the low and high parts. 2386 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 2387 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) 2388 Result = Tmp2; // 1 -> Hi 2389 else 2390 Result = Tmp1; // 0 -> Lo 2391 break; 2392 } 2393 break; 2394 } 2395 2396 case ISD::CopyToReg: 2397 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2398 2399 assert(isTypeLegal(Node->getOperand(2).getValueType()) && 2400 "Register type must be legal!"); 2401 // Legalize the incoming value (must be a legal type). 2402 Tmp2 = LegalizeOp(Node->getOperand(2)); 2403 if (Node->getNumValues() == 1) { 2404 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2); 2405 } else { 2406 assert(Node->getNumValues() == 2 && "Unknown CopyToReg"); 2407 if (Node->getNumOperands() == 4) { 2408 Tmp3 = LegalizeOp(Node->getOperand(3)); 2409 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2, 2410 Tmp3); 2411 } else { 2412 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2); 2413 } 2414 2415 // Since this produces two values, make sure to remember that we legalized 2416 // both of them. 2417 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); 2418 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); 2419 return Result; 2420 } 2421 break; 2422 2423 case ISD::RET: 2424 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2425 2426 // Ensure that libcalls are emitted before a return. 2427 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1, LastCALLSEQ_END); 2428 Tmp1 = LegalizeOp(Tmp1); 2429 LastCALLSEQ_END = DAG.getEntryNode(); 2430 2431 switch (Node->getNumOperands()) { 2432 case 3: // ret val 2433 Tmp2 = Node->getOperand(1); 2434 Tmp3 = Node->getOperand(2); // Signness 2435 switch (getTypeAction(Tmp2.getValueType())) { 2436 case Legal: 2437 Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3); 2438 break; 2439 case Expand: 2440 if (!Tmp2.getValueType().isVector()) { 2441 SDValue Lo, Hi; 2442 ExpandOp(Tmp2, Lo, Hi); 2443 2444 // Big endian systems want the hi reg first. 2445 if (TLI.isBigEndian()) 2446 std::swap(Lo, Hi); 2447 2448 if (Hi.getNode()) 2449 Result = DAG.getNode(ISD::RET, dl, MVT::Other, 2450 Tmp1, Lo, Tmp3, Hi, Tmp3); 2451 else 2452 Result = DAG.getNode(ISD::RET, dl, MVT::Other, Tmp1, Lo, Tmp3); 2453 Result = LegalizeOp(Result); 2454 } else { 2455 SDNode *InVal = Tmp2.getNode(); 2456 int InIx = Tmp2.getResNo(); 2457 unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements(); 2458 MVT EVT = InVal->getValueType(InIx).getVectorElementType(); 2459 2460 // Figure out if there is a simple type corresponding to this Vector 2461 // type. If so, convert to the vector type. 2462 MVT TVT = MVT::getVectorVT(EVT, NumElems); 2463 if (TLI.isTypeLegal(TVT)) { 2464 // Turn this into a return of the vector type. 2465 Tmp2 = LegalizeOp(Tmp2); 2466 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2467 } else if (NumElems == 1) { 2468 // Turn this into a return of the scalar type. 2469 Tmp2 = ScalarizeVectorOp(Tmp2); 2470 Tmp2 = LegalizeOp(Tmp2); 2471 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2472 2473 // FIXME: Returns of gcc generic vectors smaller than a legal type 2474 // should be returned in integer registers! 2475 2476 // The scalarized value type may not be legal, e.g. it might require 2477 // promotion or expansion. Relegalize the return. 2478 Result = LegalizeOp(Result); 2479 } else { 2480 // FIXME: Returns of gcc generic vectors larger than a legal vector 2481 // type should be returned by reference! 2482 SDValue Lo, Hi; 2483 SplitVectorOp(Tmp2, Lo, Hi); 2484 Result = DAG.getNode(ISD::RET, dl, MVT::Other, 2485 Tmp1, Lo, Tmp3, Hi, Tmp3); 2486 Result = LegalizeOp(Result); 2487 } 2488 } 2489 break; 2490 case Promote: 2491 Tmp2 = PromoteOp(Node->getOperand(1)); 2492 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2493 Result = LegalizeOp(Result); 2494 break; 2495 } 2496 break; 2497 case 1: // ret void 2498 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2499 break; 2500 default: { // ret <values> 2501 SmallVector<SDValue, 8> NewValues; 2502 NewValues.push_back(Tmp1); 2503 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) 2504 switch (getTypeAction(Node->getOperand(i).getValueType())) { 2505 case Legal: 2506 NewValues.push_back(LegalizeOp(Node->getOperand(i))); 2507 NewValues.push_back(Node->getOperand(i+1)); 2508 break; 2509 case Expand: { 2510 SDValue Lo, Hi; 2511 assert(!Node->getOperand(i).getValueType().isExtended() && 2512 "FIXME: TODO: implement returning non-legal vector types!"); 2513 ExpandOp(Node->getOperand(i), Lo, Hi); 2514 NewValues.push_back(Lo); 2515 NewValues.push_back(Node->getOperand(i+1)); 2516 if (Hi.getNode()) { 2517 NewValues.push_back(Hi); 2518 NewValues.push_back(Node->getOperand(i+1)); 2519 } 2520 break; 2521 } 2522 case Promote: 2523 assert(0 && "Can't promote multiple return value yet!"); 2524 } 2525 2526 if (NewValues.size() == Node->getNumOperands()) 2527 Result = DAG.UpdateNodeOperands(Result, &NewValues[0],NewValues.size()); 2528 else 2529 Result = DAG.getNode(ISD::RET, dl, MVT::Other, 2530 &NewValues[0], NewValues.size()); 2531 break; 2532 } 2533 } 2534 2535 if (Result.getOpcode() == ISD::RET) { 2536 switch (TLI.getOperationAction(Result.getOpcode(), MVT::Other)) { 2537 default: assert(0 && "This action is not supported yet!"); 2538 case TargetLowering::Legal: break; 2539 case TargetLowering::Custom: 2540 Tmp1 = TLI.LowerOperation(Result, DAG); 2541 if (Tmp1.getNode()) Result = Tmp1; 2542 break; 2543 } 2544 } 2545 break; 2546 case ISD::STORE: { 2547 StoreSDNode *ST = cast<StoreSDNode>(Node); 2548 Tmp1 = LegalizeOp(ST->getChain()); // Legalize the chain. 2549 Tmp2 = LegalizeOp(ST->getBasePtr()); // Legalize the pointer. 2550 int SVOffset = ST->getSrcValueOffset(); 2551 unsigned Alignment = ST->getAlignment(); 2552 bool isVolatile = ST->isVolatile(); 2553 2554 if (!ST->isTruncatingStore()) { 2555 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 2556 // FIXME: We shouldn't do this for TargetConstantFP's. 2557 // FIXME: move this to the DAG Combiner! Note that we can't regress due 2558 // to phase ordering between legalized code and the dag combiner. This 2559 // probably means that we need to integrate dag combiner and legalizer 2560 // together. 2561 // We generally can't do this one for long doubles. 2562 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 2563 if (CFP->getValueType(0) == MVT::f32 && 2564 getTypeAction(MVT::i32) == Legal) { 2565 Tmp3 = DAG.getConstant(CFP->getValueAPF(). 2566 bitcastToAPInt().zextOrTrunc(32), 2567 MVT::i32); 2568 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2569 SVOffset, isVolatile, Alignment); 2570 break; 2571 } else if (CFP->getValueType(0) == MVT::f64) { 2572 // If this target supports 64-bit registers, do a single 64-bit store. 2573 if (getTypeAction(MVT::i64) == Legal) { 2574 Tmp3 = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 2575 zextOrTrunc(64), MVT::i64); 2576 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2577 SVOffset, isVolatile, Alignment); 2578 break; 2579 } else if (getTypeAction(MVT::i32) == Legal && !ST->isVolatile()) { 2580 // Otherwise, if the target supports 32-bit registers, use 2 32-bit 2581 // stores. If the target supports neither 32- nor 64-bits, this 2582 // xform is certainly not worth it. 2583 const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); 2584 SDValue Lo = DAG.getConstant(APInt(IntVal).trunc(32), MVT::i32); 2585 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); 2586 if (TLI.isBigEndian()) std::swap(Lo, Hi); 2587 2588 Lo = DAG.getStore(Tmp1, dl, Lo, Tmp2, ST->getSrcValue(), 2589 SVOffset, isVolatile, Alignment); 2590 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2591 DAG.getIntPtrConstant(4)); 2592 Hi = DAG.getStore(Tmp1, dl, Hi, Tmp2, ST->getSrcValue(), SVOffset+4, 2593 isVolatile, MinAlign(Alignment, 4U)); 2594 2595 Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2596 break; 2597 } 2598 } 2599 } 2600 2601 switch (getTypeAction(ST->getMemoryVT())) { 2602 case Legal: { 2603 Tmp3 = LegalizeOp(ST->getValue()); 2604 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 2605 ST->getOffset()); 2606 2607 MVT VT = Tmp3.getValueType(); 2608 switch (TLI.getOperationAction(ISD::STORE, VT)) { 2609 default: assert(0 && "This action is not supported yet!"); 2610 case TargetLowering::Legal: 2611 // If this is an unaligned store and the target doesn't support it, 2612 // expand it. 2613 if (!TLI.allowsUnalignedMemoryAccesses()) { 2614 unsigned ABIAlignment = TLI.getTargetData()-> 2615 getABITypeAlignment(ST->getMemoryVT().getTypeForMVT()); 2616 if (ST->getAlignment() < ABIAlignment) 2617 Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()), DAG, 2618 TLI); 2619 } 2620 break; 2621 case TargetLowering::Custom: 2622 Tmp1 = TLI.LowerOperation(Result, DAG); 2623 if (Tmp1.getNode()) Result = Tmp1; 2624 break; 2625 case TargetLowering::Promote: 2626 assert(VT.isVector() && "Unknown legal promote case!"); 2627 Tmp3 = DAG.getNode(ISD::BIT_CONVERT, dl, 2628 TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3); 2629 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, 2630 ST->getSrcValue(), SVOffset, isVolatile, 2631 Alignment); 2632 break; 2633 } 2634 break; 2635 } 2636 case Promote: 2637 if (!ST->getMemoryVT().isVector()) { 2638 // Truncate the value and store the result. 2639 Tmp3 = PromoteOp(ST->getValue()); 2640 Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2641 SVOffset, ST->getMemoryVT(), 2642 isVolatile, Alignment); 2643 break; 2644 } 2645 // Fall thru to expand for vector 2646 case Expand: { 2647 unsigned IncrementSize = 0; 2648 SDValue Lo, Hi; 2649 2650 // If this is a vector type, then we have to calculate the increment as 2651 // the product of the element size in bytes, and the number of elements 2652 // in the high half of the vector. 2653 if (ST->getValue().getValueType().isVector()) { 2654 SDNode *InVal = ST->getValue().getNode(); 2655 int InIx = ST->getValue().getResNo(); 2656 MVT InVT = InVal->getValueType(InIx); 2657 unsigned NumElems = InVT.getVectorNumElements(); 2658 MVT EVT = InVT.getVectorElementType(); 2659 2660 // Figure out if there is a simple type corresponding to this Vector 2661 // type. If so, convert to the vector type. 2662 MVT TVT = MVT::getVectorVT(EVT, NumElems); 2663 if (TLI.isTypeLegal(TVT)) { 2664 // Turn this into a normal store of the vector type. 2665 Tmp3 = LegalizeOp(ST->getValue()); 2666 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2667 SVOffset, isVolatile, Alignment); 2668 Result = LegalizeOp(Result); 2669 break; 2670 } else if (NumElems == 1) { 2671 // Turn this into a normal store of the scalar type. 2672 Tmp3 = ScalarizeVectorOp(ST->getValue()); 2673 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2674 SVOffset, isVolatile, Alignment); 2675 // The scalarized value type may not be legal, e.g. it might require 2676 // promotion or expansion. Relegalize the scalar store. 2677 Result = LegalizeOp(Result); 2678 break; 2679 } else { 2680 // Check if we have widen this node with another value 2681 std::map<SDValue, SDValue>::iterator I = 2682 WidenNodes.find(ST->getValue()); 2683 if (I != WidenNodes.end()) { 2684 Result = StoreWidenVectorOp(ST, Tmp1, Tmp2); 2685 break; 2686 } 2687 else { 2688 SplitVectorOp(ST->getValue(), Lo, Hi); 2689 IncrementSize = Lo.getNode()->getValueType(0).getVectorNumElements() * 2690 EVT.getSizeInBits()/8; 2691 } 2692 } 2693 } else { 2694 ExpandOp(ST->getValue(), Lo, Hi); 2695 IncrementSize = Hi.getNode() ? Hi.getValueType().getSizeInBits()/8 : 0; 2696 2697 if (Hi.getNode() && TLI.isBigEndian()) 2698 std::swap(Lo, Hi); 2699 } 2700 2701 Lo = DAG.getStore(Tmp1, dl, Lo, Tmp2, ST->getSrcValue(), 2702 SVOffset, isVolatile, Alignment); 2703 2704 if (Hi.getNode() == NULL) { 2705 // Must be int <-> float one-to-one expansion. 2706 Result = Lo; 2707 break; 2708 } 2709 2710 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2711 DAG.getIntPtrConstant(IncrementSize)); 2712 assert(isTypeLegal(Tmp2.getValueType()) && 2713 "Pointers must be legal!"); 2714 SVOffset += IncrementSize; 2715 Alignment = MinAlign(Alignment, IncrementSize); 2716 Hi = DAG.getStore(Tmp1, dl, Hi, Tmp2, ST->getSrcValue(), 2717 SVOffset, isVolatile, Alignment); 2718 Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2719 break; 2720 } // case Expand 2721 } 2722 } else { 2723 switch (getTypeAction(ST->getValue().getValueType())) { 2724 case Legal: 2725 Tmp3 = LegalizeOp(ST->getValue()); 2726 break; 2727 case Promote: 2728 if (!ST->getValue().getValueType().isVector()) { 2729 // We can promote the value, the truncstore will still take care of it. 2730 Tmp3 = PromoteOp(ST->getValue()); 2731 break; 2732 } 2733 // Vector case falls through to expand 2734 case Expand: 2735 // Just store the low part. This may become a non-trunc store, so make 2736 // sure to use getTruncStore, not UpdateNodeOperands below. 2737 ExpandOp(ST->getValue(), Tmp3, Tmp4); 2738 return DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2739 SVOffset, MVT::i8, isVolatile, Alignment); 2740 } 2741 2742 MVT StVT = ST->getMemoryVT(); 2743 unsigned StWidth = StVT.getSizeInBits(); 2744 2745 if (StWidth != StVT.getStoreSizeInBits()) { 2746 // Promote to a byte-sized store with upper bits zero if not 2747 // storing an integral number of bytes. For example, promote 2748 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 2749 MVT NVT = MVT::getIntegerVT(StVT.getStoreSizeInBits()); 2750 Tmp3 = DAG.getZeroExtendInReg(Tmp3, dl, StVT); 2751 Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2752 SVOffset, NVT, isVolatile, Alignment); 2753 } else if (StWidth & (StWidth - 1)) { 2754 // If not storing a power-of-2 number of bits, expand as two stores. 2755 assert(StVT.isExtended() && !StVT.isVector() && 2756 "Unsupported truncstore!"); 2757 unsigned RoundWidth = 1 << Log2_32(StWidth); 2758 assert(RoundWidth < StWidth); 2759 unsigned ExtraWidth = StWidth - RoundWidth; 2760 assert(ExtraWidth < RoundWidth); 2761 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 2762 "Store size not an integral number of bytes!"); 2763 MVT RoundVT = MVT::getIntegerVT(RoundWidth); 2764 MVT ExtraVT = MVT::getIntegerVT(ExtraWidth); 2765 SDValue Lo, Hi; 2766 unsigned IncrementSize; 2767 2768 if (TLI.isLittleEndian()) { 2769 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 2770 // Store the bottom RoundWidth bits. 2771 Lo = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2772 SVOffset, RoundVT, 2773 isVolatile, Alignment); 2774 2775 // Store the remaining ExtraWidth bits. 2776 IncrementSize = RoundWidth / 8; 2777 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2778 DAG.getIntPtrConstant(IncrementSize)); 2779 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3, 2780 DAG.getConstant(RoundWidth, TLI.getShiftAmountTy())); 2781 Hi = DAG.getTruncStore(Tmp1, dl, Hi, Tmp2, ST->getSrcValue(), 2782 SVOffset + IncrementSize, ExtraVT, isVolatile, 2783 MinAlign(Alignment, IncrementSize)); 2784 } else { 2785 // Big endian - avoid unaligned stores. 2786 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 2787 // Store the top RoundWidth bits. 2788 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3, 2789 DAG.getConstant(ExtraWidth, TLI.getShiftAmountTy())); 2790 Hi = DAG.getTruncStore(Tmp1, dl, Hi, Tmp2, ST->getSrcValue(), 2791 SVOffset, RoundVT, isVolatile, Alignment); 2792 2793 // Store the remaining ExtraWidth bits. 2794 IncrementSize = RoundWidth / 8; 2795 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, 2796 DAG.getIntPtrConstant(IncrementSize)); 2797 Lo = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2798 SVOffset + IncrementSize, ExtraVT, isVolatile, 2799 MinAlign(Alignment, IncrementSize)); 2800 } 2801 2802 // The order of the stores doesn't matter. 2803 Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2804 } else { 2805 if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() || 2806 Tmp2 != ST->getBasePtr()) 2807 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 2808 ST->getOffset()); 2809 2810 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { 2811 default: assert(0 && "This action is not supported yet!"); 2812 case TargetLowering::Legal: 2813 // If this is an unaligned store and the target doesn't support it, 2814 // expand it. 2815 if (!TLI.allowsUnalignedMemoryAccesses()) { 2816 unsigned ABIAlignment = TLI.getTargetData()-> 2817 getABITypeAlignment(ST->getMemoryVT().getTypeForMVT()); 2818 if (ST->getAlignment() < ABIAlignment) 2819 Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()), DAG, 2820 TLI); 2821 } 2822 break; 2823 case TargetLowering::Custom: 2824 Result = TLI.LowerOperation(Result, DAG); 2825 break; 2826 case Expand: 2827 // TRUNCSTORE:i16 i32 -> STORE i16 2828 assert(isTypeLegal(StVT) && "Do not know how to expand this store!"); 2829 Tmp3 = DAG.getNode(ISD::TRUNCATE, dl, StVT, Tmp3); 2830 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), 2831 SVOffset, isVolatile, Alignment); 2832 break; 2833 } 2834 } 2835 } 2836 break; 2837 } 2838 case ISD::PCMARKER: 2839 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2840 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 2841 break; 2842 case ISD::STACKSAVE: 2843 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2844 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2845 Tmp1 = Result.getValue(0); 2846 Tmp2 = Result.getValue(1); 2847 2848 switch (TLI.getOperationAction(ISD::STACKSAVE, MVT::Other)) { 2849 default: assert(0 && "This action is not supported yet!"); 2850 case TargetLowering::Legal: break; 2851 case TargetLowering::Custom: 2852 Tmp3 = TLI.LowerOperation(Result, DAG); 2853 if (Tmp3.getNode()) { 2854 Tmp1 = LegalizeOp(Tmp3); 2855 Tmp2 = LegalizeOp(Tmp3.getValue(1)); 2856 } 2857 break; 2858 case TargetLowering::Expand: 2859 // Expand to CopyFromReg if the target set 2860 // StackPointerRegisterToSaveRestore. 2861 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 2862 Tmp1 = DAG.getCopyFromReg(Result.getOperand(0), dl, SP, 2863 Node->getValueType(0)); 2864 Tmp2 = Tmp1.getValue(1); 2865 } else { 2866 Tmp1 = DAG.getUNDEF(Node->getValueType(0)); 2867 Tmp2 = Node->getOperand(0); 2868 } 2869 break; 2870 } 2871 2872 // Since stacksave produce two values, make sure to remember that we 2873 // legalized both of them. 2874 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2875 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2876 return Op.getResNo() ? Tmp2 : Tmp1; 2877 2878 case ISD::STACKRESTORE: 2879 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 2880 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 2881 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 2882 2883 switch (TLI.getOperationAction(ISD::STACKRESTORE, MVT::Other)) { 2884 default: assert(0 && "This action is not supported yet!"); 2885 case TargetLowering::Legal: break; 2886 case TargetLowering::Custom: 2887 Tmp1 = TLI.LowerOperation(Result, DAG); 2888 if (Tmp1.getNode()) Result = Tmp1; 2889 break; 2890 case TargetLowering::Expand: 2891 // Expand to CopyToReg if the target set 2892 // StackPointerRegisterToSaveRestore. 2893 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 2894 Result = DAG.getCopyToReg(Tmp1, dl, SP, Tmp2); 2895 } else { 2896 Result = Tmp1; 2897 } 2898 break; 2899 } 2900 break; 2901 2902 case ISD::READCYCLECOUNTER: 2903 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain 2904 Result = DAG.UpdateNodeOperands(Result, Tmp1); 2905 switch (TLI.getOperationAction(ISD::READCYCLECOUNTER, 2906 Node->getValueType(0))) { 2907 default: assert(0 && "This action is not supported yet!"); 2908 case TargetLowering::Legal: 2909 Tmp1 = Result.getValue(0); 2910 Tmp2 = Result.getValue(1); 2911 break; 2912 case TargetLowering::Custom: 2913 Result = TLI.LowerOperation(Result, DAG); 2914 Tmp1 = LegalizeOp(Result.getValue(0)); 2915 Tmp2 = LegalizeOp(Result.getValue(1)); 2916 break; 2917 } 2918 2919 // Since rdcc produce two values, make sure to remember that we legalized 2920 // both of them. 2921 AddLegalizedOperand(SDValue(Node, 0), Tmp1); 2922 AddLegalizedOperand(SDValue(Node, 1), Tmp2); 2923 return Result; 2924 2925 case ISD::SELECT: 2926 switch (getTypeAction(Node->getOperand(0).getValueType())) { 2927 case Expand: assert(0 && "It's impossible to expand bools"); 2928 case Legal: 2929 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition. 2930 break; 2931 case Promote: { 2932 assert(!Node->getOperand(0).getValueType().isVector() && "not possible"); 2933 Tmp1 = PromoteOp(Node->getOperand(0)); // Promote the condition. 2934 // Make sure the condition is either zero or one. 2935 unsigned BitWidth = Tmp1.getValueSizeInBits(); 2936 if (!DAG.MaskedValueIsZero(Tmp1, 2937 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 2938 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, MVT::i1); 2939 break; 2940 } 2941 } 2942 Tmp2 = LegalizeOp(Node->getOperand(1)); // TrueVal 2943 Tmp3 = LegalizeOp(Node->getOperand(2)); // FalseVal 2944 2945 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 2946 2947 switch (TLI.getOperationAction(ISD::SELECT, Tmp2.getValueType())) { 2948 default: assert(0 && "This action is not supported yet!"); 2949 case TargetLowering::Legal: break; 2950 case TargetLowering::Custom: { 2951 Tmp1 = TLI.LowerOperation(Result, DAG); 2952 if (Tmp1.getNode()) Result = Tmp1; 2953 break; 2954 } 2955 case TargetLowering::Expand: 2956 if (Tmp1.getOpcode() == ISD::SETCC) { 2957 Result = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 2958 Tmp2, Tmp3, 2959 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 2960 } else { 2961 Result = DAG.getSelectCC(dl, Tmp1, 2962 DAG.getConstant(0, Tmp1.getValueType()), 2963 Tmp2, Tmp3, ISD::SETNE); 2964 } 2965 break; 2966 case TargetLowering::Promote: { 2967 MVT NVT = 2968 TLI.getTypeToPromoteTo(ISD::SELECT, Tmp2.getValueType()); 2969 unsigned ExtOp, TruncOp; 2970 if (Tmp2.getValueType().isVector()) { 2971 ExtOp = ISD::BIT_CONVERT; 2972 TruncOp = ISD::BIT_CONVERT; 2973 } else if (Tmp2.getValueType().isInteger()) { 2974 ExtOp = ISD::ANY_EXTEND; 2975 TruncOp = ISD::TRUNCATE; 2976 } else { 2977 ExtOp = ISD::FP_EXTEND; 2978 TruncOp = ISD::FP_ROUND; 2979 } 2980 // Promote each of the values to the new type. 2981 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Tmp2); 2982 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Tmp3); 2983 // Perform the larger operation, then round down. 2984 Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2, Tmp3); 2985 if (TruncOp != ISD::FP_ROUND) 2986 Result = DAG.getNode(TruncOp, dl, Node->getValueType(0), Result); 2987 else 2988 Result = DAG.getNode(TruncOp, dl, Node->getValueType(0), Result, 2989 DAG.getIntPtrConstant(0)); 2990 break; 2991 } 2992 } 2993 break; 2994 case ISD::SELECT_CC: { 2995 Tmp1 = Node->getOperand(0); // LHS 2996 Tmp2 = Node->getOperand(1); // RHS 2997 Tmp3 = LegalizeOp(Node->getOperand(2)); // True 2998 Tmp4 = LegalizeOp(Node->getOperand(3)); // False 2999 SDValue CC = Node->getOperand(4); 3000 3001 LegalizeSetCC(TLI.getSetCCResultType(Tmp1.getValueType()), 3002 Tmp1, Tmp2, CC, dl); 3003 3004 // If we didn't get both a LHS and RHS back from LegalizeSetCC, 3005 // the LHS is a legal SETCC itself. In this case, we need to compare 3006 // the result against zero to select between true and false values. 3007 if (Tmp2.getNode() == 0) { 3008 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 3009 CC = DAG.getCondCode(ISD::SETNE); 3010 } 3011 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, CC); 3012 3013 // Everything is legal, see if we should expand this op or something. 3014 switch (TLI.getOperationAction(ISD::SELECT_CC, Tmp3.getValueType())) { 3015 default: assert(0 && "This action is not supported yet!"); 3016 case TargetLowering::Legal: break; 3017 case TargetLowering::Custom: 3018 Tmp1 = TLI.LowerOperation(Result, DAG); 3019 if (Tmp1.getNode()) Result = Tmp1; 3020 break; 3021 } 3022 break; 3023 } 3024 case ISD::SETCC: 3025 Tmp1 = Node->getOperand(0); 3026 Tmp2 = Node->getOperand(1); 3027 Tmp3 = Node->getOperand(2); 3028 LegalizeSetCC(Node->getValueType(0), Tmp1, Tmp2, Tmp3, dl); 3029 3030 // If we had to Expand the SetCC operands into a SELECT node, then it may 3031 // not always be possible to return a true LHS & RHS. In this case, just 3032 // return the value we legalized, returned in the LHS 3033 if (Tmp2.getNode() == 0) { 3034 Result = Tmp1; 3035 break; 3036 } 3037 3038 switch (TLI.getOperationAction(ISD::SETCC, Tmp1.getValueType())) { 3039 default: assert(0 && "Cannot handle this action for SETCC yet!"); 3040 case TargetLowering::Custom: 3041 isCustom = true; 3042 // FALLTHROUGH. 3043 case TargetLowering::Legal: 3044 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 3045 if (isCustom) { 3046 Tmp4 = TLI.LowerOperation(Result, DAG); 3047 if (Tmp4.getNode()) Result = Tmp4; 3048 } 3049 break; 3050 case TargetLowering::Promote: { 3051 // First step, figure out the appropriate operation to use. 3052 // Allow SETCC to not be supported for all legal data types 3053 // Mostly this targets FP 3054 MVT NewInTy = Node->getOperand(0).getValueType(); 3055 MVT OldVT = NewInTy; OldVT = OldVT; 3056 3057 // Scan for the appropriate larger type to use. 3058 while (1) { 3059 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT()+1); 3060 3061 assert(NewInTy.isInteger() == OldVT.isInteger() && 3062 "Fell off of the edge of the integer world"); 3063 assert(NewInTy.isFloatingPoint() == OldVT.isFloatingPoint() && 3064 "Fell off of the edge of the floating point world"); 3065 3066 // If the target supports SETCC of this type, use it. 3067 if (TLI.isOperationLegalOrCustom(ISD::SETCC, NewInTy)) 3068 break; 3069 } 3070 if (NewInTy.isInteger()) 3071 assert(0 && "Cannot promote Legal Integer SETCC yet"); 3072 else { 3073 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewInTy, Tmp1); 3074 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NewInTy, Tmp2); 3075 } 3076 Tmp1 = LegalizeOp(Tmp1); 3077 Tmp2 = LegalizeOp(Tmp2); 3078 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 3079 Result = LegalizeOp(Result); 3080 break; 3081 } 3082 case TargetLowering::Expand: 3083 // Expand a setcc node into a select_cc of the same condition, lhs, and 3084 // rhs that selects between const 1 (true) and const 0 (false). 3085 MVT VT = Node->getValueType(0); 3086 Result = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3087 DAG.getConstant(1, VT), DAG.getConstant(0, VT), 3088 Tmp3); 3089 break; 3090 } 3091 break; 3092 case ISD::VSETCC: { 3093 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3094 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3095 SDValue CC = Node->getOperand(2); 3096 3097 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, CC); 3098 3099 // Everything is legal, see if we should expand this op or something. 3100 switch (TLI.getOperationAction(ISD::VSETCC, Tmp1.getValueType())) { 3101 default: assert(0 && "This action is not supported yet!"); 3102 case TargetLowering::Legal: break; 3103 case TargetLowering::Custom: 3104 Tmp1 = TLI.LowerOperation(Result, DAG); 3105 if (Tmp1.getNode()) Result = Tmp1; 3106 break; 3107 case TargetLowering::Expand: { 3108 // Unroll into a nasty set of scalar code for now. 3109 MVT VT = Node->getValueType(0); 3110 unsigned NumElems = VT.getVectorNumElements(); 3111 MVT EltVT = VT.getVectorElementType(); 3112 MVT TmpEltVT = Tmp1.getValueType().getVectorElementType(); 3113 SmallVector<SDValue, 8> Ops(NumElems); 3114 for (unsigned i = 0; i < NumElems; ++i) { 3115 SDValue In1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, 3116 Tmp1, DAG.getIntPtrConstant(i)); 3117 Ops[i] = DAG.getNode(ISD::SETCC, dl, TLI.getSetCCResultType(TmpEltVT), 3118 In1, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3119 TmpEltVT, Tmp2, 3120 DAG.getIntPtrConstant(i)), 3121 CC); 3122 Ops[i] = DAG.getNode(ISD::SELECT, dl, EltVT, Ops[i], 3123 DAG.getConstant(APInt::getAllOnesValue 3124 (EltVT.getSizeInBits()), EltVT), 3125 DAG.getConstant(0, EltVT)); 3126 } 3127 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElems); 3128 break; 3129 } 3130 } 3131 break; 3132 } 3133 3134 case ISD::SHL_PARTS: 3135 case ISD::SRA_PARTS: 3136 case ISD::SRL_PARTS: { 3137 SmallVector<SDValue, 8> Ops; 3138 bool Changed = false; 3139 unsigned N = Node->getNumOperands(); 3140 for (unsigned i = 0; i + 1 < N; ++i) { 3141 Ops.push_back(LegalizeOp(Node->getOperand(i))); 3142 Changed |= Ops.back() != Node->getOperand(i); 3143 } 3144 Ops.push_back(LegalizeOp(DAG.getShiftAmountOperand(Node->getOperand(N-1)))); 3145 Changed |= Ops.back() != Node->getOperand(N-1); 3146 if (Changed) 3147 Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); 3148 3149 switch (TLI.getOperationAction(Node->getOpcode(), 3150 Node->getValueType(0))) { 3151 default: assert(0 && "This action is not supported yet!"); 3152 case TargetLowering::Legal: break; 3153 case TargetLowering::Custom: 3154 Tmp1 = TLI.LowerOperation(Result, DAG); 3155 if (Tmp1.getNode()) { 3156 SDValue Tmp2, RetVal(0, 0); 3157 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { 3158 Tmp2 = LegalizeOp(Tmp1.getValue(i)); 3159 AddLegalizedOperand(SDValue(Node, i), Tmp2); 3160 if (i == Op.getResNo()) 3161 RetVal = Tmp2; 3162 } 3163 assert(RetVal.getNode() && "Illegal result number"); 3164 return RetVal; 3165 } 3166 break; 3167 } 3168 3169 // Since these produce multiple values, make sure to remember that we 3170 // legalized all of them. 3171 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 3172 AddLegalizedOperand(SDValue(Node, i), Result.getValue(i)); 3173 return Result.getValue(Op.getResNo()); 3174 } 3175 3176 // Binary operators 3177 case ISD::ADD: 3178 case ISD::SUB: 3179 case ISD::MUL: 3180 case ISD::MULHS: 3181 case ISD::MULHU: 3182 case ISD::UDIV: 3183 case ISD::SDIV: 3184 case ISD::AND: 3185 case ISD::OR: 3186 case ISD::XOR: 3187 case ISD::SHL: 3188 case ISD::SRL: 3189 case ISD::SRA: 3190 case ISD::FADD: 3191 case ISD::FSUB: 3192 case ISD::FMUL: 3193 case ISD::FDIV: 3194 case ISD::FPOW: 3195 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3196 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3197 3198 if ((Node->getOpcode() == ISD::SHL || 3199 Node->getOpcode() == ISD::SRL || 3200 Node->getOpcode() == ISD::SRA) && 3201 !Node->getValueType(0).isVector()) 3202 Tmp2 = DAG.getShiftAmountOperand(Tmp2); 3203 3204 switch (getTypeAction(Tmp2.getValueType())) { 3205 case Expand: assert(0 && "Not possible"); 3206 case Legal: 3207 Tmp2 = LegalizeOp(Tmp2); // Legalize the RHS. 3208 break; 3209 case Promote: 3210 Tmp2 = PromoteOp(Tmp2); // Promote the RHS. 3211 break; 3212 } 3213 3214 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3215 3216 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3217 default: assert(0 && "BinOp legalize operation not supported"); 3218 case TargetLowering::Legal: break; 3219 case TargetLowering::Custom: 3220 Tmp1 = TLI.LowerOperation(Result, DAG); 3221 if (Tmp1.getNode()) { 3222 Result = Tmp1; 3223 break; 3224 } 3225 // Fall through if the custom lower can't deal with the operation 3226 case TargetLowering::Expand: { 3227 MVT VT = Op.getValueType(); 3228 3229 // See if multiply or divide can be lowered using two-result operations. 3230 SDVTList VTs = DAG.getVTList(VT, VT); 3231 if (Node->getOpcode() == ISD::MUL) { 3232 // We just need the low half of the multiply; try both the signed 3233 // and unsigned forms. If the target supports both SMUL_LOHI and 3234 // UMUL_LOHI, form a preference by checking which forms of plain 3235 // MULH it supports. 3236 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3237 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3238 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3239 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3240 unsigned OpToUse = 0; 3241 if (HasSMUL_LOHI && !HasMULHS) { 3242 OpToUse = ISD::SMUL_LOHI; 3243 } else if (HasUMUL_LOHI && !HasMULHU) { 3244 OpToUse = ISD::UMUL_LOHI; 3245 } else if (HasSMUL_LOHI) { 3246 OpToUse = ISD::SMUL_LOHI; 3247 } else if (HasUMUL_LOHI) { 3248 OpToUse = ISD::UMUL_LOHI; 3249 } 3250 if (OpToUse) { 3251 Result = SDValue(DAG.getNode(OpToUse, dl, VTs, Tmp1, Tmp2).getNode(), 3252 0); 3253 break; 3254 } 3255 } 3256 if (Node->getOpcode() == ISD::MULHS && 3257 TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) { 3258 Result = SDValue(DAG.getNode(ISD::SMUL_LOHI, dl, 3259 VTs, Tmp1, Tmp2).getNode(), 3260 1); 3261 break; 3262 } 3263 if (Node->getOpcode() == ISD::MULHU && 3264 TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT)) { 3265 Result = SDValue(DAG.getNode(ISD::UMUL_LOHI, dl, 3266 VTs, Tmp1, Tmp2).getNode(), 3267 1); 3268 break; 3269 } 3270 if (Node->getOpcode() == ISD::SDIV && 3271 TLI.isOperationLegalOrCustom(ISD::SDIVREM, VT)) { 3272 Result = SDValue(DAG.getNode(ISD::SDIVREM, dl, 3273 VTs, Tmp1, Tmp2).getNode(), 3274 0); 3275 break; 3276 } 3277 if (Node->getOpcode() == ISD::UDIV && 3278 TLI.isOperationLegalOrCustom(ISD::UDIVREM, VT)) { 3279 Result = SDValue(DAG.getNode(ISD::UDIVREM, dl, 3280 VTs, Tmp1, Tmp2).getNode(), 3281 0); 3282 break; 3283 } 3284 3285 // Check to see if we have a libcall for this operator. 3286 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3287 bool isSigned = false; 3288 switch (Node->getOpcode()) { 3289 case ISD::UDIV: 3290 case ISD::SDIV: 3291 if (VT == MVT::i32) { 3292 LC = Node->getOpcode() == ISD::UDIV 3293 ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32; 3294 isSigned = Node->getOpcode() == ISD::SDIV; 3295 } 3296 break; 3297 case ISD::MUL: 3298 if (VT == MVT::i32) 3299 LC = RTLIB::MUL_I32; 3300 else if (VT == MVT::i64) 3301 LC = RTLIB::MUL_I64; 3302 break; 3303 case ISD::FPOW: 3304 LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, 3305 RTLIB::POW_PPCF128); 3306 break; 3307 case ISD::FDIV: 3308 LC = GetFPLibCall(VT, RTLIB::DIV_F32, RTLIB::DIV_F64, RTLIB::DIV_F80, 3309 RTLIB::DIV_PPCF128); 3310 break; 3311 default: break; 3312 } 3313 if (LC != RTLIB::UNKNOWN_LIBCALL) { 3314 SDValue Dummy; 3315 Result = ExpandLibCall(LC, Node, isSigned, Dummy); 3316 break; 3317 } 3318 3319 assert(Node->getValueType(0).isVector() && 3320 "Cannot expand this binary operator!"); 3321 // Expand the operation into a bunch of nasty scalar code. 3322 Result = LegalizeOp(UnrollVectorOp(Op)); 3323 break; 3324 } 3325 case TargetLowering::Promote: { 3326 switch (Node->getOpcode()) { 3327 default: assert(0 && "Do not know how to promote this BinOp!"); 3328 case ISD::AND: 3329 case ISD::OR: 3330 case ISD::XOR: { 3331 MVT OVT = Node->getValueType(0); 3332 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3333 assert(OVT.isVector() && "Cannot promote this BinOp!"); 3334 // Bit convert each of the values to the new type. 3335 Tmp1 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp1); 3336 Tmp2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp2); 3337 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3338 // Bit convert the result back the original type. 3339 Result = DAG.getNode(ISD::BIT_CONVERT, dl, OVT, Result); 3340 break; 3341 } 3342 } 3343 } 3344 } 3345 break; 3346 3347 case ISD::SMUL_LOHI: 3348 case ISD::UMUL_LOHI: 3349 case ISD::SDIVREM: 3350 case ISD::UDIVREM: 3351 // These nodes will only be produced by target-specific lowering, so 3352 // they shouldn't be here if they aren't legal. 3353 assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) && 3354 "This must be legal!"); 3355 3356 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3357 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3358 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3359 break; 3360 3361 case ISD::FCOPYSIGN: // FCOPYSIGN does not require LHS/RHS to match type! 3362 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3363 switch (getTypeAction(Node->getOperand(1).getValueType())) { 3364 case Expand: assert(0 && "Not possible"); 3365 case Legal: 3366 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS. 3367 break; 3368 case Promote: 3369 Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the RHS. 3370 break; 3371 } 3372 3373 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3374 3375 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3376 default: assert(0 && "Operation not supported"); 3377 case TargetLowering::Custom: 3378 Tmp1 = TLI.LowerOperation(Result, DAG); 3379 if (Tmp1.getNode()) Result = Tmp1; 3380 break; 3381 case TargetLowering::Legal: break; 3382 case TargetLowering::Expand: { 3383 // If this target supports fabs/fneg natively and select is cheap, 3384 // do this efficiently. 3385 if (!TLI.isSelectExpensive() && 3386 TLI.getOperationAction(ISD::FABS, Tmp1.getValueType()) == 3387 TargetLowering::Legal && 3388 TLI.getOperationAction(ISD::FNEG, Tmp1.getValueType()) == 3389 TargetLowering::Legal) { 3390 // Get the sign bit of the RHS. 3391 MVT IVT = 3392 Tmp2.getValueType() == MVT::f32 ? MVT::i32 : MVT::i64; 3393 SDValue SignBit = DAG.getNode(ISD::BIT_CONVERT, dl, IVT, Tmp2); 3394 SignBit = DAG.getSetCC(dl, TLI.getSetCCResultType(IVT), 3395 SignBit, DAG.getConstant(0, IVT), ISD::SETLT); 3396 // Get the absolute value of the result. 3397 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); 3398 // Select between the nabs and abs value based on the sign bit of 3399 // the input. 3400 Result = DAG.getNode(ISD::SELECT, dl, AbsVal.getValueType(), SignBit, 3401 DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), 3402 AbsVal), 3403 AbsVal); 3404 Result = LegalizeOp(Result); 3405 break; 3406 } 3407 3408 // Otherwise, do bitwise ops! 3409 MVT NVT = 3410 Node->getValueType(0) == MVT::f32 ? MVT::i32 : MVT::i64; 3411 Result = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI); 3412 Result = DAG.getNode(ISD::BIT_CONVERT, dl, Node->getValueType(0), Result); 3413 Result = LegalizeOp(Result); 3414 break; 3415 } 3416 } 3417 break; 3418 3419 case ISD::ADDC: 3420 case ISD::SUBC: 3421 Tmp1 = LegalizeOp(Node->getOperand(0)); 3422 Tmp2 = LegalizeOp(Node->getOperand(1)); 3423 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3424 Tmp3 = Result.getValue(0); 3425 Tmp4 = Result.getValue(1); 3426 3427 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3428 default: assert(0 && "This action is not supported yet!"); 3429 case TargetLowering::Legal: 3430 break; 3431 case TargetLowering::Custom: 3432 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 3433 if (Tmp1.getNode() != NULL) { 3434 Tmp3 = LegalizeOp(Tmp1); 3435 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 3436 } 3437 break; 3438 } 3439 // Since this produces two values, make sure to remember that we legalized 3440 // both of them. 3441 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 3442 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 3443 return Op.getResNo() ? Tmp4 : Tmp3; 3444 3445 case ISD::ADDE: 3446 case ISD::SUBE: 3447 Tmp1 = LegalizeOp(Node->getOperand(0)); 3448 Tmp2 = LegalizeOp(Node->getOperand(1)); 3449 Tmp3 = LegalizeOp(Node->getOperand(2)); 3450 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); 3451 Tmp3 = Result.getValue(0); 3452 Tmp4 = Result.getValue(1); 3453 3454 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3455 default: assert(0 && "This action is not supported yet!"); 3456 case TargetLowering::Legal: 3457 break; 3458 case TargetLowering::Custom: 3459 Tmp1 = TLI.LowerOperation(Tmp3, DAG); 3460 if (Tmp1.getNode() != NULL) { 3461 Tmp3 = LegalizeOp(Tmp1); 3462 Tmp4 = LegalizeOp(Tmp1.getValue(1)); 3463 } 3464 break; 3465 } 3466 // Since this produces two values, make sure to remember that we legalized 3467 // both of them. 3468 AddLegalizedOperand(SDValue(Node, 0), Tmp3); 3469 AddLegalizedOperand(SDValue(Node, 1), Tmp4); 3470 return Op.getResNo() ? Tmp4 : Tmp3; 3471 3472 case ISD::BUILD_PAIR: { 3473 MVT PairTy = Node->getValueType(0); 3474 // TODO: handle the case where the Lo and Hi operands are not of legal type 3475 Tmp1 = LegalizeOp(Node->getOperand(0)); // Lo 3476 Tmp2 = LegalizeOp(Node->getOperand(1)); // Hi 3477 switch (TLI.getOperationAction(ISD::BUILD_PAIR, PairTy)) { 3478 case TargetLowering::Promote: 3479 case TargetLowering::Custom: 3480 assert(0 && "Cannot promote/custom this yet!"); 3481 case TargetLowering::Legal: 3482 if (Tmp1 != Node->getOperand(0) || Tmp2 != Node->getOperand(1)) 3483 Result = DAG.getNode(ISD::BUILD_PAIR, dl, PairTy, Tmp1, Tmp2); 3484 break; 3485 case TargetLowering::Expand: 3486 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Tmp1); 3487 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Tmp2); 3488 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, 3489 DAG.getConstant(PairTy.getSizeInBits()/2, 3490 TLI.getShiftAmountTy())); 3491 Result = DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2); 3492 break; 3493 } 3494 break; 3495 } 3496 3497 case ISD::UREM: 3498 case ISD::SREM: 3499 case ISD::FREM: 3500 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3501 Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS 3502 3503 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3504 case TargetLowering::Promote: assert(0 && "Cannot promote this yet!"); 3505 case TargetLowering::Custom: 3506 isCustom = true; 3507 // FALLTHROUGH 3508 case TargetLowering::Legal: 3509 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3510 if (isCustom) { 3511 Tmp1 = TLI.LowerOperation(Result, DAG); 3512 if (Tmp1.getNode()) Result = Tmp1; 3513 } 3514 break; 3515 case TargetLowering::Expand: { 3516 unsigned DivOpc= (Node->getOpcode() == ISD::UREM) ? ISD::UDIV : ISD::SDIV; 3517 bool isSigned = DivOpc == ISD::SDIV; 3518 MVT VT = Node->getValueType(0); 3519 3520 // See if remainder can be lowered using two-result operations. 3521 SDVTList VTs = DAG.getVTList(VT, VT); 3522 if (Node->getOpcode() == ISD::SREM && 3523 TLI.isOperationLegalOrCustom(ISD::SDIVREM, VT)) { 3524 Result = SDValue(DAG.getNode(ISD::SDIVREM, dl, 3525 VTs, Tmp1, Tmp2).getNode(), 1); 3526 break; 3527 } 3528 if (Node->getOpcode() == ISD::UREM && 3529 TLI.isOperationLegalOrCustom(ISD::UDIVREM, VT)) { 3530 Result = SDValue(DAG.getNode(ISD::UDIVREM, dl, 3531 VTs, Tmp1, Tmp2).getNode(), 1); 3532 break; 3533 } 3534 3535 if (VT.isInteger()) { 3536 if (TLI.getOperationAction(DivOpc, VT) == 3537 TargetLowering::Legal) { 3538 // X % Y -> X-X/Y*Y 3539 Result = DAG.getNode(DivOpc, dl, VT, Tmp1, Tmp2); 3540 Result = DAG.getNode(ISD::MUL, dl, VT, Result, Tmp2); 3541 Result = DAG.getNode(ISD::SUB, dl, VT, Tmp1, Result); 3542 } else if (VT.isVector()) { 3543 Result = LegalizeOp(UnrollVectorOp(Op)); 3544 } else { 3545 assert(VT == MVT::i32 && 3546 "Cannot expand this binary operator!"); 3547 RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM 3548 ? RTLIB::UREM_I32 : RTLIB::SREM_I32; 3549 SDValue Dummy; 3550 Result = ExpandLibCall(LC, Node, isSigned, Dummy); 3551 } 3552 } else { 3553 assert(VT.isFloatingPoint() && 3554 "remainder op must have integer or floating-point type"); 3555 if (VT.isVector()) { 3556 Result = LegalizeOp(UnrollVectorOp(Op)); 3557 } else { 3558 // Floating point mod -> fmod libcall. 3559 RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::REM_F32, RTLIB::REM_F64, 3560 RTLIB::REM_F80, RTLIB::REM_PPCF128); 3561 SDValue Dummy; 3562 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3563 } 3564 } 3565 break; 3566 } 3567 } 3568 break; 3569 case ISD::VAARG: { 3570 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3571 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3572 3573 MVT VT = Node->getValueType(0); 3574 switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) { 3575 default: assert(0 && "This action is not supported yet!"); 3576 case TargetLowering::Custom: 3577 isCustom = true; 3578 // FALLTHROUGH 3579 case TargetLowering::Legal: 3580 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3581 Result = Result.getValue(0); 3582 Tmp1 = Result.getValue(1); 3583 3584 if (isCustom) { 3585 Tmp2 = TLI.LowerOperation(Result, DAG); 3586 if (Tmp2.getNode()) { 3587 Result = LegalizeOp(Tmp2); 3588 Tmp1 = LegalizeOp(Tmp2.getValue(1)); 3589 } 3590 } 3591 break; 3592 case TargetLowering::Expand: { 3593 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 3594 SDValue VAList = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, V, 0); 3595 // Increment the pointer, VAList, to the next vaarg 3596 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 3597 DAG.getConstant(TLI.getTargetData()-> 3598 getTypePaddedSize(VT.getTypeForMVT()), 3599 TLI.getPointerTy())); 3600 // Store the incremented VAList to the legalized pointer 3601 Tmp3 = DAG.getStore(VAList.getValue(1), dl, Tmp3, Tmp2, V, 0); 3602 // Load the actual argument out of the pointer VAList 3603 Result = DAG.getLoad(VT, dl, Tmp3, VAList, NULL, 0); 3604 Tmp1 = LegalizeOp(Result.getValue(1)); 3605 Result = LegalizeOp(Result); 3606 break; 3607 } 3608 } 3609 // Since VAARG produces two values, make sure to remember that we 3610 // legalized both of them. 3611 AddLegalizedOperand(SDValue(Node, 0), Result); 3612 AddLegalizedOperand(SDValue(Node, 1), Tmp1); 3613 return Op.getResNo() ? Tmp1 : Result; 3614 } 3615 3616 case ISD::VACOPY: 3617 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3618 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the dest pointer. 3619 Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the source pointer. 3620 3621 switch (TLI.getOperationAction(ISD::VACOPY, MVT::Other)) { 3622 default: assert(0 && "This action is not supported yet!"); 3623 case TargetLowering::Custom: 3624 isCustom = true; 3625 // FALLTHROUGH 3626 case TargetLowering::Legal: 3627 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, 3628 Node->getOperand(3), Node->getOperand(4)); 3629 if (isCustom) { 3630 Tmp1 = TLI.LowerOperation(Result, DAG); 3631 if (Tmp1.getNode()) Result = Tmp1; 3632 } 3633 break; 3634 case TargetLowering::Expand: 3635 // This defaults to loading a pointer from the input and storing it to the 3636 // output, returning the chain. 3637 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 3638 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 3639 Tmp4 = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp3, VS, 0); 3640 Result = DAG.getStore(Tmp4.getValue(1), dl, Tmp4, Tmp2, VD, 0); 3641 break; 3642 } 3643 break; 3644 3645 case ISD::VAEND: 3646 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3647 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3648 3649 switch (TLI.getOperationAction(ISD::VAEND, MVT::Other)) { 3650 default: assert(0 && "This action is not supported yet!"); 3651 case TargetLowering::Custom: 3652 isCustom = true; 3653 // FALLTHROUGH 3654 case TargetLowering::Legal: 3655 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3656 if (isCustom) { 3657 Tmp1 = TLI.LowerOperation(Tmp1, DAG); 3658 if (Tmp1.getNode()) Result = Tmp1; 3659 } 3660 break; 3661 case TargetLowering::Expand: 3662 Result = Tmp1; // Default to a no-op, return the chain 3663 break; 3664 } 3665 break; 3666 3667 case ISD::VASTART: 3668 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. 3669 Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. 3670 3671 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); 3672 3673 switch (TLI.getOperationAction(ISD::VASTART, MVT::Other)) { 3674 default: assert(0 && "This action is not supported yet!"); 3675 case TargetLowering::Legal: break; 3676 case TargetLowering::Custom: 3677 Tmp1 = TLI.LowerOperation(Result, DAG); 3678 if (Tmp1.getNode()) Result = Tmp1; 3679 break; 3680 } 3681 break; 3682 3683 case ISD::ROTL: 3684 case ISD::ROTR: 3685 Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS 3686 Tmp2 = LegalizeOp(DAG.getShiftAmountOperand(Node->getOperand(1))); // RHS 3687 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); 3688 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3689 default: 3690 assert(0 && "ROTL/ROTR legalize operation not supported"); 3691 break; 3692 case TargetLowering::Legal: 3693 break; 3694 case TargetLowering::Custom: 3695 Tmp1 = TLI.LowerOperation(Result, DAG); 3696 if (Tmp1.getNode()) Result = Tmp1; 3697 break; 3698 case TargetLowering::Promote: 3699 assert(0 && "Do not know how to promote ROTL/ROTR"); 3700 break; 3701 case TargetLowering::Expand: 3702 assert(0 && "Do not know how to expand ROTL/ROTR"); 3703 break; 3704 } 3705 break; 3706 3707 case ISD::BSWAP: 3708 Tmp1 = LegalizeOp(Node->getOperand(0)); // Op 3709 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3710 case TargetLowering::Custom: 3711 assert(0 && "Cannot custom legalize this yet!"); 3712 case TargetLowering::Legal: 3713 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3714 break; 3715 case TargetLowering::Promote: { 3716 MVT OVT = Tmp1.getValueType(); 3717 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3718 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 3719 3720 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1); 3721 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 3722 Result = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 3723 DAG.getConstant(DiffBits, TLI.getShiftAmountTy())); 3724 break; 3725 } 3726 case TargetLowering::Expand: 3727 Result = ExpandBSWAP(Tmp1, dl); 3728 break; 3729 } 3730 break; 3731 3732 case ISD::CTPOP: 3733 case ISD::CTTZ: 3734 case ISD::CTLZ: 3735 Tmp1 = LegalizeOp(Node->getOperand(0)); // Op 3736 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3737 case TargetLowering::Custom: 3738 case TargetLowering::Legal: 3739 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3740 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 3741 TargetLowering::Custom) { 3742 Tmp1 = TLI.LowerOperation(Result, DAG); 3743 if (Tmp1.getNode()) { 3744 Result = Tmp1; 3745 } 3746 } 3747 break; 3748 case TargetLowering::Promote: { 3749 MVT OVT = Tmp1.getValueType(); 3750 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3751 3752 // Zero extend the argument. 3753 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1); 3754 // Perform the larger operation, then subtract if needed. 3755 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1); 3756 switch (Node->getOpcode()) { 3757 case ISD::CTPOP: 3758 Result = Tmp1; 3759 break; 3760 case ISD::CTTZ: 3761 //if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT) 3762 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), 3763 Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), 3764 ISD::SETEQ); 3765 Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, 3766 DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); 3767 break; 3768 case ISD::CTLZ: 3769 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 3770 Result = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 3771 DAG.getConstant(NVT.getSizeInBits() - 3772 OVT.getSizeInBits(), NVT)); 3773 break; 3774 } 3775 break; 3776 } 3777 case TargetLowering::Expand: 3778 Result = ExpandBitCount(Node->getOpcode(), Tmp1, dl); 3779 break; 3780 } 3781 break; 3782 3783 // Unary operators 3784 case ISD::FABS: 3785 case ISD::FNEG: 3786 case ISD::FSQRT: 3787 case ISD::FSIN: 3788 case ISD::FCOS: 3789 case ISD::FLOG: 3790 case ISD::FLOG2: 3791 case ISD::FLOG10: 3792 case ISD::FEXP: 3793 case ISD::FEXP2: 3794 case ISD::FTRUNC: 3795 case ISD::FFLOOR: 3796 case ISD::FCEIL: 3797 case ISD::FRINT: 3798 case ISD::FNEARBYINT: 3799 Tmp1 = LegalizeOp(Node->getOperand(0)); 3800 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 3801 case TargetLowering::Promote: 3802 case TargetLowering::Custom: 3803 isCustom = true; 3804 // FALLTHROUGH 3805 case TargetLowering::Legal: 3806 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3807 if (isCustom) { 3808 Tmp1 = TLI.LowerOperation(Result, DAG); 3809 if (Tmp1.getNode()) Result = Tmp1; 3810 } 3811 break; 3812 case TargetLowering::Expand: 3813 switch (Node->getOpcode()) { 3814 default: assert(0 && "Unreachable!"); 3815 case ISD::FNEG: 3816 // Expand Y = FNEG(X) -> Y = SUB -0.0, X 3817 Tmp2 = DAG.getConstantFP(-0.0, Node->getValueType(0)); 3818 Result = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp2, Tmp1); 3819 break; 3820 case ISD::FABS: { 3821 // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). 3822 MVT VT = Node->getValueType(0); 3823 Tmp2 = DAG.getConstantFP(0.0, VT); 3824 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), 3825 Tmp1, Tmp2, ISD::SETUGT); 3826 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); 3827 Result = DAG.getNode(ISD::SELECT, dl, VT, Tmp2, Tmp1, Tmp3); 3828 break; 3829 } 3830 case ISD::FSQRT: 3831 case ISD::FSIN: 3832 case ISD::FCOS: 3833 case ISD::FLOG: 3834 case ISD::FLOG2: 3835 case ISD::FLOG10: 3836 case ISD::FEXP: 3837 case ISD::FEXP2: 3838 case ISD::FTRUNC: 3839 case ISD::FFLOOR: 3840 case ISD::FCEIL: 3841 case ISD::FRINT: 3842 case ISD::FNEARBYINT: { 3843 MVT VT = Node->getValueType(0); 3844 3845 // Expand unsupported unary vector operators by unrolling them. 3846 if (VT.isVector()) { 3847 Result = LegalizeOp(UnrollVectorOp(Op)); 3848 break; 3849 } 3850 3851 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3852 switch(Node->getOpcode()) { 3853 case ISD::FSQRT: 3854 LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 3855 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128); 3856 break; 3857 case ISD::FSIN: 3858 LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64, 3859 RTLIB::SIN_F80, RTLIB::SIN_PPCF128); 3860 break; 3861 case ISD::FCOS: 3862 LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, 3863 RTLIB::COS_F80, RTLIB::COS_PPCF128); 3864 break; 3865 case ISD::FLOG: 3866 LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, 3867 RTLIB::LOG_F80, RTLIB::LOG_PPCF128); 3868 break; 3869 case ISD::FLOG2: 3870 LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 3871 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); 3872 break; 3873 case ISD::FLOG10: 3874 LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 3875 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); 3876 break; 3877 case ISD::FEXP: 3878 LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, 3879 RTLIB::EXP_F80, RTLIB::EXP_PPCF128); 3880 break; 3881 case ISD::FEXP2: 3882 LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 3883 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); 3884 break; 3885 case ISD::FTRUNC: 3886 LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 3887 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); 3888 break; 3889 case ISD::FFLOOR: 3890 LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 3891 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); 3892 break; 3893 case ISD::FCEIL: 3894 LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 3895 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); 3896 break; 3897 case ISD::FRINT: 3898 LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, 3899 RTLIB::RINT_F80, RTLIB::RINT_PPCF128); 3900 break; 3901 case ISD::FNEARBYINT: 3902 LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, 3903 RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); 3904 break; 3905 break; 3906 default: assert(0 && "Unreachable!"); 3907 } 3908 SDValue Dummy; 3909 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3910 break; 3911 } 3912 } 3913 break; 3914 } 3915 break; 3916 case ISD::FPOWI: { 3917 MVT VT = Node->getValueType(0); 3918 3919 // Expand unsupported unary vector operators by unrolling them. 3920 if (VT.isVector()) { 3921 Result = LegalizeOp(UnrollVectorOp(Op)); 3922 break; 3923 } 3924 3925 // We always lower FPOWI into a libcall. No target support for it yet. 3926 RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, 3927 RTLIB::POWI_F80, RTLIB::POWI_PPCF128); 3928 SDValue Dummy; 3929 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 3930 break; 3931 } 3932 case ISD::BIT_CONVERT: 3933 if (!isTypeLegal(Node->getOperand(0).getValueType())) { 3934 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 3935 Node->getValueType(0), dl); 3936 } else if (Op.getOperand(0).getValueType().isVector()) { 3937 // The input has to be a vector type, we have to either scalarize it, pack 3938 // it, or convert it based on whether the input vector type is legal. 3939 SDNode *InVal = Node->getOperand(0).getNode(); 3940 int InIx = Node->getOperand(0).getResNo(); 3941 unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements(); 3942 MVT EVT = InVal->getValueType(InIx).getVectorElementType(); 3943 3944 // Figure out if there is a simple type corresponding to this Vector 3945 // type. If so, convert to the vector type. 3946 MVT TVT = MVT::getVectorVT(EVT, NumElems); 3947 if (TLI.isTypeLegal(TVT)) { 3948 // Turn this into a bit convert of the vector input. 3949 Tmp1 = LegalizeOp(Node->getOperand(0)); 3950 Result = DAG.getNode(ISD::BIT_CONVERT, dl, Node->getValueType(0), Tmp1); 3951 break; 3952 } else if (NumElems == 1) { 3953 // Turn this into a bit convert of the scalar input. 3954 Result = DAG.getNode(ISD::BIT_CONVERT, dl, Node->getValueType(0), 3955 ScalarizeVectorOp(Node->getOperand(0))); 3956 break; 3957 } else { 3958 // FIXME: UNIMP! Store then reload 3959 assert(0 && "Cast from unsupported vector type not implemented yet!"); 3960 } 3961 } else { 3962 switch (TLI.getOperationAction(ISD::BIT_CONVERT, 3963 Node->getOperand(0).getValueType())) { 3964 default: assert(0 && "Unknown operation action!"); 3965 case TargetLowering::Expand: 3966 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 3967 Node->getValueType(0), dl); 3968 break; 3969 case TargetLowering::Legal: 3970 Tmp1 = LegalizeOp(Node->getOperand(0)); 3971 Result = DAG.UpdateNodeOperands(Result, Tmp1); 3972 break; 3973 } 3974 } 3975 break; 3976 case ISD::CONVERT_RNDSAT: { 3977 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 3978 switch (CvtCode) { 3979 default: assert(0 && "Unknown cvt code!"); 3980 case ISD::CVT_SF: 3981 case ISD::CVT_UF: 3982 case ISD::CVT_FF: 3983 break; 3984 case ISD::CVT_FS: 3985 case ISD::CVT_FU: 3986 case ISD::CVT_SS: 3987 case ISD::CVT_SU: 3988 case ISD::CVT_US: 3989 case ISD::CVT_UU: { 3990 SDValue DTyOp = Node->getOperand(1); 3991 SDValue STyOp = Node->getOperand(2); 3992 SDValue RndOp = Node->getOperand(3); 3993 SDValue SatOp = Node->getOperand(4); 3994 switch (getTypeAction(Node->getOperand(0).getValueType())) { 3995 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 3996 case Legal: 3997 Tmp1 = LegalizeOp(Node->getOperand(0)); 3998 Result = DAG.UpdateNodeOperands(Result, Tmp1, DTyOp, STyOp, 3999 RndOp, SatOp); 4000 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 4001 TargetLowering::Custom) { 4002 Tmp1 = TLI.LowerOperation(Result, DAG); 4003 if (Tmp1.getNode()) Result = Tmp1; 4004 } 4005 break; 4006 case Promote: 4007 Result = PromoteOp(Node->getOperand(0)); 4008 // For FP, make Op1 a i32 4009 4010 Result = DAG.getConvertRndSat(Op.getValueType(), dl, Result, 4011 DTyOp, STyOp, RndOp, SatOp, CvtCode); 4012 break; 4013 } 4014 break; 4015 } 4016 } // end switch CvtCode 4017 break; 4018 } 4019 // Conversion operators. The source and destination have different types. 4020 case ISD::SINT_TO_FP: 4021 case ISD::UINT_TO_FP: { 4022 bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; 4023 Result = LegalizeINT_TO_FP(Result, isSigned, 4024 Node->getValueType(0), Node->getOperand(0), dl); 4025 break; 4026 } 4027 case ISD::TRUNCATE: 4028 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4029 case Legal: 4030 Tmp1 = LegalizeOp(Node->getOperand(0)); 4031 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { 4032 default: assert(0 && "Unknown TRUNCATE legalization operation action!"); 4033 case TargetLowering::Custom: 4034 isCustom = true; 4035 // FALLTHROUGH 4036 case TargetLowering::Legal: 4037 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4038 if (isCustom) { 4039 Tmp1 = TLI.LowerOperation(Result, DAG); 4040 if (Tmp1.getNode()) Result = Tmp1; 4041 } 4042 break; 4043 case TargetLowering::Expand: 4044 assert(Result.getValueType().isVector() && "must be vector type"); 4045 // Unroll the truncate. We should do better. 4046 Result = LegalizeOp(UnrollVectorOp(Result)); 4047 } 4048 break; 4049 case Expand: 4050 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 4051 4052 // Since the result is legal, we should just be able to truncate the low 4053 // part of the source. 4054 Result = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 4055 break; 4056 case Promote: 4057 Result = PromoteOp(Node->getOperand(0)); 4058 Result = DAG.getNode(ISD::TRUNCATE, dl, Op.getValueType(), Result); 4059 break; 4060 } 4061 break; 4062 4063 case ISD::FP_TO_SINT: 4064 case ISD::FP_TO_UINT: 4065 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4066 case Legal: 4067 Tmp1 = LegalizeOp(Node->getOperand(0)); 4068 4069 switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))){ 4070 default: assert(0 && "Unknown operation action!"); 4071 case TargetLowering::Custom: 4072 isCustom = true; 4073 // FALLTHROUGH 4074 case TargetLowering::Legal: 4075 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4076 if (isCustom) { 4077 Tmp1 = TLI.LowerOperation(Result, DAG); 4078 if (Tmp1.getNode()) Result = Tmp1; 4079 } 4080 break; 4081 case TargetLowering::Promote: 4082 Result = PromoteLegalFP_TO_INT(Tmp1, Node->getValueType(0), 4083 Node->getOpcode() == ISD::FP_TO_SINT, 4084 dl); 4085 break; 4086 case TargetLowering::Expand: 4087 if (Node->getOpcode() == ISD::FP_TO_UINT) { 4088 SDValue True, False; 4089 MVT VT = Node->getOperand(0).getValueType(); 4090 MVT NVT = Node->getValueType(0); 4091 const uint64_t zero[] = {0, 0}; 4092 APFloat apf = APFloat(APInt(VT.getSizeInBits(), 2, zero)); 4093 APInt x = APInt::getSignBit(NVT.getSizeInBits()); 4094 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); 4095 Tmp2 = DAG.getConstantFP(apf, VT); 4096 Tmp3 = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), 4097 Node->getOperand(0), 4098 Tmp2, ISD::SETLT); 4099 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); 4100 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, 4101 DAG.getNode(ISD::FSUB, dl, VT, 4102 Node->getOperand(0), Tmp2)); 4103 False = DAG.getNode(ISD::XOR, dl, NVT, False, 4104 DAG.getConstant(x, NVT)); 4105 Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp3, True, False); 4106 break; 4107 } else { 4108 assert(0 && "Do not know how to expand FP_TO_SINT yet!"); 4109 } 4110 break; 4111 } 4112 break; 4113 case Expand: { 4114 MVT VT = Op.getValueType(); 4115 MVT OVT = Node->getOperand(0).getValueType(); 4116 // Convert ppcf128 to i32 4117 if (OVT == MVT::ppcf128 && VT == MVT::i32) { 4118 if (Node->getOpcode() == ISD::FP_TO_SINT) { 4119 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, MVT::ppcf128, 4120 Node->getOperand(0), DAG.getValueType(MVT::f64)); 4121 Result = DAG.getNode(ISD::FP_ROUND, dl, MVT::f64, Result, 4122 DAG.getIntPtrConstant(1)); 4123 Result = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Result); 4124 } else { 4125 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; 4126 APFloat apf = APFloat(APInt(128, 2, TwoE31)); 4127 Tmp2 = DAG.getConstantFP(apf, OVT); 4128 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X 4129 // FIXME: generated code sucks. 4130 Result = DAG.getNode(ISD::SELECT_CC, dl, VT, Node->getOperand(0), 4131 Tmp2, 4132 DAG.getNode(ISD::ADD, dl, MVT::i32, 4133 DAG.getNode(ISD::FP_TO_SINT, dl, VT, 4134 DAG.getNode(ISD::FSUB, dl, OVT, 4135 Node->getOperand(0), Tmp2)), 4136 DAG.getConstant(0x80000000, MVT::i32)), 4137 DAG.getNode(ISD::FP_TO_SINT, dl, VT, 4138 Node->getOperand(0)), 4139 DAG.getCondCode(ISD::SETGE)); 4140 } 4141 break; 4142 } 4143 // Convert f32 / f64 to i32 / i64 / i128. 4144 RTLIB::Libcall LC = (Node->getOpcode() == ISD::FP_TO_SINT) ? 4145 RTLIB::getFPTOSINT(OVT, VT) : RTLIB::getFPTOUINT(OVT, VT); 4146 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpectd fp-to-int conversion!"); 4147 SDValue Dummy; 4148 Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); 4149 break; 4150 } 4151 case Promote: 4152 Tmp1 = PromoteOp(Node->getOperand(0)); 4153 Result = DAG.UpdateNodeOperands(Result, LegalizeOp(Tmp1)); 4154 Result = LegalizeOp(Result); 4155 break; 4156 } 4157 break; 4158 4159 case ISD::FP_EXTEND: { 4160 MVT DstVT = Op.getValueType(); 4161 MVT SrcVT = Op.getOperand(0).getValueType(); 4162 if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) { 4163 // The only other way we can lower this is to turn it into a STORE, 4164 // LOAD pair, targetting a temporary location (a stack slot). 4165 Result = EmitStackConvert(Node->getOperand(0), SrcVT, DstVT, dl); 4166 break; 4167 } 4168 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4169 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4170 case Legal: 4171 Tmp1 = LegalizeOp(Node->getOperand(0)); 4172 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4173 break; 4174 case Promote: 4175 Tmp1 = PromoteOp(Node->getOperand(0)); 4176 Result = DAG.getNode(ISD::FP_EXTEND, dl, Op.getValueType(), Tmp1); 4177 break; 4178 } 4179 break; 4180 } 4181 case ISD::FP_ROUND: { 4182 MVT DstVT = Op.getValueType(); 4183 MVT SrcVT = Op.getOperand(0).getValueType(); 4184 if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) { 4185 if (SrcVT == MVT::ppcf128) { 4186 SDValue Lo; 4187 ExpandOp(Node->getOperand(0), Lo, Result); 4188 // Round it the rest of the way (e.g. to f32) if needed. 4189 if (DstVT!=MVT::f64) 4190 Result = DAG.getNode(ISD::FP_ROUND, dl, 4191 DstVT, Result, Op.getOperand(1)); 4192 break; 4193 } 4194 // The only other way we can lower this is to turn it into a STORE, 4195 // LOAD pair, targetting a temporary location (a stack slot). 4196 Result = EmitStackConvert(Node->getOperand(0), DstVT, DstVT, dl); 4197 break; 4198 } 4199 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4200 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4201 case Legal: 4202 Tmp1 = LegalizeOp(Node->getOperand(0)); 4203 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 4204 break; 4205 case Promote: 4206 Tmp1 = PromoteOp(Node->getOperand(0)); 4207 Result = DAG.getNode(ISD::FP_ROUND, dl, Op.getValueType(), Tmp1, 4208 Node->getOperand(1)); 4209 break; 4210 } 4211 break; 4212 } 4213 case ISD::ANY_EXTEND: 4214 case ISD::ZERO_EXTEND: 4215 case ISD::SIGN_EXTEND: 4216 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4217 case Expand: assert(0 && "Shouldn't need to expand other operators here!"); 4218 case Legal: 4219 Tmp1 = LegalizeOp(Node->getOperand(0)); 4220 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4221 if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == 4222 TargetLowering::Custom) { 4223 Tmp1 = TLI.LowerOperation(Result, DAG); 4224 if (Tmp1.getNode()) Result = Tmp1; 4225 } 4226 break; 4227 case Promote: 4228 switch (Node->getOpcode()) { 4229 case ISD::ANY_EXTEND: 4230 Tmp1 = PromoteOp(Node->getOperand(0)); 4231 Result = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), Tmp1); 4232 break; 4233 case ISD::ZERO_EXTEND: 4234 Result = PromoteOp(Node->getOperand(0)); 4235 Result = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), Result); 4236 Result = DAG.getZeroExtendInReg(Result, dl, 4237 Node->getOperand(0).getValueType()); 4238 break; 4239 case ISD::SIGN_EXTEND: 4240 Result = PromoteOp(Node->getOperand(0)); 4241 Result = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), Result); 4242 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Result.getValueType(), 4243 Result, 4244 DAG.getValueType(Node->getOperand(0).getValueType())); 4245 break; 4246 } 4247 } 4248 break; 4249 case ISD::FP_ROUND_INREG: 4250 case ISD::SIGN_EXTEND_INREG: { 4251 Tmp1 = LegalizeOp(Node->getOperand(0)); 4252 MVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 4253 4254 // If this operation is not supported, convert it to a shl/shr or load/store 4255 // pair. 4256 switch (TLI.getOperationAction(Node->getOpcode(), ExtraVT)) { 4257 default: assert(0 && "This action not supported for this op yet!"); 4258 case TargetLowering::Legal: 4259 Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1)); 4260 break; 4261 case TargetLowering::Expand: 4262 // If this is an integer extend and shifts are supported, do that. 4263 if (Node->getOpcode() == ISD::SIGN_EXTEND_INREG) { 4264 // NOTE: we could fall back on load/store here too for targets without 4265 // SAR. However, it is doubtful that any exist. 4266 unsigned BitsDiff = Node->getValueType(0).getSizeInBits() - 4267 ExtraVT.getSizeInBits(); 4268 SDValue ShiftCst = DAG.getConstant(BitsDiff, TLI.getShiftAmountTy()); 4269 Result = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 4270 Node->getOperand(0), ShiftCst); 4271 Result = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), 4272 Result, ShiftCst); 4273 } else if (Node->getOpcode() == ISD::FP_ROUND_INREG) { 4274 // The only way we can lower this is to turn it into a TRUNCSTORE, 4275 // EXTLOAD pair, targetting a temporary location (a stack slot). 4276 4277 // NOTE: there is a choice here between constantly creating new stack 4278 // slots and always reusing the same one. We currently always create 4279 // new ones, as reuse may inhibit scheduling. 4280 Result = EmitStackConvert(Node->getOperand(0), ExtraVT, 4281 Node->getValueType(0), dl); 4282 } else { 4283 assert(0 && "Unknown op"); 4284 } 4285 break; 4286 } 4287 break; 4288 } 4289 case ISD::TRAMPOLINE: { 4290 SDValue Ops[6]; 4291 for (unsigned i = 0; i != 6; ++i) 4292 Ops[i] = LegalizeOp(Node->getOperand(i)); 4293 Result = DAG.UpdateNodeOperands(Result, Ops, 6); 4294 // The only option for this node is to custom lower it. 4295 Result = TLI.LowerOperation(Result, DAG); 4296 assert(Result.getNode() && "Should always custom lower!"); 4297 4298 // Since trampoline produces two values, make sure to remember that we 4299 // legalized both of them. 4300 Tmp1 = LegalizeOp(Result.getValue(1)); 4301 Result = LegalizeOp(Result); 4302 AddLegalizedOperand(SDValue(Node, 0), Result); 4303 AddLegalizedOperand(SDValue(Node, 1), Tmp1); 4304 return Op.getResNo() ? Tmp1 : Result; 4305 } 4306 case ISD::FLT_ROUNDS_: { 4307 MVT VT = Node->getValueType(0); 4308 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4309 default: assert(0 && "This action not supported for this op yet!"); 4310 case TargetLowering::Custom: 4311 Result = TLI.LowerOperation(Op, DAG); 4312 if (Result.getNode()) break; 4313 // Fall Thru 4314 case TargetLowering::Legal: 4315 // If this operation is not supported, lower it to constant 1 4316 Result = DAG.getConstant(1, VT); 4317 break; 4318 } 4319 break; 4320 } 4321 case ISD::TRAP: { 4322 MVT VT = Node->getValueType(0); 4323 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4324 default: assert(0 && "This action not supported for this op yet!"); 4325 case TargetLowering::Legal: 4326 Tmp1 = LegalizeOp(Node->getOperand(0)); 4327 Result = DAG.UpdateNodeOperands(Result, Tmp1); 4328 break; 4329 case TargetLowering::Custom: 4330 Result = TLI.LowerOperation(Op, DAG); 4331 if (Result.getNode()) break; 4332 // Fall Thru 4333 case TargetLowering::Expand: 4334 // If this operation is not supported, lower it to 'abort()' call 4335 Tmp1 = LegalizeOp(Node->getOperand(0)); 4336 TargetLowering::ArgListTy Args; 4337 std::pair<SDValue, SDValue> CallResult = 4338 TLI.LowerCallTo(Tmp1, Type::VoidTy, 4339 false, false, false, false, CallingConv::C, false, 4340 DAG.getExternalSymbol("abort", TLI.getPointerTy()), 4341 Args, DAG, dl); 4342 Result = CallResult.second; 4343 break; 4344 } 4345 break; 4346 } 4347 4348 case ISD::SADDO: 4349 case ISD::SSUBO: { 4350 MVT VT = Node->getValueType(0); 4351 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4352 default: assert(0 && "This action not supported for this op yet!"); 4353 case TargetLowering::Custom: 4354 Result = TLI.LowerOperation(Op, DAG); 4355 if (Result.getNode()) break; 4356 // FALLTHROUGH 4357 case TargetLowering::Legal: { 4358 SDValue LHS = LegalizeOp(Node->getOperand(0)); 4359 SDValue RHS = LegalizeOp(Node->getOperand(1)); 4360 4361 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 4362 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 4363 LHS, RHS); 4364 MVT OType = Node->getValueType(1); 4365 4366 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 4367 4368 // LHSSign -> LHS >= 0 4369 // RHSSign -> RHS >= 0 4370 // SumSign -> Sum >= 0 4371 // 4372 // Add: 4373 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 4374 // Sub: 4375 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 4376 // 4377 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 4378 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 4379 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 4380 Node->getOpcode() == ISD::SADDO ? 4381 ISD::SETEQ : ISD::SETNE); 4382 4383 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 4384 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 4385 4386 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 4387 4388 MVT ValueVTs[] = { LHS.getValueType(), OType }; 4389 SDValue Ops[] = { Sum, Cmp }; 4390 4391 Result = DAG.getNode(ISD::MERGE_VALUES, dl, 4392 DAG.getVTList(&ValueVTs[0], 2), 4393 &Ops[0], 2); 4394 SDNode *RNode = Result.getNode(); 4395 DAG.ReplaceAllUsesWith(Node, RNode); 4396 break; 4397 } 4398 } 4399 4400 break; 4401 } 4402 case ISD::UADDO: 4403 case ISD::USUBO: { 4404 MVT VT = Node->getValueType(0); 4405 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4406 default: assert(0 && "This action not supported for this op yet!"); 4407 case TargetLowering::Custom: 4408 Result = TLI.LowerOperation(Op, DAG); 4409 if (Result.getNode()) break; 4410 // FALLTHROUGH 4411 case TargetLowering::Legal: { 4412 SDValue LHS = LegalizeOp(Node->getOperand(0)); 4413 SDValue RHS = LegalizeOp(Node->getOperand(1)); 4414 4415 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? 4416 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 4417 LHS, RHS); 4418 MVT OType = Node->getValueType(1); 4419 SDValue Cmp = DAG.getSetCC(dl, OType, Sum, LHS, 4420 Node->getOpcode () == ISD::UADDO ? 4421 ISD::SETULT : ISD::SETUGT); 4422 4423 MVT ValueVTs[] = { LHS.getValueType(), OType }; 4424 SDValue Ops[] = { Sum, Cmp }; 4425 4426 Result = DAG.getNode(ISD::MERGE_VALUES, dl, 4427 DAG.getVTList(&ValueVTs[0], 2), 4428 &Ops[0], 2); 4429 SDNode *RNode = Result.getNode(); 4430 DAG.ReplaceAllUsesWith(Node, RNode); 4431 break; 4432 } 4433 } 4434 4435 break; 4436 } 4437 case ISD::SMULO: 4438 case ISD::UMULO: { 4439 MVT VT = Node->getValueType(0); 4440 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 4441 default: assert(0 && "This action is not supported at all!"); 4442 case TargetLowering::Custom: 4443 Result = TLI.LowerOperation(Op, DAG); 4444 if (Result.getNode()) break; 4445 // Fall Thru 4446 case TargetLowering::Legal: 4447 // FIXME: According to Hacker's Delight, this can be implemented in 4448 // target independent lowering, but it would be inefficient, since it 4449 // requires a division + a branch. 4450 assert(0 && "Target independent lowering is not supported for SMULO/UMULO!"); 4451 break; 4452 } 4453 break; 4454 } 4455 4456 } 4457 4458 assert(Result.getValueType() == Op.getValueType() && 4459 "Bad legalization!"); 4460 4461 // Make sure that the generated code is itself legal. 4462 if (Result != Op) 4463 Result = LegalizeOp(Result); 4464 4465 // Note that LegalizeOp may be reentered even from single-use nodes, which 4466 // means that we always must cache transformed nodes. 4467 AddLegalizedOperand(Op, Result); 4468 return Result; 4469} 4470 4471/// PromoteOp - Given an operation that produces a value in an invalid type, 4472/// promote it to compute the value into a larger type. The produced value will 4473/// have the correct bits for the low portion of the register, but no guarantee 4474/// is made about the top bits: it may be zero, sign-extended, or garbage. 4475SDValue SelectionDAGLegalize::PromoteOp(SDValue Op) { 4476 MVT VT = Op.getValueType(); 4477 MVT NVT = TLI.getTypeToTransformTo(VT); 4478 assert(getTypeAction(VT) == Promote && 4479 "Caller should expand or legalize operands that are not promotable!"); 4480 assert(NVT.bitsGT(VT) && NVT.isInteger() == VT.isInteger() && 4481 "Cannot promote to smaller type!"); 4482 4483 SDValue Tmp1, Tmp2, Tmp3; 4484 SDValue Result; 4485 SDNode *Node = Op.getNode(); 4486 DebugLoc dl = Node->getDebugLoc(); 4487 4488 DenseMap<SDValue, SDValue>::iterator I = PromotedNodes.find(Op); 4489 if (I != PromotedNodes.end()) return I->second; 4490 4491 switch (Node->getOpcode()) { 4492 case ISD::CopyFromReg: 4493 assert(0 && "CopyFromReg must be legal!"); 4494 default: 4495#ifndef NDEBUG 4496 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 4497#endif 4498 assert(0 && "Do not know how to promote this operator!"); 4499 abort(); 4500 case ISD::UNDEF: 4501 Result = DAG.getUNDEF(NVT); 4502 break; 4503 case ISD::Constant: 4504 if (VT != MVT::i1) 4505 Result = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, Op); 4506 else 4507 Result = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Op); 4508 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold zext?"); 4509 break; 4510 case ISD::ConstantFP: 4511 Result = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Op); 4512 assert(isa<ConstantFPSDNode>(Result) && "Didn't constant fold fp_extend?"); 4513 break; 4514 4515 case ISD::SETCC: { 4516 MVT VT0 = Node->getOperand(0).getValueType(); 4517 assert(isTypeLegal(TLI.getSetCCResultType(VT0)) 4518 && "SetCC type is not legal??"); 4519 Result = DAG.getNode(ISD::SETCC, dl, TLI.getSetCCResultType(VT0), 4520 Node->getOperand(0), Node->getOperand(1), 4521 Node->getOperand(2)); 4522 break; 4523 } 4524 case ISD::TRUNCATE: 4525 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4526 case Legal: 4527 Result = LegalizeOp(Node->getOperand(0)); 4528 assert(Result.getValueType().bitsGE(NVT) && 4529 "This truncation doesn't make sense!"); 4530 if (Result.getValueType().bitsGT(NVT)) // Truncate to NVT instead of VT 4531 Result = DAG.getNode(ISD::TRUNCATE, dl, NVT, Result); 4532 break; 4533 case Promote: 4534 // The truncation is not required, because we don't guarantee anything 4535 // about high bits anyway. 4536 Result = PromoteOp(Node->getOperand(0)); 4537 break; 4538 case Expand: 4539 ExpandOp(Node->getOperand(0), Tmp1, Tmp2); 4540 // Truncate the low part of the expanded value to the result type 4541 Result = DAG.getNode(ISD::TRUNCATE, dl, NVT, Tmp1); 4542 } 4543 break; 4544 case ISD::SIGN_EXTEND: 4545 case ISD::ZERO_EXTEND: 4546 case ISD::ANY_EXTEND: 4547 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4548 case Expand: assert(0 && "BUG: Smaller reg should have been promoted!"); 4549 case Legal: 4550 // Input is legal? Just do extend all the way to the larger type. 4551 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Node->getOperand(0)); 4552 break; 4553 case Promote: 4554 // Promote the reg if it's smaller. 4555 Result = PromoteOp(Node->getOperand(0)); 4556 // The high bits are not guaranteed to be anything. Insert an extend. 4557 if (Node->getOpcode() == ISD::SIGN_EXTEND) 4558 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Result, 4559 DAG.getValueType(Node->getOperand(0).getValueType())); 4560 else if (Node->getOpcode() == ISD::ZERO_EXTEND) 4561 Result = DAG.getZeroExtendInReg(Result, dl, 4562 Node->getOperand(0).getValueType()); 4563 break; 4564 } 4565 break; 4566 case ISD::CONVERT_RNDSAT: { 4567 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 4568 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 4569 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 4570 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 4571 "can only promote integers"); 4572 Result = DAG.getConvertRndSat(NVT, dl, Node->getOperand(0), 4573 Node->getOperand(1), Node->getOperand(2), 4574 Node->getOperand(3), Node->getOperand(4), 4575 CvtCode); 4576 break; 4577 4578 } 4579 case ISD::BIT_CONVERT: 4580 Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 4581 Node->getValueType(0), dl); 4582 Result = PromoteOp(Result); 4583 break; 4584 4585 case ISD::FP_EXTEND: 4586 assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); 4587 case ISD::FP_ROUND: 4588 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4589 case Expand: assert(0 && "BUG: Cannot expand FP regs!"); 4590 case Promote: assert(0 && "Unreachable with 2 FP types!"); 4591 case Legal: 4592 if (Node->getConstantOperandVal(1) == 0) { 4593 // Input is legal? Do an FP_ROUND_INREG. 4594 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Node->getOperand(0), 4595 DAG.getValueType(VT)); 4596 } else { 4597 // Just remove the truncate, it isn't affecting the value. 4598 Result = DAG.getNode(ISD::FP_ROUND, dl, NVT, Node->getOperand(0), 4599 Node->getOperand(1)); 4600 } 4601 break; 4602 } 4603 break; 4604 case ISD::SINT_TO_FP: 4605 case ISD::UINT_TO_FP: 4606 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4607 case Legal: 4608 // No extra round required here. 4609 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Node->getOperand(0)); 4610 break; 4611 4612 case Promote: 4613 Result = PromoteOp(Node->getOperand(0)); 4614 if (Node->getOpcode() == ISD::SINT_TO_FP) 4615 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Result.getValueType(), 4616 Result, 4617 DAG.getValueType(Node->getOperand(0).getValueType())); 4618 else 4619 Result = DAG.getZeroExtendInReg(Result, dl, 4620 Node->getOperand(0).getValueType()); 4621 // No extra round required here. 4622 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Result); 4623 break; 4624 case Expand: 4625 Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, NVT, 4626 Node->getOperand(0), dl); 4627 // Round if we cannot tolerate excess precision. 4628 if (NoExcessFPPrecision) 4629 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4630 DAG.getValueType(VT)); 4631 break; 4632 } 4633 break; 4634 4635 case ISD::SIGN_EXTEND_INREG: 4636 Result = PromoteOp(Node->getOperand(0)); 4637 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Result, 4638 Node->getOperand(1)); 4639 break; 4640 case ISD::FP_TO_SINT: 4641 case ISD::FP_TO_UINT: 4642 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4643 case Legal: 4644 case Expand: 4645 Tmp1 = Node->getOperand(0); 4646 break; 4647 case Promote: 4648 // The input result is prerounded, so we don't have to do anything 4649 // special. 4650 Tmp1 = PromoteOp(Node->getOperand(0)); 4651 break; 4652 } 4653 // If we're promoting a UINT to a larger size, check to see if the new node 4654 // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since 4655 // we can use that instead. This allows us to generate better code for 4656 // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not 4657 // legal, such as PowerPC. 4658 if (Node->getOpcode() == ISD::FP_TO_UINT && 4659 !TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NVT) && 4660 (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT) || 4661 TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)){ 4662 Result = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Tmp1); 4663 } else { 4664 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4665 } 4666 break; 4667 4668 case ISD::FABS: 4669 case ISD::FNEG: 4670 Tmp1 = PromoteOp(Node->getOperand(0)); 4671 assert(Tmp1.getValueType() == NVT); 4672 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4673 // NOTE: we do not have to do any extra rounding here for 4674 // NoExcessFPPrecision, because we know the input will have the appropriate 4675 // precision, and these operations don't modify precision at all. 4676 break; 4677 4678 case ISD::FLOG: 4679 case ISD::FLOG2: 4680 case ISD::FLOG10: 4681 case ISD::FEXP: 4682 case ISD::FEXP2: 4683 case ISD::FSQRT: 4684 case ISD::FSIN: 4685 case ISD::FCOS: 4686 case ISD::FTRUNC: 4687 case ISD::FFLOOR: 4688 case ISD::FCEIL: 4689 case ISD::FRINT: 4690 case ISD::FNEARBYINT: 4691 Tmp1 = PromoteOp(Node->getOperand(0)); 4692 assert(Tmp1.getValueType() == NVT); 4693 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4694 if (NoExcessFPPrecision) 4695 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4696 DAG.getValueType(VT)); 4697 break; 4698 4699 case ISD::FPOW: 4700 case ISD::FPOWI: { 4701 // Promote f32 pow(i) to f64 pow(i). Note that this could insert a libcall 4702 // directly as well, which may be better. 4703 Tmp1 = PromoteOp(Node->getOperand(0)); 4704 Tmp2 = Node->getOperand(1); 4705 if (Node->getOpcode() == ISD::FPOW) 4706 Tmp2 = PromoteOp(Tmp2); 4707 assert(Tmp1.getValueType() == NVT); 4708 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4709 if (NoExcessFPPrecision) 4710 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4711 DAG.getValueType(VT)); 4712 break; 4713 } 4714 4715 case ISD::ATOMIC_CMP_SWAP: { 4716 AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node); 4717 Tmp2 = PromoteOp(Node->getOperand(2)); 4718 Tmp3 = PromoteOp(Node->getOperand(3)); 4719 Result = DAG.getAtomic(Node->getOpcode(), dl, AtomNode->getMemoryVT(), 4720 AtomNode->getChain(), 4721 AtomNode->getBasePtr(), Tmp2, Tmp3, 4722 AtomNode->getSrcValue(), 4723 AtomNode->getAlignment()); 4724 // Remember that we legalized the chain. 4725 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4726 break; 4727 } 4728 case ISD::ATOMIC_LOAD_ADD: 4729 case ISD::ATOMIC_LOAD_SUB: 4730 case ISD::ATOMIC_LOAD_AND: 4731 case ISD::ATOMIC_LOAD_OR: 4732 case ISD::ATOMIC_LOAD_XOR: 4733 case ISD::ATOMIC_LOAD_NAND: 4734 case ISD::ATOMIC_LOAD_MIN: 4735 case ISD::ATOMIC_LOAD_MAX: 4736 case ISD::ATOMIC_LOAD_UMIN: 4737 case ISD::ATOMIC_LOAD_UMAX: 4738 case ISD::ATOMIC_SWAP: { 4739 AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node); 4740 Tmp2 = PromoteOp(Node->getOperand(2)); 4741 Result = DAG.getAtomic(Node->getOpcode(), dl, AtomNode->getMemoryVT(), 4742 AtomNode->getChain(), 4743 AtomNode->getBasePtr(), Tmp2, 4744 AtomNode->getSrcValue(), 4745 AtomNode->getAlignment()); 4746 // Remember that we legalized the chain. 4747 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4748 break; 4749 } 4750 4751 case ISD::AND: 4752 case ISD::OR: 4753 case ISD::XOR: 4754 case ISD::ADD: 4755 case ISD::SUB: 4756 case ISD::MUL: 4757 // The input may have strange things in the top bits of the registers, but 4758 // these operations don't care. They may have weird bits going out, but 4759 // that too is okay if they are integer operations. 4760 Tmp1 = PromoteOp(Node->getOperand(0)); 4761 Tmp2 = PromoteOp(Node->getOperand(1)); 4762 assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT); 4763 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4764 break; 4765 case ISD::FADD: 4766 case ISD::FSUB: 4767 case ISD::FMUL: 4768 Tmp1 = PromoteOp(Node->getOperand(0)); 4769 Tmp2 = PromoteOp(Node->getOperand(1)); 4770 assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT); 4771 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4772 4773 // Floating point operations will give excess precision that we may not be 4774 // able to tolerate. If we DO allow excess precision, just leave it, 4775 // otherwise excise it. 4776 // FIXME: Why would we need to round FP ops more than integer ones? 4777 // Is Round(Add(Add(A,B),C)) != Round(Add(Round(Add(A,B)), C)) 4778 if (NoExcessFPPrecision) 4779 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4780 DAG.getValueType(VT)); 4781 break; 4782 4783 case ISD::SDIV: 4784 case ISD::SREM: 4785 // These operators require that their input be sign extended. 4786 Tmp1 = PromoteOp(Node->getOperand(0)); 4787 Tmp2 = PromoteOp(Node->getOperand(1)); 4788 if (NVT.isInteger()) { 4789 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, 4790 DAG.getValueType(VT)); 4791 Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp2, 4792 DAG.getValueType(VT)); 4793 } 4794 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4795 4796 // Perform FP_ROUND: this is probably overly pessimistic. 4797 if (NVT.isFloatingPoint() && NoExcessFPPrecision) 4798 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4799 DAG.getValueType(VT)); 4800 break; 4801 case ISD::FDIV: 4802 case ISD::FREM: 4803 case ISD::FCOPYSIGN: 4804 // These operators require that their input be fp extended. 4805 switch (getTypeAction(Node->getOperand(0).getValueType())) { 4806 case Expand: assert(0 && "not implemented"); 4807 case Legal: Tmp1 = LegalizeOp(Node->getOperand(0)); break; 4808 case Promote: Tmp1 = PromoteOp(Node->getOperand(0)); break; 4809 } 4810 switch (getTypeAction(Node->getOperand(1).getValueType())) { 4811 case Expand: assert(0 && "not implemented"); 4812 case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; 4813 case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; 4814 } 4815 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4816 4817 // Perform FP_ROUND: this is probably overly pessimistic. 4818 if (NoExcessFPPrecision && Node->getOpcode() != ISD::FCOPYSIGN) 4819 Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, 4820 DAG.getValueType(VT)); 4821 break; 4822 4823 case ISD::UDIV: 4824 case ISD::UREM: 4825 // These operators require that their input be zero extended. 4826 Tmp1 = PromoteOp(Node->getOperand(0)); 4827 Tmp2 = PromoteOp(Node->getOperand(1)); 4828 assert(NVT.isInteger() && "Operators don't apply to FP!"); 4829 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); 4830 Tmp2 = DAG.getZeroExtendInReg(Tmp2, dl, VT); 4831 Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4832 break; 4833 4834 case ISD::SHL: 4835 Tmp1 = PromoteOp(Node->getOperand(0)); 4836 Result = DAG.getNode(ISD::SHL, dl, NVT, Tmp1, Node->getOperand(1)); 4837 break; 4838 case ISD::SRA: 4839 // The input value must be properly sign extended. 4840 Tmp1 = PromoteOp(Node->getOperand(0)); 4841 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, 4842 DAG.getValueType(VT)); 4843 Result = DAG.getNode(ISD::SRA, dl, NVT, Tmp1, Node->getOperand(1)); 4844 break; 4845 case ISD::SRL: 4846 // The input value must be properly zero extended. 4847 Tmp1 = PromoteOp(Node->getOperand(0)); 4848 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); 4849 Result = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, Node->getOperand(1)); 4850 break; 4851 4852 case ISD::VAARG: 4853 Tmp1 = Node->getOperand(0); // Get the chain. 4854 Tmp2 = Node->getOperand(1); // Get the pointer. 4855 if (TLI.getOperationAction(ISD::VAARG, VT) == TargetLowering::Custom) { 4856 Tmp3 = DAG.getVAArg(VT, dl, Tmp1, Tmp2, Node->getOperand(2)); 4857 Result = TLI.LowerOperation(Tmp3, DAG); 4858 } else { 4859 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 4860 SDValue VAList = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, V, 0); 4861 // Increment the pointer, VAList, to the next vaarg 4862 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 4863 DAG.getConstant(VT.getSizeInBits()/8, 4864 TLI.getPointerTy())); 4865 // Store the incremented VAList to the legalized pointer 4866 Tmp3 = DAG.getStore(VAList.getValue(1), dl, Tmp3, Tmp2, V, 0); 4867 // Load the actual argument out of the pointer VAList 4868 Result = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Tmp3, VAList, NULL, 0, VT); 4869 } 4870 // Remember that we legalized the chain. 4871 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4872 break; 4873 4874 case ISD::LOAD: { 4875 LoadSDNode *LD = cast<LoadSDNode>(Node); 4876 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(Node) 4877 ? ISD::EXTLOAD : LD->getExtensionType(); 4878 Result = DAG.getExtLoad(ExtType, dl, NVT, 4879 LD->getChain(), LD->getBasePtr(), 4880 LD->getSrcValue(), LD->getSrcValueOffset(), 4881 LD->getMemoryVT(), 4882 LD->isVolatile(), 4883 LD->getAlignment()); 4884 // Remember that we legalized the chain. 4885 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 4886 break; 4887 } 4888 case ISD::SELECT: { 4889 Tmp2 = PromoteOp(Node->getOperand(1)); // Legalize the op0 4890 Tmp3 = PromoteOp(Node->getOperand(2)); // Legalize the op1 4891 4892 MVT VT2 = Tmp2.getValueType(); 4893 assert(VT2 == Tmp3.getValueType() 4894 && "PromoteOp SELECT: Operands 2 and 3 ValueTypes don't match"); 4895 // Ensure that the resulting node is at least the same size as the operands' 4896 // value types, because we cannot assume that TLI.getSetCCValueType() is 4897 // constant. 4898 Result = DAG.getNode(ISD::SELECT, dl, VT2, Node->getOperand(0), Tmp2, Tmp3); 4899 break; 4900 } 4901 case ISD::SELECT_CC: 4902 Tmp2 = PromoteOp(Node->getOperand(2)); // True 4903 Tmp3 = PromoteOp(Node->getOperand(3)); // False 4904 Result = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), 4905 Node->getOperand(1), Tmp2, Tmp3, Node->getOperand(4)); 4906 break; 4907 case ISD::BSWAP: 4908 Tmp1 = Node->getOperand(0); 4909 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1); 4910 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 4911 Result = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 4912 DAG.getConstant(NVT.getSizeInBits() - 4913 VT.getSizeInBits(), 4914 TLI.getShiftAmountTy())); 4915 break; 4916 case ISD::CTPOP: 4917 case ISD::CTTZ: 4918 case ISD::CTLZ: 4919 // Zero extend the argument 4920 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 4921 // Perform the larger operation, then subtract if needed. 4922 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4923 switch(Node->getOpcode()) { 4924 case ISD::CTPOP: 4925 Result = Tmp1; 4926 break; 4927 case ISD::CTTZ: 4928 // if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT) 4929 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), Tmp1, 4930 DAG.getConstant(NVT.getSizeInBits(), NVT), 4931 ISD::SETEQ); 4932 Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, 4933 DAG.getConstant(VT.getSizeInBits(), NVT), Tmp1); 4934 break; 4935 case ISD::CTLZ: 4936 //Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 4937 Result = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 4938 DAG.getConstant(NVT.getSizeInBits() - 4939 VT.getSizeInBits(), NVT)); 4940 break; 4941 } 4942 break; 4943 case ISD::EXTRACT_SUBVECTOR: 4944 Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op)); 4945 break; 4946 case ISD::EXTRACT_VECTOR_ELT: 4947 Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op)); 4948 break; 4949 } 4950 4951 assert(Result.getNode() && "Didn't set a result!"); 4952 4953 // Make sure the result is itself legal. 4954 Result = LegalizeOp(Result); 4955 4956 // Remember that we promoted this! 4957 AddPromotedOperand(Op, Result); 4958 return Result; 4959} 4960 4961/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into 4962/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic, 4963/// based on the vector type. The return type of this matches the element type 4964/// of the vector, which may not be legal for the target. 4965SDValue SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDValue Op) { 4966 // We know that operand #0 is the Vec vector. If the index is a constant 4967 // or if the invec is a supported hardware type, we can use it. Otherwise, 4968 // lower to a store then an indexed load. 4969 SDValue Vec = Op.getOperand(0); 4970 SDValue Idx = Op.getOperand(1); 4971 DebugLoc dl = Op.getDebugLoc(); 4972 4973 MVT TVT = Vec.getValueType(); 4974 unsigned NumElems = TVT.getVectorNumElements(); 4975 4976 switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) { 4977 default: assert(0 && "This action is not supported yet!"); 4978 case TargetLowering::Custom: { 4979 Vec = LegalizeOp(Vec); 4980 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 4981 SDValue Tmp3 = TLI.LowerOperation(Op, DAG); 4982 if (Tmp3.getNode()) 4983 return Tmp3; 4984 break; 4985 } 4986 case TargetLowering::Legal: 4987 if (isTypeLegal(TVT)) { 4988 Vec = LegalizeOp(Vec); 4989 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 4990 return Op; 4991 } 4992 break; 4993 case TargetLowering::Promote: 4994 assert(TVT.isVector() && "not vector type"); 4995 // fall thru to expand since vectors are by default are promote 4996 case TargetLowering::Expand: 4997 break; 4998 } 4999 5000 if (NumElems == 1) { 5001 // This must be an access of the only element. Return it. 5002 Op = ScalarizeVectorOp(Vec); 5003 } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) { 5004 unsigned NumLoElts = 1 << Log2_32(NumElems-1); 5005 ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx); 5006 SDValue Lo, Hi; 5007 SplitVectorOp(Vec, Lo, Hi); 5008 if (CIdx->getZExtValue() < NumLoElts) { 5009 Vec = Lo; 5010 } else { 5011 Vec = Hi; 5012 Idx = DAG.getConstant(CIdx->getZExtValue() - NumLoElts, 5013 Idx.getValueType()); 5014 } 5015 5016 // It's now an extract from the appropriate high or low part. Recurse. 5017 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 5018 Op = ExpandEXTRACT_VECTOR_ELT(Op); 5019 } else { 5020 // Store the value to a temporary stack slot, then LOAD the scalar 5021 // element back out. 5022 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 5023 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 5024 5025 // Add the offset to the index. 5026 unsigned EltSize = Op.getValueType().getSizeInBits()/8; 5027 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 5028 DAG.getConstant(EltSize, Idx.getValueType())); 5029 5030 if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 5031 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx); 5032 else 5033 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 5034 5035 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); 5036 5037 Op = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, NULL, 0); 5038 } 5039 return Op; 5040} 5041 5042/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation. For now 5043/// we assume the operation can be split if it is not already legal. 5044SDValue SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDValue Op) { 5045 // We know that operand #0 is the Vec vector. For now we assume the index 5046 // is a constant and that the extracted result is a supported hardware type. 5047 SDValue Vec = Op.getOperand(0); 5048 SDValue Idx = LegalizeOp(Op.getOperand(1)); 5049 5050 unsigned NumElems = Vec.getValueType().getVectorNumElements(); 5051 5052 if (NumElems == Op.getValueType().getVectorNumElements()) { 5053 // This must be an access of the desired vector length. Return it. 5054 return Vec; 5055 } 5056 5057 ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx); 5058 SDValue Lo, Hi; 5059 SplitVectorOp(Vec, Lo, Hi); 5060 if (CIdx->getZExtValue() < NumElems/2) { 5061 Vec = Lo; 5062 } else { 5063 Vec = Hi; 5064 Idx = DAG.getConstant(CIdx->getZExtValue() - NumElems/2, 5065 Idx.getValueType()); 5066 } 5067 5068 // It's now an extract from the appropriate high or low part. Recurse. 5069 Op = DAG.UpdateNodeOperands(Op, Vec, Idx); 5070 return ExpandEXTRACT_SUBVECTOR(Op); 5071} 5072 5073/// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC 5074/// with condition CC on the current target. This usually involves legalizing 5075/// or promoting the arguments. In the case where LHS and RHS must be expanded, 5076/// there may be no choice but to create a new SetCC node to represent the 5077/// legalized value of setcc lhs, rhs. In this case, the value is returned in 5078/// LHS, and the SDValue returned in RHS has a nil SDNode value. 5079void SelectionDAGLegalize::LegalizeSetCCOperands(SDValue &LHS, 5080 SDValue &RHS, 5081 SDValue &CC, 5082 DebugLoc dl) { 5083 SDValue Tmp1, Tmp2, Tmp3, Result; 5084 5085 switch (getTypeAction(LHS.getValueType())) { 5086 case Legal: 5087 Tmp1 = LegalizeOp(LHS); // LHS 5088 Tmp2 = LegalizeOp(RHS); // RHS 5089 break; 5090 case Promote: 5091 Tmp1 = PromoteOp(LHS); // LHS 5092 Tmp2 = PromoteOp(RHS); // RHS 5093 5094 // If this is an FP compare, the operands have already been extended. 5095 if (LHS.getValueType().isInteger()) { 5096 MVT VT = LHS.getValueType(); 5097 MVT NVT = TLI.getTypeToTransformTo(VT); 5098 5099 // Otherwise, we have to insert explicit sign or zero extends. Note 5100 // that we could insert sign extends for ALL conditions, but zero extend 5101 // is cheaper on many machines (an AND instead of two shifts), so prefer 5102 // it. 5103 switch (cast<CondCodeSDNode>(CC)->get()) { 5104 default: assert(0 && "Unknown integer comparison!"); 5105 case ISD::SETEQ: 5106 case ISD::SETNE: 5107 case ISD::SETUGE: 5108 case ISD::SETUGT: 5109 case ISD::SETULE: 5110 case ISD::SETULT: 5111 // ALL of these operations will work if we either sign or zero extend 5112 // the operands (including the unsigned comparisons!). Zero extend is 5113 // usually a simpler/cheaper operation, so prefer it. 5114 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); 5115 Tmp2 = DAG.getZeroExtendInReg(Tmp2, dl, VT); 5116 break; 5117 case ISD::SETGE: 5118 case ISD::SETGT: 5119 case ISD::SETLT: 5120 case ISD::SETLE: 5121 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, 5122 DAG.getValueType(VT)); 5123 Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp2, 5124 DAG.getValueType(VT)); 5125 Tmp1 = LegalizeOp(Tmp1); // Relegalize new nodes. 5126 Tmp2 = LegalizeOp(Tmp2); // Relegalize new nodes. 5127 break; 5128 } 5129 } 5130 break; 5131 case Expand: { 5132 MVT VT = LHS.getValueType(); 5133 if (VT == MVT::f32 || VT == MVT::f64) { 5134 // Expand into one or more soft-fp libcall(s). 5135 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL; 5136 switch (cast<CondCodeSDNode>(CC)->get()) { 5137 case ISD::SETEQ: 5138 case ISD::SETOEQ: 5139 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; 5140 break; 5141 case ISD::SETNE: 5142 case ISD::SETUNE: 5143 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : RTLIB::UNE_F64; 5144 break; 5145 case ISD::SETGE: 5146 case ISD::SETOGE: 5147 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; 5148 break; 5149 case ISD::SETLT: 5150 case ISD::SETOLT: 5151 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5152 break; 5153 case ISD::SETLE: 5154 case ISD::SETOLE: 5155 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; 5156 break; 5157 case ISD::SETGT: 5158 case ISD::SETOGT: 5159 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; 5160 break; 5161 case ISD::SETUO: 5162 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; 5163 break; 5164 case ISD::SETO: 5165 LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : RTLIB::O_F64; 5166 break; 5167 default: 5168 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; 5169 switch (cast<CondCodeSDNode>(CC)->get()) { 5170 case ISD::SETONE: 5171 // SETONE = SETOLT | SETOGT 5172 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5173 // Fallthrough 5174 case ISD::SETUGT: 5175 LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; 5176 break; 5177 case ISD::SETUGE: 5178 LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; 5179 break; 5180 case ISD::SETULT: 5181 LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; 5182 break; 5183 case ISD::SETULE: 5184 LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; 5185 break; 5186 case ISD::SETUEQ: 5187 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; 5188 break; 5189 default: assert(0 && "Unsupported FP setcc!"); 5190 } 5191 } 5192 5193 SDValue Dummy; 5194 SDValue Ops[2] = { LHS, RHS }; 5195 Tmp1 = ExpandLibCall(LC1, DAG.getMergeValues(Ops, 2, dl).getNode(), 5196 false /*sign irrelevant*/, Dummy); 5197 Tmp2 = DAG.getConstant(0, MVT::i32); 5198 CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1)); 5199 if (LC2 != RTLIB::UNKNOWN_LIBCALL) { 5200 Tmp1 = DAG.getNode(ISD::SETCC, dl, 5201 TLI.getSetCCResultType(Tmp1.getValueType()), 5202 Tmp1, Tmp2, CC); 5203 LHS = ExpandLibCall(LC2, DAG.getMergeValues(Ops, 2, dl).getNode(), 5204 false /*sign irrelevant*/, Dummy); 5205 Tmp2 = DAG.getNode(ISD::SETCC, dl, 5206 TLI.getSetCCResultType(LHS.getValueType()), LHS, 5207 Tmp2, DAG.getCondCode(TLI.getCmpLibcallCC(LC2))); 5208 Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp2); 5209 Tmp2 = SDValue(); 5210 } 5211 LHS = LegalizeOp(Tmp1); 5212 RHS = Tmp2; 5213 return; 5214 } 5215 5216 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 5217 ExpandOp(LHS, LHSLo, LHSHi); 5218 ExpandOp(RHS, RHSLo, RHSHi); 5219 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 5220 5221 if (VT==MVT::ppcf128) { 5222 // FIXME: This generated code sucks. We want to generate 5223 // FCMPU crN, hi1, hi2 5224 // BNE crN, L: 5225 // FCMPU crN, lo1, lo2 5226 // The following can be improved, but not that much. 5227 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 5228 LHSHi, RHSHi, ISD::SETOEQ); 5229 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), 5230 LHSLo, RHSLo, CCCode); 5231 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 5232 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 5233 LHSHi, RHSHi, ISD::SETUNE); 5234 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 5235 LHSHi, RHSHi, CCCode); 5236 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 5237 Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 5238 Tmp2 = SDValue(); 5239 break; 5240 } 5241 5242 switch (CCCode) { 5243 case ISD::SETEQ: 5244 case ISD::SETNE: 5245 if (RHSLo == RHSHi) 5246 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) 5247 if (RHSCST->isAllOnesValue()) { 5248 // Comparison to -1. 5249 Tmp1 = DAG.getNode(ISD::AND, dl,LHSLo.getValueType(), LHSLo, LHSHi); 5250 Tmp2 = RHSLo; 5251 break; 5252 } 5253 5254 Tmp1 = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 5255 Tmp2 = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 5256 Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp2); 5257 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 5258 break; 5259 default: 5260 // If this is a comparison of the sign bit, just look at the top part. 5261 // X > -1, x < 0 5262 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(RHS)) 5263 if ((cast<CondCodeSDNode>(CC)->get() == ISD::SETLT && 5264 CST->isNullValue()) || // X < 0 5265 (cast<CondCodeSDNode>(CC)->get() == ISD::SETGT && 5266 CST->isAllOnesValue())) { // X > -1 5267 Tmp1 = LHSHi; 5268 Tmp2 = RHSHi; 5269 break; 5270 } 5271 5272 // FIXME: This generated code sucks. 5273 ISD::CondCode LowCC; 5274 switch (CCCode) { 5275 default: assert(0 && "Unknown integer setcc!"); 5276 case ISD::SETLT: 5277 case ISD::SETULT: LowCC = ISD::SETULT; break; 5278 case ISD::SETGT: 5279 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 5280 case ISD::SETLE: 5281 case ISD::SETULE: LowCC = ISD::SETULE; break; 5282 case ISD::SETGE: 5283 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 5284 } 5285 5286 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 5287 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 5288 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 5289 5290 // NOTE: on targets without efficient SELECT of bools, we can always use 5291 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 5292 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL); 5293 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), 5294 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); 5295 if (!Tmp1.getNode()) 5296 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), 5297 LHSLo, RHSLo, LowCC); 5298 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 5299 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); 5300 if (!Tmp2.getNode()) 5301 Tmp2 = DAG.getNode(ISD::SETCC, dl, 5302 TLI.getSetCCResultType(LHSHi.getValueType()), 5303 LHSHi, RHSHi, CC); 5304 5305 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 5306 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 5307 if ((Tmp1C && Tmp1C->isNullValue()) || 5308 (Tmp2C && Tmp2C->isNullValue() && 5309 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 5310 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 5311 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 5312 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 5313 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 5314 // low part is known false, returns high part. 5315 // For LE / GE, if high part is known false, ignore the low part. 5316 // For LT / GT, if high part is known true, ignore the low part. 5317 Tmp1 = Tmp2; 5318 Tmp2 = SDValue(); 5319 } else { 5320 Result = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 5321 LHSHi, RHSHi, ISD::SETEQ, false, 5322 DagCombineInfo, dl); 5323 if (!Result.getNode()) 5324 Result=DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 5325 LHSHi, RHSHi, ISD::SETEQ); 5326 Result = LegalizeOp(DAG.getNode(ISD::SELECT, dl, Tmp1.getValueType(), 5327 Result, Tmp1, Tmp2)); 5328 Tmp1 = Result; 5329 Tmp2 = SDValue(); 5330 } 5331 } 5332 } 5333 } 5334 LHS = Tmp1; 5335 RHS = Tmp2; 5336} 5337 5338/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and 5339/// condition code CC on the current target. This routine assumes LHS and rHS 5340/// have already been legalized by LegalizeSetCCOperands. It expands SETCC with 5341/// illegal condition code into AND / OR of multiple SETCC values. 5342void SelectionDAGLegalize::LegalizeSetCCCondCode(MVT VT, 5343 SDValue &LHS, SDValue &RHS, 5344 SDValue &CC, 5345 DebugLoc dl) { 5346 MVT OpVT = LHS.getValueType(); 5347 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 5348 switch (TLI.getCondCodeAction(CCCode, OpVT)) { 5349 default: assert(0 && "Unknown condition code action!"); 5350 case TargetLowering::Legal: 5351 // Nothing to do. 5352 break; 5353 case TargetLowering::Expand: { 5354 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 5355 unsigned Opc = 0; 5356 switch (CCCode) { 5357 default: assert(0 && "Don't know how to expand this condition!"); abort(); 5358 case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break; 5359 case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break; 5360 case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5361 case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break; 5362 case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5363 case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break; 5364 case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5365 case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5366 case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5367 case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5368 case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5369 case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break; 5370 // FIXME: Implement more expansions. 5371 } 5372 5373 SDValue SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); 5374 SDValue SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); 5375 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 5376 RHS = SDValue(); 5377 CC = SDValue(); 5378 break; 5379 } 5380 } 5381} 5382 5383/// EmitStackConvert - Emit a store/load combination to the stack. This stores 5384/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 5385/// a load from the stack slot to DestVT, extending it if needed. 5386/// The resultant code need not be legal. 5387SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, 5388 MVT SlotVT, 5389 MVT DestVT, 5390 DebugLoc dl) { 5391 // Create the stack frame object. 5392 unsigned SrcAlign = 5393 TLI.getTargetData()->getPrefTypeAlignment(SrcOp.getValueType(). 5394 getTypeForMVT()); 5395 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 5396 5397 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 5398 int SPFI = StackPtrFI->getIndex(); 5399 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 5400 5401 unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); 5402 unsigned SlotSize = SlotVT.getSizeInBits(); 5403 unsigned DestSize = DestVT.getSizeInBits(); 5404 unsigned DestAlign = 5405 TLI.getTargetData()->getPrefTypeAlignment(DestVT.getTypeForMVT()); 5406 5407 // Emit a store to the stack slot. Use a truncstore if the input value is 5408 // later than DestVT. 5409 SDValue Store; 5410 5411 if (SrcSize > SlotSize) 5412 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 5413 SV, 0, SlotVT, false, SrcAlign); 5414 else { 5415 assert(SrcSize == SlotSize && "Invalid store"); 5416 Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 5417 SV, 0, false, SrcAlign); 5418 } 5419 5420 // Result is a load from the stack slot. 5421 if (SlotSize == DestSize) 5422 return DAG.getLoad(DestVT, dl, Store, FIPtr, SV, 0, false, DestAlign); 5423 5424 assert(SlotSize < DestSize && "Unknown extension!"); 5425 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, SV, 0, SlotVT, 5426 false, DestAlign); 5427} 5428 5429SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 5430 DebugLoc dl = Node->getDebugLoc(); 5431 // Create a vector sized/aligned stack slot, store the value to element #0, 5432 // then load the whole vector back out. 5433 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 5434 5435 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 5436 int SPFI = StackPtrFI->getIndex(); 5437 5438 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0), 5439 StackPtr, 5440 PseudoSourceValue::getFixedStack(SPFI), 0, 5441 Node->getValueType(0).getVectorElementType()); 5442 return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr, 5443 PseudoSourceValue::getFixedStack(SPFI), 0); 5444} 5445 5446 5447/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't 5448/// support the operation, but do support the resultant vector type. 5449SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 5450 unsigned NumElems = Node->getNumOperands(); 5451 SDValue SplatValue = Node->getOperand(0); 5452 DebugLoc dl = Node->getDebugLoc(); 5453 MVT VT = Node->getValueType(0); 5454 MVT OpVT = SplatValue.getValueType(); 5455 MVT EltVT = VT.getVectorElementType(); 5456 5457 // If the only non-undef value is the low element, turn this into a 5458 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 5459 bool isOnlyLowElement = true; 5460 5461 // FIXME: it would be far nicer to change this into map<SDValue,uint64_t> 5462 // and use a bitmask instead of a list of elements. 5463 // FIXME: this doesn't treat <0, u, 0, u> for example, as a splat. 5464 std::map<SDValue, std::vector<unsigned> > Values; 5465 Values[SplatValue].push_back(0); 5466 bool isConstant = true; 5467 if (!isa<ConstantFPSDNode>(SplatValue) && !isa<ConstantSDNode>(SplatValue) && 5468 SplatValue.getOpcode() != ISD::UNDEF) 5469 isConstant = false; 5470 5471 for (unsigned i = 1; i < NumElems; ++i) { 5472 SDValue V = Node->getOperand(i); 5473 Values[V].push_back(i); 5474 if (V.getOpcode() != ISD::UNDEF) 5475 isOnlyLowElement = false; 5476 if (SplatValue != V) 5477 SplatValue = SDValue(0, 0); 5478 5479 // If this isn't a constant element or an undef, we can't use a constant 5480 // pool load. 5481 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V) && 5482 V.getOpcode() != ISD::UNDEF) 5483 isConstant = false; 5484 } 5485 5486 if (isOnlyLowElement) { 5487 // If the low element is an undef too, then this whole things is an undef. 5488 if (Node->getOperand(0).getOpcode() == ISD::UNDEF) 5489 return DAG.getUNDEF(VT); 5490 // Otherwise, turn this into a scalar_to_vector node. 5491 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); 5492 } 5493 5494 // If all elements are constants, create a load from the constant pool. 5495 if (isConstant) { 5496 std::vector<Constant*> CV; 5497 for (unsigned i = 0, e = NumElems; i != e; ++i) { 5498 if (ConstantFPSDNode *V = 5499 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 5500 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 5501 } else if (ConstantSDNode *V = 5502 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 5503 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 5504 } else { 5505 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); 5506 const Type *OpNTy = OpVT.getTypeForMVT(); 5507 CV.push_back(UndefValue::get(OpNTy)); 5508 } 5509 } 5510 Constant *CP = ConstantVector::get(CV); 5511 SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); 5512 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 5513 return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 5514 PseudoSourceValue::getConstantPool(), 0, 5515 false, Alignment); 5516 } 5517 5518 if (SplatValue.getNode()) { // Splat of one value? 5519 // Build the shuffle constant vector: <0, 0, 0, 0> 5520 SmallVector<int, 8> ZeroVec(NumElems, 0); 5521 5522 // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it. 5523 if (TLI.isShuffleMaskLegal(ZeroVec, Node->getValueType(0))) { 5524 // Get the splatted value into the low element of a vector register. 5525 SDValue LowValVec = 5526 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, SplatValue); 5527 5528 // Return shuffle(LowValVec, undef, <0,0,0,0>) 5529 return DAG.getVectorShuffle(VT, dl, LowValVec, DAG.getUNDEF(VT), 5530 &ZeroVec[0]); 5531 } 5532 } 5533 5534 // If there are only two unique elements, we may be able to turn this into a 5535 // vector shuffle. 5536 if (Values.size() == 2) { 5537 // Get the two values in deterministic order. 5538 SDValue Val1 = Node->getOperand(1); 5539 SDValue Val2; 5540 std::map<SDValue, std::vector<unsigned> >::iterator MI = Values.begin(); 5541 if (MI->first != Val1) 5542 Val2 = MI->first; 5543 else 5544 Val2 = (++MI)->first; 5545 5546 // If Val1 is an undef, make sure it ends up as Val2, to ensure that our 5547 // vector shuffle has the undef vector on the RHS. 5548 if (Val1.getOpcode() == ISD::UNDEF) 5549 std::swap(Val1, Val2); 5550 5551 // Build the shuffle constant vector: e.g. <0, 4, 0, 4> 5552 SmallVector<int, 8> ShuffleMask(NumElems, -1); 5553 5554 // Set elements of the shuffle mask for Val1. 5555 std::vector<unsigned> &Val1Elts = Values[Val1]; 5556 for (unsigned i = 0, e = Val1Elts.size(); i != e; ++i) 5557 ShuffleMask[Val1Elts[i]] = 0; 5558 5559 // Set elements of the shuffle mask for Val2. 5560 std::vector<unsigned> &Val2Elts = Values[Val2]; 5561 for (unsigned i = 0, e = Val2Elts.size(); i != e; ++i) 5562 if (Val2.getOpcode() != ISD::UNDEF) 5563 ShuffleMask[Val2Elts[i]] = NumElems; 5564 5565 // If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it. 5566 if (TLI.isOperationLegalOrCustom(ISD::SCALAR_TO_VECTOR, VT) && 5567 TLI.isShuffleMaskLegal(ShuffleMask, VT)) { 5568 Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Val1); 5569 Val2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Val2); 5570 return DAG.getVectorShuffle(VT, dl, Val1, Val2, &ShuffleMask[0]); 5571 } 5572 } 5573 5574 // Otherwise, we can't handle this case efficiently. Allocate a sufficiently 5575 // aligned object on the stack, store each element into it, then load 5576 // the result as a vector. 5577 // Create the stack frame object. 5578 SDValue FIPtr = DAG.CreateStackTemporary(VT); 5579 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 5580 const Value *SV = PseudoSourceValue::getFixedStack(FI); 5581 5582 // Emit a store of each element to the stack slot. 5583 SmallVector<SDValue, 8> Stores; 5584 unsigned TypeByteSize = OpVT.getSizeInBits() / 8; 5585 // Store (in the right endianness) the elements to memory. 5586 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 5587 // Ignore undef elements. 5588 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; 5589 5590 unsigned Offset = TypeByteSize*i; 5591 5592 SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); 5593 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); 5594 5595 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), 5596 Idx, SV, Offset)); 5597 } 5598 5599 SDValue StoreChain; 5600 if (!Stores.empty()) // Not all undef elements? 5601 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 5602 &Stores[0], Stores.size()); 5603 else 5604 StoreChain = DAG.getEntryNode(); 5605 5606 // Result is a load from the stack slot. 5607 return DAG.getLoad(VT, dl, StoreChain, FIPtr, SV, 0); 5608} 5609 5610void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp, 5611 SDValue Op, SDValue Amt, 5612 SDValue &Lo, SDValue &Hi, 5613 DebugLoc dl) { 5614 // Expand the subcomponents. 5615 SDValue LHSL, LHSH; 5616 ExpandOp(Op, LHSL, LHSH); 5617 5618 SDValue Ops[] = { LHSL, LHSH, Amt }; 5619 MVT VT = LHSL.getValueType(); 5620 Lo = DAG.getNode(NodeOp, dl, DAG.getVTList(VT, VT), Ops, 3); 5621 Hi = Lo.getValue(1); 5622} 5623 5624 5625/// ExpandShift - Try to find a clever way to expand this shift operation out to 5626/// smaller elements. If we can't find a way that is more efficient than a 5627/// libcall on this target, return false. Otherwise, return true with the 5628/// low-parts expanded into Lo and Hi. 5629bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDValue Op, SDValue Amt, 5630 SDValue &Lo, SDValue &Hi, 5631 DebugLoc dl) { 5632 assert((Opc == ISD::SHL || Opc == ISD::SRA || Opc == ISD::SRL) && 5633 "This is not a shift!"); 5634 5635 MVT NVT = TLI.getTypeToTransformTo(Op.getValueType()); 5636 SDValue ShAmt = LegalizeOp(Amt); 5637 MVT ShTy = ShAmt.getValueType(); 5638 unsigned ShBits = ShTy.getSizeInBits(); 5639 unsigned VTBits = Op.getValueType().getSizeInBits(); 5640 unsigned NVTBits = NVT.getSizeInBits(); 5641 5642 // Handle the case when Amt is an immediate. 5643 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Amt.getNode())) { 5644 unsigned Cst = CN->getZExtValue(); 5645 // Expand the incoming operand to be shifted, so that we have its parts 5646 SDValue InL, InH; 5647 ExpandOp(Op, InL, InH); 5648 switch(Opc) { 5649 case ISD::SHL: 5650 if (Cst > VTBits) { 5651 Lo = DAG.getConstant(0, NVT); 5652 Hi = DAG.getConstant(0, NVT); 5653 } else if (Cst > NVTBits) { 5654 Lo = DAG.getConstant(0, NVT); 5655 Hi = DAG.getNode(ISD::SHL, dl, 5656 NVT, InL, DAG.getConstant(Cst-NVTBits, ShTy)); 5657 } else if (Cst == NVTBits) { 5658 Lo = DAG.getConstant(0, NVT); 5659 Hi = InL; 5660 } else { 5661 Lo = DAG.getNode(ISD::SHL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)); 5662 Hi = DAG.getNode(ISD::OR, dl, NVT, 5663 DAG.getNode(ISD::SHL, dl, NVT, InH, DAG.getConstant(Cst, ShTy)), 5664 DAG.getNode(ISD::SRL, dl, NVT, InL, 5665 DAG.getConstant(NVTBits-Cst, ShTy))); 5666 } 5667 return true; 5668 case ISD::SRL: 5669 if (Cst > VTBits) { 5670 Lo = DAG.getConstant(0, NVT); 5671 Hi = DAG.getConstant(0, NVT); 5672 } else if (Cst > NVTBits) { 5673 Lo = DAG.getNode(ISD::SRL, dl, NVT, 5674 InH, DAG.getConstant(Cst-NVTBits, ShTy)); 5675 Hi = DAG.getConstant(0, NVT); 5676 } else if (Cst == NVTBits) { 5677 Lo = InH; 5678 Hi = DAG.getConstant(0, NVT); 5679 } else { 5680 Lo = DAG.getNode(ISD::OR, dl, NVT, 5681 DAG.getNode(ISD::SRL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)), 5682 DAG.getNode(ISD::SHL, dl, NVT, InH, 5683 DAG.getConstant(NVTBits-Cst, ShTy))); 5684 Hi = DAG.getNode(ISD::SRL, dl, NVT, InH, DAG.getConstant(Cst, ShTy)); 5685 } 5686 return true; 5687 case ISD::SRA: 5688 if (Cst > VTBits) { 5689 Hi = Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, 5690 DAG.getConstant(NVTBits-1, ShTy)); 5691 } else if (Cst > NVTBits) { 5692 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, 5693 DAG.getConstant(Cst-NVTBits, ShTy)); 5694 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, 5695 DAG.getConstant(NVTBits-1, ShTy)); 5696 } else if (Cst == NVTBits) { 5697 Lo = InH; 5698 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, 5699 DAG.getConstant(NVTBits-1, ShTy)); 5700 } else { 5701 Lo = DAG.getNode(ISD::OR, dl, NVT, 5702 DAG.getNode(ISD::SRL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)), 5703 DAG.getNode(ISD::SHL, dl, 5704 NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy))); 5705 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, DAG.getConstant(Cst, ShTy)); 5706 } 5707 return true; 5708 } 5709 } 5710 5711 // Okay, the shift amount isn't constant. However, if we can tell that it is 5712 // >= 32 or < 32, we can still simplify it, without knowing the actual value. 5713 APInt Mask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 5714 APInt KnownZero, KnownOne; 5715 DAG.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne); 5716 5717 // If we know that if any of the high bits of the shift amount are one, then 5718 // we can do this as a couple of simple shifts. 5719 if (KnownOne.intersects(Mask)) { 5720 // Mask out the high bit, which we know is set. 5721 Amt = DAG.getNode(ISD::AND, dl, Amt.getValueType(), Amt, 5722 DAG.getConstant(~Mask, Amt.getValueType())); 5723 5724 // Expand the incoming operand to be shifted, so that we have its parts 5725 SDValue InL, InH; 5726 ExpandOp(Op, InL, InH); 5727 switch(Opc) { 5728 case ISD::SHL: 5729 Lo = DAG.getConstant(0, NVT); // Low part is zero. 5730 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 5731 return true; 5732 case ISD::SRL: 5733 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 5734 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 5735 return true; 5736 case ISD::SRA: 5737 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 5738 DAG.getConstant(NVTBits-1, Amt.getValueType())); 5739 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 5740 return true; 5741 } 5742 } 5743 5744 // If we know that the high bits of the shift amount are all zero, then we can 5745 // do this as a couple of simple shifts. 5746 if ((KnownZero & Mask) == Mask) { 5747 // Compute 32-amt. 5748 SDValue Amt2 = DAG.getNode(ISD::SUB, dl, Amt.getValueType(), 5749 DAG.getConstant(NVTBits, Amt.getValueType()), 5750 Amt); 5751 5752 // Expand the incoming operand to be shifted, so that we have its parts 5753 SDValue InL, InH; 5754 ExpandOp(Op, InL, InH); 5755 switch(Opc) { 5756 case ISD::SHL: 5757 Lo = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 5758 Hi = DAG.getNode(ISD::OR, dl, NVT, 5759 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 5760 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt2)); 5761 return true; 5762 case ISD::SRL: 5763 Hi = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 5764 Lo = DAG.getNode(ISD::OR, dl, NVT, 5765 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 5766 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt2)); 5767 return true; 5768 case ISD::SRA: 5769 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 5770 Lo = DAG.getNode(ISD::OR, dl, NVT, 5771 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 5772 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt2)); 5773 return true; 5774 } 5775 } 5776 5777 return false; 5778} 5779 5780 5781// ExpandLibCall - Expand a node into a call to a libcall. If the result value 5782// does not fit into a register, return the lo part and set the hi part to the 5783// by-reg argument. If it does fit into a single register, return the result 5784// and leave the Hi part unset. 5785SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 5786 bool isSigned, SDValue &Hi) { 5787 assert(!IsLegalizingCall && "Cannot overlap legalization of calls!"); 5788 // The input chain to this libcall is the entry node of the function. 5789 // Legalizing the call will automatically add the previous call to the 5790 // dependence. 5791 SDValue InChain = DAG.getEntryNode(); 5792 5793 TargetLowering::ArgListTy Args; 5794 TargetLowering::ArgListEntry Entry; 5795 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 5796 MVT ArgVT = Node->getOperand(i).getValueType(); 5797 const Type *ArgTy = ArgVT.getTypeForMVT(); 5798 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 5799 Entry.isSExt = isSigned; 5800 Entry.isZExt = !isSigned; 5801 Args.push_back(Entry); 5802 } 5803 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 5804 TLI.getPointerTy()); 5805 5806 // Splice the libcall in wherever FindInputOutputChains tells us to. 5807 const Type *RetTy = Node->getValueType(0).getTypeForMVT(); 5808 std::pair<SDValue, SDValue> CallInfo = 5809 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, 5810 CallingConv::C, false, Callee, Args, DAG, 5811 Node->getDebugLoc()); 5812 5813 // Legalize the call sequence, starting with the chain. This will advance 5814 // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that 5815 // was added by LowerCallTo (guaranteeing proper serialization of calls). 5816 LegalizeOp(CallInfo.second); 5817 SDValue Result; 5818 switch (getTypeAction(CallInfo.first.getValueType())) { 5819 default: assert(0 && "Unknown thing"); 5820 case Legal: 5821 Result = CallInfo.first; 5822 break; 5823 case Expand: 5824 ExpandOp(CallInfo.first, Result, Hi); 5825 break; 5826 } 5827 return Result; 5828} 5829 5830/// LegalizeINT_TO_FP - Legalize a [US]INT_TO_FP operation. 5831/// 5832SDValue SelectionDAGLegalize:: 5833LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op, 5834 DebugLoc dl) { 5835 bool isCustom = false; 5836 SDValue Tmp1; 5837 switch (getTypeAction(Op.getValueType())) { 5838 case Legal: 5839 switch (TLI.getOperationAction(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, 5840 Op.getValueType())) { 5841 default: assert(0 && "Unknown operation action!"); 5842 case TargetLowering::Custom: 5843 isCustom = true; 5844 // FALLTHROUGH 5845 case TargetLowering::Legal: 5846 Tmp1 = LegalizeOp(Op); 5847 if (Result.getNode()) 5848 Result = DAG.UpdateNodeOperands(Result, Tmp1); 5849 else 5850 Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, dl, 5851 DestTy, Tmp1); 5852 if (isCustom) { 5853 Tmp1 = TLI.LowerOperation(Result, DAG); 5854 if (Tmp1.getNode()) Result = Tmp1; 5855 } 5856 break; 5857 case TargetLowering::Expand: 5858 Result = ExpandLegalINT_TO_FP(isSigned, LegalizeOp(Op), DestTy, dl); 5859 break; 5860 case TargetLowering::Promote: 5861 Result = PromoteLegalINT_TO_FP(LegalizeOp(Op), DestTy, isSigned, dl); 5862 break; 5863 } 5864 break; 5865 case Expand: 5866 Result = ExpandIntToFP(isSigned, DestTy, Op, dl) ; 5867 break; 5868 case Promote: 5869 Tmp1 = PromoteOp(Op); 5870 if (isSigned) { 5871 Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Tmp1.getValueType(), 5872 Tmp1, DAG.getValueType(Op.getValueType())); 5873 } else { 5874 Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, Op.getValueType()); 5875 } 5876 if (Result.getNode()) 5877 Result = DAG.UpdateNodeOperands(Result, Tmp1); 5878 else 5879 Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, dl, 5880 DestTy, Tmp1); 5881 Result = LegalizeOp(Result); // The 'op' is not necessarily legal! 5882 break; 5883 } 5884 return Result; 5885} 5886 5887/// ExpandIntToFP - Expand a [US]INT_TO_FP operation. 5888/// 5889SDValue SelectionDAGLegalize:: 5890ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source, DebugLoc dl) { 5891 MVT SourceVT = Source.getValueType(); 5892 bool ExpandSource = getTypeAction(SourceVT) == Expand; 5893 5894 // Expand unsupported int-to-fp vector casts by unrolling them. 5895 if (DestTy.isVector()) { 5896 if (!ExpandSource) 5897 return LegalizeOp(UnrollVectorOp(Source)); 5898 MVT DestEltTy = DestTy.getVectorElementType(); 5899 if (DestTy.getVectorNumElements() == 1) { 5900 SDValue Scalar = ScalarizeVectorOp(Source); 5901 SDValue Result = LegalizeINT_TO_FP(SDValue(), isSigned, 5902 DestEltTy, Scalar, dl); 5903 return DAG.getNode(ISD::BUILD_VECTOR, dl, DestTy, Result); 5904 } 5905 SDValue Lo, Hi; 5906 SplitVectorOp(Source, Lo, Hi); 5907 MVT SplitDestTy = MVT::getVectorVT(DestEltTy, 5908 DestTy.getVectorNumElements() / 2); 5909 SDValue LoResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, 5910 Lo, dl); 5911 SDValue HiResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, 5912 Hi, dl); 5913 return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, DestTy, LoResult, 5914 HiResult)); 5915 } 5916 5917 // Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc. 5918 if (!isSigned && SourceVT != MVT::i32) { 5919 // The integer value loaded will be incorrectly if the 'sign bit' of the 5920 // incoming integer is set. To handle this, we dynamically test to see if 5921 // it is set, and, if so, add a fudge factor. 5922 SDValue Hi; 5923 if (ExpandSource) { 5924 SDValue Lo; 5925 ExpandOp(Source, Lo, Hi); 5926 Source = DAG.getNode(ISD::BUILD_PAIR, dl, SourceVT, Lo, Hi); 5927 } else { 5928 // The comparison for the sign bit will use the entire operand. 5929 Hi = Source; 5930 } 5931 5932 // Check to see if the target has a custom way to lower this. If so, use 5933 // it. (Note we've already expanded the operand in this case.) 5934 switch (TLI.getOperationAction(ISD::UINT_TO_FP, SourceVT)) { 5935 default: assert(0 && "This action not implemented for this operation!"); 5936 case TargetLowering::Legal: 5937 case TargetLowering::Expand: 5938 break; // This case is handled below. 5939 case TargetLowering::Custom: { 5940 SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::UINT_TO_FP, dl, DestTy, 5941 Source), DAG); 5942 if (NV.getNode()) 5943 return LegalizeOp(NV); 5944 break; // The target decided this was legal after all 5945 } 5946 } 5947 5948 // If this is unsigned, and not supported, first perform the conversion to 5949 // signed, then adjust the result if the sign bit is set. 5950 SDValue SignedConv = ExpandIntToFP(true, DestTy, Source, dl); 5951 5952 SDValue SignSet = DAG.getSetCC(dl, 5953 TLI.getSetCCResultType(Hi.getValueType()), 5954 Hi, DAG.getConstant(0, Hi.getValueType()), 5955 ISD::SETLT); 5956 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 5957 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), 5958 SignSet, Four, Zero); 5959 uint64_t FF = 0x5f800000ULL; 5960 if (TLI.isLittleEndian()) FF <<= 32; 5961 Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); 5962 5963 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 5964 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 5965 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset); 5966 Alignment = std::min(Alignment, 4u); 5967 SDValue FudgeInReg; 5968 if (DestTy == MVT::f32) 5969 FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 5970 PseudoSourceValue::getConstantPool(), 0, 5971 false, Alignment); 5972 else if (DestTy.bitsGT(MVT::f32)) 5973 // FIXME: Avoid the extend by construction the right constantpool? 5974 FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, dl, DestTy, DAG.getEntryNode(), 5975 CPIdx, 5976 PseudoSourceValue::getConstantPool(), 0, 5977 MVT::f32, false, Alignment); 5978 else 5979 assert(0 && "Unexpected conversion"); 5980 5981 MVT SCVT = SignedConv.getValueType(); 5982 if (SCVT != DestTy) { 5983 // Destination type needs to be expanded as well. The FADD now we are 5984 // constructing will be expanded into a libcall. 5985 if (SCVT.getSizeInBits() != DestTy.getSizeInBits()) { 5986 assert(SCVT.getSizeInBits() * 2 == DestTy.getSizeInBits()); 5987 SignedConv = DAG.getNode(ISD::BUILD_PAIR, dl, DestTy, 5988 SignedConv, SignedConv.getValue(1)); 5989 } 5990 SignedConv = DAG.getNode(ISD::BIT_CONVERT, dl, DestTy, SignedConv); 5991 } 5992 return DAG.getNode(ISD::FADD, dl, DestTy, SignedConv, FudgeInReg); 5993 } 5994 5995 // Check to see if the target has a custom way to lower this. If so, use it. 5996 switch (TLI.getOperationAction(ISD::SINT_TO_FP, SourceVT)) { 5997 default: assert(0 && "This action not implemented for this operation!"); 5998 case TargetLowering::Legal: 5999 case TargetLowering::Expand: 6000 break; // This case is handled below. 6001 case TargetLowering::Custom: { 6002 SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, dl, DestTy, 6003 Source), DAG); 6004 if (NV.getNode()) 6005 return LegalizeOp(NV); 6006 break; // The target decided this was legal after all 6007 } 6008 } 6009 6010 // Expand the source, then glue it back together for the call. We must expand 6011 // the source in case it is shared (this pass of legalize must traverse it). 6012 if (ExpandSource) { 6013 SDValue SrcLo, SrcHi; 6014 ExpandOp(Source, SrcLo, SrcHi); 6015 Source = DAG.getNode(ISD::BUILD_PAIR, dl, SourceVT, SrcLo, SrcHi); 6016 } 6017 6018 RTLIB::Libcall LC = isSigned ? 6019 RTLIB::getSINTTOFP(SourceVT, DestTy) : 6020 RTLIB::getUINTTOFP(SourceVT, DestTy); 6021 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unknown int value type"); 6022 6023 Source = DAG.getNode(ISD::SINT_TO_FP, dl, DestTy, Source); 6024 SDValue HiPart; 6025 SDValue Result = ExpandLibCall(LC, Source.getNode(), isSigned, HiPart); 6026 if (Result.getValueType() != DestTy && HiPart.getNode()) 6027 Result = DAG.getNode(ISD::BUILD_PAIR, dl, DestTy, Result, HiPart); 6028 return Result; 6029} 6030 6031/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a 6032/// INT_TO_FP operation of the specified operand when the target requests that 6033/// we expand it. At this point, we know that the result and operand types are 6034/// legal for the target. 6035SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, 6036 SDValue Op0, 6037 MVT DestVT, 6038 DebugLoc dl) { 6039 if (Op0.getValueType() == MVT::i32) { 6040 // simple 32-bit [signed|unsigned] integer to float/double expansion 6041 6042 // Get the stack frame index of a 8 byte buffer. 6043 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 6044 6045 // word offset constant for Hi/Lo address computation 6046 SDValue WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy()); 6047 // set up Hi and Lo (into buffer) address based on endian 6048 SDValue Hi = StackSlot; 6049 SDValue Lo = DAG.getNode(ISD::ADD, dl, 6050 TLI.getPointerTy(), StackSlot, WordOff); 6051 if (TLI.isLittleEndian()) 6052 std::swap(Hi, Lo); 6053 6054 // if signed map to unsigned space 6055 SDValue Op0Mapped; 6056 if (isSigned) { 6057 // constant used to invert sign bit (signed to unsigned mapping) 6058 SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); 6059 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); 6060 } else { 6061 Op0Mapped = Op0; 6062 } 6063 // store the lo of the constructed double - based on integer input 6064 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, 6065 Op0Mapped, Lo, NULL, 0); 6066 // initial hi portion of constructed double 6067 SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); 6068 // store the hi of the constructed double - biased exponent 6069 SDValue Store2=DAG.getStore(Store1, dl, InitialHi, Hi, NULL, 0); 6070 // load the constructed double 6071 SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot, NULL, 0); 6072 // FP constant to bias correct the final result 6073 SDValue Bias = DAG.getConstantFP(isSigned ? 6074 BitsToDouble(0x4330000080000000ULL) : 6075 BitsToDouble(0x4330000000000000ULL), 6076 MVT::f64); 6077 // subtract the bias 6078 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 6079 // final result 6080 SDValue Result; 6081 // handle final rounding 6082 if (DestVT == MVT::f64) { 6083 // do nothing 6084 Result = Sub; 6085 } else if (DestVT.bitsLT(MVT::f64)) { 6086 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 6087 DAG.getIntPtrConstant(0)); 6088 } else if (DestVT.bitsGT(MVT::f64)) { 6089 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 6090 } 6091 return Result; 6092 } 6093 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 6094 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 6095 6096 SDValue SignSet = DAG.getSetCC(dl, TLI.getSetCCResultType(Op0.getValueType()), 6097 Op0, DAG.getConstant(0, Op0.getValueType()), 6098 ISD::SETLT); 6099 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 6100 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), 6101 SignSet, Four, Zero); 6102 6103 // If the sign bit of the integer is set, the large number will be treated 6104 // as a negative number. To counteract this, the dynamic code adds an 6105 // offset depending on the data type. 6106 uint64_t FF; 6107 switch (Op0.getValueType().getSimpleVT()) { 6108 default: assert(0 && "Unsupported integer type!"); 6109 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 6110 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 6111 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 6112 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 6113 } 6114 if (TLI.isLittleEndian()) FF <<= 32; 6115 Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); 6116 6117 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 6118 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 6119 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset); 6120 Alignment = std::min(Alignment, 4u); 6121 SDValue FudgeInReg; 6122 if (DestVT == MVT::f32) 6123 FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 6124 PseudoSourceValue::getConstantPool(), 0, 6125 false, Alignment); 6126 else { 6127 FudgeInReg = 6128 LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, 6129 DAG.getEntryNode(), CPIdx, 6130 PseudoSourceValue::getConstantPool(), 0, 6131 MVT::f32, false, Alignment)); 6132 } 6133 6134 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 6135} 6136 6137/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a 6138/// *INT_TO_FP operation of the specified operand when the target requests that 6139/// we promote it. At this point, we know that the result and operand types are 6140/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 6141/// operation that takes a larger input. 6142SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, 6143 MVT DestVT, 6144 bool isSigned, 6145 DebugLoc dl) { 6146 // First step, figure out the appropriate *INT_TO_FP operation to use. 6147 MVT NewInTy = LegalOp.getValueType(); 6148 6149 unsigned OpToUse = 0; 6150 6151 // Scan for the appropriate larger type to use. 6152 while (1) { 6153 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT()+1); 6154 assert(NewInTy.isInteger() && "Ran out of possibilities!"); 6155 6156 // If the target supports SINT_TO_FP of this type, use it. 6157 switch (TLI.getOperationAction(ISD::SINT_TO_FP, NewInTy)) { 6158 default: break; 6159 case TargetLowering::Legal: 6160 if (!TLI.isTypeLegal(NewInTy)) 6161 break; // Can't use this datatype. 6162 // FALL THROUGH. 6163 case TargetLowering::Custom: 6164 OpToUse = ISD::SINT_TO_FP; 6165 break; 6166 } 6167 if (OpToUse) break; 6168 if (isSigned) continue; 6169 6170 // If the target supports UINT_TO_FP of this type, use it. 6171 switch (TLI.getOperationAction(ISD::UINT_TO_FP, NewInTy)) { 6172 default: break; 6173 case TargetLowering::Legal: 6174 if (!TLI.isTypeLegal(NewInTy)) 6175 break; // Can't use this datatype. 6176 // FALL THROUGH. 6177 case TargetLowering::Custom: 6178 OpToUse = ISD::UINT_TO_FP; 6179 break; 6180 } 6181 if (OpToUse) break; 6182 6183 // Otherwise, try a larger type. 6184 } 6185 6186 // Okay, we found the operation and type to use. Zero extend our input to the 6187 // desired type then run the operation on it. 6188 return DAG.getNode(OpToUse, dl, DestVT, 6189 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 6190 dl, NewInTy, LegalOp)); 6191} 6192 6193/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a 6194/// FP_TO_*INT operation of the specified operand when the target requests that 6195/// we promote it. At this point, we know that the result and operand types are 6196/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 6197/// operation that returns a larger result. 6198SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, 6199 MVT DestVT, 6200 bool isSigned, 6201 DebugLoc dl) { 6202 // First step, figure out the appropriate FP_TO*INT operation to use. 6203 MVT NewOutTy = DestVT; 6204 6205 unsigned OpToUse = 0; 6206 6207 // Scan for the appropriate larger type to use. 6208 while (1) { 6209 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT()+1); 6210 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 6211 6212 // If the target supports FP_TO_SINT returning this type, use it. 6213 switch (TLI.getOperationAction(ISD::FP_TO_SINT, NewOutTy)) { 6214 default: break; 6215 case TargetLowering::Legal: 6216 if (!TLI.isTypeLegal(NewOutTy)) 6217 break; // Can't use this datatype. 6218 // FALL THROUGH. 6219 case TargetLowering::Custom: 6220 OpToUse = ISD::FP_TO_SINT; 6221 break; 6222 } 6223 if (OpToUse) break; 6224 6225 // If the target supports FP_TO_UINT of this type, use it. 6226 switch (TLI.getOperationAction(ISD::FP_TO_UINT, NewOutTy)) { 6227 default: break; 6228 case TargetLowering::Legal: 6229 if (!TLI.isTypeLegal(NewOutTy)) 6230 break; // Can't use this datatype. 6231 // FALL THROUGH. 6232 case TargetLowering::Custom: 6233 OpToUse = ISD::FP_TO_UINT; 6234 break; 6235 } 6236 if (OpToUse) break; 6237 6238 // Otherwise, try a larger type. 6239 } 6240 6241 6242 // Okay, we found the operation and type to use. 6243 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 6244 6245 // If the operation produces an invalid type, it must be custom lowered. Use 6246 // the target lowering hooks to expand it. Just keep the low part of the 6247 // expanded operation, we know that we're truncating anyway. 6248 if (getTypeAction(NewOutTy) == Expand) { 6249 SmallVector<SDValue, 2> Results; 6250 TLI.ReplaceNodeResults(Operation.getNode(), Results, DAG); 6251 assert(Results.size() == 1 && "Incorrect FP_TO_XINT lowering!"); 6252 Operation = Results[0]; 6253 } 6254 6255 // Truncate the result of the extended FP_TO_*INT operation to the desired 6256 // size. 6257 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 6258} 6259 6260/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. 6261/// 6262SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) { 6263 MVT VT = Op.getValueType(); 6264 MVT SHVT = TLI.getShiftAmountTy(); 6265 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 6266 switch (VT.getSimpleVT()) { 6267 default: assert(0 && "Unhandled Expand type in BSWAP!"); abort(); 6268 case MVT::i16: 6269 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6270 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6271 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 6272 case MVT::i32: 6273 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 6274 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6275 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6276 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 6277 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); 6278 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); 6279 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 6280 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 6281 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 6282 case MVT::i64: 6283 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); 6284 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); 6285 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 6286 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6287 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 6288 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 6289 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); 6290 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); 6291 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); 6292 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); 6293 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); 6294 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); 6295 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); 6296 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); 6297 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 6298 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 6299 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 6300 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 6301 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 6302 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 6303 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 6304 } 6305} 6306 6307/// ExpandBitCount - Expand the specified bitcount instruction into operations. 6308/// 6309SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, 6310 DebugLoc dl) { 6311 switch (Opc) { 6312 default: assert(0 && "Cannot expand this yet!"); 6313 case ISD::CTPOP: { 6314 static const uint64_t mask[6] = { 6315 0x5555555555555555ULL, 0x3333333333333333ULL, 6316 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL, 6317 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL 6318 }; 6319 MVT VT = Op.getValueType(); 6320 MVT ShVT = TLI.getShiftAmountTy(); 6321 unsigned len = VT.getSizeInBits(); 6322 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 6323 //x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8]) 6324 unsigned EltSize = VT.isVector() ? 6325 VT.getVectorElementType().getSizeInBits() : len; 6326 SDValue Tmp2 = DAG.getConstant(APInt(EltSize, mask[i]), VT); 6327 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 6328 Op = DAG.getNode(ISD::ADD, dl, VT, 6329 DAG.getNode(ISD::AND, dl, VT, Op, Tmp2), 6330 DAG.getNode(ISD::AND, dl, VT, 6331 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3), 6332 Tmp2)); 6333 } 6334 return Op; 6335 } 6336 case ISD::CTLZ: { 6337 // for now, we do this: 6338 // x = x | (x >> 1); 6339 // x = x | (x >> 2); 6340 // ... 6341 // x = x | (x >>16); 6342 // x = x | (x >>32); // for 64-bit input 6343 // return popcount(~x); 6344 // 6345 // but see also: http://www.hackersdelight.org/HDcode/nlz.cc 6346 MVT VT = Op.getValueType(); 6347 MVT ShVT = TLI.getShiftAmountTy(); 6348 unsigned len = VT.getSizeInBits(); 6349 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 6350 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 6351 Op = DAG.getNode(ISD::OR, dl, VT, Op, 6352 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); 6353 } 6354 Op = DAG.getNOT(dl, Op, VT); 6355 return DAG.getNode(ISD::CTPOP, dl, VT, Op); 6356 } 6357 case ISD::CTTZ: { 6358 // for now, we use: { return popcount(~x & (x - 1)); } 6359 // unless the target has ctlz but not ctpop, in which case we use: 6360 // { return 32 - nlz(~x & (x-1)); } 6361 // see also http://www.hackersdelight.org/HDcode/ntz.cc 6362 MVT VT = Op.getValueType(); 6363 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, 6364 DAG.getNOT(dl, Op, VT), 6365 DAG.getNode(ISD::SUB, dl, VT, Op, 6366 DAG.getConstant(1, VT))); 6367 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. 6368 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && 6369 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) 6370 return DAG.getNode(ISD::SUB, dl, VT, 6371 DAG.getConstant(VT.getSizeInBits(), VT), 6372 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); 6373 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); 6374 } 6375 } 6376} 6377 6378/// ExpandOp - Expand the specified SDValue into its two component pieces 6379/// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, the 6380/// LegalizedNodes map is filled in for any results that are not expanded, the 6381/// ExpandedNodes map is filled in for any results that are expanded, and the 6382/// Lo/Hi values are returned. 6383void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ 6384 MVT VT = Op.getValueType(); 6385 MVT NVT = TLI.getTypeToTransformTo(VT); 6386 SDNode *Node = Op.getNode(); 6387 DebugLoc dl = Node->getDebugLoc(); 6388 assert(getTypeAction(VT) == Expand && "Not an expanded type!"); 6389 assert(((NVT.isInteger() && NVT.bitsLT(VT)) || VT.isFloatingPoint() || 6390 VT.isVector()) && "Cannot expand to FP value or to larger int value!"); 6391 6392 // See if we already expanded it. 6393 DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator I 6394 = ExpandedNodes.find(Op); 6395 if (I != ExpandedNodes.end()) { 6396 Lo = I->second.first; 6397 Hi = I->second.second; 6398 return; 6399 } 6400 6401 switch (Node->getOpcode()) { 6402 case ISD::CopyFromReg: 6403 assert(0 && "CopyFromReg must be legal!"); 6404 case ISD::FP_ROUND_INREG: 6405 if (VT == MVT::ppcf128 && 6406 TLI.getOperationAction(ISD::FP_ROUND_INREG, VT) == 6407 TargetLowering::Custom) { 6408 SDValue SrcLo, SrcHi, Src; 6409 ExpandOp(Op.getOperand(0), SrcLo, SrcHi); 6410 Src = DAG.getNode(ISD::BUILD_PAIR, dl, VT, SrcLo, SrcHi); 6411 SDValue Result = 6412 TLI.LowerOperation(DAG.getNode(ISD::FP_ROUND_INREG, dl, VT, Src, 6413 Op.getOperand(1)), DAG); 6414 assert(Result.getNode()->getOpcode() == ISD::BUILD_PAIR); 6415 Lo = Result.getNode()->getOperand(0); 6416 Hi = Result.getNode()->getOperand(1); 6417 break; 6418 } 6419 // fall through 6420 default: 6421#ifndef NDEBUG 6422 cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; 6423#endif 6424 assert(0 && "Do not know how to expand this operator!"); 6425 abort(); 6426 case ISD::EXTRACT_ELEMENT: 6427 ExpandOp(Node->getOperand(0), Lo, Hi); 6428 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) 6429 return ExpandOp(Hi, Lo, Hi); 6430 return ExpandOp(Lo, Lo, Hi); 6431 case ISD::EXTRACT_VECTOR_ELT: 6432 // ExpandEXTRACT_VECTOR_ELT tolerates invalid result types. 6433 Lo = ExpandEXTRACT_VECTOR_ELT(Op); 6434 return ExpandOp(Lo, Lo, Hi); 6435 case ISD::UNDEF: 6436 Lo = DAG.getUNDEF(NVT); 6437 Hi = DAG.getUNDEF(NVT); 6438 break; 6439 case ISD::Constant: { 6440 unsigned NVTBits = NVT.getSizeInBits(); 6441 const APInt &Cst = cast<ConstantSDNode>(Node)->getAPIntValue(); 6442 Lo = DAG.getConstant(APInt(Cst).trunc(NVTBits), NVT); 6443 Hi = DAG.getConstant(Cst.lshr(NVTBits).trunc(NVTBits), NVT); 6444 break; 6445 } 6446 case ISD::ConstantFP: { 6447 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 6448 if (CFP->getValueType(0) == MVT::ppcf128) { 6449 APInt api = CFP->getValueAPF().bitcastToAPInt(); 6450 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])), 6451 MVT::f64); 6452 Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])), 6453 MVT::f64); 6454 break; 6455 } 6456 Lo = ExpandConstantFP(CFP, false, DAG, TLI); 6457 if (getTypeAction(Lo.getValueType()) == Expand) 6458 ExpandOp(Lo, Lo, Hi); 6459 break; 6460 } 6461 case ISD::BUILD_PAIR: 6462 // Return the operands. 6463 Lo = Node->getOperand(0); 6464 Hi = Node->getOperand(1); 6465 break; 6466 6467 case ISD::MERGE_VALUES: 6468 if (Node->getNumValues() == 1) { 6469 ExpandOp(Op.getOperand(0), Lo, Hi); 6470 break; 6471 } 6472 // FIXME: For now only expand i64,chain = MERGE_VALUES (x, y) 6473 assert(Op.getResNo() == 0 && Node->getNumValues() == 2 && 6474 Op.getValue(1).getValueType() == MVT::Other && 6475 "unhandled MERGE_VALUES"); 6476 ExpandOp(Op.getOperand(0), Lo, Hi); 6477 // Remember that we legalized the chain. 6478 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Op.getOperand(1))); 6479 break; 6480 6481 case ISD::SIGN_EXTEND_INREG: 6482 ExpandOp(Node->getOperand(0), Lo, Hi); 6483 // sext_inreg the low part if needed. 6484 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Lo, Node->getOperand(1)); 6485 6486 // The high part gets the sign extension from the lo-part. This handles 6487 // things like sextinreg V:i64 from i8. 6488 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 6489 DAG.getConstant(NVT.getSizeInBits()-1, 6490 TLI.getShiftAmountTy())); 6491 break; 6492 6493 case ISD::BSWAP: { 6494 ExpandOp(Node->getOperand(0), Lo, Hi); 6495 SDValue TempLo = DAG.getNode(ISD::BSWAP, dl, NVT, Hi); 6496 Hi = DAG.getNode(ISD::BSWAP, dl, NVT, Lo); 6497 Lo = TempLo; 6498 break; 6499 } 6500 6501 case ISD::CTPOP: 6502 ExpandOp(Node->getOperand(0), Lo, Hi); 6503 Lo = DAG.getNode(ISD::ADD, dl, NVT, // ctpop(HL) -> ctpop(H)+ctpop(L) 6504 DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 6505 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 6506 Hi = DAG.getConstant(0, NVT); 6507 break; 6508 6509 case ISD::CTLZ: { 6510 // ctlz (HL) -> ctlz(H) != 32 ? ctlz(H) : (ctlz(L)+32) 6511 ExpandOp(Node->getOperand(0), Lo, Hi); 6512 SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); 6513 SDValue HLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Hi); 6514 SDValue TopNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), HLZ, 6515 BitsC, ISD::SETNE); 6516 SDValue LowPart = DAG.getNode(ISD::CTLZ, dl, NVT, Lo); 6517 LowPart = DAG.getNode(ISD::ADD, dl, NVT, LowPart, BitsC); 6518 6519 Lo = DAG.getNode(ISD::SELECT, dl, NVT, TopNotZero, HLZ, LowPart); 6520 Hi = DAG.getConstant(0, NVT); 6521 break; 6522 } 6523 6524 case ISD::CTTZ: { 6525 // cttz (HL) -> cttz(L) != 32 ? cttz(L) : (cttz(H)+32) 6526 ExpandOp(Node->getOperand(0), Lo, Hi); 6527 SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); 6528 SDValue LTZ = DAG.getNode(ISD::CTTZ, dl, NVT, Lo); 6529 SDValue BotNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), LTZ, 6530 BitsC, ISD::SETNE); 6531 SDValue HiPart = DAG.getNode(ISD::CTTZ, dl, NVT, Hi); 6532 HiPart = DAG.getNode(ISD::ADD, dl, NVT, HiPart, BitsC); 6533 6534 Lo = DAG.getNode(ISD::SELECT, dl, NVT, BotNotZero, LTZ, HiPart); 6535 Hi = DAG.getConstant(0, NVT); 6536 break; 6537 } 6538 6539 case ISD::VAARG: { 6540 SDValue Ch = Node->getOperand(0); // Legalize the chain. 6541 SDValue Ptr = Node->getOperand(1); // Legalize the pointer. 6542 Lo = DAG.getVAArg(NVT, dl, Ch, Ptr, Node->getOperand(2)); 6543 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, Node->getOperand(2)); 6544 6545 // Remember that we legalized the chain. 6546 Hi = LegalizeOp(Hi); 6547 AddLegalizedOperand(Op.getValue(1), Hi.getValue(1)); 6548 if (TLI.isBigEndian()) 6549 std::swap(Lo, Hi); 6550 break; 6551 } 6552 6553 case ISD::LOAD: { 6554 LoadSDNode *LD = cast<LoadSDNode>(Node); 6555 SDValue Ch = LD->getChain(); // Legalize the chain. 6556 SDValue Ptr = LD->getBasePtr(); // Legalize the pointer. 6557 ISD::LoadExtType ExtType = LD->getExtensionType(); 6558 const Value *SV = LD->getSrcValue(); 6559 int SVOffset = LD->getSrcValueOffset(); 6560 unsigned Alignment = LD->getAlignment(); 6561 bool isVolatile = LD->isVolatile(); 6562 6563 if (ExtType == ISD::NON_EXTLOAD) { 6564 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, SV, SVOffset, 6565 isVolatile, Alignment); 6566 if (VT == MVT::f32 || VT == MVT::f64) { 6567 // f32->i32 or f64->i64 one to one expansion. 6568 // Remember that we legalized the chain. 6569 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); 6570 // Recursively expand the new load. 6571 if (getTypeAction(NVT) == Expand) 6572 ExpandOp(Lo, Lo, Hi); 6573 break; 6574 } 6575 6576 // Increment the pointer to the other half. 6577 unsigned IncrementSize = Lo.getValueType().getSizeInBits()/8; 6578 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 6579 DAG.getIntPtrConstant(IncrementSize)); 6580 SVOffset += IncrementSize; 6581 Alignment = MinAlign(Alignment, IncrementSize); 6582 Hi = DAG.getLoad(NVT, dl, Ch, Ptr, SV, SVOffset, 6583 isVolatile, Alignment); 6584 6585 // Build a factor node to remember that this load is independent of the 6586 // other one. 6587 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 6588 Hi.getValue(1)); 6589 6590 // Remember that we legalized the chain. 6591 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); 6592 if (TLI.isBigEndian()) 6593 std::swap(Lo, Hi); 6594 } else { 6595 MVT EVT = LD->getMemoryVT(); 6596 6597 if ((VT == MVT::f64 && EVT == MVT::f32) || 6598 (VT == MVT::ppcf128 && (EVT==MVT::f64 || EVT==MVT::f32))) { 6599 // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND 6600 SDValue Load = DAG.getLoad(EVT, dl, Ch, Ptr, SV, 6601 SVOffset, isVolatile, Alignment); 6602 // Remember that we legalized the chain. 6603 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Load.getValue(1))); 6604 ExpandOp(DAG.getNode(ISD::FP_EXTEND, dl, VT, Load), Lo, Hi); 6605 break; 6606 } 6607 6608 if (EVT == NVT) 6609 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, SV, 6610 SVOffset, isVolatile, Alignment); 6611 else 6612 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, SV, 6613 SVOffset, EVT, isVolatile, 6614 Alignment); 6615 6616 // Remember that we legalized the chain. 6617 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); 6618 6619 if (ExtType == ISD::SEXTLOAD) { 6620 // The high part is obtained by SRA'ing all but one of the bits of the 6621 // lo part. 6622 unsigned LoSize = Lo.getValueType().getSizeInBits(); 6623 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 6624 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 6625 } else if (ExtType == ISD::ZEXTLOAD) { 6626 // The high part is just a zero. 6627 Hi = DAG.getConstant(0, NVT); 6628 } else /* if (ExtType == ISD::EXTLOAD) */ { 6629 // The high part is undefined. 6630 Hi = DAG.getUNDEF(NVT); 6631 } 6632 } 6633 break; 6634 } 6635 case ISD::AND: 6636 case ISD::OR: 6637 case ISD::XOR: { // Simple logical operators -> two trivial pieces. 6638 SDValue LL, LH, RL, RH; 6639 ExpandOp(Node->getOperand(0), LL, LH); 6640 ExpandOp(Node->getOperand(1), RL, RH); 6641 Lo = DAG.getNode(Node->getOpcode(), dl, NVT, LL, RL); 6642 Hi = DAG.getNode(Node->getOpcode(), dl, NVT, LH, RH); 6643 break; 6644 } 6645 case ISD::SELECT: { 6646 SDValue LL, LH, RL, RH; 6647 ExpandOp(Node->getOperand(1), LL, LH); 6648 ExpandOp(Node->getOperand(2), RL, RH); 6649 if (getTypeAction(NVT) == Expand) 6650 NVT = TLI.getTypeToExpandTo(NVT); 6651 Lo = DAG.getNode(ISD::SELECT, dl, NVT, Node->getOperand(0), LL, RL); 6652 if (VT != MVT::f32) 6653 Hi = DAG.getNode(ISD::SELECT, dl, NVT, Node->getOperand(0), LH, RH); 6654 break; 6655 } 6656 case ISD::SELECT_CC: { 6657 SDValue TL, TH, FL, FH; 6658 ExpandOp(Node->getOperand(2), TL, TH); 6659 ExpandOp(Node->getOperand(3), FL, FH); 6660 if (getTypeAction(NVT) == Expand) 6661 NVT = TLI.getTypeToExpandTo(NVT); 6662 Lo = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), 6663 Node->getOperand(1), TL, FL, Node->getOperand(4)); 6664 if (VT != MVT::f32) 6665 Hi = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), 6666 Node->getOperand(1), TH, FH, Node->getOperand(4)); 6667 break; 6668 } 6669 case ISD::ANY_EXTEND: 6670 // The low part is any extension of the input (which degenerates to a copy). 6671 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); 6672 // The high part is undefined. 6673 Hi = DAG.getUNDEF(NVT); 6674 break; 6675 case ISD::SIGN_EXTEND: { 6676 // The low part is just a sign extension of the input (which degenerates to 6677 // a copy). 6678 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, Node->getOperand(0)); 6679 6680 // The high part is obtained by SRA'ing all but one of the bits of the lo 6681 // part. 6682 unsigned LoSize = Lo.getValueType().getSizeInBits(); 6683 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 6684 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 6685 break; 6686 } 6687 case ISD::ZERO_EXTEND: 6688 // The low part is just a zero extension of the input (which degenerates to 6689 // a copy). 6690 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 6691 6692 // The high part is just a zero. 6693 Hi = DAG.getConstant(0, NVT); 6694 break; 6695 6696 case ISD::TRUNCATE: { 6697 // The input value must be larger than this value. Expand *it*. 6698 SDValue NewLo; 6699 ExpandOp(Node->getOperand(0), NewLo, Hi); 6700 6701 // The low part is now either the right size, or it is closer. If not the 6702 // right size, make an illegal truncate so we recursively expand it. 6703 if (NewLo.getValueType() != Node->getValueType(0)) 6704 NewLo = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), NewLo); 6705 ExpandOp(NewLo, Lo, Hi); 6706 break; 6707 } 6708 6709 case ISD::BIT_CONVERT: { 6710 SDValue Tmp; 6711 if (TLI.getOperationAction(ISD::BIT_CONVERT, VT) == TargetLowering::Custom){ 6712 // If the target wants to, allow it to lower this itself. 6713 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6714 case Expand: assert(0 && "cannot expand FP!"); 6715 case Legal: Tmp = LegalizeOp(Node->getOperand(0)); break; 6716 case Promote: Tmp = PromoteOp (Node->getOperand(0)); break; 6717 } 6718 Tmp = TLI.LowerOperation(DAG.getNode(ISD::BIT_CONVERT, dl, VT, Tmp), DAG); 6719 } 6720 6721 // f32 / f64 must be expanded to i32 / i64. 6722 if (VT == MVT::f32 || VT == MVT::f64) { 6723 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); 6724 if (getTypeAction(NVT) == Expand) 6725 ExpandOp(Lo, Lo, Hi); 6726 break; 6727 } 6728 6729 // If source operand will be expanded to the same type as VT, i.e. 6730 // i64 <- f64, i32 <- f32, expand the source operand instead. 6731 MVT VT0 = Node->getOperand(0).getValueType(); 6732 if (getTypeAction(VT0) == Expand && TLI.getTypeToTransformTo(VT0) == VT) { 6733 ExpandOp(Node->getOperand(0), Lo, Hi); 6734 break; 6735 } 6736 6737 // Turn this into a load/store pair by default. 6738 if (Tmp.getNode() == 0) 6739 Tmp = EmitStackConvert(Node->getOperand(0), VT, VT, dl); 6740 6741 ExpandOp(Tmp, Lo, Hi); 6742 break; 6743 } 6744 6745 case ISD::READCYCLECOUNTER: { 6746 assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) == 6747 TargetLowering::Custom && 6748 "Must custom expand ReadCycleCounter"); 6749 SDValue Tmp = TLI.LowerOperation(Op, DAG); 6750 assert(Tmp.getNode() && "Node must be custom expanded!"); 6751 ExpandOp(Tmp.getValue(0), Lo, Hi); 6752 AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. 6753 LegalizeOp(Tmp.getValue(1))); 6754 break; 6755 } 6756 6757 case ISD::ATOMIC_CMP_SWAP: { 6758 // This operation does not need a loop. 6759 SDValue Tmp = TLI.LowerOperation(Op, DAG); 6760 assert(Tmp.getNode() && "Node must be custom expanded!"); 6761 ExpandOp(Tmp.getValue(0), Lo, Hi); 6762 AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. 6763 LegalizeOp(Tmp.getValue(1))); 6764 break; 6765 } 6766 6767 case ISD::ATOMIC_LOAD_ADD: 6768 case ISD::ATOMIC_LOAD_SUB: 6769 case ISD::ATOMIC_LOAD_AND: 6770 case ISD::ATOMIC_LOAD_OR: 6771 case ISD::ATOMIC_LOAD_XOR: 6772 case ISD::ATOMIC_LOAD_NAND: 6773 case ISD::ATOMIC_SWAP: { 6774 // These operations require a loop to be generated. We can't do that yet, 6775 // so substitute a target-dependent pseudo and expand that later. 6776 SDValue In2Lo, In2Hi, In2; 6777 ExpandOp(Op.getOperand(2), In2Lo, In2Hi); 6778 In2 = DAG.getNode(ISD::BUILD_PAIR, dl, VT, In2Lo, In2Hi); 6779 AtomicSDNode* Anode = cast<AtomicSDNode>(Node); 6780 SDValue Replace = 6781 DAG.getAtomic(Op.getOpcode(), dl, Anode->getMemoryVT(), 6782 Op.getOperand(0), Op.getOperand(1), In2, 6783 Anode->getSrcValue(), Anode->getAlignment()); 6784 SDValue Result = TLI.LowerOperation(Replace, DAG); 6785 ExpandOp(Result.getValue(0), Lo, Hi); 6786 // Remember that we legalized the chain. 6787 AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Result.getValue(1))); 6788 break; 6789 } 6790 6791 // These operators cannot be expanded directly, emit them as calls to 6792 // library functions. 6793 case ISD::FP_TO_SINT: { 6794 if (TLI.getOperationAction(ISD::FP_TO_SINT, VT) == TargetLowering::Custom) { 6795 SDValue Op; 6796 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6797 case Expand: assert(0 && "cannot expand FP!"); 6798 case Legal: Op = LegalizeOp(Node->getOperand(0)); break; 6799 case Promote: Op = PromoteOp (Node->getOperand(0)); break; 6800 } 6801 6802 Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op), DAG); 6803 6804 // Now that the custom expander is done, expand the result, which is still 6805 // VT. 6806 if (Op.getNode()) { 6807 ExpandOp(Op, Lo, Hi); 6808 break; 6809 } 6810 } 6811 6812 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Node->getOperand(0).getValueType(), 6813 VT); 6814 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected uint-to-fp conversion!"); 6815 Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); 6816 break; 6817 } 6818 6819 case ISD::FP_TO_UINT: { 6820 if (TLI.getOperationAction(ISD::FP_TO_UINT, VT) == TargetLowering::Custom) { 6821 SDValue Op; 6822 switch (getTypeAction(Node->getOperand(0).getValueType())) { 6823 case Expand: assert(0 && "cannot expand FP!"); 6824 case Legal: Op = LegalizeOp(Node->getOperand(0)); break; 6825 case Promote: Op = PromoteOp (Node->getOperand(0)); break; 6826 } 6827 6828 Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op), DAG); 6829 6830 // Now that the custom expander is done, expand the result. 6831 if (Op.getNode()) { 6832 ExpandOp(Op, Lo, Hi); 6833 break; 6834 } 6835 } 6836 6837 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Node->getOperand(0).getValueType(), 6838 VT); 6839 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 6840 Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); 6841 break; 6842 } 6843 6844 case ISD::SHL: { 6845 // If the target wants custom lowering, do so. 6846 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6847 if (TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Custom) { 6848 SDValue Op = DAG.getNode(ISD::SHL, dl, VT, Node->getOperand(0), ShiftAmt); 6849 Op = TLI.LowerOperation(Op, DAG); 6850 if (Op.getNode()) { 6851 // Now that the custom expander is done, expand the result, which is 6852 // still VT. 6853 ExpandOp(Op, Lo, Hi); 6854 break; 6855 } 6856 } 6857 6858 // If ADDC/ADDE are supported and if the shift amount is a constant 1, emit 6859 // this X << 1 as X+X. 6860 if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(ShiftAmt)) { 6861 if (ShAmt->getAPIntValue() == 1 && 6862 TLI.isOperationLegalOrCustom(ISD::ADDC, NVT) && 6863 TLI.isOperationLegalOrCustom(ISD::ADDE, NVT)) { 6864 SDValue LoOps[2], HiOps[3]; 6865 ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]); 6866 SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag); 6867 LoOps[1] = LoOps[0]; 6868 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 6869 6870 HiOps[1] = HiOps[0]; 6871 HiOps[2] = Lo.getValue(1); 6872 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 6873 break; 6874 } 6875 } 6876 6877 // If we can emit an efficient shift operation, do so now. 6878 if (ExpandShift(ISD::SHL, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) 6879 break; 6880 6881 // If this target supports SHL_PARTS, use it. 6882 TargetLowering::LegalizeAction Action = 6883 TLI.getOperationAction(ISD::SHL_PARTS, NVT); 6884 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 6885 Action == TargetLowering::Custom) { 6886 ExpandShiftParts(ISD::SHL_PARTS, Node->getOperand(0), 6887 ShiftAmt, Lo, Hi, dl); 6888 break; 6889 } 6890 6891 // Otherwise, emit a libcall. 6892 Lo = ExpandLibCall(RTLIB::SHL_I64, Node, false/*left shift=unsigned*/, Hi); 6893 break; 6894 } 6895 6896 case ISD::SRA: { 6897 // If the target wants custom lowering, do so. 6898 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6899 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Custom) { 6900 SDValue Op = DAG.getNode(ISD::SRA, dl, VT, Node->getOperand(0), ShiftAmt); 6901 Op = TLI.LowerOperation(Op, DAG); 6902 if (Op.getNode()) { 6903 // Now that the custom expander is done, expand the result, which is 6904 // still VT. 6905 ExpandOp(Op, Lo, Hi); 6906 break; 6907 } 6908 } 6909 6910 // If we can emit an efficient shift operation, do so now. 6911 if (ExpandShift(ISD::SRA, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) 6912 break; 6913 6914 // If this target supports SRA_PARTS, use it. 6915 TargetLowering::LegalizeAction Action = 6916 TLI.getOperationAction(ISD::SRA_PARTS, NVT); 6917 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 6918 Action == TargetLowering::Custom) { 6919 ExpandShiftParts(ISD::SRA_PARTS, Node->getOperand(0), 6920 ShiftAmt, Lo, Hi, dl); 6921 break; 6922 } 6923 6924 // Otherwise, emit a libcall. 6925 Lo = ExpandLibCall(RTLIB::SRA_I64, Node, true/*ashr is signed*/, Hi); 6926 break; 6927 } 6928 6929 case ISD::SRL: { 6930 // If the target wants custom lowering, do so. 6931 SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); 6932 if (TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Custom) { 6933 SDValue Op = DAG.getNode(ISD::SRL, dl, VT, Node->getOperand(0), ShiftAmt); 6934 Op = TLI.LowerOperation(Op, DAG); 6935 if (Op.getNode()) { 6936 // Now that the custom expander is done, expand the result, which is 6937 // still VT. 6938 ExpandOp(Op, Lo, Hi); 6939 break; 6940 } 6941 } 6942 6943 // If we can emit an efficient shift operation, do so now. 6944 if (ExpandShift(ISD::SRL, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) 6945 break; 6946 6947 // If this target supports SRL_PARTS, use it. 6948 TargetLowering::LegalizeAction Action = 6949 TLI.getOperationAction(ISD::SRL_PARTS, NVT); 6950 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 6951 Action == TargetLowering::Custom) { 6952 ExpandShiftParts(ISD::SRL_PARTS, 6953 Node->getOperand(0), ShiftAmt, Lo, Hi, dl); 6954 break; 6955 } 6956 6957 // Otherwise, emit a libcall. 6958 Lo = ExpandLibCall(RTLIB::SRL_I64, Node, false/*lshr is unsigned*/, Hi); 6959 break; 6960 } 6961 6962 case ISD::ADD: 6963 case ISD::SUB: { 6964 // If the target wants to custom expand this, let them. 6965 if (TLI.getOperationAction(Node->getOpcode(), VT) == 6966 TargetLowering::Custom) { 6967 SDValue Result = TLI.LowerOperation(Op, DAG); 6968 if (Result.getNode()) { 6969 ExpandOp(Result, Lo, Hi); 6970 break; 6971 } 6972 } 6973 // Expand the subcomponents. 6974 SDValue LHSL, LHSH, RHSL, RHSH; 6975 ExpandOp(Node->getOperand(0), LHSL, LHSH); 6976 ExpandOp(Node->getOperand(1), RHSL, RHSH); 6977 SDValue LoOps[2], HiOps[3]; 6978 LoOps[0] = LHSL; 6979 LoOps[1] = RHSL; 6980 HiOps[0] = LHSH; 6981 HiOps[1] = RHSH; 6982 6983 //cascaded check to see if any smaller size has a a carry flag. 6984 unsigned OpV = Node->getOpcode() == ISD::ADD ? ISD::ADDC : ISD::SUBC; 6985 bool hasCarry = false; 6986 for (unsigned BitSize = NVT.getSizeInBits(); BitSize != 0; BitSize /= 2) { 6987 MVT AVT = MVT::getIntegerVT(BitSize); 6988 if (TLI.isOperationLegalOrCustom(OpV, AVT)) { 6989 hasCarry = true; 6990 break; 6991 } 6992 } 6993 6994 if(hasCarry) { 6995 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 6996 if (Node->getOpcode() == ISD::ADD) { 6997 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 6998 HiOps[2] = Lo.getValue(1); 6999 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 7000 } else { 7001 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 7002 HiOps[2] = Lo.getValue(1); 7003 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 7004 } 7005 break; 7006 } else { 7007 if (Node->getOpcode() == ISD::ADD) { 7008 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps, 2); 7009 Hi = DAG.getNode(ISD::ADD, dl, NVT, HiOps, 2); 7010 SDValue Cmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), 7011 Lo, LoOps[0], ISD::SETULT); 7012 SDValue Carry1 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp1, 7013 DAG.getConstant(1, NVT), 7014 DAG.getConstant(0, NVT)); 7015 SDValue Cmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), 7016 Lo, LoOps[1], ISD::SETULT); 7017 SDValue Carry2 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp2, 7018 DAG.getConstant(1, NVT), 7019 Carry1); 7020 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 7021 } else { 7022 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps, 2); 7023 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps, 2); 7024 SDValue Cmp = DAG.getSetCC(dl, NVT, LoOps[0], LoOps[1], ISD::SETULT); 7025 SDValue Borrow = DAG.getNode(ISD::SELECT, dl, NVT, Cmp, 7026 DAG.getConstant(1, NVT), 7027 DAG.getConstant(0, NVT)); 7028 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 7029 } 7030 break; 7031 } 7032 } 7033 7034 case ISD::ADDC: 7035 case ISD::SUBC: { 7036 // Expand the subcomponents. 7037 SDValue LHSL, LHSH, RHSL, RHSH; 7038 ExpandOp(Node->getOperand(0), LHSL, LHSH); 7039 ExpandOp(Node->getOperand(1), RHSL, RHSH); 7040 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 7041 SDValue LoOps[2] = { LHSL, RHSL }; 7042 SDValue HiOps[3] = { LHSH, RHSH }; 7043 7044 if (Node->getOpcode() == ISD::ADDC) { 7045 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 7046 HiOps[2] = Lo.getValue(1); 7047 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 7048 } else { 7049 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 7050 HiOps[2] = Lo.getValue(1); 7051 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 7052 } 7053 // Remember that we legalized the flag. 7054 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1))); 7055 break; 7056 } 7057 case ISD::ADDE: 7058 case ISD::SUBE: { 7059 // Expand the subcomponents. 7060 SDValue LHSL, LHSH, RHSL, RHSH; 7061 ExpandOp(Node->getOperand(0), LHSL, LHSH); 7062 ExpandOp(Node->getOperand(1), RHSL, RHSH); 7063 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 7064 SDValue LoOps[3] = { LHSL, RHSL, Node->getOperand(2) }; 7065 SDValue HiOps[3] = { LHSH, RHSH }; 7066 7067 Lo = DAG.getNode(Node->getOpcode(), dl, VTList, LoOps, 3); 7068 HiOps[2] = Lo.getValue(1); 7069 Hi = DAG.getNode(Node->getOpcode(), dl, VTList, HiOps, 3); 7070 7071 // Remember that we legalized the flag. 7072 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1))); 7073 break; 7074 } 7075 case ISD::MUL: { 7076 // If the target wants to custom expand this, let them. 7077 if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) { 7078 SDValue New = TLI.LowerOperation(Op, DAG); 7079 if (New.getNode()) { 7080 ExpandOp(New, Lo, Hi); 7081 break; 7082 } 7083 } 7084 7085 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT); 7086 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT); 7087 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT); 7088 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT); 7089 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 7090 SDValue LL, LH, RL, RH; 7091 ExpandOp(Node->getOperand(0), LL, LH); 7092 ExpandOp(Node->getOperand(1), RL, RH); 7093 unsigned OuterBitSize = Op.getValueSizeInBits(); 7094 unsigned InnerBitSize = RH.getValueSizeInBits(); 7095 unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0)); 7096 unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1)); 7097 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 7098 if (DAG.MaskedValueIsZero(Node->getOperand(0), HighMask) && 7099 DAG.MaskedValueIsZero(Node->getOperand(1), HighMask)) { 7100 // The inputs are both zero-extended. 7101 if (HasUMUL_LOHI) { 7102 // We can emit a umul_lohi. 7103 Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 7104 Hi = SDValue(Lo.getNode(), 1); 7105 break; 7106 } 7107 if (HasMULHU) { 7108 // We can emit a mulhu+mul. 7109 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 7110 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 7111 break; 7112 } 7113 } 7114 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 7115 // The input values are both sign-extended. 7116 if (HasSMUL_LOHI) { 7117 // We can emit a smul_lohi. 7118 Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 7119 Hi = SDValue(Lo.getNode(), 1); 7120 break; 7121 } 7122 if (HasMULHS) { 7123 // We can emit a mulhs+mul. 7124 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 7125 Hi = DAG.getNode(ISD::MULHS, dl, NVT, LL, RL); 7126 break; 7127 } 7128 } 7129 if (HasUMUL_LOHI) { 7130 // Lo,Hi = umul LHS, RHS. 7131 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl, 7132 DAG.getVTList(NVT, NVT), LL, RL); 7133 Lo = UMulLOHI; 7134 Hi = UMulLOHI.getValue(1); 7135 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 7136 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 7137 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 7138 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 7139 break; 7140 } 7141 if (HasMULHU) { 7142 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 7143 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 7144 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 7145 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 7146 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 7147 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 7148 break; 7149 } 7150 } 7151 7152 // If nothing else, we can make a libcall. 7153 Lo = ExpandLibCall(RTLIB::MUL_I64, Node, false/*sign irrelevant*/, Hi); 7154 break; 7155 } 7156 case ISD::SDIV: 7157 Lo = ExpandLibCall(RTLIB::SDIV_I64, Node, true, Hi); 7158 break; 7159 case ISD::UDIV: 7160 Lo = ExpandLibCall(RTLIB::UDIV_I64, Node, true, Hi); 7161 break; 7162 case ISD::SREM: 7163 Lo = ExpandLibCall(RTLIB::SREM_I64, Node, true, Hi); 7164 break; 7165 case ISD::UREM: 7166 Lo = ExpandLibCall(RTLIB::UREM_I64, Node, true, Hi); 7167 break; 7168 7169 case ISD::FADD: 7170 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::ADD_F32, 7171 RTLIB::ADD_F64, 7172 RTLIB::ADD_F80, 7173 RTLIB::ADD_PPCF128), 7174 Node, false, Hi); 7175 break; 7176 case ISD::FSUB: 7177 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::SUB_F32, 7178 RTLIB::SUB_F64, 7179 RTLIB::SUB_F80, 7180 RTLIB::SUB_PPCF128), 7181 Node, false, Hi); 7182 break; 7183 case ISD::FMUL: 7184 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::MUL_F32, 7185 RTLIB::MUL_F64, 7186 RTLIB::MUL_F80, 7187 RTLIB::MUL_PPCF128), 7188 Node, false, Hi); 7189 break; 7190 case ISD::FDIV: 7191 Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::DIV_F32, 7192 RTLIB::DIV_F64, 7193 RTLIB::DIV_F80, 7194 RTLIB::DIV_PPCF128), 7195 Node, false, Hi); 7196 break; 7197 case ISD::FP_EXTEND: { 7198 if (VT == MVT::ppcf128) { 7199 assert(Node->getOperand(0).getValueType()==MVT::f32 || 7200 Node->getOperand(0).getValueType()==MVT::f64); 7201 const uint64_t zero = 0; 7202 if (Node->getOperand(0).getValueType()==MVT::f32) 7203 Hi = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Node->getOperand(0)); 7204 else 7205 Hi = Node->getOperand(0); 7206 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7207 break; 7208 } 7209 RTLIB::Libcall LC = RTLIB::getFPEXT(Node->getOperand(0).getValueType(), VT); 7210 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 7211 Lo = ExpandLibCall(LC, Node, true, Hi); 7212 break; 7213 } 7214 case ISD::FP_ROUND: { 7215 RTLIB::Libcall LC = RTLIB::getFPROUND(Node->getOperand(0).getValueType(), 7216 VT); 7217 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 7218 Lo = ExpandLibCall(LC, Node, true, Hi); 7219 break; 7220 } 7221 case ISD::FSQRT: 7222 case ISD::FSIN: 7223 case ISD::FCOS: 7224 case ISD::FLOG: 7225 case ISD::FLOG2: 7226 case ISD::FLOG10: 7227 case ISD::FEXP: 7228 case ISD::FEXP2: 7229 case ISD::FTRUNC: 7230 case ISD::FFLOOR: 7231 case ISD::FCEIL: 7232 case ISD::FRINT: 7233 case ISD::FNEARBYINT: 7234 case ISD::FPOW: 7235 case ISD::FPOWI: { 7236 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 7237 switch(Node->getOpcode()) { 7238 case ISD::FSQRT: 7239 LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 7240 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128); 7241 break; 7242 case ISD::FSIN: 7243 LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64, 7244 RTLIB::SIN_F80, RTLIB::SIN_PPCF128); 7245 break; 7246 case ISD::FCOS: 7247 LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, 7248 RTLIB::COS_F80, RTLIB::COS_PPCF128); 7249 break; 7250 case ISD::FLOG: 7251 LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, 7252 RTLIB::LOG_F80, RTLIB::LOG_PPCF128); 7253 break; 7254 case ISD::FLOG2: 7255 LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 7256 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); 7257 break; 7258 case ISD::FLOG10: 7259 LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 7260 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); 7261 break; 7262 case ISD::FEXP: 7263 LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, 7264 RTLIB::EXP_F80, RTLIB::EXP_PPCF128); 7265 break; 7266 case ISD::FEXP2: 7267 LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 7268 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); 7269 break; 7270 case ISD::FTRUNC: 7271 LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 7272 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); 7273 break; 7274 case ISD::FFLOOR: 7275 LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 7276 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); 7277 break; 7278 case ISD::FCEIL: 7279 LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 7280 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); 7281 break; 7282 case ISD::FRINT: 7283 LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, 7284 RTLIB::RINT_F80, RTLIB::RINT_PPCF128); 7285 break; 7286 case ISD::FNEARBYINT: 7287 LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, 7288 RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); 7289 break; 7290 case ISD::FPOW: 7291 LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, 7292 RTLIB::POW_PPCF128); 7293 break; 7294 case ISD::FPOWI: 7295 LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, RTLIB::POWI_F80, 7296 RTLIB::POWI_PPCF128); 7297 break; 7298 default: assert(0 && "Unreachable!"); 7299 } 7300 Lo = ExpandLibCall(LC, Node, false, Hi); 7301 break; 7302 } 7303 case ISD::FABS: { 7304 if (VT == MVT::ppcf128) { 7305 SDValue Tmp; 7306 ExpandOp(Node->getOperand(0), Lo, Tmp); 7307 Hi = DAG.getNode(ISD::FABS, dl, NVT, Tmp); 7308 // lo = hi==fabs(hi) ? lo : -lo; 7309 Lo = DAG.getNode(ISD::SELECT_CC, dl, NVT, Hi, Tmp, 7310 Lo, DAG.getNode(ISD::FNEG, dl, NVT, Lo), 7311 DAG.getCondCode(ISD::SETEQ)); 7312 break; 7313 } 7314 SDValue Mask = (VT == MVT::f64) 7315 ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) 7316 : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); 7317 Mask = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask); 7318 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); 7319 Lo = DAG.getNode(ISD::AND, dl, NVT, Lo, Mask); 7320 if (getTypeAction(NVT) == Expand) 7321 ExpandOp(Lo, Lo, Hi); 7322 break; 7323 } 7324 case ISD::FNEG: { 7325 if (VT == MVT::ppcf128) { 7326 ExpandOp(Node->getOperand(0), Lo, Hi); 7327 Lo = DAG.getNode(ISD::FNEG, dl, MVT::f64, Lo); 7328 Hi = DAG.getNode(ISD::FNEG, dl, MVT::f64, Hi); 7329 break; 7330 } 7331 SDValue Mask = (VT == MVT::f64) 7332 ? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT) 7333 : DAG.getConstantFP(BitsToFloat(1U << 31), VT); 7334 Mask = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask); 7335 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); 7336 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Mask); 7337 if (getTypeAction(NVT) == Expand) 7338 ExpandOp(Lo, Lo, Hi); 7339 break; 7340 } 7341 case ISD::FCOPYSIGN: { 7342 Lo = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI); 7343 if (getTypeAction(NVT) == Expand) 7344 ExpandOp(Lo, Lo, Hi); 7345 break; 7346 } 7347 case ISD::SINT_TO_FP: 7348 case ISD::UINT_TO_FP: { 7349 bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; 7350 MVT SrcVT = Node->getOperand(0).getValueType(); 7351 7352 // Promote the operand if needed. Do this before checking for 7353 // ppcf128 so conversions of i16 and i8 work. 7354 if (getTypeAction(SrcVT) == Promote) { 7355 SDValue Tmp = PromoteOp(Node->getOperand(0)); 7356 Tmp = isSigned 7357 ? DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Tmp.getValueType(), Tmp, 7358 DAG.getValueType(SrcVT)) 7359 : DAG.getZeroExtendInReg(Tmp, dl, SrcVT); 7360 Node = DAG.UpdateNodeOperands(Op, Tmp).getNode(); 7361 SrcVT = Node->getOperand(0).getValueType(); 7362 } 7363 7364 if (VT == MVT::ppcf128 && SrcVT == MVT::i32) { 7365 static const uint64_t zero = 0; 7366 if (isSigned) { 7367 Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f64, 7368 Node->getOperand(0))); 7369 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7370 } else { 7371 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 7372 Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f64, 7373 Node->getOperand(0))); 7374 Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); 7375 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 7376 // X>=0 ? {(f64)x, 0} : {(f64)x, 0} + 2^32 7377 ExpandOp(DAG.getNode(ISD::SELECT_CC, dl, 7378 MVT::ppcf128, Node->getOperand(0), 7379 DAG.getConstant(0, MVT::i32), 7380 DAG.getNode(ISD::FADD, dl, MVT::ppcf128, Hi, 7381 DAG.getConstantFP 7382 (APFloat(APInt(128, 2, TwoE32)), 7383 MVT::ppcf128)), 7384 Hi, 7385 DAG.getCondCode(ISD::SETLT)), 7386 Lo, Hi); 7387 } 7388 break; 7389 } 7390 if (VT == MVT::ppcf128 && SrcVT == MVT::i64 && !isSigned) { 7391 // si64->ppcf128 done by libcall, below 7392 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 7393 ExpandOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::ppcf128, 7394 Node->getOperand(0)), Lo, Hi); 7395 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 7396 // x>=0 ? (ppcf128)(i64)x : (ppcf128)(i64)x + 2^64 7397 ExpandOp(DAG.getNode(ISD::SELECT_CC, dl, MVT::ppcf128, 7398 Node->getOperand(0), 7399 DAG.getConstant(0, MVT::i64), 7400 DAG.getNode(ISD::FADD, dl, MVT::ppcf128, Hi, 7401 DAG.getConstantFP 7402 (APFloat(APInt(128, 2, TwoE64)), 7403 MVT::ppcf128)), 7404 Hi, 7405 DAG.getCondCode(ISD::SETLT)), 7406 Lo, Hi); 7407 break; 7408 } 7409 7410 Lo = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, VT, 7411 Node->getOperand(0), dl); 7412 if (getTypeAction(Lo.getValueType()) == Expand) 7413 // float to i32 etc. can be 'expanded' to a single node. 7414 ExpandOp(Lo, Lo, Hi); 7415 break; 7416 } 7417 } 7418 7419 // Make sure the resultant values have been legalized themselves, unless this 7420 // is a type that requires multi-step expansion. 7421 if (getTypeAction(NVT) != Expand && NVT != MVT::isVoid) { 7422 Lo = LegalizeOp(Lo); 7423 if (Hi.getNode()) 7424 // Don't legalize the high part if it is expanded to a single node. 7425 Hi = LegalizeOp(Hi); 7426 } 7427 7428 // Remember in a map if the values will be reused later. 7429 bool isNew = 7430 ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; 7431 assert(isNew && "Value already expanded?!?"); 7432 isNew = isNew; 7433} 7434 7435/// SplitVectorOp - Given an operand of vector type, break it down into 7436/// two smaller values, still of vector type. 7437void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo, 7438 SDValue &Hi) { 7439 assert(Op.getValueType().isVector() && "Cannot split non-vector type!"); 7440 SDNode *Node = Op.getNode(); 7441 DebugLoc dl = Node->getDebugLoc(); 7442 unsigned NumElements = Op.getValueType().getVectorNumElements(); 7443 assert(NumElements > 1 && "Cannot split a single element vector!"); 7444 7445 MVT NewEltVT = Op.getValueType().getVectorElementType(); 7446 7447 unsigned NewNumElts_Lo = 1 << Log2_32(NumElements-1); 7448 unsigned NewNumElts_Hi = NumElements - NewNumElts_Lo; 7449 7450 MVT NewVT_Lo = MVT::getVectorVT(NewEltVT, NewNumElts_Lo); 7451 MVT NewVT_Hi = MVT::getVectorVT(NewEltVT, NewNumElts_Hi); 7452 7453 // See if we already split it. 7454 std::map<SDValue, std::pair<SDValue, SDValue> >::iterator I 7455 = SplitNodes.find(Op); 7456 if (I != SplitNodes.end()) { 7457 Lo = I->second.first; 7458 Hi = I->second.second; 7459 return; 7460 } 7461 7462 switch (Node->getOpcode()) { 7463 default: 7464#ifndef NDEBUG 7465 Node->dump(&DAG); 7466#endif 7467 assert(0 && "Unhandled operation in SplitVectorOp!"); 7468 case ISD::UNDEF: 7469 Lo = DAG.getUNDEF(NewVT_Lo); 7470 Hi = DAG.getUNDEF(NewVT_Hi); 7471 break; 7472 case ISD::BUILD_PAIR: 7473 Lo = Node->getOperand(0); 7474 Hi = Node->getOperand(1); 7475 break; 7476 case ISD::INSERT_VECTOR_ELT: { 7477 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Node->getOperand(2))) { 7478 SplitVectorOp(Node->getOperand(0), Lo, Hi); 7479 unsigned Index = Idx->getZExtValue(); 7480 SDValue ScalarOp = Node->getOperand(1); 7481 if (Index < NewNumElts_Lo) 7482 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVT_Lo, Lo, ScalarOp, 7483 DAG.getIntPtrConstant(Index)); 7484 else 7485 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVT_Hi, Hi, ScalarOp, 7486 DAG.getIntPtrConstant(Index - NewNumElts_Lo)); 7487 break; 7488 } 7489 SDValue Tmp = PerformInsertVectorEltInMemory(Node->getOperand(0), 7490 Node->getOperand(1), 7491 Node->getOperand(2), dl); 7492 SplitVectorOp(Tmp, Lo, Hi); 7493 break; 7494 } 7495 case ISD::VECTOR_SHUFFLE: { 7496 // Build the low part. 7497 SDValue Mask = Node->getOperand(2); 7498 SmallVector<SDValue, 8> Ops; 7499 MVT PtrVT = TLI.getPointerTy(); 7500 7501 // Insert all of the elements from the input that are needed. We use 7502 // buildvector of extractelement here because the input vectors will have 7503 // to be legalized, so this makes the code simpler. 7504 for (unsigned i = 0; i != NewNumElts_Lo; ++i) { 7505 SDValue IdxNode = Mask.getOperand(i); 7506 if (IdxNode.getOpcode() == ISD::UNDEF) { 7507 Ops.push_back(DAG.getUNDEF(NewEltVT)); 7508 continue; 7509 } 7510 unsigned Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue(); 7511 SDValue InVec = Node->getOperand(0); 7512 if (Idx >= NumElements) { 7513 InVec = Node->getOperand(1); 7514 Idx -= NumElements; 7515 } 7516 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, InVec, 7517 DAG.getConstant(Idx, PtrVT))); 7518 } 7519 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Lo, &Ops[0], Ops.size()); 7520 Ops.clear(); 7521 7522 for (unsigned i = NewNumElts_Lo; i != NumElements; ++i) { 7523 SDValue IdxNode = Mask.getOperand(i); 7524 if (IdxNode.getOpcode() == ISD::UNDEF) { 7525 Ops.push_back(DAG.getUNDEF(NewEltVT)); 7526 continue; 7527 } 7528 unsigned Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue(); 7529 SDValue InVec = Node->getOperand(0); 7530 if (Idx >= NumElements) { 7531 InVec = Node->getOperand(1); 7532 Idx -= NumElements; 7533 } 7534 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, InVec, 7535 DAG.getConstant(Idx, PtrVT))); 7536 } 7537 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Hi, &Ops[0], Ops.size()); 7538 break; 7539 } 7540 case ISD::BUILD_VECTOR: { 7541 SmallVector<SDValue, 8> LoOps(Node->op_begin(), 7542 Node->op_begin()+NewNumElts_Lo); 7543 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Lo, &LoOps[0], LoOps.size()); 7544 7545 SmallVector<SDValue, 8> HiOps(Node->op_begin()+NewNumElts_Lo, 7546 Node->op_end()); 7547 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Hi, &HiOps[0], HiOps.size()); 7548 break; 7549 } 7550 case ISD::CONCAT_VECTORS: { 7551 // FIXME: Handle non-power-of-two vectors? 7552 unsigned NewNumSubvectors = Node->getNumOperands() / 2; 7553 if (NewNumSubvectors == 1) { 7554 Lo = Node->getOperand(0); 7555 Hi = Node->getOperand(1); 7556 } else { 7557 SmallVector<SDValue, 8> LoOps(Node->op_begin(), 7558 Node->op_begin()+NewNumSubvectors); 7559 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT_Lo, 7560 &LoOps[0], LoOps.size()); 7561 7562 SmallVector<SDValue, 8> HiOps(Node->op_begin()+NewNumSubvectors, 7563 Node->op_end()); 7564 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT_Hi, 7565 &HiOps[0], HiOps.size()); 7566 } 7567 break; 7568 } 7569 case ISD::EXTRACT_SUBVECTOR: { 7570 SDValue Vec = Op.getOperand(0); 7571 SDValue Idx = Op.getOperand(1); 7572 MVT IdxVT = Idx.getValueType(); 7573 7574 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Lo, Vec, Idx); 7575 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 7576 if (CIdx) { 7577 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Hi, Vec, 7578 DAG.getConstant(CIdx->getZExtValue() + NewNumElts_Lo, 7579 IdxVT)); 7580 } else { 7581 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 7582 DAG.getConstant(NewNumElts_Lo, IdxVT)); 7583 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Hi, Vec, Idx); 7584 } 7585 break; 7586 } 7587 case ISD::SELECT: { 7588 SDValue Cond = Node->getOperand(0); 7589 7590 SDValue LL, LH, RL, RH; 7591 SplitVectorOp(Node->getOperand(1), LL, LH); 7592 SplitVectorOp(Node->getOperand(2), RL, RH); 7593 7594 if (Cond.getValueType().isVector()) { 7595 // Handle a vector merge. 7596 SDValue CL, CH; 7597 SplitVectorOp(Cond, CL, CH); 7598 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, CL, LL, RL); 7599 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, CH, LH, RH); 7600 } else { 7601 // Handle a simple select with vector operands. 7602 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, Cond, LL, RL); 7603 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, Cond, LH, RH); 7604 } 7605 break; 7606 } 7607 case ISD::SELECT_CC: { 7608 SDValue CondLHS = Node->getOperand(0); 7609 SDValue CondRHS = Node->getOperand(1); 7610 SDValue CondCode = Node->getOperand(4); 7611 7612 SDValue LL, LH, RL, RH; 7613 SplitVectorOp(Node->getOperand(2), LL, LH); 7614 SplitVectorOp(Node->getOperand(3), RL, RH); 7615 7616 // Handle a simple select with vector operands. 7617 Lo = DAG.getNode(ISD::SELECT_CC, dl, NewVT_Lo, CondLHS, CondRHS, 7618 LL, RL, CondCode); 7619 Hi = DAG.getNode(ISD::SELECT_CC, dl, NewVT_Hi, CondLHS, CondRHS, 7620 LH, RH, CondCode); 7621 break; 7622 } 7623 case ISD::VSETCC: { 7624 SDValue LL, LH, RL, RH; 7625 SplitVectorOp(Node->getOperand(0), LL, LH); 7626 SplitVectorOp(Node->getOperand(1), RL, RH); 7627 Lo = DAG.getNode(ISD::VSETCC, dl, NewVT_Lo, LL, RL, Node->getOperand(2)); 7628 Hi = DAG.getNode(ISD::VSETCC, dl, NewVT_Hi, LH, RH, Node->getOperand(2)); 7629 break; 7630 } 7631 case ISD::ADD: 7632 case ISD::SUB: 7633 case ISD::MUL: 7634 case ISD::FADD: 7635 case ISD::FSUB: 7636 case ISD::FMUL: 7637 case ISD::SDIV: 7638 case ISD::UDIV: 7639 case ISD::FDIV: 7640 case ISD::FPOW: 7641 case ISD::AND: 7642 case ISD::OR: 7643 case ISD::XOR: 7644 case ISD::UREM: 7645 case ISD::SREM: 7646 case ISD::FREM: 7647 case ISD::SHL: 7648 case ISD::SRA: 7649 case ISD::SRL: { 7650 SDValue LL, LH, RL, RH; 7651 SplitVectorOp(Node->getOperand(0), LL, LH); 7652 SplitVectorOp(Node->getOperand(1), RL, RH); 7653 7654 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, LL, RL); 7655 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, LH, RH); 7656 break; 7657 } 7658 case ISD::FP_ROUND: 7659 case ISD::FPOWI: { 7660 SDValue L, H; 7661 SplitVectorOp(Node->getOperand(0), L, H); 7662 7663 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, L, Node->getOperand(1)); 7664 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, H, Node->getOperand(1)); 7665 break; 7666 } 7667 case ISD::CTTZ: 7668 case ISD::CTLZ: 7669 case ISD::CTPOP: 7670 case ISD::FNEG: 7671 case ISD::FABS: 7672 case ISD::FSQRT: 7673 case ISD::FSIN: 7674 case ISD::FCOS: 7675 case ISD::FLOG: 7676 case ISD::FLOG2: 7677 case ISD::FLOG10: 7678 case ISD::FEXP: 7679 case ISD::FEXP2: 7680 case ISD::FP_TO_SINT: 7681 case ISD::FP_TO_UINT: 7682 case ISD::SINT_TO_FP: 7683 case ISD::UINT_TO_FP: 7684 case ISD::TRUNCATE: 7685 case ISD::ANY_EXTEND: 7686 case ISD::SIGN_EXTEND: 7687 case ISD::ZERO_EXTEND: 7688 case ISD::FP_EXTEND: { 7689 SDValue L, H; 7690 SplitVectorOp(Node->getOperand(0), L, H); 7691 7692 Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, L); 7693 Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, H); 7694 break; 7695 } 7696 case ISD::CONVERT_RNDSAT: { 7697 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 7698 SDValue L, H; 7699 SplitVectorOp(Node->getOperand(0), L, H); 7700 SDValue DTyOpL = DAG.getValueType(NewVT_Lo); 7701 SDValue DTyOpH = DAG.getValueType(NewVT_Hi); 7702 SDValue STyOpL = DAG.getValueType(L.getValueType()); 7703 SDValue STyOpH = DAG.getValueType(H.getValueType()); 7704 7705 SDValue RndOp = Node->getOperand(3); 7706 SDValue SatOp = Node->getOperand(4); 7707 7708 Lo = DAG.getConvertRndSat(NewVT_Lo, dl, L, DTyOpL, STyOpL, 7709 RndOp, SatOp, CvtCode); 7710 Hi = DAG.getConvertRndSat(NewVT_Hi, dl, H, DTyOpH, STyOpH, 7711 RndOp, SatOp, CvtCode); 7712 break; 7713 } 7714 case ISD::LOAD: { 7715 LoadSDNode *LD = cast<LoadSDNode>(Node); 7716 SDValue Ch = LD->getChain(); 7717 SDValue Ptr = LD->getBasePtr(); 7718 ISD::LoadExtType ExtType = LD->getExtensionType(); 7719 const Value *SV = LD->getSrcValue(); 7720 int SVOffset = LD->getSrcValueOffset(); 7721 MVT MemoryVT = LD->getMemoryVT(); 7722 unsigned Alignment = LD->getAlignment(); 7723 bool isVolatile = LD->isVolatile(); 7724 7725 assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); 7726 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 7727 7728 MVT MemNewEltVT = MemoryVT.getVectorElementType(); 7729 MVT MemNewVT_Lo = MVT::getVectorVT(MemNewEltVT, NewNumElts_Lo); 7730 MVT MemNewVT_Hi = MVT::getVectorVT(MemNewEltVT, NewNumElts_Hi); 7731 7732 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, 7733 NewVT_Lo, Ch, Ptr, Offset, 7734 SV, SVOffset, MemNewVT_Lo, isVolatile, Alignment); 7735 unsigned IncrementSize = NewNumElts_Lo * MemNewEltVT.getSizeInBits()/8; 7736 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 7737 DAG.getIntPtrConstant(IncrementSize)); 7738 SVOffset += IncrementSize; 7739 Alignment = MinAlign(Alignment, IncrementSize); 7740 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, 7741 NewVT_Hi, Ch, Ptr, Offset, 7742 SV, SVOffset, MemNewVT_Hi, isVolatile, Alignment); 7743 7744 // Build a factor node to remember that this load is independent of the 7745 // other one. 7746 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 7747 Hi.getValue(1)); 7748 7749 // Remember that we legalized the chain. 7750 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); 7751 break; 7752 } 7753 case ISD::BIT_CONVERT: { 7754 // We know the result is a vector. The input may be either a vector or a 7755 // scalar value. 7756 SDValue InOp = Node->getOperand(0); 7757 if (!InOp.getValueType().isVector() || 7758 InOp.getValueType().getVectorNumElements() == 1) { 7759 // The input is a scalar or single-element vector. 7760 // Lower to a store/load so that it can be split. 7761 // FIXME: this could be improved probably. 7762 unsigned LdAlign = TLI.getTargetData()-> 7763 getPrefTypeAlignment(Op.getValueType().getTypeForMVT()); 7764 SDValue Ptr = DAG.CreateStackTemporary(InOp.getValueType(), LdAlign); 7765 int FI = cast<FrameIndexSDNode>(Ptr.getNode())->getIndex(); 7766 7767 SDValue St = DAG.getStore(DAG.getEntryNode(), dl, 7768 InOp, Ptr, 7769 PseudoSourceValue::getFixedStack(FI), 0); 7770 InOp = DAG.getLoad(Op.getValueType(), dl, St, Ptr, 7771 PseudoSourceValue::getFixedStack(FI), 0); 7772 } 7773 // Split the vector and convert each of the pieces now. 7774 SplitVectorOp(InOp, Lo, Hi); 7775 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT_Lo, Lo); 7776 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT_Hi, Hi); 7777 break; 7778 } 7779 } 7780 7781 // Remember in a map if the values will be reused later. 7782 bool isNew = 7783 SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; 7784 assert(isNew && "Value already split?!?"); 7785 isNew = isNew; 7786} 7787 7788 7789/// ScalarizeVectorOp - Given an operand of single-element vector type 7790/// (e.g. v1f32), convert it into the equivalent operation that returns a 7791/// scalar (e.g. f32) value. 7792SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) { 7793 assert(Op.getValueType().isVector() && "Bad ScalarizeVectorOp invocation!"); 7794 SDNode *Node = Op.getNode(); 7795 DebugLoc dl = Node->getDebugLoc(); 7796 MVT NewVT = Op.getValueType().getVectorElementType(); 7797 assert(Op.getValueType().getVectorNumElements() == 1); 7798 7799 // See if we already scalarized it. 7800 std::map<SDValue, SDValue>::iterator I = ScalarizedNodes.find(Op); 7801 if (I != ScalarizedNodes.end()) return I->second; 7802 7803 SDValue Result; 7804 switch (Node->getOpcode()) { 7805 default: 7806#ifndef NDEBUG 7807 Node->dump(&DAG); cerr << "\n"; 7808#endif 7809 assert(0 && "Unknown vector operation in ScalarizeVectorOp!"); 7810 case ISD::ADD: 7811 case ISD::FADD: 7812 case ISD::SUB: 7813 case ISD::FSUB: 7814 case ISD::MUL: 7815 case ISD::FMUL: 7816 case ISD::SDIV: 7817 case ISD::UDIV: 7818 case ISD::FDIV: 7819 case ISD::SREM: 7820 case ISD::UREM: 7821 case ISD::FREM: 7822 case ISD::FPOW: 7823 case ISD::AND: 7824 case ISD::OR: 7825 case ISD::XOR: 7826 Result = DAG.getNode(Node->getOpcode(), dl, 7827 NewVT, 7828 ScalarizeVectorOp(Node->getOperand(0)), 7829 ScalarizeVectorOp(Node->getOperand(1))); 7830 break; 7831 case ISD::FNEG: 7832 case ISD::FABS: 7833 case ISD::FSQRT: 7834 case ISD::FSIN: 7835 case ISD::FCOS: 7836 case ISD::FLOG: 7837 case ISD::FLOG2: 7838 case ISD::FLOG10: 7839 case ISD::FEXP: 7840 case ISD::FEXP2: 7841 case ISD::FP_TO_SINT: 7842 case ISD::FP_TO_UINT: 7843 case ISD::SINT_TO_FP: 7844 case ISD::UINT_TO_FP: 7845 case ISD::SIGN_EXTEND: 7846 case ISD::ZERO_EXTEND: 7847 case ISD::ANY_EXTEND: 7848 case ISD::TRUNCATE: 7849 case ISD::FP_EXTEND: 7850 Result = DAG.getNode(Node->getOpcode(), dl, 7851 NewVT, 7852 ScalarizeVectorOp(Node->getOperand(0))); 7853 break; 7854 case ISD::CONVERT_RNDSAT: { 7855 SDValue Op0 = ScalarizeVectorOp(Node->getOperand(0)); 7856 Result = DAG.getConvertRndSat(NewVT, dl, Op0, 7857 DAG.getValueType(NewVT), 7858 DAG.getValueType(Op0.getValueType()), 7859 Node->getOperand(3), 7860 Node->getOperand(4), 7861 cast<CvtRndSatSDNode>(Node)->getCvtCode()); 7862 break; 7863 } 7864 case ISD::FPOWI: 7865 case ISD::FP_ROUND: 7866 Result = DAG.getNode(Node->getOpcode(), dl, 7867 NewVT, 7868 ScalarizeVectorOp(Node->getOperand(0)), 7869 Node->getOperand(1)); 7870 break; 7871 case ISD::LOAD: { 7872 LoadSDNode *LD = cast<LoadSDNode>(Node); 7873 SDValue Ch = LegalizeOp(LD->getChain()); // Legalize the chain. 7874 SDValue Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer. 7875 ISD::LoadExtType ExtType = LD->getExtensionType(); 7876 const Value *SV = LD->getSrcValue(); 7877 int SVOffset = LD->getSrcValueOffset(); 7878 MVT MemoryVT = LD->getMemoryVT(); 7879 unsigned Alignment = LD->getAlignment(); 7880 bool isVolatile = LD->isVolatile(); 7881 7882 assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); 7883 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 7884 7885 Result = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, 7886 NewVT, Ch, Ptr, Offset, SV, SVOffset, 7887 MemoryVT.getVectorElementType(), 7888 isVolatile, Alignment); 7889 7890 // Remember that we legalized the chain. 7891 AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); 7892 break; 7893 } 7894 case ISD::BUILD_VECTOR: 7895 Result = Node->getOperand(0); 7896 break; 7897 case ISD::INSERT_VECTOR_ELT: 7898 // Returning the inserted scalar element. 7899 Result = Node->getOperand(1); 7900 break; 7901 case ISD::CONCAT_VECTORS: 7902 assert(Node->getOperand(0).getValueType() == NewVT && 7903 "Concat of non-legal vectors not yet supported!"); 7904 Result = Node->getOperand(0); 7905 break; 7906 case ISD::VECTOR_SHUFFLE: { 7907 // Figure out if the scalar is the LHS or RHS and return it. 7908 SDValue EltNum = Node->getOperand(2).getOperand(0); 7909 if (cast<ConstantSDNode>(EltNum)->getZExtValue()) 7910 Result = ScalarizeVectorOp(Node->getOperand(1)); 7911 else 7912 Result = ScalarizeVectorOp(Node->getOperand(0)); 7913 break; 7914 } 7915 case ISD::EXTRACT_SUBVECTOR: 7916 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, 7917 Node->getOperand(0), Node->getOperand(1)); 7918 break; 7919 case ISD::BIT_CONVERT: { 7920 SDValue Op0 = Op.getOperand(0); 7921 if (Op0.getValueType().getVectorNumElements() == 1) 7922 Op0 = ScalarizeVectorOp(Op0); 7923 Result = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, Op0); 7924 break; 7925 } 7926 case ISD::SELECT: 7927 Result = DAG.getNode(ISD::SELECT, dl, NewVT, Op.getOperand(0), 7928 ScalarizeVectorOp(Op.getOperand(1)), 7929 ScalarizeVectorOp(Op.getOperand(2))); 7930 break; 7931 case ISD::SELECT_CC: 7932 Result = DAG.getNode(ISD::SELECT_CC, dl, NewVT, Node->getOperand(0), 7933 Node->getOperand(1), 7934 ScalarizeVectorOp(Op.getOperand(2)), 7935 ScalarizeVectorOp(Op.getOperand(3)), 7936 Node->getOperand(4)); 7937 break; 7938 case ISD::VSETCC: { 7939 SDValue Op0 = ScalarizeVectorOp(Op.getOperand(0)); 7940 SDValue Op1 = ScalarizeVectorOp(Op.getOperand(1)); 7941 Result = DAG.getNode(ISD::SETCC, dl, 7942 TLI.getSetCCResultType(Op0.getValueType()), 7943 Op0, Op1, Op.getOperand(2)); 7944 Result = DAG.getNode(ISD::SELECT, dl, NewVT, Result, 7945 DAG.getConstant(-1ULL, NewVT), 7946 DAG.getConstant(0ULL, NewVT)); 7947 break; 7948 } 7949 } 7950 7951 if (TLI.isTypeLegal(NewVT)) 7952 Result = LegalizeOp(Result); 7953 bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second; 7954 assert(isNew && "Value already scalarized?"); 7955 isNew = isNew; 7956 return Result; 7957} 7958 7959 7960SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) { 7961 std::map<SDValue, SDValue>::iterator I = WidenNodes.find(Op); 7962 if (I != WidenNodes.end()) return I->second; 7963 7964 MVT VT = Op.getValueType(); 7965 assert(VT.isVector() && "Cannot widen non-vector type!"); 7966 7967 SDValue Result; 7968 SDNode *Node = Op.getNode(); 7969 DebugLoc dl = Node->getDebugLoc(); 7970 MVT EVT = VT.getVectorElementType(); 7971 7972 unsigned NumElts = VT.getVectorNumElements(); 7973 unsigned NewNumElts = WidenVT.getVectorNumElements(); 7974 assert(NewNumElts > NumElts && "Cannot widen to smaller type!"); 7975 assert(NewNumElts < 17); 7976 7977 // When widen is called, it is assumed that it is more efficient to use a 7978 // wide type. The default action is to widen to operation to a wider legal 7979 // vector type and then do the operation if it is legal by calling LegalizeOp 7980 // again. If there is no vector equivalent, we will unroll the operation, do 7981 // it, and rebuild the vector. If most of the operations are vectorizible to 7982 // the legal type, the resulting code will be more efficient. If this is not 7983 // the case, the resulting code will preform badly as we end up generating 7984 // code to pack/unpack the results. It is the function that calls widen 7985 // that is responsible for seeing this doesn't happen. 7986 switch (Node->getOpcode()) { 7987 default: 7988#ifndef NDEBUG 7989 Node->dump(&DAG); 7990#endif 7991 assert(0 && "Unexpected operation in WidenVectorOp!"); 7992 break; 7993 case ISD::CopyFromReg: 7994 assert(0 && "CopyFromReg doesn't need widening!"); 7995 case ISD::Constant: 7996 case ISD::ConstantFP: 7997 // To build a vector of these elements, clients should call BuildVector 7998 // and with each element instead of creating a node with a vector type 7999 assert(0 && "Unexpected operation in WidenVectorOp!"); 8000 case ISD::VAARG: 8001 // Variable Arguments with vector types doesn't make any sense to me 8002 assert(0 && "Unexpected operation in WidenVectorOp!"); 8003 break; 8004 case ISD::UNDEF: 8005 Result = DAG.getUNDEF(WidenVT); 8006 break; 8007 case ISD::BUILD_VECTOR: { 8008 // Build a vector with undefined for the new nodes 8009 SDValueVector NewOps(Node->op_begin(), Node->op_end()); 8010 for (unsigned i = NumElts; i < NewNumElts; ++i) { 8011 NewOps.push_back(DAG.getUNDEF(EVT)); 8012 } 8013 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, 8014 &NewOps[0], NewOps.size()); 8015 break; 8016 } 8017 case ISD::INSERT_VECTOR_ELT: { 8018 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8019 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WidenVT, Tmp1, 8020 Node->getOperand(1), Node->getOperand(2)); 8021 break; 8022 } 8023 case ISD::VECTOR_SHUFFLE: { 8024 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8025 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); 8026 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Node); 8027 SmallVector<int, 8> NewMask; 8028 for (unsigned i = 0; i < NumElts; ++i) { 8029 int Idx = SVOp->getMaskElt(i); 8030 if (Idx < (int)NumElts) 8031 NewMask.push_back(Idx); 8032 else 8033 NewMask.push_back(Idx + NewNumElts - NumElts); 8034 } 8035 for (unsigned i = NumElts; i < NewNumElts; ++i) 8036 NewMask.push_back(-1); 8037 8038 Result = DAG.getVectorShuffle(WidenVT, dl, Tmp1, Tmp2, &NewMask[0]); 8039 break; 8040 } 8041 case ISD::LOAD: { 8042 // If the load widen returns true, we can use a single load for the 8043 // vector. Otherwise, it is returning a token factor for multiple 8044 // loads. 8045 SDValue TFOp; 8046 if (LoadWidenVectorOp(Result, TFOp, Op, WidenVT)) 8047 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(1))); 8048 else 8049 AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(0))); 8050 break; 8051 } 8052 8053 case ISD::BIT_CONVERT: { 8054 SDValue Tmp1 = Node->getOperand(0); 8055 // Converts between two different types so we need to determine 8056 // the correct widen type for the input operand. 8057 MVT InVT = Tmp1.getValueType(); 8058 unsigned WidenSize = WidenVT.getSizeInBits(); 8059 if (InVT.isVector()) { 8060 MVT InEltVT = InVT.getVectorElementType(); 8061 unsigned InEltSize = InEltVT.getSizeInBits(); 8062 assert(WidenSize % InEltSize == 0 && 8063 "can not widen bit convert that are not multiple of element type"); 8064 MVT NewInWidenVT = MVT::getVectorVT(InEltVT, WidenSize / InEltSize); 8065 Tmp1 = WidenVectorOp(Tmp1, NewInWidenVT); 8066 assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits()); 8067 Result = DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, Tmp1); 8068 } else { 8069 // If the result size is a multiple of the input size, widen the input 8070 // and then convert. 8071 unsigned InSize = InVT.getSizeInBits(); 8072 assert(WidenSize % InSize == 0 && 8073 "can not widen bit convert that are not multiple of element type"); 8074 unsigned NewNumElts = WidenSize / InSize; 8075 SmallVector<SDValue, 16> Ops(NewNumElts); 8076 SDValue UndefVal = DAG.getUNDEF(InVT); 8077 Ops[0] = Tmp1; 8078 for (unsigned i = 1; i < NewNumElts; ++i) 8079 Ops[i] = UndefVal; 8080 8081 MVT NewInVT = MVT::getVectorVT(InVT, NewNumElts); 8082 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, &Ops[0], NewNumElts); 8083 Result = DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, Result); 8084 } 8085 break; 8086 } 8087 8088 case ISD::SINT_TO_FP: 8089 case ISD::UINT_TO_FP: 8090 case ISD::FP_TO_SINT: 8091 case ISD::FP_TO_UINT: 8092 case ISD::FP_ROUND: { 8093 SDValue Tmp1 = Node->getOperand(0); 8094 // Converts between two different types so we need to determine 8095 // the correct widen type for the input operand. 8096 MVT TVT = Tmp1.getValueType(); 8097 assert(TVT.isVector() && "can not widen non vector type"); 8098 MVT TEVT = TVT.getVectorElementType(); 8099 MVT TWidenVT = MVT::getVectorVT(TEVT, NewNumElts); 8100 Tmp1 = WidenVectorOp(Tmp1, TWidenVT); 8101 assert(Tmp1.getValueType().getVectorNumElements() == NewNumElts); 8102 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1); 8103 break; 8104 } 8105 8106 case ISD::FP_EXTEND: 8107 assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); 8108 case ISD::TRUNCATE: 8109 case ISD::SIGN_EXTEND: 8110 case ISD::ZERO_EXTEND: 8111 case ISD::ANY_EXTEND: 8112 case ISD::SIGN_EXTEND_INREG: 8113 case ISD::FABS: 8114 case ISD::FNEG: 8115 case ISD::FSQRT: 8116 case ISD::FSIN: 8117 case ISD::FCOS: 8118 case ISD::CTPOP: 8119 case ISD::CTTZ: 8120 case ISD::CTLZ: { 8121 // Unary op widening 8122 SDValue Tmp1; 8123 Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8124 assert(Tmp1.getValueType() == WidenVT); 8125 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1); 8126 break; 8127 } 8128 case ISD::CONVERT_RNDSAT: { 8129 SDValue RndOp = Node->getOperand(3); 8130 SDValue SatOp = Node->getOperand(4); 8131 SDValue SrcOp = Node->getOperand(0); 8132 8133 // Converts between two different types so we need to determine 8134 // the correct widen type for the input operand. 8135 MVT SVT = SrcOp.getValueType(); 8136 assert(SVT.isVector() && "can not widen non vector type"); 8137 MVT SEVT = SVT.getVectorElementType(); 8138 MVT SWidenVT = MVT::getVectorVT(SEVT, NewNumElts); 8139 8140 SrcOp = WidenVectorOp(SrcOp, SWidenVT); 8141 assert(SrcOp.getValueType() == WidenVT); 8142 SDValue DTyOp = DAG.getValueType(WidenVT); 8143 SDValue STyOp = DAG.getValueType(SrcOp.getValueType()); 8144 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode(); 8145 8146 Result = DAG.getConvertRndSat(WidenVT, dl, SrcOp, DTyOp, STyOp, 8147 RndOp, SatOp, CvtCode); 8148 break; 8149 } 8150 case ISD::FPOW: 8151 case ISD::FPOWI: 8152 case ISD::ADD: 8153 case ISD::SUB: 8154 case ISD::MUL: 8155 case ISD::MULHS: 8156 case ISD::MULHU: 8157 case ISD::AND: 8158 case ISD::OR: 8159 case ISD::XOR: 8160 case ISD::FADD: 8161 case ISD::FSUB: 8162 case ISD::FMUL: 8163 case ISD::SDIV: 8164 case ISD::SREM: 8165 case ISD::FDIV: 8166 case ISD::FREM: 8167 case ISD::FCOPYSIGN: 8168 case ISD::UDIV: 8169 case ISD::UREM: 8170 case ISD::BSWAP: { 8171 // Binary op widening 8172 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8173 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); 8174 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); 8175 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, Tmp2); 8176 break; 8177 } 8178 8179 case ISD::SHL: 8180 case ISD::SRA: 8181 case ISD::SRL: { 8182 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8183 assert(Tmp1.getValueType() == WidenVT); 8184 SDValue ShOp = Node->getOperand(1); 8185 MVT ShVT = ShOp.getValueType(); 8186 MVT NewShVT = MVT::getVectorVT(ShVT.getVectorElementType(), 8187 WidenVT.getVectorNumElements()); 8188 ShOp = WidenVectorOp(ShOp, NewShVT); 8189 assert(ShOp.getValueType() == NewShVT); 8190 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, ShOp); 8191 break; 8192 } 8193 8194 case ISD::EXTRACT_VECTOR_ELT: { 8195 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); 8196 assert(Tmp1.getValueType() == WidenVT); 8197 Result = DAG.getNode(Node->getOpcode(), dl, EVT, Tmp1, Node->getOperand(1)); 8198 break; 8199 } 8200 case ISD::CONCAT_VECTORS: { 8201 // We concurrently support only widen on a multiple of the incoming vector. 8202 // We could widen on a multiple of the incoming operand if necessary. 8203 unsigned NumConcat = NewNumElts / NumElts; 8204 assert(NewNumElts % NumElts == 0 && "Can widen only a multiple of vector"); 8205 SDValue UndefVal = DAG.getUNDEF(VT); 8206 SmallVector<SDValue, 8> MOps; 8207 MOps.push_back(Op); 8208 for (unsigned i = 1; i != NumConcat; ++i) { 8209 MOps.push_back(UndefVal); 8210 } 8211 Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 8212 &MOps[0], MOps.size())); 8213 break; 8214 } 8215 case ISD::EXTRACT_SUBVECTOR: { 8216 SDValue Tmp1 = Node->getOperand(0); 8217 SDValue Idx = Node->getOperand(1); 8218 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 8219 if (CIdx && CIdx->getZExtValue() == 0) { 8220 // Since we are access the start of the vector, the incoming 8221 // vector type might be the proper. 8222 MVT Tmp1VT = Tmp1.getValueType(); 8223 if (Tmp1VT == WidenVT) 8224 return Tmp1; 8225 else { 8226 unsigned Tmp1VTNumElts = Tmp1VT.getVectorNumElements(); 8227 if (Tmp1VTNumElts < NewNumElts) 8228 Result = WidenVectorOp(Tmp1, WidenVT); 8229 else 8230 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, Tmp1, Idx); 8231 } 8232 } else if (NewNumElts % NumElts == 0) { 8233 // Widen the extracted subvector. 8234 unsigned NumConcat = NewNumElts / NumElts; 8235 SDValue UndefVal = DAG.getUNDEF(VT); 8236 SmallVector<SDValue, 8> MOps; 8237 MOps.push_back(Op); 8238 for (unsigned i = 1; i != NumConcat; ++i) { 8239 MOps.push_back(UndefVal); 8240 } 8241 Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 8242 &MOps[0], MOps.size())); 8243 } else { 8244 assert(0 && "can not widen extract subvector"); 8245 // This could be implemented using insert and build vector but I would 8246 // like to see when this happens. 8247 } 8248 break; 8249 } 8250 8251 case ISD::SELECT: { 8252 // Determine new condition widen type and widen 8253 SDValue Cond1 = Node->getOperand(0); 8254 MVT CondVT = Cond1.getValueType(); 8255 assert(CondVT.isVector() && "can not widen non vector type"); 8256 MVT CondEVT = CondVT.getVectorElementType(); 8257 MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); 8258 Cond1 = WidenVectorOp(Cond1, CondWidenVT); 8259 assert(Cond1.getValueType() == CondWidenVT && "Condition not widen"); 8260 8261 SDValue Tmp1 = WidenVectorOp(Node->getOperand(1), WidenVT); 8262 SDValue Tmp2 = WidenVectorOp(Node->getOperand(2), WidenVT); 8263 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); 8264 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Cond1, Tmp1, Tmp2); 8265 break; 8266 } 8267 8268 case ISD::SELECT_CC: { 8269 // Determine new condition widen type and widen 8270 SDValue Cond1 = Node->getOperand(0); 8271 SDValue Cond2 = Node->getOperand(1); 8272 MVT CondVT = Cond1.getValueType(); 8273 assert(CondVT.isVector() && "can not widen non vector type"); 8274 assert(CondVT == Cond2.getValueType() && "mismatch lhs/rhs"); 8275 MVT CondEVT = CondVT.getVectorElementType(); 8276 MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); 8277 Cond1 = WidenVectorOp(Cond1, CondWidenVT); 8278 Cond2 = WidenVectorOp(Cond2, CondWidenVT); 8279 assert(Cond1.getValueType() == CondWidenVT && 8280 Cond2.getValueType() == CondWidenVT && "condition not widen"); 8281 8282 SDValue Tmp1 = WidenVectorOp(Node->getOperand(2), WidenVT); 8283 SDValue Tmp2 = WidenVectorOp(Node->getOperand(3), WidenVT); 8284 assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT && 8285 "operands not widen"); 8286 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Cond1, Cond2, Tmp1, 8287 Tmp2, Node->getOperand(4)); 8288 break; 8289 } 8290 case ISD::VSETCC: { 8291 // Determine widen for the operand 8292 SDValue Tmp1 = Node->getOperand(0); 8293 MVT TmpVT = Tmp1.getValueType(); 8294 assert(TmpVT.isVector() && "can not widen non vector type"); 8295 MVT TmpEVT = TmpVT.getVectorElementType(); 8296 MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts); 8297 Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT); 8298 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT); 8299 Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, Tmp2, 8300 Node->getOperand(2)); 8301 break; 8302 } 8303 case ISD::ATOMIC_CMP_SWAP: 8304 case ISD::ATOMIC_LOAD_ADD: 8305 case ISD::ATOMIC_LOAD_SUB: 8306 case ISD::ATOMIC_LOAD_AND: 8307 case ISD::ATOMIC_LOAD_OR: 8308 case ISD::ATOMIC_LOAD_XOR: 8309 case ISD::ATOMIC_LOAD_NAND: 8310 case ISD::ATOMIC_LOAD_MIN: 8311 case ISD::ATOMIC_LOAD_MAX: 8312 case ISD::ATOMIC_LOAD_UMIN: 8313 case ISD::ATOMIC_LOAD_UMAX: 8314 case ISD::ATOMIC_SWAP: { 8315 // For now, we assume that using vectors for these operations don't make 8316 // much sense so we just split it. We return an empty result 8317 SDValue X, Y; 8318 SplitVectorOp(Op, X, Y); 8319 return Result; 8320 break; 8321 } 8322 8323 } // end switch (Node->getOpcode()) 8324 8325 assert(Result.getNode() && "Didn't set a result!"); 8326 if (Result != Op) 8327 Result = LegalizeOp(Result); 8328 8329 AddWidenedOperand(Op, Result); 8330 return Result; 8331} 8332 8333// Utility function to find a legal vector type and its associated element 8334// type from a preferred width and whose vector type must be the same size 8335// as the VVT. 8336// TLI: Target lowering used to determine legal types 8337// Width: Preferred width of element type 8338// VVT: Vector value type whose size we must match. 8339// Returns VecEVT and EVT - the vector type and its associated element type 8340static void FindWidenVecType(const TargetLowering &TLI, unsigned Width, MVT VVT, 8341 MVT& EVT, MVT& VecEVT) { 8342 // We start with the preferred width, make it a power of 2 and see if 8343 // we can find a vector type of that width. If not, we reduce it by 8344 // another power of 2. If we have widen the type, a vector of bytes should 8345 // always be legal. 8346 assert(TLI.isTypeLegal(VVT)); 8347 unsigned EWidth = Width + 1; 8348 do { 8349 assert(EWidth > 0); 8350 EWidth = (1 << Log2_32(EWidth-1)); 8351 EVT = MVT::getIntegerVT(EWidth); 8352 unsigned NumEVT = VVT.getSizeInBits()/EWidth; 8353 VecEVT = MVT::getVectorVT(EVT, NumEVT); 8354 } while (!TLI.isTypeLegal(VecEVT) || 8355 VVT.getSizeInBits() != VecEVT.getSizeInBits()); 8356} 8357 8358SDValue SelectionDAGLegalize::genWidenVectorLoads(SDValueVector& LdChain, 8359 SDValue Chain, 8360 SDValue BasePtr, 8361 const Value *SV, 8362 int SVOffset, 8363 unsigned Alignment, 8364 bool isVolatile, 8365 unsigned LdWidth, 8366 MVT ResType, 8367 DebugLoc dl) { 8368 // We assume that we have good rules to handle loading power of two loads so 8369 // we break down the operations to power of 2 loads. The strategy is to 8370 // load the largest power of 2 that we can easily transform to a legal vector 8371 // and then insert into that vector, and the cast the result into the legal 8372 // vector that we want. This avoids unnecessary stack converts. 8373 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 8374 // the load is nonvolatile, we an use a wider load for the value. 8375 // Find a vector length we can load a large chunk 8376 MVT EVT, VecEVT; 8377 unsigned EVTWidth; 8378 FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); 8379 EVTWidth = EVT.getSizeInBits(); 8380 8381 SDValue LdOp = DAG.getLoad(EVT, dl, Chain, BasePtr, SV, SVOffset, 8382 isVolatile, Alignment); 8383 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecEVT, LdOp); 8384 LdChain.push_back(LdOp.getValue(1)); 8385 8386 // Check if we can load the element with one instruction 8387 if (LdWidth == EVTWidth) { 8388 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 8389 } 8390 8391 // The vector element order is endianness dependent. 8392 unsigned Idx = 1; 8393 LdWidth -= EVTWidth; 8394 unsigned Offset = 0; 8395 8396 while (LdWidth > 0) { 8397 unsigned Increment = EVTWidth / 8; 8398 Offset += Increment; 8399 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 8400 DAG.getIntPtrConstant(Increment)); 8401 8402 if (LdWidth < EVTWidth) { 8403 // Our current type we are using is too large, use a smaller size by 8404 // using a smaller power of 2 8405 unsigned oEVTWidth = EVTWidth; 8406 FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); 8407 EVTWidth = EVT.getSizeInBits(); 8408 // Readjust position and vector position based on new load type 8409 Idx = Idx * (oEVTWidth/EVTWidth); 8410 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, VecOp); 8411 } 8412 8413 SDValue LdOp = DAG.getLoad(EVT, dl, Chain, BasePtr, SV, 8414 SVOffset+Offset, isVolatile, 8415 MinAlign(Alignment, Offset)); 8416 LdChain.push_back(LdOp.getValue(1)); 8417 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecEVT, VecOp, LdOp, 8418 DAG.getIntPtrConstant(Idx++)); 8419 8420 LdWidth -= EVTWidth; 8421 } 8422 8423 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 8424} 8425 8426bool SelectionDAGLegalize::LoadWidenVectorOp(SDValue& Result, 8427 SDValue& TFOp, 8428 SDValue Op, 8429 MVT NVT) { 8430 // TODO: Add support for ConcatVec and the ability to load many vector 8431 // types (e.g., v4i8). This will not work when a vector register 8432 // to memory mapping is strange (e.g., vector elements are not 8433 // stored in some sequential order). 8434 8435 // It must be true that the widen vector type is bigger than where 8436 // we need to load from. 8437 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 8438 MVT LdVT = LD->getMemoryVT(); 8439 DebugLoc dl = LD->getDebugLoc(); 8440 assert(LdVT.isVector() && NVT.isVector()); 8441 assert(LdVT.getVectorElementType() == NVT.getVectorElementType()); 8442 8443 // Load information 8444 SDValue Chain = LD->getChain(); 8445 SDValue BasePtr = LD->getBasePtr(); 8446 int SVOffset = LD->getSrcValueOffset(); 8447 unsigned Alignment = LD->getAlignment(); 8448 bool isVolatile = LD->isVolatile(); 8449 const Value *SV = LD->getSrcValue(); 8450 unsigned int LdWidth = LdVT.getSizeInBits(); 8451 8452 // Load value as a large register 8453 SDValueVector LdChain; 8454 Result = genWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 8455 Alignment, isVolatile, LdWidth, NVT, dl); 8456 8457 if (LdChain.size() == 1) { 8458 TFOp = LdChain[0]; 8459 return true; 8460 } 8461 else { 8462 TFOp=DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 8463 &LdChain[0], LdChain.size()); 8464 return false; 8465 } 8466} 8467 8468 8469void SelectionDAGLegalize::genWidenVectorStores(SDValueVector& StChain, 8470 SDValue Chain, 8471 SDValue BasePtr, 8472 const Value *SV, 8473 int SVOffset, 8474 unsigned Alignment, 8475 bool isVolatile, 8476 SDValue ValOp, 8477 unsigned StWidth, 8478 DebugLoc dl) { 8479 // Breaks the stores into a series of power of 2 width stores. For any 8480 // width, we convert the vector to the vector of element size that we 8481 // want to store. This avoids requiring a stack convert. 8482 8483 // Find a width of the element type we can store with 8484 MVT VVT = ValOp.getValueType(); 8485 MVT EVT, VecEVT; 8486 unsigned EVTWidth; 8487 FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); 8488 EVTWidth = EVT.getSizeInBits(); 8489 8490 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, ValOp); 8491 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EVT, VecOp, 8492 DAG.getIntPtrConstant(0)); 8493 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 8494 isVolatile, Alignment); 8495 StChain.push_back(StOp); 8496 8497 // Check if we are done 8498 if (StWidth == EVTWidth) { 8499 return; 8500 } 8501 8502 unsigned Idx = 1; 8503 StWidth -= EVTWidth; 8504 unsigned Offset = 0; 8505 8506 while (StWidth > 0) { 8507 unsigned Increment = EVTWidth / 8; 8508 Offset += Increment; 8509 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 8510 DAG.getIntPtrConstant(Increment)); 8511 8512 if (StWidth < EVTWidth) { 8513 // Our current type we are using is too large, use a smaller size by 8514 // using a smaller power of 2 8515 unsigned oEVTWidth = EVTWidth; 8516 FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); 8517 EVTWidth = EVT.getSizeInBits(); 8518 // Readjust position and vector position based on new load type 8519 Idx = Idx * (oEVTWidth/EVTWidth); 8520 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, VecOp); 8521 } 8522 8523 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EVT, VecOp, 8524 DAG.getIntPtrConstant(Idx++)); 8525 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 8526 SVOffset + Offset, isVolatile, 8527 MinAlign(Alignment, Offset))); 8528 StWidth -= EVTWidth; 8529 } 8530} 8531 8532 8533SDValue SelectionDAGLegalize::StoreWidenVectorOp(StoreSDNode *ST, 8534 SDValue Chain, 8535 SDValue BasePtr) { 8536 // TODO: It might be cleaner if we can use SplitVector and have more legal 8537 // vector types that can be stored into memory (e.g., v4xi8 can 8538 // be stored as a word). This will not work when a vector register 8539 // to memory mapping is strange (e.g., vector elements are not 8540 // stored in some sequential order). 8541 8542 MVT StVT = ST->getMemoryVT(); 8543 SDValue ValOp = ST->getValue(); 8544 DebugLoc dl = ST->getDebugLoc(); 8545 8546 // Check if we have widen this node with another value 8547 std::map<SDValue, SDValue>::iterator I = WidenNodes.find(ValOp); 8548 if (I != WidenNodes.end()) 8549 ValOp = I->second; 8550 8551 MVT VVT = ValOp.getValueType(); 8552 8553 // It must be true that we the widen vector type is bigger than where 8554 // we need to store. 8555 assert(StVT.isVector() && VVT.isVector()); 8556 assert(StVT.bitsLT(VVT)); 8557 assert(StVT.getVectorElementType() == VVT.getVectorElementType()); 8558 8559 // Store value 8560 SDValueVector StChain; 8561 genWidenVectorStores(StChain, Chain, BasePtr, ST->getSrcValue(), 8562 ST->getSrcValueOffset(), ST->getAlignment(), 8563 ST->isVolatile(), ValOp, StVT.getSizeInBits(), dl); 8564 if (StChain.size() == 1) 8565 return StChain[0]; 8566 else 8567 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 8568 &StChain[0], StChain.size()); 8569} 8570 8571 8572// SelectionDAG::Legalize - This is the entry point for the file. 8573// 8574void SelectionDAG::Legalize(bool TypesNeedLegalizing, unsigned OptLevel) { 8575 /// run - This is the main entry point to this class. 8576 /// 8577 SelectionDAGLegalize(*this, TypesNeedLegalizing, OptLevel).LegalizeDAG(); 8578} 8579 8580