LegalizeIntegerTypes.cpp revision 5fa796a3be8ed1e2a521831f49b920d2fb07916f
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22using namespace llvm; 23 24//===----------------------------------------------------------------------===// 25// Integer Result Promotion 26//===----------------------------------------------------------------------===// 27 28/// PromoteIntegerResult - This method is called when a result of a node is 29/// found to be in need of promotion to a larger type. At this point, the node 30/// may also have invalid operands or may have other results that need 31/// expansion, we just know that (at least) one result needs promotion. 32void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 33 DEBUG(cerr << "Promote integer result: "; N->dump(&DAG); cerr << "\n"); 34 SDValue Result = SDValue(); 35 36 // See if the target wants to custom expand this node. 37 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(ResNo)) == 38 TargetLowering::Custom) { 39 // If the target wants to, allow it to lower this itself. 40 if (SDNode *P = TLI.ReplaceNodeResults(N, DAG)) { 41 // Everything that once used N now uses P. We are guaranteed that the 42 // result value types of N and the result value types of P match. 43 ReplaceNodeWith(N, P); 44 return; 45 } 46 } 47 48 switch (N->getOpcode()) { 49 default: 50#ifndef NDEBUG 51 cerr << "PromoteIntegerResult #" << ResNo << ": "; 52 N->dump(&DAG); cerr << "\n"; 53#endif 54 assert(0 && "Do not know how to promote this operator!"); 55 abort(); 56 case ISD::AssertSext: Result = PromoteIntRes_AssertSext(N); break; 57 case ISD::AssertZext: Result = PromoteIntRes_AssertZext(N); break; 58 case ISD::BIT_CONVERT: Result = PromoteIntRes_BIT_CONVERT(N); break; 59 case ISD::BSWAP: Result = PromoteIntRes_BSWAP(N); break; 60 case ISD::BUILD_PAIR: Result = PromoteIntRes_BUILD_PAIR(N); break; 61 case ISD::Constant: Result = PromoteIntRes_Constant(N); break; 62 case ISD::CTLZ: Result = PromoteIntRes_CTLZ(N); break; 63 case ISD::CTPOP: Result = PromoteIntRes_CTPOP(N); break; 64 case ISD::CTTZ: Result = PromoteIntRes_CTTZ(N); break; 65 case ISD::EXTRACT_VECTOR_ELT: 66 Result = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 67 case ISD::LOAD: Result = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break; 68 case ISD::SELECT: Result = PromoteIntRes_SELECT(N); break; 69 case ISD::SELECT_CC: Result = PromoteIntRes_SELECT_CC(N); break; 70 case ISD::SETCC: Result = PromoteIntRes_SETCC(N); break; 71 case ISD::SHL: Result = PromoteIntRes_SHL(N); break; 72 case ISD::SIGN_EXTEND_INREG: 73 Result = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 74 case ISD::SRA: Result = PromoteIntRes_SRA(N); break; 75 case ISD::SRL: Result = PromoteIntRes_SRL(N); break; 76 case ISD::TRUNCATE: Result = PromoteIntRes_TRUNCATE(N); break; 77 case ISD::UNDEF: Result = PromoteIntRes_UNDEF(N); break; 78 case ISD::VAARG: Result = PromoteIntRes_VAARG(N); break; 79 80 case ISD::SIGN_EXTEND: 81 case ISD::ZERO_EXTEND: 82 case ISD::ANY_EXTEND: Result = PromoteIntRes_INT_EXTEND(N); break; 83 84 case ISD::FP_TO_SINT: 85 case ISD::FP_TO_UINT: Result = PromoteIntRes_FP_TO_XINT(N); break; 86 87 case ISD::AND: 88 case ISD::OR: 89 case ISD::XOR: 90 case ISD::ADD: 91 case ISD::SUB: 92 case ISD::MUL: Result = PromoteIntRes_SimpleIntBinOp(N); break; 93 94 case ISD::SDIV: 95 case ISD::SREM: Result = PromoteIntRes_SDIV(N); break; 96 97 case ISD::UDIV: 98 case ISD::UREM: Result = PromoteIntRes_UDIV(N); break; 99 } 100 101 // If Result is null, the sub-method took care of registering the result. 102 if (Result.getNode()) 103 SetPromotedInteger(SDValue(N, ResNo), Result); 104} 105 106SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 107 // Sign-extend the new bits, and continue the assertion. 108 MVT OldVT = N->getValueType(0); 109 SDValue Op = GetPromotedInteger(N->getOperand(0)); 110 return DAG.getNode(ISD::AssertSext, Op.getValueType(), 111 DAG.getNode(ISD::SIGN_EXTEND_INREG, Op.getValueType(), Op, 112 DAG.getValueType(OldVT)), N->getOperand(1)); 113} 114 115SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 116 // Zero the new bits, and continue the assertion. 117 MVT OldVT = N->getValueType(0); 118 SDValue Op = GetPromotedInteger(N->getOperand(0)); 119 return DAG.getNode(ISD::AssertZext, Op.getValueType(), 120 DAG.getZeroExtendInReg(Op, OldVT), N->getOperand(1)); 121} 122 123SDValue DAGTypeLegalizer::PromoteIntRes_BIT_CONVERT(SDNode *N) { 124 SDValue InOp = N->getOperand(0); 125 MVT InVT = InOp.getValueType(); 126 MVT NInVT = TLI.getTypeToTransformTo(InVT); 127 MVT OutVT = TLI.getTypeToTransformTo(N->getValueType(0)); 128 129 switch (getTypeAction(InVT)) { 130 default: 131 assert(false && "Unknown type action!"); 132 break; 133 case Legal: 134 break; 135 case PromoteInteger: 136 if (OutVT.getSizeInBits() == NInVT.getSizeInBits()) 137 // The input promotes to the same size. Convert the promoted value. 138 return DAG.getNode(ISD::BIT_CONVERT, OutVT, GetPromotedInteger(InOp)); 139 break; 140 case SoftenFloat: 141 // Promote the integer operand by hand. 142 return DAG.getNode(ISD::ANY_EXTEND, OutVT, GetSoftenedFloat(InOp)); 143 case ExpandInteger: 144 case ExpandFloat: 145 break; 146 case ScalarizeVector: 147 // Convert the element to an integer and promote it by hand. 148 return DAG.getNode(ISD::ANY_EXTEND, OutVT, 149 BitConvertToInteger(GetScalarizedVector(InOp))); 150 case SplitVector: 151 // For example, i32 = BIT_CONVERT v2i16 on alpha. Convert the split 152 // pieces of the input into integers and reassemble in the final type. 153 SDValue Lo, Hi; 154 GetSplitVector(N->getOperand(0), Lo, Hi); 155 Lo = BitConvertToInteger(Lo); 156 Hi = BitConvertToInteger(Hi); 157 158 if (TLI.isBigEndian()) 159 std::swap(Lo, Hi); 160 161 InOp = DAG.getNode(ISD::ANY_EXTEND, 162 MVT::getIntegerVT(OutVT.getSizeInBits()), 163 JoinIntegers(Lo, Hi)); 164 return DAG.getNode(ISD::BIT_CONVERT, OutVT, InOp); 165 } 166 167 // Otherwise, lower the bit-convert to a store/load from the stack, then 168 // promote the load. 169 SDValue Op = CreateStackStoreLoad(InOp, N->getValueType(0)); 170 return PromoteIntRes_LOAD(cast<LoadSDNode>(Op.getNode())); 171} 172 173SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 174 SDValue Op = GetPromotedInteger(N->getOperand(0)); 175 MVT OVT = N->getValueType(0); 176 MVT NVT = Op.getValueType(); 177 178 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 179 return DAG.getNode(ISD::SRL, NVT, DAG.getNode(ISD::BSWAP, NVT, Op), 180 DAG.getConstant(DiffBits, TLI.getShiftAmountTy())); 181} 182 183SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 184 // The pair element type may be legal, or may not promote to the same type as 185 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 186 return DAG.getNode(ISD::ANY_EXTEND, 187 TLI.getTypeToTransformTo(N->getValueType(0)), 188 JoinIntegers(N->getOperand(0), N->getOperand(1))); 189} 190 191SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 192 MVT VT = N->getValueType(0); 193 // Zero extend things like i1, sign extend everything else. It shouldn't 194 // matter in theory which one we pick, but this tends to give better code? 195 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 196 SDValue Result = DAG.getNode(Opc, TLI.getTypeToTransformTo(VT), 197 SDValue(N, 0)); 198 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 199 return Result; 200} 201 202SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 203 SDValue Op = GetPromotedInteger(N->getOperand(0)); 204 MVT OVT = N->getValueType(0); 205 MVT NVT = Op.getValueType(); 206 // Zero extend to the promoted type and do the count there. 207 Op = DAG.getNode(ISD::CTLZ, NVT, DAG.getZeroExtendInReg(Op, OVT)); 208 // Subtract off the extra leading bits in the bigger type. 209 return DAG.getNode(ISD::SUB, NVT, Op, 210 DAG.getConstant(NVT.getSizeInBits() - 211 OVT.getSizeInBits(), NVT)); 212} 213 214SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 215 SDValue Op = GetPromotedInteger(N->getOperand(0)); 216 MVT OVT = N->getValueType(0); 217 MVT NVT = Op.getValueType(); 218 // Zero extend to the promoted type and do the count there. 219 return DAG.getNode(ISD::CTPOP, NVT, DAG.getZeroExtendInReg(Op, OVT)); 220} 221 222SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 223 SDValue Op = GetPromotedInteger(N->getOperand(0)); 224 MVT OVT = N->getValueType(0); 225 MVT NVT = Op.getValueType(); 226 // The count is the same in the promoted type except if the original 227 // value was zero. This can be handled by setting the bit just off 228 // the top of the original type. 229 APInt TopBit(NVT.getSizeInBits(), 0); 230 TopBit.set(OVT.getSizeInBits()); 231 Op = DAG.getNode(ISD::OR, NVT, Op, DAG.getConstant(TopBit, NVT)); 232 return DAG.getNode(ISD::CTTZ, NVT, Op); 233} 234 235SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 236 MVT OldVT = N->getValueType(0); 237 SDValue OldVec = N->getOperand(0); 238 unsigned OldElts = OldVec.getValueType().getVectorNumElements(); 239 240 if (OldElts == 1) { 241 assert(!isTypeLegal(OldVec.getValueType()) && 242 "Legal one-element vector of a type needing promotion!"); 243 // It is tempting to follow GetScalarizedVector by a call to 244 // GetPromotedInteger, but this would be wrong because the 245 // scalarized value may not yet have been processed. 246 return DAG.getNode(ISD::ANY_EXTEND, TLI.getTypeToTransformTo(OldVT), 247 GetScalarizedVector(OldVec)); 248 } 249 250 // Convert to a vector half as long with an element type of twice the width, 251 // for example <4 x i16> -> <2 x i32>. 252 assert(!(OldElts & 1) && "Odd length vectors not supported!"); 253 MVT NewVT = MVT::getIntegerVT(2 * OldVT.getSizeInBits()); 254 assert(OldVT.isSimple() && NewVT.isSimple()); 255 256 SDValue NewVec = DAG.getNode(ISD::BIT_CONVERT, 257 MVT::getVectorVT(NewVT, OldElts / 2), 258 OldVec); 259 260 // Extract the element at OldIdx / 2 from the new vector. 261 SDValue OldIdx = N->getOperand(1); 262 SDValue NewIdx = DAG.getNode(ISD::SRL, OldIdx.getValueType(), OldIdx, 263 DAG.getConstant(1, TLI.getShiftAmountTy())); 264 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, NewIdx); 265 266 // Select the appropriate half of the element: Lo if OldIdx was even, 267 // Hi if it was odd. 268 SDValue Lo = Elt; 269 SDValue Hi = DAG.getNode(ISD::SRL, NewVT, Elt, 270 DAG.getConstant(OldVT.getSizeInBits(), 271 TLI.getShiftAmountTy())); 272 if (TLI.isBigEndian()) 273 std::swap(Lo, Hi); 274 275 SDValue Odd = DAG.getNode(ISD::TRUNCATE, MVT::i1, OldIdx); 276 return DAG.getNode(ISD::SELECT, NewVT, Odd, Hi, Lo); 277} 278 279SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 280 unsigned NewOpc = N->getOpcode(); 281 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 282 283 // If we're promoting a UINT to a larger size, check to see if the new node 284 // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since 285 // we can use that instead. This allows us to generate better code for 286 // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not 287 // legal, such as PowerPC. 288 if (N->getOpcode() == ISD::FP_TO_UINT) { 289 if (!TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 290 (TLI.isOperationLegal(ISD::FP_TO_SINT, NVT) || 291 TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)) 292 NewOpc = ISD::FP_TO_SINT; 293 } 294 295 return DAG.getNode(NewOpc, NVT, N->getOperand(0)); 296} 297 298SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 299 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 300 301 if (getTypeAction(N->getOperand(0).getValueType()) == PromoteInteger) { 302 SDValue Res = GetPromotedInteger(N->getOperand(0)); 303 assert(Res.getValueType().getSizeInBits() <= NVT.getSizeInBits() && 304 "Extension doesn't make sense!"); 305 306 // If the result and operand types are the same after promotion, simplify 307 // to an in-register extension. 308 if (NVT == Res.getValueType()) { 309 // The high bits are not guaranteed to be anything. Insert an extend. 310 if (N->getOpcode() == ISD::SIGN_EXTEND) 311 return DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Res, 312 DAG.getValueType(N->getOperand(0).getValueType())); 313 if (N->getOpcode() == ISD::ZERO_EXTEND) 314 return DAG.getZeroExtendInReg(Res, N->getOperand(0).getValueType()); 315 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 316 return Res; 317 } 318 } 319 320 // Otherwise, just extend the original operand all the way to the larger type. 321 return DAG.getNode(N->getOpcode(), NVT, N->getOperand(0)); 322} 323 324SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 325 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 326 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 327 ISD::LoadExtType ExtType = 328 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 329 SDValue Res = DAG.getExtLoad(ExtType, NVT, N->getChain(), N->getBasePtr(), 330 N->getSrcValue(), N->getSrcValueOffset(), 331 N->getMemoryVT(), N->isVolatile(), 332 N->getAlignment()); 333 334 // Legalized the chain result - switch anything that used the old chain to 335 // use the new one. 336 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 337 return Res; 338} 339 340SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) { 341 // Sign extend the input. 342 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 343 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 344 MVT VT = N->getValueType(0); 345 LHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, LHS.getValueType(), LHS, 346 DAG.getValueType(VT)); 347 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, RHS.getValueType(), RHS, 348 DAG.getValueType(VT)); 349 350 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 351} 352 353SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 354 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 355 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 356 return DAG.getNode(ISD::SELECT, LHS.getValueType(), N->getOperand(0),LHS,RHS); 357} 358 359SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 360 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 361 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 362 return DAG.getNode(ISD::SELECT_CC, LHS.getValueType(), N->getOperand(0), 363 N->getOperand(1), LHS, RHS, N->getOperand(4)); 364} 365 366SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 367 MVT SVT = TLI.getSetCCResultType(N->getOperand(0)); 368 assert(isTypeLegal(SVT) && "Illegal SetCC type!"); 369 370 // Get the SETCC result using the canonical SETCC type. 371 SDValue SetCC = DAG.getNode(ISD::SETCC, SVT, N->getOperand(0), 372 N->getOperand(1), N->getOperand(2)); 373 374 // Convert to the expected type. 375 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 376 assert(NVT.getSizeInBits() <= SVT.getSizeInBits() && 377 "Integer type overpromoted?"); 378 return DAG.getNode(ISD::TRUNCATE, NVT, SetCC); 379} 380 381SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 382 return DAG.getNode(ISD::SHL, TLI.getTypeToTransformTo(N->getValueType(0)), 383 GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); 384} 385 386SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 387 SDValue Op = GetPromotedInteger(N->getOperand(0)); 388 return DAG.getNode(ISD::SIGN_EXTEND_INREG, Op.getValueType(), Op, 389 N->getOperand(1)); 390} 391 392SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 393 // The input may have strange things in the top bits of the registers, but 394 // these operations don't care. They may have weird bits going out, but 395 // that too is okay if they are integer operations. 396 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 397 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 398 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 399} 400 401SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 402 // The input value must be properly sign extended. 403 MVT VT = N->getValueType(0); 404 MVT NVT = TLI.getTypeToTransformTo(VT); 405 SDValue Res = GetPromotedInteger(N->getOperand(0)); 406 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Res, DAG.getValueType(VT)); 407 return DAG.getNode(ISD::SRA, NVT, Res, N->getOperand(1)); 408} 409 410SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 411 // The input value must be properly zero extended. 412 MVT VT = N->getValueType(0); 413 MVT NVT = TLI.getTypeToTransformTo(VT); 414 SDValue Res = ZExtPromotedInteger(N->getOperand(0)); 415 return DAG.getNode(ISD::SRL, NVT, Res, N->getOperand(1)); 416} 417 418SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 419 SDValue Res; 420 421 switch (getTypeAction(N->getOperand(0).getValueType())) { 422 default: assert(0 && "Unknown type action!"); 423 case Legal: 424 case ExpandInteger: 425 Res = N->getOperand(0); 426 break; 427 case PromoteInteger: 428 Res = GetPromotedInteger(N->getOperand(0)); 429 break; 430 } 431 432 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 433 assert(Res.getValueType().getSizeInBits() >= NVT.getSizeInBits() && 434 "Truncation doesn't make sense!"); 435 if (Res.getValueType() == NVT) 436 return Res; 437 438 // Truncate to NVT instead of VT 439 return DAG.getNode(ISD::TRUNCATE, NVT, Res); 440} 441 442SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { 443 // Zero extend the input. 444 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 445 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 446 MVT VT = N->getValueType(0); 447 LHS = DAG.getZeroExtendInReg(LHS, VT); 448 RHS = DAG.getZeroExtendInReg(RHS, VT); 449 450 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 451} 452 453SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 454 return DAG.getNode(ISD::UNDEF, TLI.getTypeToTransformTo(N->getValueType(0))); 455} 456 457SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 458 SDValue Chain = N->getOperand(0); // Get the chain. 459 SDValue Ptr = N->getOperand(1); // Get the pointer. 460 MVT VT = N->getValueType(0); 461 462 const Value *V = cast<SrcValueSDNode>(N->getOperand(2))->getValue(); 463 SDValue VAList = DAG.getLoad(TLI.getPointerTy(), Chain, Ptr, V, 0); 464 465 // Increment the arg pointer, VAList, to the next vaarg 466 // FIXME: should the ABI size be used for the increment? Think of 467 // x86 long double (10 bytes long, but aligned on 4 or 8 bytes) or 468 // integers of unusual size (such MVT::i1, which gives an increment 469 // of zero here!). 470 unsigned Increment = VT.getSizeInBits() / 8; 471 SDValue Tmp = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, 472 DAG.getIntPtrConstant(Increment)); 473 474 // Store the incremented VAList to the pointer. 475 Tmp = DAG.getStore(VAList.getValue(1), Tmp, Ptr, V, 0); 476 477 // Load the actual argument out of the arg pointer VAList. 478 Tmp = DAG.getExtLoad(ISD::EXTLOAD, TLI.getTypeToTransformTo(VT), Tmp, 479 VAList, NULL, 0, VT); 480 481 // Legalized the chain result - switch anything that used the old chain to 482 // use the new one. 483 ReplaceValueWith(SDValue(N, 1), Tmp.getValue(1)); 484 return Tmp; 485} 486 487 488//===----------------------------------------------------------------------===// 489// Integer Operand Promotion 490//===----------------------------------------------------------------------===// 491 492/// PromoteIntegerOperand - This method is called when the specified operand of 493/// the specified node is found to need promotion. At this point, all of the 494/// result types of the node are known to be legal, but other operands of the 495/// node may need promotion or expansion as well as the specified one. 496bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 497 DEBUG(cerr << "Promote integer operand: "; N->dump(&DAG); cerr << "\n"); 498 SDValue Res = SDValue(); 499 500 if (TLI.getOperationAction(N->getOpcode(), N->getOperand(OpNo).getValueType()) 501 == TargetLowering::Custom) 502 Res = TLI.LowerOperation(SDValue(N, OpNo), DAG); 503 504 if (Res.getNode() == 0) { 505 switch (N->getOpcode()) { 506 default: 507 #ifndef NDEBUG 508 cerr << "PromoteIntegerOperand Op #" << OpNo << ": "; 509 N->dump(&DAG); cerr << "\n"; 510 #endif 511 assert(0 && "Do not know how to promote this operator's operand!"); 512 abort(); 513 514 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 515 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 516 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 517 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 518 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 519 case ISD::FP_EXTEND: Res = PromoteIntOp_FP_EXTEND(N); break; 520 case ISD::FP_ROUND: Res = PromoteIntOp_FP_ROUND(N); break; 521 case ISD::INSERT_VECTOR_ELT: 522 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 523 case ISD::MEMBARRIER: Res = PromoteIntOp_MEMBARRIER(N); break; 524 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 525 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 526 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 527 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 528 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 529 OpNo); break; 530 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 531 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 532 533 case ISD::SINT_TO_FP: 534 case ISD::UINT_TO_FP: Res = PromoteIntOp_INT_TO_FP(N); break; 535 } 536 } 537 538 // If the result is null, the sub-method took care of registering results etc. 539 if (!Res.getNode()) return false; 540 // If the result is N, the sub-method updated N in place. 541 if (Res.getNode() == N) { 542 // Mark N as new and remark N and its operands. This allows us to correctly 543 // revisit N if it needs another step of promotion and allows us to visit 544 // any new operands to N. 545 ReanalyzeNode(N); 546 return true; 547 } 548 549 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 550 "Invalid operand expansion"); 551 552 ReplaceValueWith(SDValue(N, 0), Res); 553 return false; 554} 555 556/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 557/// shared among BR_CC, SELECT_CC, and SETCC handlers. 558void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 559 ISD::CondCode CCCode) { 560 MVT VT = NewLHS.getValueType(); 561 562 // Get the promoted values. 563 NewLHS = GetPromotedInteger(NewLHS); 564 NewRHS = GetPromotedInteger(NewRHS); 565 566 // Otherwise, we have to insert explicit sign or zero extends. Note 567 // that we could insert sign extends for ALL conditions, but zero extend 568 // is cheaper on many machines (an AND instead of two shifts), so prefer 569 // it. 570 switch (CCCode) { 571 default: assert(0 && "Unknown integer comparison!"); 572 case ISD::SETEQ: 573 case ISD::SETNE: 574 case ISD::SETUGE: 575 case ISD::SETUGT: 576 case ISD::SETULE: 577 case ISD::SETULT: 578 // ALL of these operations will work if we either sign or zero extend 579 // the operands (including the unsigned comparisons!). Zero extend is 580 // usually a simpler/cheaper operation, so prefer it. 581 NewLHS = DAG.getZeroExtendInReg(NewLHS, VT); 582 NewRHS = DAG.getZeroExtendInReg(NewRHS, VT); 583 break; 584 case ISD::SETGE: 585 case ISD::SETGT: 586 case ISD::SETLT: 587 case ISD::SETLE: 588 NewLHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, NewLHS.getValueType(), NewLHS, 589 DAG.getValueType(VT)); 590 NewRHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, NewRHS.getValueType(), NewRHS, 591 DAG.getValueType(VT)); 592 break; 593 } 594} 595 596SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 597 SDValue Op = GetPromotedInteger(N->getOperand(0)); 598 return DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 599} 600 601SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 602 assert(OpNo == 2 && "Don't know how to promote this operand!"); 603 604 SDValue LHS = N->getOperand(2); 605 SDValue RHS = N->getOperand(3); 606 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 607 608 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 609 // legal types. 610 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 611 N->getOperand(1), LHS, RHS, N->getOperand(4)); 612} 613 614SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 615 assert(OpNo == 1 && "only know how to promote condition"); 616 SDValue Cond = GetPromotedInteger(N->getOperand(1)); // Promote condition. 617 618 // The top bits of the promoted condition are not necessarily zero, ensure 619 // that the value is properly zero extended. 620 unsigned BitWidth = Cond.getValueSizeInBits(); 621 if (!DAG.MaskedValueIsZero(Cond, 622 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 623 Cond = DAG.getZeroExtendInReg(Cond, MVT::i1); 624 625 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 626 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), Cond, 627 N->getOperand(2)); 628} 629 630SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 631 // Since the result type is legal, the operands must promote to it. 632 MVT OVT = N->getOperand(0).getValueType(); 633 SDValue Lo = GetPromotedInteger(N->getOperand(0)); 634 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 635 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 636 637 Lo = DAG.getZeroExtendInReg(Lo, OVT); 638 Hi = DAG.getNode(ISD::SHL, N->getValueType(0), Hi, 639 DAG.getConstant(OVT.getSizeInBits(), 640 TLI.getShiftAmountTy())); 641 return DAG.getNode(ISD::OR, N->getValueType(0), Lo, Hi); 642} 643 644SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 645 // The vector type is legal but the element type is not. This implies 646 // that the vector is a power-of-two in length and that the element 647 // type does not have a strange size (eg: it is not i1). 648 MVT VecVT = N->getValueType(0); 649 unsigned NumElts = VecVT.getVectorNumElements(); 650 assert(!(NumElts & 1) && "Legal vector of one illegal element?"); 651 652 // Build a vector of half the length out of elements of twice the bitwidth. 653 // For example <4 x i16> -> <2 x i32>. 654 MVT OldVT = N->getOperand(0).getValueType(); 655 MVT NewVT = MVT::getIntegerVT(2 * OldVT.getSizeInBits()); 656 assert(OldVT.isSimple() && NewVT.isSimple()); 657 658 std::vector<SDValue> NewElts; 659 NewElts.reserve(NumElts/2); 660 661 for (unsigned i = 0; i < NumElts; i += 2) { 662 // Combine two successive elements into one promoted element. 663 SDValue Lo = N->getOperand(i); 664 SDValue Hi = N->getOperand(i+1); 665 if (TLI.isBigEndian()) 666 std::swap(Lo, Hi); 667 NewElts.push_back(JoinIntegers(Lo, Hi)); 668 } 669 670 SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, 671 MVT::getVectorVT(NewVT, NewElts.size()), 672 &NewElts[0], NewElts.size()); 673 674 // Convert the new vector to the old vector type. 675 return DAG.getNode(ISD::BIT_CONVERT, VecVT, NewVec); 676} 677 678SDValue DAGTypeLegalizer::PromoteIntOp_FP_EXTEND(SDNode *N) { 679 SDValue Op = GetPromotedInteger(N->getOperand(0)); 680 return DAG.getNode(ISD::FP_EXTEND, N->getValueType(0), Op); 681} 682 683SDValue DAGTypeLegalizer::PromoteIntOp_FP_ROUND(SDNode *N) { 684 SDValue Op = GetPromotedInteger(N->getOperand(0)); 685 return DAG.getNode(ISD::FP_ROUND, N->getValueType(0), Op, 686 DAG.getIntPtrConstant(0)); 687} 688 689SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 690 unsigned OpNo) { 691 if (OpNo == 1) { 692 // Promote the inserted value. This is valid because the type does not 693 // have to match the vector element type. 694 695 // Check that any extra bits introduced will be truncated away. 696 assert(N->getOperand(1).getValueType().getSizeInBits() >= 697 N->getValueType(0).getVectorElementType().getSizeInBits() && 698 "Type of inserted value narrower than vector element type!"); 699 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 700 GetPromotedInteger(N->getOperand(1)), 701 N->getOperand(2)); 702 } 703 704 assert(OpNo == 2 && "Different operand and result vector types?"); 705 706 // Promote the index. 707 SDValue Idx = N->getOperand(2); 708 Idx = DAG.getZeroExtendInReg(GetPromotedInteger(Idx), Idx.getValueType()); 709 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 710 N->getOperand(1), Idx); 711} 712 713SDValue DAGTypeLegalizer::PromoteIntOp_INT_TO_FP(SDNode *N) { 714 SDValue In = GetPromotedInteger(N->getOperand(0)); 715 MVT OpVT = N->getOperand(0).getValueType(); 716 if (N->getOpcode() == ISD::UINT_TO_FP) 717 In = DAG.getZeroExtendInReg(In, OpVT); 718 else 719 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, In.getValueType(), 720 In, DAG.getValueType(OpVT)); 721 722 return DAG.UpdateNodeOperands(SDValue(N, 0), In); 723} 724 725SDValue DAGTypeLegalizer::PromoteIntOp_MEMBARRIER(SDNode *N) { 726 SDValue NewOps[6]; 727 NewOps[0] = N->getOperand(0); 728 for (unsigned i = 1; i < array_lengthof(NewOps); ++i) { 729 SDValue Flag = GetPromotedInteger(N->getOperand(i)); 730 NewOps[i] = DAG.getZeroExtendInReg(Flag, MVT::i1); 731 } 732 return DAG.UpdateNodeOperands(SDValue (N, 0), NewOps, 733 array_lengthof(NewOps)); 734} 735 736SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 737 assert(OpNo == 0 && "Only know how to promote condition"); 738 assert(N->getOperand(0).getValueType() == MVT::i1 && 739 "SetCC type is not legal??"); 740 SDValue Cond = GetPromotedInteger(N->getOperand(0)); 741 742 // Promote all the way up to SVT, the canonical SetCC type. 743 MVT SVT = TLI.getSetCCResultType(Cond); 744 assert(isTypeLegal(SVT) && "Illegal SetCC type!"); 745 assert(Cond.getValueSizeInBits() <= SVT.getSizeInBits() && 746 "Integer type overpromoted?"); 747 748 // Make sure the extra bits conform to getSetCCResultContents. There are 749 // two sets of extra bits: those in Cond, which come from type promotion, 750 // and those we need to add to have the final type be SVT (for most targets 751 // this last set of bits is empty). 752 unsigned CondBits = Cond.getValueSizeInBits(); 753 ISD::NodeType ExtendCode; 754 switch (TLI.getSetCCResultContents()) { 755 default: 756 assert(false && "Unknown SetCCResultValue!"); 757 case TargetLowering::UndefinedSetCCResult: 758 // Extend to SVT by adding rubbish. 759 ExtendCode = ISD::ANY_EXTEND; 760 break; 761 case TargetLowering::ZeroOrOneSetCCResult: 762 ExtendCode = ISD::ZERO_EXTEND; 763 if (!DAG.MaskedValueIsZero(Cond,APInt::getHighBitsSet(CondBits,CondBits-1))) 764 // All extra bits need to be cleared. Do this by zero extending the 765 // original MVT::i1 condition value all the way to SVT. 766 Cond = N->getOperand(0); 767 break; 768 case TargetLowering::ZeroOrNegativeOneSetCCResult: { 769 ExtendCode = ISD::SIGN_EXTEND; 770 unsigned SignBits = DAG.ComputeNumSignBits(Cond); 771 if (SignBits != CondBits) 772 // All extra bits need to be sign extended. Do this by sign extending the 773 // original MVT::i1 condition value all the way to SVT. 774 Cond = N->getOperand(0); 775 break; 776 } 777 } 778 Cond = DAG.getNode(ExtendCode, SVT, Cond); 779 780 return DAG.UpdateNodeOperands(SDValue(N, 0), Cond, 781 N->getOperand(1), N->getOperand(2)); 782} 783 784SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 785 assert(OpNo == 0 && "Don't know how to promote this operand!"); 786 787 SDValue LHS = N->getOperand(0); 788 SDValue RHS = N->getOperand(1); 789 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 790 791 // The CC (#4) and the possible return values (#2 and #3) have legal types. 792 return DAG.UpdateNodeOperands(SDValue(N, 0), LHS, RHS, N->getOperand(2), 793 N->getOperand(3), N->getOperand(4)); 794} 795 796SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 797 assert(OpNo == 0 && "Don't know how to promote this operand!"); 798 799 SDValue LHS = N->getOperand(0); 800 SDValue RHS = N->getOperand(1); 801 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 802 803 // The CC (#2) is always legal. 804 return DAG.UpdateNodeOperands(SDValue(N, 0), LHS, RHS, N->getOperand(2)); 805} 806 807SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 808 SDValue Op = GetPromotedInteger(N->getOperand(0)); 809 Op = DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 810 return DAG.getNode(ISD::SIGN_EXTEND_INREG, Op.getValueType(), 811 Op, DAG.getValueType(N->getOperand(0).getValueType())); 812} 813 814SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 815 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 816 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 817 int SVOffset = N->getSrcValueOffset(); 818 unsigned Alignment = N->getAlignment(); 819 bool isVolatile = N->isVolatile(); 820 821 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 822 823 assert(!N->isTruncatingStore() && "Cannot promote this store operand!"); 824 825 // Truncate the value and store the result. 826 return DAG.getTruncStore(Ch, Val, Ptr, N->getSrcValue(), 827 SVOffset, N->getMemoryVT(), 828 isVolatile, Alignment); 829} 830 831SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 832 SDValue Op = GetPromotedInteger(N->getOperand(0)); 833 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), Op); 834} 835 836SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 837 SDValue Op = GetPromotedInteger(N->getOperand(0)); 838 Op = DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 839 return DAG.getZeroExtendInReg(Op, N->getOperand(0).getValueType()); 840} 841 842 843//===----------------------------------------------------------------------===// 844// Integer Result Expansion 845//===----------------------------------------------------------------------===// 846 847/// ExpandIntegerResult - This method is called when the specified result of the 848/// specified node is found to need expansion. At this point, the node may also 849/// have invalid operands or may have other results that need promotion, we just 850/// know that (at least) one result needs expansion. 851void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 852 DEBUG(cerr << "Expand integer result: "; N->dump(&DAG); cerr << "\n"); 853 SDValue Lo, Hi; 854 Lo = Hi = SDValue(); 855 856 // See if the target wants to custom expand this node. 857 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(ResNo)) == 858 TargetLowering::Custom) { 859 // If the target wants to, allow it to lower this itself. 860 if (SDNode *P = TLI.ReplaceNodeResults(N, DAG)) { 861 // Everything that once used N now uses P. We are guaranteed that the 862 // result value types of N and the result value types of P match. 863 ReplaceNodeWith(N, P); 864 return; 865 } 866 } 867 868 switch (N->getOpcode()) { 869 default: 870#ifndef NDEBUG 871 cerr << "ExpandIntegerResult #" << ResNo << ": "; 872 N->dump(&DAG); cerr << "\n"; 873#endif 874 assert(0 && "Do not know how to expand the result of this operator!"); 875 abort(); 876 877 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 878 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 879 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 880 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 881 882 case ISD::BIT_CONVERT: ExpandRes_BIT_CONVERT(N, Lo, Hi); break; 883 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 884 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 885 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 886 887 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 888 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 889 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 890 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 891 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 892 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 893 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 894 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 895 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 896 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 897 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 898 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 899 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 900 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 901 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 902 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 903 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 904 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 905 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 906 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 907 908 case ISD::AND: 909 case ISD::OR: 910 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 911 912 case ISD::ADD: 913 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 914 915 case ISD::ADDC: 916 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 917 918 case ISD::ADDE: 919 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 920 921 case ISD::SHL: 922 case ISD::SRA: 923 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 924 } 925 926 // If Lo/Hi is null, the sub-method took care of registering results etc. 927 if (Lo.getNode()) 928 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 929} 930 931/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded, 932/// and the shift amount is a constant 'Amt'. Expand the operation. 933void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 934 SDValue &Lo, SDValue &Hi) { 935 // Expand the incoming operand to be shifted, so that we have its parts 936 SDValue InL, InH; 937 GetExpandedInteger(N->getOperand(0), InL, InH); 938 939 MVT NVT = InL.getValueType(); 940 unsigned VTBits = N->getValueType(0).getSizeInBits(); 941 unsigned NVTBits = NVT.getSizeInBits(); 942 MVT ShTy = N->getOperand(1).getValueType(); 943 944 if (N->getOpcode() == ISD::SHL) { 945 if (Amt > VTBits) { 946 Lo = Hi = DAG.getConstant(0, NVT); 947 } else if (Amt > NVTBits) { 948 Lo = DAG.getConstant(0, NVT); 949 Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt-NVTBits,ShTy)); 950 } else if (Amt == NVTBits) { 951 Lo = DAG.getConstant(0, NVT); 952 Hi = InL; 953 } else if (Amt == 1) { 954 // Emit this X << 1 as X+X. 955 SDVTList VTList = DAG.getVTList(NVT, MVT::Flag); 956 SDValue LoOps[2] = { InL, InL }; 957 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 958 SDValue HiOps[3] = { InH, InH, Lo.getValue(1) }; 959 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 960 } else { 961 Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt, ShTy)); 962 Hi = DAG.getNode(ISD::OR, NVT, 963 DAG.getNode(ISD::SHL, NVT, InH, 964 DAG.getConstant(Amt, ShTy)), 965 DAG.getNode(ISD::SRL, NVT, InL, 966 DAG.getConstant(NVTBits-Amt, ShTy))); 967 } 968 return; 969 } 970 971 if (N->getOpcode() == ISD::SRL) { 972 if (Amt > VTBits) { 973 Lo = DAG.getConstant(0, NVT); 974 Hi = DAG.getConstant(0, NVT); 975 } else if (Amt > NVTBits) { 976 Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy)); 977 Hi = DAG.getConstant(0, NVT); 978 } else if (Amt == NVTBits) { 979 Lo = InH; 980 Hi = DAG.getConstant(0, NVT); 981 } else { 982 Lo = DAG.getNode(ISD::OR, NVT, 983 DAG.getNode(ISD::SRL, NVT, InL, 984 DAG.getConstant(Amt, ShTy)), 985 DAG.getNode(ISD::SHL, NVT, InH, 986 DAG.getConstant(NVTBits-Amt, ShTy))); 987 Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt, ShTy)); 988 } 989 return; 990 } 991 992 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 993 if (Amt > VTBits) { 994 Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH, 995 DAG.getConstant(NVTBits-1, ShTy)); 996 } else if (Amt > NVTBits) { 997 Lo = DAG.getNode(ISD::SRA, NVT, InH, 998 DAG.getConstant(Amt-NVTBits, ShTy)); 999 Hi = DAG.getNode(ISD::SRA, NVT, InH, 1000 DAG.getConstant(NVTBits-1, ShTy)); 1001 } else if (Amt == NVTBits) { 1002 Lo = InH; 1003 Hi = DAG.getNode(ISD::SRA, NVT, InH, 1004 DAG.getConstant(NVTBits-1, ShTy)); 1005 } else { 1006 Lo = DAG.getNode(ISD::OR, NVT, 1007 DAG.getNode(ISD::SRL, NVT, InL, 1008 DAG.getConstant(Amt, ShTy)), 1009 DAG.getNode(ISD::SHL, NVT, InH, 1010 DAG.getConstant(NVTBits-Amt, ShTy))); 1011 Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Amt, ShTy)); 1012 } 1013} 1014 1015/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1016/// this shift based on knowledge of the high bit of the shift amount. If we 1017/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1018/// shift amount. 1019bool DAGTypeLegalizer:: 1020ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1021 SDValue Amt = N->getOperand(1); 1022 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1023 MVT ShTy = Amt.getValueType(); 1024 unsigned ShBits = ShTy.getSizeInBits(); 1025 unsigned NVTBits = NVT.getSizeInBits(); 1026 assert(isPowerOf2_32(NVTBits) && 1027 "Expanded integer type size not a power of two!"); 1028 1029 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1030 APInt KnownZero, KnownOne; 1031 DAG.ComputeMaskedBits(N->getOperand(1), HighBitMask, KnownZero, KnownOne); 1032 1033 // If we don't know anything about the high bits, exit. 1034 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1035 return false; 1036 1037 // Get the incoming operand to be shifted. 1038 SDValue InL, InH; 1039 GetExpandedInteger(N->getOperand(0), InL, InH); 1040 1041 // If we know that any of the high bits of the shift amount are one, then we 1042 // can do this as a couple of simple shifts. 1043 if (KnownOne.intersects(HighBitMask)) { 1044 // Mask out the high bit, which we know is set. 1045 Amt = DAG.getNode(ISD::AND, ShTy, Amt, 1046 DAG.getConstant(~HighBitMask, ShTy)); 1047 1048 switch (N->getOpcode()) { 1049 default: assert(0 && "Unknown shift"); 1050 case ISD::SHL: 1051 Lo = DAG.getConstant(0, NVT); // Low part is zero. 1052 Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part. 1053 return true; 1054 case ISD::SRL: 1055 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 1056 Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part. 1057 return true; 1058 case ISD::SRA: 1059 Hi = DAG.getNode(ISD::SRA, NVT, InH, // Sign extend high part. 1060 DAG.getConstant(NVTBits-1, ShTy)); 1061 Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part. 1062 return true; 1063 } 1064 } 1065 1066 // If we know that all of the high bits of the shift amount are zero, then we 1067 // can do this as a couple of simple shifts. 1068 if ((KnownZero & HighBitMask) == HighBitMask) { 1069 // Compute 32-amt. 1070 SDValue Amt2 = DAG.getNode(ISD::SUB, ShTy, 1071 DAG.getConstant(NVTBits, ShTy), 1072 Amt); 1073 unsigned Op1, Op2; 1074 switch (N->getOpcode()) { 1075 default: assert(0 && "Unknown shift"); 1076 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1077 case ISD::SRL: 1078 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1079 } 1080 1081 Lo = DAG.getNode(N->getOpcode(), NVT, InL, Amt); 1082 Hi = DAG.getNode(ISD::OR, NVT, 1083 DAG.getNode(Op1, NVT, InH, Amt), 1084 DAG.getNode(Op2, NVT, InL, Amt2)); 1085 return true; 1086 } 1087 1088 return false; 1089} 1090 1091void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1092 SDValue &Lo, SDValue &Hi) { 1093 // Expand the subcomponents. 1094 SDValue LHSL, LHSH, RHSL, RHSH; 1095 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1096 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1097 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1098 SDValue LoOps[2] = { LHSL, RHSL }; 1099 SDValue HiOps[3] = { LHSH, RHSH }; 1100 1101 if (N->getOpcode() == ISD::ADD) { 1102 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 1103 HiOps[2] = Lo.getValue(1); 1104 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 1105 } else { 1106 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); 1107 HiOps[2] = Lo.getValue(1); 1108 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); 1109 } 1110} 1111 1112void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1113 SDValue &Lo, SDValue &Hi) { 1114 // Expand the subcomponents. 1115 SDValue LHSL, LHSH, RHSL, RHSH; 1116 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1117 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1118 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1119 SDValue LoOps[2] = { LHSL, RHSL }; 1120 SDValue HiOps[3] = { LHSH, RHSH }; 1121 1122 if (N->getOpcode() == ISD::ADDC) { 1123 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 1124 HiOps[2] = Lo.getValue(1); 1125 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 1126 } else { 1127 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); 1128 HiOps[2] = Lo.getValue(1); 1129 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); 1130 } 1131 1132 // Legalized the flag result - switch anything that used the old flag to 1133 // use the new one. 1134 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1135} 1136 1137void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1138 SDValue &Lo, SDValue &Hi) { 1139 // Expand the subcomponents. 1140 SDValue LHSL, LHSH, RHSL, RHSH; 1141 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1142 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1143 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1144 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1145 SDValue HiOps[3] = { LHSH, RHSH }; 1146 1147 Lo = DAG.getNode(N->getOpcode(), VTList, LoOps, 3); 1148 HiOps[2] = Lo.getValue(1); 1149 Hi = DAG.getNode(N->getOpcode(), VTList, HiOps, 3); 1150 1151 // Legalized the flag result - switch anything that used the old flag to 1152 // use the new one. 1153 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1154} 1155 1156void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 1157 SDValue &Lo, SDValue &Hi) { 1158 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1159 SDValue Op = N->getOperand(0); 1160 if (Op.getValueType().bitsLE(NVT)) { 1161 // The low part is any extension of the input (which degenerates to a copy). 1162 Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Op); 1163 Hi = DAG.getNode(ISD::UNDEF, NVT); // The high part is undefined. 1164 } else { 1165 // For example, extension of an i48 to an i64. The operand type necessarily 1166 // promotes to the result type, so will end up being expanded too. 1167 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1168 "Only know how to promote this result!"); 1169 SDValue Res = GetPromotedInteger(Op); 1170 assert(Res.getValueType() == N->getValueType(0) && 1171 "Operand over promoted?"); 1172 // Split the promoted operand. This will simplify when it is expanded. 1173 SplitInteger(Res, Lo, Hi); 1174 } 1175} 1176 1177void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 1178 SDValue &Lo, SDValue &Hi) { 1179 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1180 MVT NVT = Lo.getValueType(); 1181 MVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1182 unsigned NVTBits = NVT.getSizeInBits(); 1183 unsigned EVTBits = EVT.getSizeInBits(); 1184 1185 if (NVTBits < EVTBits) { 1186 Hi = DAG.getNode(ISD::AssertSext, NVT, Hi, 1187 DAG.getValueType(MVT::getIntegerVT(EVTBits - NVTBits))); 1188 } else { 1189 Lo = DAG.getNode(ISD::AssertSext, NVT, Lo, DAG.getValueType(EVT)); 1190 // The high part replicates the sign bit of Lo, make it explicit. 1191 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 1192 DAG.getConstant(NVTBits-1, TLI.getShiftAmountTy())); 1193 } 1194} 1195 1196void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 1197 SDValue &Lo, SDValue &Hi) { 1198 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1199 MVT NVT = Lo.getValueType(); 1200 MVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1201 unsigned NVTBits = NVT.getSizeInBits(); 1202 unsigned EVTBits = EVT.getSizeInBits(); 1203 1204 if (NVTBits < EVTBits) { 1205 Hi = DAG.getNode(ISD::AssertZext, NVT, Hi, 1206 DAG.getValueType(MVT::getIntegerVT(EVTBits - NVTBits))); 1207 } else { 1208 Lo = DAG.getNode(ISD::AssertZext, NVT, Lo, DAG.getValueType(EVT)); 1209 // The high part must be zero, make it explicit. 1210 Hi = DAG.getConstant(0, NVT); 1211 } 1212} 1213 1214void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1215 SDValue &Lo, SDValue &Hi) { 1216 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1217 Lo = DAG.getNode(ISD::BSWAP, Lo.getValueType(), Lo); 1218 Hi = DAG.getNode(ISD::BSWAP, Hi.getValueType(), Hi); 1219} 1220 1221void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 1222 SDValue &Lo, SDValue &Hi) { 1223 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1224 unsigned NBitWidth = NVT.getSizeInBits(); 1225 const APInt &Cst = cast<ConstantSDNode>(N)->getAPIntValue(); 1226 Lo = DAG.getConstant(APInt(Cst).trunc(NBitWidth), NVT); 1227 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT); 1228} 1229 1230void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1231 SDValue &Lo, SDValue &Hi) { 1232 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1233 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1234 MVT NVT = Lo.getValueType(); 1235 1236 SDValue HiNotZero = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi, 1237 DAG.getConstant(0, NVT), ISD::SETNE); 1238 1239 SDValue LoLZ = DAG.getNode(ISD::CTLZ, NVT, Lo); 1240 SDValue HiLZ = DAG.getNode(ISD::CTLZ, NVT, Hi); 1241 1242 Lo = DAG.getNode(ISD::SELECT, NVT, HiNotZero, HiLZ, 1243 DAG.getNode(ISD::ADD, NVT, LoLZ, 1244 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1245 Hi = DAG.getConstant(0, NVT); 1246} 1247 1248void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1249 SDValue &Lo, SDValue &Hi) { 1250 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1251 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1252 MVT NVT = Lo.getValueType(); 1253 Lo = DAG.getNode(ISD::ADD, NVT, DAG.getNode(ISD::CTPOP, NVT, Lo), 1254 DAG.getNode(ISD::CTPOP, NVT, Hi)); 1255 Hi = DAG.getConstant(0, NVT); 1256} 1257 1258void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1259 SDValue &Lo, SDValue &Hi) { 1260 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1261 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1262 MVT NVT = Lo.getValueType(); 1263 1264 SDValue LoNotZero = DAG.getSetCC(TLI.getSetCCResultType(Lo), Lo, 1265 DAG.getConstant(0, NVT), ISD::SETNE); 1266 1267 SDValue LoLZ = DAG.getNode(ISD::CTTZ, NVT, Lo); 1268 SDValue HiLZ = DAG.getNode(ISD::CTTZ, NVT, Hi); 1269 1270 Lo = DAG.getNode(ISD::SELECT, NVT, LoNotZero, LoLZ, 1271 DAG.getNode(ISD::ADD, NVT, HiLZ, 1272 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1273 Hi = DAG.getConstant(0, NVT); 1274} 1275 1276void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 1277 SDValue &Hi) { 1278 MVT VT = N->getValueType(0); 1279 SDValue Op = N->getOperand(0); 1280 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 1281 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 1282 SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*sign irrelevant*/), Lo, Hi); 1283} 1284 1285void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 1286 SDValue &Hi) { 1287 MVT VT = N->getValueType(0); 1288 SDValue Op = N->getOperand(0); 1289 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 1290 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 1291 SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*sign irrelevant*/), Lo, Hi); 1292} 1293 1294void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 1295 SDValue &Lo, SDValue &Hi) { 1296 if (ISD::isNormalLoad(N)) { 1297 ExpandRes_NormalLoad(N, Lo, Hi); 1298 return; 1299 } 1300 1301 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1302 1303 MVT VT = N->getValueType(0); 1304 MVT NVT = TLI.getTypeToTransformTo(VT); 1305 SDValue Ch = N->getChain(); 1306 SDValue Ptr = N->getBasePtr(); 1307 ISD::LoadExtType ExtType = N->getExtensionType(); 1308 int SVOffset = N->getSrcValueOffset(); 1309 unsigned Alignment = N->getAlignment(); 1310 bool isVolatile = N->isVolatile(); 1311 1312 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1313 1314 if (N->getMemoryVT().bitsLE(NVT)) { 1315 MVT EVT = N->getMemoryVT(); 1316 1317 Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, EVT, 1318 isVolatile, Alignment); 1319 1320 // Remember the chain. 1321 Ch = Lo.getValue(1); 1322 1323 if (ExtType == ISD::SEXTLOAD) { 1324 // The high part is obtained by SRA'ing all but one of the bits of the 1325 // lo part. 1326 unsigned LoSize = Lo.getValueType().getSizeInBits(); 1327 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 1328 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 1329 } else if (ExtType == ISD::ZEXTLOAD) { 1330 // The high part is just a zero. 1331 Hi = DAG.getConstant(0, NVT); 1332 } else { 1333 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 1334 // The high part is undefined. 1335 Hi = DAG.getNode(ISD::UNDEF, NVT); 1336 } 1337 } else if (TLI.isLittleEndian()) { 1338 // Little-endian - low bits are at low addresses. 1339 Lo = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset, 1340 isVolatile, Alignment); 1341 1342 unsigned ExcessBits = 1343 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1344 MVT NEVT = MVT::getIntegerVT(ExcessBits); 1345 1346 // Increment the pointer to the other half. 1347 unsigned IncrementSize = NVT.getSizeInBits()/8; 1348 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 1349 DAG.getIntPtrConstant(IncrementSize)); 1350 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), 1351 SVOffset+IncrementSize, NEVT, 1352 isVolatile, MinAlign(Alignment, IncrementSize)); 1353 1354 // Build a factor node to remember that this load is independent of the 1355 // other one. 1356 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 1357 Hi.getValue(1)); 1358 } else { 1359 // Big-endian - high bits are at low addresses. Favor aligned loads at 1360 // the cost of some bit-fiddling. 1361 MVT EVT = N->getMemoryVT(); 1362 unsigned EBytes = EVT.getStoreSizeInBits()/8; 1363 unsigned IncrementSize = NVT.getSizeInBits()/8; 1364 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1365 1366 // Load both the high bits and maybe some of the low bits. 1367 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, 1368 MVT::getIntegerVT(EVT.getSizeInBits() - ExcessBits), 1369 isVolatile, Alignment); 1370 1371 // Increment the pointer to the other half. 1372 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 1373 DAG.getIntPtrConstant(IncrementSize)); 1374 // Load the rest of the low bits. 1375 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, NVT, Ch, Ptr, N->getSrcValue(), 1376 SVOffset+IncrementSize, 1377 MVT::getIntegerVT(ExcessBits), 1378 isVolatile, MinAlign(Alignment, IncrementSize)); 1379 1380 // Build a factor node to remember that this load is independent of the 1381 // other one. 1382 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 1383 Hi.getValue(1)); 1384 1385 if (ExcessBits < NVT.getSizeInBits()) { 1386 // Transfer low bits from the bottom of Hi to the top of Lo. 1387 Lo = DAG.getNode(ISD::OR, NVT, Lo, 1388 DAG.getNode(ISD::SHL, NVT, Hi, 1389 DAG.getConstant(ExcessBits, 1390 TLI.getShiftAmountTy()))); 1391 // Move high bits to the right position in Hi. 1392 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, NVT, Hi, 1393 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 1394 TLI.getShiftAmountTy())); 1395 } 1396 } 1397 1398 // Legalized the chain result - switch anything that used the old chain to 1399 // use the new one. 1400 ReplaceValueWith(SDValue(N, 1), Ch); 1401} 1402 1403void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 1404 SDValue &Lo, SDValue &Hi) { 1405 SDValue LL, LH, RL, RH; 1406 GetExpandedInteger(N->getOperand(0), LL, LH); 1407 GetExpandedInteger(N->getOperand(1), RL, RH); 1408 Lo = DAG.getNode(N->getOpcode(), LL.getValueType(), LL, RL); 1409 Hi = DAG.getNode(N->getOpcode(), LL.getValueType(), LH, RH); 1410} 1411 1412void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 1413 SDValue &Lo, SDValue &Hi) { 1414 MVT VT = N->getValueType(0); 1415 MVT NVT = TLI.getTypeToTransformTo(VT); 1416 1417 bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT); 1418 bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT); 1419 bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT); 1420 bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT); 1421 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 1422 SDValue LL, LH, RL, RH; 1423 GetExpandedInteger(N->getOperand(0), LL, LH); 1424 GetExpandedInteger(N->getOperand(1), RL, RH); 1425 unsigned OuterBitSize = VT.getSizeInBits(); 1426 unsigned InnerBitSize = NVT.getSizeInBits(); 1427 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0)); 1428 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1)); 1429 1430 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 1431 if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) && 1432 DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) { 1433 // The inputs are both zero-extended. 1434 if (HasUMUL_LOHI) { 1435 // We can emit a umul_lohi. 1436 Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); 1437 Hi = SDValue(Lo.getNode(), 1); 1438 return; 1439 } 1440 if (HasMULHU) { 1441 // We can emit a mulhu+mul. 1442 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1443 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); 1444 return; 1445 } 1446 } 1447 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 1448 // The input values are both sign-extended. 1449 if (HasSMUL_LOHI) { 1450 // We can emit a smul_lohi. 1451 Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); 1452 Hi = SDValue(Lo.getNode(), 1); 1453 return; 1454 } 1455 if (HasMULHS) { 1456 // We can emit a mulhs+mul. 1457 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1458 Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL); 1459 return; 1460 } 1461 } 1462 if (HasUMUL_LOHI) { 1463 // Lo,Hi = umul LHS, RHS. 1464 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, 1465 DAG.getVTList(NVT, NVT), LL, RL); 1466 Lo = UMulLOHI; 1467 Hi = UMulLOHI.getValue(1); 1468 RH = DAG.getNode(ISD::MUL, NVT, LL, RH); 1469 LH = DAG.getNode(ISD::MUL, NVT, LH, RL); 1470 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); 1471 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH); 1472 return; 1473 } 1474 if (HasMULHU) { 1475 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1476 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); 1477 RH = DAG.getNode(ISD::MUL, NVT, LL, RH); 1478 LH = DAG.getNode(ISD::MUL, NVT, LH, RL); 1479 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); 1480 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH); 1481 return; 1482 } 1483 } 1484 1485 // If nothing else, we can make a libcall. 1486 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1487 if (VT == MVT::i32) 1488 LC = RTLIB::MUL_I32; 1489 else if (VT == MVT::i64) 1490 LC = RTLIB::MUL_I64; 1491 else if (VT == MVT::i128) 1492 LC = RTLIB::MUL_I128; 1493 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!"); 1494 1495 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1496 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*sign irrelevant*/), Lo, Hi); 1497} 1498 1499void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 1500 SDValue &Lo, SDValue &Hi) { 1501 MVT VT = N->getValueType(0); 1502 1503 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1504 if (VT == MVT::i32) 1505 LC = RTLIB::SDIV_I32; 1506 else if (VT == MVT::i64) 1507 LC = RTLIB::SDIV_I64; 1508 else if (VT == MVT::i128) 1509 LC = RTLIB::SDIV_I128; 1510 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 1511 1512 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1513 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true), Lo, Hi); 1514} 1515 1516void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 1517 SDValue &Lo, SDValue &Hi) { 1518 MVT VT = N->getValueType(0); 1519 1520 // If we can emit an efficient shift operation, do so now. Check to see if 1521 // the RHS is a constant. 1522 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 1523 return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi); 1524 1525 // If we can determine that the high bit of the shift is zero or one, even if 1526 // the low bits are variable, emit this shift in an optimized form. 1527 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 1528 return; 1529 1530 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 1531 unsigned PartsOpc; 1532 if (N->getOpcode() == ISD::SHL) { 1533 PartsOpc = ISD::SHL_PARTS; 1534 } else if (N->getOpcode() == ISD::SRL) { 1535 PartsOpc = ISD::SRL_PARTS; 1536 } else { 1537 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1538 PartsOpc = ISD::SRA_PARTS; 1539 } 1540 1541 // Next check to see if the target supports this SHL_PARTS operation or if it 1542 // will custom expand it. 1543 MVT NVT = TLI.getTypeToTransformTo(VT); 1544 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 1545 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 1546 Action == TargetLowering::Custom) { 1547 // Expand the subcomponents. 1548 SDValue LHSL, LHSH; 1549 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1550 1551 SDValue Ops[] = { LHSL, LHSH, N->getOperand(1) }; 1552 MVT VT = LHSL.getValueType(); 1553 Lo = DAG.getNode(PartsOpc, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3); 1554 Hi = Lo.getValue(1); 1555 return; 1556 } 1557 1558 // Otherwise, emit a libcall. 1559 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1560 bool isSigned; 1561 if (N->getOpcode() == ISD::SHL) { 1562 isSigned = false; /*sign irrelevant*/ 1563 if (VT == MVT::i32) 1564 LC = RTLIB::SHL_I32; 1565 else if (VT == MVT::i64) 1566 LC = RTLIB::SHL_I64; 1567 else if (VT == MVT::i128) 1568 LC = RTLIB::SHL_I128; 1569 } else if (N->getOpcode() == ISD::SRL) { 1570 isSigned = false; 1571 if (VT == MVT::i32) 1572 LC = RTLIB::SRL_I32; 1573 else if (VT == MVT::i64) 1574 LC = RTLIB::SRL_I64; 1575 else if (VT == MVT::i128) 1576 LC = RTLIB::SRL_I128; 1577 } else { 1578 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1579 isSigned = true; 1580 if (VT == MVT::i32) 1581 LC = RTLIB::SRA_I32; 1582 else if (VT == MVT::i64) 1583 LC = RTLIB::SRA_I64; 1584 else if (VT == MVT::i128) 1585 LC = RTLIB::SRA_I128; 1586 } 1587 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported shift!"); 1588 1589 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1590 SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned), Lo, Hi); 1591} 1592 1593void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 1594 SDValue &Lo, SDValue &Hi) { 1595 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1596 SDValue Op = N->getOperand(0); 1597 if (Op.getValueType().bitsLE(NVT)) { 1598 // The low part is sign extension of the input (which degenerates to a copy). 1599 Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, N->getOperand(0)); 1600 // The high part is obtained by SRA'ing all but one of the bits of low part. 1601 unsigned LoSize = NVT.getSizeInBits(); 1602 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 1603 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 1604 } else { 1605 // For example, extension of an i48 to an i64. The operand type necessarily 1606 // promotes to the result type, so will end up being expanded too. 1607 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1608 "Only know how to promote this result!"); 1609 SDValue Res = GetPromotedInteger(Op); 1610 assert(Res.getValueType() == N->getValueType(0) && 1611 "Operand over promoted?"); 1612 // Split the promoted operand. This will simplify when it is expanded. 1613 SplitInteger(Res, Lo, Hi); 1614 unsigned ExcessBits = 1615 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 1616 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi, 1617 DAG.getValueType(MVT::getIntegerVT(ExcessBits))); 1618 } 1619} 1620 1621void DAGTypeLegalizer:: 1622ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 1623 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1624 MVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1625 1626 if (EVT.bitsLE(Lo.getValueType())) { 1627 // sext_inreg the low part if needed. 1628 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, Lo.getValueType(), Lo, 1629 N->getOperand(1)); 1630 1631 // The high part gets the sign extension from the lo-part. This handles 1632 // things like sextinreg V:i64 from i8. 1633 Hi = DAG.getNode(ISD::SRA, Hi.getValueType(), Lo, 1634 DAG.getConstant(Hi.getValueType().getSizeInBits()-1, 1635 TLI.getShiftAmountTy())); 1636 } else { 1637 // For example, extension of an i48 to an i64. Leave the low part alone, 1638 // sext_inreg the high part. 1639 unsigned ExcessBits = 1640 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 1641 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi, 1642 DAG.getValueType(MVT::getIntegerVT(ExcessBits))); 1643 } 1644} 1645 1646void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 1647 SDValue &Lo, SDValue &Hi) { 1648 MVT VT = N->getValueType(0); 1649 1650 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1651 if (VT == MVT::i32) 1652 LC = RTLIB::SREM_I32; 1653 else if (VT == MVT::i64) 1654 LC = RTLIB::SREM_I64; 1655 else if (VT == MVT::i128) 1656 LC = RTLIB::SREM_I128; 1657 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 1658 1659 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1660 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true), Lo, Hi); 1661} 1662 1663void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 1664 SDValue &Lo, SDValue &Hi) { 1665 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1666 Lo = DAG.getNode(ISD::TRUNCATE, NVT, N->getOperand(0)); 1667 Hi = DAG.getNode(ISD::SRL, N->getOperand(0).getValueType(), N->getOperand(0), 1668 DAG.getConstant(NVT.getSizeInBits(), 1669 TLI.getShiftAmountTy())); 1670 Hi = DAG.getNode(ISD::TRUNCATE, NVT, Hi); 1671} 1672 1673void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 1674 SDValue &Lo, SDValue &Hi) { 1675 MVT VT = N->getValueType(0); 1676 1677 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1678 if (VT == MVT::i32) 1679 LC = RTLIB::UDIV_I32; 1680 else if (VT == MVT::i64) 1681 LC = RTLIB::UDIV_I64; 1682 else if (VT == MVT::i128) 1683 LC = RTLIB::UDIV_I128; 1684 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 1685 1686 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1687 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false), Lo, Hi); 1688} 1689 1690void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 1691 SDValue &Lo, SDValue &Hi) { 1692 MVT VT = N->getValueType(0); 1693 1694 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1695 if (VT == MVT::i32) 1696 LC = RTLIB::UREM_I32; 1697 else if (VT == MVT::i64) 1698 LC = RTLIB::UREM_I64; 1699 else if (VT == MVT::i128) 1700 LC = RTLIB::UREM_I128; 1701 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 1702 1703 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1704 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false), Lo, Hi); 1705} 1706 1707void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 1708 SDValue &Lo, SDValue &Hi) { 1709 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1710 SDValue Op = N->getOperand(0); 1711 if (Op.getValueType().bitsLE(NVT)) { 1712 // The low part is zero extension of the input (which degenerates to a copy). 1713 Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, N->getOperand(0)); 1714 Hi = DAG.getConstant(0, NVT); // The high part is just a zero. 1715 } else { 1716 // For example, extension of an i48 to an i64. The operand type necessarily 1717 // promotes to the result type, so will end up being expanded too. 1718 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1719 "Only know how to promote this result!"); 1720 SDValue Res = GetPromotedInteger(Op); 1721 assert(Res.getValueType() == N->getValueType(0) && 1722 "Operand over promoted?"); 1723 // Split the promoted operand. This will simplify when it is expanded. 1724 SplitInteger(Res, Lo, Hi); 1725 unsigned ExcessBits = 1726 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 1727 Hi = DAG.getZeroExtendInReg(Hi, MVT::getIntegerVT(ExcessBits)); 1728 } 1729} 1730 1731 1732//===----------------------------------------------------------------------===// 1733// Integer Operand Expansion 1734//===----------------------------------------------------------------------===// 1735 1736/// ExpandIntegerOperand - This method is called when the specified operand of 1737/// the specified node is found to need expansion. At this point, all of the 1738/// result types of the node are known to be legal, but other operands of the 1739/// node may need promotion or expansion as well as the specified one. 1740bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 1741 DEBUG(cerr << "Expand integer operand: "; N->dump(&DAG); cerr << "\n"); 1742 SDValue Res = SDValue(); 1743 1744 if (TLI.getOperationAction(N->getOpcode(), N->getOperand(OpNo).getValueType()) 1745 == TargetLowering::Custom) 1746 Res = TLI.LowerOperation(SDValue(N, OpNo), DAG); 1747 1748 if (Res.getNode() == 0) { 1749 switch (N->getOpcode()) { 1750 default: 1751 #ifndef NDEBUG 1752 cerr << "ExpandIntegerOperand Op #" << OpNo << ": "; 1753 N->dump(&DAG); cerr << "\n"; 1754 #endif 1755 assert(0 && "Do not know how to expand this operator's operand!"); 1756 abort(); 1757 1758 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1759 case ISD::BIT_CONVERT: Res = ExpandOp_BIT_CONVERT(N); break; 1760 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1761 1762 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 1763 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 1764 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 1765 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 1766 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); 1767 break; 1768 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 1769 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 1770 } 1771 } 1772 1773 // If the result is null, the sub-method took care of registering results etc. 1774 if (!Res.getNode()) return false; 1775 // If the result is N, the sub-method updated N in place. Check to see if any 1776 // operands are new, and if so, mark them. 1777 if (Res.getNode() == N) { 1778 // Mark N as new and remark N and its operands. This allows us to correctly 1779 // revisit N if it needs another step of expansion and allows us to visit 1780 // any new operands to N. 1781 ReanalyzeNode(N); 1782 return true; 1783 } 1784 1785 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1786 "Invalid operand expansion"); 1787 1788 ReplaceValueWith(SDValue(N, 0), Res); 1789 return false; 1790} 1791 1792/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 1793/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1794void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 1795 SDValue &NewRHS, 1796 ISD::CondCode &CCCode) { 1797 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 1798 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 1799 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 1800 1801 MVT VT = NewLHS.getValueType(); 1802 1803 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 1804 if (RHSLo == RHSHi) { 1805 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 1806 if (RHSCST->isAllOnesValue()) { 1807 // Equality comparison to -1. 1808 NewLHS = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi); 1809 NewRHS = RHSLo; 1810 return; 1811 } 1812 } 1813 } 1814 1815 NewLHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo); 1816 NewRHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi); 1817 NewLHS = DAG.getNode(ISD::OR, NewLHS.getValueType(), NewLHS, NewRHS); 1818 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 1819 return; 1820 } 1821 1822 // If this is a comparison of the sign bit, just look at the top part. 1823 // X > -1, x < 0 1824 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 1825 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 1826 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 1827 NewLHS = LHSHi; 1828 NewRHS = RHSHi; 1829 return; 1830 } 1831 1832 // FIXME: This generated code sucks. 1833 ISD::CondCode LowCC; 1834 switch (CCCode) { 1835 default: assert(0 && "Unknown integer setcc!"); 1836 case ISD::SETLT: 1837 case ISD::SETULT: LowCC = ISD::SETULT; break; 1838 case ISD::SETGT: 1839 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 1840 case ISD::SETLE: 1841 case ISD::SETULE: LowCC = ISD::SETULE; break; 1842 case ISD::SETGE: 1843 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 1844 } 1845 1846 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 1847 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 1848 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 1849 1850 // NOTE: on targets without efficient SELECT of bools, we can always use 1851 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 1852 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL); 1853 SDValue Tmp1, Tmp2; 1854 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC, 1855 false, DagCombineInfo); 1856 if (!Tmp1.getNode()) 1857 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC); 1858 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 1859 CCCode, false, DagCombineInfo); 1860 if (!Tmp2.getNode()) 1861 Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 1862 DAG.getCondCode(CCCode)); 1863 1864 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 1865 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 1866 if ((Tmp1C && Tmp1C->isNullValue()) || 1867 (Tmp2C && Tmp2C->isNullValue() && 1868 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 1869 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 1870 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 1871 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 1872 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 1873 // low part is known false, returns high part. 1874 // For LE / GE, if high part is known false, ignore the low part. 1875 // For LT / GT, if high part is known true, ignore the low part. 1876 NewLHS = Tmp2; 1877 NewRHS = SDValue(); 1878 return; 1879 } 1880 1881 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 1882 ISD::SETEQ, false, DagCombineInfo); 1883 if (!NewLHS.getNode()) 1884 NewLHS = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 1885 ISD::SETEQ); 1886 NewLHS = DAG.getNode(ISD::SELECT, Tmp1.getValueType(), 1887 NewLHS, Tmp1, Tmp2); 1888 NewRHS = SDValue(); 1889} 1890 1891SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 1892 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1893 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1894 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 1895 1896 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1897 // against zero to select between true and false values. 1898 if (NewRHS.getNode() == 0) { 1899 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 1900 CCCode = ISD::SETNE; 1901 } 1902 1903 // Update N to have the operands specified. 1904 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 1905 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1906 N->getOperand(4)); 1907} 1908 1909SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 1910 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1911 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1912 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 1913 1914 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1915 // against zero to select between true and false values. 1916 if (NewRHS.getNode() == 0) { 1917 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 1918 CCCode = ISD::SETNE; 1919 } 1920 1921 // Update N to have the operands specified. 1922 return DAG.UpdateNodeOperands(SDValue(N, 0), NewLHS, NewRHS, 1923 N->getOperand(2), N->getOperand(3), 1924 DAG.getCondCode(CCCode)); 1925} 1926 1927SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 1928 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1929 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 1930 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 1931 1932 // If ExpandSetCCOperands returned a scalar, use it. 1933 if (NewRHS.getNode() == 0) { 1934 assert(NewLHS.getValueType() == N->getValueType(0) && 1935 "Unexpected setcc expansion!"); 1936 return NewLHS; 1937 } 1938 1939 // Otherwise, update N to have the operands specified. 1940 return DAG.UpdateNodeOperands(SDValue(N, 0), NewLHS, NewRHS, 1941 DAG.getCondCode(CCCode)); 1942} 1943 1944SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 1945 SDValue Op = N->getOperand(0); 1946 MVT DstVT = N->getValueType(0); 1947 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 1948 assert(LC != RTLIB::UNKNOWN_LIBCALL && 1949 "Don't know how to expand this SINT_TO_FP!"); 1950 return MakeLibCall(LC, DstVT, &Op, 1, true); 1951} 1952 1953SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 1954 if (ISD::isNormalStore(N)) 1955 return ExpandOp_NormalStore(N, OpNo); 1956 1957 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1958 assert(OpNo == 1 && "Can only expand the stored value so far"); 1959 1960 MVT VT = N->getOperand(1).getValueType(); 1961 MVT NVT = TLI.getTypeToTransformTo(VT); 1962 SDValue Ch = N->getChain(); 1963 SDValue Ptr = N->getBasePtr(); 1964 int SVOffset = N->getSrcValueOffset(); 1965 unsigned Alignment = N->getAlignment(); 1966 bool isVolatile = N->isVolatile(); 1967 SDValue Lo, Hi; 1968 1969 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1970 1971 if (N->getMemoryVT().bitsLE(NVT)) { 1972 GetExpandedInteger(N->getValue(), Lo, Hi); 1973 return DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 1974 N->getMemoryVT(), isVolatile, Alignment); 1975 } else if (TLI.isLittleEndian()) { 1976 // Little-endian - low bits are at low addresses. 1977 GetExpandedInteger(N->getValue(), Lo, Hi); 1978 1979 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 1980 isVolatile, Alignment); 1981 1982 unsigned ExcessBits = 1983 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1984 MVT NEVT = MVT::getIntegerVT(ExcessBits); 1985 1986 // Increment the pointer to the other half. 1987 unsigned IncrementSize = NVT.getSizeInBits()/8; 1988 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 1989 DAG.getIntPtrConstant(IncrementSize)); 1990 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(), 1991 SVOffset+IncrementSize, NEVT, 1992 isVolatile, MinAlign(Alignment, IncrementSize)); 1993 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 1994 } else { 1995 // Big-endian - high bits are at low addresses. Favor aligned stores at 1996 // the cost of some bit-fiddling. 1997 GetExpandedInteger(N->getValue(), Lo, Hi); 1998 1999 MVT EVT = N->getMemoryVT(); 2000 unsigned EBytes = EVT.getStoreSizeInBits()/8; 2001 unsigned IncrementSize = NVT.getSizeInBits()/8; 2002 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2003 MVT HiVT = MVT::getIntegerVT(EVT.getSizeInBits() - ExcessBits); 2004 2005 if (ExcessBits < NVT.getSizeInBits()) { 2006 // Transfer high bits from the top of Lo to the bottom of Hi. 2007 Hi = DAG.getNode(ISD::SHL, NVT, Hi, 2008 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 2009 TLI.getShiftAmountTy())); 2010 Hi = DAG.getNode(ISD::OR, NVT, Hi, 2011 DAG.getNode(ISD::SRL, NVT, Lo, 2012 DAG.getConstant(ExcessBits, 2013 TLI.getShiftAmountTy()))); 2014 } 2015 2016 // Store both the high bits and maybe some of the low bits. 2017 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(), 2018 SVOffset, HiVT, isVolatile, Alignment); 2019 2020 // Increment the pointer to the other half. 2021 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 2022 DAG.getIntPtrConstant(IncrementSize)); 2023 // Store the lowest ExcessBits bits in the second half. 2024 Lo = DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), 2025 SVOffset+IncrementSize, 2026 MVT::getIntegerVT(ExcessBits), 2027 isVolatile, MinAlign(Alignment, IncrementSize)); 2028 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 2029 } 2030} 2031 2032SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 2033 SDValue InL, InH; 2034 GetExpandedInteger(N->getOperand(0), InL, InH); 2035 // Just truncate the low part of the source. 2036 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), InL); 2037} 2038 2039SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 2040 SDValue Op = N->getOperand(0); 2041 MVT SrcVT = Op.getValueType(); 2042 MVT DstVT = N->getValueType(0); 2043 2044 if (TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 2045 // Do a signed conversion then adjust the result. 2046 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, DstVT, Op); 2047 SignedConv = TLI.LowerOperation(SignedConv, DAG); 2048 2049 // The result of the signed conversion needs adjusting if the 'sign bit' of 2050 // the incoming integer was set. To handle this, we dynamically test to see 2051 // if it is set, and, if so, add a fudge factor. 2052 2053 const uint64_t F32TwoE32 = 0x4F800000ULL; 2054 const uint64_t F32TwoE64 = 0x5F800000ULL; 2055 const uint64_t F32TwoE128 = 0x7F800000ULL; 2056 2057 APInt FF(32, 0); 2058 if (SrcVT == MVT::i32) 2059 FF = APInt(32, F32TwoE32); 2060 else if (SrcVT == MVT::i64) 2061 FF = APInt(32, F32TwoE64); 2062 else if (SrcVT == MVT::i128) 2063 FF = APInt(32, F32TwoE128); 2064 else 2065 assert(false && "Unsupported UINT_TO_FP!"); 2066 2067 // Check whether the sign bit is set. 2068 SDValue Lo, Hi; 2069 GetExpandedInteger(Op, Lo, Hi); 2070 SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi, 2071 DAG.getConstant(0, Hi.getValueType()), 2072 ISD::SETLT); 2073 2074 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 2075 SDValue FudgePtr = DAG.getConstantPool(ConstantInt::get(FF.zext(64)), 2076 TLI.getPointerTy()); 2077 2078 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 2079 SDValue Zero = DAG.getIntPtrConstant(0); 2080 SDValue Four = DAG.getIntPtrConstant(4); 2081 if (TLI.isBigEndian()) std::swap(Zero, Four); 2082 SDValue Offset = DAG.getNode(ISD::SELECT, Zero.getValueType(), SignSet, 2083 Zero, Four); 2084 unsigned Alignment = 2085 1 << cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 2086 FudgePtr = DAG.getNode(ISD::ADD, TLI.getPointerTy(), FudgePtr, Offset); 2087 Alignment = std::min(Alignment, 4u); 2088 2089 // Load the value out, extending it from f32 to the destination float type. 2090 // FIXME: Avoid the extend by constructing the right constant pool? 2091 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, DstVT, DAG.getEntryNode(), 2092 FudgePtr, NULL, 0, MVT::f32, 2093 false, Alignment); 2094 return DAG.getNode(ISD::FADD, DstVT, SignedConv, Fudge); 2095 } 2096 2097 // Otherwise, use a libcall. 2098 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 2099 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2100 "Don't know how to expand this UINT_TO_FP!"); 2101 return MakeLibCall(LC, DstVT, &Op, 1, true); 2102} 2103