LegalizeIntegerTypes.cpp revision 899c5cfbecf4221a3268384e417e85c29c130594
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22#include "llvm/CodeGen/PseudoSourceValue.h" 23#include "llvm/Support/ErrorHandling.h" 24#include "llvm/Support/raw_ostream.h" 25using namespace llvm; 26 27//===----------------------------------------------------------------------===// 28// Integer Result Promotion 29//===----------------------------------------------------------------------===// 30 31/// PromoteIntegerResult - This method is called when a result of a node is 32/// found to be in need of promotion to a larger type. At this point, the node 33/// may also have invalid operands or may have other results that need 34/// expansion, we just know that (at least) one result needs promotion. 35void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 36 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); 37 SDValue Res = SDValue(); 38 39 // See if the target wants to custom expand this node. 40 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 41 return; 42 43 switch (N->getOpcode()) { 44 default: 45#ifndef NDEBUG 46 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 47 N->dump(&DAG); dbgs() << "\n"; 48#endif 49 llvm_unreachable("Do not know how to promote this operator!"); 50 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 51 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 52 case ISD::BIT_CONVERT: Res = PromoteIntRes_BIT_CONVERT(N); break; 53 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 54 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 55 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 56 case ISD::CONVERT_RNDSAT: 57 Res = PromoteIntRes_CONVERT_RNDSAT(N); break; 58 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 59 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 60 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 61 case ISD::EXTRACT_VECTOR_ELT: 62 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 63 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break; 64 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 65 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 66 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 67 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 68 case ISD::SIGN_EXTEND_INREG: 69 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 70 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 71 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 72 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 73 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 74 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 75 76 case ISD::SIGN_EXTEND: 77 case ISD::ZERO_EXTEND: 78 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 79 80 case ISD::FP_TO_SINT: 81 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 82 83 case ISD::FP32_TO_FP16:Res = PromoteIntRes_FP32_TO_FP16(N); break; 84 85 case ISD::AND: 86 case ISD::OR: 87 case ISD::XOR: 88 case ISD::ADD: 89 case ISD::SUB: 90 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 91 92 case ISD::SDIV: 93 case ISD::SREM: Res = PromoteIntRes_SDIV(N); break; 94 95 case ISD::UDIV: 96 case ISD::UREM: Res = PromoteIntRes_UDIV(N); break; 97 98 case ISD::SADDO: 99 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 100 case ISD::UADDO: 101 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 102 case ISD::SMULO: 103 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 104 105 case ISD::ATOMIC_LOAD_ADD: 106 case ISD::ATOMIC_LOAD_SUB: 107 case ISD::ATOMIC_LOAD_AND: 108 case ISD::ATOMIC_LOAD_OR: 109 case ISD::ATOMIC_LOAD_XOR: 110 case ISD::ATOMIC_LOAD_NAND: 111 case ISD::ATOMIC_LOAD_MIN: 112 case ISD::ATOMIC_LOAD_MAX: 113 case ISD::ATOMIC_LOAD_UMIN: 114 case ISD::ATOMIC_LOAD_UMAX: 115 case ISD::ATOMIC_SWAP: 116 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 117 118 case ISD::ATOMIC_CMP_SWAP: 119 Res = PromoteIntRes_Atomic2(cast<AtomicSDNode>(N)); break; 120 } 121 122 // If the result is null then the sub-method took care of registering it. 123 if (Res.getNode()) 124 SetPromotedInteger(SDValue(N, ResNo), Res); 125} 126 127SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 128 // Sign-extend the new bits, and continue the assertion. 129 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 130 return DAG.getNode(ISD::AssertSext, N->getDebugLoc(), 131 Op.getValueType(), Op, N->getOperand(1)); 132} 133 134SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 135 // Zero the new bits, and continue the assertion. 136 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 137 return DAG.getNode(ISD::AssertZext, N->getDebugLoc(), 138 Op.getValueType(), Op, N->getOperand(1)); 139} 140 141SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 142 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 143 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 144 N->getMemoryVT(), 145 N->getChain(), N->getBasePtr(), 146 Op2, N->getSrcValue(), N->getAlignment()); 147 // Legalized the chain result - switch anything that used the old chain to 148 // use the new one. 149 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 150 return Res; 151} 152 153SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) { 154 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 155 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 156 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 157 N->getMemoryVT(), N->getChain(), N->getBasePtr(), 158 Op2, Op3, N->getSrcValue(), N->getAlignment()); 159 // Legalized the chain result - switch anything that used the old chain to 160 // use the new one. 161 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 162 return Res; 163} 164 165SDValue DAGTypeLegalizer::PromoteIntRes_BIT_CONVERT(SDNode *N) { 166 SDValue InOp = N->getOperand(0); 167 EVT InVT = InOp.getValueType(); 168 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 169 EVT OutVT = N->getValueType(0); 170 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 171 DebugLoc dl = N->getDebugLoc(); 172 173 switch (getTypeAction(InVT)) { 174 default: 175 assert(false && "Unknown type action!"); 176 break; 177 case Legal: 178 break; 179 case PromoteInteger: 180 if (NOutVT.bitsEq(NInVT)) 181 // The input promotes to the same size. Convert the promoted value. 182 return DAG.getNode(ISD::BIT_CONVERT, dl, 183 NOutVT, GetPromotedInteger(InOp)); 184 break; 185 case SoftenFloat: 186 // Promote the integer operand by hand. 187 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 188 case ExpandInteger: 189 case ExpandFloat: 190 break; 191 case ScalarizeVector: 192 // Convert the element to an integer and promote it by hand. 193 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 194 BitConvertToInteger(GetScalarizedVector(InOp))); 195 case SplitVector: { 196 // For example, i32 = BIT_CONVERT v2i16 on alpha. Convert the split 197 // pieces of the input into integers and reassemble in the final type. 198 SDValue Lo, Hi; 199 GetSplitVector(N->getOperand(0), Lo, Hi); 200 Lo = BitConvertToInteger(Lo); 201 Hi = BitConvertToInteger(Hi); 202 203 if (TLI.isBigEndian()) 204 std::swap(Lo, Hi); 205 206 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 207 EVT::getIntegerVT(*DAG.getContext(), NOutVT.getSizeInBits()), 208 JoinIntegers(Lo, Hi)); 209 return DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, InOp); 210 } 211 case WidenVector: 212 if (OutVT.bitsEq(NInVT)) 213 // The input is widened to the same size. Convert to the widened value. 214 return DAG.getNode(ISD::BIT_CONVERT, dl, OutVT, GetWidenedVector(InOp)); 215 } 216 217 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 218 CreateStackStoreLoad(InOp, OutVT)); 219} 220 221SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 222 SDValue Op = GetPromotedInteger(N->getOperand(0)); 223 EVT OVT = N->getValueType(0); 224 EVT NVT = Op.getValueType(); 225 DebugLoc dl = N->getDebugLoc(); 226 227 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 228 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 229 DAG.getConstant(DiffBits, TLI.getPointerTy())); 230} 231 232SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 233 // The pair element type may be legal, or may not promote to the same type as 234 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 235 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), 236 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), 237 JoinIntegers(N->getOperand(0), N->getOperand(1))); 238} 239 240SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 241 EVT VT = N->getValueType(0); 242 // FIXME there is no actual debug info here 243 DebugLoc dl = N->getDebugLoc(); 244 // Zero extend things like i1, sign extend everything else. It shouldn't 245 // matter in theory which one we pick, but this tends to give better code? 246 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 247 SDValue Result = DAG.getNode(Opc, dl, TLI.getTypeToTransformTo(*DAG.getContext(), VT), 248 SDValue(N, 0)); 249 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 250 return Result; 251} 252 253SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) { 254 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 255 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 256 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 257 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 258 "can only promote integers"); 259 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 260 return DAG.getConvertRndSat(OutVT, N->getDebugLoc(), N->getOperand(0), 261 N->getOperand(1), N->getOperand(2), 262 N->getOperand(3), N->getOperand(4), CvtCode); 263} 264 265SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 266 // Zero extend to the promoted type and do the count there. 267 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 268 DebugLoc dl = N->getDebugLoc(); 269 EVT OVT = N->getValueType(0); 270 EVT NVT = Op.getValueType(); 271 Op = DAG.getNode(ISD::CTLZ, dl, NVT, Op); 272 // Subtract off the extra leading bits in the bigger type. 273 return DAG.getNode(ISD::SUB, dl, NVT, Op, 274 DAG.getConstant(NVT.getSizeInBits() - 275 OVT.getSizeInBits(), NVT)); 276} 277 278SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 279 // Zero extend to the promoted type and do the count there. 280 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 281 return DAG.getNode(ISD::CTPOP, N->getDebugLoc(), Op.getValueType(), Op); 282} 283 284SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 285 SDValue Op = GetPromotedInteger(N->getOperand(0)); 286 EVT OVT = N->getValueType(0); 287 EVT NVT = Op.getValueType(); 288 DebugLoc dl = N->getDebugLoc(); 289 // The count is the same in the promoted type except if the original 290 // value was zero. This can be handled by setting the bit just off 291 // the top of the original type. 292 APInt TopBit(NVT.getSizeInBits(), 0); 293 TopBit.set(OVT.getSizeInBits()); 294 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT)); 295 return DAG.getNode(ISD::CTTZ, dl, NVT, Op); 296} 297 298SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 299 DebugLoc dl = N->getDebugLoc(); 300 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 301 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), 302 N->getOperand(1)); 303} 304 305SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 306 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 307 unsigned NewOpc = N->getOpcode(); 308 DebugLoc dl = N->getDebugLoc(); 309 310 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 311 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 312 // and SINT conversions are Custom, there is no way to tell which is preferable. 313 // We choose SINT because that's the right thing on PPC.) 314 if (N->getOpcode() == ISD::FP_TO_UINT && 315 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 316 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 317 NewOpc = ISD::FP_TO_SINT; 318 319 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 320 321 // Assert that the converted value fits in the original type. If it doesn't 322 // (eg: because the value being converted is too big), then the result of the 323 // original operation was undefined anyway, so the assert is still correct. 324 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 325 ISD::AssertZext : ISD::AssertSext, dl, 326 NVT, Res, DAG.getValueType(N->getValueType(0))); 327} 328 329SDValue DAGTypeLegalizer::PromoteIntRes_FP32_TO_FP16(SDNode *N) { 330 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 331 DebugLoc dl = N->getDebugLoc(); 332 333 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 334 335 return DAG.getNode(ISD::AssertZext, dl, 336 NVT, Res, DAG.getValueType(N->getValueType(0))); 337} 338 339SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 340 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 341 DebugLoc dl = N->getDebugLoc(); 342 343 if (getTypeAction(N->getOperand(0).getValueType()) == PromoteInteger) { 344 SDValue Res = GetPromotedInteger(N->getOperand(0)); 345 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 346 347 // If the result and operand types are the same after promotion, simplify 348 // to an in-register extension. 349 if (NVT == Res.getValueType()) { 350 // The high bits are not guaranteed to be anything. Insert an extend. 351 if (N->getOpcode() == ISD::SIGN_EXTEND) 352 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 353 DAG.getValueType(N->getOperand(0).getValueType())); 354 if (N->getOpcode() == ISD::ZERO_EXTEND) 355 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); 356 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 357 return Res; 358 } 359 } 360 361 // Otherwise, just extend the original operand all the way to the larger type. 362 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 363} 364 365SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 366 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 367 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 368 ISD::LoadExtType ExtType = 369 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 370 DebugLoc dl = N->getDebugLoc(); 371 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 372 N->getSrcValue(), N->getSrcValueOffset(), 373 N->getMemoryVT(), N->isVolatile(), 374 N->isNonTemporal(), N->getAlignment()); 375 376 // Legalized the chain result - switch anything that used the old chain to 377 // use the new one. 378 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 379 return Res; 380} 381 382/// Promote the overflow flag of an overflowing arithmetic node. 383SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 384 // Simply change the return type of the boolean result. 385 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 386 EVT ValueVTs[] = { N->getValueType(0), NVT }; 387 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) }; 388 SDValue Res = DAG.getNode(N->getOpcode(), N->getDebugLoc(), 389 DAG.getVTList(ValueVTs, 2), Ops, 2); 390 391 // Modified the sum result - switch anything that used the old sum to use 392 // the new one. 393 ReplaceValueWith(SDValue(N, 0), Res); 394 395 return SDValue(Res.getNode(), 1); 396} 397 398SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 399 if (ResNo == 1) 400 return PromoteIntRes_Overflow(N); 401 402 // The operation overflowed iff the result in the larger type is not the 403 // sign extension of its truncation to the original type. 404 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 405 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 406 EVT OVT = N->getOperand(0).getValueType(); 407 EVT NVT = LHS.getValueType(); 408 DebugLoc dl = N->getDebugLoc(); 409 410 // Do the arithmetic in the larger type. 411 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 412 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 413 414 // Calculate the overflow flag: sign extend the arithmetic result from 415 // the original type. 416 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 417 DAG.getValueType(OVT)); 418 // Overflowed if and only if this is not equal to Res. 419 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 420 421 // Use the calculated overflow everywhere. 422 ReplaceValueWith(SDValue(N, 1), Ofl); 423 424 return Res; 425} 426 427SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) { 428 // Sign extend the input. 429 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 430 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 431 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 432 LHS.getValueType(), LHS, RHS); 433} 434 435SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 436 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 437 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 438 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 439 LHS.getValueType(), N->getOperand(0),LHS,RHS); 440} 441 442SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 443 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 444 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 445 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 446 LHS.getValueType(), N->getOperand(0), 447 N->getOperand(1), LHS, RHS, N->getOperand(4)); 448} 449 450SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 451 EVT SVT = TLI.getSetCCResultType(N->getOperand(0).getValueType()); 452 assert(isTypeLegal(SVT) && "Illegal SetCC type!"); 453 DebugLoc dl = N->getDebugLoc(); 454 455 // Get the SETCC result using the canonical SETCC type. 456 SDValue SetCC = DAG.getNode(ISD::SETCC, dl, SVT, N->getOperand(0), 457 N->getOperand(1), N->getOperand(2)); 458 459 // Convert to the expected type. 460 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 461 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?"); 462 return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC); 463} 464 465SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 466 return DAG.getNode(ISD::SHL, N->getDebugLoc(), 467 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), 468 GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); 469} 470 471SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 472 SDValue Op = GetPromotedInteger(N->getOperand(0)); 473 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), 474 Op.getValueType(), Op, N->getOperand(1)); 475} 476 477SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 478 // The input may have strange things in the top bits of the registers, but 479 // these operations don't care. They may have weird bits going out, but 480 // that too is okay if they are integer operations. 481 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 482 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 483 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 484 LHS.getValueType(), LHS, RHS); 485} 486 487SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 488 // The input value must be properly sign extended. 489 SDValue Res = SExtPromotedInteger(N->getOperand(0)); 490 return DAG.getNode(ISD::SRA, N->getDebugLoc(), 491 Res.getValueType(), Res, N->getOperand(1)); 492} 493 494SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 495 // The input value must be properly zero extended. 496 EVT VT = N->getValueType(0); 497 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 498 SDValue Res = ZExtPromotedInteger(N->getOperand(0)); 499 return DAG.getNode(ISD::SRL, N->getDebugLoc(), NVT, Res, N->getOperand(1)); 500} 501 502SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 503 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 504 SDValue Res; 505 506 switch (getTypeAction(N->getOperand(0).getValueType())) { 507 default: llvm_unreachable("Unknown type action!"); 508 case Legal: 509 case ExpandInteger: 510 Res = N->getOperand(0); 511 break; 512 case PromoteInteger: 513 Res = GetPromotedInteger(N->getOperand(0)); 514 break; 515 } 516 517 // Truncate to NVT instead of VT 518 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), NVT, Res); 519} 520 521SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 522 if (ResNo == 1) 523 return PromoteIntRes_Overflow(N); 524 525 // The operation overflowed iff the result in the larger type is not the 526 // zero extension of its truncation to the original type. 527 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 528 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 529 EVT OVT = N->getOperand(0).getValueType(); 530 EVT NVT = LHS.getValueType(); 531 DebugLoc dl = N->getDebugLoc(); 532 533 // Do the arithmetic in the larger type. 534 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 535 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 536 537 // Calculate the overflow flag: zero extend the arithmetic result from 538 // the original type. 539 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 540 // Overflowed if and only if this is not equal to Res. 541 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 542 543 // Use the calculated overflow everywhere. 544 ReplaceValueWith(SDValue(N, 1), Ofl); 545 546 return Res; 547} 548 549SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { 550 // Zero extend the input. 551 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 552 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 553 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 554 LHS.getValueType(), LHS, RHS); 555} 556 557SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 558 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0))); 559} 560 561SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 562 SDValue Chain = N->getOperand(0); // Get the chain. 563 SDValue Ptr = N->getOperand(1); // Get the pointer. 564 EVT VT = N->getValueType(0); 565 DebugLoc dl = N->getDebugLoc(); 566 567 EVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 568 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 569 // The argument is passed as NumRegs registers of type RegVT. 570 571 SmallVector<SDValue, 8> Parts(NumRegs); 572 for (unsigned i = 0; i < NumRegs; ++i) { 573 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2)); 574 Chain = Parts[i].getValue(1); 575 } 576 577 // Handle endianness of the load. 578 if (TLI.isBigEndian()) 579 std::reverse(Parts.begin(), Parts.end()); 580 581 // Assemble the parts in the promoted type. 582 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 583 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 584 for (unsigned i = 1; i < NumRegs; ++i) { 585 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 586 // Shift it to the right position and "or" it in. 587 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 588 DAG.getConstant(i * RegVT.getSizeInBits(), 589 TLI.getPointerTy())); 590 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 591 } 592 593 // Modified the chain result - switch anything that used the old chain to 594 // use the new one. 595 ReplaceValueWith(SDValue(N, 1), Chain); 596 597 return Res; 598} 599 600SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 601 assert(ResNo == 1 && "Only boolean result promotion currently supported!"); 602 return PromoteIntRes_Overflow(N); 603} 604 605//===----------------------------------------------------------------------===// 606// Integer Operand Promotion 607//===----------------------------------------------------------------------===// 608 609/// PromoteIntegerOperand - This method is called when the specified operand of 610/// the specified node is found to need promotion. At this point, all of the 611/// result types of the node are known to be legal, but other operands of the 612/// node may need promotion or expansion as well as the specified one. 613bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 614 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); 615 SDValue Res = SDValue(); 616 617 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 618 return false; 619 620 switch (N->getOpcode()) { 621 default: 622 #ifndef NDEBUG 623 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 624 N->dump(&DAG); dbgs() << "\n"; 625 #endif 626 llvm_unreachable("Do not know how to promote this operator's operand!"); 627 628 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 629 case ISD::BIT_CONVERT: Res = PromoteIntOp_BIT_CONVERT(N); break; 630 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 631 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 632 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 633 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 634 case ISD::CONVERT_RNDSAT: 635 Res = PromoteIntOp_CONVERT_RNDSAT(N); break; 636 case ISD::INSERT_VECTOR_ELT: 637 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 638 case ISD::MEMBARRIER: Res = PromoteIntOp_MEMBARRIER(N); break; 639 case ISD::SCALAR_TO_VECTOR: 640 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 641 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 642 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 643 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 644 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 645 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 646 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 647 OpNo); break; 648 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 649 case ISD::FP16_TO_FP32: 650 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 651 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 652 653 case ISD::SHL: 654 case ISD::SRA: 655 case ISD::SRL: 656 case ISD::ROTL: 657 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 658 } 659 660 // If the result is null, the sub-method took care of registering results etc. 661 if (!Res.getNode()) return false; 662 663 // If the result is N, the sub-method updated N in place. Tell the legalizer 664 // core about this. 665 if (Res.getNode() == N) 666 return true; 667 668 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 669 "Invalid operand expansion"); 670 671 ReplaceValueWith(SDValue(N, 0), Res); 672 return false; 673} 674 675/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 676/// shared among BR_CC, SELECT_CC, and SETCC handlers. 677void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 678 ISD::CondCode CCCode) { 679 // We have to insert explicit sign or zero extends. Note that we could 680 // insert sign extends for ALL conditions, but zero extend is cheaper on 681 // many machines (an AND instead of two shifts), so prefer it. 682 switch (CCCode) { 683 default: llvm_unreachable("Unknown integer comparison!"); 684 case ISD::SETEQ: 685 case ISD::SETNE: 686 case ISD::SETUGE: 687 case ISD::SETUGT: 688 case ISD::SETULE: 689 case ISD::SETULT: 690 // ALL of these operations will work if we either sign or zero extend 691 // the operands (including the unsigned comparisons!). Zero extend is 692 // usually a simpler/cheaper operation, so prefer it. 693 NewLHS = ZExtPromotedInteger(NewLHS); 694 NewRHS = ZExtPromotedInteger(NewRHS); 695 break; 696 case ISD::SETGE: 697 case ISD::SETGT: 698 case ISD::SETLT: 699 case ISD::SETLE: 700 NewLHS = SExtPromotedInteger(NewLHS); 701 NewRHS = SExtPromotedInteger(NewRHS); 702 break; 703 } 704} 705 706SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 707 SDValue Op = GetPromotedInteger(N->getOperand(0)); 708 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), Op); 709} 710 711SDValue DAGTypeLegalizer::PromoteIntOp_BIT_CONVERT(SDNode *N) { 712 // This should only occur in unusual situations like bitcasting to an 713 // x86_fp80, so just turn it into a store+load 714 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 715} 716 717SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 718 assert(OpNo == 2 && "Don't know how to promote this operand!"); 719 720 SDValue LHS = N->getOperand(2); 721 SDValue RHS = N->getOperand(3); 722 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 723 724 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 725 // legal types. 726 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 727 N->getOperand(1), LHS, RHS, N->getOperand(4)); 728} 729 730SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 731 assert(OpNo == 1 && "only know how to promote condition"); 732 733 // Promote all the way up to the canonical SetCC type. 734 EVT SVT = TLI.getSetCCResultType(MVT::Other); 735 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), SVT); 736 737 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 738 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), Cond, 739 N->getOperand(2)); 740} 741 742SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 743 // Since the result type is legal, the operands must promote to it. 744 EVT OVT = N->getOperand(0).getValueType(); 745 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 746 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 747 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 748 DebugLoc dl = N->getDebugLoc(); 749 750 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 751 DAG.getConstant(OVT.getSizeInBits(), TLI.getPointerTy())); 752 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 753} 754 755SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 756 // The vector type is legal but the element type is not. This implies 757 // that the vector is a power-of-two in length and that the element 758 // type does not have a strange size (eg: it is not i1). 759 EVT VecVT = N->getValueType(0); 760 unsigned NumElts = VecVT.getVectorNumElements(); 761 assert(!(NumElts & 1) && "Legal vector of one illegal element?"); 762 763 // Promote the inserted value. The type does not need to match the 764 // vector element type. Check that any extra bits introduced will be 765 // truncated away. 766 assert(N->getOperand(0).getValueType().getSizeInBits() >= 767 N->getValueType(0).getVectorElementType().getSizeInBits() && 768 "Type of inserted value narrower than vector element type!"); 769 770 SmallVector<SDValue, 16> NewOps; 771 for (unsigned i = 0; i < NumElts; ++i) 772 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 773 774 return DAG.UpdateNodeOperands(SDValue(N, 0), &NewOps[0], NumElts); 775} 776 777SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) { 778 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 779 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 780 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 781 CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) && 782 "can only promote integer arguments"); 783 SDValue InOp = GetPromotedInteger(N->getOperand(0)); 784 return DAG.getConvertRndSat(N->getValueType(0), N->getDebugLoc(), InOp, 785 N->getOperand(1), N->getOperand(2), 786 N->getOperand(3), N->getOperand(4), CvtCode); 787} 788 789SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 790 unsigned OpNo) { 791 if (OpNo == 1) { 792 // Promote the inserted value. This is valid because the type does not 793 // have to match the vector element type. 794 795 // Check that any extra bits introduced will be truncated away. 796 assert(N->getOperand(1).getValueType().getSizeInBits() >= 797 N->getValueType(0).getVectorElementType().getSizeInBits() && 798 "Type of inserted value narrower than vector element type!"); 799 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 800 GetPromotedInteger(N->getOperand(1)), 801 N->getOperand(2)); 802 } 803 804 assert(OpNo == 2 && "Different operand and result vector types?"); 805 806 // Promote the index. 807 SDValue Idx = ZExtPromotedInteger(N->getOperand(2)); 808 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 809 N->getOperand(1), Idx); 810} 811 812SDValue DAGTypeLegalizer::PromoteIntOp_MEMBARRIER(SDNode *N) { 813 SDValue NewOps[6]; 814 DebugLoc dl = N->getDebugLoc(); 815 NewOps[0] = N->getOperand(0); 816 for (unsigned i = 1; i < array_lengthof(NewOps); ++i) { 817 SDValue Flag = GetPromotedInteger(N->getOperand(i)); 818 NewOps[i] = DAG.getZeroExtendInReg(Flag, dl, MVT::i1); 819 } 820 return DAG.UpdateNodeOperands(SDValue (N, 0), NewOps, 821 array_lengthof(NewOps)); 822} 823 824SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 825 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 826 // the operand in place. 827 return DAG.UpdateNodeOperands(SDValue(N, 0), 828 GetPromotedInteger(N->getOperand(0))); 829} 830 831SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 832 assert(OpNo == 0 && "Only know how to promote condition"); 833 834 // Promote all the way up to the canonical SetCC type. 835 EVT SVT = TLI.getSetCCResultType(N->getOperand(1).getValueType()); 836 SDValue Cond = PromoteTargetBoolean(N->getOperand(0), SVT); 837 838 return DAG.UpdateNodeOperands(SDValue(N, 0), Cond, 839 N->getOperand(1), N->getOperand(2)); 840} 841 842SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 843 assert(OpNo == 0 && "Don't know how to promote this operand!"); 844 845 SDValue LHS = N->getOperand(0); 846 SDValue RHS = N->getOperand(1); 847 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 848 849 // The CC (#4) and the possible return values (#2 and #3) have legal types. 850 return DAG.UpdateNodeOperands(SDValue(N, 0), LHS, RHS, N->getOperand(2), 851 N->getOperand(3), N->getOperand(4)); 852} 853 854SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 855 assert(OpNo == 0 && "Don't know how to promote this operand!"); 856 857 SDValue LHS = N->getOperand(0); 858 SDValue RHS = N->getOperand(1); 859 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 860 861 // The CC (#2) is always legal. 862 return DAG.UpdateNodeOperands(SDValue(N, 0), LHS, RHS, N->getOperand(2)); 863} 864 865SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 866 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 867 ZExtPromotedInteger(N->getOperand(1))); 868} 869 870SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 871 SDValue Op = GetPromotedInteger(N->getOperand(0)); 872 DebugLoc dl = N->getDebugLoc(); 873 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 874 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 875 Op, DAG.getValueType(N->getOperand(0).getValueType())); 876} 877 878SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 879 return DAG.UpdateNodeOperands(SDValue(N, 0), 880 SExtPromotedInteger(N->getOperand(0))); 881} 882 883SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 884 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 885 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 886 int SVOffset = N->getSrcValueOffset(); 887 unsigned Alignment = N->getAlignment(); 888 bool isVolatile = N->isVolatile(); 889 bool isNonTemporal = N->isNonTemporal(); 890 DebugLoc dl = N->getDebugLoc(); 891 892 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 893 894 // Truncate the value and store the result. 895 return DAG.getTruncStore(Ch, dl, Val, Ptr, N->getSrcValue(), 896 SVOffset, N->getMemoryVT(), 897 isVolatile, isNonTemporal, Alignment); 898} 899 900SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 901 SDValue Op = GetPromotedInteger(N->getOperand(0)); 902 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), Op); 903} 904 905SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 906 return DAG.UpdateNodeOperands(SDValue(N, 0), 907 ZExtPromotedInteger(N->getOperand(0))); 908} 909 910SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 911 DebugLoc dl = N->getDebugLoc(); 912 SDValue Op = GetPromotedInteger(N->getOperand(0)); 913 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 914 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); 915} 916 917 918//===----------------------------------------------------------------------===// 919// Integer Result Expansion 920//===----------------------------------------------------------------------===// 921 922/// ExpandIntegerResult - This method is called when the specified result of the 923/// specified node is found to need expansion. At this point, the node may also 924/// have invalid operands or may have other results that need promotion, we just 925/// know that (at least) one result needs expansion. 926void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 927 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); 928 SDValue Lo, Hi; 929 Lo = Hi = SDValue(); 930 931 // See if the target wants to custom expand this node. 932 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 933 return; 934 935 switch (N->getOpcode()) { 936 default: 937#ifndef NDEBUG 938 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 939 N->dump(&DAG); dbgs() << "\n"; 940#endif 941 llvm_unreachable("Do not know how to expand the result of this operator!"); 942 943 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 944 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 945 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 946 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 947 948 case ISD::BIT_CONVERT: ExpandRes_BIT_CONVERT(N, Lo, Hi); break; 949 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 950 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 951 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 952 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 953 954 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 955 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 956 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 957 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 958 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 959 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 960 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 961 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 962 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 963 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 964 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 965 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 966 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 967 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 968 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 969 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 970 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 971 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 972 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 973 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 974 975 case ISD::AND: 976 case ISD::OR: 977 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 978 979 case ISD::ADD: 980 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 981 982 case ISD::ADDC: 983 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 984 985 case ISD::ADDE: 986 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 987 988 case ISD::SHL: 989 case ISD::SRA: 990 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 991 } 992 993 // If Lo/Hi is null, the sub-method took care of registering results etc. 994 if (Lo.getNode()) 995 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 996} 997 998/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded, 999/// and the shift amount is a constant 'Amt'. Expand the operation. 1000void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 1001 SDValue &Lo, SDValue &Hi) { 1002 DebugLoc dl = N->getDebugLoc(); 1003 // Expand the incoming operand to be shifted, so that we have its parts 1004 SDValue InL, InH; 1005 GetExpandedInteger(N->getOperand(0), InL, InH); 1006 1007 EVT NVT = InL.getValueType(); 1008 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1009 unsigned NVTBits = NVT.getSizeInBits(); 1010 EVT ShTy = N->getOperand(1).getValueType(); 1011 1012 if (N->getOpcode() == ISD::SHL) { 1013 if (Amt > VTBits) { 1014 Lo = Hi = DAG.getConstant(0, NVT); 1015 } else if (Amt > NVTBits) { 1016 Lo = DAG.getConstant(0, NVT); 1017 Hi = DAG.getNode(ISD::SHL, dl, 1018 NVT, InL, DAG.getConstant(Amt-NVTBits,ShTy)); 1019 } else if (Amt == NVTBits) { 1020 Lo = DAG.getConstant(0, NVT); 1021 Hi = InL; 1022 } else if (Amt == 1 && 1023 TLI.isOperationLegalOrCustom(ISD::ADDC, 1024 TLI.getTypeToExpandTo(*DAG.getContext(), NVT))) { 1025 // Emit this X << 1 as X+X. 1026 SDVTList VTList = DAG.getVTList(NVT, MVT::Flag); 1027 SDValue LoOps[2] = { InL, InL }; 1028 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1029 SDValue HiOps[3] = { InH, InH, Lo.getValue(1) }; 1030 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1031 } else { 1032 Lo = DAG.getNode(ISD::SHL, dl, NVT, InL, DAG.getConstant(Amt, ShTy)); 1033 Hi = DAG.getNode(ISD::OR, dl, NVT, 1034 DAG.getNode(ISD::SHL, dl, NVT, InH, 1035 DAG.getConstant(Amt, ShTy)), 1036 DAG.getNode(ISD::SRL, dl, NVT, InL, 1037 DAG.getConstant(NVTBits-Amt, ShTy))); 1038 } 1039 return; 1040 } 1041 1042 if (N->getOpcode() == ISD::SRL) { 1043 if (Amt > VTBits) { 1044 Lo = DAG.getConstant(0, NVT); 1045 Hi = DAG.getConstant(0, NVT); 1046 } else if (Amt > NVTBits) { 1047 Lo = DAG.getNode(ISD::SRL, dl, 1048 NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy)); 1049 Hi = DAG.getConstant(0, NVT); 1050 } else if (Amt == NVTBits) { 1051 Lo = InH; 1052 Hi = DAG.getConstant(0, NVT); 1053 } else { 1054 Lo = DAG.getNode(ISD::OR, dl, NVT, 1055 DAG.getNode(ISD::SRL, dl, NVT, InL, 1056 DAG.getConstant(Amt, ShTy)), 1057 DAG.getNode(ISD::SHL, dl, NVT, InH, 1058 DAG.getConstant(NVTBits-Amt, ShTy))); 1059 Hi = DAG.getNode(ISD::SRL, dl, NVT, InH, DAG.getConstant(Amt, ShTy)); 1060 } 1061 return; 1062 } 1063 1064 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1065 if (Amt > VTBits) { 1066 Hi = Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, 1067 DAG.getConstant(NVTBits-1, ShTy)); 1068 } else if (Amt > NVTBits) { 1069 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, 1070 DAG.getConstant(Amt-NVTBits, ShTy)); 1071 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, 1072 DAG.getConstant(NVTBits-1, ShTy)); 1073 } else if (Amt == NVTBits) { 1074 Lo = InH; 1075 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, 1076 DAG.getConstant(NVTBits-1, ShTy)); 1077 } else { 1078 Lo = DAG.getNode(ISD::OR, dl, NVT, 1079 DAG.getNode(ISD::SRL, dl, NVT, InL, 1080 DAG.getConstant(Amt, ShTy)), 1081 DAG.getNode(ISD::SHL, dl, NVT, InH, 1082 DAG.getConstant(NVTBits-Amt, ShTy))); 1083 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, DAG.getConstant(Amt, ShTy)); 1084 } 1085} 1086 1087/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1088/// this shift based on knowledge of the high bit of the shift amount. If we 1089/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1090/// shift amount. 1091bool DAGTypeLegalizer:: 1092ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1093 SDValue Amt = N->getOperand(1); 1094 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1095 EVT ShTy = Amt.getValueType(); 1096 unsigned ShBits = ShTy.getScalarType().getSizeInBits(); 1097 unsigned NVTBits = NVT.getScalarType().getSizeInBits(); 1098 assert(isPowerOf2_32(NVTBits) && 1099 "Expanded integer type size not a power of two!"); 1100 DebugLoc dl = N->getDebugLoc(); 1101 1102 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1103 APInt KnownZero, KnownOne; 1104 DAG.ComputeMaskedBits(N->getOperand(1), HighBitMask, KnownZero, KnownOne); 1105 1106 // If we don't know anything about the high bits, exit. 1107 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1108 return false; 1109 1110 // Get the incoming operand to be shifted. 1111 SDValue InL, InH; 1112 GetExpandedInteger(N->getOperand(0), InL, InH); 1113 1114 // If we know that any of the high bits of the shift amount are one, then we 1115 // can do this as a couple of simple shifts. 1116 if (KnownOne.intersects(HighBitMask)) { 1117 // Mask out the high bit, which we know is set. 1118 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 1119 DAG.getConstant(~HighBitMask, ShTy)); 1120 1121 switch (N->getOpcode()) { 1122 default: llvm_unreachable("Unknown shift"); 1123 case ISD::SHL: 1124 Lo = DAG.getConstant(0, NVT); // Low part is zero. 1125 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 1126 return true; 1127 case ISD::SRL: 1128 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 1129 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 1130 return true; 1131 case ISD::SRA: 1132 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 1133 DAG.getConstant(NVTBits-1, ShTy)); 1134 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 1135 return true; 1136 } 1137 } 1138 1139#if 0 1140 // FIXME: This code is broken for shifts with a zero amount! 1141 // If we know that all of the high bits of the shift amount are zero, then we 1142 // can do this as a couple of simple shifts. 1143 if ((KnownZero & HighBitMask) == HighBitMask) { 1144 // Compute 32-amt. 1145 SDValue Amt2 = DAG.getNode(ISD::SUB, ShTy, 1146 DAG.getConstant(NVTBits, ShTy), 1147 Amt); 1148 unsigned Op1, Op2; 1149 switch (N->getOpcode()) { 1150 default: llvm_unreachable("Unknown shift"); 1151 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1152 case ISD::SRL: 1153 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1154 } 1155 1156 Lo = DAG.getNode(N->getOpcode(), NVT, InL, Amt); 1157 Hi = DAG.getNode(ISD::OR, NVT, 1158 DAG.getNode(Op1, NVT, InH, Amt), 1159 DAG.getNode(Op2, NVT, InL, Amt2)); 1160 return true; 1161 } 1162#endif 1163 1164 return false; 1165} 1166 1167/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 1168/// of any size. 1169bool DAGTypeLegalizer:: 1170ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1171 SDValue Amt = N->getOperand(1); 1172 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1173 EVT ShTy = Amt.getValueType(); 1174 unsigned NVTBits = NVT.getSizeInBits(); 1175 assert(isPowerOf2_32(NVTBits) && 1176 "Expanded integer type size not a power of two!"); 1177 DebugLoc dl = N->getDebugLoc(); 1178 1179 // Get the incoming operand to be shifted. 1180 SDValue InL, InH; 1181 GetExpandedInteger(N->getOperand(0), InL, InH); 1182 1183 SDValue NVBitsNode = DAG.getConstant(NVTBits, ShTy); 1184 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 1185 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 1186 SDValue isShort = DAG.getSetCC(dl, TLI.getSetCCResultType(ShTy), 1187 Amt, NVBitsNode, ISD::SETULT); 1188 1189 SDValue LoS, HiS, LoL, HiL; 1190 switch (N->getOpcode()) { 1191 default: llvm_unreachable("Unknown shift"); 1192 case ISD::SHL: 1193 // Short: ShAmt < NVTBits 1194 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 1195 HiS = DAG.getNode(ISD::OR, dl, NVT, 1196 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 1197 // FIXME: If Amt is zero, the following shift generates an undefined result 1198 // on some architectures. 1199 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 1200 1201 // Long: ShAmt >= NVTBits 1202 LoL = DAG.getConstant(0, NVT); // Lo part is zero. 1203 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 1204 1205 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1206 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1207 return true; 1208 case ISD::SRL: 1209 // Short: ShAmt < NVTBits 1210 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 1211 LoS = DAG.getNode(ISD::OR, dl, NVT, 1212 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1213 // FIXME: If Amt is zero, the following shift generates an undefined result 1214 // on some architectures. 1215 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1216 1217 // Long: ShAmt >= NVTBits 1218 HiL = DAG.getConstant(0, NVT); // Hi part is zero. 1219 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1220 1221 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1222 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1223 return true; 1224 case ISD::SRA: 1225 // Short: ShAmt < NVTBits 1226 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 1227 LoS = DAG.getNode(ISD::OR, dl, NVT, 1228 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1229 // FIXME: If Amt is zero, the following shift generates an undefined result 1230 // on some architectures. 1231 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1232 1233 // Long: ShAmt >= NVTBits 1234 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 1235 DAG.getConstant(NVTBits-1, ShTy)); 1236 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1237 1238 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1239 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1240 return true; 1241 } 1242 1243 return false; 1244} 1245 1246void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1247 SDValue &Lo, SDValue &Hi) { 1248 DebugLoc dl = N->getDebugLoc(); 1249 // Expand the subcomponents. 1250 SDValue LHSL, LHSH, RHSL, RHSH; 1251 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1252 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1253 1254 EVT NVT = LHSL.getValueType(); 1255 SDValue LoOps[2] = { LHSL, RHSL }; 1256 SDValue HiOps[3] = { LHSH, RHSH }; 1257 1258 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1259 // them. TODO: Teach operation legalization how to expand unsupported 1260 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1261 // a carry of type MVT::Flag, but there doesn't seem to be any way to 1262 // generate a value of this type in the expanded code sequence. 1263 bool hasCarry = 1264 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1265 ISD::ADDC : ISD::SUBC, 1266 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1267 1268 if (hasCarry) { 1269 SDVTList VTList = DAG.getVTList(NVT, MVT::Flag); 1270 if (N->getOpcode() == ISD::ADD) { 1271 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1272 HiOps[2] = Lo.getValue(1); 1273 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1274 } else { 1275 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 1276 HiOps[2] = Lo.getValue(1); 1277 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 1278 } 1279 } else { 1280 if (N->getOpcode() == ISD::ADD) { 1281 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps, 2); 1282 Hi = DAG.getNode(ISD::ADD, dl, NVT, HiOps, 2); 1283 SDValue Cmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[0], 1284 ISD::SETULT); 1285 SDValue Carry1 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp1, 1286 DAG.getConstant(1, NVT), 1287 DAG.getConstant(0, NVT)); 1288 SDValue Cmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[1], 1289 ISD::SETULT); 1290 SDValue Carry2 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp2, 1291 DAG.getConstant(1, NVT), Carry1); 1292 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 1293 } else { 1294 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps, 2); 1295 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps, 2); 1296 SDValue Cmp = 1297 DAG.getSetCC(dl, TLI.getSetCCResultType(LoOps[0].getValueType()), 1298 LoOps[0], LoOps[1], ISD::SETULT); 1299 SDValue Borrow = DAG.getNode(ISD::SELECT, dl, NVT, Cmp, 1300 DAG.getConstant(1, NVT), 1301 DAG.getConstant(0, NVT)); 1302 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 1303 } 1304 } 1305} 1306 1307void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1308 SDValue &Lo, SDValue &Hi) { 1309 // Expand the subcomponents. 1310 SDValue LHSL, LHSH, RHSL, RHSH; 1311 DebugLoc dl = N->getDebugLoc(); 1312 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1313 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1314 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1315 SDValue LoOps[2] = { LHSL, RHSL }; 1316 SDValue HiOps[3] = { LHSH, RHSH }; 1317 1318 if (N->getOpcode() == ISD::ADDC) { 1319 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1320 HiOps[2] = Lo.getValue(1); 1321 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1322 } else { 1323 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 1324 HiOps[2] = Lo.getValue(1); 1325 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 1326 } 1327 1328 // Legalized the flag result - switch anything that used the old flag to 1329 // use the new one. 1330 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1331} 1332 1333void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1334 SDValue &Lo, SDValue &Hi) { 1335 // Expand the subcomponents. 1336 SDValue LHSL, LHSH, RHSL, RHSH; 1337 DebugLoc dl = N->getDebugLoc(); 1338 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1339 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1340 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1341 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1342 SDValue HiOps[3] = { LHSH, RHSH }; 1343 1344 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps, 3); 1345 HiOps[2] = Lo.getValue(1); 1346 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps, 3); 1347 1348 // Legalized the flag result - switch anything that used the old flag to 1349 // use the new one. 1350 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1351} 1352 1353void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 1354 SDValue &Lo, SDValue &Hi) { 1355 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1356 DebugLoc dl = N->getDebugLoc(); 1357 SDValue Op = N->getOperand(0); 1358 if (Op.getValueType().bitsLE(NVT)) { 1359 // The low part is any extension of the input (which degenerates to a copy). 1360 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 1361 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 1362 } else { 1363 // For example, extension of an i48 to an i64. The operand type necessarily 1364 // promotes to the result type, so will end up being expanded too. 1365 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1366 "Only know how to promote this result!"); 1367 SDValue Res = GetPromotedInteger(Op); 1368 assert(Res.getValueType() == N->getValueType(0) && 1369 "Operand over promoted?"); 1370 // Split the promoted operand. This will simplify when it is expanded. 1371 SplitInteger(Res, Lo, Hi); 1372 } 1373} 1374 1375void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 1376 SDValue &Lo, SDValue &Hi) { 1377 DebugLoc dl = N->getDebugLoc(); 1378 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1379 EVT NVT = Lo.getValueType(); 1380 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1381 unsigned NVTBits = NVT.getSizeInBits(); 1382 unsigned EVTBits = EVT.getSizeInBits(); 1383 1384 if (NVTBits < EVTBits) { 1385 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 1386 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), EVTBits - NVTBits))); 1387 } else { 1388 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 1389 // The high part replicates the sign bit of Lo, make it explicit. 1390 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1391 DAG.getConstant(NVTBits-1, TLI.getPointerTy())); 1392 } 1393} 1394 1395void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 1396 SDValue &Lo, SDValue &Hi) { 1397 DebugLoc dl = N->getDebugLoc(); 1398 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1399 EVT NVT = Lo.getValueType(); 1400 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1401 unsigned NVTBits = NVT.getSizeInBits(); 1402 unsigned EVTBits = EVT.getSizeInBits(); 1403 1404 if (NVTBits < EVTBits) { 1405 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 1406 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), EVTBits - NVTBits))); 1407 } else { 1408 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 1409 // The high part must be zero, make it explicit. 1410 Hi = DAG.getConstant(0, NVT); 1411 } 1412} 1413 1414void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1415 SDValue &Lo, SDValue &Hi) { 1416 DebugLoc dl = N->getDebugLoc(); 1417 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1418 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 1419 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 1420} 1421 1422void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 1423 SDValue &Lo, SDValue &Hi) { 1424 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1425 unsigned NBitWidth = NVT.getSizeInBits(); 1426 const APInt &Cst = cast<ConstantSDNode>(N)->getAPIntValue(); 1427 Lo = DAG.getConstant(APInt(Cst).trunc(NBitWidth), NVT); 1428 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT); 1429} 1430 1431void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1432 SDValue &Lo, SDValue &Hi) { 1433 DebugLoc dl = N->getDebugLoc(); 1434 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1435 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1436 EVT NVT = Lo.getValueType(); 1437 1438 SDValue HiNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Hi, 1439 DAG.getConstant(0, NVT), ISD::SETNE); 1440 1441 SDValue LoLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Lo); 1442 SDValue HiLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Hi); 1443 1444 Lo = DAG.getNode(ISD::SELECT, dl, NVT, HiNotZero, HiLZ, 1445 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 1446 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1447 Hi = DAG.getConstant(0, NVT); 1448} 1449 1450void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1451 SDValue &Lo, SDValue &Hi) { 1452 DebugLoc dl = N->getDebugLoc(); 1453 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1454 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1455 EVT NVT = Lo.getValueType(); 1456 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 1457 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 1458 Hi = DAG.getConstant(0, NVT); 1459} 1460 1461void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1462 SDValue &Lo, SDValue &Hi) { 1463 DebugLoc dl = N->getDebugLoc(); 1464 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1465 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1466 EVT NVT = Lo.getValueType(); 1467 1468 SDValue LoNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, 1469 DAG.getConstant(0, NVT), ISD::SETNE); 1470 1471 SDValue LoLZ = DAG.getNode(ISD::CTTZ, dl, NVT, Lo); 1472 SDValue HiLZ = DAG.getNode(ISD::CTTZ, dl, NVT, Hi); 1473 1474 Lo = DAG.getNode(ISD::SELECT, dl, NVT, LoNotZero, LoLZ, 1475 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 1476 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1477 Hi = DAG.getConstant(0, NVT); 1478} 1479 1480void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 1481 SDValue &Hi) { 1482 DebugLoc dl = N->getDebugLoc(); 1483 EVT VT = N->getValueType(0); 1484 SDValue Op = N->getOperand(0); 1485 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 1486 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 1487 SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*irrelevant*/, dl), Lo, Hi); 1488} 1489 1490void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 1491 SDValue &Hi) { 1492 DebugLoc dl = N->getDebugLoc(); 1493 EVT VT = N->getValueType(0); 1494 SDValue Op = N->getOperand(0); 1495 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 1496 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 1497 SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*irrelevant*/, dl), Lo, Hi); 1498} 1499 1500void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 1501 SDValue &Lo, SDValue &Hi) { 1502 if (ISD::isNormalLoad(N)) { 1503 ExpandRes_NormalLoad(N, Lo, Hi); 1504 return; 1505 } 1506 1507 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1508 1509 EVT VT = N->getValueType(0); 1510 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1511 SDValue Ch = N->getChain(); 1512 SDValue Ptr = N->getBasePtr(); 1513 ISD::LoadExtType ExtType = N->getExtensionType(); 1514 int SVOffset = N->getSrcValueOffset(); 1515 unsigned Alignment = N->getAlignment(); 1516 bool isVolatile = N->isVolatile(); 1517 bool isNonTemporal = N->isNonTemporal(); 1518 DebugLoc dl = N->getDebugLoc(); 1519 1520 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1521 1522 if (N->getMemoryVT().bitsLE(NVT)) { 1523 EVT MemVT = N->getMemoryVT(); 1524 1525 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, 1526 MemVT, isVolatile, isNonTemporal, Alignment); 1527 1528 // Remember the chain. 1529 Ch = Lo.getValue(1); 1530 1531 if (ExtType == ISD::SEXTLOAD) { 1532 // The high part is obtained by SRA'ing all but one of the bits of the 1533 // lo part. 1534 unsigned LoSize = Lo.getValueType().getSizeInBits(); 1535 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1536 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 1537 } else if (ExtType == ISD::ZEXTLOAD) { 1538 // The high part is just a zero. 1539 Hi = DAG.getConstant(0, NVT); 1540 } else { 1541 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 1542 // The high part is undefined. 1543 Hi = DAG.getUNDEF(NVT); 1544 } 1545 } else if (TLI.isLittleEndian()) { 1546 // Little-endian - low bits are at low addresses. 1547 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getSrcValue(), SVOffset, 1548 isVolatile, isNonTemporal, Alignment); 1549 1550 unsigned ExcessBits = 1551 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1552 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 1553 1554 // Increment the pointer to the other half. 1555 unsigned IncrementSize = NVT.getSizeInBits()/8; 1556 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1557 DAG.getIntPtrConstant(IncrementSize)); 1558 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getSrcValue(), 1559 SVOffset+IncrementSize, NEVT, 1560 isVolatile, isNonTemporal, 1561 MinAlign(Alignment, IncrementSize)); 1562 1563 // Build a factor node to remember that this load is independent of the 1564 // other one. 1565 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1566 Hi.getValue(1)); 1567 } else { 1568 // Big-endian - high bits are at low addresses. Favor aligned loads at 1569 // the cost of some bit-fiddling. 1570 EVT MemVT = N->getMemoryVT(); 1571 unsigned EBytes = MemVT.getStoreSize(); 1572 unsigned IncrementSize = NVT.getSizeInBits()/8; 1573 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1574 1575 // Load both the high bits and maybe some of the low bits. 1576 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, 1577 EVT::getIntegerVT(*DAG.getContext(), 1578 MemVT.getSizeInBits() - ExcessBits), 1579 isVolatile, isNonTemporal, Alignment); 1580 1581 // Increment the pointer to the other half. 1582 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1583 DAG.getIntPtrConstant(IncrementSize)); 1584 // Load the rest of the low bits. 1585 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, N->getSrcValue(), 1586 SVOffset+IncrementSize, 1587 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 1588 isVolatile, isNonTemporal, 1589 MinAlign(Alignment, IncrementSize)); 1590 1591 // Build a factor node to remember that this load is independent of the 1592 // other one. 1593 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1594 Hi.getValue(1)); 1595 1596 if (ExcessBits < NVT.getSizeInBits()) { 1597 // Transfer low bits from the bottom of Hi to the top of Lo. 1598 Lo = DAG.getNode(ISD::OR, dl, NVT, Lo, 1599 DAG.getNode(ISD::SHL, dl, NVT, Hi, 1600 DAG.getConstant(ExcessBits, 1601 TLI.getPointerTy()))); 1602 // Move high bits to the right position in Hi. 1603 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, 1604 NVT, Hi, 1605 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 1606 TLI.getPointerTy())); 1607 } 1608 } 1609 1610 // Legalized the chain result - switch anything that used the old chain to 1611 // use the new one. 1612 ReplaceValueWith(SDValue(N, 1), Ch); 1613} 1614 1615void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 1616 SDValue &Lo, SDValue &Hi) { 1617 DebugLoc dl = N->getDebugLoc(); 1618 SDValue LL, LH, RL, RH; 1619 GetExpandedInteger(N->getOperand(0), LL, LH); 1620 GetExpandedInteger(N->getOperand(1), RL, RH); 1621 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 1622 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 1623} 1624 1625void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 1626 SDValue &Lo, SDValue &Hi) { 1627 EVT VT = N->getValueType(0); 1628 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1629 DebugLoc dl = N->getDebugLoc(); 1630 1631 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT); 1632 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT); 1633 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT); 1634 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT); 1635 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 1636 SDValue LL, LH, RL, RH; 1637 GetExpandedInteger(N->getOperand(0), LL, LH); 1638 GetExpandedInteger(N->getOperand(1), RL, RH); 1639 unsigned OuterBitSize = VT.getSizeInBits(); 1640 unsigned InnerBitSize = NVT.getSizeInBits(); 1641 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0)); 1642 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1)); 1643 1644 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 1645 if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) && 1646 DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) { 1647 // The inputs are both zero-extended. 1648 if (HasUMUL_LOHI) { 1649 // We can emit a umul_lohi. 1650 Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 1651 Hi = SDValue(Lo.getNode(), 1); 1652 return; 1653 } 1654 if (HasMULHU) { 1655 // We can emit a mulhu+mul. 1656 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1657 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 1658 return; 1659 } 1660 } 1661 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 1662 // The input values are both sign-extended. 1663 if (HasSMUL_LOHI) { 1664 // We can emit a smul_lohi. 1665 Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 1666 Hi = SDValue(Lo.getNode(), 1); 1667 return; 1668 } 1669 if (HasMULHS) { 1670 // We can emit a mulhs+mul. 1671 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1672 Hi = DAG.getNode(ISD::MULHS, dl, NVT, LL, RL); 1673 return; 1674 } 1675 } 1676 if (HasUMUL_LOHI) { 1677 // Lo,Hi = umul LHS, RHS. 1678 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl, 1679 DAG.getVTList(NVT, NVT), LL, RL); 1680 Lo = UMulLOHI; 1681 Hi = UMulLOHI.getValue(1); 1682 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 1683 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 1684 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 1685 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 1686 return; 1687 } 1688 if (HasMULHU) { 1689 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1690 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 1691 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 1692 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 1693 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 1694 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 1695 return; 1696 } 1697 } 1698 1699 // If nothing else, we can make a libcall. 1700 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1701 if (VT == MVT::i16) 1702 LC = RTLIB::MUL_I16; 1703 else if (VT == MVT::i32) 1704 LC = RTLIB::MUL_I32; 1705 else if (VT == MVT::i64) 1706 LC = RTLIB::MUL_I64; 1707 else if (VT == MVT::i128) 1708 LC = RTLIB::MUL_I128; 1709 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!"); 1710 1711 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1712 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*irrelevant*/, dl), Lo, Hi); 1713} 1714 1715void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 1716 SDValue &Lo, SDValue &Hi) { 1717 EVT VT = N->getValueType(0); 1718 DebugLoc dl = N->getDebugLoc(); 1719 1720 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1721 if (VT == MVT::i16) 1722 LC = RTLIB::SDIV_I16; 1723 else if (VT == MVT::i32) 1724 LC = RTLIB::SDIV_I32; 1725 else if (VT == MVT::i64) 1726 LC = RTLIB::SDIV_I64; 1727 else if (VT == MVT::i128) 1728 LC = RTLIB::SDIV_I128; 1729 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 1730 1731 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1732 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); 1733} 1734 1735void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 1736 SDValue &Lo, SDValue &Hi) { 1737 EVT VT = N->getValueType(0); 1738 DebugLoc dl = N->getDebugLoc(); 1739 1740 // If we can emit an efficient shift operation, do so now. Check to see if 1741 // the RHS is a constant. 1742 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 1743 return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi); 1744 1745 // If we can determine that the high bit of the shift is zero or one, even if 1746 // the low bits are variable, emit this shift in an optimized form. 1747 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 1748 return; 1749 1750 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 1751 unsigned PartsOpc; 1752 if (N->getOpcode() == ISD::SHL) { 1753 PartsOpc = ISD::SHL_PARTS; 1754 } else if (N->getOpcode() == ISD::SRL) { 1755 PartsOpc = ISD::SRL_PARTS; 1756 } else { 1757 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1758 PartsOpc = ISD::SRA_PARTS; 1759 } 1760 1761 // Next check to see if the target supports this SHL_PARTS operation or if it 1762 // will custom expand it. 1763 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1764 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 1765 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 1766 Action == TargetLowering::Custom) { 1767 // Expand the subcomponents. 1768 SDValue LHSL, LHSH; 1769 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1770 1771 SDValue Ops[] = { LHSL, LHSH, N->getOperand(1) }; 1772 EVT VT = LHSL.getValueType(); 1773 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops, 3); 1774 Hi = Lo.getValue(1); 1775 return; 1776 } 1777 1778 // Otherwise, emit a libcall. 1779 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1780 bool isSigned; 1781 if (N->getOpcode() == ISD::SHL) { 1782 isSigned = false; /*sign irrelevant*/ 1783 if (VT == MVT::i16) 1784 LC = RTLIB::SHL_I16; 1785 else if (VT == MVT::i32) 1786 LC = RTLIB::SHL_I32; 1787 else if (VT == MVT::i64) 1788 LC = RTLIB::SHL_I64; 1789 else if (VT == MVT::i128) 1790 LC = RTLIB::SHL_I128; 1791 } else if (N->getOpcode() == ISD::SRL) { 1792 isSigned = false; 1793 if (VT == MVT::i16) 1794 LC = RTLIB::SRL_I16; 1795 else if (VT == MVT::i32) 1796 LC = RTLIB::SRL_I32; 1797 else if (VT == MVT::i64) 1798 LC = RTLIB::SRL_I64; 1799 else if (VT == MVT::i128) 1800 LC = RTLIB::SRL_I128; 1801 } else { 1802 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1803 isSigned = true; 1804 if (VT == MVT::i16) 1805 LC = RTLIB::SRA_I16; 1806 else if (VT == MVT::i32) 1807 LC = RTLIB::SRA_I32; 1808 else if (VT == MVT::i64) 1809 LC = RTLIB::SRA_I64; 1810 else if (VT == MVT::i128) 1811 LC = RTLIB::SRA_I128; 1812 } 1813 1814 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 1815 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1816 SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned, dl), Lo, Hi); 1817 return; 1818 } 1819 1820 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 1821 llvm_unreachable("Unsupported shift!"); 1822} 1823 1824void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 1825 SDValue &Lo, SDValue &Hi) { 1826 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1827 DebugLoc dl = N->getDebugLoc(); 1828 SDValue Op = N->getOperand(0); 1829 if (Op.getValueType().bitsLE(NVT)) { 1830 // The low part is sign extension of the input (degenerates to a copy). 1831 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 1832 // The high part is obtained by SRA'ing all but one of the bits of low part. 1833 unsigned LoSize = NVT.getSizeInBits(); 1834 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1835 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 1836 } else { 1837 // For example, extension of an i48 to an i64. The operand type necessarily 1838 // promotes to the result type, so will end up being expanded too. 1839 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1840 "Only know how to promote this result!"); 1841 SDValue Res = GetPromotedInteger(Op); 1842 assert(Res.getValueType() == N->getValueType(0) && 1843 "Operand over promoted?"); 1844 // Split the promoted operand. This will simplify when it is expanded. 1845 SplitInteger(Res, Lo, Hi); 1846 unsigned ExcessBits = 1847 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 1848 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 1849 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), ExcessBits))); 1850 } 1851} 1852 1853void DAGTypeLegalizer:: 1854ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 1855 DebugLoc dl = N->getDebugLoc(); 1856 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1857 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1858 1859 if (EVT.bitsLE(Lo.getValueType())) { 1860 // sext_inreg the low part if needed. 1861 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 1862 N->getOperand(1)); 1863 1864 // The high part gets the sign extension from the lo-part. This handles 1865 // things like sextinreg V:i64 from i8. 1866 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 1867 DAG.getConstant(Hi.getValueType().getSizeInBits()-1, 1868 TLI.getPointerTy())); 1869 } else { 1870 // For example, extension of an i48 to an i64. Leave the low part alone, 1871 // sext_inreg the high part. 1872 unsigned ExcessBits = 1873 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 1874 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 1875 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), ExcessBits))); 1876 } 1877} 1878 1879void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 1880 SDValue &Lo, SDValue &Hi) { 1881 EVT VT = N->getValueType(0); 1882 DebugLoc dl = N->getDebugLoc(); 1883 1884 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1885 if (VT == MVT::i16) 1886 LC = RTLIB::SREM_I16; 1887 else if (VT == MVT::i32) 1888 LC = RTLIB::SREM_I32; 1889 else if (VT == MVT::i64) 1890 LC = RTLIB::SREM_I64; 1891 else if (VT == MVT::i128) 1892 LC = RTLIB::SREM_I128; 1893 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 1894 1895 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1896 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); 1897} 1898 1899void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 1900 SDValue &Lo, SDValue &Hi) { 1901 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1902 DebugLoc dl = N->getDebugLoc(); 1903 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 1904 Hi = DAG.getNode(ISD::SRL, dl, 1905 N->getOperand(0).getValueType(), N->getOperand(0), 1906 DAG.getConstant(NVT.getSizeInBits(), TLI.getPointerTy())); 1907 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 1908} 1909 1910void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 1911 SDValue &Lo, SDValue &Hi) { 1912 EVT VT = N->getValueType(0); 1913 DebugLoc dl = N->getDebugLoc(); 1914 1915 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1916 if (VT == MVT::i16) 1917 LC = RTLIB::UDIV_I16; 1918 else if (VT == MVT::i32) 1919 LC = RTLIB::UDIV_I32; 1920 else if (VT == MVT::i64) 1921 LC = RTLIB::UDIV_I64; 1922 else if (VT == MVT::i128) 1923 LC = RTLIB::UDIV_I128; 1924 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 1925 1926 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1927 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); 1928} 1929 1930void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 1931 SDValue &Lo, SDValue &Hi) { 1932 EVT VT = N->getValueType(0); 1933 DebugLoc dl = N->getDebugLoc(); 1934 1935 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1936 if (VT == MVT::i16) 1937 LC = RTLIB::UREM_I16; 1938 else if (VT == MVT::i32) 1939 LC = RTLIB::UREM_I32; 1940 else if (VT == MVT::i64) 1941 LC = RTLIB::UREM_I64; 1942 else if (VT == MVT::i128) 1943 LC = RTLIB::UREM_I128; 1944 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 1945 1946 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1947 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); 1948} 1949 1950void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 1951 SDValue &Lo, SDValue &Hi) { 1952 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1953 DebugLoc dl = N->getDebugLoc(); 1954 SDValue Op = N->getOperand(0); 1955 if (Op.getValueType().bitsLE(NVT)) { 1956 // The low part is zero extension of the input (degenerates to a copy). 1957 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 1958 Hi = DAG.getConstant(0, NVT); // The high part is just a zero. 1959 } else { 1960 // For example, extension of an i48 to an i64. The operand type necessarily 1961 // promotes to the result type, so will end up being expanded too. 1962 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1963 "Only know how to promote this result!"); 1964 SDValue Res = GetPromotedInteger(Op); 1965 assert(Res.getValueType() == N->getValueType(0) && 1966 "Operand over promoted?"); 1967 // Split the promoted operand. This will simplify when it is expanded. 1968 SplitInteger(Res, Lo, Hi); 1969 unsigned ExcessBits = 1970 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 1971 Hi = DAG.getZeroExtendInReg(Hi, dl, EVT::getIntegerVT(*DAG.getContext(), ExcessBits)); 1972 } 1973} 1974 1975 1976//===----------------------------------------------------------------------===// 1977// Integer Operand Expansion 1978//===----------------------------------------------------------------------===// 1979 1980/// ExpandIntegerOperand - This method is called when the specified operand of 1981/// the specified node is found to need expansion. At this point, all of the 1982/// result types of the node are known to be legal, but other operands of the 1983/// node may need promotion or expansion as well as the specified one. 1984bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 1985 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); 1986 SDValue Res = SDValue(); 1987 1988 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1989 return false; 1990 1991 switch (N->getOpcode()) { 1992 default: 1993 #ifndef NDEBUG 1994 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 1995 N->dump(&DAG); dbgs() << "\n"; 1996 #endif 1997 llvm_unreachable("Do not know how to expand this operator's operand!"); 1998 1999 case ISD::BIT_CONVERT: Res = ExpandOp_BIT_CONVERT(N); break; 2000 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 2001 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2002 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2003 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 2004 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 2005 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 2006 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 2007 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 2008 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 2009 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 2010 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 2011 2012 case ISD::SHL: 2013 case ISD::SRA: 2014 case ISD::SRL: 2015 case ISD::ROTL: 2016 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 2017 case ISD::RETURNADDR: 2018 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 2019 } 2020 2021 // If the result is null, the sub-method took care of registering results etc. 2022 if (!Res.getNode()) return false; 2023 2024 // If the result is N, the sub-method updated N in place. Tell the legalizer 2025 // core about this. 2026 if (Res.getNode() == N) 2027 return true; 2028 2029 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2030 "Invalid operand expansion"); 2031 2032 ReplaceValueWith(SDValue(N, 0), Res); 2033 return false; 2034} 2035 2036/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 2037/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2038void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 2039 SDValue &NewRHS, 2040 ISD::CondCode &CCCode, 2041 DebugLoc dl) { 2042 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2043 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 2044 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 2045 2046 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 2047 if (RHSLo == RHSHi) { 2048 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 2049 if (RHSCST->isAllOnesValue()) { 2050 // Equality comparison to -1. 2051 NewLHS = DAG.getNode(ISD::AND, dl, 2052 LHSLo.getValueType(), LHSLo, LHSHi); 2053 NewRHS = RHSLo; 2054 return; 2055 } 2056 } 2057 } 2058 2059 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 2060 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 2061 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 2062 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2063 return; 2064 } 2065 2066 // If this is a comparison of the sign bit, just look at the top part. 2067 // X > -1, x < 0 2068 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 2069 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 2070 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 2071 NewLHS = LHSHi; 2072 NewRHS = RHSHi; 2073 return; 2074 } 2075 2076 // FIXME: This generated code sucks. 2077 ISD::CondCode LowCC; 2078 switch (CCCode) { 2079 default: llvm_unreachable("Unknown integer setcc!"); 2080 case ISD::SETLT: 2081 case ISD::SETULT: LowCC = ISD::SETULT; break; 2082 case ISD::SETGT: 2083 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 2084 case ISD::SETLE: 2085 case ISD::SETULE: LowCC = ISD::SETULE; break; 2086 case ISD::SETGE: 2087 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 2088 } 2089 2090 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 2091 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 2092 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 2093 2094 // NOTE: on targets without efficient SELECT of bools, we can always use 2095 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 2096 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, true, NULL); 2097 SDValue Tmp1, Tmp2; 2098 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), 2099 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); 2100 if (!Tmp1.getNode()) 2101 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), 2102 LHSLo, RHSLo, LowCC); 2103 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 2104 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); 2105 if (!Tmp2.getNode()) 2106 Tmp2 = DAG.getNode(ISD::SETCC, dl, 2107 TLI.getSetCCResultType(LHSHi.getValueType()), 2108 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 2109 2110 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 2111 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 2112 if ((Tmp1C && Tmp1C->isNullValue()) || 2113 (Tmp2C && Tmp2C->isNullValue() && 2114 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 2115 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 2116 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 2117 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 2118 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 2119 // low part is known false, returns high part. 2120 // For LE / GE, if high part is known false, ignore the low part. 2121 // For LT / GT, if high part is known true, ignore the low part. 2122 NewLHS = Tmp2; 2123 NewRHS = SDValue(); 2124 return; 2125 } 2126 2127 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 2128 LHSHi, RHSHi, ISD::SETEQ, false, 2129 DagCombineInfo, dl); 2130 if (!NewLHS.getNode()) 2131 NewLHS = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 2132 LHSHi, RHSHi, ISD::SETEQ); 2133 NewLHS = DAG.getNode(ISD::SELECT, dl, Tmp1.getValueType(), 2134 NewLHS, Tmp1, Tmp2); 2135 NewRHS = SDValue(); 2136} 2137 2138SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 2139 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 2140 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 2141 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2142 2143 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2144 // against zero to select between true and false values. 2145 if (NewRHS.getNode() == 0) { 2146 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2147 CCCode = ISD::SETNE; 2148 } 2149 2150 // Update N to have the operands specified. 2151 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 2152 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2153 N->getOperand(4)); 2154} 2155 2156SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 2157 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2158 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2159 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2160 2161 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2162 // against zero to select between true and false values. 2163 if (NewRHS.getNode() == 0) { 2164 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2165 CCCode = ISD::SETNE; 2166 } 2167 2168 // Update N to have the operands specified. 2169 return DAG.UpdateNodeOperands(SDValue(N, 0), NewLHS, NewRHS, 2170 N->getOperand(2), N->getOperand(3), 2171 DAG.getCondCode(CCCode)); 2172} 2173 2174SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 2175 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2176 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2177 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2178 2179 // If ExpandSetCCOperands returned a scalar, use it. 2180 if (NewRHS.getNode() == 0) { 2181 assert(NewLHS.getValueType() == N->getValueType(0) && 2182 "Unexpected setcc expansion!"); 2183 return NewLHS; 2184 } 2185 2186 // Otherwise, update N to have the operands specified. 2187 return DAG.UpdateNodeOperands(SDValue(N, 0), NewLHS, NewRHS, 2188 DAG.getCondCode(CCCode)); 2189} 2190 2191SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 2192 // The value being shifted is legal, but the shift amount is too big. 2193 // It follows that either the result of the shift is undefined, or the 2194 // upper half of the shift amount is zero. Just use the lower half. 2195 SDValue Lo, Hi; 2196 GetExpandedInteger(N->getOperand(1), Lo, Hi); 2197 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), Lo); 2198} 2199 2200SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 2201 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 2202 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 2203 // constant to valid type. 2204 SDValue Lo, Hi; 2205 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2206 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo); 2207} 2208 2209SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 2210 SDValue Op = N->getOperand(0); 2211 EVT DstVT = N->getValueType(0); 2212 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 2213 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2214 "Don't know how to expand this SINT_TO_FP!"); 2215 return MakeLibCall(LC, DstVT, &Op, 1, true, N->getDebugLoc()); 2216} 2217 2218SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 2219 if (ISD::isNormalStore(N)) 2220 return ExpandOp_NormalStore(N, OpNo); 2221 2222 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2223 assert(OpNo == 1 && "Can only expand the stored value so far"); 2224 2225 EVT VT = N->getOperand(1).getValueType(); 2226 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2227 SDValue Ch = N->getChain(); 2228 SDValue Ptr = N->getBasePtr(); 2229 int SVOffset = N->getSrcValueOffset(); 2230 unsigned Alignment = N->getAlignment(); 2231 bool isVolatile = N->isVolatile(); 2232 bool isNonTemporal = N->isNonTemporal(); 2233 DebugLoc dl = N->getDebugLoc(); 2234 SDValue Lo, Hi; 2235 2236 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2237 2238 if (N->getMemoryVT().bitsLE(NVT)) { 2239 GetExpandedInteger(N->getValue(), Lo, Hi); 2240 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 2241 N->getMemoryVT(), isVolatile, isNonTemporal, 2242 Alignment); 2243 } else if (TLI.isLittleEndian()) { 2244 // Little-endian - low bits are at low addresses. 2245 GetExpandedInteger(N->getValue(), Lo, Hi); 2246 2247 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 2248 isVolatile, isNonTemporal, Alignment); 2249 2250 unsigned ExcessBits = 2251 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2252 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2253 2254 // Increment the pointer to the other half. 2255 unsigned IncrementSize = NVT.getSizeInBits()/8; 2256 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2257 DAG.getIntPtrConstant(IncrementSize)); 2258 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getSrcValue(), 2259 SVOffset+IncrementSize, NEVT, 2260 isVolatile, isNonTemporal, 2261 MinAlign(Alignment, IncrementSize)); 2262 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2263 } else { 2264 // Big-endian - high bits are at low addresses. Favor aligned stores at 2265 // the cost of some bit-fiddling. 2266 GetExpandedInteger(N->getValue(), Lo, Hi); 2267 2268 EVT ExtVT = N->getMemoryVT(); 2269 unsigned EBytes = ExtVT.getStoreSize(); 2270 unsigned IncrementSize = NVT.getSizeInBits()/8; 2271 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2272 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), ExtVT.getSizeInBits() - ExcessBits); 2273 2274 if (ExcessBits < NVT.getSizeInBits()) { 2275 // Transfer high bits from the top of Lo to the bottom of Hi. 2276 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 2277 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 2278 TLI.getPointerTy())); 2279 Hi = DAG.getNode(ISD::OR, dl, NVT, Hi, 2280 DAG.getNode(ISD::SRL, dl, NVT, Lo, 2281 DAG.getConstant(ExcessBits, 2282 TLI.getPointerTy()))); 2283 } 2284 2285 // Store both the high bits and maybe some of the low bits. 2286 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getSrcValue(), 2287 SVOffset, HiVT, isVolatile, isNonTemporal, 2288 Alignment); 2289 2290 // Increment the pointer to the other half. 2291 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2292 DAG.getIntPtrConstant(IncrementSize)); 2293 // Store the lowest ExcessBits bits in the second half. 2294 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), 2295 SVOffset+IncrementSize, 2296 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2297 isVolatile, isNonTemporal, 2298 MinAlign(Alignment, IncrementSize)); 2299 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2300 } 2301} 2302 2303SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 2304 SDValue InL, InH; 2305 GetExpandedInteger(N->getOperand(0), InL, InH); 2306 // Just truncate the low part of the source. 2307 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), InL); 2308} 2309 2310SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 2311 SDValue Op = N->getOperand(0); 2312 EVT SrcVT = Op.getValueType(); 2313 EVT DstVT = N->getValueType(0); 2314 DebugLoc dl = N->getDebugLoc(); 2315 2316 if (TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 2317 // Do a signed conversion then adjust the result. 2318 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); 2319 SignedConv = TLI.LowerOperation(SignedConv, DAG); 2320 2321 // The result of the signed conversion needs adjusting if the 'sign bit' of 2322 // the incoming integer was set. To handle this, we dynamically test to see 2323 // if it is set, and, if so, add a fudge factor. 2324 2325 const uint64_t F32TwoE32 = 0x4F800000ULL; 2326 const uint64_t F32TwoE64 = 0x5F800000ULL; 2327 const uint64_t F32TwoE128 = 0x7F800000ULL; 2328 2329 APInt FF(32, 0); 2330 if (SrcVT == MVT::i32) 2331 FF = APInt(32, F32TwoE32); 2332 else if (SrcVT == MVT::i64) 2333 FF = APInt(32, F32TwoE64); 2334 else if (SrcVT == MVT::i128) 2335 FF = APInt(32, F32TwoE128); 2336 else 2337 assert(false && "Unsupported UINT_TO_FP!"); 2338 2339 // Check whether the sign bit is set. 2340 SDValue Lo, Hi; 2341 GetExpandedInteger(Op, Lo, Hi); 2342 SDValue SignSet = DAG.getSetCC(dl, 2343 TLI.getSetCCResultType(Hi.getValueType()), 2344 Hi, DAG.getConstant(0, Hi.getValueType()), 2345 ISD::SETLT); 2346 2347 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 2348 SDValue FudgePtr = DAG.getConstantPool( 2349 ConstantInt::get(*DAG.getContext(), FF.zext(64)), 2350 TLI.getPointerTy()); 2351 2352 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 2353 SDValue Zero = DAG.getIntPtrConstant(0); 2354 SDValue Four = DAG.getIntPtrConstant(4); 2355 if (TLI.isBigEndian()) std::swap(Zero, Four); 2356 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet, 2357 Zero, Four); 2358 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 2359 FudgePtr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), FudgePtr, Offset); 2360 Alignment = std::min(Alignment, 4u); 2361 2362 // Load the value out, extending it from f32 to the destination float type. 2363 // FIXME: Avoid the extend by constructing the right constant pool? 2364 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), 2365 FudgePtr, NULL, 0, MVT::f32, 2366 false, false, Alignment); 2367 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); 2368 } 2369 2370 // Otherwise, use a libcall. 2371 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 2372 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2373 "Don't know how to expand this UINT_TO_FP!"); 2374 return MakeLibCall(LC, DstVT, &Op, 1, true, dl); 2375} 2376