LegalizeIntegerTypes.cpp revision 5b82c2f8911e7e7bcaa563cf15b5d03cf0292379
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/CodeGen/PseudoSourceValue.h" 24#include "llvm/Support/ErrorHandling.h" 25#include "llvm/Support/raw_ostream.h" 26using namespace llvm; 27 28//===----------------------------------------------------------------------===// 29// Integer Result Promotion 30//===----------------------------------------------------------------------===// 31 32/// PromoteIntegerResult - This method is called when a result of a node is 33/// found to be in need of promotion to a larger type. At this point, the node 34/// may also have invalid operands or may have other results that need 35/// expansion, we just know that (at least) one result needs promotion. 36void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 37 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); 38 SDValue Res = SDValue(); 39 40 // See if the target wants to custom expand this node. 41 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 42 return; 43 44 switch (N->getOpcode()) { 45 default: 46#ifndef NDEBUG 47 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 48 N->dump(&DAG); dbgs() << "\n"; 49#endif 50 llvm_unreachable("Do not know how to promote this operator!"); 51 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 52 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 53 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 54 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 55 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 56 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 57 case ISD::CONVERT_RNDSAT: 58 Res = PromoteIntRes_CONVERT_RNDSAT(N); break; 59 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 60 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 61 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 62 case ISD::EXTRACT_VECTOR_ELT: 63 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 64 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break; 65 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 66 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 67 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 68 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 69 case ISD::SIGN_EXTEND_INREG: 70 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 71 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 72 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 73 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 74 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 75 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 76 77 case ISD::EXTRACT_SUBVECTOR: 78 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 79 case ISD::VECTOR_SHUFFLE: 80 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 81 case ISD::INSERT_VECTOR_ELT: 82 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 83 case ISD::BUILD_VECTOR: 84 Res = PromoteIntRes_BUILD_VECTOR(N); break; 85 case ISD::SCALAR_TO_VECTOR: 86 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 87 88 case ISD::SIGN_EXTEND: 89 case ISD::ZERO_EXTEND: 90 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 91 92 case ISD::FP_TO_SINT: 93 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 94 95 case ISD::FP32_TO_FP16:Res = PromoteIntRes_FP32_TO_FP16(N); break; 96 97 case ISD::AND: 98 case ISD::OR: 99 case ISD::XOR: 100 case ISD::ADD: 101 case ISD::SUB: 102 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 103 104 case ISD::SDIV: 105 case ISD::SREM: Res = PromoteIntRes_SDIV(N); break; 106 107 case ISD::UDIV: 108 case ISD::UREM: Res = PromoteIntRes_UDIV(N); break; 109 110 case ISD::SADDO: 111 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 112 case ISD::UADDO: 113 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 114 case ISD::SMULO: 115 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 116 117 case ISD::ATOMIC_LOAD_ADD: 118 case ISD::ATOMIC_LOAD_SUB: 119 case ISD::ATOMIC_LOAD_AND: 120 case ISD::ATOMIC_LOAD_OR: 121 case ISD::ATOMIC_LOAD_XOR: 122 case ISD::ATOMIC_LOAD_NAND: 123 case ISD::ATOMIC_LOAD_MIN: 124 case ISD::ATOMIC_LOAD_MAX: 125 case ISD::ATOMIC_LOAD_UMIN: 126 case ISD::ATOMIC_LOAD_UMAX: 127 case ISD::ATOMIC_SWAP: 128 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 129 130 case ISD::ATOMIC_CMP_SWAP: 131 Res = PromoteIntRes_Atomic2(cast<AtomicSDNode>(N)); break; 132 } 133 134 // If the result is null then the sub-method took care of registering it. 135 if (Res.getNode()) 136 SetPromotedInteger(SDValue(N, ResNo), Res); 137} 138 139SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 140 // Sign-extend the new bits, and continue the assertion. 141 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 142 return DAG.getNode(ISD::AssertSext, N->getDebugLoc(), 143 Op.getValueType(), Op, N->getOperand(1)); 144} 145 146SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 147 // Zero the new bits, and continue the assertion. 148 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 149 return DAG.getNode(ISD::AssertZext, N->getDebugLoc(), 150 Op.getValueType(), Op, N->getOperand(1)); 151} 152 153SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 154 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 155 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 156 N->getMemoryVT(), 157 N->getChain(), N->getBasePtr(), 158 Op2, N->getMemOperand()); 159 // Legalized the chain result - switch anything that used the old chain to 160 // use the new one. 161 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 162 return Res; 163} 164 165SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) { 166 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 167 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 168 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 169 N->getMemoryVT(), N->getChain(), N->getBasePtr(), 170 Op2, Op3, N->getMemOperand()); 171 // Legalized the chain result - switch anything that used the old chain to 172 // use the new one. 173 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 174 return Res; 175} 176 177SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 178 SDValue InOp = N->getOperand(0); 179 EVT InVT = InOp.getValueType(); 180 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 181 EVT OutVT = N->getValueType(0); 182 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 183 DebugLoc dl = N->getDebugLoc(); 184 185 switch (getTypeAction(InVT)) { 186 default: 187 assert(false && "Unknown type action!"); 188 break; 189 case TargetLowering::TypeLegal: 190 break; 191 case TargetLowering::TypePromoteInteger: 192 if (NOutVT.bitsEq(NInVT)) 193 // The input promotes to the same size. Convert the promoted value. 194 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 195 break; 196 case TargetLowering::TypeSoftenFloat: 197 // Promote the integer operand by hand. 198 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 199 case TargetLowering::TypeExpandInteger: 200 case TargetLowering::TypeExpandFloat: 201 break; 202 case TargetLowering::TypeScalarizeVector: 203 // Convert the element to an integer and promote it by hand. 204 if (!NOutVT.isVector()) 205 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 206 BitConvertToInteger(GetScalarizedVector(InOp))); 207 break; 208 case TargetLowering::TypeSplitVector: { 209 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 210 // pieces of the input into integers and reassemble in the final type. 211 SDValue Lo, Hi; 212 GetSplitVector(N->getOperand(0), Lo, Hi); 213 Lo = BitConvertToInteger(Lo); 214 Hi = BitConvertToInteger(Hi); 215 216 if (TLI.isBigEndian()) 217 std::swap(Lo, Hi); 218 219 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 220 EVT::getIntegerVT(*DAG.getContext(), 221 NOutVT.getSizeInBits()), 222 JoinIntegers(Lo, Hi)); 223 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 224 } 225 case TargetLowering::TypeWidenVector: 226 if (OutVT.bitsEq(NInVT)) 227 // The input is widened to the same size. Convert to the widened value. 228 return DAG.getNode(ISD::BITCAST, dl, OutVT, GetWidenedVector(InOp)); 229 } 230 231 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 232 CreateStackStoreLoad(InOp, OutVT)); 233} 234 235SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 236 SDValue Op = GetPromotedInteger(N->getOperand(0)); 237 EVT OVT = N->getValueType(0); 238 EVT NVT = Op.getValueType(); 239 DebugLoc dl = N->getDebugLoc(); 240 241 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 242 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 243 DAG.getConstant(DiffBits, TLI.getPointerTy())); 244} 245 246SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 247 // The pair element type may be legal, or may not promote to the same type as 248 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 249 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), 250 TLI.getTypeToTransformTo(*DAG.getContext(), 251 N->getValueType(0)), JoinIntegers(N->getOperand(0), 252 N->getOperand(1))); 253} 254 255SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 256 EVT VT = N->getValueType(0); 257 // FIXME there is no actual debug info here 258 DebugLoc dl = N->getDebugLoc(); 259 // Zero extend things like i1, sign extend everything else. It shouldn't 260 // matter in theory which one we pick, but this tends to give better code? 261 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 262 SDValue Result = DAG.getNode(Opc, dl, 263 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 264 SDValue(N, 0)); 265 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 266 return Result; 267} 268 269SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) { 270 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 271 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 272 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 273 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 274 "can only promote integers"); 275 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 276 return DAG.getConvertRndSat(OutVT, N->getDebugLoc(), N->getOperand(0), 277 N->getOperand(1), N->getOperand(2), 278 N->getOperand(3), N->getOperand(4), CvtCode); 279} 280 281SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 282 // Zero extend to the promoted type and do the count there. 283 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 284 DebugLoc dl = N->getDebugLoc(); 285 EVT OVT = N->getValueType(0); 286 EVT NVT = Op.getValueType(); 287 Op = DAG.getNode(ISD::CTLZ, dl, NVT, Op); 288 // Subtract off the extra leading bits in the bigger type. 289 return DAG.getNode(ISD::SUB, dl, NVT, Op, 290 DAG.getConstant(NVT.getSizeInBits() - 291 OVT.getSizeInBits(), NVT)); 292} 293 294SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 295 // Zero extend to the promoted type and do the count there. 296 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 297 return DAG.getNode(ISD::CTPOP, N->getDebugLoc(), Op.getValueType(), Op); 298} 299 300SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 301 SDValue Op = GetPromotedInteger(N->getOperand(0)); 302 EVT OVT = N->getValueType(0); 303 EVT NVT = Op.getValueType(); 304 DebugLoc dl = N->getDebugLoc(); 305 // The count is the same in the promoted type except if the original 306 // value was zero. This can be handled by setting the bit just off 307 // the top of the original type. 308 APInt TopBit(NVT.getSizeInBits(), 0); 309 TopBit.setBit(OVT.getSizeInBits()); 310 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT)); 311 return DAG.getNode(ISD::CTTZ, dl, NVT, Op); 312} 313 314SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 315 DebugLoc dl = N->getDebugLoc(); 316 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 317 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), 318 N->getOperand(1)); 319} 320 321SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 322 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 323 unsigned NewOpc = N->getOpcode(); 324 DebugLoc dl = N->getDebugLoc(); 325 326 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 327 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 328 // and SINT conversions are Custom, there is no way to tell which is 329 // preferable. We choose SINT because that's the right thing on PPC.) 330 if (N->getOpcode() == ISD::FP_TO_UINT && 331 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 332 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 333 NewOpc = ISD::FP_TO_SINT; 334 335 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 336 337 // Assert that the converted value fits in the original type. If it doesn't 338 // (eg: because the value being converted is too big), then the result of the 339 // original operation was undefined anyway, so the assert is still correct. 340 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 341 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 342 DAG.getValueType(N->getValueType(0).getScalarType())); 343} 344 345SDValue DAGTypeLegalizer::PromoteIntRes_FP32_TO_FP16(SDNode *N) { 346 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 347 DebugLoc dl = N->getDebugLoc(); 348 349 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 350 351 return DAG.getNode(ISD::AssertZext, dl, 352 NVT, Res, DAG.getValueType(N->getValueType(0))); 353} 354 355SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 356 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 357 DebugLoc dl = N->getDebugLoc(); 358 359 if (getTypeAction(N->getOperand(0).getValueType()) 360 == TargetLowering::TypePromoteInteger) { 361 SDValue Res = GetPromotedInteger(N->getOperand(0)); 362 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 363 364 // If the result and operand types are the same after promotion, simplify 365 // to an in-register extension. 366 if (NVT == Res.getValueType()) { 367 // The high bits are not guaranteed to be anything. Insert an extend. 368 if (N->getOpcode() == ISD::SIGN_EXTEND) 369 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 370 DAG.getValueType(N->getOperand(0).getValueType())); 371 if (N->getOpcode() == ISD::ZERO_EXTEND) 372 return DAG.getZeroExtendInReg(Res, dl, 373 N->getOperand(0).getValueType().getScalarType()); 374 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 375 return Res; 376 } 377 } 378 379 // Otherwise, just extend the original operand all the way to the larger type. 380 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 381} 382 383SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 384 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 385 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 386 ISD::LoadExtType ExtType = 387 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 388 DebugLoc dl = N->getDebugLoc(); 389 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 390 N->getPointerInfo(), 391 N->getMemoryVT(), N->isVolatile(), 392 N->isNonTemporal(), N->getAlignment()); 393 394 // Legalized the chain result - switch anything that used the old chain to 395 // use the new one. 396 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 397 return Res; 398} 399 400/// Promote the overflow flag of an overflowing arithmetic node. 401SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 402 // Simply change the return type of the boolean result. 403 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 404 EVT ValueVTs[] = { N->getValueType(0), NVT }; 405 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) }; 406 SDValue Res = DAG.getNode(N->getOpcode(), N->getDebugLoc(), 407 DAG.getVTList(ValueVTs, 2), Ops, 2); 408 409 // Modified the sum result - switch anything that used the old sum to use 410 // the new one. 411 ReplaceValueWith(SDValue(N, 0), Res); 412 413 return SDValue(Res.getNode(), 1); 414} 415 416SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 417 if (ResNo == 1) 418 return PromoteIntRes_Overflow(N); 419 420 // The operation overflowed iff the result in the larger type is not the 421 // sign extension of its truncation to the original type. 422 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 423 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 424 EVT OVT = N->getOperand(0).getValueType(); 425 EVT NVT = LHS.getValueType(); 426 DebugLoc dl = N->getDebugLoc(); 427 428 // Do the arithmetic in the larger type. 429 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 430 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 431 432 // Calculate the overflow flag: sign extend the arithmetic result from 433 // the original type. 434 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 435 DAG.getValueType(OVT)); 436 // Overflowed if and only if this is not equal to Res. 437 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 438 439 // Use the calculated overflow everywhere. 440 ReplaceValueWith(SDValue(N, 1), Ofl); 441 442 return Res; 443} 444 445SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) { 446 // Sign extend the input. 447 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 448 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 449 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 450 LHS.getValueType(), LHS, RHS); 451} 452 453SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 454 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 455 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 456 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 457 LHS.getValueType(), N->getOperand(0),LHS,RHS); 458} 459 460SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 461 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 462 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 463 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 464 LHS.getValueType(), N->getOperand(0), 465 N->getOperand(1), LHS, RHS, N->getOperand(4)); 466} 467 468SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 469 EVT SVT = TLI.getSetCCResultType(N->getOperand(0).getValueType()); 470 assert(isTypeLegal(SVT) && "Illegal SetCC type!"); 471 DebugLoc dl = N->getDebugLoc(); 472 473 // Get the SETCC result using the canonical SETCC type. 474 SDValue SetCC = DAG.getNode(ISD::SETCC, dl, SVT, N->getOperand(0), 475 N->getOperand(1), N->getOperand(2)); 476 477 // Convert to the expected type. 478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 479 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?"); 480 return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC); 481} 482 483SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 484 return DAG.getNode(ISD::SHL, N->getDebugLoc(), 485 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), 486 GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); 487} 488 489SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 490 SDValue Op = GetPromotedInteger(N->getOperand(0)); 491 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), 492 Op.getValueType(), Op, N->getOperand(1)); 493} 494 495SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 496 // The input may have strange things in the top bits of the registers, but 497 // these operations don't care. They may have weird bits going out, but 498 // that too is okay if they are integer operations. 499 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 500 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 501 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 502 LHS.getValueType(), LHS, RHS); 503} 504 505SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 506 // The input value must be properly sign extended. 507 SDValue Res = SExtPromotedInteger(N->getOperand(0)); 508 return DAG.getNode(ISD::SRA, N->getDebugLoc(), 509 Res.getValueType(), Res, N->getOperand(1)); 510} 511 512SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 513 // The input value must be properly zero extended. 514 EVT VT = N->getValueType(0); 515 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 516 SDValue Res = ZExtPromotedInteger(N->getOperand(0)); 517 return DAG.getNode(ISD::SRL, N->getDebugLoc(), NVT, Res, N->getOperand(1)); 518} 519 520SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 521 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 522 SDValue Res; 523 524 switch (getTypeAction(N->getOperand(0).getValueType())) { 525 default: llvm_unreachable("Unknown type action!"); 526 case TargetLowering::TypeLegal: 527 case TargetLowering::TypeExpandInteger: 528 Res = N->getOperand(0); 529 break; 530 case TargetLowering::TypePromoteInteger: 531 Res = GetPromotedInteger(N->getOperand(0)); 532 break; 533 } 534 535 // Truncate to NVT instead of VT 536 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), NVT, Res); 537} 538 539SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 540 if (ResNo == 1) 541 return PromoteIntRes_Overflow(N); 542 543 // The operation overflowed iff the result in the larger type is not the 544 // zero extension of its truncation to the original type. 545 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 546 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 547 EVT OVT = N->getOperand(0).getValueType(); 548 EVT NVT = LHS.getValueType(); 549 DebugLoc dl = N->getDebugLoc(); 550 551 // Do the arithmetic in the larger type. 552 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 553 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 554 555 // Calculate the overflow flag: zero extend the arithmetic result from 556 // the original type. 557 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 558 // Overflowed if and only if this is not equal to Res. 559 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 560 561 // Use the calculated overflow everywhere. 562 ReplaceValueWith(SDValue(N, 1), Ofl); 563 564 return Res; 565} 566 567SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 568 // Promote the overflow bit trivially. 569 if (ResNo == 1) 570 return PromoteIntRes_Overflow(N); 571 572 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 573 DebugLoc DL = N->getDebugLoc(); 574 EVT SmallVT = LHS.getValueType(); 575 576 // To determine if the result overflowed in a larger type, we extend the 577 // input to the larger type, do the multiply, then check the high bits of 578 // the result to see if the overflow happened. 579 if (N->getOpcode() == ISD::SMULO) { 580 LHS = SExtPromotedInteger(LHS); 581 RHS = SExtPromotedInteger(RHS); 582 } else { 583 LHS = ZExtPromotedInteger(LHS); 584 RHS = ZExtPromotedInteger(RHS); 585 } 586 SDValue Mul = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS); 587 588 // Overflow occurred iff the high part of the result does not 589 // zero/sign-extend the low part. 590 SDValue Overflow; 591 if (N->getOpcode() == ISD::UMULO) { 592 // Unsigned overflow occurred iff the high part is non-zero. 593 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 594 DAG.getIntPtrConstant(SmallVT.getSizeInBits())); 595 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 596 DAG.getConstant(0, Hi.getValueType()), ISD::SETNE); 597 } else { 598 // Signed overflow occurred iff the high part does not sign extend the low. 599 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 600 Mul, DAG.getValueType(SmallVT)); 601 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 602 } 603 604 // Use the calculated overflow everywhere. 605 ReplaceValueWith(SDValue(N, 1), Overflow); 606 return Mul; 607} 608 609SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { 610 // Zero extend the input. 611 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 612 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 613 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 614 LHS.getValueType(), LHS, RHS); 615} 616 617SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 618 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 619 N->getValueType(0))); 620} 621 622SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 623 SDValue Chain = N->getOperand(0); // Get the chain. 624 SDValue Ptr = N->getOperand(1); // Get the pointer. 625 EVT VT = N->getValueType(0); 626 DebugLoc dl = N->getDebugLoc(); 627 628 EVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 629 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 630 // The argument is passed as NumRegs registers of type RegVT. 631 632 SmallVector<SDValue, 8> Parts(NumRegs); 633 for (unsigned i = 0; i < NumRegs; ++i) { 634 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 635 N->getConstantOperandVal(3)); 636 Chain = Parts[i].getValue(1); 637 } 638 639 // Handle endianness of the load. 640 if (TLI.isBigEndian()) 641 std::reverse(Parts.begin(), Parts.end()); 642 643 // Assemble the parts in the promoted type. 644 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 645 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 646 for (unsigned i = 1; i < NumRegs; ++i) { 647 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 648 // Shift it to the right position and "or" it in. 649 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 650 DAG.getConstant(i * RegVT.getSizeInBits(), 651 TLI.getPointerTy())); 652 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 653 } 654 655 // Modified the chain result - switch anything that used the old chain to 656 // use the new one. 657 ReplaceValueWith(SDValue(N, 1), Chain); 658 659 return Res; 660} 661 662//===----------------------------------------------------------------------===// 663// Integer Operand Promotion 664//===----------------------------------------------------------------------===// 665 666/// PromoteIntegerOperand - This method is called when the specified operand of 667/// the specified node is found to need promotion. At this point, all of the 668/// result types of the node are known to be legal, but other operands of the 669/// node may need promotion or expansion as well as the specified one. 670bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 671 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); 672 SDValue Res = SDValue(); 673 674 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 675 return false; 676 677 switch (N->getOpcode()) { 678 default: 679 #ifndef NDEBUG 680 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 681 N->dump(&DAG); dbgs() << "\n"; 682 #endif 683 llvm_unreachable("Do not know how to promote this operator's operand!"); 684 685 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 686 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 687 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 688 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 689 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 690 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 691 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 692 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 693 case ISD::CONVERT_RNDSAT: 694 Res = PromoteIntOp_CONVERT_RNDSAT(N); break; 695 case ISD::INSERT_VECTOR_ELT: 696 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 697 case ISD::MEMBARRIER: Res = PromoteIntOp_MEMBARRIER(N); break; 698 case ISD::SCALAR_TO_VECTOR: 699 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 700 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 701 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 702 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 703 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 704 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 705 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 706 OpNo); break; 707 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 708 case ISD::FP16_TO_FP32: 709 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 710 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 711 712 case ISD::SHL: 713 case ISD::SRA: 714 case ISD::SRL: 715 case ISD::ROTL: 716 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 717 } 718 719 // If the result is null, the sub-method took care of registering results etc. 720 if (!Res.getNode()) return false; 721 722 // If the result is N, the sub-method updated N in place. Tell the legalizer 723 // core about this. 724 if (Res.getNode() == N) 725 return true; 726 727 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 728 "Invalid operand expansion"); 729 730 ReplaceValueWith(SDValue(N, 0), Res); 731 return false; 732} 733 734/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 735/// shared among BR_CC, SELECT_CC, and SETCC handlers. 736void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 737 ISD::CondCode CCCode) { 738 // We have to insert explicit sign or zero extends. Note that we could 739 // insert sign extends for ALL conditions, but zero extend is cheaper on 740 // many machines (an AND instead of two shifts), so prefer it. 741 switch (CCCode) { 742 default: llvm_unreachable("Unknown integer comparison!"); 743 case ISD::SETEQ: 744 case ISD::SETNE: 745 case ISD::SETUGE: 746 case ISD::SETUGT: 747 case ISD::SETULE: 748 case ISD::SETULT: 749 // ALL of these operations will work if we either sign or zero extend 750 // the operands (including the unsigned comparisons!). Zero extend is 751 // usually a simpler/cheaper operation, so prefer it. 752 NewLHS = ZExtPromotedInteger(NewLHS); 753 NewRHS = ZExtPromotedInteger(NewRHS); 754 break; 755 case ISD::SETGE: 756 case ISD::SETGT: 757 case ISD::SETLT: 758 case ISD::SETLE: 759 NewLHS = SExtPromotedInteger(NewLHS); 760 NewRHS = SExtPromotedInteger(NewRHS); 761 break; 762 } 763} 764 765SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 766 SDValue Op = GetPromotedInteger(N->getOperand(0)); 767 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), Op); 768} 769 770SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 771 // This should only occur in unusual situations like bitcasting to an 772 // x86_fp80, so just turn it into a store+load 773 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 774} 775 776SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 777 assert(OpNo == 2 && "Don't know how to promote this operand!"); 778 779 SDValue LHS = N->getOperand(2); 780 SDValue RHS = N->getOperand(3); 781 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 782 783 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 784 // legal types. 785 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 786 N->getOperand(1), LHS, RHS, N->getOperand(4)), 787 0); 788} 789 790SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 791 assert(OpNo == 1 && "only know how to promote condition"); 792 793 // Promote all the way up to the canonical SetCC type. 794 EVT SVT = TLI.getSetCCResultType(MVT::Other); 795 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), SVT); 796 797 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 798 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 799 N->getOperand(2)), 0); 800} 801 802SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 803 // Since the result type is legal, the operands must promote to it. 804 EVT OVT = N->getOperand(0).getValueType(); 805 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 806 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 807 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 808 DebugLoc dl = N->getDebugLoc(); 809 810 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 811 DAG.getConstant(OVT.getSizeInBits(), TLI.getPointerTy())); 812 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 813} 814 815SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 816 // The vector type is legal but the element type is not. This implies 817 // that the vector is a power-of-two in length and that the element 818 // type does not have a strange size (eg: it is not i1). 819 EVT VecVT = N->getValueType(0); 820 unsigned NumElts = VecVT.getVectorNumElements(); 821 assert(!(NumElts & 1) && "Legal vector of one illegal element?"); 822 823 // Promote the inserted value. The type does not need to match the 824 // vector element type. Check that any extra bits introduced will be 825 // truncated away. 826 assert(N->getOperand(0).getValueType().getSizeInBits() >= 827 N->getValueType(0).getVectorElementType().getSizeInBits() && 828 "Type of inserted value narrower than vector element type!"); 829 830 SmallVector<SDValue, 16> NewOps; 831 for (unsigned i = 0; i < NumElts; ++i) 832 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 833 834 return SDValue(DAG.UpdateNodeOperands(N, &NewOps[0], NumElts), 0); 835} 836 837SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) { 838 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 839 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 840 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 841 CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) && 842 "can only promote integer arguments"); 843 SDValue InOp = GetPromotedInteger(N->getOperand(0)); 844 return DAG.getConvertRndSat(N->getValueType(0), N->getDebugLoc(), InOp, 845 N->getOperand(1), N->getOperand(2), 846 N->getOperand(3), N->getOperand(4), CvtCode); 847} 848 849SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 850 unsigned OpNo) { 851 if (OpNo == 1) { 852 // Promote the inserted value. This is valid because the type does not 853 // have to match the vector element type. 854 855 // Check that any extra bits introduced will be truncated away. 856 assert(N->getOperand(1).getValueType().getSizeInBits() >= 857 N->getValueType(0).getVectorElementType().getSizeInBits() && 858 "Type of inserted value narrower than vector element type!"); 859 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 860 GetPromotedInteger(N->getOperand(1)), 861 N->getOperand(2)), 862 0); 863 } 864 865 assert(OpNo == 2 && "Different operand and result vector types?"); 866 867 // Promote the index. 868 SDValue Idx = ZExtPromotedInteger(N->getOperand(2)); 869 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 870 N->getOperand(1), Idx), 0); 871} 872 873SDValue DAGTypeLegalizer::PromoteIntOp_MEMBARRIER(SDNode *N) { 874 SDValue NewOps[6]; 875 DebugLoc dl = N->getDebugLoc(); 876 NewOps[0] = N->getOperand(0); 877 for (unsigned i = 1; i < array_lengthof(NewOps); ++i) { 878 SDValue Flag = GetPromotedInteger(N->getOperand(i)); 879 NewOps[i] = DAG.getZeroExtendInReg(Flag, dl, MVT::i1); 880 } 881 return SDValue(DAG.UpdateNodeOperands(N, NewOps, array_lengthof(NewOps)), 0); 882} 883 884SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 885 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 886 // the operand in place. 887 return SDValue(DAG.UpdateNodeOperands(N, 888 GetPromotedInteger(N->getOperand(0))), 0); 889} 890 891SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 892 assert(OpNo == 0 && "Only know how to promote condition"); 893 894 // Promote all the way up to the canonical SetCC type. 895 EVT SVT = TLI.getSetCCResultType(N->getOperand(1).getValueType()); 896 SDValue Cond = PromoteTargetBoolean(N->getOperand(0), SVT); 897 898 return SDValue(DAG.UpdateNodeOperands(N, Cond, 899 N->getOperand(1), N->getOperand(2)), 0); 900} 901 902SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 903 assert(OpNo == 0 && "Don't know how to promote this operand!"); 904 905 SDValue LHS = N->getOperand(0); 906 SDValue RHS = N->getOperand(1); 907 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 908 909 // The CC (#4) and the possible return values (#2 and #3) have legal types. 910 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 911 N->getOperand(3), N->getOperand(4)), 0); 912} 913 914SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 915 assert(OpNo == 0 && "Don't know how to promote this operand!"); 916 917 SDValue LHS = N->getOperand(0); 918 SDValue RHS = N->getOperand(1); 919 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 920 921 // The CC (#2) is always legal. 922 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 923} 924 925SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 926 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 927 ZExtPromotedInteger(N->getOperand(1))), 0); 928} 929 930SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 931 SDValue Op = GetPromotedInteger(N->getOperand(0)); 932 DebugLoc dl = N->getDebugLoc(); 933 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 934 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 935 Op, DAG.getValueType(N->getOperand(0).getValueType())); 936} 937 938SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 939 return SDValue(DAG.UpdateNodeOperands(N, 940 SExtPromotedInteger(N->getOperand(0))), 0); 941} 942 943SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 944 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 945 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 946 unsigned Alignment = N->getAlignment(); 947 bool isVolatile = N->isVolatile(); 948 bool isNonTemporal = N->isNonTemporal(); 949 DebugLoc dl = N->getDebugLoc(); 950 951 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 952 953 // Truncate the value and store the result. 954 return DAG.getTruncStore(Ch, dl, Val, Ptr, N->getPointerInfo(), 955 N->getMemoryVT(), 956 isVolatile, isNonTemporal, Alignment); 957} 958 959SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 960 SDValue Op = GetPromotedInteger(N->getOperand(0)); 961 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), Op); 962} 963 964SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 965 return SDValue(DAG.UpdateNodeOperands(N, 966 ZExtPromotedInteger(N->getOperand(0))), 0); 967} 968 969SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 970 DebugLoc dl = N->getDebugLoc(); 971 SDValue Op = GetPromotedInteger(N->getOperand(0)); 972 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 973 return DAG.getZeroExtendInReg(Op, dl, 974 N->getOperand(0).getValueType().getScalarType()); 975} 976 977 978//===----------------------------------------------------------------------===// 979// Integer Result Expansion 980//===----------------------------------------------------------------------===// 981 982/// ExpandIntegerResult - This method is called when the specified result of the 983/// specified node is found to need expansion. At this point, the node may also 984/// have invalid operands or may have other results that need promotion, we just 985/// know that (at least) one result needs expansion. 986void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 987 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); 988 SDValue Lo, Hi; 989 Lo = Hi = SDValue(); 990 991 // See if the target wants to custom expand this node. 992 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 993 return; 994 995 switch (N->getOpcode()) { 996 default: 997#ifndef NDEBUG 998 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 999 N->dump(&DAG); dbgs() << "\n"; 1000#endif 1001 llvm_unreachable("Do not know how to expand the result of this operator!"); 1002 1003 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 1004 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1005 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1006 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1007 1008 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1009 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1010 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1011 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1012 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1013 1014 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 1015 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 1016 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 1017 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 1018 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 1019 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 1020 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 1021 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 1022 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 1023 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 1024 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 1025 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 1026 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 1027 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 1028 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 1029 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 1030 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 1031 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 1032 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 1033 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 1034 1035 case ISD::ATOMIC_LOAD_ADD: 1036 case ISD::ATOMIC_LOAD_SUB: 1037 case ISD::ATOMIC_LOAD_AND: 1038 case ISD::ATOMIC_LOAD_OR: 1039 case ISD::ATOMIC_LOAD_XOR: 1040 case ISD::ATOMIC_LOAD_NAND: 1041 case ISD::ATOMIC_LOAD_MIN: 1042 case ISD::ATOMIC_LOAD_MAX: 1043 case ISD::ATOMIC_LOAD_UMIN: 1044 case ISD::ATOMIC_LOAD_UMAX: 1045 case ISD::ATOMIC_SWAP: { 1046 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 1047 SplitInteger(Tmp.first, Lo, Hi); 1048 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1049 break; 1050 } 1051 1052 case ISD::AND: 1053 case ISD::OR: 1054 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 1055 1056 case ISD::ADD: 1057 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 1058 1059 case ISD::ADDC: 1060 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 1061 1062 case ISD::ADDE: 1063 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 1064 1065 case ISD::SHL: 1066 case ISD::SRA: 1067 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 1068 1069 case ISD::SADDO: 1070 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 1071 case ISD::UADDO: 1072 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 1073 case ISD::UMULO: 1074 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 1075 } 1076 1077 // If Lo/Hi is null, the sub-method took care of registering results etc. 1078 if (Lo.getNode()) 1079 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 1080} 1081 1082/// Lower an atomic node to the appropriate builtin call. 1083std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 1084 unsigned Opc = Node->getOpcode(); 1085 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 1086 RTLIB::Libcall LC; 1087 1088 switch (Opc) { 1089 default: 1090 llvm_unreachable("Unhandled atomic intrinsic Expand!"); 1091 break; 1092 case ISD::ATOMIC_SWAP: 1093 switch (VT.SimpleTy) { 1094 default: llvm_unreachable("Unexpected value type for atomic!"); 1095 case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 1096 case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 1097 case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 1098 case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 1099 } 1100 break; 1101 case ISD::ATOMIC_CMP_SWAP: 1102 switch (VT.SimpleTy) { 1103 default: llvm_unreachable("Unexpected value type for atomic!"); 1104 case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 1105 case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 1106 case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 1107 case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 1108 } 1109 break; 1110 case ISD::ATOMIC_LOAD_ADD: 1111 switch (VT.SimpleTy) { 1112 default: llvm_unreachable("Unexpected value type for atomic!"); 1113 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 1114 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 1115 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 1116 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 1117 } 1118 break; 1119 case ISD::ATOMIC_LOAD_SUB: 1120 switch (VT.SimpleTy) { 1121 default: llvm_unreachable("Unexpected value type for atomic!"); 1122 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 1123 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 1124 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 1125 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 1126 } 1127 break; 1128 case ISD::ATOMIC_LOAD_AND: 1129 switch (VT.SimpleTy) { 1130 default: llvm_unreachable("Unexpected value type for atomic!"); 1131 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 1132 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 1133 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 1134 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 1135 } 1136 break; 1137 case ISD::ATOMIC_LOAD_OR: 1138 switch (VT.SimpleTy) { 1139 default: llvm_unreachable("Unexpected value type for atomic!"); 1140 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 1141 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 1142 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 1143 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 1144 } 1145 break; 1146 case ISD::ATOMIC_LOAD_XOR: 1147 switch (VT.SimpleTy) { 1148 default: llvm_unreachable("Unexpected value type for atomic!"); 1149 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 1150 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 1151 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 1152 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 1153 } 1154 break; 1155 case ISD::ATOMIC_LOAD_NAND: 1156 switch (VT.SimpleTy) { 1157 default: llvm_unreachable("Unexpected value type for atomic!"); 1158 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 1159 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 1160 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 1161 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 1162 } 1163 break; 1164 } 1165 1166 return ExpandChainLibCall(LC, Node, false); 1167} 1168 1169/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded, 1170/// and the shift amount is a constant 'Amt'. Expand the operation. 1171void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 1172 SDValue &Lo, SDValue &Hi) { 1173 DebugLoc DL = N->getDebugLoc(); 1174 // Expand the incoming operand to be shifted, so that we have its parts 1175 SDValue InL, InH; 1176 GetExpandedInteger(N->getOperand(0), InL, InH); 1177 1178 EVT NVT = InL.getValueType(); 1179 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1180 unsigned NVTBits = NVT.getSizeInBits(); 1181 EVT ShTy = N->getOperand(1).getValueType(); 1182 1183 if (N->getOpcode() == ISD::SHL) { 1184 if (Amt > VTBits) { 1185 Lo = Hi = DAG.getConstant(0, NVT); 1186 } else if (Amt > NVTBits) { 1187 Lo = DAG.getConstant(0, NVT); 1188 Hi = DAG.getNode(ISD::SHL, DL, 1189 NVT, InL, DAG.getConstant(Amt-NVTBits, ShTy)); 1190 } else if (Amt == NVTBits) { 1191 Lo = DAG.getConstant(0, NVT); 1192 Hi = InL; 1193 } else if (Amt == 1 && 1194 TLI.isOperationLegalOrCustom(ISD::ADDC, 1195 TLI.getTypeToExpandTo(*DAG.getContext(), NVT))) { 1196 // Emit this X << 1 as X+X. 1197 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1198 SDValue LoOps[2] = { InL, InL }; 1199 Lo = DAG.getNode(ISD::ADDC, DL, VTList, LoOps, 2); 1200 SDValue HiOps[3] = { InH, InH, Lo.getValue(1) }; 1201 Hi = DAG.getNode(ISD::ADDE, DL, VTList, HiOps, 3); 1202 } else { 1203 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, ShTy)); 1204 Hi = DAG.getNode(ISD::OR, DL, NVT, 1205 DAG.getNode(ISD::SHL, DL, NVT, InH, 1206 DAG.getConstant(Amt, ShTy)), 1207 DAG.getNode(ISD::SRL, DL, NVT, InL, 1208 DAG.getConstant(NVTBits-Amt, ShTy))); 1209 } 1210 return; 1211 } 1212 1213 if (N->getOpcode() == ISD::SRL) { 1214 if (Amt > VTBits) { 1215 Lo = DAG.getConstant(0, NVT); 1216 Hi = DAG.getConstant(0, NVT); 1217 } else if (Amt > NVTBits) { 1218 Lo = DAG.getNode(ISD::SRL, DL, 1219 NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy)); 1220 Hi = DAG.getConstant(0, NVT); 1221 } else if (Amt == NVTBits) { 1222 Lo = InH; 1223 Hi = DAG.getConstant(0, NVT); 1224 } else { 1225 Lo = DAG.getNode(ISD::OR, DL, NVT, 1226 DAG.getNode(ISD::SRL, DL, NVT, InL, 1227 DAG.getConstant(Amt, ShTy)), 1228 DAG.getNode(ISD::SHL, DL, NVT, InH, 1229 DAG.getConstant(NVTBits-Amt, ShTy))); 1230 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1231 } 1232 return; 1233 } 1234 1235 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1236 if (Amt > VTBits) { 1237 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1238 DAG.getConstant(NVTBits-1, ShTy)); 1239 } else if (Amt > NVTBits) { 1240 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1241 DAG.getConstant(Amt-NVTBits, ShTy)); 1242 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1243 DAG.getConstant(NVTBits-1, ShTy)); 1244 } else if (Amt == NVTBits) { 1245 Lo = InH; 1246 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1247 DAG.getConstant(NVTBits-1, ShTy)); 1248 } else { 1249 Lo = DAG.getNode(ISD::OR, DL, NVT, 1250 DAG.getNode(ISD::SRL, DL, NVT, InL, 1251 DAG.getConstant(Amt, ShTy)), 1252 DAG.getNode(ISD::SHL, DL, NVT, InH, 1253 DAG.getConstant(NVTBits-Amt, ShTy))); 1254 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1255 } 1256} 1257 1258/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1259/// this shift based on knowledge of the high bit of the shift amount. If we 1260/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1261/// shift amount. 1262bool DAGTypeLegalizer:: 1263ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1264 SDValue Amt = N->getOperand(1); 1265 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1266 EVT ShTy = Amt.getValueType(); 1267 unsigned ShBits = ShTy.getScalarType().getSizeInBits(); 1268 unsigned NVTBits = NVT.getScalarType().getSizeInBits(); 1269 assert(isPowerOf2_32(NVTBits) && 1270 "Expanded integer type size not a power of two!"); 1271 DebugLoc dl = N->getDebugLoc(); 1272 1273 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1274 APInt KnownZero, KnownOne; 1275 DAG.ComputeMaskedBits(N->getOperand(1), HighBitMask, KnownZero, KnownOne); 1276 1277 // If we don't know anything about the high bits, exit. 1278 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1279 return false; 1280 1281 // Get the incoming operand to be shifted. 1282 SDValue InL, InH; 1283 GetExpandedInteger(N->getOperand(0), InL, InH); 1284 1285 // If we know that any of the high bits of the shift amount are one, then we 1286 // can do this as a couple of simple shifts. 1287 if (KnownOne.intersects(HighBitMask)) { 1288 // Mask out the high bit, which we know is set. 1289 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 1290 DAG.getConstant(~HighBitMask, ShTy)); 1291 1292 switch (N->getOpcode()) { 1293 default: llvm_unreachable("Unknown shift"); 1294 case ISD::SHL: 1295 Lo = DAG.getConstant(0, NVT); // Low part is zero. 1296 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 1297 return true; 1298 case ISD::SRL: 1299 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 1300 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 1301 return true; 1302 case ISD::SRA: 1303 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 1304 DAG.getConstant(NVTBits-1, ShTy)); 1305 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 1306 return true; 1307 } 1308 } 1309 1310#if 0 1311 // FIXME: This code is broken for shifts with a zero amount! 1312 // If we know that all of the high bits of the shift amount are zero, then we 1313 // can do this as a couple of simple shifts. 1314 if ((KnownZero & HighBitMask) == HighBitMask) { 1315 // Compute 32-amt. 1316 SDValue Amt2 = DAG.getNode(ISD::SUB, ShTy, 1317 DAG.getConstant(NVTBits, ShTy), 1318 Amt); 1319 unsigned Op1, Op2; 1320 switch (N->getOpcode()) { 1321 default: llvm_unreachable("Unknown shift"); 1322 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1323 case ISD::SRL: 1324 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1325 } 1326 1327 Lo = DAG.getNode(N->getOpcode(), NVT, InL, Amt); 1328 Hi = DAG.getNode(ISD::OR, NVT, 1329 DAG.getNode(Op1, NVT, InH, Amt), 1330 DAG.getNode(Op2, NVT, InL, Amt2)); 1331 return true; 1332 } 1333#endif 1334 1335 return false; 1336} 1337 1338/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 1339/// of any size. 1340bool DAGTypeLegalizer:: 1341ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1342 SDValue Amt = N->getOperand(1); 1343 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1344 EVT ShTy = Amt.getValueType(); 1345 unsigned NVTBits = NVT.getSizeInBits(); 1346 assert(isPowerOf2_32(NVTBits) && 1347 "Expanded integer type size not a power of two!"); 1348 DebugLoc dl = N->getDebugLoc(); 1349 1350 // Get the incoming operand to be shifted. 1351 SDValue InL, InH; 1352 GetExpandedInteger(N->getOperand(0), InL, InH); 1353 1354 SDValue NVBitsNode = DAG.getConstant(NVTBits, ShTy); 1355 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 1356 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 1357 SDValue isShort = DAG.getSetCC(dl, TLI.getSetCCResultType(ShTy), 1358 Amt, NVBitsNode, ISD::SETULT); 1359 1360 SDValue LoS, HiS, LoL, HiL; 1361 switch (N->getOpcode()) { 1362 default: llvm_unreachable("Unknown shift"); 1363 case ISD::SHL: 1364 // Short: ShAmt < NVTBits 1365 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 1366 HiS = DAG.getNode(ISD::OR, dl, NVT, 1367 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 1368 // FIXME: If Amt is zero, the following shift generates an undefined result 1369 // on some architectures. 1370 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 1371 1372 // Long: ShAmt >= NVTBits 1373 LoL = DAG.getConstant(0, NVT); // Lo part is zero. 1374 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 1375 1376 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1377 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1378 return true; 1379 case ISD::SRL: 1380 // Short: ShAmt < NVTBits 1381 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 1382 LoS = DAG.getNode(ISD::OR, dl, NVT, 1383 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1384 // FIXME: If Amt is zero, the following shift generates an undefined result 1385 // on some architectures. 1386 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1387 1388 // Long: ShAmt >= NVTBits 1389 HiL = DAG.getConstant(0, NVT); // Hi part is zero. 1390 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1391 1392 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1393 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1394 return true; 1395 case ISD::SRA: 1396 // Short: ShAmt < NVTBits 1397 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 1398 LoS = DAG.getNode(ISD::OR, dl, NVT, 1399 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1400 // FIXME: If Amt is zero, the following shift generates an undefined result 1401 // on some architectures. 1402 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1403 1404 // Long: ShAmt >= NVTBits 1405 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 1406 DAG.getConstant(NVTBits-1, ShTy)); 1407 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1408 1409 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1410 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1411 return true; 1412 } 1413 1414 return false; 1415} 1416 1417void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1418 SDValue &Lo, SDValue &Hi) { 1419 DebugLoc dl = N->getDebugLoc(); 1420 // Expand the subcomponents. 1421 SDValue LHSL, LHSH, RHSL, RHSH; 1422 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1423 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1424 1425 EVT NVT = LHSL.getValueType(); 1426 SDValue LoOps[2] = { LHSL, RHSL }; 1427 SDValue HiOps[3] = { LHSH, RHSH }; 1428 1429 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1430 // them. TODO: Teach operation legalization how to expand unsupported 1431 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1432 // a carry of type MVT::Glue, but there doesn't seem to be any way to 1433 // generate a value of this type in the expanded code sequence. 1434 bool hasCarry = 1435 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1436 ISD::ADDC : ISD::SUBC, 1437 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1438 1439 if (hasCarry) { 1440 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1441 if (N->getOpcode() == ISD::ADD) { 1442 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1443 HiOps[2] = Lo.getValue(1); 1444 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1445 } else { 1446 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 1447 HiOps[2] = Lo.getValue(1); 1448 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 1449 } 1450 return; 1451 } 1452 1453 if (N->getOpcode() == ISD::ADD) { 1454 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps, 2); 1455 Hi = DAG.getNode(ISD::ADD, dl, NVT, HiOps, 2); 1456 SDValue Cmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[0], 1457 ISD::SETULT); 1458 SDValue Carry1 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp1, 1459 DAG.getConstant(1, NVT), 1460 DAG.getConstant(0, NVT)); 1461 SDValue Cmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[1], 1462 ISD::SETULT); 1463 SDValue Carry2 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp2, 1464 DAG.getConstant(1, NVT), Carry1); 1465 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 1466 } else { 1467 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps, 2); 1468 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps, 2); 1469 SDValue Cmp = 1470 DAG.getSetCC(dl, TLI.getSetCCResultType(LoOps[0].getValueType()), 1471 LoOps[0], LoOps[1], ISD::SETULT); 1472 SDValue Borrow = DAG.getNode(ISD::SELECT, dl, NVT, Cmp, 1473 DAG.getConstant(1, NVT), 1474 DAG.getConstant(0, NVT)); 1475 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 1476 } 1477} 1478 1479void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1480 SDValue &Lo, SDValue &Hi) { 1481 // Expand the subcomponents. 1482 SDValue LHSL, LHSH, RHSL, RHSH; 1483 DebugLoc dl = N->getDebugLoc(); 1484 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1485 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1486 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1487 SDValue LoOps[2] = { LHSL, RHSL }; 1488 SDValue HiOps[3] = { LHSH, RHSH }; 1489 1490 if (N->getOpcode() == ISD::ADDC) { 1491 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1492 HiOps[2] = Lo.getValue(1); 1493 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1494 } else { 1495 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 1496 HiOps[2] = Lo.getValue(1); 1497 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 1498 } 1499 1500 // Legalized the flag result - switch anything that used the old flag to 1501 // use the new one. 1502 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1503} 1504 1505void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1506 SDValue &Lo, SDValue &Hi) { 1507 // Expand the subcomponents. 1508 SDValue LHSL, LHSH, RHSL, RHSH; 1509 DebugLoc dl = N->getDebugLoc(); 1510 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1511 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1512 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1513 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1514 SDValue HiOps[3] = { LHSH, RHSH }; 1515 1516 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps, 3); 1517 HiOps[2] = Lo.getValue(1); 1518 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps, 3); 1519 1520 // Legalized the flag result - switch anything that used the old flag to 1521 // use the new one. 1522 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1523} 1524 1525void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 1526 SDValue &Lo, SDValue &Hi) { 1527 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1528 DebugLoc dl = N->getDebugLoc(); 1529 SDValue Op = N->getOperand(0); 1530 if (Op.getValueType().bitsLE(NVT)) { 1531 // The low part is any extension of the input (which degenerates to a copy). 1532 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 1533 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 1534 } else { 1535 // For example, extension of an i48 to an i64. The operand type necessarily 1536 // promotes to the result type, so will end up being expanded too. 1537 assert(getTypeAction(Op.getValueType()) == 1538 TargetLowering::TypePromoteInteger && 1539 "Only know how to promote this result!"); 1540 SDValue Res = GetPromotedInteger(Op); 1541 assert(Res.getValueType() == N->getValueType(0) && 1542 "Operand over promoted?"); 1543 // Split the promoted operand. This will simplify when it is expanded. 1544 SplitInteger(Res, Lo, Hi); 1545 } 1546} 1547 1548void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 1549 SDValue &Lo, SDValue &Hi) { 1550 DebugLoc dl = N->getDebugLoc(); 1551 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1552 EVT NVT = Lo.getValueType(); 1553 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1554 unsigned NVTBits = NVT.getSizeInBits(); 1555 unsigned EVTBits = EVT.getSizeInBits(); 1556 1557 if (NVTBits < EVTBits) { 1558 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 1559 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1560 EVTBits - NVTBits))); 1561 } else { 1562 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 1563 // The high part replicates the sign bit of Lo, make it explicit. 1564 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1565 DAG.getConstant(NVTBits-1, TLI.getPointerTy())); 1566 } 1567} 1568 1569void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 1570 SDValue &Lo, SDValue &Hi) { 1571 DebugLoc dl = N->getDebugLoc(); 1572 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1573 EVT NVT = Lo.getValueType(); 1574 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1575 unsigned NVTBits = NVT.getSizeInBits(); 1576 unsigned EVTBits = EVT.getSizeInBits(); 1577 1578 if (NVTBits < EVTBits) { 1579 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 1580 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1581 EVTBits - NVTBits))); 1582 } else { 1583 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 1584 // The high part must be zero, make it explicit. 1585 Hi = DAG.getConstant(0, NVT); 1586 } 1587} 1588 1589void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1590 SDValue &Lo, SDValue &Hi) { 1591 DebugLoc dl = N->getDebugLoc(); 1592 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1593 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 1594 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 1595} 1596 1597void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 1598 SDValue &Lo, SDValue &Hi) { 1599 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1600 unsigned NBitWidth = NVT.getSizeInBits(); 1601 const APInt &Cst = cast<ConstantSDNode>(N)->getAPIntValue(); 1602 Lo = DAG.getConstant(Cst.trunc(NBitWidth), NVT); 1603 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT); 1604} 1605 1606void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1607 SDValue &Lo, SDValue &Hi) { 1608 DebugLoc dl = N->getDebugLoc(); 1609 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1610 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1611 EVT NVT = Lo.getValueType(); 1612 1613 SDValue HiNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Hi, 1614 DAG.getConstant(0, NVT), ISD::SETNE); 1615 1616 SDValue LoLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Lo); 1617 SDValue HiLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Hi); 1618 1619 Lo = DAG.getNode(ISD::SELECT, dl, NVT, HiNotZero, HiLZ, 1620 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 1621 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1622 Hi = DAG.getConstant(0, NVT); 1623} 1624 1625void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1626 SDValue &Lo, SDValue &Hi) { 1627 DebugLoc dl = N->getDebugLoc(); 1628 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1629 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1630 EVT NVT = Lo.getValueType(); 1631 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 1632 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 1633 Hi = DAG.getConstant(0, NVT); 1634} 1635 1636void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1637 SDValue &Lo, SDValue &Hi) { 1638 DebugLoc dl = N->getDebugLoc(); 1639 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1640 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1641 EVT NVT = Lo.getValueType(); 1642 1643 SDValue LoNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, 1644 DAG.getConstant(0, NVT), ISD::SETNE); 1645 1646 SDValue LoLZ = DAG.getNode(ISD::CTTZ, dl, NVT, Lo); 1647 SDValue HiLZ = DAG.getNode(ISD::CTTZ, dl, NVT, Hi); 1648 1649 Lo = DAG.getNode(ISD::SELECT, dl, NVT, LoNotZero, LoLZ, 1650 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 1651 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1652 Hi = DAG.getConstant(0, NVT); 1653} 1654 1655void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 1656 SDValue &Hi) { 1657 DebugLoc dl = N->getDebugLoc(); 1658 EVT VT = N->getValueType(0); 1659 SDValue Op = N->getOperand(0); 1660 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 1661 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 1662 SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*irrelevant*/, dl), Lo, Hi); 1663} 1664 1665void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 1666 SDValue &Hi) { 1667 DebugLoc dl = N->getDebugLoc(); 1668 EVT VT = N->getValueType(0); 1669 SDValue Op = N->getOperand(0); 1670 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 1671 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 1672 SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*irrelevant*/, dl), Lo, Hi); 1673} 1674 1675void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 1676 SDValue &Lo, SDValue &Hi) { 1677 if (ISD::isNormalLoad(N)) { 1678 ExpandRes_NormalLoad(N, Lo, Hi); 1679 return; 1680 } 1681 1682 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1683 1684 EVT VT = N->getValueType(0); 1685 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1686 SDValue Ch = N->getChain(); 1687 SDValue Ptr = N->getBasePtr(); 1688 ISD::LoadExtType ExtType = N->getExtensionType(); 1689 unsigned Alignment = N->getAlignment(); 1690 bool isVolatile = N->isVolatile(); 1691 bool isNonTemporal = N->isNonTemporal(); 1692 DebugLoc dl = N->getDebugLoc(); 1693 1694 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1695 1696 if (N->getMemoryVT().bitsLE(NVT)) { 1697 EVT MemVT = N->getMemoryVT(); 1698 1699 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 1700 MemVT, isVolatile, isNonTemporal, Alignment); 1701 1702 // Remember the chain. 1703 Ch = Lo.getValue(1); 1704 1705 if (ExtType == ISD::SEXTLOAD) { 1706 // The high part is obtained by SRA'ing all but one of the bits of the 1707 // lo part. 1708 unsigned LoSize = Lo.getValueType().getSizeInBits(); 1709 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1710 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 1711 } else if (ExtType == ISD::ZEXTLOAD) { 1712 // The high part is just a zero. 1713 Hi = DAG.getConstant(0, NVT); 1714 } else { 1715 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 1716 // The high part is undefined. 1717 Hi = DAG.getUNDEF(NVT); 1718 } 1719 } else if (TLI.isLittleEndian()) { 1720 // Little-endian - low bits are at low addresses. 1721 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 1722 isVolatile, isNonTemporal, Alignment); 1723 1724 unsigned ExcessBits = 1725 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1726 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 1727 1728 // Increment the pointer to the other half. 1729 unsigned IncrementSize = NVT.getSizeInBits()/8; 1730 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1731 DAG.getIntPtrConstant(IncrementSize)); 1732 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 1733 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 1734 isVolatile, isNonTemporal, 1735 MinAlign(Alignment, IncrementSize)); 1736 1737 // Build a factor node to remember that this load is independent of the 1738 // other one. 1739 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1740 Hi.getValue(1)); 1741 } else { 1742 // Big-endian - high bits are at low addresses. Favor aligned loads at 1743 // the cost of some bit-fiddling. 1744 EVT MemVT = N->getMemoryVT(); 1745 unsigned EBytes = MemVT.getStoreSize(); 1746 unsigned IncrementSize = NVT.getSizeInBits()/8; 1747 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1748 1749 // Load both the high bits and maybe some of the low bits. 1750 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 1751 EVT::getIntegerVT(*DAG.getContext(), 1752 MemVT.getSizeInBits() - ExcessBits), 1753 isVolatile, isNonTemporal, Alignment); 1754 1755 // Increment the pointer to the other half. 1756 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1757 DAG.getIntPtrConstant(IncrementSize)); 1758 // Load the rest of the low bits. 1759 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 1760 N->getPointerInfo().getWithOffset(IncrementSize), 1761 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 1762 isVolatile, isNonTemporal, 1763 MinAlign(Alignment, IncrementSize)); 1764 1765 // Build a factor node to remember that this load is independent of the 1766 // other one. 1767 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1768 Hi.getValue(1)); 1769 1770 if (ExcessBits < NVT.getSizeInBits()) { 1771 // Transfer low bits from the bottom of Hi to the top of Lo. 1772 Lo = DAG.getNode(ISD::OR, dl, NVT, Lo, 1773 DAG.getNode(ISD::SHL, dl, NVT, Hi, 1774 DAG.getConstant(ExcessBits, 1775 TLI.getPointerTy()))); 1776 // Move high bits to the right position in Hi. 1777 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, 1778 NVT, Hi, 1779 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 1780 TLI.getPointerTy())); 1781 } 1782 } 1783 1784 // Legalized the chain result - switch anything that used the old chain to 1785 // use the new one. 1786 ReplaceValueWith(SDValue(N, 1), Ch); 1787} 1788 1789void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 1790 SDValue &Lo, SDValue &Hi) { 1791 DebugLoc dl = N->getDebugLoc(); 1792 SDValue LL, LH, RL, RH; 1793 GetExpandedInteger(N->getOperand(0), LL, LH); 1794 GetExpandedInteger(N->getOperand(1), RL, RH); 1795 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 1796 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 1797} 1798 1799void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 1800 SDValue &Lo, SDValue &Hi) { 1801 EVT VT = N->getValueType(0); 1802 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1803 DebugLoc dl = N->getDebugLoc(); 1804 1805 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT); 1806 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT); 1807 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT); 1808 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT); 1809 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 1810 SDValue LL, LH, RL, RH; 1811 GetExpandedInteger(N->getOperand(0), LL, LH); 1812 GetExpandedInteger(N->getOperand(1), RL, RH); 1813 unsigned OuterBitSize = VT.getSizeInBits(); 1814 unsigned InnerBitSize = NVT.getSizeInBits(); 1815 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0)); 1816 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1)); 1817 1818 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 1819 if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) && 1820 DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) { 1821 // The inputs are both zero-extended. 1822 if (HasUMUL_LOHI) { 1823 // We can emit a umul_lohi. 1824 Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 1825 Hi = SDValue(Lo.getNode(), 1); 1826 return; 1827 } 1828 if (HasMULHU) { 1829 // We can emit a mulhu+mul. 1830 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1831 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 1832 return; 1833 } 1834 } 1835 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 1836 // The input values are both sign-extended. 1837 if (HasSMUL_LOHI) { 1838 // We can emit a smul_lohi. 1839 Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 1840 Hi = SDValue(Lo.getNode(), 1); 1841 return; 1842 } 1843 if (HasMULHS) { 1844 // We can emit a mulhs+mul. 1845 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1846 Hi = DAG.getNode(ISD::MULHS, dl, NVT, LL, RL); 1847 return; 1848 } 1849 } 1850 if (HasUMUL_LOHI) { 1851 // Lo,Hi = umul LHS, RHS. 1852 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl, 1853 DAG.getVTList(NVT, NVT), LL, RL); 1854 Lo = UMulLOHI; 1855 Hi = UMulLOHI.getValue(1); 1856 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 1857 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 1858 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 1859 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 1860 return; 1861 } 1862 if (HasMULHU) { 1863 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1864 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 1865 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 1866 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 1867 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 1868 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 1869 return; 1870 } 1871 } 1872 1873 // If nothing else, we can make a libcall. 1874 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1875 if (VT == MVT::i16) 1876 LC = RTLIB::MUL_I16; 1877 else if (VT == MVT::i32) 1878 LC = RTLIB::MUL_I32; 1879 else if (VT == MVT::i64) 1880 LC = RTLIB::MUL_I64; 1881 else if (VT == MVT::i128) 1882 LC = RTLIB::MUL_I128; 1883 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!"); 1884 1885 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1886 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*irrelevant*/, dl), Lo, Hi); 1887} 1888 1889void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 1890 SDValue &Lo, SDValue &Hi) { 1891 SDValue LHS = Node->getOperand(0); 1892 SDValue RHS = Node->getOperand(1); 1893 DebugLoc dl = Node->getDebugLoc(); 1894 1895 // Expand the result by simply replacing it with the equivalent 1896 // non-overflow-checking operation. 1897 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 1898 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 1899 LHS, RHS); 1900 SplitInteger(Sum, Lo, Hi); 1901 1902 // Compute the overflow. 1903 // 1904 // LHSSign -> LHS >= 0 1905 // RHSSign -> RHS >= 0 1906 // SumSign -> Sum >= 0 1907 // 1908 // Add: 1909 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 1910 // Sub: 1911 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 1912 // 1913 EVT OType = Node->getValueType(1); 1914 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 1915 1916 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 1917 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 1918 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 1919 Node->getOpcode() == ISD::SADDO ? 1920 ISD::SETEQ : ISD::SETNE); 1921 1922 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 1923 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 1924 1925 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 1926 1927 // Use the calculated overflow everywhere. 1928 ReplaceValueWith(SDValue(Node, 1), Cmp); 1929} 1930 1931void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 1932 SDValue &Lo, SDValue &Hi) { 1933 EVT VT = N->getValueType(0); 1934 DebugLoc dl = N->getDebugLoc(); 1935 1936 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1937 if (VT == MVT::i16) 1938 LC = RTLIB::SDIV_I16; 1939 else if (VT == MVT::i32) 1940 LC = RTLIB::SDIV_I32; 1941 else if (VT == MVT::i64) 1942 LC = RTLIB::SDIV_I64; 1943 else if (VT == MVT::i128) 1944 LC = RTLIB::SDIV_I128; 1945 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 1946 1947 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1948 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); 1949} 1950 1951void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 1952 SDValue &Lo, SDValue &Hi) { 1953 EVT VT = N->getValueType(0); 1954 DebugLoc dl = N->getDebugLoc(); 1955 1956 // If we can emit an efficient shift operation, do so now. Check to see if 1957 // the RHS is a constant. 1958 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 1959 return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi); 1960 1961 // If we can determine that the high bit of the shift is zero or one, even if 1962 // the low bits are variable, emit this shift in an optimized form. 1963 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 1964 return; 1965 1966 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 1967 unsigned PartsOpc; 1968 if (N->getOpcode() == ISD::SHL) { 1969 PartsOpc = ISD::SHL_PARTS; 1970 } else if (N->getOpcode() == ISD::SRL) { 1971 PartsOpc = ISD::SRL_PARTS; 1972 } else { 1973 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1974 PartsOpc = ISD::SRA_PARTS; 1975 } 1976 1977 // Next check to see if the target supports this SHL_PARTS operation or if it 1978 // will custom expand it. 1979 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1980 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 1981 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 1982 Action == TargetLowering::Custom) { 1983 // Expand the subcomponents. 1984 SDValue LHSL, LHSH; 1985 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1986 1987 SDValue Ops[] = { LHSL, LHSH, N->getOperand(1) }; 1988 EVT VT = LHSL.getValueType(); 1989 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops, 3); 1990 Hi = Lo.getValue(1); 1991 return; 1992 } 1993 1994 // Otherwise, emit a libcall. 1995 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1996 bool isSigned; 1997 if (N->getOpcode() == ISD::SHL) { 1998 isSigned = false; /*sign irrelevant*/ 1999 if (VT == MVT::i16) 2000 LC = RTLIB::SHL_I16; 2001 else if (VT == MVT::i32) 2002 LC = RTLIB::SHL_I32; 2003 else if (VT == MVT::i64) 2004 LC = RTLIB::SHL_I64; 2005 else if (VT == MVT::i128) 2006 LC = RTLIB::SHL_I128; 2007 } else if (N->getOpcode() == ISD::SRL) { 2008 isSigned = false; 2009 if (VT == MVT::i16) 2010 LC = RTLIB::SRL_I16; 2011 else if (VT == MVT::i32) 2012 LC = RTLIB::SRL_I32; 2013 else if (VT == MVT::i64) 2014 LC = RTLIB::SRL_I64; 2015 else if (VT == MVT::i128) 2016 LC = RTLIB::SRL_I128; 2017 } else { 2018 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2019 isSigned = true; 2020 if (VT == MVT::i16) 2021 LC = RTLIB::SRA_I16; 2022 else if (VT == MVT::i32) 2023 LC = RTLIB::SRA_I32; 2024 else if (VT == MVT::i64) 2025 LC = RTLIB::SRA_I64; 2026 else if (VT == MVT::i128) 2027 LC = RTLIB::SRA_I128; 2028 } 2029 2030 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 2031 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2032 SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned, dl), Lo, Hi); 2033 return; 2034 } 2035 2036 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 2037 llvm_unreachable("Unsupported shift!"); 2038} 2039 2040void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 2041 SDValue &Lo, SDValue &Hi) { 2042 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2043 DebugLoc dl = N->getDebugLoc(); 2044 SDValue Op = N->getOperand(0); 2045 if (Op.getValueType().bitsLE(NVT)) { 2046 // The low part is sign extension of the input (degenerates to a copy). 2047 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 2048 // The high part is obtained by SRA'ing all but one of the bits of low part. 2049 unsigned LoSize = NVT.getSizeInBits(); 2050 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2051 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 2052 } else { 2053 // For example, extension of an i48 to an i64. The operand type necessarily 2054 // promotes to the result type, so will end up being expanded too. 2055 assert(getTypeAction(Op.getValueType()) == 2056 TargetLowering::TypePromoteInteger && 2057 "Only know how to promote this result!"); 2058 SDValue Res = GetPromotedInteger(Op); 2059 assert(Res.getValueType() == N->getValueType(0) && 2060 "Operand over promoted?"); 2061 // Split the promoted operand. This will simplify when it is expanded. 2062 SplitInteger(Res, Lo, Hi); 2063 unsigned ExcessBits = 2064 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2065 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2066 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2067 ExcessBits))); 2068 } 2069} 2070 2071void DAGTypeLegalizer:: 2072ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2073 DebugLoc dl = N->getDebugLoc(); 2074 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2075 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2076 2077 if (EVT.bitsLE(Lo.getValueType())) { 2078 // sext_inreg the low part if needed. 2079 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 2080 N->getOperand(1)); 2081 2082 // The high part gets the sign extension from the lo-part. This handles 2083 // things like sextinreg V:i64 from i8. 2084 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 2085 DAG.getConstant(Hi.getValueType().getSizeInBits()-1, 2086 TLI.getPointerTy())); 2087 } else { 2088 // For example, extension of an i48 to an i64. Leave the low part alone, 2089 // sext_inreg the high part. 2090 unsigned ExcessBits = 2091 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 2092 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2093 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2094 ExcessBits))); 2095 } 2096} 2097 2098void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 2099 SDValue &Lo, SDValue &Hi) { 2100 EVT VT = N->getValueType(0); 2101 DebugLoc dl = N->getDebugLoc(); 2102 2103 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2104 if (VT == MVT::i16) 2105 LC = RTLIB::SREM_I16; 2106 else if (VT == MVT::i32) 2107 LC = RTLIB::SREM_I32; 2108 else if (VT == MVT::i64) 2109 LC = RTLIB::SREM_I64; 2110 else if (VT == MVT::i128) 2111 LC = RTLIB::SREM_I128; 2112 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 2113 2114 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2115 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); 2116} 2117 2118void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 2119 SDValue &Lo, SDValue &Hi) { 2120 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2121 DebugLoc dl = N->getDebugLoc(); 2122 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 2123 Hi = DAG.getNode(ISD::SRL, dl, 2124 N->getOperand(0).getValueType(), N->getOperand(0), 2125 DAG.getConstant(NVT.getSizeInBits(), TLI.getPointerTy())); 2126 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 2127} 2128 2129void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 2130 SDValue &Lo, SDValue &Hi) { 2131 SDValue LHS = N->getOperand(0); 2132 SDValue RHS = N->getOperand(1); 2133 DebugLoc dl = N->getDebugLoc(); 2134 2135 // Expand the result by simply replacing it with the equivalent 2136 // non-overflow-checking operation. 2137 SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ? 2138 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2139 LHS, RHS); 2140 SplitInteger(Sum, Lo, Hi); 2141 2142 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2143 // overflows iff a - b > a. 2144 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, 2145 N->getOpcode () == ISD::UADDO ? 2146 ISD::SETULT : ISD::SETUGT); 2147 2148 // Use the calculated overflow everywhere. 2149 ReplaceValueWith(SDValue(N, 1), Ofl); 2150} 2151 2152void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 2153 SDValue &Lo, SDValue &Hi) { 2154 EVT VT = N->getValueType(0); 2155 const Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 2156 EVT PtrVT = TLI.getPointerTy(); 2157 const Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 2158 DebugLoc dl = N->getDebugLoc(); 2159 2160 // A divide for UMULO should be faster than a function call. 2161 if (N->getOpcode() == ISD::UMULO) { 2162 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 2163 DebugLoc DL = N->getDebugLoc(); 2164 2165 SDValue MUL = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS); 2166 SplitInteger(MUL, Lo, Hi); 2167 2168 // A divide for UMULO will be faster than a function call. Select to 2169 // make sure we aren't using 0. 2170 SDValue isZero = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), 2171 RHS, DAG.getConstant(0, VT), ISD::SETNE); 2172 SDValue NotZero = DAG.getNode(ISD::SELECT, dl, VT, isZero, 2173 DAG.getConstant(1, VT), RHS); 2174 SDValue DIV = DAG.getNode(ISD::UDIV, DL, LHS.getValueType(), MUL, NotZero); 2175 SDValue Overflow; 2176 Overflow = DAG.getSetCC(DL, N->getValueType(1), DIV, LHS, ISD::SETNE); 2177 ReplaceValueWith(SDValue(N, 1), Overflow); 2178 return; 2179 } 2180 2181 // Replace this with a libcall that will check overflow. 2182 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2183 if (VT == MVT::i32) 2184 LC = RTLIB::MULO_I32; 2185 else if (VT == MVT::i64) 2186 LC = RTLIB::MULO_I64; 2187 else if (VT == MVT::i128) 2188 LC = RTLIB::MULO_I128; 2189 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 2190 2191 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 2192 // Temporary for the overflow value, default it to zero. 2193 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, 2194 DAG.getConstant(0, PtrVT), Temp, 2195 MachinePointerInfo(), false, false, 0); 2196 2197 TargetLowering::ArgListTy Args; 2198 TargetLowering::ArgListEntry Entry; 2199 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2200 EVT ArgVT = N->getOperand(i).getValueType(); 2201 const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2202 Entry.Node = N->getOperand(i); 2203 Entry.Ty = ArgTy; 2204 Entry.isSExt = true; 2205 Entry.isZExt = false; 2206 Args.push_back(Entry); 2207 } 2208 2209 // Also pass the address of the overflow check. 2210 Entry.Node = Temp; 2211 Entry.Ty = PtrTy->getPointerTo(); 2212 Entry.isSExt = true; 2213 Entry.isZExt = false; 2214 Args.push_back(Entry); 2215 2216 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 2217 std::pair<SDValue, SDValue> CallInfo = 2218 TLI.LowerCallTo(Chain, RetTy, true, false, false, false, 2219 0, TLI.getLibcallCallingConv(LC), false, 2220 true, Func, Args, DAG, dl); 2221 2222 SplitInteger(CallInfo.first, Lo, Hi); 2223 SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, 2224 MachinePointerInfo(), false, false, 0); 2225 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 2226 DAG.getConstant(0, PtrVT), 2227 ISD::SETNE); 2228 // Use the overflow from the libcall everywhere. 2229 ReplaceValueWith(SDValue(N, 1), Ofl); 2230} 2231 2232void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 2233 SDValue &Lo, SDValue &Hi) { 2234 EVT VT = N->getValueType(0); 2235 DebugLoc dl = N->getDebugLoc(); 2236 2237 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2238 if (VT == MVT::i16) 2239 LC = RTLIB::UDIV_I16; 2240 else if (VT == MVT::i32) 2241 LC = RTLIB::UDIV_I32; 2242 else if (VT == MVT::i64) 2243 LC = RTLIB::UDIV_I64; 2244 else if (VT == MVT::i128) 2245 LC = RTLIB::UDIV_I128; 2246 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 2247 2248 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2249 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); 2250} 2251 2252void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 2253 SDValue &Lo, SDValue &Hi) { 2254 EVT VT = N->getValueType(0); 2255 DebugLoc dl = N->getDebugLoc(); 2256 2257 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2258 if (VT == MVT::i16) 2259 LC = RTLIB::UREM_I16; 2260 else if (VT == MVT::i32) 2261 LC = RTLIB::UREM_I32; 2262 else if (VT == MVT::i64) 2263 LC = RTLIB::UREM_I64; 2264 else if (VT == MVT::i128) 2265 LC = RTLIB::UREM_I128; 2266 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 2267 2268 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2269 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); 2270} 2271 2272void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 2273 SDValue &Lo, SDValue &Hi) { 2274 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2275 DebugLoc dl = N->getDebugLoc(); 2276 SDValue Op = N->getOperand(0); 2277 if (Op.getValueType().bitsLE(NVT)) { 2278 // The low part is zero extension of the input (degenerates to a copy). 2279 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 2280 Hi = DAG.getConstant(0, NVT); // The high part is just a zero. 2281 } else { 2282 // For example, extension of an i48 to an i64. The operand type necessarily 2283 // promotes to the result type, so will end up being expanded too. 2284 assert(getTypeAction(Op.getValueType()) == 2285 TargetLowering::TypePromoteInteger && 2286 "Only know how to promote this result!"); 2287 SDValue Res = GetPromotedInteger(Op); 2288 assert(Res.getValueType() == N->getValueType(0) && 2289 "Operand over promoted?"); 2290 // Split the promoted operand. This will simplify when it is expanded. 2291 SplitInteger(Res, Lo, Hi); 2292 unsigned ExcessBits = 2293 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2294 Hi = DAG.getZeroExtendInReg(Hi, dl, 2295 EVT::getIntegerVT(*DAG.getContext(), 2296 ExcessBits)); 2297 } 2298} 2299 2300 2301//===----------------------------------------------------------------------===// 2302// Integer Operand Expansion 2303//===----------------------------------------------------------------------===// 2304 2305/// ExpandIntegerOperand - This method is called when the specified operand of 2306/// the specified node is found to need expansion. At this point, all of the 2307/// result types of the node are known to be legal, but other operands of the 2308/// node may need promotion or expansion as well as the specified one. 2309bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 2310 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); 2311 SDValue Res = SDValue(); 2312 2313 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2314 return false; 2315 2316 switch (N->getOpcode()) { 2317 default: 2318 #ifndef NDEBUG 2319 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 2320 N->dump(&DAG); dbgs() << "\n"; 2321 #endif 2322 llvm_unreachable("Do not know how to expand this operator's operand!"); 2323 2324 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2325 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 2326 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2327 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2328 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 2329 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 2330 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 2331 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 2332 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 2333 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 2334 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 2335 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 2336 2337 case ISD::SHL: 2338 case ISD::SRA: 2339 case ISD::SRL: 2340 case ISD::ROTL: 2341 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 2342 case ISD::RETURNADDR: 2343 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 2344 } 2345 2346 // If the result is null, the sub-method took care of registering results etc. 2347 if (!Res.getNode()) return false; 2348 2349 // If the result is N, the sub-method updated N in place. Tell the legalizer 2350 // core about this. 2351 if (Res.getNode() == N) 2352 return true; 2353 2354 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2355 "Invalid operand expansion"); 2356 2357 ReplaceValueWith(SDValue(N, 0), Res); 2358 return false; 2359} 2360 2361/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 2362/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2363void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 2364 SDValue &NewRHS, 2365 ISD::CondCode &CCCode, 2366 DebugLoc dl) { 2367 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2368 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 2369 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 2370 2371 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 2372 if (RHSLo == RHSHi) { 2373 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 2374 if (RHSCST->isAllOnesValue()) { 2375 // Equality comparison to -1. 2376 NewLHS = DAG.getNode(ISD::AND, dl, 2377 LHSLo.getValueType(), LHSLo, LHSHi); 2378 NewRHS = RHSLo; 2379 return; 2380 } 2381 } 2382 } 2383 2384 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 2385 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 2386 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 2387 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2388 return; 2389 } 2390 2391 // If this is a comparison of the sign bit, just look at the top part. 2392 // X > -1, x < 0 2393 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 2394 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 2395 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 2396 NewLHS = LHSHi; 2397 NewRHS = RHSHi; 2398 return; 2399 } 2400 2401 // FIXME: This generated code sucks. 2402 ISD::CondCode LowCC; 2403 switch (CCCode) { 2404 default: llvm_unreachable("Unknown integer setcc!"); 2405 case ISD::SETLT: 2406 case ISD::SETULT: LowCC = ISD::SETULT; break; 2407 case ISD::SETGT: 2408 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 2409 case ISD::SETLE: 2410 case ISD::SETULE: LowCC = ISD::SETULE; break; 2411 case ISD::SETGE: 2412 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 2413 } 2414 2415 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 2416 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 2417 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 2418 2419 // NOTE: on targets without efficient SELECT of bools, we can always use 2420 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 2421 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, true, NULL); 2422 SDValue Tmp1, Tmp2; 2423 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), 2424 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); 2425 if (!Tmp1.getNode()) 2426 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), 2427 LHSLo, RHSLo, LowCC); 2428 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 2429 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); 2430 if (!Tmp2.getNode()) 2431 Tmp2 = DAG.getNode(ISD::SETCC, dl, 2432 TLI.getSetCCResultType(LHSHi.getValueType()), 2433 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 2434 2435 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 2436 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 2437 if ((Tmp1C && Tmp1C->isNullValue()) || 2438 (Tmp2C && Tmp2C->isNullValue() && 2439 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 2440 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 2441 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 2442 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 2443 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 2444 // low part is known false, returns high part. 2445 // For LE / GE, if high part is known false, ignore the low part. 2446 // For LT / GT, if high part is known true, ignore the low part. 2447 NewLHS = Tmp2; 2448 NewRHS = SDValue(); 2449 return; 2450 } 2451 2452 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 2453 LHSHi, RHSHi, ISD::SETEQ, false, 2454 DagCombineInfo, dl); 2455 if (!NewLHS.getNode()) 2456 NewLHS = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 2457 LHSHi, RHSHi, ISD::SETEQ); 2458 NewLHS = DAG.getNode(ISD::SELECT, dl, Tmp1.getValueType(), 2459 NewLHS, Tmp1, Tmp2); 2460 NewRHS = SDValue(); 2461} 2462 2463SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 2464 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 2465 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 2466 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2467 2468 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2469 // against zero to select between true and false values. 2470 if (NewRHS.getNode() == 0) { 2471 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2472 CCCode = ISD::SETNE; 2473 } 2474 2475 // Update N to have the operands specified. 2476 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2477 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2478 N->getOperand(4)), 0); 2479} 2480 2481SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 2482 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2483 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2484 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2485 2486 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2487 // against zero to select between true and false values. 2488 if (NewRHS.getNode() == 0) { 2489 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2490 CCCode = ISD::SETNE; 2491 } 2492 2493 // Update N to have the operands specified. 2494 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2495 N->getOperand(2), N->getOperand(3), 2496 DAG.getCondCode(CCCode)), 0); 2497} 2498 2499SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 2500 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2501 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2502 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2503 2504 // If ExpandSetCCOperands returned a scalar, use it. 2505 if (NewRHS.getNode() == 0) { 2506 assert(NewLHS.getValueType() == N->getValueType(0) && 2507 "Unexpected setcc expansion!"); 2508 return NewLHS; 2509 } 2510 2511 // Otherwise, update N to have the operands specified. 2512 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2513 DAG.getCondCode(CCCode)), 0); 2514} 2515 2516SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 2517 // The value being shifted is legal, but the shift amount is too big. 2518 // It follows that either the result of the shift is undefined, or the 2519 // upper half of the shift amount is zero. Just use the lower half. 2520 SDValue Lo, Hi; 2521 GetExpandedInteger(N->getOperand(1), Lo, Hi); 2522 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 2523} 2524 2525SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 2526 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 2527 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 2528 // constant to valid type. 2529 SDValue Lo, Hi; 2530 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2531 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 2532} 2533 2534SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 2535 SDValue Op = N->getOperand(0); 2536 EVT DstVT = N->getValueType(0); 2537 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 2538 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2539 "Don't know how to expand this SINT_TO_FP!"); 2540 return MakeLibCall(LC, DstVT, &Op, 1, true, N->getDebugLoc()); 2541} 2542 2543SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 2544 if (ISD::isNormalStore(N)) 2545 return ExpandOp_NormalStore(N, OpNo); 2546 2547 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2548 assert(OpNo == 1 && "Can only expand the stored value so far"); 2549 2550 EVT VT = N->getOperand(1).getValueType(); 2551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2552 SDValue Ch = N->getChain(); 2553 SDValue Ptr = N->getBasePtr(); 2554 unsigned Alignment = N->getAlignment(); 2555 bool isVolatile = N->isVolatile(); 2556 bool isNonTemporal = N->isNonTemporal(); 2557 DebugLoc dl = N->getDebugLoc(); 2558 SDValue Lo, Hi; 2559 2560 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2561 2562 if (N->getMemoryVT().bitsLE(NVT)) { 2563 GetExpandedInteger(N->getValue(), Lo, Hi); 2564 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 2565 N->getMemoryVT(), isVolatile, isNonTemporal, 2566 Alignment); 2567 } 2568 2569 if (TLI.isLittleEndian()) { 2570 // Little-endian - low bits are at low addresses. 2571 GetExpandedInteger(N->getValue(), Lo, Hi); 2572 2573 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 2574 isVolatile, isNonTemporal, Alignment); 2575 2576 unsigned ExcessBits = 2577 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2578 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2579 2580 // Increment the pointer to the other half. 2581 unsigned IncrementSize = NVT.getSizeInBits()/8; 2582 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2583 DAG.getIntPtrConstant(IncrementSize)); 2584 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 2585 N->getPointerInfo().getWithOffset(IncrementSize), 2586 NEVT, isVolatile, isNonTemporal, 2587 MinAlign(Alignment, IncrementSize)); 2588 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2589 } 2590 2591 // Big-endian - high bits are at low addresses. Favor aligned stores at 2592 // the cost of some bit-fiddling. 2593 GetExpandedInteger(N->getValue(), Lo, Hi); 2594 2595 EVT ExtVT = N->getMemoryVT(); 2596 unsigned EBytes = ExtVT.getStoreSize(); 2597 unsigned IncrementSize = NVT.getSizeInBits()/8; 2598 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2599 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 2600 ExtVT.getSizeInBits() - ExcessBits); 2601 2602 if (ExcessBits < NVT.getSizeInBits()) { 2603 // Transfer high bits from the top of Lo to the bottom of Hi. 2604 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 2605 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 2606 TLI.getPointerTy())); 2607 Hi = DAG.getNode(ISD::OR, dl, NVT, Hi, 2608 DAG.getNode(ISD::SRL, dl, NVT, Lo, 2609 DAG.getConstant(ExcessBits, 2610 TLI.getPointerTy()))); 2611 } 2612 2613 // Store both the high bits and maybe some of the low bits. 2614 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), 2615 HiVT, isVolatile, isNonTemporal, Alignment); 2616 2617 // Increment the pointer to the other half. 2618 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2619 DAG.getIntPtrConstant(IncrementSize)); 2620 // Store the lowest ExcessBits bits in the second half. 2621 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 2622 N->getPointerInfo().getWithOffset(IncrementSize), 2623 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2624 isVolatile, isNonTemporal, 2625 MinAlign(Alignment, IncrementSize)); 2626 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2627} 2628 2629SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 2630 SDValue InL, InH; 2631 GetExpandedInteger(N->getOperand(0), InL, InH); 2632 // Just truncate the low part of the source. 2633 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), InL); 2634} 2635 2636static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { 2637 switch (VT.getSimpleVT().SimpleTy) { 2638 default: llvm_unreachable("Unknown FP format"); 2639 case MVT::f32: return &APFloat::IEEEsingle; 2640 case MVT::f64: return &APFloat::IEEEdouble; 2641 case MVT::f80: return &APFloat::x87DoubleExtended; 2642 case MVT::f128: return &APFloat::IEEEquad; 2643 case MVT::ppcf128: return &APFloat::PPCDoubleDouble; 2644 } 2645} 2646 2647SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 2648 SDValue Op = N->getOperand(0); 2649 EVT SrcVT = Op.getValueType(); 2650 EVT DstVT = N->getValueType(0); 2651 DebugLoc dl = N->getDebugLoc(); 2652 2653 // The following optimization is valid only if every value in SrcVT (when 2654 // treated as signed) is representable in DstVT. Check that the mantissa 2655 // size of DstVT is >= than the number of bits in SrcVT -1. 2656 const fltSemantics *sem = EVTToAPFloatSemantics(DstVT); 2657 if (APFloat::semanticsPrecision(*sem) >= SrcVT.getSizeInBits()-1 && 2658 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 2659 // Do a signed conversion then adjust the result. 2660 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); 2661 SignedConv = TLI.LowerOperation(SignedConv, DAG); 2662 2663 // The result of the signed conversion needs adjusting if the 'sign bit' of 2664 // the incoming integer was set. To handle this, we dynamically test to see 2665 // if it is set, and, if so, add a fudge factor. 2666 2667 const uint64_t F32TwoE32 = 0x4F800000ULL; 2668 const uint64_t F32TwoE64 = 0x5F800000ULL; 2669 const uint64_t F32TwoE128 = 0x7F800000ULL; 2670 2671 APInt FF(32, 0); 2672 if (SrcVT == MVT::i32) 2673 FF = APInt(32, F32TwoE32); 2674 else if (SrcVT == MVT::i64) 2675 FF = APInt(32, F32TwoE64); 2676 else if (SrcVT == MVT::i128) 2677 FF = APInt(32, F32TwoE128); 2678 else 2679 assert(false && "Unsupported UINT_TO_FP!"); 2680 2681 // Check whether the sign bit is set. 2682 SDValue Lo, Hi; 2683 GetExpandedInteger(Op, Lo, Hi); 2684 SDValue SignSet = DAG.getSetCC(dl, 2685 TLI.getSetCCResultType(Hi.getValueType()), 2686 Hi, DAG.getConstant(0, Hi.getValueType()), 2687 ISD::SETLT); 2688 2689 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 2690 SDValue FudgePtr = DAG.getConstantPool( 2691 ConstantInt::get(*DAG.getContext(), FF.zext(64)), 2692 TLI.getPointerTy()); 2693 2694 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 2695 SDValue Zero = DAG.getIntPtrConstant(0); 2696 SDValue Four = DAG.getIntPtrConstant(4); 2697 if (TLI.isBigEndian()) std::swap(Zero, Four); 2698 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet, 2699 Zero, Four); 2700 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 2701 FudgePtr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), FudgePtr, Offset); 2702 Alignment = std::min(Alignment, 4u); 2703 2704 // Load the value out, extending it from f32 to the destination float type. 2705 // FIXME: Avoid the extend by constructing the right constant pool? 2706 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), 2707 FudgePtr, 2708 MachinePointerInfo::getConstantPool(), 2709 MVT::f32, 2710 false, false, Alignment); 2711 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); 2712 } 2713 2714 // Otherwise, use a libcall. 2715 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 2716 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2717 "Don't know how to expand this UINT_TO_FP!"); 2718 return MakeLibCall(LC, DstVT, &Op, 1, true, dl); 2719} 2720 2721SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 2722 SDValue InOp0 = N->getOperand(0); 2723 EVT InVT = InOp0.getValueType(); 2724 2725 EVT OutVT = N->getValueType(0); 2726 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2727 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2728 unsigned OutNumElems = OutVT.getVectorNumElements(); 2729 EVT NOutVTElem = NOutVT.getVectorElementType(); 2730 2731 DebugLoc dl = N->getDebugLoc(); 2732 SDValue BaseIdx = N->getOperand(1); 2733 2734 SmallVector<SDValue, 8> Ops; 2735 Ops.reserve(OutNumElems); 2736 for (unsigned i = 0; i != OutNumElems; ++i) { 2737 2738 // Extract the element from the original vector. 2739 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 2740 BaseIdx, DAG.getIntPtrConstant(i)); 2741 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 2742 InVT.getVectorElementType(), N->getOperand(0), Index); 2743 2744 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); 2745 // Insert the converted element to the new vector. 2746 Ops.push_back(Op); 2747 } 2748 2749 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size()); 2750} 2751 2752 2753SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 2754 2755 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 2756 EVT VT = N->getValueType(0); 2757 DebugLoc dl = N->getDebugLoc(); 2758 2759 unsigned NumElts = VT.getVectorNumElements(); 2760 SmallVector<int, 8> NewMask; 2761 for (unsigned i = 0; i != NumElts; ++i) { 2762 NewMask.push_back(SV->getMaskElt(i)); 2763 } 2764 2765 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 2766 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 2767 EVT OutVT = V0.getValueType(); 2768 2769 return DAG.getVectorShuffle(OutVT, dl, V0, V1, &NewMask[0]); 2770} 2771 2772 2773SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 2774 EVT OutVT = N->getValueType(0); 2775 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2776 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2777 unsigned NumElems = N->getNumOperands(); 2778 EVT NOutVTElem = NOutVT.getVectorElementType(); 2779 2780 DebugLoc dl = N->getDebugLoc(); 2781 2782 SmallVector<SDValue, 8> Ops; 2783 Ops.reserve(NumElems); 2784 for (unsigned i = 0; i != NumElems; ++i) { 2785 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 2786 Ops.push_back(Op); 2787 } 2788 2789 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size()); 2790} 2791 2792SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 2793 2794 DebugLoc dl = N->getDebugLoc(); 2795 2796 assert(!N->getOperand(0).getValueType().isVector() && 2797 "Input must be a scalar"); 2798 2799 EVT OutVT = N->getValueType(0); 2800 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2801 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2802 EVT NOutVTElem = NOutVT.getVectorElementType(); 2803 2804 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 2805 2806 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 2807} 2808 2809SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 2810 EVT OutVT = N->getValueType(0); 2811 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2812 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2813 2814 EVT NOutVTElem = NOutVT.getVectorElementType(); 2815 2816 DebugLoc dl = N->getDebugLoc(); 2817 2818 SDValue ConvertedVector = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 2819 N->getOperand(0)); 2820 2821 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 2822 NOutVTElem, N->getOperand(1)); 2823 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,NOutVT, 2824 ConvertedVector, ConvElem, N->getOperand(2)); 2825} 2826 2827SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2828 DebugLoc dl = N->getDebugLoc(); 2829 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 2830 SDValue V1 = N->getOperand(1); 2831 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 2832 V0->getValueType(0).getScalarType(), V0, V1); 2833 2834 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 2835 2836} 2837 2838SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 2839 2840 DebugLoc dl = N->getDebugLoc(); 2841 2842 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 2843 2844 SmallVector<SDValue, 8> NewOps; 2845 2846 // For each incoming vector 2847 for (unsigned VecIdx = 0, E = N->getNumOperands(); VecIdx!= E; ++VecIdx) { 2848 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 2849 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 2850 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 2851 2852 for (unsigned i=0; i<NumElem; ++i) { 2853 // Extract element from incoming vector 2854 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, 2855 Incoming, DAG.getIntPtrConstant(i)); 2856 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 2857 NewOps.push_back(Tr); 2858 } 2859 } 2860 2861 return DAG.getNode(ISD::BUILD_VECTOR, dl, N->getValueType(0), 2862 &NewOps[0], NewOps.size()); 2863 } 2864