LegalizeIntegerTypes.cpp revision cd81d94322a39503e4a3e87b6ee03d4fcb3465fb
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22#include "llvm/IR/DerivedTypes.h" 23#include "llvm/Support/ErrorHandling.h" 24#include "llvm/Support/raw_ostream.h" 25using namespace llvm; 26 27#define DEBUG_TYPE "legalize-types" 28 29//===----------------------------------------------------------------------===// 30// Integer Result Promotion 31//===----------------------------------------------------------------------===// 32 33/// PromoteIntegerResult - This method is called when a result of a node is 34/// found to be in need of promotion to a larger type. At this point, the node 35/// may also have invalid operands or may have other results that need 36/// expansion, we just know that (at least) one result needs promotion. 37void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 38 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); 39 SDValue Res = SDValue(); 40 41 // See if the target wants to custom expand this node. 42 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 43 return; 44 45 switch (N->getOpcode()) { 46 default: 47#ifndef NDEBUG 48 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 49 N->dump(&DAG); dbgs() << "\n"; 50#endif 51 llvm_unreachable("Do not know how to promote this operator!"); 52 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 53 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 54 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 55 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 56 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 57 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 58 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 59 case ISD::CONVERT_RNDSAT: 60 Res = PromoteIntRes_CONVERT_RNDSAT(N); break; 61 case ISD::CTLZ_ZERO_UNDEF: 62 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 63 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 64 case ISD::CTTZ_ZERO_UNDEF: 65 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 66 case ISD::EXTRACT_VECTOR_ELT: 67 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 68 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break; 69 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 70 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; 71 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 72 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 73 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 74 case ISD::SIGN_EXTEND_INREG: 75 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 76 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 77 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 78 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 79 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 80 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 81 82 case ISD::EXTRACT_SUBVECTOR: 83 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 84 case ISD::VECTOR_SHUFFLE: 85 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 86 case ISD::INSERT_VECTOR_ELT: 87 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 88 case ISD::BUILD_VECTOR: 89 Res = PromoteIntRes_BUILD_VECTOR(N); break; 90 case ISD::SCALAR_TO_VECTOR: 91 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 92 case ISD::CONCAT_VECTORS: 93 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 94 95 case ISD::SIGN_EXTEND: 96 case ISD::ZERO_EXTEND: 97 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 98 99 case ISD::FP_TO_SINT: 100 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 101 102 case ISD::FP32_TO_FP16:Res = PromoteIntRes_FP32_TO_FP16(N); break; 103 104 case ISD::AND: 105 case ISD::OR: 106 case ISD::XOR: 107 case ISD::ADD: 108 case ISD::SUB: 109 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 110 111 case ISD::SDIV: 112 case ISD::SREM: Res = PromoteIntRes_SDIV(N); break; 113 114 case ISD::UDIV: 115 case ISD::UREM: Res = PromoteIntRes_UDIV(N); break; 116 117 case ISD::SADDO: 118 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 119 case ISD::UADDO: 120 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 121 case ISD::SMULO: 122 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 123 124 case ISD::ATOMIC_LOAD: 125 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 126 127 case ISD::ATOMIC_LOAD_ADD: 128 case ISD::ATOMIC_LOAD_SUB: 129 case ISD::ATOMIC_LOAD_AND: 130 case ISD::ATOMIC_LOAD_OR: 131 case ISD::ATOMIC_LOAD_XOR: 132 case ISD::ATOMIC_LOAD_NAND: 133 case ISD::ATOMIC_LOAD_MIN: 134 case ISD::ATOMIC_LOAD_MAX: 135 case ISD::ATOMIC_LOAD_UMIN: 136 case ISD::ATOMIC_LOAD_UMAX: 137 case ISD::ATOMIC_SWAP: 138 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 139 140 case ISD::ATOMIC_CMP_SWAP: 141 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 142 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 143 break; 144 } 145 146 // If the result is null then the sub-method took care of registering it. 147 if (Res.getNode()) 148 SetPromotedInteger(SDValue(N, ResNo), Res); 149} 150 151SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 152 unsigned ResNo) { 153 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 154 return GetPromotedInteger(Op); 155} 156 157SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 158 // Sign-extend the new bits, and continue the assertion. 159 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 160 return DAG.getNode(ISD::AssertSext, SDLoc(N), 161 Op.getValueType(), Op, N->getOperand(1)); 162} 163 164SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 165 // Zero the new bits, and continue the assertion. 166 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 167 return DAG.getNode(ISD::AssertZext, SDLoc(N), 168 Op.getValueType(), Op, N->getOperand(1)); 169} 170 171SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 172 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 173 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 174 N->getMemoryVT(), ResVT, 175 N->getChain(), N->getBasePtr(), 176 N->getMemOperand(), N->getOrdering(), 177 N->getSynchScope()); 178 // Legalized the chain result - switch anything that used the old chain to 179 // use the new one. 180 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 181 return Res; 182} 183 184SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 185 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 186 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 187 N->getMemoryVT(), 188 N->getChain(), N->getBasePtr(), 189 Op2, N->getMemOperand(), N->getOrdering(), 190 N->getSynchScope()); 191 // Legalized the chain result - switch anything that used the old chain to 192 // use the new one. 193 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 194 return Res; 195} 196 197SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 198 unsigned ResNo) { 199 if (ResNo == 1) { 200 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 201 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 202 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 203 204 // Only use the result of getSetCCResultType if it is legal, 205 // otherwise just use the promoted result type (NVT). 206 if (!TLI.isTypeLegal(SVT)) 207 SVT = NVT; 208 209 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 210 SDValue Res = DAG.getAtomicCmpSwap( 211 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 212 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 213 N->getMemOperand(), N->getSuccessOrdering(), N->getFailureOrdering(), 214 N->getSynchScope()); 215 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 216 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 217 return Res.getValue(1); 218 } 219 220 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 221 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 222 SDVTList VTs = 223 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 224 SDValue Res = DAG.getAtomicCmpSwap( 225 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 226 N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(), 227 N->getFailureOrdering(), N->getSynchScope()); 228 // Legalized the chain result - switch anything that used the old chain to 229 // use the new one. 230 unsigned ChainOp = N->getNumValues() - 1; 231 ReplaceValueWith(SDValue(N, ChainOp), Res.getValue(ChainOp)); 232 return Res; 233} 234 235SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 236 SDValue InOp = N->getOperand(0); 237 EVT InVT = InOp.getValueType(); 238 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 239 EVT OutVT = N->getValueType(0); 240 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 241 SDLoc dl(N); 242 243 switch (getTypeAction(InVT)) { 244 case TargetLowering::TypeLegal: 245 break; 246 case TargetLowering::TypePromoteInteger: 247 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 248 // The input promotes to the same size. Convert the promoted value. 249 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 250 break; 251 case TargetLowering::TypeSoftenFloat: 252 // Promote the integer operand by hand. 253 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 254 case TargetLowering::TypeExpandInteger: 255 case TargetLowering::TypeExpandFloat: 256 break; 257 case TargetLowering::TypeScalarizeVector: 258 // Convert the element to an integer and promote it by hand. 259 if (!NOutVT.isVector()) 260 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 261 BitConvertToInteger(GetScalarizedVector(InOp))); 262 break; 263 case TargetLowering::TypeSplitVector: { 264 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 265 // pieces of the input into integers and reassemble in the final type. 266 SDValue Lo, Hi; 267 GetSplitVector(N->getOperand(0), Lo, Hi); 268 Lo = BitConvertToInteger(Lo); 269 Hi = BitConvertToInteger(Hi); 270 271 if (TLI.isBigEndian()) 272 std::swap(Lo, Hi); 273 274 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 275 EVT::getIntegerVT(*DAG.getContext(), 276 NOutVT.getSizeInBits()), 277 JoinIntegers(Lo, Hi)); 278 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 279 } 280 case TargetLowering::TypeWidenVector: 281 // The input is widened to the same size. Convert to the widened value. 282 // Make sure that the outgoing value is not a vector, because this would 283 // make us bitcast between two vectors which are legalized in different ways. 284 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) 285 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 286 } 287 288 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 289 CreateStackStoreLoad(InOp, OutVT)); 290} 291 292SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 293 SDValue Op = GetPromotedInteger(N->getOperand(0)); 294 EVT OVT = N->getValueType(0); 295 EVT NVT = Op.getValueType(); 296 SDLoc dl(N); 297 298 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 299 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 300 DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT))); 301} 302 303SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 304 // The pair element type may be legal, or may not promote to the same type as 305 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 306 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 307 TLI.getTypeToTransformTo(*DAG.getContext(), 308 N->getValueType(0)), JoinIntegers(N->getOperand(0), 309 N->getOperand(1))); 310} 311 312SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 313 EVT VT = N->getValueType(0); 314 // FIXME there is no actual debug info here 315 SDLoc dl(N); 316 // Zero extend things like i1, sign extend everything else. It shouldn't 317 // matter in theory which one we pick, but this tends to give better code? 318 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 319 SDValue Result = DAG.getNode(Opc, dl, 320 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 321 SDValue(N, 0)); 322 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 323 return Result; 324} 325 326SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) { 327 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 328 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 329 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 330 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 331 "can only promote integers"); 332 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 333 return DAG.getConvertRndSat(OutVT, SDLoc(N), N->getOperand(0), 334 N->getOperand(1), N->getOperand(2), 335 N->getOperand(3), N->getOperand(4), CvtCode); 336} 337 338SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 339 // Zero extend to the promoted type and do the count there. 340 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 341 SDLoc dl(N); 342 EVT OVT = N->getValueType(0); 343 EVT NVT = Op.getValueType(); 344 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 345 // Subtract off the extra leading bits in the bigger type. 346 return DAG.getNode(ISD::SUB, dl, NVT, Op, 347 DAG.getConstant(NVT.getSizeInBits() - 348 OVT.getSizeInBits(), NVT)); 349} 350 351SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 352 // Zero extend to the promoted type and do the count there. 353 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 354 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op); 355} 356 357SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 358 SDValue Op = GetPromotedInteger(N->getOperand(0)); 359 EVT OVT = N->getValueType(0); 360 EVT NVT = Op.getValueType(); 361 SDLoc dl(N); 362 if (N->getOpcode() == ISD::CTTZ) { 363 // The count is the same in the promoted type except if the original 364 // value was zero. This can be handled by setting the bit just off 365 // the top of the original type. 366 APInt TopBit(NVT.getSizeInBits(), 0); 367 TopBit.setBit(OVT.getSizeInBits()); 368 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT)); 369 } 370 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 371} 372 373SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 374 SDLoc dl(N); 375 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 376 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), 377 N->getOperand(1)); 378} 379 380SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 381 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 382 unsigned NewOpc = N->getOpcode(); 383 SDLoc dl(N); 384 385 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 386 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 387 // and SINT conversions are Custom, there is no way to tell which is 388 // preferable. We choose SINT because that's the right thing on PPC.) 389 if (N->getOpcode() == ISD::FP_TO_UINT && 390 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 391 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 392 NewOpc = ISD::FP_TO_SINT; 393 394 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 395 396 // Assert that the converted value fits in the original type. If it doesn't 397 // (eg: because the value being converted is too big), then the result of the 398 // original operation was undefined anyway, so the assert is still correct. 399 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 400 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 401 DAG.getValueType(N->getValueType(0).getScalarType())); 402} 403 404SDValue DAGTypeLegalizer::PromoteIntRes_FP32_TO_FP16(SDNode *N) { 405 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 406 SDLoc dl(N); 407 408 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 409 410 return DAG.getNode(ISD::AssertZext, dl, 411 NVT, Res, DAG.getValueType(N->getValueType(0))); 412} 413 414SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 415 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 416 SDLoc dl(N); 417 418 if (getTypeAction(N->getOperand(0).getValueType()) 419 == TargetLowering::TypePromoteInteger) { 420 SDValue Res = GetPromotedInteger(N->getOperand(0)); 421 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 422 423 // If the result and operand types are the same after promotion, simplify 424 // to an in-register extension. 425 if (NVT == Res.getValueType()) { 426 // The high bits are not guaranteed to be anything. Insert an extend. 427 if (N->getOpcode() == ISD::SIGN_EXTEND) 428 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 429 DAG.getValueType(N->getOperand(0).getValueType())); 430 if (N->getOpcode() == ISD::ZERO_EXTEND) 431 return DAG.getZeroExtendInReg(Res, dl, 432 N->getOperand(0).getValueType().getScalarType()); 433 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 434 return Res; 435 } 436 } 437 438 // Otherwise, just extend the original operand all the way to the larger type. 439 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 440} 441 442SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 443 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 444 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 445 ISD::LoadExtType ExtType = 446 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 447 SDLoc dl(N); 448 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 449 N->getMemoryVT(), N->getMemOperand()); 450 451 // Legalized the chain result - switch anything that used the old chain to 452 // use the new one. 453 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 454 return Res; 455} 456 457/// Promote the overflow flag of an overflowing arithmetic node. 458SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 459 // Simply change the return type of the boolean result. 460 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 461 EVT ValueVTs[] = { N->getValueType(0), NVT }; 462 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) }; 463 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), 464 DAG.getVTList(ValueVTs), Ops); 465 466 // Modified the sum result - switch anything that used the old sum to use 467 // the new one. 468 ReplaceValueWith(SDValue(N, 0), Res); 469 470 return SDValue(Res.getNode(), 1); 471} 472 473SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 474 if (ResNo == 1) 475 return PromoteIntRes_Overflow(N); 476 477 // The operation overflowed iff the result in the larger type is not the 478 // sign extension of its truncation to the original type. 479 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 480 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 481 EVT OVT = N->getOperand(0).getValueType(); 482 EVT NVT = LHS.getValueType(); 483 SDLoc dl(N); 484 485 // Do the arithmetic in the larger type. 486 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 487 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 488 489 // Calculate the overflow flag: sign extend the arithmetic result from 490 // the original type. 491 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 492 DAG.getValueType(OVT)); 493 // Overflowed if and only if this is not equal to Res. 494 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 495 496 // Use the calculated overflow everywhere. 497 ReplaceValueWith(SDValue(N, 1), Ofl); 498 499 return Res; 500} 501 502SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) { 503 // Sign extend the input. 504 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 505 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 506 return DAG.getNode(N->getOpcode(), SDLoc(N), 507 LHS.getValueType(), LHS, RHS); 508} 509 510SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 511 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 512 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 513 return DAG.getSelect(SDLoc(N), 514 LHS.getValueType(), N->getOperand(0), LHS, RHS); 515} 516 517SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { 518 SDValue Mask = N->getOperand(0); 519 EVT OpTy = N->getOperand(1).getValueType(); 520 521 // Promote all the way up to the canonical SetCC type. 522 Mask = PromoteTargetBoolean(Mask, OpTy); 523 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 524 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 525 return DAG.getNode(ISD::VSELECT, SDLoc(N), 526 LHS.getValueType(), Mask, LHS, RHS); 527} 528 529SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 530 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 531 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 532 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 533 LHS.getValueType(), N->getOperand(0), 534 N->getOperand(1), LHS, RHS, N->getOperand(4)); 535} 536 537SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 538 EVT SVT = getSetCCResultType(N->getOperand(0).getValueType()); 539 540 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 541 542 // Only use the result of getSetCCResultType if it is legal, 543 // otherwise just use the promoted result type (NVT). 544 if (!TLI.isTypeLegal(SVT)) 545 SVT = NVT; 546 547 SDLoc dl(N); 548 assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() && 549 "Vector compare must return a vector result!"); 550 551 SDValue LHS = N->getOperand(0); 552 SDValue RHS = N->getOperand(1); 553 if (LHS.getValueType() != RHS.getValueType()) { 554 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger && 555 !LHS.getValueType().isVector()) 556 LHS = GetPromotedInteger(LHS); 557 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger && 558 !RHS.getValueType().isVector()) 559 RHS = GetPromotedInteger(RHS); 560 } 561 562 // Get the SETCC result using the canonical SETCC type. 563 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS, 564 N->getOperand(2)); 565 566 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?"); 567 // Convert to the expected type. 568 return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC); 569} 570 571SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 572 SDValue Res = GetPromotedInteger(N->getOperand(0)); 573 SDValue Amt = N->getOperand(1); 574 Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt; 575 return DAG.getNode(ISD::SHL, SDLoc(N), Res.getValueType(), Res, Amt); 576} 577 578SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 579 SDValue Op = GetPromotedInteger(N->getOperand(0)); 580 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 581 Op.getValueType(), Op, N->getOperand(1)); 582} 583 584SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 585 // The input may have strange things in the top bits of the registers, but 586 // these operations don't care. They may have weird bits going out, but 587 // that too is okay if they are integer operations. 588 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 589 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 590 return DAG.getNode(N->getOpcode(), SDLoc(N), 591 LHS.getValueType(), LHS, RHS); 592} 593 594SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 595 // The input value must be properly sign extended. 596 SDValue Res = SExtPromotedInteger(N->getOperand(0)); 597 SDValue Amt = N->getOperand(1); 598 Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt; 599 return DAG.getNode(ISD::SRA, SDLoc(N), Res.getValueType(), Res, Amt); 600} 601 602SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 603 // The input value must be properly zero extended. 604 SDValue Res = ZExtPromotedInteger(N->getOperand(0)); 605 SDValue Amt = N->getOperand(1); 606 Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt; 607 return DAG.getNode(ISD::SRL, SDLoc(N), Res.getValueType(), Res, Amt); 608} 609 610SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 611 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 612 SDValue Res; 613 SDValue InOp = N->getOperand(0); 614 SDLoc dl(N); 615 616 switch (getTypeAction(InOp.getValueType())) { 617 default: llvm_unreachable("Unknown type action!"); 618 case TargetLowering::TypeLegal: 619 case TargetLowering::TypeExpandInteger: 620 Res = InOp; 621 break; 622 case TargetLowering::TypePromoteInteger: 623 Res = GetPromotedInteger(InOp); 624 break; 625 case TargetLowering::TypeSplitVector: 626 EVT InVT = InOp.getValueType(); 627 assert(InVT.isVector() && "Cannot split scalar types"); 628 unsigned NumElts = InVT.getVectorNumElements(); 629 assert(NumElts == NVT.getVectorNumElements() && 630 "Dst and Src must have the same number of elements"); 631 assert(isPowerOf2_32(NumElts) && 632 "Promoted vector type must be a power of two"); 633 634 SDValue EOp1, EOp2; 635 GetSplitVector(InOp, EOp1, EOp2); 636 637 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 638 NumElts/2); 639 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 640 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 641 642 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 643 } 644 645 // Truncate to NVT instead of VT 646 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 647} 648 649SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 650 if (ResNo == 1) 651 return PromoteIntRes_Overflow(N); 652 653 // The operation overflowed iff the result in the larger type is not the 654 // zero extension of its truncation to the original type. 655 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 656 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 657 EVT OVT = N->getOperand(0).getValueType(); 658 EVT NVT = LHS.getValueType(); 659 SDLoc dl(N); 660 661 // Do the arithmetic in the larger type. 662 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 663 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 664 665 // Calculate the overflow flag: zero extend the arithmetic result from 666 // the original type. 667 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 668 // Overflowed if and only if this is not equal to Res. 669 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 670 671 // Use the calculated overflow everywhere. 672 ReplaceValueWith(SDValue(N, 1), Ofl); 673 674 return Res; 675} 676 677SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 678 // Promote the overflow bit trivially. 679 if (ResNo == 1) 680 return PromoteIntRes_Overflow(N); 681 682 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 683 SDLoc DL(N); 684 EVT SmallVT = LHS.getValueType(); 685 686 // To determine if the result overflowed in a larger type, we extend the 687 // input to the larger type, do the multiply (checking if it overflows), 688 // then also check the high bits of the result to see if overflow happened 689 // there. 690 if (N->getOpcode() == ISD::SMULO) { 691 LHS = SExtPromotedInteger(LHS); 692 RHS = SExtPromotedInteger(RHS); 693 } else { 694 LHS = ZExtPromotedInteger(LHS); 695 RHS = ZExtPromotedInteger(RHS); 696 } 697 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 698 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 699 700 // Overflow occurred if it occurred in the larger type, or if the high part 701 // of the result does not zero/sign-extend the low part. Check this second 702 // possibility first. 703 SDValue Overflow; 704 if (N->getOpcode() == ISD::UMULO) { 705 // Unsigned overflow occurred if the high part is non-zero. 706 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 707 DAG.getIntPtrConstant(SmallVT.getSizeInBits())); 708 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 709 DAG.getConstant(0, Hi.getValueType()), ISD::SETNE); 710 } else { 711 // Signed overflow occurred if the high part does not sign extend the low. 712 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 713 Mul, DAG.getValueType(SmallVT)); 714 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 715 } 716 717 // The only other way for overflow to occur is if the multiplication in the 718 // larger type itself overflowed. 719 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 720 SDValue(Mul.getNode(), 1)); 721 722 // Use the calculated overflow everywhere. 723 ReplaceValueWith(SDValue(N, 1), Overflow); 724 return Mul; 725} 726 727SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { 728 // Zero extend the input. 729 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 730 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 731 return DAG.getNode(N->getOpcode(), SDLoc(N), 732 LHS.getValueType(), LHS, RHS); 733} 734 735SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 736 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 737 N->getValueType(0))); 738} 739 740SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 741 SDValue Chain = N->getOperand(0); // Get the chain. 742 SDValue Ptr = N->getOperand(1); // Get the pointer. 743 EVT VT = N->getValueType(0); 744 SDLoc dl(N); 745 746 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 747 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 748 // The argument is passed as NumRegs registers of type RegVT. 749 750 SmallVector<SDValue, 8> Parts(NumRegs); 751 for (unsigned i = 0; i < NumRegs; ++i) { 752 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 753 N->getConstantOperandVal(3)); 754 Chain = Parts[i].getValue(1); 755 } 756 757 // Handle endianness of the load. 758 if (TLI.isBigEndian()) 759 std::reverse(Parts.begin(), Parts.end()); 760 761 // Assemble the parts in the promoted type. 762 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 763 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 764 for (unsigned i = 1; i < NumRegs; ++i) { 765 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 766 // Shift it to the right position and "or" it in. 767 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 768 DAG.getConstant(i * RegVT.getSizeInBits(), 769 TLI.getPointerTy())); 770 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 771 } 772 773 // Modified the chain result - switch anything that used the old chain to 774 // use the new one. 775 ReplaceValueWith(SDValue(N, 1), Chain); 776 777 return Res; 778} 779 780//===----------------------------------------------------------------------===// 781// Integer Operand Promotion 782//===----------------------------------------------------------------------===// 783 784/// PromoteIntegerOperand - This method is called when the specified operand of 785/// the specified node is found to need promotion. At this point, all of the 786/// result types of the node are known to be legal, but other operands of the 787/// node may need promotion or expansion as well as the specified one. 788bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 789 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); 790 SDValue Res = SDValue(); 791 792 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 793 return false; 794 795 switch (N->getOpcode()) { 796 default: 797 #ifndef NDEBUG 798 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 799 N->dump(&DAG); dbgs() << "\n"; 800 #endif 801 llvm_unreachable("Do not know how to promote this operator's operand!"); 802 803 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 804 case ISD::ATOMIC_STORE: 805 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 806 break; 807 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 808 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 809 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 810 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 811 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 812 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 813 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 814 case ISD::CONVERT_RNDSAT: 815 Res = PromoteIntOp_CONVERT_RNDSAT(N); break; 816 case ISD::INSERT_VECTOR_ELT: 817 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 818 case ISD::SCALAR_TO_VECTOR: 819 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 820 case ISD::VSELECT: 821 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 822 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 823 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 824 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 825 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 826 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 827 OpNo); break; 828 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 829 case ISD::FP16_TO_FP32: 830 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 831 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 832 833 case ISD::SHL: 834 case ISD::SRA: 835 case ISD::SRL: 836 case ISD::ROTL: 837 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 838 } 839 840 // If the result is null, the sub-method took care of registering results etc. 841 if (!Res.getNode()) return false; 842 843 // If the result is N, the sub-method updated N in place. Tell the legalizer 844 // core about this. 845 if (Res.getNode() == N) 846 return true; 847 848 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 849 "Invalid operand expansion"); 850 851 ReplaceValueWith(SDValue(N, 0), Res); 852 return false; 853} 854 855/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 856/// shared among BR_CC, SELECT_CC, and SETCC handlers. 857void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 858 ISD::CondCode CCCode) { 859 // We have to insert explicit sign or zero extends. Note that we could 860 // insert sign extends for ALL conditions, but zero extend is cheaper on 861 // many machines (an AND instead of two shifts), so prefer it. 862 switch (CCCode) { 863 default: llvm_unreachable("Unknown integer comparison!"); 864 case ISD::SETEQ: 865 case ISD::SETNE: 866 case ISD::SETUGE: 867 case ISD::SETUGT: 868 case ISD::SETULE: 869 case ISD::SETULT: 870 // ALL of these operations will work if we either sign or zero extend 871 // the operands (including the unsigned comparisons!). Zero extend is 872 // usually a simpler/cheaper operation, so prefer it. 873 NewLHS = ZExtPromotedInteger(NewLHS); 874 NewRHS = ZExtPromotedInteger(NewRHS); 875 break; 876 case ISD::SETGE: 877 case ISD::SETGT: 878 case ISD::SETLT: 879 case ISD::SETLE: 880 NewLHS = SExtPromotedInteger(NewLHS); 881 NewRHS = SExtPromotedInteger(NewRHS); 882 break; 883 } 884} 885 886SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 887 SDValue Op = GetPromotedInteger(N->getOperand(0)); 888 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 889} 890 891SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 892 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 893 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 894 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(), 895 N->getOrdering(), N->getSynchScope()); 896} 897 898SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 899 // This should only occur in unusual situations like bitcasting to an 900 // x86_fp80, so just turn it into a store+load 901 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 902} 903 904SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 905 assert(OpNo == 2 && "Don't know how to promote this operand!"); 906 907 SDValue LHS = N->getOperand(2); 908 SDValue RHS = N->getOperand(3); 909 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 910 911 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 912 // legal types. 913 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 914 N->getOperand(1), LHS, RHS, N->getOperand(4)), 915 0); 916} 917 918SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 919 assert(OpNo == 1 && "only know how to promote condition"); 920 921 // Promote all the way up to the canonical SetCC type. 922 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 923 924 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 925 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 926 N->getOperand(2)), 0); 927} 928 929SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 930 // Since the result type is legal, the operands must promote to it. 931 EVT OVT = N->getOperand(0).getValueType(); 932 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 933 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 934 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 935 SDLoc dl(N); 936 937 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 938 DAG.getConstant(OVT.getSizeInBits(), TLI.getPointerTy())); 939 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 940} 941 942SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 943 // The vector type is legal but the element type is not. This implies 944 // that the vector is a power-of-two in length and that the element 945 // type does not have a strange size (eg: it is not i1). 946 EVT VecVT = N->getValueType(0); 947 unsigned NumElts = VecVT.getVectorNumElements(); 948 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 949 "Legal vector of one illegal element?"); 950 951 // Promote the inserted value. The type does not need to match the 952 // vector element type. Check that any extra bits introduced will be 953 // truncated away. 954 assert(N->getOperand(0).getValueType().getSizeInBits() >= 955 N->getValueType(0).getVectorElementType().getSizeInBits() && 956 "Type of inserted value narrower than vector element type!"); 957 958 SmallVector<SDValue, 16> NewOps; 959 for (unsigned i = 0; i < NumElts; ++i) 960 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 961 962 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 963} 964 965SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) { 966 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 967 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 968 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 969 CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) && 970 "can only promote integer arguments"); 971 SDValue InOp = GetPromotedInteger(N->getOperand(0)); 972 return DAG.getConvertRndSat(N->getValueType(0), SDLoc(N), InOp, 973 N->getOperand(1), N->getOperand(2), 974 N->getOperand(3), N->getOperand(4), CvtCode); 975} 976 977SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 978 unsigned OpNo) { 979 if (OpNo == 1) { 980 // Promote the inserted value. This is valid because the type does not 981 // have to match the vector element type. 982 983 // Check that any extra bits introduced will be truncated away. 984 assert(N->getOperand(1).getValueType().getSizeInBits() >= 985 N->getValueType(0).getVectorElementType().getSizeInBits() && 986 "Type of inserted value narrower than vector element type!"); 987 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 988 GetPromotedInteger(N->getOperand(1)), 989 N->getOperand(2)), 990 0); 991 } 992 993 assert(OpNo == 2 && "Different operand and result vector types?"); 994 995 // Promote the index. 996 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 997 TLI.getVectorIdxTy()); 998 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 999 N->getOperand(1), Idx), 0); 1000} 1001 1002SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 1003 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 1004 // the operand in place. 1005 return SDValue(DAG.UpdateNodeOperands(N, 1006 GetPromotedInteger(N->getOperand(0))), 0); 1007} 1008 1009SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1010 assert(OpNo == 0 && "Only know how to promote the condition!"); 1011 SDValue Cond = N->getOperand(0); 1012 EVT OpTy = N->getOperand(1).getValueType(); 1013 1014 // Promote all the way up to the canonical SetCC type. 1015 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1016 Cond = PromoteTargetBoolean(Cond, OpVT); 1017 1018 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1019 N->getOperand(2)), 0); 1020} 1021 1022SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1023 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1024 1025 SDValue LHS = N->getOperand(0); 1026 SDValue RHS = N->getOperand(1); 1027 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1028 1029 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1030 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1031 N->getOperand(3), N->getOperand(4)), 0); 1032} 1033 1034SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1035 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1036 1037 SDValue LHS = N->getOperand(0); 1038 SDValue RHS = N->getOperand(1); 1039 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1040 1041 // The CC (#2) is always legal. 1042 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1043} 1044 1045SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1046 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1047 ZExtPromotedInteger(N->getOperand(1))), 0); 1048} 1049 1050SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1051 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1052 SDLoc dl(N); 1053 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1054 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1055 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1056} 1057 1058SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 1059 return SDValue(DAG.UpdateNodeOperands(N, 1060 SExtPromotedInteger(N->getOperand(0))), 0); 1061} 1062 1063SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 1064 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1065 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 1066 SDLoc dl(N); 1067 1068 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 1069 1070 // Truncate the value and store the result. 1071 return DAG.getTruncStore(Ch, dl, Val, Ptr, 1072 N->getMemoryVT(), N->getMemOperand()); 1073} 1074 1075SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 1076 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1077 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 1078} 1079 1080SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 1081 return SDValue(DAG.UpdateNodeOperands(N, 1082 ZExtPromotedInteger(N->getOperand(0))), 0); 1083} 1084 1085SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1086 SDLoc dl(N); 1087 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1088 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1089 return DAG.getZeroExtendInReg(Op, dl, 1090 N->getOperand(0).getValueType().getScalarType()); 1091} 1092 1093 1094//===----------------------------------------------------------------------===// 1095// Integer Result Expansion 1096//===----------------------------------------------------------------------===// 1097 1098/// ExpandIntegerResult - This method is called when the specified result of the 1099/// specified node is found to need expansion. At this point, the node may also 1100/// have invalid operands or may have other results that need promotion, we just 1101/// know that (at least) one result needs expansion. 1102void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 1103 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); 1104 SDValue Lo, Hi; 1105 Lo = Hi = SDValue(); 1106 1107 // See if the target wants to custom expand this node. 1108 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1109 return; 1110 1111 switch (N->getOpcode()) { 1112 default: 1113#ifndef NDEBUG 1114 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 1115 N->dump(&DAG); dbgs() << "\n"; 1116#endif 1117 llvm_unreachable("Do not know how to expand the result of this operator!"); 1118 1119 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1120 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1121 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1122 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1123 1124 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1125 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1126 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1127 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1128 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1129 1130 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 1131 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 1132 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 1133 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 1134 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 1135 case ISD::CTLZ_ZERO_UNDEF: 1136 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 1137 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 1138 case ISD::CTTZ_ZERO_UNDEF: 1139 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 1140 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 1141 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 1142 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 1143 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 1144 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 1145 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 1146 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 1147 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 1148 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 1149 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 1150 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 1151 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 1152 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 1153 1154 case ISD::ATOMIC_LOAD_ADD: 1155 case ISD::ATOMIC_LOAD_SUB: 1156 case ISD::ATOMIC_LOAD_AND: 1157 case ISD::ATOMIC_LOAD_OR: 1158 case ISD::ATOMIC_LOAD_XOR: 1159 case ISD::ATOMIC_LOAD_NAND: 1160 case ISD::ATOMIC_LOAD_MIN: 1161 case ISD::ATOMIC_LOAD_MAX: 1162 case ISD::ATOMIC_LOAD_UMIN: 1163 case ISD::ATOMIC_LOAD_UMAX: 1164 case ISD::ATOMIC_SWAP: 1165 case ISD::ATOMIC_CMP_SWAP: { 1166 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 1167 SplitInteger(Tmp.first, Lo, Hi); 1168 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1169 break; 1170 } 1171 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 1172 AtomicSDNode *AN = cast<AtomicSDNode>(N); 1173 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 1174 SDValue Tmp = DAG.getAtomicCmpSwap( 1175 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 1176 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 1177 AN->getMemOperand(), AN->getSuccessOrdering(), AN->getFailureOrdering(), 1178 AN->getSynchScope()); 1179 1180 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 1181 // success simply by comparing the loaded value against the ingoing 1182 // comparison. 1183 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 1184 N->getOperand(2), ISD::SETEQ); 1185 1186 SplitInteger(Tmp, Lo, Hi); 1187 ReplaceValueWith(SDValue(N, 1), Success); 1188 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 1189 break; 1190 } 1191 1192 case ISD::AND: 1193 case ISD::OR: 1194 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 1195 1196 case ISD::ADD: 1197 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 1198 1199 case ISD::ADDC: 1200 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 1201 1202 case ISD::ADDE: 1203 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 1204 1205 case ISD::SHL: 1206 case ISD::SRA: 1207 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 1208 1209 case ISD::SADDO: 1210 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 1211 case ISD::UADDO: 1212 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 1213 case ISD::UMULO: 1214 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 1215 } 1216 1217 // If Lo/Hi is null, the sub-method took care of registering results etc. 1218 if (Lo.getNode()) 1219 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 1220} 1221 1222/// Lower an atomic node to the appropriate builtin call. 1223std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 1224 unsigned Opc = Node->getOpcode(); 1225 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 1226 RTLIB::Libcall LC; 1227 1228 switch (Opc) { 1229 default: 1230 llvm_unreachable("Unhandled atomic intrinsic Expand!"); 1231 case ISD::ATOMIC_SWAP: 1232 switch (VT.SimpleTy) { 1233 default: llvm_unreachable("Unexpected value type for atomic!"); 1234 case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 1235 case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 1236 case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 1237 case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 1238 case MVT::i128:LC = RTLIB::SYNC_LOCK_TEST_AND_SET_16;break; 1239 } 1240 break; 1241 case ISD::ATOMIC_CMP_SWAP: 1242 switch (VT.SimpleTy) { 1243 default: llvm_unreachable("Unexpected value type for atomic!"); 1244 case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 1245 case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 1246 case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 1247 case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 1248 case MVT::i128:LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_16;break; 1249 } 1250 break; 1251 case ISD::ATOMIC_LOAD_ADD: 1252 switch (VT.SimpleTy) { 1253 default: llvm_unreachable("Unexpected value type for atomic!"); 1254 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 1255 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 1256 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 1257 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 1258 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_ADD_16;break; 1259 } 1260 break; 1261 case ISD::ATOMIC_LOAD_SUB: 1262 switch (VT.SimpleTy) { 1263 default: llvm_unreachable("Unexpected value type for atomic!"); 1264 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 1265 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 1266 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 1267 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 1268 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_SUB_16;break; 1269 } 1270 break; 1271 case ISD::ATOMIC_LOAD_AND: 1272 switch (VT.SimpleTy) { 1273 default: llvm_unreachable("Unexpected value type for atomic!"); 1274 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 1275 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 1276 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 1277 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 1278 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_AND_16;break; 1279 } 1280 break; 1281 case ISD::ATOMIC_LOAD_OR: 1282 switch (VT.SimpleTy) { 1283 default: llvm_unreachable("Unexpected value type for atomic!"); 1284 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 1285 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 1286 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 1287 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 1288 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_OR_16;break; 1289 } 1290 break; 1291 case ISD::ATOMIC_LOAD_XOR: 1292 switch (VT.SimpleTy) { 1293 default: llvm_unreachable("Unexpected value type for atomic!"); 1294 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 1295 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 1296 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 1297 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 1298 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_XOR_16;break; 1299 } 1300 break; 1301 case ISD::ATOMIC_LOAD_NAND: 1302 switch (VT.SimpleTy) { 1303 default: llvm_unreachable("Unexpected value type for atomic!"); 1304 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 1305 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 1306 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 1307 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 1308 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_NAND_16;break; 1309 } 1310 break; 1311 } 1312 1313 return ExpandChainLibCall(LC, Node, false); 1314} 1315 1316/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded, 1317/// and the shift amount is a constant 'Amt'. Expand the operation. 1318void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 1319 SDValue &Lo, SDValue &Hi) { 1320 assert(Amt && "Expected zero shifts to be already optimized away."); 1321 SDLoc DL(N); 1322 // Expand the incoming operand to be shifted, so that we have its parts 1323 SDValue InL, InH; 1324 GetExpandedInteger(N->getOperand(0), InL, InH); 1325 1326 EVT NVT = InL.getValueType(); 1327 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1328 unsigned NVTBits = NVT.getSizeInBits(); 1329 EVT ShTy = N->getOperand(1).getValueType(); 1330 1331 if (N->getOpcode() == ISD::SHL) { 1332 if (Amt > VTBits) { 1333 Lo = Hi = DAG.getConstant(0, NVT); 1334 } else if (Amt > NVTBits) { 1335 Lo = DAG.getConstant(0, NVT); 1336 Hi = DAG.getNode(ISD::SHL, DL, 1337 NVT, InL, DAG.getConstant(Amt-NVTBits, ShTy)); 1338 } else if (Amt == NVTBits) { 1339 Lo = DAG.getConstant(0, NVT); 1340 Hi = InL; 1341 } else if (Amt == 1 && 1342 TLI.isOperationLegalOrCustom(ISD::ADDC, 1343 TLI.getTypeToExpandTo(*DAG.getContext(), NVT))) { 1344 // Emit this X << 1 as X+X. 1345 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1346 SDValue LoOps[2] = { InL, InL }; 1347 Lo = DAG.getNode(ISD::ADDC, DL, VTList, LoOps); 1348 SDValue HiOps[3] = { InH, InH, Lo.getValue(1) }; 1349 Hi = DAG.getNode(ISD::ADDE, DL, VTList, HiOps); 1350 } else { 1351 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, ShTy)); 1352 Hi = DAG.getNode(ISD::OR, DL, NVT, 1353 DAG.getNode(ISD::SHL, DL, NVT, InH, 1354 DAG.getConstant(Amt, ShTy)), 1355 DAG.getNode(ISD::SRL, DL, NVT, InL, 1356 DAG.getConstant(NVTBits-Amt, ShTy))); 1357 } 1358 return; 1359 } 1360 1361 if (N->getOpcode() == ISD::SRL) { 1362 if (Amt > VTBits) { 1363 Lo = DAG.getConstant(0, NVT); 1364 Hi = DAG.getConstant(0, NVT); 1365 } else if (Amt > NVTBits) { 1366 Lo = DAG.getNode(ISD::SRL, DL, 1367 NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy)); 1368 Hi = DAG.getConstant(0, NVT); 1369 } else if (Amt == NVTBits) { 1370 Lo = InH; 1371 Hi = DAG.getConstant(0, NVT); 1372 } else { 1373 Lo = DAG.getNode(ISD::OR, DL, NVT, 1374 DAG.getNode(ISD::SRL, DL, NVT, InL, 1375 DAG.getConstant(Amt, ShTy)), 1376 DAG.getNode(ISD::SHL, DL, NVT, InH, 1377 DAG.getConstant(NVTBits-Amt, ShTy))); 1378 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1379 } 1380 return; 1381 } 1382 1383 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1384 if (Amt > VTBits) { 1385 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1386 DAG.getConstant(NVTBits-1, ShTy)); 1387 } else if (Amt > NVTBits) { 1388 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1389 DAG.getConstant(Amt-NVTBits, ShTy)); 1390 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1391 DAG.getConstant(NVTBits-1, ShTy)); 1392 } else if (Amt == NVTBits) { 1393 Lo = InH; 1394 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1395 DAG.getConstant(NVTBits-1, ShTy)); 1396 } else { 1397 Lo = DAG.getNode(ISD::OR, DL, NVT, 1398 DAG.getNode(ISD::SRL, DL, NVT, InL, 1399 DAG.getConstant(Amt, ShTy)), 1400 DAG.getNode(ISD::SHL, DL, NVT, InH, 1401 DAG.getConstant(NVTBits-Amt, ShTy))); 1402 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1403 } 1404} 1405 1406/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1407/// this shift based on knowledge of the high bit of the shift amount. If we 1408/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1409/// shift amount. 1410bool DAGTypeLegalizer:: 1411ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1412 SDValue Amt = N->getOperand(1); 1413 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1414 EVT ShTy = Amt.getValueType(); 1415 unsigned ShBits = ShTy.getScalarType().getSizeInBits(); 1416 unsigned NVTBits = NVT.getScalarType().getSizeInBits(); 1417 assert(isPowerOf2_32(NVTBits) && 1418 "Expanded integer type size not a power of two!"); 1419 SDLoc dl(N); 1420 1421 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1422 APInt KnownZero, KnownOne; 1423 DAG.computeKnownBits(N->getOperand(1), KnownZero, KnownOne); 1424 1425 // If we don't know anything about the high bits, exit. 1426 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1427 return false; 1428 1429 // Get the incoming operand to be shifted. 1430 SDValue InL, InH; 1431 GetExpandedInteger(N->getOperand(0), InL, InH); 1432 1433 // If we know that any of the high bits of the shift amount are one, then we 1434 // can do this as a couple of simple shifts. 1435 if (KnownOne.intersects(HighBitMask)) { 1436 // Mask out the high bit, which we know is set. 1437 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 1438 DAG.getConstant(~HighBitMask, ShTy)); 1439 1440 switch (N->getOpcode()) { 1441 default: llvm_unreachable("Unknown shift"); 1442 case ISD::SHL: 1443 Lo = DAG.getConstant(0, NVT); // Low part is zero. 1444 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 1445 return true; 1446 case ISD::SRL: 1447 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 1448 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 1449 return true; 1450 case ISD::SRA: 1451 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 1452 DAG.getConstant(NVTBits-1, ShTy)); 1453 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 1454 return true; 1455 } 1456 } 1457 1458 // If we know that all of the high bits of the shift amount are zero, then we 1459 // can do this as a couple of simple shifts. 1460 if ((KnownZero & HighBitMask) == HighBitMask) { 1461 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 1462 // shift if x is zero. We can use XOR here because x is known to be smaller 1463 // than 32. 1464 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 1465 DAG.getConstant(NVTBits-1, ShTy)); 1466 1467 unsigned Op1, Op2; 1468 switch (N->getOpcode()) { 1469 default: llvm_unreachable("Unknown shift"); 1470 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1471 case ISD::SRL: 1472 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1473 } 1474 1475 // When shifting right the arithmetic for Lo and Hi is swapped. 1476 if (N->getOpcode() != ISD::SHL) 1477 std::swap(InL, InH); 1478 1479 // Use a little trick to get the bits that move from Lo to Hi. First 1480 // shift by one bit. 1481 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, ShTy)); 1482 // Then compute the remaining shift with amount-1. 1483 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 1484 1485 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 1486 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 1487 1488 if (N->getOpcode() != ISD::SHL) 1489 std::swap(Hi, Lo); 1490 return true; 1491 } 1492 1493 return false; 1494} 1495 1496/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 1497/// of any size. 1498bool DAGTypeLegalizer:: 1499ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1500 SDValue Amt = N->getOperand(1); 1501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1502 EVT ShTy = Amt.getValueType(); 1503 unsigned NVTBits = NVT.getSizeInBits(); 1504 assert(isPowerOf2_32(NVTBits) && 1505 "Expanded integer type size not a power of two!"); 1506 SDLoc dl(N); 1507 1508 // Get the incoming operand to be shifted. 1509 SDValue InL, InH; 1510 GetExpandedInteger(N->getOperand(0), InL, InH); 1511 1512 SDValue NVBitsNode = DAG.getConstant(NVTBits, ShTy); 1513 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 1514 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 1515 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 1516 Amt, NVBitsNode, ISD::SETULT); 1517 1518 SDValue LoS, HiS, LoL, HiL; 1519 switch (N->getOpcode()) { 1520 default: llvm_unreachable("Unknown shift"); 1521 case ISD::SHL: 1522 // Short: ShAmt < NVTBits 1523 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 1524 HiS = DAG.getNode(ISD::OR, dl, NVT, 1525 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 1526 // FIXME: If Amt is zero, the following shift generates an undefined result 1527 // on some architectures. 1528 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 1529 1530 // Long: ShAmt >= NVTBits 1531 LoL = DAG.getConstant(0, NVT); // Lo part is zero. 1532 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 1533 1534 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 1535 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1536 return true; 1537 case ISD::SRL: 1538 // Short: ShAmt < NVTBits 1539 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 1540 LoS = DAG.getNode(ISD::OR, dl, NVT, 1541 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1542 // FIXME: If Amt is zero, the following shift generates an undefined result 1543 // on some architectures. 1544 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1545 1546 // Long: ShAmt >= NVTBits 1547 HiL = DAG.getConstant(0, NVT); // Hi part is zero. 1548 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1549 1550 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 1551 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1552 return true; 1553 case ISD::SRA: 1554 // Short: ShAmt < NVTBits 1555 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 1556 LoS = DAG.getNode(ISD::OR, dl, NVT, 1557 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1558 // FIXME: If Amt is zero, the following shift generates an undefined result 1559 // on some architectures. 1560 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1561 1562 // Long: ShAmt >= NVTBits 1563 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 1564 DAG.getConstant(NVTBits-1, ShTy)); 1565 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1566 1567 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 1568 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1569 return true; 1570 } 1571} 1572 1573void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1574 SDValue &Lo, SDValue &Hi) { 1575 SDLoc dl(N); 1576 // Expand the subcomponents. 1577 SDValue LHSL, LHSH, RHSL, RHSH; 1578 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1579 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1580 1581 EVT NVT = LHSL.getValueType(); 1582 SDValue LoOps[2] = { LHSL, RHSL }; 1583 SDValue HiOps[3] = { LHSH, RHSH }; 1584 1585 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1586 // them. TODO: Teach operation legalization how to expand unsupported 1587 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1588 // a carry of type MVT::Glue, but there doesn't seem to be any way to 1589 // generate a value of this type in the expanded code sequence. 1590 bool hasCarry = 1591 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1592 ISD::ADDC : ISD::SUBC, 1593 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1594 1595 if (hasCarry) { 1596 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1597 if (N->getOpcode() == ISD::ADD) { 1598 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 1599 HiOps[2] = Lo.getValue(1); 1600 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 1601 } else { 1602 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 1603 HiOps[2] = Lo.getValue(1); 1604 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 1605 } 1606 return; 1607 } 1608 1609 if (N->getOpcode() == ISD::ADD) { 1610 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 1611 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 1612 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 1613 ISD::SETULT); 1614 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, 1615 DAG.getConstant(1, NVT), 1616 DAG.getConstant(0, NVT)); 1617 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], 1618 ISD::SETULT); 1619 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, 1620 DAG.getConstant(1, NVT), Carry1); 1621 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 1622 } else { 1623 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 1624 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 1625 SDValue Cmp = 1626 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 1627 LoOps[0], LoOps[1], ISD::SETULT); 1628 SDValue Borrow = DAG.getSelect(dl, NVT, Cmp, 1629 DAG.getConstant(1, NVT), 1630 DAG.getConstant(0, NVT)); 1631 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 1632 } 1633} 1634 1635void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1636 SDValue &Lo, SDValue &Hi) { 1637 // Expand the subcomponents. 1638 SDValue LHSL, LHSH, RHSL, RHSH; 1639 SDLoc dl(N); 1640 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1641 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1642 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1643 SDValue LoOps[2] = { LHSL, RHSL }; 1644 SDValue HiOps[3] = { LHSH, RHSH }; 1645 1646 if (N->getOpcode() == ISD::ADDC) { 1647 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 1648 HiOps[2] = Lo.getValue(1); 1649 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 1650 } else { 1651 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 1652 HiOps[2] = Lo.getValue(1); 1653 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 1654 } 1655 1656 // Legalized the flag result - switch anything that used the old flag to 1657 // use the new one. 1658 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1659} 1660 1661void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1662 SDValue &Lo, SDValue &Hi) { 1663 // Expand the subcomponents. 1664 SDValue LHSL, LHSH, RHSL, RHSH; 1665 SDLoc dl(N); 1666 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1667 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1668 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1669 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1670 SDValue HiOps[3] = { LHSH, RHSH }; 1671 1672 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 1673 HiOps[2] = Lo.getValue(1); 1674 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 1675 1676 // Legalized the flag result - switch anything that used the old flag to 1677 // use the new one. 1678 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1679} 1680 1681void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 1682 SDValue &Lo, SDValue &Hi) { 1683 SDValue Res = DisintegrateMERGE_VALUES(N, ResNo); 1684 SplitInteger(Res, Lo, Hi); 1685} 1686 1687void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 1688 SDValue &Lo, SDValue &Hi) { 1689 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1690 SDLoc dl(N); 1691 SDValue Op = N->getOperand(0); 1692 if (Op.getValueType().bitsLE(NVT)) { 1693 // The low part is any extension of the input (which degenerates to a copy). 1694 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 1695 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 1696 } else { 1697 // For example, extension of an i48 to an i64. The operand type necessarily 1698 // promotes to the result type, so will end up being expanded too. 1699 assert(getTypeAction(Op.getValueType()) == 1700 TargetLowering::TypePromoteInteger && 1701 "Only know how to promote this result!"); 1702 SDValue Res = GetPromotedInteger(Op); 1703 assert(Res.getValueType() == N->getValueType(0) && 1704 "Operand over promoted?"); 1705 // Split the promoted operand. This will simplify when it is expanded. 1706 SplitInteger(Res, Lo, Hi); 1707 } 1708} 1709 1710void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 1711 SDValue &Lo, SDValue &Hi) { 1712 SDLoc dl(N); 1713 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1714 EVT NVT = Lo.getValueType(); 1715 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1716 unsigned NVTBits = NVT.getSizeInBits(); 1717 unsigned EVTBits = EVT.getSizeInBits(); 1718 1719 if (NVTBits < EVTBits) { 1720 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 1721 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1722 EVTBits - NVTBits))); 1723 } else { 1724 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 1725 // The high part replicates the sign bit of Lo, make it explicit. 1726 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1727 DAG.getConstant(NVTBits-1, TLI.getPointerTy())); 1728 } 1729} 1730 1731void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 1732 SDValue &Lo, SDValue &Hi) { 1733 SDLoc dl(N); 1734 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1735 EVT NVT = Lo.getValueType(); 1736 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1737 unsigned NVTBits = NVT.getSizeInBits(); 1738 unsigned EVTBits = EVT.getSizeInBits(); 1739 1740 if (NVTBits < EVTBits) { 1741 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 1742 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1743 EVTBits - NVTBits))); 1744 } else { 1745 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 1746 // The high part must be zero, make it explicit. 1747 Hi = DAG.getConstant(0, NVT); 1748 } 1749} 1750 1751void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1752 SDValue &Lo, SDValue &Hi) { 1753 SDLoc dl(N); 1754 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1755 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 1756 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 1757} 1758 1759void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 1760 SDValue &Lo, SDValue &Hi) { 1761 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1762 unsigned NBitWidth = NVT.getSizeInBits(); 1763 auto Constant = cast<ConstantSDNode>(N); 1764 const APInt &Cst = Constant->getAPIntValue(); 1765 bool IsTarget = Constant->isTargetOpcode(); 1766 bool IsOpaque = Constant->isOpaque(); 1767 Lo = DAG.getConstant(Cst.trunc(NBitWidth), NVT, IsTarget, IsOpaque); 1768 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT, IsTarget, 1769 IsOpaque); 1770} 1771 1772void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1773 SDValue &Lo, SDValue &Hi) { 1774 SDLoc dl(N); 1775 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1776 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1777 EVT NVT = Lo.getValueType(); 1778 1779 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 1780 DAG.getConstant(0, NVT), ISD::SETNE); 1781 1782 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 1783 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 1784 1785 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 1786 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 1787 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1788 Hi = DAG.getConstant(0, NVT); 1789} 1790 1791void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1792 SDValue &Lo, SDValue &Hi) { 1793 SDLoc dl(N); 1794 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1795 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1796 EVT NVT = Lo.getValueType(); 1797 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 1798 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 1799 Hi = DAG.getConstant(0, NVT); 1800} 1801 1802void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1803 SDValue &Lo, SDValue &Hi) { 1804 SDLoc dl(N); 1805 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1806 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1807 EVT NVT = Lo.getValueType(); 1808 1809 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 1810 DAG.getConstant(0, NVT), ISD::SETNE); 1811 1812 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 1813 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 1814 1815 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 1816 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 1817 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1818 Hi = DAG.getConstant(0, NVT); 1819} 1820 1821void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 1822 SDValue &Hi) { 1823 SDLoc dl(N); 1824 EVT VT = N->getValueType(0); 1825 SDValue Op = N->getOperand(0); 1826 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 1827 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 1828 SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, true/*irrelevant*/, 1829 dl).first, 1830 Lo, Hi); 1831} 1832 1833void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 1834 SDValue &Hi) { 1835 SDLoc dl(N); 1836 EVT VT = N->getValueType(0); 1837 SDValue Op = N->getOperand(0); 1838 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 1839 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 1840 SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, false/*irrelevant*/, 1841 dl).first, 1842 Lo, Hi); 1843} 1844 1845void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 1846 SDValue &Lo, SDValue &Hi) { 1847 if (ISD::isNormalLoad(N)) { 1848 ExpandRes_NormalLoad(N, Lo, Hi); 1849 return; 1850 } 1851 1852 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1853 1854 EVT VT = N->getValueType(0); 1855 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1856 SDValue Ch = N->getChain(); 1857 SDValue Ptr = N->getBasePtr(); 1858 ISD::LoadExtType ExtType = N->getExtensionType(); 1859 unsigned Alignment = N->getAlignment(); 1860 bool isVolatile = N->isVolatile(); 1861 bool isNonTemporal = N->isNonTemporal(); 1862 bool isInvariant = N->isInvariant(); 1863 const MDNode *TBAAInfo = N->getTBAAInfo(); 1864 SDLoc dl(N); 1865 1866 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1867 1868 if (N->getMemoryVT().bitsLE(NVT)) { 1869 EVT MemVT = N->getMemoryVT(); 1870 1871 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 1872 MemVT, isVolatile, isNonTemporal, Alignment, TBAAInfo); 1873 1874 // Remember the chain. 1875 Ch = Lo.getValue(1); 1876 1877 if (ExtType == ISD::SEXTLOAD) { 1878 // The high part is obtained by SRA'ing all but one of the bits of the 1879 // lo part. 1880 unsigned LoSize = Lo.getValueType().getSizeInBits(); 1881 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1882 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 1883 } else if (ExtType == ISD::ZEXTLOAD) { 1884 // The high part is just a zero. 1885 Hi = DAG.getConstant(0, NVT); 1886 } else { 1887 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 1888 // The high part is undefined. 1889 Hi = DAG.getUNDEF(NVT); 1890 } 1891 } else if (TLI.isLittleEndian()) { 1892 // Little-endian - low bits are at low addresses. 1893 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 1894 isVolatile, isNonTemporal, isInvariant, Alignment, 1895 TBAAInfo); 1896 1897 unsigned ExcessBits = 1898 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1899 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 1900 1901 // Increment the pointer to the other half. 1902 unsigned IncrementSize = NVT.getSizeInBits()/8; 1903 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1904 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1905 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 1906 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 1907 isVolatile, isNonTemporal, 1908 MinAlign(Alignment, IncrementSize), TBAAInfo); 1909 1910 // Build a factor node to remember that this load is independent of the 1911 // other one. 1912 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1913 Hi.getValue(1)); 1914 } else { 1915 // Big-endian - high bits are at low addresses. Favor aligned loads at 1916 // the cost of some bit-fiddling. 1917 EVT MemVT = N->getMemoryVT(); 1918 unsigned EBytes = MemVT.getStoreSize(); 1919 unsigned IncrementSize = NVT.getSizeInBits()/8; 1920 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1921 1922 // Load both the high bits and maybe some of the low bits. 1923 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 1924 EVT::getIntegerVT(*DAG.getContext(), 1925 MemVT.getSizeInBits() - ExcessBits), 1926 isVolatile, isNonTemporal, Alignment, TBAAInfo); 1927 1928 // Increment the pointer to the other half. 1929 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1930 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1931 // Load the rest of the low bits. 1932 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 1933 N->getPointerInfo().getWithOffset(IncrementSize), 1934 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 1935 isVolatile, isNonTemporal, 1936 MinAlign(Alignment, IncrementSize), TBAAInfo); 1937 1938 // Build a factor node to remember that this load is independent of the 1939 // other one. 1940 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1941 Hi.getValue(1)); 1942 1943 if (ExcessBits < NVT.getSizeInBits()) { 1944 // Transfer low bits from the bottom of Hi to the top of Lo. 1945 Lo = DAG.getNode(ISD::OR, dl, NVT, Lo, 1946 DAG.getNode(ISD::SHL, dl, NVT, Hi, 1947 DAG.getConstant(ExcessBits, 1948 TLI.getPointerTy()))); 1949 // Move high bits to the right position in Hi. 1950 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, 1951 NVT, Hi, 1952 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 1953 TLI.getPointerTy())); 1954 } 1955 } 1956 1957 // Legalized the chain result - switch anything that used the old chain to 1958 // use the new one. 1959 ReplaceValueWith(SDValue(N, 1), Ch); 1960} 1961 1962void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 1963 SDValue &Lo, SDValue &Hi) { 1964 SDLoc dl(N); 1965 SDValue LL, LH, RL, RH; 1966 GetExpandedInteger(N->getOperand(0), LL, LH); 1967 GetExpandedInteger(N->getOperand(1), RL, RH); 1968 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 1969 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 1970} 1971 1972void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 1973 SDValue &Lo, SDValue &Hi) { 1974 EVT VT = N->getValueType(0); 1975 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1976 SDLoc dl(N); 1977 1978 SDValue LL, LH, RL, RH; 1979 GetExpandedInteger(N->getOperand(0), LL, LH); 1980 GetExpandedInteger(N->getOperand(1), RL, RH); 1981 1982 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, LL, LH, RL, RH)) 1983 return; 1984 1985 // If nothing else, we can make a libcall. 1986 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1987 if (VT == MVT::i16) 1988 LC = RTLIB::MUL_I16; 1989 else if (VT == MVT::i32) 1990 LC = RTLIB::MUL_I32; 1991 else if (VT == MVT::i64) 1992 LC = RTLIB::MUL_I64; 1993 else if (VT == MVT::i128) 1994 LC = RTLIB::MUL_I128; 1995 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!"); 1996 1997 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1998 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true/*irrelevant*/, 1999 dl).first, 2000 Lo, Hi); 2001} 2002 2003void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 2004 SDValue &Lo, SDValue &Hi) { 2005 SDValue LHS = Node->getOperand(0); 2006 SDValue RHS = Node->getOperand(1); 2007 SDLoc dl(Node); 2008 2009 // Expand the result by simply replacing it with the equivalent 2010 // non-overflow-checking operation. 2011 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 2012 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2013 LHS, RHS); 2014 SplitInteger(Sum, Lo, Hi); 2015 2016 // Compute the overflow. 2017 // 2018 // LHSSign -> LHS >= 0 2019 // RHSSign -> RHS >= 0 2020 // SumSign -> Sum >= 0 2021 // 2022 // Add: 2023 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 2024 // Sub: 2025 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 2026 // 2027 EVT OType = Node->getValueType(1); 2028 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 2029 2030 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 2031 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 2032 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 2033 Node->getOpcode() == ISD::SADDO ? 2034 ISD::SETEQ : ISD::SETNE); 2035 2036 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 2037 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 2038 2039 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 2040 2041 // Use the calculated overflow everywhere. 2042 ReplaceValueWith(SDValue(Node, 1), Cmp); 2043} 2044 2045void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 2046 SDValue &Lo, SDValue &Hi) { 2047 EVT VT = N->getValueType(0); 2048 SDLoc dl(N); 2049 2050 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2051 if (VT == MVT::i16) 2052 LC = RTLIB::SDIV_I16; 2053 else if (VT == MVT::i32) 2054 LC = RTLIB::SDIV_I32; 2055 else if (VT == MVT::i64) 2056 LC = RTLIB::SDIV_I64; 2057 else if (VT == MVT::i128) 2058 LC = RTLIB::SDIV_I128; 2059 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 2060 2061 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2062 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl).first, Lo, Hi); 2063} 2064 2065void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 2066 SDValue &Lo, SDValue &Hi) { 2067 EVT VT = N->getValueType(0); 2068 SDLoc dl(N); 2069 2070 // If we can emit an efficient shift operation, do so now. Check to see if 2071 // the RHS is a constant. 2072 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 2073 return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi); 2074 2075 // If we can determine that the high bit of the shift is zero or one, even if 2076 // the low bits are variable, emit this shift in an optimized form. 2077 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 2078 return; 2079 2080 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 2081 unsigned PartsOpc; 2082 if (N->getOpcode() == ISD::SHL) { 2083 PartsOpc = ISD::SHL_PARTS; 2084 } else if (N->getOpcode() == ISD::SRL) { 2085 PartsOpc = ISD::SRL_PARTS; 2086 } else { 2087 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2088 PartsOpc = ISD::SRA_PARTS; 2089 } 2090 2091 // Next check to see if the target supports this SHL_PARTS operation or if it 2092 // will custom expand it. 2093 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2094 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 2095 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 2096 Action == TargetLowering::Custom) { 2097 // Expand the subcomponents. 2098 SDValue LHSL, LHSH; 2099 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2100 EVT VT = LHSL.getValueType(); 2101 2102 // If the shift amount operand is coming from a vector legalization it may 2103 // have an illegal type. Fix that first by casting the operand, otherwise 2104 // the new SHL_PARTS operation would need further legalization. 2105 SDValue ShiftOp = N->getOperand(1); 2106 EVT ShiftTy = TLI.getShiftAmountTy(VT); 2107 assert(ShiftTy.getScalarType().getSizeInBits() >= 2108 Log2_32_Ceil(VT.getScalarType().getSizeInBits()) && 2109 "ShiftAmountTy is too small to cover the range of this type!"); 2110 if (ShiftOp.getValueType() != ShiftTy) 2111 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 2112 2113 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 2114 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 2115 Hi = Lo.getValue(1); 2116 return; 2117 } 2118 2119 // Otherwise, emit a libcall. 2120 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2121 bool isSigned; 2122 if (N->getOpcode() == ISD::SHL) { 2123 isSigned = false; /*sign irrelevant*/ 2124 if (VT == MVT::i16) 2125 LC = RTLIB::SHL_I16; 2126 else if (VT == MVT::i32) 2127 LC = RTLIB::SHL_I32; 2128 else if (VT == MVT::i64) 2129 LC = RTLIB::SHL_I64; 2130 else if (VT == MVT::i128) 2131 LC = RTLIB::SHL_I128; 2132 } else if (N->getOpcode() == ISD::SRL) { 2133 isSigned = false; 2134 if (VT == MVT::i16) 2135 LC = RTLIB::SRL_I16; 2136 else if (VT == MVT::i32) 2137 LC = RTLIB::SRL_I32; 2138 else if (VT == MVT::i64) 2139 LC = RTLIB::SRL_I64; 2140 else if (VT == MVT::i128) 2141 LC = RTLIB::SRL_I128; 2142 } else { 2143 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2144 isSigned = true; 2145 if (VT == MVT::i16) 2146 LC = RTLIB::SRA_I16; 2147 else if (VT == MVT::i32) 2148 LC = RTLIB::SRA_I32; 2149 else if (VT == MVT::i64) 2150 LC = RTLIB::SRA_I64; 2151 else if (VT == MVT::i128) 2152 LC = RTLIB::SRA_I128; 2153 } 2154 2155 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 2156 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2157 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, isSigned, dl).first, Lo, 2158 Hi); 2159 return; 2160 } 2161 2162 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 2163 llvm_unreachable("Unsupported shift!"); 2164} 2165 2166void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 2167 SDValue &Lo, SDValue &Hi) { 2168 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2169 SDLoc dl(N); 2170 SDValue Op = N->getOperand(0); 2171 if (Op.getValueType().bitsLE(NVT)) { 2172 // The low part is sign extension of the input (degenerates to a copy). 2173 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 2174 // The high part is obtained by SRA'ing all but one of the bits of low part. 2175 unsigned LoSize = NVT.getSizeInBits(); 2176 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2177 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 2178 } else { 2179 // For example, extension of an i48 to an i64. The operand type necessarily 2180 // promotes to the result type, so will end up being expanded too. 2181 assert(getTypeAction(Op.getValueType()) == 2182 TargetLowering::TypePromoteInteger && 2183 "Only know how to promote this result!"); 2184 SDValue Res = GetPromotedInteger(Op); 2185 assert(Res.getValueType() == N->getValueType(0) && 2186 "Operand over promoted?"); 2187 // Split the promoted operand. This will simplify when it is expanded. 2188 SplitInteger(Res, Lo, Hi); 2189 unsigned ExcessBits = 2190 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2191 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2192 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2193 ExcessBits))); 2194 } 2195} 2196 2197void DAGTypeLegalizer:: 2198ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2199 SDLoc dl(N); 2200 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2201 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2202 2203 if (EVT.bitsLE(Lo.getValueType())) { 2204 // sext_inreg the low part if needed. 2205 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 2206 N->getOperand(1)); 2207 2208 // The high part gets the sign extension from the lo-part. This handles 2209 // things like sextinreg V:i64 from i8. 2210 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 2211 DAG.getConstant(Hi.getValueType().getSizeInBits()-1, 2212 TLI.getPointerTy())); 2213 } else { 2214 // For example, extension of an i48 to an i64. Leave the low part alone, 2215 // sext_inreg the high part. 2216 unsigned ExcessBits = 2217 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 2218 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2219 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2220 ExcessBits))); 2221 } 2222} 2223 2224void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 2225 SDValue &Lo, SDValue &Hi) { 2226 EVT VT = N->getValueType(0); 2227 SDLoc dl(N); 2228 2229 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2230 if (VT == MVT::i16) 2231 LC = RTLIB::SREM_I16; 2232 else if (VT == MVT::i32) 2233 LC = RTLIB::SREM_I32; 2234 else if (VT == MVT::i64) 2235 LC = RTLIB::SREM_I64; 2236 else if (VT == MVT::i128) 2237 LC = RTLIB::SREM_I128; 2238 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 2239 2240 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2241 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl).first, Lo, Hi); 2242} 2243 2244void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 2245 SDValue &Lo, SDValue &Hi) { 2246 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2247 SDLoc dl(N); 2248 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 2249 Hi = DAG.getNode(ISD::SRL, dl, 2250 N->getOperand(0).getValueType(), N->getOperand(0), 2251 DAG.getConstant(NVT.getSizeInBits(), TLI.getPointerTy())); 2252 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 2253} 2254 2255void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 2256 SDValue &Lo, SDValue &Hi) { 2257 SDValue LHS = N->getOperand(0); 2258 SDValue RHS = N->getOperand(1); 2259 SDLoc dl(N); 2260 2261 // Expand the result by simply replacing it with the equivalent 2262 // non-overflow-checking operation. 2263 SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ? 2264 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2265 LHS, RHS); 2266 SplitInteger(Sum, Lo, Hi); 2267 2268 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2269 // overflows iff a - b > a. 2270 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, 2271 N->getOpcode () == ISD::UADDO ? 2272 ISD::SETULT : ISD::SETUGT); 2273 2274 // Use the calculated overflow everywhere. 2275 ReplaceValueWith(SDValue(N, 1), Ofl); 2276} 2277 2278void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 2279 SDValue &Lo, SDValue &Hi) { 2280 EVT VT = N->getValueType(0); 2281 SDLoc dl(N); 2282 2283 // A divide for UMULO should be faster than a function call. 2284 if (N->getOpcode() == ISD::UMULO) { 2285 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 2286 2287 SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS); 2288 SplitInteger(MUL, Lo, Hi); 2289 2290 // A divide for UMULO will be faster than a function call. Select to 2291 // make sure we aren't using 0. 2292 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT), 2293 RHS, DAG.getConstant(0, VT), ISD::SETEQ); 2294 SDValue NotZero = DAG.getSelect(dl, VT, isZero, 2295 DAG.getConstant(1, VT), RHS); 2296 SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero); 2297 SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS, 2298 ISD::SETNE); 2299 Overflow = DAG.getSelect(dl, N->getValueType(1), isZero, 2300 DAG.getConstant(0, N->getValueType(1)), 2301 Overflow); 2302 ReplaceValueWith(SDValue(N, 1), Overflow); 2303 return; 2304 } 2305 2306 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 2307 EVT PtrVT = TLI.getPointerTy(); 2308 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 2309 2310 // Replace this with a libcall that will check overflow. 2311 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2312 if (VT == MVT::i32) 2313 LC = RTLIB::MULO_I32; 2314 else if (VT == MVT::i64) 2315 LC = RTLIB::MULO_I64; 2316 else if (VT == MVT::i128) 2317 LC = RTLIB::MULO_I128; 2318 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 2319 2320 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 2321 // Temporary for the overflow value, default it to zero. 2322 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, 2323 DAG.getConstant(0, PtrVT), Temp, 2324 MachinePointerInfo(), false, false, 0); 2325 2326 TargetLowering::ArgListTy Args; 2327 TargetLowering::ArgListEntry Entry; 2328 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2329 EVT ArgVT = N->getOperand(i).getValueType(); 2330 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2331 Entry.Node = N->getOperand(i); 2332 Entry.Ty = ArgTy; 2333 Entry.isSExt = true; 2334 Entry.isZExt = false; 2335 Args.push_back(Entry); 2336 } 2337 2338 // Also pass the address of the overflow check. 2339 Entry.Node = Temp; 2340 Entry.Ty = PtrTy->getPointerTo(); 2341 Entry.isSExt = true; 2342 Entry.isZExt = false; 2343 Args.push_back(Entry); 2344 2345 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 2346 2347 TargetLowering::CallLoweringInfo CLI(DAG); 2348 CLI.setDebugLoc(dl).setChain(Chain) 2349 .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args), 0) 2350 .setSExtResult(); 2351 2352 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2353 2354 SplitInteger(CallInfo.first, Lo, Hi); 2355 SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, 2356 MachinePointerInfo(), false, false, false, 0); 2357 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 2358 DAG.getConstant(0, PtrVT), 2359 ISD::SETNE); 2360 // Use the overflow from the libcall everywhere. 2361 ReplaceValueWith(SDValue(N, 1), Ofl); 2362} 2363 2364void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 2365 SDValue &Lo, SDValue &Hi) { 2366 EVT VT = N->getValueType(0); 2367 SDLoc dl(N); 2368 2369 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2370 if (VT == MVT::i16) 2371 LC = RTLIB::UDIV_I16; 2372 else if (VT == MVT::i32) 2373 LC = RTLIB::UDIV_I32; 2374 else if (VT == MVT::i64) 2375 LC = RTLIB::UDIV_I64; 2376 else if (VT == MVT::i128) 2377 LC = RTLIB::UDIV_I128; 2378 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 2379 2380 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2381 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl).first, Lo, Hi); 2382} 2383 2384void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 2385 SDValue &Lo, SDValue &Hi) { 2386 EVT VT = N->getValueType(0); 2387 SDLoc dl(N); 2388 2389 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2390 if (VT == MVT::i16) 2391 LC = RTLIB::UREM_I16; 2392 else if (VT == MVT::i32) 2393 LC = RTLIB::UREM_I32; 2394 else if (VT == MVT::i64) 2395 LC = RTLIB::UREM_I64; 2396 else if (VT == MVT::i128) 2397 LC = RTLIB::UREM_I128; 2398 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 2399 2400 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2401 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl).first, Lo, Hi); 2402} 2403 2404void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 2405 SDValue &Lo, SDValue &Hi) { 2406 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2407 SDLoc dl(N); 2408 SDValue Op = N->getOperand(0); 2409 if (Op.getValueType().bitsLE(NVT)) { 2410 // The low part is zero extension of the input (degenerates to a copy). 2411 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 2412 Hi = DAG.getConstant(0, NVT); // The high part is just a zero. 2413 } else { 2414 // For example, extension of an i48 to an i64. The operand type necessarily 2415 // promotes to the result type, so will end up being expanded too. 2416 assert(getTypeAction(Op.getValueType()) == 2417 TargetLowering::TypePromoteInteger && 2418 "Only know how to promote this result!"); 2419 SDValue Res = GetPromotedInteger(Op); 2420 assert(Res.getValueType() == N->getValueType(0) && 2421 "Operand over promoted?"); 2422 // Split the promoted operand. This will simplify when it is expanded. 2423 SplitInteger(Res, Lo, Hi); 2424 unsigned ExcessBits = 2425 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2426 Hi = DAG.getZeroExtendInReg(Hi, dl, 2427 EVT::getIntegerVT(*DAG.getContext(), 2428 ExcessBits)); 2429 } 2430} 2431 2432void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 2433 SDValue &Lo, SDValue &Hi) { 2434 SDLoc dl(N); 2435 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 2436 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 2437 SDValue Zero = DAG.getConstant(0, VT); 2438 SDValue Swap = DAG.getAtomicCmpSwap( 2439 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 2440 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 2441 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand(), 2442 cast<AtomicSDNode>(N)->getOrdering(), 2443 cast<AtomicSDNode>(N)->getOrdering(), 2444 cast<AtomicSDNode>(N)->getSynchScope()); 2445 2446 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 2447 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 2448} 2449 2450//===----------------------------------------------------------------------===// 2451// Integer Operand Expansion 2452//===----------------------------------------------------------------------===// 2453 2454/// ExpandIntegerOperand - This method is called when the specified operand of 2455/// the specified node is found to need expansion. At this point, all of the 2456/// result types of the node are known to be legal, but other operands of the 2457/// node may need promotion or expansion as well as the specified one. 2458bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 2459 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); 2460 SDValue Res = SDValue(); 2461 2462 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2463 return false; 2464 2465 switch (N->getOpcode()) { 2466 default: 2467 #ifndef NDEBUG 2468 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 2469 N->dump(&DAG); dbgs() << "\n"; 2470 #endif 2471 llvm_unreachable("Do not know how to expand this operator's operand!"); 2472 2473 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2474 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 2475 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2476 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2477 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 2478 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 2479 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 2480 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 2481 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 2482 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 2483 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 2484 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 2485 2486 case ISD::SHL: 2487 case ISD::SRA: 2488 case ISD::SRL: 2489 case ISD::ROTL: 2490 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 2491 case ISD::RETURNADDR: 2492 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 2493 2494 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 2495 } 2496 2497 // If the result is null, the sub-method took care of registering results etc. 2498 if (!Res.getNode()) return false; 2499 2500 // If the result is N, the sub-method updated N in place. Tell the legalizer 2501 // core about this. 2502 if (Res.getNode() == N) 2503 return true; 2504 2505 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2506 "Invalid operand expansion"); 2507 2508 ReplaceValueWith(SDValue(N, 0), Res); 2509 return false; 2510} 2511 2512/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 2513/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2514void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 2515 SDValue &NewRHS, 2516 ISD::CondCode &CCCode, 2517 SDLoc dl) { 2518 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2519 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 2520 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 2521 2522 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 2523 if (RHSLo == RHSHi) { 2524 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 2525 if (RHSCST->isAllOnesValue()) { 2526 // Equality comparison to -1. 2527 NewLHS = DAG.getNode(ISD::AND, dl, 2528 LHSLo.getValueType(), LHSLo, LHSHi); 2529 NewRHS = RHSLo; 2530 return; 2531 } 2532 } 2533 } 2534 2535 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 2536 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 2537 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 2538 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2539 return; 2540 } 2541 2542 // If this is a comparison of the sign bit, just look at the top part. 2543 // X > -1, x < 0 2544 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 2545 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 2546 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 2547 NewLHS = LHSHi; 2548 NewRHS = RHSHi; 2549 return; 2550 } 2551 2552 // FIXME: This generated code sucks. 2553 ISD::CondCode LowCC; 2554 switch (CCCode) { 2555 default: llvm_unreachable("Unknown integer setcc!"); 2556 case ISD::SETLT: 2557 case ISD::SETULT: LowCC = ISD::SETULT; break; 2558 case ISD::SETGT: 2559 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 2560 case ISD::SETLE: 2561 case ISD::SETULE: LowCC = ISD::SETULE; break; 2562 case ISD::SETGE: 2563 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 2564 } 2565 2566 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 2567 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 2568 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 2569 2570 // NOTE: on targets without efficient SELECT of bools, we can always use 2571 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 2572 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 2573 nullptr); 2574 SDValue Tmp1, Tmp2; 2575 if (TLI.isTypeLegal(LHSLo.getValueType()) && 2576 TLI.isTypeLegal(RHSLo.getValueType())) 2577 Tmp1 = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), 2578 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); 2579 if (!Tmp1.getNode()) 2580 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), 2581 LHSLo, RHSLo, LowCC); 2582 if (TLI.isTypeLegal(LHSHi.getValueType()) && 2583 TLI.isTypeLegal(RHSHi.getValueType())) 2584 Tmp2 = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), 2585 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); 2586 if (!Tmp2.getNode()) 2587 Tmp2 = DAG.getNode(ISD::SETCC, dl, 2588 getSetCCResultType(LHSHi.getValueType()), 2589 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 2590 2591 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 2592 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 2593 if ((Tmp1C && Tmp1C->isNullValue()) || 2594 (Tmp2C && Tmp2C->isNullValue() && 2595 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 2596 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 2597 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 2598 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 2599 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 2600 // low part is known false, returns high part. 2601 // For LE / GE, if high part is known false, ignore the low part. 2602 // For LT / GT, if high part is known true, ignore the low part. 2603 NewLHS = Tmp2; 2604 NewRHS = SDValue(); 2605 return; 2606 } 2607 2608 NewLHS = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), 2609 LHSHi, RHSHi, ISD::SETEQ, false, 2610 DagCombineInfo, dl); 2611 if (!NewLHS.getNode()) 2612 NewLHS = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), 2613 LHSHi, RHSHi, ISD::SETEQ); 2614 NewLHS = DAG.getSelect(dl, Tmp1.getValueType(), 2615 NewLHS, Tmp1, Tmp2); 2616 NewRHS = SDValue(); 2617} 2618 2619SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 2620 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 2621 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 2622 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 2623 2624 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2625 // against zero to select between true and false values. 2626 if (!NewRHS.getNode()) { 2627 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2628 CCCode = ISD::SETNE; 2629 } 2630 2631 // Update N to have the operands specified. 2632 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2633 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2634 N->getOperand(4)), 0); 2635} 2636 2637SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 2638 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2639 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2640 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 2641 2642 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2643 // against zero to select between true and false values. 2644 if (!NewRHS.getNode()) { 2645 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2646 CCCode = ISD::SETNE; 2647 } 2648 2649 // Update N to have the operands specified. 2650 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2651 N->getOperand(2), N->getOperand(3), 2652 DAG.getCondCode(CCCode)), 0); 2653} 2654 2655SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 2656 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2657 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2658 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 2659 2660 // If ExpandSetCCOperands returned a scalar, use it. 2661 if (!NewRHS.getNode()) { 2662 assert(NewLHS.getValueType() == N->getValueType(0) && 2663 "Unexpected setcc expansion!"); 2664 return NewLHS; 2665 } 2666 2667 // Otherwise, update N to have the operands specified. 2668 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2669 DAG.getCondCode(CCCode)), 0); 2670} 2671 2672SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 2673 // The value being shifted is legal, but the shift amount is too big. 2674 // It follows that either the result of the shift is undefined, or the 2675 // upper half of the shift amount is zero. Just use the lower half. 2676 SDValue Lo, Hi; 2677 GetExpandedInteger(N->getOperand(1), Lo, Hi); 2678 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 2679} 2680 2681SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 2682 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 2683 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 2684 // constant to valid type. 2685 SDValue Lo, Hi; 2686 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2687 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 2688} 2689 2690SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 2691 SDValue Op = N->getOperand(0); 2692 EVT DstVT = N->getValueType(0); 2693 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 2694 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2695 "Don't know how to expand this SINT_TO_FP!"); 2696 return TLI.makeLibCall(DAG, LC, DstVT, &Op, 1, true, SDLoc(N)).first; 2697} 2698 2699SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 2700 if (ISD::isNormalStore(N)) 2701 return ExpandOp_NormalStore(N, OpNo); 2702 2703 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2704 assert(OpNo == 1 && "Can only expand the stored value so far"); 2705 2706 EVT VT = N->getOperand(1).getValueType(); 2707 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2708 SDValue Ch = N->getChain(); 2709 SDValue Ptr = N->getBasePtr(); 2710 unsigned Alignment = N->getAlignment(); 2711 bool isVolatile = N->isVolatile(); 2712 bool isNonTemporal = N->isNonTemporal(); 2713 const MDNode *TBAAInfo = N->getTBAAInfo(); 2714 SDLoc dl(N); 2715 SDValue Lo, Hi; 2716 2717 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2718 2719 if (N->getMemoryVT().bitsLE(NVT)) { 2720 GetExpandedInteger(N->getValue(), Lo, Hi); 2721 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 2722 N->getMemoryVT(), isVolatile, isNonTemporal, 2723 Alignment, TBAAInfo); 2724 } 2725 2726 if (TLI.isLittleEndian()) { 2727 // Little-endian - low bits are at low addresses. 2728 GetExpandedInteger(N->getValue(), Lo, Hi); 2729 2730 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 2731 isVolatile, isNonTemporal, Alignment, TBAAInfo); 2732 2733 unsigned ExcessBits = 2734 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2735 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2736 2737 // Increment the pointer to the other half. 2738 unsigned IncrementSize = NVT.getSizeInBits()/8; 2739 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2740 DAG.getConstant(IncrementSize, Ptr.getValueType())); 2741 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 2742 N->getPointerInfo().getWithOffset(IncrementSize), 2743 NEVT, isVolatile, isNonTemporal, 2744 MinAlign(Alignment, IncrementSize), TBAAInfo); 2745 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2746 } 2747 2748 // Big-endian - high bits are at low addresses. Favor aligned stores at 2749 // the cost of some bit-fiddling. 2750 GetExpandedInteger(N->getValue(), Lo, Hi); 2751 2752 EVT ExtVT = N->getMemoryVT(); 2753 unsigned EBytes = ExtVT.getStoreSize(); 2754 unsigned IncrementSize = NVT.getSizeInBits()/8; 2755 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2756 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 2757 ExtVT.getSizeInBits() - ExcessBits); 2758 2759 if (ExcessBits < NVT.getSizeInBits()) { 2760 // Transfer high bits from the top of Lo to the bottom of Hi. 2761 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 2762 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 2763 TLI.getPointerTy())); 2764 Hi = DAG.getNode(ISD::OR, dl, NVT, Hi, 2765 DAG.getNode(ISD::SRL, dl, NVT, Lo, 2766 DAG.getConstant(ExcessBits, 2767 TLI.getPointerTy()))); 2768 } 2769 2770 // Store both the high bits and maybe some of the low bits. 2771 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), 2772 HiVT, isVolatile, isNonTemporal, Alignment, TBAAInfo); 2773 2774 // Increment the pointer to the other half. 2775 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2776 DAG.getConstant(IncrementSize, Ptr.getValueType())); 2777 // Store the lowest ExcessBits bits in the second half. 2778 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 2779 N->getPointerInfo().getWithOffset(IncrementSize), 2780 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2781 isVolatile, isNonTemporal, 2782 MinAlign(Alignment, IncrementSize), TBAAInfo); 2783 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2784} 2785 2786SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 2787 SDValue InL, InH; 2788 GetExpandedInteger(N->getOperand(0), InL, InH); 2789 // Just truncate the low part of the source. 2790 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 2791} 2792 2793SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 2794 SDValue Op = N->getOperand(0); 2795 EVT SrcVT = Op.getValueType(); 2796 EVT DstVT = N->getValueType(0); 2797 SDLoc dl(N); 2798 2799 // The following optimization is valid only if every value in SrcVT (when 2800 // treated as signed) is representable in DstVT. Check that the mantissa 2801 // size of DstVT is >= than the number of bits in SrcVT -1. 2802 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT); 2803 if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 && 2804 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 2805 // Do a signed conversion then adjust the result. 2806 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); 2807 SignedConv = TLI.LowerOperation(SignedConv, DAG); 2808 2809 // The result of the signed conversion needs adjusting if the 'sign bit' of 2810 // the incoming integer was set. To handle this, we dynamically test to see 2811 // if it is set, and, if so, add a fudge factor. 2812 2813 const uint64_t F32TwoE32 = 0x4F800000ULL; 2814 const uint64_t F32TwoE64 = 0x5F800000ULL; 2815 const uint64_t F32TwoE128 = 0x7F800000ULL; 2816 2817 APInt FF(32, 0); 2818 if (SrcVT == MVT::i32) 2819 FF = APInt(32, F32TwoE32); 2820 else if (SrcVT == MVT::i64) 2821 FF = APInt(32, F32TwoE64); 2822 else if (SrcVT == MVT::i128) 2823 FF = APInt(32, F32TwoE128); 2824 else 2825 llvm_unreachable("Unsupported UINT_TO_FP!"); 2826 2827 // Check whether the sign bit is set. 2828 SDValue Lo, Hi; 2829 GetExpandedInteger(Op, Lo, Hi); 2830 SDValue SignSet = DAG.getSetCC(dl, 2831 getSetCCResultType(Hi.getValueType()), 2832 Hi, DAG.getConstant(0, Hi.getValueType()), 2833 ISD::SETLT); 2834 2835 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 2836 SDValue FudgePtr = DAG.getConstantPool( 2837 ConstantInt::get(*DAG.getContext(), FF.zext(64)), 2838 TLI.getPointerTy()); 2839 2840 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 2841 SDValue Zero = DAG.getIntPtrConstant(0); 2842 SDValue Four = DAG.getIntPtrConstant(4); 2843 if (TLI.isBigEndian()) std::swap(Zero, Four); 2844 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, 2845 Zero, Four); 2846 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 2847 FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(), 2848 FudgePtr, Offset); 2849 Alignment = std::min(Alignment, 4u); 2850 2851 // Load the value out, extending it from f32 to the destination float type. 2852 // FIXME: Avoid the extend by constructing the right constant pool? 2853 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), 2854 FudgePtr, 2855 MachinePointerInfo::getConstantPool(), 2856 MVT::f32, 2857 false, false, Alignment); 2858 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); 2859 } 2860 2861 // Otherwise, use a libcall. 2862 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 2863 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2864 "Don't know how to expand this UINT_TO_FP!"); 2865 return TLI.makeLibCall(DAG, LC, DstVT, &Op, 1, true, dl).first; 2866} 2867 2868SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 2869 SDLoc dl(N); 2870 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 2871 cast<AtomicSDNode>(N)->getMemoryVT(), 2872 N->getOperand(0), 2873 N->getOperand(1), N->getOperand(2), 2874 cast<AtomicSDNode>(N)->getMemOperand(), 2875 cast<AtomicSDNode>(N)->getOrdering(), 2876 cast<AtomicSDNode>(N)->getSynchScope()); 2877 return Swap.getValue(1); 2878} 2879 2880 2881SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 2882 SDValue InOp0 = N->getOperand(0); 2883 EVT InVT = InOp0.getValueType(); 2884 2885 EVT OutVT = N->getValueType(0); 2886 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2887 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2888 unsigned OutNumElems = OutVT.getVectorNumElements(); 2889 EVT NOutVTElem = NOutVT.getVectorElementType(); 2890 2891 SDLoc dl(N); 2892 SDValue BaseIdx = N->getOperand(1); 2893 2894 SmallVector<SDValue, 8> Ops; 2895 Ops.reserve(OutNumElems); 2896 for (unsigned i = 0; i != OutNumElems; ++i) { 2897 2898 // Extract the element from the original vector. 2899 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 2900 BaseIdx, DAG.getConstant(i, BaseIdx.getValueType())); 2901 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 2902 InVT.getVectorElementType(), N->getOperand(0), Index); 2903 2904 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); 2905 // Insert the converted element to the new vector. 2906 Ops.push_back(Op); 2907 } 2908 2909 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops); 2910} 2911 2912 2913SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 2914 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 2915 EVT VT = N->getValueType(0); 2916 SDLoc dl(N); 2917 2918 unsigned NumElts = VT.getVectorNumElements(); 2919 SmallVector<int, 8> NewMask; 2920 for (unsigned i = 0; i != NumElts; ++i) { 2921 NewMask.push_back(SV->getMaskElt(i)); 2922 } 2923 2924 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 2925 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 2926 EVT OutVT = V0.getValueType(); 2927 2928 return DAG.getVectorShuffle(OutVT, dl, V0, V1, &NewMask[0]); 2929} 2930 2931 2932SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 2933 EVT OutVT = N->getValueType(0); 2934 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2935 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2936 unsigned NumElems = N->getNumOperands(); 2937 EVT NOutVTElem = NOutVT.getVectorElementType(); 2938 2939 SDLoc dl(N); 2940 2941 SmallVector<SDValue, 8> Ops; 2942 Ops.reserve(NumElems); 2943 for (unsigned i = 0; i != NumElems; ++i) { 2944 SDValue Op; 2945 // BUILD_VECTOR integer operand types are allowed to be larger than the 2946 // result's element type. This may still be true after the promotion. For 2947 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 2948 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 2949 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) 2950 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 2951 else 2952 Op = N->getOperand(i); 2953 Ops.push_back(Op); 2954 } 2955 2956 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops); 2957} 2958 2959SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 2960 2961 SDLoc dl(N); 2962 2963 assert(!N->getOperand(0).getValueType().isVector() && 2964 "Input must be a scalar"); 2965 2966 EVT OutVT = N->getValueType(0); 2967 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2968 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2969 EVT NOutVTElem = NOutVT.getVectorElementType(); 2970 2971 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 2972 2973 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 2974} 2975 2976SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 2977 SDLoc dl(N); 2978 2979 EVT OutVT = N->getValueType(0); 2980 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2981 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2982 2983 EVT InElemTy = OutVT.getVectorElementType(); 2984 EVT OutElemTy = NOutVT.getVectorElementType(); 2985 2986 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 2987 unsigned NumOutElem = NOutVT.getVectorNumElements(); 2988 unsigned NumOperands = N->getNumOperands(); 2989 assert(NumElem * NumOperands == NumOutElem && 2990 "Unexpected number of elements"); 2991 2992 // Take the elements from the first vector. 2993 SmallVector<SDValue, 8> Ops(NumOutElem); 2994 for (unsigned i = 0; i < NumOperands; ++i) { 2995 SDValue Op = N->getOperand(i); 2996 for (unsigned j = 0; j < NumElem; ++j) { 2997 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 2998 InElemTy, Op, DAG.getConstant(j, 2999 TLI.getVectorIdxTy())); 3000 Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext); 3001 } 3002 } 3003 3004 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops); 3005} 3006 3007SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 3008 EVT OutVT = N->getValueType(0); 3009 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3010 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3011 3012 EVT NOutVTElem = NOutVT.getVectorElementType(); 3013 3014 SDLoc dl(N); 3015 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3016 3017 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 3018 NOutVTElem, N->getOperand(1)); 3019 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 3020 V0, ConvElem, N->getOperand(2)); 3021} 3022 3023SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 3024 SDLoc dl(N); 3025 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3026 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, TLI.getVectorIdxTy()); 3027 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3028 V0->getValueType(0).getScalarType(), V0, V1); 3029 3030 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 3031 // element types. If this is the case then we need to expand the outgoing 3032 // value and not truncate it. 3033 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 3034} 3035 3036SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 3037 SDLoc dl(N); 3038 unsigned NumElems = N->getNumOperands(); 3039 3040 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 3041 3042 SmallVector<SDValue, 8> NewOps; 3043 NewOps.reserve(NumElems); 3044 3045 // For each incoming vector 3046 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 3047 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 3048 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 3049 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 3050 3051 for (unsigned i=0; i<NumElem; ++i) { 3052 // Extract element from incoming vector 3053 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, 3054 Incoming, DAG.getConstant(i, TLI.getVectorIdxTy())); 3055 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 3056 NewOps.push_back(Tr); 3057 } 3058 } 3059 3060 return DAG.getNode(ISD::BUILD_VECTOR, dl, N->getValueType(0), NewOps); 3061} 3062