LegalizeIntegerTypes.cpp revision fcd75e5efb482f35cfc22fd4b64e047930130fd6
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/CodeGen/PseudoSourceValue.h" 24#include "llvm/Support/ErrorHandling.h" 25#include "llvm/Support/raw_ostream.h" 26using namespace llvm; 27 28//===----------------------------------------------------------------------===// 29// Integer Result Promotion 30//===----------------------------------------------------------------------===// 31 32/// PromoteIntegerResult - This method is called when a result of a node is 33/// found to be in need of promotion to a larger type. At this point, the node 34/// may also have invalid operands or may have other results that need 35/// expansion, we just know that (at least) one result needs promotion. 36void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 37 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); 38 SDValue Res = SDValue(); 39 40 // See if the target wants to custom expand this node. 41 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 42 return; 43 44 switch (N->getOpcode()) { 45 default: 46#ifndef NDEBUG 47 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 48 N->dump(&DAG); dbgs() << "\n"; 49#endif 50 llvm_unreachable("Do not know how to promote this operator!"); 51 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N); break; 52 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 53 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 54 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 55 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 56 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 57 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 58 case ISD::CONVERT_RNDSAT: 59 Res = PromoteIntRes_CONVERT_RNDSAT(N); break; 60 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 61 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 62 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 63 case ISD::EXTRACT_VECTOR_ELT: 64 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 65 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break; 66 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 67 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 68 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 69 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 70 case ISD::SIGN_EXTEND_INREG: 71 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 72 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 73 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 74 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 75 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 76 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 77 78 case ISD::EXTRACT_SUBVECTOR: 79 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 80 case ISD::VECTOR_SHUFFLE: 81 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 82 case ISD::INSERT_VECTOR_ELT: 83 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 84 case ISD::BUILD_VECTOR: 85 Res = PromoteIntRes_BUILD_VECTOR(N); break; 86 case ISD::SCALAR_TO_VECTOR: 87 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 88 89 case ISD::SIGN_EXTEND: 90 case ISD::ZERO_EXTEND: 91 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 92 93 case ISD::FP_TO_SINT: 94 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 95 96 case ISD::FP32_TO_FP16:Res = PromoteIntRes_FP32_TO_FP16(N); break; 97 98 case ISD::AND: 99 case ISD::OR: 100 case ISD::XOR: 101 case ISD::ADD: 102 case ISD::SUB: 103 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 104 105 case ISD::SDIV: 106 case ISD::SREM: Res = PromoteIntRes_SDIV(N); break; 107 108 case ISD::UDIV: 109 case ISD::UREM: Res = PromoteIntRes_UDIV(N); break; 110 111 case ISD::SADDO: 112 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 113 case ISD::UADDO: 114 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 115 case ISD::SMULO: 116 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 117 118 case ISD::ATOMIC_LOAD_ADD: 119 case ISD::ATOMIC_LOAD_SUB: 120 case ISD::ATOMIC_LOAD_AND: 121 case ISD::ATOMIC_LOAD_OR: 122 case ISD::ATOMIC_LOAD_XOR: 123 case ISD::ATOMIC_LOAD_NAND: 124 case ISD::ATOMIC_LOAD_MIN: 125 case ISD::ATOMIC_LOAD_MAX: 126 case ISD::ATOMIC_LOAD_UMIN: 127 case ISD::ATOMIC_LOAD_UMAX: 128 case ISD::ATOMIC_SWAP: 129 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 130 131 case ISD::ATOMIC_CMP_SWAP: 132 Res = PromoteIntRes_Atomic2(cast<AtomicSDNode>(N)); break; 133 } 134 135 // If the result is null then the sub-method took care of registering it. 136 if (Res.getNode()) 137 SetPromotedInteger(SDValue(N, ResNo), Res); 138} 139 140SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N) { 141 SDValue Op = DecomposeMERGE_VALUES(N); 142 return GetPromotedInteger(Op); 143} 144 145SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 146 // Sign-extend the new bits, and continue the assertion. 147 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 148 return DAG.getNode(ISD::AssertSext, N->getDebugLoc(), 149 Op.getValueType(), Op, N->getOperand(1)); 150} 151 152SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 153 // Zero the new bits, and continue the assertion. 154 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 155 return DAG.getNode(ISD::AssertZext, N->getDebugLoc(), 156 Op.getValueType(), Op, N->getOperand(1)); 157} 158 159SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 160 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 161 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 162 N->getMemoryVT(), 163 N->getChain(), N->getBasePtr(), 164 Op2, N->getMemOperand(), N->getOrdering(), 165 N->getSynchScope()); 166 // Legalized the chain result - switch anything that used the old chain to 167 // use the new one. 168 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 169 return Res; 170} 171 172SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) { 173 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 174 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 175 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 176 N->getMemoryVT(), N->getChain(), N->getBasePtr(), 177 Op2, Op3, N->getMemOperand(), N->getOrdering(), 178 N->getSynchScope()); 179 // Legalized the chain result - switch anything that used the old chain to 180 // use the new one. 181 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 182 return Res; 183} 184 185SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 186 SDValue InOp = N->getOperand(0); 187 EVT InVT = InOp.getValueType(); 188 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 189 EVT OutVT = N->getValueType(0); 190 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 191 DebugLoc dl = N->getDebugLoc(); 192 193 switch (getTypeAction(InVT)) { 194 default: 195 assert(false && "Unknown type action!"); 196 break; 197 case TargetLowering::TypeLegal: 198 break; 199 case TargetLowering::TypePromoteInteger: 200 if (NOutVT.bitsEq(NInVT)) 201 // The input promotes to the same size. Convert the promoted value. 202 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 203 break; 204 case TargetLowering::TypeSoftenFloat: 205 // Promote the integer operand by hand. 206 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 207 case TargetLowering::TypeExpandInteger: 208 case TargetLowering::TypeExpandFloat: 209 break; 210 case TargetLowering::TypeScalarizeVector: 211 // Convert the element to an integer and promote it by hand. 212 if (!NOutVT.isVector()) 213 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 214 BitConvertToInteger(GetScalarizedVector(InOp))); 215 break; 216 case TargetLowering::TypeSplitVector: { 217 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 218 // pieces of the input into integers and reassemble in the final type. 219 SDValue Lo, Hi; 220 GetSplitVector(N->getOperand(0), Lo, Hi); 221 Lo = BitConvertToInteger(Lo); 222 Hi = BitConvertToInteger(Hi); 223 224 if (TLI.isBigEndian()) 225 std::swap(Lo, Hi); 226 227 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 228 EVT::getIntegerVT(*DAG.getContext(), 229 NOutVT.getSizeInBits()), 230 JoinIntegers(Lo, Hi)); 231 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 232 } 233 case TargetLowering::TypeWidenVector: 234 if (OutVT.bitsEq(NInVT)) 235 // The input is widened to the same size. Convert to the widened value. 236 return DAG.getNode(ISD::BITCAST, dl, OutVT, GetWidenedVector(InOp)); 237 } 238 239 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 240 CreateStackStoreLoad(InOp, OutVT)); 241} 242 243SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 244 SDValue Op = GetPromotedInteger(N->getOperand(0)); 245 EVT OVT = N->getValueType(0); 246 EVT NVT = Op.getValueType(); 247 DebugLoc dl = N->getDebugLoc(); 248 249 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 250 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 251 DAG.getConstant(DiffBits, TLI.getPointerTy())); 252} 253 254SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 255 // The pair element type may be legal, or may not promote to the same type as 256 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 257 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), 258 TLI.getTypeToTransformTo(*DAG.getContext(), 259 N->getValueType(0)), JoinIntegers(N->getOperand(0), 260 N->getOperand(1))); 261} 262 263SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 264 EVT VT = N->getValueType(0); 265 // FIXME there is no actual debug info here 266 DebugLoc dl = N->getDebugLoc(); 267 // Zero extend things like i1, sign extend everything else. It shouldn't 268 // matter in theory which one we pick, but this tends to give better code? 269 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 270 SDValue Result = DAG.getNode(Opc, dl, 271 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 272 SDValue(N, 0)); 273 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 274 return Result; 275} 276 277SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) { 278 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 279 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 280 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 281 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 282 "can only promote integers"); 283 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 284 return DAG.getConvertRndSat(OutVT, N->getDebugLoc(), N->getOperand(0), 285 N->getOperand(1), N->getOperand(2), 286 N->getOperand(3), N->getOperand(4), CvtCode); 287} 288 289SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 290 // Zero extend to the promoted type and do the count there. 291 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 292 DebugLoc dl = N->getDebugLoc(); 293 EVT OVT = N->getValueType(0); 294 EVT NVT = Op.getValueType(); 295 Op = DAG.getNode(ISD::CTLZ, dl, NVT, Op); 296 // Subtract off the extra leading bits in the bigger type. 297 return DAG.getNode(ISD::SUB, dl, NVT, Op, 298 DAG.getConstant(NVT.getSizeInBits() - 299 OVT.getSizeInBits(), NVT)); 300} 301 302SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 303 // Zero extend to the promoted type and do the count there. 304 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 305 return DAG.getNode(ISD::CTPOP, N->getDebugLoc(), Op.getValueType(), Op); 306} 307 308SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 309 SDValue Op = GetPromotedInteger(N->getOperand(0)); 310 EVT OVT = N->getValueType(0); 311 EVT NVT = Op.getValueType(); 312 DebugLoc dl = N->getDebugLoc(); 313 // The count is the same in the promoted type except if the original 314 // value was zero. This can be handled by setting the bit just off 315 // the top of the original type. 316 APInt TopBit(NVT.getSizeInBits(), 0); 317 TopBit.setBit(OVT.getSizeInBits()); 318 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT)); 319 return DAG.getNode(ISD::CTTZ, dl, NVT, Op); 320} 321 322SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 323 DebugLoc dl = N->getDebugLoc(); 324 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 325 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), 326 N->getOperand(1)); 327} 328 329SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 330 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 331 unsigned NewOpc = N->getOpcode(); 332 DebugLoc dl = N->getDebugLoc(); 333 334 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 335 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 336 // and SINT conversions are Custom, there is no way to tell which is 337 // preferable. We choose SINT because that's the right thing on PPC.) 338 if (N->getOpcode() == ISD::FP_TO_UINT && 339 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 340 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 341 NewOpc = ISD::FP_TO_SINT; 342 343 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 344 345 // Assert that the converted value fits in the original type. If it doesn't 346 // (eg: because the value being converted is too big), then the result of the 347 // original operation was undefined anyway, so the assert is still correct. 348 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 349 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 350 DAG.getValueType(N->getValueType(0).getScalarType())); 351} 352 353SDValue DAGTypeLegalizer::PromoteIntRes_FP32_TO_FP16(SDNode *N) { 354 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 355 DebugLoc dl = N->getDebugLoc(); 356 357 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 358 359 return DAG.getNode(ISD::AssertZext, dl, 360 NVT, Res, DAG.getValueType(N->getValueType(0))); 361} 362 363SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 364 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 365 DebugLoc dl = N->getDebugLoc(); 366 367 if (getTypeAction(N->getOperand(0).getValueType()) 368 == TargetLowering::TypePromoteInteger) { 369 SDValue Res = GetPromotedInteger(N->getOperand(0)); 370 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 371 372 // If the result and operand types are the same after promotion, simplify 373 // to an in-register extension. 374 if (NVT == Res.getValueType()) { 375 // The high bits are not guaranteed to be anything. Insert an extend. 376 if (N->getOpcode() == ISD::SIGN_EXTEND) 377 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 378 DAG.getValueType(N->getOperand(0).getValueType())); 379 if (N->getOpcode() == ISD::ZERO_EXTEND) 380 return DAG.getZeroExtendInReg(Res, dl, 381 N->getOperand(0).getValueType().getScalarType()); 382 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 383 return Res; 384 } 385 } 386 387 // Otherwise, just extend the original operand all the way to the larger type. 388 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 389} 390 391SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 392 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 394 ISD::LoadExtType ExtType = 395 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 396 DebugLoc dl = N->getDebugLoc(); 397 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 398 N->getPointerInfo(), 399 N->getMemoryVT(), N->isVolatile(), 400 N->isNonTemporal(), N->getAlignment()); 401 402 // Legalized the chain result - switch anything that used the old chain to 403 // use the new one. 404 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 405 return Res; 406} 407 408/// Promote the overflow flag of an overflowing arithmetic node. 409SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 410 // Simply change the return type of the boolean result. 411 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 412 EVT ValueVTs[] = { N->getValueType(0), NVT }; 413 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) }; 414 SDValue Res = DAG.getNode(N->getOpcode(), N->getDebugLoc(), 415 DAG.getVTList(ValueVTs, 2), Ops, 2); 416 417 // Modified the sum result - switch anything that used the old sum to use 418 // the new one. 419 ReplaceValueWith(SDValue(N, 0), Res); 420 421 return SDValue(Res.getNode(), 1); 422} 423 424SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 425 if (ResNo == 1) 426 return PromoteIntRes_Overflow(N); 427 428 // The operation overflowed iff the result in the larger type is not the 429 // sign extension of its truncation to the original type. 430 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 431 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 432 EVT OVT = N->getOperand(0).getValueType(); 433 EVT NVT = LHS.getValueType(); 434 DebugLoc dl = N->getDebugLoc(); 435 436 // Do the arithmetic in the larger type. 437 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 438 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 439 440 // Calculate the overflow flag: sign extend the arithmetic result from 441 // the original type. 442 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 443 DAG.getValueType(OVT)); 444 // Overflowed if and only if this is not equal to Res. 445 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 446 447 // Use the calculated overflow everywhere. 448 ReplaceValueWith(SDValue(N, 1), Ofl); 449 450 return Res; 451} 452 453SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) { 454 // Sign extend the input. 455 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 456 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 457 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 458 LHS.getValueType(), LHS, RHS); 459} 460 461SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 462 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 463 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 464 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 465 LHS.getValueType(), N->getOperand(0),LHS,RHS); 466} 467 468SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 469 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 470 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 471 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 472 LHS.getValueType(), N->getOperand(0), 473 N->getOperand(1), LHS, RHS, N->getOperand(4)); 474} 475 476SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 477 EVT SVT = TLI.getSetCCResultType(N->getOperand(0).getValueType()); 478 assert(isTypeLegal(SVT) && "Illegal SetCC type!"); 479 DebugLoc dl = N->getDebugLoc(); 480 481 // Get the SETCC result using the canonical SETCC type. 482 SDValue SetCC = DAG.getNode(ISD::SETCC, dl, SVT, N->getOperand(0), 483 N->getOperand(1), N->getOperand(2)); 484 485 // Convert to the expected type. 486 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 487 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?"); 488 return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC); 489} 490 491SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 492 return DAG.getNode(ISD::SHL, N->getDebugLoc(), 493 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), 494 GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); 495} 496 497SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 498 SDValue Op = GetPromotedInteger(N->getOperand(0)); 499 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), 500 Op.getValueType(), Op, N->getOperand(1)); 501} 502 503SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 504 // The input may have strange things in the top bits of the registers, but 505 // these operations don't care. They may have weird bits going out, but 506 // that too is okay if they are integer operations. 507 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 508 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 509 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 510 LHS.getValueType(), LHS, RHS); 511} 512 513SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 514 // The input value must be properly sign extended. 515 SDValue Res = SExtPromotedInteger(N->getOperand(0)); 516 return DAG.getNode(ISD::SRA, N->getDebugLoc(), 517 Res.getValueType(), Res, N->getOperand(1)); 518} 519 520SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 521 // The input value must be properly zero extended. 522 EVT VT = N->getValueType(0); 523 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 524 SDValue Res = ZExtPromotedInteger(N->getOperand(0)); 525 return DAG.getNode(ISD::SRL, N->getDebugLoc(), NVT, Res, N->getOperand(1)); 526} 527 528SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 529 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 530 SDValue Res; 531 SDValue InOp = N->getOperand(0); 532 DebugLoc dl = N->getDebugLoc(); 533 534 switch (getTypeAction(InOp.getValueType())) { 535 default: llvm_unreachable("Unknown type action!"); 536 case TargetLowering::TypeLegal: 537 case TargetLowering::TypeExpandInteger: 538 Res = InOp; 539 break; 540 case TargetLowering::TypePromoteInteger: 541 Res = GetPromotedInteger(InOp); 542 break; 543 case TargetLowering::TypeSplitVector: 544 EVT InVT = InOp.getValueType(); 545 assert(InVT.isVector() && "Cannot split scalar types"); 546 unsigned NumElts = InVT.getVectorNumElements(); 547 assert(NumElts == NVT.getVectorNumElements() && 548 "Dst and Src must have the same number of elements"); 549 EVT EltVT = InVT.getScalarType(); 550 assert(isPowerOf2_32(NumElts) && 551 "Promoted vector type must be a power of two"); 552 553 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts/2); 554 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 555 NumElts/2); 556 557 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, InOp, 558 DAG.getIntPtrConstant(0)); 559 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, InOp, 560 DAG.getIntPtrConstant(NumElts/2)); 561 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 562 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 563 564 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 565 } 566 567 // Truncate to NVT instead of VT 568 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 569} 570 571SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 572 if (ResNo == 1) 573 return PromoteIntRes_Overflow(N); 574 575 // The operation overflowed iff the result in the larger type is not the 576 // zero extension of its truncation to the original type. 577 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 578 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 579 EVT OVT = N->getOperand(0).getValueType(); 580 EVT NVT = LHS.getValueType(); 581 DebugLoc dl = N->getDebugLoc(); 582 583 // Do the arithmetic in the larger type. 584 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 585 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 586 587 // Calculate the overflow flag: zero extend the arithmetic result from 588 // the original type. 589 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 590 // Overflowed if and only if this is not equal to Res. 591 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 592 593 // Use the calculated overflow everywhere. 594 ReplaceValueWith(SDValue(N, 1), Ofl); 595 596 return Res; 597} 598 599SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 600 // Promote the overflow bit trivially. 601 if (ResNo == 1) 602 return PromoteIntRes_Overflow(N); 603 604 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 605 DebugLoc DL = N->getDebugLoc(); 606 EVT SmallVT = LHS.getValueType(); 607 608 // To determine if the result overflowed in a larger type, we extend the 609 // input to the larger type, do the multiply, then check the high bits of 610 // the result to see if the overflow happened. 611 if (N->getOpcode() == ISD::SMULO) { 612 LHS = SExtPromotedInteger(LHS); 613 RHS = SExtPromotedInteger(RHS); 614 } else { 615 LHS = ZExtPromotedInteger(LHS); 616 RHS = ZExtPromotedInteger(RHS); 617 } 618 SDValue Mul = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS); 619 620 // Overflow occurred iff the high part of the result does not 621 // zero/sign-extend the low part. 622 SDValue Overflow; 623 if (N->getOpcode() == ISD::UMULO) { 624 // Unsigned overflow occurred iff the high part is non-zero. 625 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 626 DAG.getIntPtrConstant(SmallVT.getSizeInBits())); 627 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 628 DAG.getConstant(0, Hi.getValueType()), ISD::SETNE); 629 } else { 630 // Signed overflow occurred iff the high part does not sign extend the low. 631 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 632 Mul, DAG.getValueType(SmallVT)); 633 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 634 } 635 636 // Use the calculated overflow everywhere. 637 ReplaceValueWith(SDValue(N, 1), Overflow); 638 return Mul; 639} 640 641SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { 642 // Zero extend the input. 643 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 644 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 645 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 646 LHS.getValueType(), LHS, RHS); 647} 648 649SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 650 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 651 N->getValueType(0))); 652} 653 654SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 655 SDValue Chain = N->getOperand(0); // Get the chain. 656 SDValue Ptr = N->getOperand(1); // Get the pointer. 657 EVT VT = N->getValueType(0); 658 DebugLoc dl = N->getDebugLoc(); 659 660 EVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 661 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 662 // The argument is passed as NumRegs registers of type RegVT. 663 664 SmallVector<SDValue, 8> Parts(NumRegs); 665 for (unsigned i = 0; i < NumRegs; ++i) { 666 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 667 N->getConstantOperandVal(3)); 668 Chain = Parts[i].getValue(1); 669 } 670 671 // Handle endianness of the load. 672 if (TLI.isBigEndian()) 673 std::reverse(Parts.begin(), Parts.end()); 674 675 // Assemble the parts in the promoted type. 676 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 677 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 678 for (unsigned i = 1; i < NumRegs; ++i) { 679 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 680 // Shift it to the right position and "or" it in. 681 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 682 DAG.getConstant(i * RegVT.getSizeInBits(), 683 TLI.getPointerTy())); 684 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 685 } 686 687 // Modified the chain result - switch anything that used the old chain to 688 // use the new one. 689 ReplaceValueWith(SDValue(N, 1), Chain); 690 691 return Res; 692} 693 694//===----------------------------------------------------------------------===// 695// Integer Operand Promotion 696//===----------------------------------------------------------------------===// 697 698/// PromoteIntegerOperand - This method is called when the specified operand of 699/// the specified node is found to need promotion. At this point, all of the 700/// result types of the node are known to be legal, but other operands of the 701/// node may need promotion or expansion as well as the specified one. 702bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 703 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); 704 SDValue Res = SDValue(); 705 706 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 707 return false; 708 709 switch (N->getOpcode()) { 710 default: 711 #ifndef NDEBUG 712 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 713 N->dump(&DAG); dbgs() << "\n"; 714 #endif 715 llvm_unreachable("Do not know how to promote this operator's operand!"); 716 717 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 718 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 719 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 720 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 721 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 722 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 723 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 724 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 725 case ISD::CONVERT_RNDSAT: 726 Res = PromoteIntOp_CONVERT_RNDSAT(N); break; 727 case ISD::INSERT_VECTOR_ELT: 728 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 729 case ISD::MEMBARRIER: Res = PromoteIntOp_MEMBARRIER(N); break; 730 case ISD::SCALAR_TO_VECTOR: 731 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 732 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 733 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 734 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 735 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 736 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 737 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 738 OpNo); break; 739 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 740 case ISD::FP16_TO_FP32: 741 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 742 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 743 744 case ISD::SHL: 745 case ISD::SRA: 746 case ISD::SRL: 747 case ISD::ROTL: 748 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 749 } 750 751 // If the result is null, the sub-method took care of registering results etc. 752 if (!Res.getNode()) return false; 753 754 // If the result is N, the sub-method updated N in place. Tell the legalizer 755 // core about this. 756 if (Res.getNode() == N) 757 return true; 758 759 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 760 "Invalid operand expansion"); 761 762 ReplaceValueWith(SDValue(N, 0), Res); 763 return false; 764} 765 766/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 767/// shared among BR_CC, SELECT_CC, and SETCC handlers. 768void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 769 ISD::CondCode CCCode) { 770 // We have to insert explicit sign or zero extends. Note that we could 771 // insert sign extends for ALL conditions, but zero extend is cheaper on 772 // many machines (an AND instead of two shifts), so prefer it. 773 switch (CCCode) { 774 default: llvm_unreachable("Unknown integer comparison!"); 775 case ISD::SETEQ: 776 case ISD::SETNE: 777 case ISD::SETUGE: 778 case ISD::SETUGT: 779 case ISD::SETULE: 780 case ISD::SETULT: 781 // ALL of these operations will work if we either sign or zero extend 782 // the operands (including the unsigned comparisons!). Zero extend is 783 // usually a simpler/cheaper operation, so prefer it. 784 NewLHS = ZExtPromotedInteger(NewLHS); 785 NewRHS = ZExtPromotedInteger(NewRHS); 786 break; 787 case ISD::SETGE: 788 case ISD::SETGT: 789 case ISD::SETLT: 790 case ISD::SETLE: 791 NewLHS = SExtPromotedInteger(NewLHS); 792 NewRHS = SExtPromotedInteger(NewRHS); 793 break; 794 } 795} 796 797SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 798 SDValue Op = GetPromotedInteger(N->getOperand(0)); 799 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), Op); 800} 801 802SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 803 // This should only occur in unusual situations like bitcasting to an 804 // x86_fp80, so just turn it into a store+load 805 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 806} 807 808SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 809 assert(OpNo == 2 && "Don't know how to promote this operand!"); 810 811 SDValue LHS = N->getOperand(2); 812 SDValue RHS = N->getOperand(3); 813 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 814 815 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 816 // legal types. 817 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 818 N->getOperand(1), LHS, RHS, N->getOperand(4)), 819 0); 820} 821 822SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 823 assert(OpNo == 1 && "only know how to promote condition"); 824 825 // Promote all the way up to the canonical SetCC type. 826 EVT SVT = TLI.getSetCCResultType(MVT::Other); 827 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), SVT); 828 829 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 830 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 831 N->getOperand(2)), 0); 832} 833 834SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 835 // Since the result type is legal, the operands must promote to it. 836 EVT OVT = N->getOperand(0).getValueType(); 837 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 838 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 839 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 840 DebugLoc dl = N->getDebugLoc(); 841 842 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 843 DAG.getConstant(OVT.getSizeInBits(), TLI.getPointerTy())); 844 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 845} 846 847SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 848 // The vector type is legal but the element type is not. This implies 849 // that the vector is a power-of-two in length and that the element 850 // type does not have a strange size (eg: it is not i1). 851 EVT VecVT = N->getValueType(0); 852 unsigned NumElts = VecVT.getVectorNumElements(); 853 assert(!(NumElts & 1) && "Legal vector of one illegal element?"); 854 855 // Promote the inserted value. The type does not need to match the 856 // vector element type. Check that any extra bits introduced will be 857 // truncated away. 858 assert(N->getOperand(0).getValueType().getSizeInBits() >= 859 N->getValueType(0).getVectorElementType().getSizeInBits() && 860 "Type of inserted value narrower than vector element type!"); 861 862 SmallVector<SDValue, 16> NewOps; 863 for (unsigned i = 0; i < NumElts; ++i) 864 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 865 866 return SDValue(DAG.UpdateNodeOperands(N, &NewOps[0], NumElts), 0); 867} 868 869SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) { 870 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 871 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 872 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 873 CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) && 874 "can only promote integer arguments"); 875 SDValue InOp = GetPromotedInteger(N->getOperand(0)); 876 return DAG.getConvertRndSat(N->getValueType(0), N->getDebugLoc(), InOp, 877 N->getOperand(1), N->getOperand(2), 878 N->getOperand(3), N->getOperand(4), CvtCode); 879} 880 881SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 882 unsigned OpNo) { 883 if (OpNo == 1) { 884 // Promote the inserted value. This is valid because the type does not 885 // have to match the vector element type. 886 887 // Check that any extra bits introduced will be truncated away. 888 assert(N->getOperand(1).getValueType().getSizeInBits() >= 889 N->getValueType(0).getVectorElementType().getSizeInBits() && 890 "Type of inserted value narrower than vector element type!"); 891 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 892 GetPromotedInteger(N->getOperand(1)), 893 N->getOperand(2)), 894 0); 895 } 896 897 assert(OpNo == 2 && "Different operand and result vector types?"); 898 899 // Promote the index. 900 SDValue Idx = ZExtPromotedInteger(N->getOperand(2)); 901 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 902 N->getOperand(1), Idx), 0); 903} 904 905SDValue DAGTypeLegalizer::PromoteIntOp_MEMBARRIER(SDNode *N) { 906 SDValue NewOps[6]; 907 DebugLoc dl = N->getDebugLoc(); 908 NewOps[0] = N->getOperand(0); 909 for (unsigned i = 1; i < array_lengthof(NewOps); ++i) { 910 SDValue Flag = GetPromotedInteger(N->getOperand(i)); 911 NewOps[i] = DAG.getZeroExtendInReg(Flag, dl, MVT::i1); 912 } 913 return SDValue(DAG.UpdateNodeOperands(N, NewOps, array_lengthof(NewOps)), 0); 914} 915 916SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 917 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 918 // the operand in place. 919 return SDValue(DAG.UpdateNodeOperands(N, 920 GetPromotedInteger(N->getOperand(0))), 0); 921} 922 923SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 924 assert(OpNo == 0 && "Only know how to promote condition"); 925 926 // Promote all the way up to the canonical SetCC type. 927 EVT SVT = TLI.getSetCCResultType(N->getOperand(1).getValueType()); 928 SDValue Cond = PromoteTargetBoolean(N->getOperand(0), SVT); 929 930 return SDValue(DAG.UpdateNodeOperands(N, Cond, 931 N->getOperand(1), N->getOperand(2)), 0); 932} 933 934SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 935 assert(OpNo == 0 && "Don't know how to promote this operand!"); 936 937 SDValue LHS = N->getOperand(0); 938 SDValue RHS = N->getOperand(1); 939 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 940 941 // The CC (#4) and the possible return values (#2 and #3) have legal types. 942 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 943 N->getOperand(3), N->getOperand(4)), 0); 944} 945 946SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 947 assert(OpNo == 0 && "Don't know how to promote this operand!"); 948 949 SDValue LHS = N->getOperand(0); 950 SDValue RHS = N->getOperand(1); 951 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 952 953 // The CC (#2) is always legal. 954 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 955} 956 957SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 958 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 959 ZExtPromotedInteger(N->getOperand(1))), 0); 960} 961 962SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 963 SDValue Op = GetPromotedInteger(N->getOperand(0)); 964 DebugLoc dl = N->getDebugLoc(); 965 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 966 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 967 Op, DAG.getValueType(N->getOperand(0).getValueType())); 968} 969 970SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 971 return SDValue(DAG.UpdateNodeOperands(N, 972 SExtPromotedInteger(N->getOperand(0))), 0); 973} 974 975SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 976 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 977 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 978 unsigned Alignment = N->getAlignment(); 979 bool isVolatile = N->isVolatile(); 980 bool isNonTemporal = N->isNonTemporal(); 981 DebugLoc dl = N->getDebugLoc(); 982 983 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 984 985 // Truncate the value and store the result. 986 return DAG.getTruncStore(Ch, dl, Val, Ptr, N->getPointerInfo(), 987 N->getMemoryVT(), 988 isVolatile, isNonTemporal, Alignment); 989} 990 991SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 992 SDValue Op = GetPromotedInteger(N->getOperand(0)); 993 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), Op); 994} 995 996SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 997 return SDValue(DAG.UpdateNodeOperands(N, 998 ZExtPromotedInteger(N->getOperand(0))), 0); 999} 1000 1001SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1002 DebugLoc dl = N->getDebugLoc(); 1003 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1004 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1005 return DAG.getZeroExtendInReg(Op, dl, 1006 N->getOperand(0).getValueType().getScalarType()); 1007} 1008 1009 1010//===----------------------------------------------------------------------===// 1011// Integer Result Expansion 1012//===----------------------------------------------------------------------===// 1013 1014/// ExpandIntegerResult - This method is called when the specified result of the 1015/// specified node is found to need expansion. At this point, the node may also 1016/// have invalid operands or may have other results that need promotion, we just 1017/// know that (at least) one result needs expansion. 1018void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 1019 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); 1020 SDValue Lo, Hi; 1021 Lo = Hi = SDValue(); 1022 1023 // See if the target wants to custom expand this node. 1024 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1025 return; 1026 1027 switch (N->getOpcode()) { 1028 default: 1029#ifndef NDEBUG 1030 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 1031 N->dump(&DAG); dbgs() << "\n"; 1032#endif 1033 llvm_unreachable("Do not know how to expand the result of this operator!"); 1034 1035 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 1036 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1037 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1038 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1039 1040 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1041 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1042 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1043 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1044 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1045 1046 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 1047 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 1048 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 1049 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 1050 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 1051 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 1052 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 1053 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 1054 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 1055 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 1056 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 1057 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 1058 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 1059 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 1060 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 1061 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 1062 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 1063 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 1064 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 1065 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 1066 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 1067 1068 case ISD::ATOMIC_LOAD_ADD: 1069 case ISD::ATOMIC_LOAD_SUB: 1070 case ISD::ATOMIC_LOAD_AND: 1071 case ISD::ATOMIC_LOAD_OR: 1072 case ISD::ATOMIC_LOAD_XOR: 1073 case ISD::ATOMIC_LOAD_NAND: 1074 case ISD::ATOMIC_LOAD_MIN: 1075 case ISD::ATOMIC_LOAD_MAX: 1076 case ISD::ATOMIC_LOAD_UMIN: 1077 case ISD::ATOMIC_LOAD_UMAX: 1078 case ISD::ATOMIC_SWAP: { 1079 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 1080 SplitInteger(Tmp.first, Lo, Hi); 1081 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1082 break; 1083 } 1084 1085 case ISD::AND: 1086 case ISD::OR: 1087 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 1088 1089 case ISD::ADD: 1090 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 1091 1092 case ISD::ADDC: 1093 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 1094 1095 case ISD::ADDE: 1096 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 1097 1098 case ISD::SHL: 1099 case ISD::SRA: 1100 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 1101 1102 case ISD::SADDO: 1103 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 1104 case ISD::UADDO: 1105 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 1106 case ISD::UMULO: 1107 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 1108 } 1109 1110 // If Lo/Hi is null, the sub-method took care of registering results etc. 1111 if (Lo.getNode()) 1112 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 1113} 1114 1115/// Lower an atomic node to the appropriate builtin call. 1116std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 1117 unsigned Opc = Node->getOpcode(); 1118 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 1119 RTLIB::Libcall LC; 1120 1121 switch (Opc) { 1122 default: 1123 llvm_unreachable("Unhandled atomic intrinsic Expand!"); 1124 break; 1125 case ISD::ATOMIC_SWAP: 1126 switch (VT.SimpleTy) { 1127 default: llvm_unreachable("Unexpected value type for atomic!"); 1128 case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 1129 case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 1130 case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 1131 case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 1132 } 1133 break; 1134 case ISD::ATOMIC_CMP_SWAP: 1135 switch (VT.SimpleTy) { 1136 default: llvm_unreachable("Unexpected value type for atomic!"); 1137 case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 1138 case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 1139 case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 1140 case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 1141 } 1142 break; 1143 case ISD::ATOMIC_LOAD_ADD: 1144 switch (VT.SimpleTy) { 1145 default: llvm_unreachable("Unexpected value type for atomic!"); 1146 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 1147 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 1148 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 1149 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 1150 } 1151 break; 1152 case ISD::ATOMIC_LOAD_SUB: 1153 switch (VT.SimpleTy) { 1154 default: llvm_unreachable("Unexpected value type for atomic!"); 1155 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 1156 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 1157 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 1158 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 1159 } 1160 break; 1161 case ISD::ATOMIC_LOAD_AND: 1162 switch (VT.SimpleTy) { 1163 default: llvm_unreachable("Unexpected value type for atomic!"); 1164 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 1165 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 1166 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 1167 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 1168 } 1169 break; 1170 case ISD::ATOMIC_LOAD_OR: 1171 switch (VT.SimpleTy) { 1172 default: llvm_unreachable("Unexpected value type for atomic!"); 1173 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 1174 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 1175 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 1176 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 1177 } 1178 break; 1179 case ISD::ATOMIC_LOAD_XOR: 1180 switch (VT.SimpleTy) { 1181 default: llvm_unreachable("Unexpected value type for atomic!"); 1182 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 1183 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 1184 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 1185 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 1186 } 1187 break; 1188 case ISD::ATOMIC_LOAD_NAND: 1189 switch (VT.SimpleTy) { 1190 default: llvm_unreachable("Unexpected value type for atomic!"); 1191 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 1192 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 1193 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 1194 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 1195 } 1196 break; 1197 } 1198 1199 return ExpandChainLibCall(LC, Node, false); 1200} 1201 1202/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded, 1203/// and the shift amount is a constant 'Amt'. Expand the operation. 1204void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 1205 SDValue &Lo, SDValue &Hi) { 1206 DebugLoc DL = N->getDebugLoc(); 1207 // Expand the incoming operand to be shifted, so that we have its parts 1208 SDValue InL, InH; 1209 GetExpandedInteger(N->getOperand(0), InL, InH); 1210 1211 EVT NVT = InL.getValueType(); 1212 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1213 unsigned NVTBits = NVT.getSizeInBits(); 1214 EVT ShTy = N->getOperand(1).getValueType(); 1215 1216 if (N->getOpcode() == ISD::SHL) { 1217 if (Amt > VTBits) { 1218 Lo = Hi = DAG.getConstant(0, NVT); 1219 } else if (Amt > NVTBits) { 1220 Lo = DAG.getConstant(0, NVT); 1221 Hi = DAG.getNode(ISD::SHL, DL, 1222 NVT, InL, DAG.getConstant(Amt-NVTBits, ShTy)); 1223 } else if (Amt == NVTBits) { 1224 Lo = DAG.getConstant(0, NVT); 1225 Hi = InL; 1226 } else if (Amt == 1 && 1227 TLI.isOperationLegalOrCustom(ISD::ADDC, 1228 TLI.getTypeToExpandTo(*DAG.getContext(), NVT))) { 1229 // Emit this X << 1 as X+X. 1230 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1231 SDValue LoOps[2] = { InL, InL }; 1232 Lo = DAG.getNode(ISD::ADDC, DL, VTList, LoOps, 2); 1233 SDValue HiOps[3] = { InH, InH, Lo.getValue(1) }; 1234 Hi = DAG.getNode(ISD::ADDE, DL, VTList, HiOps, 3); 1235 } else { 1236 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, ShTy)); 1237 Hi = DAG.getNode(ISD::OR, DL, NVT, 1238 DAG.getNode(ISD::SHL, DL, NVT, InH, 1239 DAG.getConstant(Amt, ShTy)), 1240 DAG.getNode(ISD::SRL, DL, NVT, InL, 1241 DAG.getConstant(NVTBits-Amt, ShTy))); 1242 } 1243 return; 1244 } 1245 1246 if (N->getOpcode() == ISD::SRL) { 1247 if (Amt > VTBits) { 1248 Lo = DAG.getConstant(0, NVT); 1249 Hi = DAG.getConstant(0, NVT); 1250 } else if (Amt > NVTBits) { 1251 Lo = DAG.getNode(ISD::SRL, DL, 1252 NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy)); 1253 Hi = DAG.getConstant(0, NVT); 1254 } else if (Amt == NVTBits) { 1255 Lo = InH; 1256 Hi = DAG.getConstant(0, NVT); 1257 } else { 1258 Lo = DAG.getNode(ISD::OR, DL, NVT, 1259 DAG.getNode(ISD::SRL, DL, NVT, InL, 1260 DAG.getConstant(Amt, ShTy)), 1261 DAG.getNode(ISD::SHL, DL, NVT, InH, 1262 DAG.getConstant(NVTBits-Amt, ShTy))); 1263 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1264 } 1265 return; 1266 } 1267 1268 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1269 if (Amt > VTBits) { 1270 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1271 DAG.getConstant(NVTBits-1, ShTy)); 1272 } else if (Amt > NVTBits) { 1273 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1274 DAG.getConstant(Amt-NVTBits, ShTy)); 1275 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1276 DAG.getConstant(NVTBits-1, ShTy)); 1277 } else if (Amt == NVTBits) { 1278 Lo = InH; 1279 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1280 DAG.getConstant(NVTBits-1, ShTy)); 1281 } else { 1282 Lo = DAG.getNode(ISD::OR, DL, NVT, 1283 DAG.getNode(ISD::SRL, DL, NVT, InL, 1284 DAG.getConstant(Amt, ShTy)), 1285 DAG.getNode(ISD::SHL, DL, NVT, InH, 1286 DAG.getConstant(NVTBits-Amt, ShTy))); 1287 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1288 } 1289} 1290 1291/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1292/// this shift based on knowledge of the high bit of the shift amount. If we 1293/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1294/// shift amount. 1295bool DAGTypeLegalizer:: 1296ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1297 SDValue Amt = N->getOperand(1); 1298 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1299 EVT ShTy = Amt.getValueType(); 1300 unsigned ShBits = ShTy.getScalarType().getSizeInBits(); 1301 unsigned NVTBits = NVT.getScalarType().getSizeInBits(); 1302 assert(isPowerOf2_32(NVTBits) && 1303 "Expanded integer type size not a power of two!"); 1304 DebugLoc dl = N->getDebugLoc(); 1305 1306 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1307 APInt KnownZero, KnownOne; 1308 DAG.ComputeMaskedBits(N->getOperand(1), HighBitMask, KnownZero, KnownOne); 1309 1310 // If we don't know anything about the high bits, exit. 1311 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1312 return false; 1313 1314 // Get the incoming operand to be shifted. 1315 SDValue InL, InH; 1316 GetExpandedInteger(N->getOperand(0), InL, InH); 1317 1318 // If we know that any of the high bits of the shift amount are one, then we 1319 // can do this as a couple of simple shifts. 1320 if (KnownOne.intersects(HighBitMask)) { 1321 // Mask out the high bit, which we know is set. 1322 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 1323 DAG.getConstant(~HighBitMask, ShTy)); 1324 1325 switch (N->getOpcode()) { 1326 default: llvm_unreachable("Unknown shift"); 1327 case ISD::SHL: 1328 Lo = DAG.getConstant(0, NVT); // Low part is zero. 1329 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 1330 return true; 1331 case ISD::SRL: 1332 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 1333 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 1334 return true; 1335 case ISD::SRA: 1336 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 1337 DAG.getConstant(NVTBits-1, ShTy)); 1338 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 1339 return true; 1340 } 1341 } 1342 1343#if 0 1344 // FIXME: This code is broken for shifts with a zero amount! 1345 // If we know that all of the high bits of the shift amount are zero, then we 1346 // can do this as a couple of simple shifts. 1347 if ((KnownZero & HighBitMask) == HighBitMask) { 1348 // Compute 32-amt. 1349 SDValue Amt2 = DAG.getNode(ISD::SUB, ShTy, 1350 DAG.getConstant(NVTBits, ShTy), 1351 Amt); 1352 unsigned Op1, Op2; 1353 switch (N->getOpcode()) { 1354 default: llvm_unreachable("Unknown shift"); 1355 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1356 case ISD::SRL: 1357 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1358 } 1359 1360 Lo = DAG.getNode(N->getOpcode(), NVT, InL, Amt); 1361 Hi = DAG.getNode(ISD::OR, NVT, 1362 DAG.getNode(Op1, NVT, InH, Amt), 1363 DAG.getNode(Op2, NVT, InL, Amt2)); 1364 return true; 1365 } 1366#endif 1367 1368 return false; 1369} 1370 1371/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 1372/// of any size. 1373bool DAGTypeLegalizer:: 1374ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1375 SDValue Amt = N->getOperand(1); 1376 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1377 EVT ShTy = Amt.getValueType(); 1378 unsigned NVTBits = NVT.getSizeInBits(); 1379 assert(isPowerOf2_32(NVTBits) && 1380 "Expanded integer type size not a power of two!"); 1381 DebugLoc dl = N->getDebugLoc(); 1382 1383 // Get the incoming operand to be shifted. 1384 SDValue InL, InH; 1385 GetExpandedInteger(N->getOperand(0), InL, InH); 1386 1387 SDValue NVBitsNode = DAG.getConstant(NVTBits, ShTy); 1388 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 1389 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 1390 SDValue isShort = DAG.getSetCC(dl, TLI.getSetCCResultType(ShTy), 1391 Amt, NVBitsNode, ISD::SETULT); 1392 1393 SDValue LoS, HiS, LoL, HiL; 1394 switch (N->getOpcode()) { 1395 default: llvm_unreachable("Unknown shift"); 1396 case ISD::SHL: 1397 // Short: ShAmt < NVTBits 1398 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 1399 HiS = DAG.getNode(ISD::OR, dl, NVT, 1400 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 1401 // FIXME: If Amt is zero, the following shift generates an undefined result 1402 // on some architectures. 1403 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 1404 1405 // Long: ShAmt >= NVTBits 1406 LoL = DAG.getConstant(0, NVT); // Lo part is zero. 1407 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 1408 1409 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1410 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1411 return true; 1412 case ISD::SRL: 1413 // Short: ShAmt < NVTBits 1414 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 1415 LoS = DAG.getNode(ISD::OR, dl, NVT, 1416 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1417 // FIXME: If Amt is zero, the following shift generates an undefined result 1418 // on some architectures. 1419 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1420 1421 // Long: ShAmt >= NVTBits 1422 HiL = DAG.getConstant(0, NVT); // Hi part is zero. 1423 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1424 1425 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1426 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1427 return true; 1428 case ISD::SRA: 1429 // Short: ShAmt < NVTBits 1430 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 1431 LoS = DAG.getNode(ISD::OR, dl, NVT, 1432 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1433 // FIXME: If Amt is zero, the following shift generates an undefined result 1434 // on some architectures. 1435 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1436 1437 // Long: ShAmt >= NVTBits 1438 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 1439 DAG.getConstant(NVTBits-1, ShTy)); 1440 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1441 1442 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1443 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1444 return true; 1445 } 1446 1447 return false; 1448} 1449 1450void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1451 SDValue &Lo, SDValue &Hi) { 1452 DebugLoc dl = N->getDebugLoc(); 1453 // Expand the subcomponents. 1454 SDValue LHSL, LHSH, RHSL, RHSH; 1455 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1456 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1457 1458 EVT NVT = LHSL.getValueType(); 1459 SDValue LoOps[2] = { LHSL, RHSL }; 1460 SDValue HiOps[3] = { LHSH, RHSH }; 1461 1462 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1463 // them. TODO: Teach operation legalization how to expand unsupported 1464 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1465 // a carry of type MVT::Glue, but there doesn't seem to be any way to 1466 // generate a value of this type in the expanded code sequence. 1467 bool hasCarry = 1468 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1469 ISD::ADDC : ISD::SUBC, 1470 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1471 1472 if (hasCarry) { 1473 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1474 if (N->getOpcode() == ISD::ADD) { 1475 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1476 HiOps[2] = Lo.getValue(1); 1477 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1478 } else { 1479 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 1480 HiOps[2] = Lo.getValue(1); 1481 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 1482 } 1483 return; 1484 } 1485 1486 if (N->getOpcode() == ISD::ADD) { 1487 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps, 2); 1488 Hi = DAG.getNode(ISD::ADD, dl, NVT, HiOps, 2); 1489 SDValue Cmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[0], 1490 ISD::SETULT); 1491 SDValue Carry1 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp1, 1492 DAG.getConstant(1, NVT), 1493 DAG.getConstant(0, NVT)); 1494 SDValue Cmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[1], 1495 ISD::SETULT); 1496 SDValue Carry2 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp2, 1497 DAG.getConstant(1, NVT), Carry1); 1498 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 1499 } else { 1500 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps, 2); 1501 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps, 2); 1502 SDValue Cmp = 1503 DAG.getSetCC(dl, TLI.getSetCCResultType(LoOps[0].getValueType()), 1504 LoOps[0], LoOps[1], ISD::SETULT); 1505 SDValue Borrow = DAG.getNode(ISD::SELECT, dl, NVT, Cmp, 1506 DAG.getConstant(1, NVT), 1507 DAG.getConstant(0, NVT)); 1508 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 1509 } 1510} 1511 1512void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1513 SDValue &Lo, SDValue &Hi) { 1514 // Expand the subcomponents. 1515 SDValue LHSL, LHSH, RHSL, RHSH; 1516 DebugLoc dl = N->getDebugLoc(); 1517 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1518 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1519 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1520 SDValue LoOps[2] = { LHSL, RHSL }; 1521 SDValue HiOps[3] = { LHSH, RHSH }; 1522 1523 if (N->getOpcode() == ISD::ADDC) { 1524 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1525 HiOps[2] = Lo.getValue(1); 1526 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1527 } else { 1528 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 1529 HiOps[2] = Lo.getValue(1); 1530 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 1531 } 1532 1533 // Legalized the flag result - switch anything that used the old flag to 1534 // use the new one. 1535 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1536} 1537 1538void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1539 SDValue &Lo, SDValue &Hi) { 1540 // Expand the subcomponents. 1541 SDValue LHSL, LHSH, RHSL, RHSH; 1542 DebugLoc dl = N->getDebugLoc(); 1543 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1544 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1545 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1546 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1547 SDValue HiOps[3] = { LHSH, RHSH }; 1548 1549 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps, 3); 1550 HiOps[2] = Lo.getValue(1); 1551 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps, 3); 1552 1553 // Legalized the flag result - switch anything that used the old flag to 1554 // use the new one. 1555 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1556} 1557 1558void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(SDNode *N, 1559 SDValue &Lo, SDValue &Hi) { 1560 SDValue Res = DecomposeMERGE_VALUES(N); 1561 SplitInteger(Res, Lo, Hi); 1562} 1563 1564void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 1565 SDValue &Lo, SDValue &Hi) { 1566 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1567 DebugLoc dl = N->getDebugLoc(); 1568 SDValue Op = N->getOperand(0); 1569 if (Op.getValueType().bitsLE(NVT)) { 1570 // The low part is any extension of the input (which degenerates to a copy). 1571 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 1572 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 1573 } else { 1574 // For example, extension of an i48 to an i64. The operand type necessarily 1575 // promotes to the result type, so will end up being expanded too. 1576 assert(getTypeAction(Op.getValueType()) == 1577 TargetLowering::TypePromoteInteger && 1578 "Only know how to promote this result!"); 1579 SDValue Res = GetPromotedInteger(Op); 1580 assert(Res.getValueType() == N->getValueType(0) && 1581 "Operand over promoted?"); 1582 // Split the promoted operand. This will simplify when it is expanded. 1583 SplitInteger(Res, Lo, Hi); 1584 } 1585} 1586 1587void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 1588 SDValue &Lo, SDValue &Hi) { 1589 DebugLoc dl = N->getDebugLoc(); 1590 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1591 EVT NVT = Lo.getValueType(); 1592 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1593 unsigned NVTBits = NVT.getSizeInBits(); 1594 unsigned EVTBits = EVT.getSizeInBits(); 1595 1596 if (NVTBits < EVTBits) { 1597 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 1598 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1599 EVTBits - NVTBits))); 1600 } else { 1601 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 1602 // The high part replicates the sign bit of Lo, make it explicit. 1603 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1604 DAG.getConstant(NVTBits-1, TLI.getPointerTy())); 1605 } 1606} 1607 1608void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 1609 SDValue &Lo, SDValue &Hi) { 1610 DebugLoc dl = N->getDebugLoc(); 1611 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1612 EVT NVT = Lo.getValueType(); 1613 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1614 unsigned NVTBits = NVT.getSizeInBits(); 1615 unsigned EVTBits = EVT.getSizeInBits(); 1616 1617 if (NVTBits < EVTBits) { 1618 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 1619 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1620 EVTBits - NVTBits))); 1621 } else { 1622 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 1623 // The high part must be zero, make it explicit. 1624 Hi = DAG.getConstant(0, NVT); 1625 } 1626} 1627 1628void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1629 SDValue &Lo, SDValue &Hi) { 1630 DebugLoc dl = N->getDebugLoc(); 1631 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1632 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 1633 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 1634} 1635 1636void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 1637 SDValue &Lo, SDValue &Hi) { 1638 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1639 unsigned NBitWidth = NVT.getSizeInBits(); 1640 const APInt &Cst = cast<ConstantSDNode>(N)->getAPIntValue(); 1641 Lo = DAG.getConstant(Cst.trunc(NBitWidth), NVT); 1642 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT); 1643} 1644 1645void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1646 SDValue &Lo, SDValue &Hi) { 1647 DebugLoc dl = N->getDebugLoc(); 1648 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1649 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1650 EVT NVT = Lo.getValueType(); 1651 1652 SDValue HiNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Hi, 1653 DAG.getConstant(0, NVT), ISD::SETNE); 1654 1655 SDValue LoLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Lo); 1656 SDValue HiLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Hi); 1657 1658 Lo = DAG.getNode(ISD::SELECT, dl, NVT, HiNotZero, HiLZ, 1659 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 1660 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1661 Hi = DAG.getConstant(0, NVT); 1662} 1663 1664void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1665 SDValue &Lo, SDValue &Hi) { 1666 DebugLoc dl = N->getDebugLoc(); 1667 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1668 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1669 EVT NVT = Lo.getValueType(); 1670 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 1671 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 1672 Hi = DAG.getConstant(0, NVT); 1673} 1674 1675void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1676 SDValue &Lo, SDValue &Hi) { 1677 DebugLoc dl = N->getDebugLoc(); 1678 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1679 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1680 EVT NVT = Lo.getValueType(); 1681 1682 SDValue LoNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, 1683 DAG.getConstant(0, NVT), ISD::SETNE); 1684 1685 SDValue LoLZ = DAG.getNode(ISD::CTTZ, dl, NVT, Lo); 1686 SDValue HiLZ = DAG.getNode(ISD::CTTZ, dl, NVT, Hi); 1687 1688 Lo = DAG.getNode(ISD::SELECT, dl, NVT, LoNotZero, LoLZ, 1689 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 1690 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1691 Hi = DAG.getConstant(0, NVT); 1692} 1693 1694void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 1695 SDValue &Hi) { 1696 DebugLoc dl = N->getDebugLoc(); 1697 EVT VT = N->getValueType(0); 1698 SDValue Op = N->getOperand(0); 1699 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 1700 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 1701 SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*irrelevant*/, dl), Lo, Hi); 1702} 1703 1704void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 1705 SDValue &Hi) { 1706 DebugLoc dl = N->getDebugLoc(); 1707 EVT VT = N->getValueType(0); 1708 SDValue Op = N->getOperand(0); 1709 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 1710 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 1711 SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*irrelevant*/, dl), Lo, Hi); 1712} 1713 1714void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 1715 SDValue &Lo, SDValue &Hi) { 1716 if (ISD::isNormalLoad(N)) { 1717 ExpandRes_NormalLoad(N, Lo, Hi); 1718 return; 1719 } 1720 1721 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1722 1723 EVT VT = N->getValueType(0); 1724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1725 SDValue Ch = N->getChain(); 1726 SDValue Ptr = N->getBasePtr(); 1727 ISD::LoadExtType ExtType = N->getExtensionType(); 1728 unsigned Alignment = N->getAlignment(); 1729 bool isVolatile = N->isVolatile(); 1730 bool isNonTemporal = N->isNonTemporal(); 1731 DebugLoc dl = N->getDebugLoc(); 1732 1733 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1734 1735 if (N->getMemoryVT().bitsLE(NVT)) { 1736 EVT MemVT = N->getMemoryVT(); 1737 1738 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 1739 MemVT, isVolatile, isNonTemporal, Alignment); 1740 1741 // Remember the chain. 1742 Ch = Lo.getValue(1); 1743 1744 if (ExtType == ISD::SEXTLOAD) { 1745 // The high part is obtained by SRA'ing all but one of the bits of the 1746 // lo part. 1747 unsigned LoSize = Lo.getValueType().getSizeInBits(); 1748 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1749 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 1750 } else if (ExtType == ISD::ZEXTLOAD) { 1751 // The high part is just a zero. 1752 Hi = DAG.getConstant(0, NVT); 1753 } else { 1754 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 1755 // The high part is undefined. 1756 Hi = DAG.getUNDEF(NVT); 1757 } 1758 } else if (TLI.isLittleEndian()) { 1759 // Little-endian - low bits are at low addresses. 1760 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 1761 isVolatile, isNonTemporal, Alignment); 1762 1763 unsigned ExcessBits = 1764 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1765 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 1766 1767 // Increment the pointer to the other half. 1768 unsigned IncrementSize = NVT.getSizeInBits()/8; 1769 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1770 DAG.getIntPtrConstant(IncrementSize)); 1771 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 1772 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 1773 isVolatile, isNonTemporal, 1774 MinAlign(Alignment, IncrementSize)); 1775 1776 // Build a factor node to remember that this load is independent of the 1777 // other one. 1778 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1779 Hi.getValue(1)); 1780 } else { 1781 // Big-endian - high bits are at low addresses. Favor aligned loads at 1782 // the cost of some bit-fiddling. 1783 EVT MemVT = N->getMemoryVT(); 1784 unsigned EBytes = MemVT.getStoreSize(); 1785 unsigned IncrementSize = NVT.getSizeInBits()/8; 1786 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1787 1788 // Load both the high bits and maybe some of the low bits. 1789 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 1790 EVT::getIntegerVT(*DAG.getContext(), 1791 MemVT.getSizeInBits() - ExcessBits), 1792 isVolatile, isNonTemporal, Alignment); 1793 1794 // Increment the pointer to the other half. 1795 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1796 DAG.getIntPtrConstant(IncrementSize)); 1797 // Load the rest of the low bits. 1798 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 1799 N->getPointerInfo().getWithOffset(IncrementSize), 1800 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 1801 isVolatile, isNonTemporal, 1802 MinAlign(Alignment, IncrementSize)); 1803 1804 // Build a factor node to remember that this load is independent of the 1805 // other one. 1806 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1807 Hi.getValue(1)); 1808 1809 if (ExcessBits < NVT.getSizeInBits()) { 1810 // Transfer low bits from the bottom of Hi to the top of Lo. 1811 Lo = DAG.getNode(ISD::OR, dl, NVT, Lo, 1812 DAG.getNode(ISD::SHL, dl, NVT, Hi, 1813 DAG.getConstant(ExcessBits, 1814 TLI.getPointerTy()))); 1815 // Move high bits to the right position in Hi. 1816 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, 1817 NVT, Hi, 1818 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 1819 TLI.getPointerTy())); 1820 } 1821 } 1822 1823 // Legalized the chain result - switch anything that used the old chain to 1824 // use the new one. 1825 ReplaceValueWith(SDValue(N, 1), Ch); 1826} 1827 1828void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 1829 SDValue &Lo, SDValue &Hi) { 1830 DebugLoc dl = N->getDebugLoc(); 1831 SDValue LL, LH, RL, RH; 1832 GetExpandedInteger(N->getOperand(0), LL, LH); 1833 GetExpandedInteger(N->getOperand(1), RL, RH); 1834 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 1835 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 1836} 1837 1838void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 1839 SDValue &Lo, SDValue &Hi) { 1840 EVT VT = N->getValueType(0); 1841 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1842 DebugLoc dl = N->getDebugLoc(); 1843 1844 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT); 1845 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT); 1846 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT); 1847 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT); 1848 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 1849 SDValue LL, LH, RL, RH; 1850 GetExpandedInteger(N->getOperand(0), LL, LH); 1851 GetExpandedInteger(N->getOperand(1), RL, RH); 1852 unsigned OuterBitSize = VT.getSizeInBits(); 1853 unsigned InnerBitSize = NVT.getSizeInBits(); 1854 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0)); 1855 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1)); 1856 1857 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 1858 if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) && 1859 DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) { 1860 // The inputs are both zero-extended. 1861 if (HasUMUL_LOHI) { 1862 // We can emit a umul_lohi. 1863 Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 1864 Hi = SDValue(Lo.getNode(), 1); 1865 return; 1866 } 1867 if (HasMULHU) { 1868 // We can emit a mulhu+mul. 1869 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1870 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 1871 return; 1872 } 1873 } 1874 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 1875 // The input values are both sign-extended. 1876 if (HasSMUL_LOHI) { 1877 // We can emit a smul_lohi. 1878 Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 1879 Hi = SDValue(Lo.getNode(), 1); 1880 return; 1881 } 1882 if (HasMULHS) { 1883 // We can emit a mulhs+mul. 1884 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1885 Hi = DAG.getNode(ISD::MULHS, dl, NVT, LL, RL); 1886 return; 1887 } 1888 } 1889 if (HasUMUL_LOHI) { 1890 // Lo,Hi = umul LHS, RHS. 1891 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl, 1892 DAG.getVTList(NVT, NVT), LL, RL); 1893 Lo = UMulLOHI; 1894 Hi = UMulLOHI.getValue(1); 1895 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 1896 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 1897 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 1898 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 1899 return; 1900 } 1901 if (HasMULHU) { 1902 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1903 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 1904 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 1905 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 1906 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 1907 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 1908 return; 1909 } 1910 } 1911 1912 // If nothing else, we can make a libcall. 1913 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1914 if (VT == MVT::i16) 1915 LC = RTLIB::MUL_I16; 1916 else if (VT == MVT::i32) 1917 LC = RTLIB::MUL_I32; 1918 else if (VT == MVT::i64) 1919 LC = RTLIB::MUL_I64; 1920 else if (VT == MVT::i128) 1921 LC = RTLIB::MUL_I128; 1922 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!"); 1923 1924 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1925 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*irrelevant*/, dl), Lo, Hi); 1926} 1927 1928void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 1929 SDValue &Lo, SDValue &Hi) { 1930 SDValue LHS = Node->getOperand(0); 1931 SDValue RHS = Node->getOperand(1); 1932 DebugLoc dl = Node->getDebugLoc(); 1933 1934 // Expand the result by simply replacing it with the equivalent 1935 // non-overflow-checking operation. 1936 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 1937 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 1938 LHS, RHS); 1939 SplitInteger(Sum, Lo, Hi); 1940 1941 // Compute the overflow. 1942 // 1943 // LHSSign -> LHS >= 0 1944 // RHSSign -> RHS >= 0 1945 // SumSign -> Sum >= 0 1946 // 1947 // Add: 1948 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 1949 // Sub: 1950 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 1951 // 1952 EVT OType = Node->getValueType(1); 1953 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 1954 1955 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 1956 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 1957 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 1958 Node->getOpcode() == ISD::SADDO ? 1959 ISD::SETEQ : ISD::SETNE); 1960 1961 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 1962 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 1963 1964 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 1965 1966 // Use the calculated overflow everywhere. 1967 ReplaceValueWith(SDValue(Node, 1), Cmp); 1968} 1969 1970void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 1971 SDValue &Lo, SDValue &Hi) { 1972 EVT VT = N->getValueType(0); 1973 DebugLoc dl = N->getDebugLoc(); 1974 1975 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1976 if (VT == MVT::i16) 1977 LC = RTLIB::SDIV_I16; 1978 else if (VT == MVT::i32) 1979 LC = RTLIB::SDIV_I32; 1980 else if (VT == MVT::i64) 1981 LC = RTLIB::SDIV_I64; 1982 else if (VT == MVT::i128) 1983 LC = RTLIB::SDIV_I128; 1984 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 1985 1986 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1987 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); 1988} 1989 1990void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 1991 SDValue &Lo, SDValue &Hi) { 1992 EVT VT = N->getValueType(0); 1993 DebugLoc dl = N->getDebugLoc(); 1994 1995 // If we can emit an efficient shift operation, do so now. Check to see if 1996 // the RHS is a constant. 1997 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 1998 return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi); 1999 2000 // If we can determine that the high bit of the shift is zero or one, even if 2001 // the low bits are variable, emit this shift in an optimized form. 2002 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 2003 return; 2004 2005 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 2006 unsigned PartsOpc; 2007 if (N->getOpcode() == ISD::SHL) { 2008 PartsOpc = ISD::SHL_PARTS; 2009 } else if (N->getOpcode() == ISD::SRL) { 2010 PartsOpc = ISD::SRL_PARTS; 2011 } else { 2012 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2013 PartsOpc = ISD::SRA_PARTS; 2014 } 2015 2016 // Next check to see if the target supports this SHL_PARTS operation or if it 2017 // will custom expand it. 2018 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2019 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 2020 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 2021 Action == TargetLowering::Custom) { 2022 // Expand the subcomponents. 2023 SDValue LHSL, LHSH; 2024 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2025 2026 SDValue Ops[] = { LHSL, LHSH, N->getOperand(1) }; 2027 EVT VT = LHSL.getValueType(); 2028 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops, 3); 2029 Hi = Lo.getValue(1); 2030 return; 2031 } 2032 2033 // Otherwise, emit a libcall. 2034 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2035 bool isSigned; 2036 if (N->getOpcode() == ISD::SHL) { 2037 isSigned = false; /*sign irrelevant*/ 2038 if (VT == MVT::i16) 2039 LC = RTLIB::SHL_I16; 2040 else if (VT == MVT::i32) 2041 LC = RTLIB::SHL_I32; 2042 else if (VT == MVT::i64) 2043 LC = RTLIB::SHL_I64; 2044 else if (VT == MVT::i128) 2045 LC = RTLIB::SHL_I128; 2046 } else if (N->getOpcode() == ISD::SRL) { 2047 isSigned = false; 2048 if (VT == MVT::i16) 2049 LC = RTLIB::SRL_I16; 2050 else if (VT == MVT::i32) 2051 LC = RTLIB::SRL_I32; 2052 else if (VT == MVT::i64) 2053 LC = RTLIB::SRL_I64; 2054 else if (VT == MVT::i128) 2055 LC = RTLIB::SRL_I128; 2056 } else { 2057 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2058 isSigned = true; 2059 if (VT == MVT::i16) 2060 LC = RTLIB::SRA_I16; 2061 else if (VT == MVT::i32) 2062 LC = RTLIB::SRA_I32; 2063 else if (VT == MVT::i64) 2064 LC = RTLIB::SRA_I64; 2065 else if (VT == MVT::i128) 2066 LC = RTLIB::SRA_I128; 2067 } 2068 2069 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 2070 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2071 SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned, dl), Lo, Hi); 2072 return; 2073 } 2074 2075 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 2076 llvm_unreachable("Unsupported shift!"); 2077} 2078 2079void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 2080 SDValue &Lo, SDValue &Hi) { 2081 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2082 DebugLoc dl = N->getDebugLoc(); 2083 SDValue Op = N->getOperand(0); 2084 if (Op.getValueType().bitsLE(NVT)) { 2085 // The low part is sign extension of the input (degenerates to a copy). 2086 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 2087 // The high part is obtained by SRA'ing all but one of the bits of low part. 2088 unsigned LoSize = NVT.getSizeInBits(); 2089 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2090 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 2091 } else { 2092 // For example, extension of an i48 to an i64. The operand type necessarily 2093 // promotes to the result type, so will end up being expanded too. 2094 assert(getTypeAction(Op.getValueType()) == 2095 TargetLowering::TypePromoteInteger && 2096 "Only know how to promote this result!"); 2097 SDValue Res = GetPromotedInteger(Op); 2098 assert(Res.getValueType() == N->getValueType(0) && 2099 "Operand over promoted?"); 2100 // Split the promoted operand. This will simplify when it is expanded. 2101 SplitInteger(Res, Lo, Hi); 2102 unsigned ExcessBits = 2103 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2104 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2105 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2106 ExcessBits))); 2107 } 2108} 2109 2110void DAGTypeLegalizer:: 2111ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2112 DebugLoc dl = N->getDebugLoc(); 2113 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2114 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2115 2116 if (EVT.bitsLE(Lo.getValueType())) { 2117 // sext_inreg the low part if needed. 2118 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 2119 N->getOperand(1)); 2120 2121 // The high part gets the sign extension from the lo-part. This handles 2122 // things like sextinreg V:i64 from i8. 2123 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 2124 DAG.getConstant(Hi.getValueType().getSizeInBits()-1, 2125 TLI.getPointerTy())); 2126 } else { 2127 // For example, extension of an i48 to an i64. Leave the low part alone, 2128 // sext_inreg the high part. 2129 unsigned ExcessBits = 2130 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 2131 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2132 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2133 ExcessBits))); 2134 } 2135} 2136 2137void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 2138 SDValue &Lo, SDValue &Hi) { 2139 EVT VT = N->getValueType(0); 2140 DebugLoc dl = N->getDebugLoc(); 2141 2142 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2143 if (VT == MVT::i16) 2144 LC = RTLIB::SREM_I16; 2145 else if (VT == MVT::i32) 2146 LC = RTLIB::SREM_I32; 2147 else if (VT == MVT::i64) 2148 LC = RTLIB::SREM_I64; 2149 else if (VT == MVT::i128) 2150 LC = RTLIB::SREM_I128; 2151 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 2152 2153 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2154 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); 2155} 2156 2157void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 2158 SDValue &Lo, SDValue &Hi) { 2159 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2160 DebugLoc dl = N->getDebugLoc(); 2161 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 2162 Hi = DAG.getNode(ISD::SRL, dl, 2163 N->getOperand(0).getValueType(), N->getOperand(0), 2164 DAG.getConstant(NVT.getSizeInBits(), TLI.getPointerTy())); 2165 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 2166} 2167 2168void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 2169 SDValue &Lo, SDValue &Hi) { 2170 SDValue LHS = N->getOperand(0); 2171 SDValue RHS = N->getOperand(1); 2172 DebugLoc dl = N->getDebugLoc(); 2173 2174 // Expand the result by simply replacing it with the equivalent 2175 // non-overflow-checking operation. 2176 SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ? 2177 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2178 LHS, RHS); 2179 SplitInteger(Sum, Lo, Hi); 2180 2181 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2182 // overflows iff a - b > a. 2183 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, 2184 N->getOpcode () == ISD::UADDO ? 2185 ISD::SETULT : ISD::SETUGT); 2186 2187 // Use the calculated overflow everywhere. 2188 ReplaceValueWith(SDValue(N, 1), Ofl); 2189} 2190 2191void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 2192 SDValue &Lo, SDValue &Hi) { 2193 EVT VT = N->getValueType(0); 2194 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 2195 EVT PtrVT = TLI.getPointerTy(); 2196 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 2197 DebugLoc dl = N->getDebugLoc(); 2198 2199 // A divide for UMULO should be faster than a function call. 2200 if (N->getOpcode() == ISD::UMULO) { 2201 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 2202 DebugLoc DL = N->getDebugLoc(); 2203 2204 SDValue MUL = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS); 2205 SplitInteger(MUL, Lo, Hi); 2206 2207 // A divide for UMULO will be faster than a function call. Select to 2208 // make sure we aren't using 0. 2209 SDValue isZero = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), 2210 RHS, DAG.getConstant(0, VT), ISD::SETNE); 2211 SDValue NotZero = DAG.getNode(ISD::SELECT, dl, VT, isZero, 2212 DAG.getConstant(1, VT), RHS); 2213 SDValue DIV = DAG.getNode(ISD::UDIV, DL, LHS.getValueType(), MUL, NotZero); 2214 SDValue Overflow; 2215 Overflow = DAG.getSetCC(DL, N->getValueType(1), DIV, LHS, ISD::SETNE); 2216 ReplaceValueWith(SDValue(N, 1), Overflow); 2217 return; 2218 } 2219 2220 // Replace this with a libcall that will check overflow. 2221 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2222 if (VT == MVT::i32) 2223 LC = RTLIB::MULO_I32; 2224 else if (VT == MVT::i64) 2225 LC = RTLIB::MULO_I64; 2226 else if (VT == MVT::i128) 2227 LC = RTLIB::MULO_I128; 2228 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 2229 2230 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 2231 // Temporary for the overflow value, default it to zero. 2232 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, 2233 DAG.getConstant(0, PtrVT), Temp, 2234 MachinePointerInfo(), false, false, 0); 2235 2236 TargetLowering::ArgListTy Args; 2237 TargetLowering::ArgListEntry Entry; 2238 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2239 EVT ArgVT = N->getOperand(i).getValueType(); 2240 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2241 Entry.Node = N->getOperand(i); 2242 Entry.Ty = ArgTy; 2243 Entry.isSExt = true; 2244 Entry.isZExt = false; 2245 Args.push_back(Entry); 2246 } 2247 2248 // Also pass the address of the overflow check. 2249 Entry.Node = Temp; 2250 Entry.Ty = PtrTy->getPointerTo(); 2251 Entry.isSExt = true; 2252 Entry.isZExt = false; 2253 Args.push_back(Entry); 2254 2255 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 2256 std::pair<SDValue, SDValue> CallInfo = 2257 TLI.LowerCallTo(Chain, RetTy, true, false, false, false, 2258 0, TLI.getLibcallCallingConv(LC), false, 2259 true, Func, Args, DAG, dl); 2260 2261 SplitInteger(CallInfo.first, Lo, Hi); 2262 SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, 2263 MachinePointerInfo(), false, false, 0); 2264 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 2265 DAG.getConstant(0, PtrVT), 2266 ISD::SETNE); 2267 // Use the overflow from the libcall everywhere. 2268 ReplaceValueWith(SDValue(N, 1), Ofl); 2269} 2270 2271void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 2272 SDValue &Lo, SDValue &Hi) { 2273 EVT VT = N->getValueType(0); 2274 DebugLoc dl = N->getDebugLoc(); 2275 2276 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2277 if (VT == MVT::i16) 2278 LC = RTLIB::UDIV_I16; 2279 else if (VT == MVT::i32) 2280 LC = RTLIB::UDIV_I32; 2281 else if (VT == MVT::i64) 2282 LC = RTLIB::UDIV_I64; 2283 else if (VT == MVT::i128) 2284 LC = RTLIB::UDIV_I128; 2285 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 2286 2287 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2288 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); 2289} 2290 2291void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 2292 SDValue &Lo, SDValue &Hi) { 2293 EVT VT = N->getValueType(0); 2294 DebugLoc dl = N->getDebugLoc(); 2295 2296 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2297 if (VT == MVT::i16) 2298 LC = RTLIB::UREM_I16; 2299 else if (VT == MVT::i32) 2300 LC = RTLIB::UREM_I32; 2301 else if (VT == MVT::i64) 2302 LC = RTLIB::UREM_I64; 2303 else if (VT == MVT::i128) 2304 LC = RTLIB::UREM_I128; 2305 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 2306 2307 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2308 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); 2309} 2310 2311void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 2312 SDValue &Lo, SDValue &Hi) { 2313 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2314 DebugLoc dl = N->getDebugLoc(); 2315 SDValue Op = N->getOperand(0); 2316 if (Op.getValueType().bitsLE(NVT)) { 2317 // The low part is zero extension of the input (degenerates to a copy). 2318 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 2319 Hi = DAG.getConstant(0, NVT); // The high part is just a zero. 2320 } else { 2321 // For example, extension of an i48 to an i64. The operand type necessarily 2322 // promotes to the result type, so will end up being expanded too. 2323 assert(getTypeAction(Op.getValueType()) == 2324 TargetLowering::TypePromoteInteger && 2325 "Only know how to promote this result!"); 2326 SDValue Res = GetPromotedInteger(Op); 2327 assert(Res.getValueType() == N->getValueType(0) && 2328 "Operand over promoted?"); 2329 // Split the promoted operand. This will simplify when it is expanded. 2330 SplitInteger(Res, Lo, Hi); 2331 unsigned ExcessBits = 2332 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2333 Hi = DAG.getZeroExtendInReg(Hi, dl, 2334 EVT::getIntegerVT(*DAG.getContext(), 2335 ExcessBits)); 2336 } 2337} 2338 2339void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 2340 SDValue &Lo, SDValue &Hi) { 2341 DebugLoc dl = N->getDebugLoc(); 2342 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 2343 SDValue Zero = DAG.getConstant(0, VT); 2344 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT, 2345 N->getOperand(0), 2346 N->getOperand(1), Zero, Zero, 2347 cast<AtomicSDNode>(N)->getMemOperand(), 2348 cast<AtomicSDNode>(N)->getOrdering(), 2349 cast<AtomicSDNode>(N)->getSynchScope()); 2350 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 2351 ReplaceValueWith(SDValue(N, 1), Swap.getValue(1)); 2352} 2353 2354//===----------------------------------------------------------------------===// 2355// Integer Operand Expansion 2356//===----------------------------------------------------------------------===// 2357 2358/// ExpandIntegerOperand - This method is called when the specified operand of 2359/// the specified node is found to need expansion. At this point, all of the 2360/// result types of the node are known to be legal, but other operands of the 2361/// node may need promotion or expansion as well as the specified one. 2362bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 2363 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); 2364 SDValue Res = SDValue(); 2365 2366 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2367 return false; 2368 2369 switch (N->getOpcode()) { 2370 default: 2371 #ifndef NDEBUG 2372 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 2373 N->dump(&DAG); dbgs() << "\n"; 2374 #endif 2375 llvm_unreachable("Do not know how to expand this operator's operand!"); 2376 2377 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2378 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 2379 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2380 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2381 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 2382 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 2383 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 2384 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 2385 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 2386 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 2387 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 2388 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 2389 2390 case ISD::SHL: 2391 case ISD::SRA: 2392 case ISD::SRL: 2393 case ISD::ROTL: 2394 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 2395 case ISD::RETURNADDR: 2396 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 2397 2398 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 2399 } 2400 2401 // If the result is null, the sub-method took care of registering results etc. 2402 if (!Res.getNode()) return false; 2403 2404 // If the result is N, the sub-method updated N in place. Tell the legalizer 2405 // core about this. 2406 if (Res.getNode() == N) 2407 return true; 2408 2409 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2410 "Invalid operand expansion"); 2411 2412 ReplaceValueWith(SDValue(N, 0), Res); 2413 return false; 2414} 2415 2416/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 2417/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2418void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 2419 SDValue &NewRHS, 2420 ISD::CondCode &CCCode, 2421 DebugLoc dl) { 2422 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2423 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 2424 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 2425 2426 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 2427 if (RHSLo == RHSHi) { 2428 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 2429 if (RHSCST->isAllOnesValue()) { 2430 // Equality comparison to -1. 2431 NewLHS = DAG.getNode(ISD::AND, dl, 2432 LHSLo.getValueType(), LHSLo, LHSHi); 2433 NewRHS = RHSLo; 2434 return; 2435 } 2436 } 2437 } 2438 2439 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 2440 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 2441 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 2442 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2443 return; 2444 } 2445 2446 // If this is a comparison of the sign bit, just look at the top part. 2447 // X > -1, x < 0 2448 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 2449 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 2450 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 2451 NewLHS = LHSHi; 2452 NewRHS = RHSHi; 2453 return; 2454 } 2455 2456 // FIXME: This generated code sucks. 2457 ISD::CondCode LowCC; 2458 switch (CCCode) { 2459 default: llvm_unreachable("Unknown integer setcc!"); 2460 case ISD::SETLT: 2461 case ISD::SETULT: LowCC = ISD::SETULT; break; 2462 case ISD::SETGT: 2463 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 2464 case ISD::SETLE: 2465 case ISD::SETULE: LowCC = ISD::SETULE; break; 2466 case ISD::SETGE: 2467 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 2468 } 2469 2470 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 2471 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 2472 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 2473 2474 // NOTE: on targets without efficient SELECT of bools, we can always use 2475 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 2476 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, true, NULL); 2477 SDValue Tmp1, Tmp2; 2478 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), 2479 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); 2480 if (!Tmp1.getNode()) 2481 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), 2482 LHSLo, RHSLo, LowCC); 2483 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 2484 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); 2485 if (!Tmp2.getNode()) 2486 Tmp2 = DAG.getNode(ISD::SETCC, dl, 2487 TLI.getSetCCResultType(LHSHi.getValueType()), 2488 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 2489 2490 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 2491 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 2492 if ((Tmp1C && Tmp1C->isNullValue()) || 2493 (Tmp2C && Tmp2C->isNullValue() && 2494 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 2495 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 2496 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 2497 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 2498 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 2499 // low part is known false, returns high part. 2500 // For LE / GE, if high part is known false, ignore the low part. 2501 // For LT / GT, if high part is known true, ignore the low part. 2502 NewLHS = Tmp2; 2503 NewRHS = SDValue(); 2504 return; 2505 } 2506 2507 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 2508 LHSHi, RHSHi, ISD::SETEQ, false, 2509 DagCombineInfo, dl); 2510 if (!NewLHS.getNode()) 2511 NewLHS = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 2512 LHSHi, RHSHi, ISD::SETEQ); 2513 NewLHS = DAG.getNode(ISD::SELECT, dl, Tmp1.getValueType(), 2514 NewLHS, Tmp1, Tmp2); 2515 NewRHS = SDValue(); 2516} 2517 2518SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 2519 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 2520 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 2521 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2522 2523 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2524 // against zero to select between true and false values. 2525 if (NewRHS.getNode() == 0) { 2526 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2527 CCCode = ISD::SETNE; 2528 } 2529 2530 // Update N to have the operands specified. 2531 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2532 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2533 N->getOperand(4)), 0); 2534} 2535 2536SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 2537 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2538 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2539 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2540 2541 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2542 // against zero to select between true and false values. 2543 if (NewRHS.getNode() == 0) { 2544 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2545 CCCode = ISD::SETNE; 2546 } 2547 2548 // Update N to have the operands specified. 2549 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2550 N->getOperand(2), N->getOperand(3), 2551 DAG.getCondCode(CCCode)), 0); 2552} 2553 2554SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 2555 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2556 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2557 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2558 2559 // If ExpandSetCCOperands returned a scalar, use it. 2560 if (NewRHS.getNode() == 0) { 2561 assert(NewLHS.getValueType() == N->getValueType(0) && 2562 "Unexpected setcc expansion!"); 2563 return NewLHS; 2564 } 2565 2566 // Otherwise, update N to have the operands specified. 2567 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2568 DAG.getCondCode(CCCode)), 0); 2569} 2570 2571SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 2572 // The value being shifted is legal, but the shift amount is too big. 2573 // It follows that either the result of the shift is undefined, or the 2574 // upper half of the shift amount is zero. Just use the lower half. 2575 SDValue Lo, Hi; 2576 GetExpandedInteger(N->getOperand(1), Lo, Hi); 2577 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 2578} 2579 2580SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 2581 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 2582 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 2583 // constant to valid type. 2584 SDValue Lo, Hi; 2585 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2586 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 2587} 2588 2589SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 2590 SDValue Op = N->getOperand(0); 2591 EVT DstVT = N->getValueType(0); 2592 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 2593 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2594 "Don't know how to expand this SINT_TO_FP!"); 2595 return MakeLibCall(LC, DstVT, &Op, 1, true, N->getDebugLoc()); 2596} 2597 2598SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 2599 if (ISD::isNormalStore(N)) 2600 return ExpandOp_NormalStore(N, OpNo); 2601 2602 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2603 assert(OpNo == 1 && "Can only expand the stored value so far"); 2604 2605 EVT VT = N->getOperand(1).getValueType(); 2606 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2607 SDValue Ch = N->getChain(); 2608 SDValue Ptr = N->getBasePtr(); 2609 unsigned Alignment = N->getAlignment(); 2610 bool isVolatile = N->isVolatile(); 2611 bool isNonTemporal = N->isNonTemporal(); 2612 DebugLoc dl = N->getDebugLoc(); 2613 SDValue Lo, Hi; 2614 2615 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2616 2617 if (N->getMemoryVT().bitsLE(NVT)) { 2618 GetExpandedInteger(N->getValue(), Lo, Hi); 2619 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 2620 N->getMemoryVT(), isVolatile, isNonTemporal, 2621 Alignment); 2622 } 2623 2624 if (TLI.isLittleEndian()) { 2625 // Little-endian - low bits are at low addresses. 2626 GetExpandedInteger(N->getValue(), Lo, Hi); 2627 2628 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 2629 isVolatile, isNonTemporal, Alignment); 2630 2631 unsigned ExcessBits = 2632 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2633 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2634 2635 // Increment the pointer to the other half. 2636 unsigned IncrementSize = NVT.getSizeInBits()/8; 2637 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2638 DAG.getIntPtrConstant(IncrementSize)); 2639 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 2640 N->getPointerInfo().getWithOffset(IncrementSize), 2641 NEVT, isVolatile, isNonTemporal, 2642 MinAlign(Alignment, IncrementSize)); 2643 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2644 } 2645 2646 // Big-endian - high bits are at low addresses. Favor aligned stores at 2647 // the cost of some bit-fiddling. 2648 GetExpandedInteger(N->getValue(), Lo, Hi); 2649 2650 EVT ExtVT = N->getMemoryVT(); 2651 unsigned EBytes = ExtVT.getStoreSize(); 2652 unsigned IncrementSize = NVT.getSizeInBits()/8; 2653 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2654 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 2655 ExtVT.getSizeInBits() - ExcessBits); 2656 2657 if (ExcessBits < NVT.getSizeInBits()) { 2658 // Transfer high bits from the top of Lo to the bottom of Hi. 2659 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 2660 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 2661 TLI.getPointerTy())); 2662 Hi = DAG.getNode(ISD::OR, dl, NVT, Hi, 2663 DAG.getNode(ISD::SRL, dl, NVT, Lo, 2664 DAG.getConstant(ExcessBits, 2665 TLI.getPointerTy()))); 2666 } 2667 2668 // Store both the high bits and maybe some of the low bits. 2669 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), 2670 HiVT, isVolatile, isNonTemporal, Alignment); 2671 2672 // Increment the pointer to the other half. 2673 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2674 DAG.getIntPtrConstant(IncrementSize)); 2675 // Store the lowest ExcessBits bits in the second half. 2676 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 2677 N->getPointerInfo().getWithOffset(IncrementSize), 2678 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2679 isVolatile, isNonTemporal, 2680 MinAlign(Alignment, IncrementSize)); 2681 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2682} 2683 2684SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 2685 SDValue InL, InH; 2686 GetExpandedInteger(N->getOperand(0), InL, InH); 2687 // Just truncate the low part of the source. 2688 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), InL); 2689} 2690 2691static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { 2692 switch (VT.getSimpleVT().SimpleTy) { 2693 default: llvm_unreachable("Unknown FP format"); 2694 case MVT::f32: return &APFloat::IEEEsingle; 2695 case MVT::f64: return &APFloat::IEEEdouble; 2696 case MVT::f80: return &APFloat::x87DoubleExtended; 2697 case MVT::f128: return &APFloat::IEEEquad; 2698 case MVT::ppcf128: return &APFloat::PPCDoubleDouble; 2699 } 2700} 2701 2702SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 2703 SDValue Op = N->getOperand(0); 2704 EVT SrcVT = Op.getValueType(); 2705 EVT DstVT = N->getValueType(0); 2706 DebugLoc dl = N->getDebugLoc(); 2707 2708 // The following optimization is valid only if every value in SrcVT (when 2709 // treated as signed) is representable in DstVT. Check that the mantissa 2710 // size of DstVT is >= than the number of bits in SrcVT -1. 2711 const fltSemantics *sem = EVTToAPFloatSemantics(DstVT); 2712 if (APFloat::semanticsPrecision(*sem) >= SrcVT.getSizeInBits()-1 && 2713 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 2714 // Do a signed conversion then adjust the result. 2715 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); 2716 SignedConv = TLI.LowerOperation(SignedConv, DAG); 2717 2718 // The result of the signed conversion needs adjusting if the 'sign bit' of 2719 // the incoming integer was set. To handle this, we dynamically test to see 2720 // if it is set, and, if so, add a fudge factor. 2721 2722 const uint64_t F32TwoE32 = 0x4F800000ULL; 2723 const uint64_t F32TwoE64 = 0x5F800000ULL; 2724 const uint64_t F32TwoE128 = 0x7F800000ULL; 2725 2726 APInt FF(32, 0); 2727 if (SrcVT == MVT::i32) 2728 FF = APInt(32, F32TwoE32); 2729 else if (SrcVT == MVT::i64) 2730 FF = APInt(32, F32TwoE64); 2731 else if (SrcVT == MVT::i128) 2732 FF = APInt(32, F32TwoE128); 2733 else 2734 assert(false && "Unsupported UINT_TO_FP!"); 2735 2736 // Check whether the sign bit is set. 2737 SDValue Lo, Hi; 2738 GetExpandedInteger(Op, Lo, Hi); 2739 SDValue SignSet = DAG.getSetCC(dl, 2740 TLI.getSetCCResultType(Hi.getValueType()), 2741 Hi, DAG.getConstant(0, Hi.getValueType()), 2742 ISD::SETLT); 2743 2744 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 2745 SDValue FudgePtr = DAG.getConstantPool( 2746 ConstantInt::get(*DAG.getContext(), FF.zext(64)), 2747 TLI.getPointerTy()); 2748 2749 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 2750 SDValue Zero = DAG.getIntPtrConstant(0); 2751 SDValue Four = DAG.getIntPtrConstant(4); 2752 if (TLI.isBigEndian()) std::swap(Zero, Four); 2753 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet, 2754 Zero, Four); 2755 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 2756 FudgePtr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), FudgePtr, Offset); 2757 Alignment = std::min(Alignment, 4u); 2758 2759 // Load the value out, extending it from f32 to the destination float type. 2760 // FIXME: Avoid the extend by constructing the right constant pool? 2761 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), 2762 FudgePtr, 2763 MachinePointerInfo::getConstantPool(), 2764 MVT::f32, 2765 false, false, Alignment); 2766 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); 2767 } 2768 2769 // Otherwise, use a libcall. 2770 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 2771 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2772 "Don't know how to expand this UINT_TO_FP!"); 2773 return MakeLibCall(LC, DstVT, &Op, 1, true, dl); 2774} 2775 2776SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 2777 DebugLoc dl = N->getDebugLoc(); 2778 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 2779 cast<AtomicSDNode>(N)->getMemoryVT(), 2780 N->getOperand(0), 2781 N->getOperand(1), N->getOperand(2), 2782 cast<AtomicSDNode>(N)->getMemOperand(), 2783 cast<AtomicSDNode>(N)->getOrdering(), 2784 cast<AtomicSDNode>(N)->getSynchScope()); 2785 return Swap.getValue(1); 2786} 2787 2788 2789SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 2790 SDValue InOp0 = N->getOperand(0); 2791 EVT InVT = InOp0.getValueType(); 2792 2793 EVT OutVT = N->getValueType(0); 2794 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2795 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2796 unsigned OutNumElems = OutVT.getVectorNumElements(); 2797 EVT NOutVTElem = NOutVT.getVectorElementType(); 2798 2799 DebugLoc dl = N->getDebugLoc(); 2800 SDValue BaseIdx = N->getOperand(1); 2801 2802 SmallVector<SDValue, 8> Ops; 2803 Ops.reserve(OutNumElems); 2804 for (unsigned i = 0; i != OutNumElems; ++i) { 2805 2806 // Extract the element from the original vector. 2807 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 2808 BaseIdx, DAG.getIntPtrConstant(i)); 2809 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 2810 InVT.getVectorElementType(), N->getOperand(0), Index); 2811 2812 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); 2813 // Insert the converted element to the new vector. 2814 Ops.push_back(Op); 2815 } 2816 2817 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size()); 2818} 2819 2820 2821SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 2822 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 2823 EVT VT = N->getValueType(0); 2824 DebugLoc dl = N->getDebugLoc(); 2825 2826 unsigned NumElts = VT.getVectorNumElements(); 2827 SmallVector<int, 8> NewMask; 2828 for (unsigned i = 0; i != NumElts; ++i) { 2829 NewMask.push_back(SV->getMaskElt(i)); 2830 } 2831 2832 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 2833 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 2834 EVT OutVT = V0.getValueType(); 2835 2836 return DAG.getVectorShuffle(OutVT, dl, V0, V1, &NewMask[0]); 2837} 2838 2839 2840SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 2841 EVT OutVT = N->getValueType(0); 2842 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2843 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2844 unsigned NumElems = N->getNumOperands(); 2845 EVT NOutVTElem = NOutVT.getVectorElementType(); 2846 2847 DebugLoc dl = N->getDebugLoc(); 2848 2849 SmallVector<SDValue, 8> Ops; 2850 Ops.reserve(NumElems); 2851 for (unsigned i = 0; i != NumElems; ++i) { 2852 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 2853 Ops.push_back(Op); 2854 } 2855 2856 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size()); 2857} 2858 2859SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 2860 2861 DebugLoc dl = N->getDebugLoc(); 2862 2863 assert(!N->getOperand(0).getValueType().isVector() && 2864 "Input must be a scalar"); 2865 2866 EVT OutVT = N->getValueType(0); 2867 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2868 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2869 EVT NOutVTElem = NOutVT.getVectorElementType(); 2870 2871 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 2872 2873 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 2874} 2875 2876SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 2877 EVT OutVT = N->getValueType(0); 2878 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2879 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2880 2881 EVT NOutVTElem = NOutVT.getVectorElementType(); 2882 2883 DebugLoc dl = N->getDebugLoc(); 2884 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 2885 2886 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 2887 NOutVTElem, N->getOperand(1)); 2888 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 2889 V0, ConvElem, N->getOperand(2)); 2890} 2891 2892SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2893 DebugLoc dl = N->getDebugLoc(); 2894 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 2895 SDValue V1 = N->getOperand(1); 2896 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 2897 V0->getValueType(0).getScalarType(), V0, V1); 2898 2899 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 2900 2901} 2902 2903SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 2904 DebugLoc dl = N->getDebugLoc(); 2905 unsigned NumElems = N->getNumOperands(); 2906 2907 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 2908 2909 SmallVector<SDValue, 8> NewOps; 2910 NewOps.reserve(NumElems); 2911 2912 // For each incoming vector 2913 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 2914 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 2915 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 2916 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 2917 2918 for (unsigned i=0; i<NumElem; ++i) { 2919 // Extract element from incoming vector 2920 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, 2921 Incoming, DAG.getIntPtrConstant(i)); 2922 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 2923 NewOps.push_back(Tr); 2924 } 2925 } 2926 2927 return DAG.getNode(ISD::BUILD_VECTOR, dl, N->getValueType(0), 2928 &NewOps[0], NewOps.size()); 2929 } 2930