LegalizeIntegerTypes.cpp revision f54ff27d4aa92d5dde327b0ba524a4a4ec2ae00c
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22using namespace llvm; 23 24//===----------------------------------------------------------------------===// 25// Integer Result Promotion 26//===----------------------------------------------------------------------===// 27 28/// PromoteIntegerResult - This method is called when a result of a node is 29/// found to be in need of promotion to a larger type. At this point, the node 30/// may also have invalid operands or may have other results that need 31/// expansion, we just know that (at least) one result needs promotion. 32void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 33 DEBUG(cerr << "Promote integer result: "; N->dump(&DAG); cerr << "\n"); 34 SDValue Result = SDValue(); 35 36 // See if the target wants to custom expand this node. 37 if (CustomLowerResults(N, ResNo)) 38 return; 39 40 switch (N->getOpcode()) { 41 default: 42#ifndef NDEBUG 43 cerr << "PromoteIntegerResult #" << ResNo << ": "; 44 N->dump(&DAG); cerr << "\n"; 45#endif 46 assert(0 && "Do not know how to promote this operator!"); 47 abort(); 48 case ISD::AssertSext: Result = PromoteIntRes_AssertSext(N); break; 49 case ISD::AssertZext: Result = PromoteIntRes_AssertZext(N); break; 50 case ISD::BIT_CONVERT: Result = PromoteIntRes_BIT_CONVERT(N); break; 51 case ISD::BSWAP: Result = PromoteIntRes_BSWAP(N); break; 52 case ISD::BUILD_PAIR: Result = PromoteIntRes_BUILD_PAIR(N); break; 53 case ISD::Constant: Result = PromoteIntRes_Constant(N); break; 54 case ISD::CONVERT_RNDSAT: 55 Result = PromoteIntRes_CONVERT_RNDSAT(N); break; 56 case ISD::CTLZ: Result = PromoteIntRes_CTLZ(N); break; 57 case ISD::CTPOP: Result = PromoteIntRes_CTPOP(N); break; 58 case ISD::CTTZ: Result = PromoteIntRes_CTTZ(N); break; 59 case ISD::EXTRACT_VECTOR_ELT: 60 Result = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 61 case ISD::LOAD: Result = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break; 62 case ISD::SELECT: Result = PromoteIntRes_SELECT(N); break; 63 case ISD::SELECT_CC: Result = PromoteIntRes_SELECT_CC(N); break; 64 case ISD::SETCC: Result = PromoteIntRes_SETCC(N); break; 65 case ISD::SHL: Result = PromoteIntRes_SHL(N); break; 66 case ISD::SIGN_EXTEND_INREG: 67 Result = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 68 case ISD::SRA: Result = PromoteIntRes_SRA(N); break; 69 case ISD::SRL: Result = PromoteIntRes_SRL(N); break; 70 case ISD::TRUNCATE: Result = PromoteIntRes_TRUNCATE(N); break; 71 case ISD::UNDEF: Result = PromoteIntRes_UNDEF(N); break; 72 case ISD::VAARG: Result = PromoteIntRes_VAARG(N); break; 73 74 case ISD::SIGN_EXTEND: 75 case ISD::ZERO_EXTEND: 76 case ISD::ANY_EXTEND: Result = PromoteIntRes_INT_EXTEND(N); break; 77 78 case ISD::FP_TO_SINT: 79 case ISD::FP_TO_UINT: Result = PromoteIntRes_FP_TO_XINT(N); break; 80 81 case ISD::AND: 82 case ISD::OR: 83 case ISD::XOR: 84 case ISD::ADD: 85 case ISD::SUB: 86 case ISD::MUL: Result = PromoteIntRes_SimpleIntBinOp(N); break; 87 88 case ISD::SDIV: 89 case ISD::SREM: Result = PromoteIntRes_SDIV(N); break; 90 91 case ISD::UDIV: 92 case ISD::UREM: Result = PromoteIntRes_UDIV(N); break; 93 94 case ISD::SADDO: 95 case ISD::SSUBO: Result = PromoteIntRes_SADDSUBO(N, ResNo); break; 96 case ISD::UADDO: 97 case ISD::USUBO: Result = PromoteIntRes_UADDSUBO(N, ResNo); break; 98 case ISD::SMULO: 99 case ISD::UMULO: Result = PromoteIntRes_XMULO(N, ResNo); break; 100 101 case ISD::ATOMIC_LOAD_ADD: 102 case ISD::ATOMIC_LOAD_SUB: 103 case ISD::ATOMIC_LOAD_AND: 104 case ISD::ATOMIC_LOAD_OR: 105 case ISD::ATOMIC_LOAD_XOR: 106 case ISD::ATOMIC_LOAD_NAND: 107 case ISD::ATOMIC_LOAD_MIN: 108 case ISD::ATOMIC_LOAD_MAX: 109 case ISD::ATOMIC_LOAD_UMIN: 110 case ISD::ATOMIC_LOAD_UMAX: 111 case ISD::ATOMIC_SWAP: 112 Result = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 113 114 case ISD::ATOMIC_CMP_SWAP: 115 Result = PromoteIntRes_Atomic2(cast<AtomicSDNode>(N)); break; 116 } 117 118 // If Result is null, the sub-method took care of registering the result. 119 if (Result.getNode()) 120 SetPromotedInteger(SDValue(N, ResNo), Result); 121} 122 123SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 124 // Sign-extend the new bits, and continue the assertion. 125 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 126 return DAG.getNode(ISD::AssertSext, Op.getValueType(), Op, N->getOperand(1)); 127} 128 129SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 130 // Zero the new bits, and continue the assertion. 131 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 132 return DAG.getNode(ISD::AssertZext, Op.getValueType(), Op, N->getOperand(1)); 133} 134 135SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 136 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 137 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getMemoryVT(), 138 N->getChain(), N->getBasePtr(), 139 Op2, N->getSrcValue(), N->getAlignment()); 140 // Legalized the chain result - switch anything that used the old chain to 141 // use the new one. 142 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 143 return Res; 144} 145 146SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) { 147 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 148 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 149 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getMemoryVT(), 150 N->getChain(), N->getBasePtr(), 151 Op2, Op3, N->getSrcValue(), N->getAlignment()); 152 // Legalized the chain result - switch anything that used the old chain to 153 // use the new one. 154 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 155 return Res; 156} 157 158SDValue DAGTypeLegalizer::PromoteIntRes_BIT_CONVERT(SDNode *N) { 159 SDValue InOp = N->getOperand(0); 160 MVT InVT = InOp.getValueType(); 161 MVT NInVT = TLI.getTypeToTransformTo(InVT); 162 MVT OutVT = N->getValueType(0); 163 MVT NOutVT = TLI.getTypeToTransformTo(OutVT); 164 165 switch (getTypeAction(InVT)) { 166 default: 167 assert(false && "Unknown type action!"); 168 break; 169 case Legal: 170 break; 171 case PromoteInteger: 172 if (NOutVT.bitsEq(NInVT)) 173 // The input promotes to the same size. Convert the promoted value. 174 return DAG.getNode(ISD::BIT_CONVERT, NOutVT, GetPromotedInteger(InOp)); 175 break; 176 case SoftenFloat: 177 // Promote the integer operand by hand. 178 return DAG.getNode(ISD::ANY_EXTEND, NOutVT, GetSoftenedFloat(InOp)); 179 case ExpandInteger: 180 case ExpandFloat: 181 break; 182 case ScalarizeVector: 183 // Convert the element to an integer and promote it by hand. 184 return DAG.getNode(ISD::ANY_EXTEND, NOutVT, 185 BitConvertToInteger(GetScalarizedVector(InOp))); 186 case SplitVector: { 187 // For example, i32 = BIT_CONVERT v2i16 on alpha. Convert the split 188 // pieces of the input into integers and reassemble in the final type. 189 SDValue Lo, Hi; 190 GetSplitVector(N->getOperand(0), Lo, Hi); 191 Lo = BitConvertToInteger(Lo); 192 Hi = BitConvertToInteger(Hi); 193 194 if (TLI.isBigEndian()) 195 std::swap(Lo, Hi); 196 197 InOp = DAG.getNode(ISD::ANY_EXTEND, 198 MVT::getIntegerVT(NOutVT.getSizeInBits()), 199 JoinIntegers(Lo, Hi)); 200 return DAG.getNode(ISD::BIT_CONVERT, NOutVT, InOp); 201 } 202 case WidenVector: 203 if (OutVT.bitsEq(NInVT)) 204 // The input is widened to the same size. Convert to the widened value. 205 return DAG.getNode(ISD::BIT_CONVERT, OutVT, GetWidenedVector(InOp)); 206 } 207 208 // Otherwise, lower the bit-convert to a store/load from the stack. 209 // Create the stack frame object. Make sure it is aligned for both 210 // the source and destination types. 211 SDValue FIPtr = DAG.CreateStackTemporary(InVT, OutVT); 212 213 // Emit a store to the stack slot. 214 SDValue Store = DAG.getStore(DAG.getEntryNode(), InOp, FIPtr, NULL, 0); 215 216 // Result is an extending load from the stack slot. 217 return DAG.getExtLoad(ISD::EXTLOAD, NOutVT, Store, FIPtr, NULL, 0, OutVT); 218} 219 220SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 221 SDValue Op = GetPromotedInteger(N->getOperand(0)); 222 MVT OVT = N->getValueType(0); 223 MVT NVT = Op.getValueType(); 224 225 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 226 return DAG.getNode(ISD::SRL, NVT, DAG.getNode(ISD::BSWAP, NVT, Op), 227 DAG.getConstant(DiffBits, TLI.getShiftAmountTy())); 228} 229 230SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 231 // The pair element type may be legal, or may not promote to the same type as 232 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 233 return DAG.getNode(ISD::ANY_EXTEND, 234 TLI.getTypeToTransformTo(N->getValueType(0)), 235 JoinIntegers(N->getOperand(0), N->getOperand(1))); 236} 237 238SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 239 MVT VT = N->getValueType(0); 240 // Zero extend things like i1, sign extend everything else. It shouldn't 241 // matter in theory which one we pick, but this tends to give better code? 242 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 243 SDValue Result = DAG.getNode(Opc, TLI.getTypeToTransformTo(VT), 244 SDValue(N, 0)); 245 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 246 return Result; 247} 248 249SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) { 250 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 251 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 252 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 253 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 254 "can only promote integers"); 255 MVT OutVT = TLI.getTypeToTransformTo(N->getValueType(0)); 256 return DAG.getConvertRndSat(OutVT, N->getOperand(0), 257 N->getOperand(1), N->getOperand(2), 258 N->getOperand(3), N->getOperand(4), CvtCode); 259} 260 261SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 262 // Zero extend to the promoted type and do the count there. 263 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 264 MVT OVT = N->getValueType(0); 265 MVT NVT = Op.getValueType(); 266 Op = DAG.getNode(ISD::CTLZ, NVT, Op); 267 // Subtract off the extra leading bits in the bigger type. 268 return DAG.getNode(ISD::SUB, NVT, Op, 269 DAG.getConstant(NVT.getSizeInBits() - 270 OVT.getSizeInBits(), NVT)); 271} 272 273SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 274 // Zero extend to the promoted type and do the count there. 275 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 276 return DAG.getNode(ISD::CTPOP, Op.getValueType(), Op); 277} 278 279SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 280 SDValue Op = GetPromotedInteger(N->getOperand(0)); 281 MVT OVT = N->getValueType(0); 282 MVT NVT = Op.getValueType(); 283 // The count is the same in the promoted type except if the original 284 // value was zero. This can be handled by setting the bit just off 285 // the top of the original type. 286 APInt TopBit(NVT.getSizeInBits(), 0); 287 TopBit.set(OVT.getSizeInBits()); 288 Op = DAG.getNode(ISD::OR, NVT, Op, DAG.getConstant(TopBit, NVT)); 289 return DAG.getNode(ISD::CTTZ, NVT, Op); 290} 291 292SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 293 MVT OldVT = N->getValueType(0); 294 SDValue OldVec = N->getOperand(0); 295 unsigned OldElts = OldVec.getValueType().getVectorNumElements(); 296 297 if (OldElts == 1) { 298 assert(!isTypeLegal(OldVec.getValueType()) && 299 "Legal one-element vector of a type needing promotion!"); 300 // It is tempting to follow GetScalarizedVector by a call to 301 // GetPromotedInteger, but this would be wrong because the 302 // scalarized value may not yet have been processed. 303 return DAG.getNode(ISD::ANY_EXTEND, TLI.getTypeToTransformTo(OldVT), 304 GetScalarizedVector(OldVec)); 305 } 306 307 // Convert to a vector half as long with an element type of twice the width, 308 // for example <4 x i16> -> <2 x i32>. 309 assert(!(OldElts & 1) && "Odd length vectors not supported!"); 310 MVT NewVT = MVT::getIntegerVT(2 * OldVT.getSizeInBits()); 311 assert(OldVT.isSimple() && NewVT.isSimple()); 312 313 SDValue NewVec = DAG.getNode(ISD::BIT_CONVERT, 314 MVT::getVectorVT(NewVT, OldElts / 2), 315 OldVec); 316 317 // Extract the element at OldIdx / 2 from the new vector. 318 SDValue OldIdx = N->getOperand(1); 319 SDValue NewIdx = DAG.getNode(ISD::SRL, OldIdx.getValueType(), OldIdx, 320 DAG.getConstant(1, TLI.getShiftAmountTy())); 321 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, NewIdx); 322 323 // Select the appropriate half of the element: Lo if OldIdx was even, 324 // Hi if it was odd. 325 SDValue Lo = Elt; 326 SDValue Hi = DAG.getNode(ISD::SRL, NewVT, Elt, 327 DAG.getConstant(OldVT.getSizeInBits(), 328 TLI.getShiftAmountTy())); 329 if (TLI.isBigEndian()) 330 std::swap(Lo, Hi); 331 332 // Extend to the promoted type. 333 SDValue Odd = DAG.getNode(ISD::TRUNCATE, MVT::i1, OldIdx); 334 SDValue Res = DAG.getNode(ISD::SELECT, NewVT, Odd, Hi, Lo); 335 return DAG.getNode(ISD::ANY_EXTEND, TLI.getTypeToTransformTo(OldVT), Res); 336} 337 338SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 339 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 340 unsigned NewOpc = N->getOpcode(); 341 342 // If we're promoting a UINT to a larger size, check to see if the new node 343 // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since 344 // we can use that instead. This allows us to generate better code for 345 // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not 346 // legal, such as PowerPC. 347 if (N->getOpcode() == ISD::FP_TO_UINT && 348 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 349 TLI.isOperationLegal(ISD::FP_TO_SINT, NVT)) 350 NewOpc = ISD::FP_TO_SINT; 351 352 SDValue Res = DAG.getNode(NewOpc, NVT, N->getOperand(0)); 353 354 // Assert that the converted value fits in the original type. If it doesn't 355 // (eg: because the value being converted is too big), then the result of the 356 // original operation was undefined anyway, so the assert is still correct. 357 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 358 ISD::AssertZext : ISD::AssertSext, 359 NVT, Res, DAG.getValueType(N->getValueType(0))); 360} 361 362SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 363 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 364 365 if (getTypeAction(N->getOperand(0).getValueType()) == PromoteInteger) { 366 SDValue Res = GetPromotedInteger(N->getOperand(0)); 367 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 368 369 // If the result and operand types are the same after promotion, simplify 370 // to an in-register extension. 371 if (NVT == Res.getValueType()) { 372 // The high bits are not guaranteed to be anything. Insert an extend. 373 if (N->getOpcode() == ISD::SIGN_EXTEND) 374 return DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Res, 375 DAG.getValueType(N->getOperand(0).getValueType())); 376 if (N->getOpcode() == ISD::ZERO_EXTEND) 377 return DAG.getZeroExtendInReg(Res, N->getOperand(0).getValueType()); 378 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 379 return Res; 380 } 381 } 382 383 // Otherwise, just extend the original operand all the way to the larger type. 384 return DAG.getNode(N->getOpcode(), NVT, N->getOperand(0)); 385} 386 387SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 388 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 389 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 390 ISD::LoadExtType ExtType = 391 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 392 SDValue Res = DAG.getExtLoad(ExtType, NVT, N->getChain(), N->getBasePtr(), 393 N->getSrcValue(), N->getSrcValueOffset(), 394 N->getMemoryVT(), N->isVolatile(), 395 N->getAlignment()); 396 397 // Legalized the chain result - switch anything that used the old chain to 398 // use the new one. 399 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 400 return Res; 401} 402 403/// Promote the overflow flag of an overflowing arithmetic node. 404SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 405 // Simply change the return type of the boolean result. 406 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(1)); 407 MVT ValueVTs[] = { N->getValueType(0), NVT }; 408 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) }; 409 SDValue Res = DAG.getNode(N->getOpcode(), DAG.getVTList(ValueVTs, 2), Ops, 2); 410 411 // Modified the sum result - switch anything that used the old sum to use 412 // the new one. 413 ReplaceValueWith(SDValue(N, 0), Res); 414 415 return SDValue(Res.getNode(), 1); 416} 417 418SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 419 if (ResNo == 1) 420 return PromoteIntRes_Overflow(N); 421 422 // The operation overflowed iff the result in the larger type is not the 423 // sign extension of its truncation to the original type. 424 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 425 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 426 MVT OVT = N->getOperand(0).getValueType(); 427 MVT NVT = LHS.getValueType(); 428 429 // Do the arithmetic in the larger type. 430 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 431 SDValue Res = DAG.getNode(Opcode, NVT, LHS, RHS); 432 433 // Calculate the overflow flag: sign extend the arithmetic result from 434 // the original type. 435 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Res, 436 DAG.getValueType(OVT)); 437 // Overflowed if and only if this is not equal to Res. 438 Ofl = DAG.getSetCC(N->getValueType(1), Ofl, Res, ISD::SETNE); 439 440 // Use the calculated overflow everywhere. 441 ReplaceValueWith(SDValue(N, 1), Ofl); 442 443 return Res; 444} 445 446SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) { 447 // Sign extend the input. 448 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 449 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 450 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 451} 452 453SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 454 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 455 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 456 return DAG.getNode(ISD::SELECT, LHS.getValueType(), N->getOperand(0),LHS,RHS); 457} 458 459SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 460 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 461 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 462 return DAG.getNode(ISD::SELECT_CC, LHS.getValueType(), N->getOperand(0), 463 N->getOperand(1), LHS, RHS, N->getOperand(4)); 464} 465 466SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 467 MVT SVT = TLI.getSetCCResultType(N->getOperand(0).getValueType()); 468 assert(isTypeLegal(SVT) && "Illegal SetCC type!"); 469 470 // Get the SETCC result using the canonical SETCC type. 471 SDValue SetCC = DAG.getNode(ISD::SETCC, SVT, N->getOperand(0), 472 N->getOperand(1), N->getOperand(2)); 473 474 // Convert to the expected type. 475 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 476 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?"); 477 return DAG.getNode(ISD::TRUNCATE, NVT, SetCC); 478} 479 480SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 481 return DAG.getNode(ISD::SHL, TLI.getTypeToTransformTo(N->getValueType(0)), 482 GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); 483} 484 485SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 486 SDValue Op = GetPromotedInteger(N->getOperand(0)); 487 return DAG.getNode(ISD::SIGN_EXTEND_INREG, Op.getValueType(), Op, 488 N->getOperand(1)); 489} 490 491SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 492 // The input may have strange things in the top bits of the registers, but 493 // these operations don't care. They may have weird bits going out, but 494 // that too is okay if they are integer operations. 495 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 496 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 497 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 498} 499 500SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 501 // The input value must be properly sign extended. 502 SDValue Res = SExtPromotedInteger(N->getOperand(0)); 503 return DAG.getNode(ISD::SRA, Res.getValueType(), Res, N->getOperand(1)); 504} 505 506SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 507 // The input value must be properly zero extended. 508 MVT VT = N->getValueType(0); 509 MVT NVT = TLI.getTypeToTransformTo(VT); 510 SDValue Res = ZExtPromotedInteger(N->getOperand(0)); 511 return DAG.getNode(ISD::SRL, NVT, Res, N->getOperand(1)); 512} 513 514SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 515 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 516 SDValue Res; 517 518 switch (getTypeAction(N->getOperand(0).getValueType())) { 519 default: assert(0 && "Unknown type action!"); 520 case Legal: 521 case ExpandInteger: 522 Res = N->getOperand(0); 523 break; 524 case PromoteInteger: 525 Res = GetPromotedInteger(N->getOperand(0)); 526 break; 527 } 528 529 // Truncate to NVT instead of VT 530 return DAG.getNode(ISD::TRUNCATE, NVT, Res); 531} 532 533SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 534 if (ResNo == 1) 535 return PromoteIntRes_Overflow(N); 536 537 // The operation overflowed iff the result in the larger type is not the 538 // zero extension of its truncation to the original type. 539 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 540 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 541 MVT OVT = N->getOperand(0).getValueType(); 542 MVT NVT = LHS.getValueType(); 543 544 // Do the arithmetic in the larger type. 545 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 546 SDValue Res = DAG.getNode(Opcode, NVT, LHS, RHS); 547 548 // Calculate the overflow flag: zero extend the arithmetic result from 549 // the original type. 550 SDValue Ofl = DAG.getZeroExtendInReg(Res, OVT); 551 // Overflowed if and only if this is not equal to Res. 552 Ofl = DAG.getSetCC(N->getValueType(1), Ofl, Res, ISD::SETNE); 553 554 // Use the calculated overflow everywhere. 555 ReplaceValueWith(SDValue(N, 1), Ofl); 556 557 return Res; 558} 559 560SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { 561 // Zero extend the input. 562 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 563 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 564 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 565} 566 567SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 568 return DAG.getNode(ISD::UNDEF, TLI.getTypeToTransformTo(N->getValueType(0))); 569} 570 571SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 572 SDValue Chain = N->getOperand(0); // Get the chain. 573 SDValue Ptr = N->getOperand(1); // Get the pointer. 574 MVT VT = N->getValueType(0); 575 576 MVT RegVT = TLI.getRegisterType(VT); 577 unsigned NumRegs = TLI.getNumRegisters(VT); 578 // The argument is passed as NumRegs registers of type RegVT. 579 580 SmallVector<SDValue, 8> Parts(NumRegs); 581 for (unsigned i = 0; i < NumRegs; ++i) { 582 Parts[i] = DAG.getVAArg(RegVT, Chain, Ptr, N->getOperand(2)); 583 Chain = Parts[i].getValue(1); 584 } 585 586 // Handle endianness of the load. 587 if (TLI.isBigEndian()) 588 std::reverse(Parts.begin(), Parts.end()); 589 590 // Assemble the parts in the promoted type. 591 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 592 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, NVT, Parts[0]); 593 for (unsigned i = 1; i < NumRegs; ++i) { 594 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, NVT, Parts[i]); 595 // Shift it to the right position and "or" it in. 596 Part = DAG.getNode(ISD::SHL, NVT, Part, 597 DAG.getConstant(i * RegVT.getSizeInBits(), 598 TLI.getShiftAmountTy())); 599 Res = DAG.getNode(ISD::OR, NVT, Res, Part); 600 } 601 602 // Modified the chain result - switch anything that used the old chain to 603 // use the new one. 604 ReplaceValueWith(SDValue(N, 1), Chain); 605 606 return Res; 607} 608 609SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 610 assert(ResNo == 1 && "Only boolean result promotion currently supported!"); 611 return PromoteIntRes_Overflow(N); 612} 613 614//===----------------------------------------------------------------------===// 615// Integer Operand Promotion 616//===----------------------------------------------------------------------===// 617 618/// PromoteIntegerOperand - This method is called when the specified operand of 619/// the specified node is found to need promotion. At this point, all of the 620/// result types of the node are known to be legal, but other operands of the 621/// node may need promotion or expansion as well as the specified one. 622bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 623 DEBUG(cerr << "Promote integer operand: "; N->dump(&DAG); cerr << "\n"); 624 SDValue Res = SDValue(); 625 626 if (TLI.getOperationAction(N->getOpcode(), N->getOperand(OpNo).getValueType()) 627 == TargetLowering::Custom) 628 Res = TLI.LowerOperation(SDValue(N, 0), DAG); 629 630 if (Res.getNode() == 0) { 631 switch (N->getOpcode()) { 632 default: 633 #ifndef NDEBUG 634 cerr << "PromoteIntegerOperand Op #" << OpNo << ": "; 635 N->dump(&DAG); cerr << "\n"; 636 #endif 637 assert(0 && "Do not know how to promote this operator's operand!"); 638 abort(); 639 640 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 641 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 642 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 643 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 644 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 645 case ISD::CONVERT_RNDSAT: 646 Res = PromoteIntOp_CONVERT_RNDSAT(N); break; 647 case ISD::INSERT_VECTOR_ELT: 648 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 649 case ISD::MEMBARRIER: Res = PromoteIntOp_MEMBARRIER(N); break; 650 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 651 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 652 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 653 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 654 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 655 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 656 OpNo); break; 657 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 658 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 659 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 660 } 661 } 662 663 // If the result is null, the sub-method took care of registering results etc. 664 if (!Res.getNode()) return false; 665 666 // If the result is N, the sub-method updated N in place. Tell the legalizer 667 // core about this. 668 if (Res.getNode() == N) 669 return true; 670 671 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 672 "Invalid operand expansion"); 673 674 ReplaceValueWith(SDValue(N, 0), Res); 675 return false; 676} 677 678/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 679/// shared among BR_CC, SELECT_CC, and SETCC handlers. 680void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 681 ISD::CondCode CCCode) { 682 // We have to insert explicit sign or zero extends. Note that we could 683 // insert sign extends for ALL conditions, but zero extend is cheaper on 684 // many machines (an AND instead of two shifts), so prefer it. 685 switch (CCCode) { 686 default: assert(0 && "Unknown integer comparison!"); 687 case ISD::SETEQ: 688 case ISD::SETNE: 689 case ISD::SETUGE: 690 case ISD::SETUGT: 691 case ISD::SETULE: 692 case ISD::SETULT: 693 // ALL of these operations will work if we either sign or zero extend 694 // the operands (including the unsigned comparisons!). Zero extend is 695 // usually a simpler/cheaper operation, so prefer it. 696 NewLHS = ZExtPromotedInteger(NewLHS); 697 NewRHS = ZExtPromotedInteger(NewRHS); 698 break; 699 case ISD::SETGE: 700 case ISD::SETGT: 701 case ISD::SETLT: 702 case ISD::SETLE: 703 NewLHS = SExtPromotedInteger(NewLHS); 704 NewRHS = SExtPromotedInteger(NewRHS); 705 break; 706 } 707} 708 709SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 710 SDValue Op = GetPromotedInteger(N->getOperand(0)); 711 return DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 712} 713 714SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 715 assert(OpNo == 2 && "Don't know how to promote this operand!"); 716 717 SDValue LHS = N->getOperand(2); 718 SDValue RHS = N->getOperand(3); 719 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 720 721 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 722 // legal types. 723 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 724 N->getOperand(1), LHS, RHS, N->getOperand(4)); 725} 726 727SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 728 assert(OpNo == 1 && "only know how to promote condition"); 729 730 // Promote all the way up to the canonical SetCC type. 731 MVT SVT = TLI.getSetCCResultType(MVT::Other); 732 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), SVT); 733 734 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 735 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), Cond, 736 N->getOperand(2)); 737} 738 739SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 740 // Since the result type is legal, the operands must promote to it. 741 MVT OVT = N->getOperand(0).getValueType(); 742 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 743 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 744 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 745 746 Hi = DAG.getNode(ISD::SHL, N->getValueType(0), Hi, 747 DAG.getConstant(OVT.getSizeInBits(), 748 TLI.getShiftAmountTy())); 749 return DAG.getNode(ISD::OR, N->getValueType(0), Lo, Hi); 750} 751 752SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 753 // The vector type is legal but the element type is not. This implies 754 // that the vector is a power-of-two in length and that the element 755 // type does not have a strange size (eg: it is not i1). 756 MVT VecVT = N->getValueType(0); 757 unsigned NumElts = VecVT.getVectorNumElements(); 758 assert(!(NumElts & 1) && "Legal vector of one illegal element?"); 759 760 // Build a vector of half the length out of elements of twice the bitwidth. 761 // For example <4 x i16> -> <2 x i32>. 762 MVT OldVT = N->getOperand(0).getValueType(); 763 MVT NewVT = MVT::getIntegerVT(2 * OldVT.getSizeInBits()); 764 assert(OldVT.isSimple() && NewVT.isSimple()); 765 766 std::vector<SDValue> NewElts; 767 NewElts.reserve(NumElts/2); 768 769 for (unsigned i = 0; i < NumElts; i += 2) { 770 // Combine two successive elements into one promoted element. 771 SDValue Lo = N->getOperand(i); 772 SDValue Hi = N->getOperand(i+1); 773 if (TLI.isBigEndian()) 774 std::swap(Lo, Hi); 775 NewElts.push_back(JoinIntegers(Lo, Hi)); 776 } 777 778 SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, 779 MVT::getVectorVT(NewVT, NewElts.size()), 780 &NewElts[0], NewElts.size()); 781 782 // Convert the new vector to the old vector type. 783 return DAG.getNode(ISD::BIT_CONVERT, VecVT, NewVec); 784} 785 786SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) { 787 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 788 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 789 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 790 CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) && 791 "can only promote integer arguments"); 792 SDValue InOp = GetPromotedInteger(N->getOperand(0)); 793 return DAG.getConvertRndSat(N->getValueType(0), InOp, 794 N->getOperand(1), N->getOperand(2), 795 N->getOperand(3), N->getOperand(4), CvtCode); 796} 797 798SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 799 unsigned OpNo) { 800 if (OpNo == 1) { 801 // Promote the inserted value. This is valid because the type does not 802 // have to match the vector element type. 803 804 // Check that any extra bits introduced will be truncated away. 805 assert(N->getOperand(1).getValueType().getSizeInBits() >= 806 N->getValueType(0).getVectorElementType().getSizeInBits() && 807 "Type of inserted value narrower than vector element type!"); 808 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 809 GetPromotedInteger(N->getOperand(1)), 810 N->getOperand(2)); 811 } 812 813 assert(OpNo == 2 && "Different operand and result vector types?"); 814 815 // Promote the index. 816 SDValue Idx = ZExtPromotedInteger(N->getOperand(2)); 817 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 818 N->getOperand(1), Idx); 819} 820 821SDValue DAGTypeLegalizer::PromoteIntOp_MEMBARRIER(SDNode *N) { 822 SDValue NewOps[6]; 823 NewOps[0] = N->getOperand(0); 824 for (unsigned i = 1; i < array_lengthof(NewOps); ++i) { 825 SDValue Flag = GetPromotedInteger(N->getOperand(i)); 826 NewOps[i] = DAG.getZeroExtendInReg(Flag, MVT::i1); 827 } 828 return DAG.UpdateNodeOperands(SDValue (N, 0), NewOps, 829 array_lengthof(NewOps)); 830} 831 832SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 833 assert(OpNo == 0 && "Only know how to promote condition"); 834 835 // Promote all the way up to the canonical SetCC type. 836 MVT SVT = TLI.getSetCCResultType(N->getOperand(1).getValueType()); 837 SDValue Cond = PromoteTargetBoolean(N->getOperand(0), SVT); 838 839 return DAG.UpdateNodeOperands(SDValue(N, 0), Cond, 840 N->getOperand(1), N->getOperand(2)); 841} 842 843SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 844 assert(OpNo == 0 && "Don't know how to promote this operand!"); 845 846 SDValue LHS = N->getOperand(0); 847 SDValue RHS = N->getOperand(1); 848 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 849 850 // The CC (#4) and the possible return values (#2 and #3) have legal types. 851 return DAG.UpdateNodeOperands(SDValue(N, 0), LHS, RHS, N->getOperand(2), 852 N->getOperand(3), N->getOperand(4)); 853} 854 855SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 856 assert(OpNo == 0 && "Don't know how to promote this operand!"); 857 858 SDValue LHS = N->getOperand(0); 859 SDValue RHS = N->getOperand(1); 860 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 861 862 // The CC (#2) is always legal. 863 return DAG.UpdateNodeOperands(SDValue(N, 0), LHS, RHS, N->getOperand(2)); 864} 865 866SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 867 SDValue Op = GetPromotedInteger(N->getOperand(0)); 868 Op = DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 869 return DAG.getNode(ISD::SIGN_EXTEND_INREG, Op.getValueType(), 870 Op, DAG.getValueType(N->getOperand(0).getValueType())); 871} 872 873SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 874 return DAG.UpdateNodeOperands(SDValue(N, 0), 875 SExtPromotedInteger(N->getOperand(0))); 876} 877 878SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 879 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 880 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 881 int SVOffset = N->getSrcValueOffset(); 882 unsigned Alignment = N->getAlignment(); 883 bool isVolatile = N->isVolatile(); 884 885 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 886 887 // Truncate the value and store the result. 888 return DAG.getTruncStore(Ch, Val, Ptr, N->getSrcValue(), 889 SVOffset, N->getMemoryVT(), 890 isVolatile, Alignment); 891} 892 893SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 894 SDValue Op = GetPromotedInteger(N->getOperand(0)); 895 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), Op); 896} 897 898SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 899 return DAG.UpdateNodeOperands(SDValue(N, 0), 900 ZExtPromotedInteger(N->getOperand(0))); 901} 902 903SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 904 SDValue Op = GetPromotedInteger(N->getOperand(0)); 905 Op = DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 906 return DAG.getZeroExtendInReg(Op, N->getOperand(0).getValueType()); 907} 908 909 910//===----------------------------------------------------------------------===// 911// Integer Result Expansion 912//===----------------------------------------------------------------------===// 913 914/// ExpandIntegerResult - This method is called when the specified result of the 915/// specified node is found to need expansion. At this point, the node may also 916/// have invalid operands or may have other results that need promotion, we just 917/// know that (at least) one result needs expansion. 918void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 919 DEBUG(cerr << "Expand integer result: "; N->dump(&DAG); cerr << "\n"); 920 SDValue Lo, Hi; 921 Lo = Hi = SDValue(); 922 923 // See if the target wants to custom expand this node. 924 if (CustomLowerResults(N, ResNo)) 925 return; 926 927 switch (N->getOpcode()) { 928 default: 929#ifndef NDEBUG 930 cerr << "ExpandIntegerResult #" << ResNo << ": "; 931 N->dump(&DAG); cerr << "\n"; 932#endif 933 assert(0 && "Do not know how to expand the result of this operator!"); 934 abort(); 935 936 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 937 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 938 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 939 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 940 941 case ISD::BIT_CONVERT: ExpandRes_BIT_CONVERT(N, Lo, Hi); break; 942 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 943 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 944 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 945 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 946 947 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 948 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 949 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 950 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 951 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 952 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 953 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 954 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 955 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 956 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 957 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 958 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 959 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 960 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 961 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 962 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 963 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 964 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 965 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 966 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 967 968 case ISD::AND: 969 case ISD::OR: 970 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 971 972 case ISD::ADD: 973 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 974 975 case ISD::ADDC: 976 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 977 978 case ISD::ADDE: 979 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 980 981 case ISD::SHL: 982 case ISD::SRA: 983 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 984 } 985 986 // If Lo/Hi is null, the sub-method took care of registering results etc. 987 if (Lo.getNode()) 988 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 989} 990 991/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded, 992/// and the shift amount is a constant 'Amt'. Expand the operation. 993void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 994 SDValue &Lo, SDValue &Hi) { 995 // Expand the incoming operand to be shifted, so that we have its parts 996 SDValue InL, InH; 997 GetExpandedInteger(N->getOperand(0), InL, InH); 998 999 MVT NVT = InL.getValueType(); 1000 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1001 unsigned NVTBits = NVT.getSizeInBits(); 1002 MVT ShTy = N->getOperand(1).getValueType(); 1003 1004 if (N->getOpcode() == ISD::SHL) { 1005 if (Amt > VTBits) { 1006 Lo = Hi = DAG.getConstant(0, NVT); 1007 } else if (Amt > NVTBits) { 1008 Lo = DAG.getConstant(0, NVT); 1009 Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt-NVTBits,ShTy)); 1010 } else if (Amt == NVTBits) { 1011 Lo = DAG.getConstant(0, NVT); 1012 Hi = InL; 1013 } else if (Amt == 1 && 1014 TLI.isOperationLegal(ISD::ADDC, TLI.getTypeToExpandTo(NVT))) { 1015 // Emit this X << 1 as X+X. 1016 SDVTList VTList = DAG.getVTList(NVT, MVT::Flag); 1017 SDValue LoOps[2] = { InL, InL }; 1018 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 1019 SDValue HiOps[3] = { InH, InH, Lo.getValue(1) }; 1020 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 1021 } else { 1022 Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt, ShTy)); 1023 Hi = DAG.getNode(ISD::OR, NVT, 1024 DAG.getNode(ISD::SHL, NVT, InH, 1025 DAG.getConstant(Amt, ShTy)), 1026 DAG.getNode(ISD::SRL, NVT, InL, 1027 DAG.getConstant(NVTBits-Amt, ShTy))); 1028 } 1029 return; 1030 } 1031 1032 if (N->getOpcode() == ISD::SRL) { 1033 if (Amt > VTBits) { 1034 Lo = DAG.getConstant(0, NVT); 1035 Hi = DAG.getConstant(0, NVT); 1036 } else if (Amt > NVTBits) { 1037 Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy)); 1038 Hi = DAG.getConstant(0, NVT); 1039 } else if (Amt == NVTBits) { 1040 Lo = InH; 1041 Hi = DAG.getConstant(0, NVT); 1042 } else { 1043 Lo = DAG.getNode(ISD::OR, NVT, 1044 DAG.getNode(ISD::SRL, NVT, InL, 1045 DAG.getConstant(Amt, ShTy)), 1046 DAG.getNode(ISD::SHL, NVT, InH, 1047 DAG.getConstant(NVTBits-Amt, ShTy))); 1048 Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1049 } 1050 return; 1051 } 1052 1053 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1054 if (Amt > VTBits) { 1055 Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH, 1056 DAG.getConstant(NVTBits-1, ShTy)); 1057 } else if (Amt > NVTBits) { 1058 Lo = DAG.getNode(ISD::SRA, NVT, InH, 1059 DAG.getConstant(Amt-NVTBits, ShTy)); 1060 Hi = DAG.getNode(ISD::SRA, NVT, InH, 1061 DAG.getConstant(NVTBits-1, ShTy)); 1062 } else if (Amt == NVTBits) { 1063 Lo = InH; 1064 Hi = DAG.getNode(ISD::SRA, NVT, InH, 1065 DAG.getConstant(NVTBits-1, ShTy)); 1066 } else { 1067 Lo = DAG.getNode(ISD::OR, NVT, 1068 DAG.getNode(ISD::SRL, NVT, InL, 1069 DAG.getConstant(Amt, ShTy)), 1070 DAG.getNode(ISD::SHL, NVT, InH, 1071 DAG.getConstant(NVTBits-Amt, ShTy))); 1072 Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Amt, ShTy)); 1073 } 1074} 1075 1076/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1077/// this shift based on knowledge of the high bit of the shift amount. If we 1078/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1079/// shift amount. 1080bool DAGTypeLegalizer:: 1081ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1082 SDValue Amt = N->getOperand(1); 1083 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1084 MVT ShTy = Amt.getValueType(); 1085 unsigned ShBits = ShTy.getSizeInBits(); 1086 unsigned NVTBits = NVT.getSizeInBits(); 1087 assert(isPowerOf2_32(NVTBits) && 1088 "Expanded integer type size not a power of two!"); 1089 1090 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1091 APInt KnownZero, KnownOne; 1092 DAG.ComputeMaskedBits(N->getOperand(1), HighBitMask, KnownZero, KnownOne); 1093 1094 // If we don't know anything about the high bits, exit. 1095 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1096 return false; 1097 1098 // Get the incoming operand to be shifted. 1099 SDValue InL, InH; 1100 GetExpandedInteger(N->getOperand(0), InL, InH); 1101 1102 // If we know that any of the high bits of the shift amount are one, then we 1103 // can do this as a couple of simple shifts. 1104 if (KnownOne.intersects(HighBitMask)) { 1105 // Mask out the high bit, which we know is set. 1106 Amt = DAG.getNode(ISD::AND, ShTy, Amt, 1107 DAG.getConstant(~HighBitMask, ShTy)); 1108 1109 switch (N->getOpcode()) { 1110 default: assert(0 && "Unknown shift"); 1111 case ISD::SHL: 1112 Lo = DAG.getConstant(0, NVT); // Low part is zero. 1113 Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part. 1114 return true; 1115 case ISD::SRL: 1116 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 1117 Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part. 1118 return true; 1119 case ISD::SRA: 1120 Hi = DAG.getNode(ISD::SRA, NVT, InH, // Sign extend high part. 1121 DAG.getConstant(NVTBits-1, ShTy)); 1122 Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part. 1123 return true; 1124 } 1125 } 1126 1127#if 0 1128 // FIXME: This code is broken for shifts with a zero amount! 1129 // If we know that all of the high bits of the shift amount are zero, then we 1130 // can do this as a couple of simple shifts. 1131 if ((KnownZero & HighBitMask) == HighBitMask) { 1132 // Compute 32-amt. 1133 SDValue Amt2 = DAG.getNode(ISD::SUB, ShTy, 1134 DAG.getConstant(NVTBits, ShTy), 1135 Amt); 1136 unsigned Op1, Op2; 1137 switch (N->getOpcode()) { 1138 default: assert(0 && "Unknown shift"); 1139 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1140 case ISD::SRL: 1141 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1142 } 1143 1144 Lo = DAG.getNode(N->getOpcode(), NVT, InL, Amt); 1145 Hi = DAG.getNode(ISD::OR, NVT, 1146 DAG.getNode(Op1, NVT, InH, Amt), 1147 DAG.getNode(Op2, NVT, InL, Amt2)); 1148 return true; 1149 } 1150#endif 1151 1152 return false; 1153} 1154 1155void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1156 SDValue &Lo, SDValue &Hi) { 1157 // Expand the subcomponents. 1158 SDValue LHSL, LHSH, RHSL, RHSH; 1159 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1160 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1161 1162 MVT NVT = LHSL.getValueType(); 1163 SDValue LoOps[2] = { LHSL, RHSL }; 1164 SDValue HiOps[3] = { LHSH, RHSH }; 1165 1166 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1167 // them. TODO: Teach operation legalization how to expand unsupported 1168 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1169 // a carry of type MVT::Flag, but there doesn't seem to be any way to 1170 // generate a value of this type in the expanded code sequence. 1171 bool hasCarry = 1172 TLI.isOperationLegal(N->getOpcode() == ISD::ADD ? ISD::ADDC : ISD::SUBC, 1173 TLI.getTypeToExpandTo(NVT)); 1174 1175 if (hasCarry) { 1176 SDVTList VTList = DAG.getVTList(NVT, MVT::Flag); 1177 if (N->getOpcode() == ISD::ADD) { 1178 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 1179 HiOps[2] = Lo.getValue(1); 1180 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 1181 } else { 1182 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); 1183 HiOps[2] = Lo.getValue(1); 1184 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); 1185 } 1186 } else { 1187 if (N->getOpcode() == ISD::ADD) { 1188 Lo = DAG.getNode(ISD::ADD, NVT, LoOps, 2); 1189 Hi = DAG.getNode(ISD::ADD, NVT, HiOps, 2); 1190 SDValue Cmp1 = DAG.getSetCC(TLI.getSetCCResultType(NVT), Lo, LoOps[0], 1191 ISD::SETULT); 1192 SDValue Carry1 = DAG.getNode(ISD::SELECT, NVT, Cmp1, 1193 DAG.getConstant(1, NVT), 1194 DAG.getConstant(0, NVT)); 1195 SDValue Cmp2 = DAG.getSetCC(TLI.getSetCCResultType(NVT), Lo, LoOps[1], 1196 ISD::SETULT); 1197 SDValue Carry2 = DAG.getNode(ISD::SELECT, NVT, Cmp2, 1198 DAG.getConstant(1, NVT), Carry1); 1199 Hi = DAG.getNode(ISD::ADD, NVT, Hi, Carry2); 1200 } else { 1201 Lo = DAG.getNode(ISD::SUB, NVT, LoOps, 2); 1202 Hi = DAG.getNode(ISD::SUB, NVT, HiOps, 2); 1203 SDValue Cmp = 1204 DAG.getSetCC(TLI.getSetCCResultType(LoOps[0].getValueType()), 1205 LoOps[0], LoOps[1], ISD::SETULT); 1206 SDValue Borrow = DAG.getNode(ISD::SELECT, NVT, Cmp, 1207 DAG.getConstant(1, NVT), 1208 DAG.getConstant(0, NVT)); 1209 Hi = DAG.getNode(ISD::SUB, NVT, Hi, Borrow); 1210 } 1211 } 1212} 1213 1214void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1215 SDValue &Lo, SDValue &Hi) { 1216 // Expand the subcomponents. 1217 SDValue LHSL, LHSH, RHSL, RHSH; 1218 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1219 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1220 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1221 SDValue LoOps[2] = { LHSL, RHSL }; 1222 SDValue HiOps[3] = { LHSH, RHSH }; 1223 1224 if (N->getOpcode() == ISD::ADDC) { 1225 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 1226 HiOps[2] = Lo.getValue(1); 1227 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 1228 } else { 1229 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); 1230 HiOps[2] = Lo.getValue(1); 1231 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); 1232 } 1233 1234 // Legalized the flag result - switch anything that used the old flag to 1235 // use the new one. 1236 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1237} 1238 1239void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1240 SDValue &Lo, SDValue &Hi) { 1241 // Expand the subcomponents. 1242 SDValue LHSL, LHSH, RHSL, RHSH; 1243 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1244 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1245 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1246 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1247 SDValue HiOps[3] = { LHSH, RHSH }; 1248 1249 Lo = DAG.getNode(N->getOpcode(), VTList, LoOps, 3); 1250 HiOps[2] = Lo.getValue(1); 1251 Hi = DAG.getNode(N->getOpcode(), VTList, HiOps, 3); 1252 1253 // Legalized the flag result - switch anything that used the old flag to 1254 // use the new one. 1255 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1256} 1257 1258void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 1259 SDValue &Lo, SDValue &Hi) { 1260 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1261 SDValue Op = N->getOperand(0); 1262 if (Op.getValueType().bitsLE(NVT)) { 1263 // The low part is any extension of the input (which degenerates to a copy). 1264 Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Op); 1265 Hi = DAG.getNode(ISD::UNDEF, NVT); // The high part is undefined. 1266 } else { 1267 // For example, extension of an i48 to an i64. The operand type necessarily 1268 // promotes to the result type, so will end up being expanded too. 1269 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1270 "Only know how to promote this result!"); 1271 SDValue Res = GetPromotedInteger(Op); 1272 assert(Res.getValueType() == N->getValueType(0) && 1273 "Operand over promoted?"); 1274 // Split the promoted operand. This will simplify when it is expanded. 1275 SplitInteger(Res, Lo, Hi); 1276 } 1277} 1278 1279void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 1280 SDValue &Lo, SDValue &Hi) { 1281 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1282 MVT NVT = Lo.getValueType(); 1283 MVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1284 unsigned NVTBits = NVT.getSizeInBits(); 1285 unsigned EVTBits = EVT.getSizeInBits(); 1286 1287 if (NVTBits < EVTBits) { 1288 Hi = DAG.getNode(ISD::AssertSext, NVT, Hi, 1289 DAG.getValueType(MVT::getIntegerVT(EVTBits - NVTBits))); 1290 } else { 1291 Lo = DAG.getNode(ISD::AssertSext, NVT, Lo, DAG.getValueType(EVT)); 1292 // The high part replicates the sign bit of Lo, make it explicit. 1293 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 1294 DAG.getConstant(NVTBits-1, TLI.getShiftAmountTy())); 1295 } 1296} 1297 1298void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 1299 SDValue &Lo, SDValue &Hi) { 1300 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1301 MVT NVT = Lo.getValueType(); 1302 MVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1303 unsigned NVTBits = NVT.getSizeInBits(); 1304 unsigned EVTBits = EVT.getSizeInBits(); 1305 1306 if (NVTBits < EVTBits) { 1307 Hi = DAG.getNode(ISD::AssertZext, NVT, Hi, 1308 DAG.getValueType(MVT::getIntegerVT(EVTBits - NVTBits))); 1309 } else { 1310 Lo = DAG.getNode(ISD::AssertZext, NVT, Lo, DAG.getValueType(EVT)); 1311 // The high part must be zero, make it explicit. 1312 Hi = DAG.getConstant(0, NVT); 1313 } 1314} 1315 1316void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1317 SDValue &Lo, SDValue &Hi) { 1318 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1319 Lo = DAG.getNode(ISD::BSWAP, Lo.getValueType(), Lo); 1320 Hi = DAG.getNode(ISD::BSWAP, Hi.getValueType(), Hi); 1321} 1322 1323void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 1324 SDValue &Lo, SDValue &Hi) { 1325 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1326 unsigned NBitWidth = NVT.getSizeInBits(); 1327 const APInt &Cst = cast<ConstantSDNode>(N)->getAPIntValue(); 1328 Lo = DAG.getConstant(APInt(Cst).trunc(NBitWidth), NVT); 1329 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT); 1330} 1331 1332void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1333 SDValue &Lo, SDValue &Hi) { 1334 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1335 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1336 MVT NVT = Lo.getValueType(); 1337 1338 SDValue HiNotZero = DAG.getSetCC(TLI.getSetCCResultType(NVT), Hi, 1339 DAG.getConstant(0, NVT), ISD::SETNE); 1340 1341 SDValue LoLZ = DAG.getNode(ISD::CTLZ, NVT, Lo); 1342 SDValue HiLZ = DAG.getNode(ISD::CTLZ, NVT, Hi); 1343 1344 Lo = DAG.getNode(ISD::SELECT, NVT, HiNotZero, HiLZ, 1345 DAG.getNode(ISD::ADD, NVT, LoLZ, 1346 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1347 Hi = DAG.getConstant(0, NVT); 1348} 1349 1350void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1351 SDValue &Lo, SDValue &Hi) { 1352 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1353 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1354 MVT NVT = Lo.getValueType(); 1355 Lo = DAG.getNode(ISD::ADD, NVT, DAG.getNode(ISD::CTPOP, NVT, Lo), 1356 DAG.getNode(ISD::CTPOP, NVT, Hi)); 1357 Hi = DAG.getConstant(0, NVT); 1358} 1359 1360void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1361 SDValue &Lo, SDValue &Hi) { 1362 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1363 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1364 MVT NVT = Lo.getValueType(); 1365 1366 SDValue LoNotZero = DAG.getSetCC(TLI.getSetCCResultType(NVT), Lo, 1367 DAG.getConstant(0, NVT), ISD::SETNE); 1368 1369 SDValue LoLZ = DAG.getNode(ISD::CTTZ, NVT, Lo); 1370 SDValue HiLZ = DAG.getNode(ISD::CTTZ, NVT, Hi); 1371 1372 Lo = DAG.getNode(ISD::SELECT, NVT, LoNotZero, LoLZ, 1373 DAG.getNode(ISD::ADD, NVT, HiLZ, 1374 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1375 Hi = DAG.getConstant(0, NVT); 1376} 1377 1378void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 1379 SDValue &Hi) { 1380 MVT VT = N->getValueType(0); 1381 SDValue Op = N->getOperand(0); 1382 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 1383 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 1384 SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*sign irrelevant*/), Lo, Hi); 1385} 1386 1387void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 1388 SDValue &Hi) { 1389 MVT VT = N->getValueType(0); 1390 SDValue Op = N->getOperand(0); 1391 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 1392 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 1393 SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*sign irrelevant*/), Lo, Hi); 1394} 1395 1396void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 1397 SDValue &Lo, SDValue &Hi) { 1398 if (ISD::isNormalLoad(N)) { 1399 ExpandRes_NormalLoad(N, Lo, Hi); 1400 return; 1401 } 1402 1403 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1404 1405 MVT VT = N->getValueType(0); 1406 MVT NVT = TLI.getTypeToTransformTo(VT); 1407 SDValue Ch = N->getChain(); 1408 SDValue Ptr = N->getBasePtr(); 1409 ISD::LoadExtType ExtType = N->getExtensionType(); 1410 int SVOffset = N->getSrcValueOffset(); 1411 unsigned Alignment = N->getAlignment(); 1412 bool isVolatile = N->isVolatile(); 1413 1414 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1415 1416 if (N->getMemoryVT().bitsLE(NVT)) { 1417 MVT EVT = N->getMemoryVT(); 1418 1419 Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, EVT, 1420 isVolatile, Alignment); 1421 1422 // Remember the chain. 1423 Ch = Lo.getValue(1); 1424 1425 if (ExtType == ISD::SEXTLOAD) { 1426 // The high part is obtained by SRA'ing all but one of the bits of the 1427 // lo part. 1428 unsigned LoSize = Lo.getValueType().getSizeInBits(); 1429 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 1430 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 1431 } else if (ExtType == ISD::ZEXTLOAD) { 1432 // The high part is just a zero. 1433 Hi = DAG.getConstant(0, NVT); 1434 } else { 1435 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 1436 // The high part is undefined. 1437 Hi = DAG.getNode(ISD::UNDEF, NVT); 1438 } 1439 } else if (TLI.isLittleEndian()) { 1440 // Little-endian - low bits are at low addresses. 1441 Lo = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset, 1442 isVolatile, Alignment); 1443 1444 unsigned ExcessBits = 1445 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1446 MVT NEVT = MVT::getIntegerVT(ExcessBits); 1447 1448 // Increment the pointer to the other half. 1449 unsigned IncrementSize = NVT.getSizeInBits()/8; 1450 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 1451 DAG.getIntPtrConstant(IncrementSize)); 1452 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), 1453 SVOffset+IncrementSize, NEVT, 1454 isVolatile, MinAlign(Alignment, IncrementSize)); 1455 1456 // Build a factor node to remember that this load is independent of the 1457 // other one. 1458 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 1459 Hi.getValue(1)); 1460 } else { 1461 // Big-endian - high bits are at low addresses. Favor aligned loads at 1462 // the cost of some bit-fiddling. 1463 MVT EVT = N->getMemoryVT(); 1464 unsigned EBytes = EVT.getStoreSizeInBits()/8; 1465 unsigned IncrementSize = NVT.getSizeInBits()/8; 1466 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1467 1468 // Load both the high bits and maybe some of the low bits. 1469 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, 1470 MVT::getIntegerVT(EVT.getSizeInBits() - ExcessBits), 1471 isVolatile, Alignment); 1472 1473 // Increment the pointer to the other half. 1474 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 1475 DAG.getIntPtrConstant(IncrementSize)); 1476 // Load the rest of the low bits. 1477 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, NVT, Ch, Ptr, N->getSrcValue(), 1478 SVOffset+IncrementSize, 1479 MVT::getIntegerVT(ExcessBits), 1480 isVolatile, MinAlign(Alignment, IncrementSize)); 1481 1482 // Build a factor node to remember that this load is independent of the 1483 // other one. 1484 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 1485 Hi.getValue(1)); 1486 1487 if (ExcessBits < NVT.getSizeInBits()) { 1488 // Transfer low bits from the bottom of Hi to the top of Lo. 1489 Lo = DAG.getNode(ISD::OR, NVT, Lo, 1490 DAG.getNode(ISD::SHL, NVT, Hi, 1491 DAG.getConstant(ExcessBits, 1492 TLI.getShiftAmountTy()))); 1493 // Move high bits to the right position in Hi. 1494 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, NVT, Hi, 1495 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 1496 TLI.getShiftAmountTy())); 1497 } 1498 } 1499 1500 // Legalized the chain result - switch anything that used the old chain to 1501 // use the new one. 1502 ReplaceValueWith(SDValue(N, 1), Ch); 1503} 1504 1505void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 1506 SDValue &Lo, SDValue &Hi) { 1507 SDValue LL, LH, RL, RH; 1508 GetExpandedInteger(N->getOperand(0), LL, LH); 1509 GetExpandedInteger(N->getOperand(1), RL, RH); 1510 Lo = DAG.getNode(N->getOpcode(), LL.getValueType(), LL, RL); 1511 Hi = DAG.getNode(N->getOpcode(), LL.getValueType(), LH, RH); 1512} 1513 1514void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 1515 SDValue &Lo, SDValue &Hi) { 1516 MVT VT = N->getValueType(0); 1517 MVT NVT = TLI.getTypeToTransformTo(VT); 1518 1519 bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT); 1520 bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT); 1521 bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT); 1522 bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT); 1523 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 1524 SDValue LL, LH, RL, RH; 1525 GetExpandedInteger(N->getOperand(0), LL, LH); 1526 GetExpandedInteger(N->getOperand(1), RL, RH); 1527 unsigned OuterBitSize = VT.getSizeInBits(); 1528 unsigned InnerBitSize = NVT.getSizeInBits(); 1529 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0)); 1530 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1)); 1531 1532 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 1533 if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) && 1534 DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) { 1535 // The inputs are both zero-extended. 1536 if (HasUMUL_LOHI) { 1537 // We can emit a umul_lohi. 1538 Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); 1539 Hi = SDValue(Lo.getNode(), 1); 1540 return; 1541 } 1542 if (HasMULHU) { 1543 // We can emit a mulhu+mul. 1544 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1545 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); 1546 return; 1547 } 1548 } 1549 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 1550 // The input values are both sign-extended. 1551 if (HasSMUL_LOHI) { 1552 // We can emit a smul_lohi. 1553 Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); 1554 Hi = SDValue(Lo.getNode(), 1); 1555 return; 1556 } 1557 if (HasMULHS) { 1558 // We can emit a mulhs+mul. 1559 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1560 Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL); 1561 return; 1562 } 1563 } 1564 if (HasUMUL_LOHI) { 1565 // Lo,Hi = umul LHS, RHS. 1566 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, 1567 DAG.getVTList(NVT, NVT), LL, RL); 1568 Lo = UMulLOHI; 1569 Hi = UMulLOHI.getValue(1); 1570 RH = DAG.getNode(ISD::MUL, NVT, LL, RH); 1571 LH = DAG.getNode(ISD::MUL, NVT, LH, RL); 1572 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); 1573 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH); 1574 return; 1575 } 1576 if (HasMULHU) { 1577 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1578 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); 1579 RH = DAG.getNode(ISD::MUL, NVT, LL, RH); 1580 LH = DAG.getNode(ISD::MUL, NVT, LH, RL); 1581 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); 1582 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH); 1583 return; 1584 } 1585 } 1586 1587 // If nothing else, we can make a libcall. 1588 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1589 if (VT == MVT::i32) 1590 LC = RTLIB::MUL_I32; 1591 else if (VT == MVT::i64) 1592 LC = RTLIB::MUL_I64; 1593 else if (VT == MVT::i128) 1594 LC = RTLIB::MUL_I128; 1595 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!"); 1596 1597 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1598 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*sign irrelevant*/), Lo, Hi); 1599} 1600 1601void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 1602 SDValue &Lo, SDValue &Hi) { 1603 MVT VT = N->getValueType(0); 1604 1605 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1606 if (VT == MVT::i32) 1607 LC = RTLIB::SDIV_I32; 1608 else if (VT == MVT::i64) 1609 LC = RTLIB::SDIV_I64; 1610 else if (VT == MVT::i128) 1611 LC = RTLIB::SDIV_I128; 1612 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 1613 1614 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1615 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true), Lo, Hi); 1616} 1617 1618void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 1619 SDValue &Lo, SDValue &Hi) { 1620 MVT VT = N->getValueType(0); 1621 1622 // If we can emit an efficient shift operation, do so now. Check to see if 1623 // the RHS is a constant. 1624 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 1625 return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi); 1626 1627 // If we can determine that the high bit of the shift is zero or one, even if 1628 // the low bits are variable, emit this shift in an optimized form. 1629 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 1630 return; 1631 1632 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 1633 unsigned PartsOpc; 1634 if (N->getOpcode() == ISD::SHL) { 1635 PartsOpc = ISD::SHL_PARTS; 1636 } else if (N->getOpcode() == ISD::SRL) { 1637 PartsOpc = ISD::SRL_PARTS; 1638 } else { 1639 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1640 PartsOpc = ISD::SRA_PARTS; 1641 } 1642 1643 // Next check to see if the target supports this SHL_PARTS operation or if it 1644 // will custom expand it. 1645 MVT NVT = TLI.getTypeToTransformTo(VT); 1646 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 1647 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 1648 Action == TargetLowering::Custom) { 1649 // Expand the subcomponents. 1650 SDValue LHSL, LHSH; 1651 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1652 1653 SDValue Ops[] = { LHSL, LHSH, N->getOperand(1) }; 1654 MVT VT = LHSL.getValueType(); 1655 Lo = DAG.getNode(PartsOpc, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3); 1656 Hi = Lo.getValue(1); 1657 return; 1658 } 1659 1660 // Otherwise, emit a libcall. 1661 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1662 bool isSigned; 1663 if (N->getOpcode() == ISD::SHL) { 1664 isSigned = false; /*sign irrelevant*/ 1665 if (VT == MVT::i32) 1666 LC = RTLIB::SHL_I32; 1667 else if (VT == MVT::i64) 1668 LC = RTLIB::SHL_I64; 1669 else if (VT == MVT::i128) 1670 LC = RTLIB::SHL_I128; 1671 } else if (N->getOpcode() == ISD::SRL) { 1672 isSigned = false; 1673 if (VT == MVT::i32) 1674 LC = RTLIB::SRL_I32; 1675 else if (VT == MVT::i64) 1676 LC = RTLIB::SRL_I64; 1677 else if (VT == MVT::i128) 1678 LC = RTLIB::SRL_I128; 1679 } else { 1680 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1681 isSigned = true; 1682 if (VT == MVT::i32) 1683 LC = RTLIB::SRA_I32; 1684 else if (VT == MVT::i64) 1685 LC = RTLIB::SRA_I64; 1686 else if (VT == MVT::i128) 1687 LC = RTLIB::SRA_I128; 1688 } 1689 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported shift!"); 1690 1691 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1692 SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned), Lo, Hi); 1693} 1694 1695void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 1696 SDValue &Lo, SDValue &Hi) { 1697 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1698 SDValue Op = N->getOperand(0); 1699 if (Op.getValueType().bitsLE(NVT)) { 1700 // The low part is sign extension of the input (degenerates to a copy). 1701 Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, N->getOperand(0)); 1702 // The high part is obtained by SRA'ing all but one of the bits of low part. 1703 unsigned LoSize = NVT.getSizeInBits(); 1704 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 1705 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 1706 } else { 1707 // For example, extension of an i48 to an i64. The operand type necessarily 1708 // promotes to the result type, so will end up being expanded too. 1709 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1710 "Only know how to promote this result!"); 1711 SDValue Res = GetPromotedInteger(Op); 1712 assert(Res.getValueType() == N->getValueType(0) && 1713 "Operand over promoted?"); 1714 // Split the promoted operand. This will simplify when it is expanded. 1715 SplitInteger(Res, Lo, Hi); 1716 unsigned ExcessBits = 1717 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 1718 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi, 1719 DAG.getValueType(MVT::getIntegerVT(ExcessBits))); 1720 } 1721} 1722 1723void DAGTypeLegalizer:: 1724ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 1725 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1726 MVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1727 1728 if (EVT.bitsLE(Lo.getValueType())) { 1729 // sext_inreg the low part if needed. 1730 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, Lo.getValueType(), Lo, 1731 N->getOperand(1)); 1732 1733 // The high part gets the sign extension from the lo-part. This handles 1734 // things like sextinreg V:i64 from i8. 1735 Hi = DAG.getNode(ISD::SRA, Hi.getValueType(), Lo, 1736 DAG.getConstant(Hi.getValueType().getSizeInBits()-1, 1737 TLI.getShiftAmountTy())); 1738 } else { 1739 // For example, extension of an i48 to an i64. Leave the low part alone, 1740 // sext_inreg the high part. 1741 unsigned ExcessBits = 1742 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 1743 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi, 1744 DAG.getValueType(MVT::getIntegerVT(ExcessBits))); 1745 } 1746} 1747 1748void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 1749 SDValue &Lo, SDValue &Hi) { 1750 MVT VT = N->getValueType(0); 1751 1752 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1753 if (VT == MVT::i32) 1754 LC = RTLIB::SREM_I32; 1755 else if (VT == MVT::i64) 1756 LC = RTLIB::SREM_I64; 1757 else if (VT == MVT::i128) 1758 LC = RTLIB::SREM_I128; 1759 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 1760 1761 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1762 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true), Lo, Hi); 1763} 1764 1765void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 1766 SDValue &Lo, SDValue &Hi) { 1767 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1768 Lo = DAG.getNode(ISD::TRUNCATE, NVT, N->getOperand(0)); 1769 Hi = DAG.getNode(ISD::SRL, N->getOperand(0).getValueType(), N->getOperand(0), 1770 DAG.getConstant(NVT.getSizeInBits(), 1771 TLI.getShiftAmountTy())); 1772 Hi = DAG.getNode(ISD::TRUNCATE, NVT, Hi); 1773} 1774 1775void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 1776 SDValue &Lo, SDValue &Hi) { 1777 MVT VT = N->getValueType(0); 1778 1779 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1780 if (VT == MVT::i32) 1781 LC = RTLIB::UDIV_I32; 1782 else if (VT == MVT::i64) 1783 LC = RTLIB::UDIV_I64; 1784 else if (VT == MVT::i128) 1785 LC = RTLIB::UDIV_I128; 1786 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 1787 1788 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1789 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false), Lo, Hi); 1790} 1791 1792void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 1793 SDValue &Lo, SDValue &Hi) { 1794 MVT VT = N->getValueType(0); 1795 1796 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1797 if (VT == MVT::i32) 1798 LC = RTLIB::UREM_I32; 1799 else if (VT == MVT::i64) 1800 LC = RTLIB::UREM_I64; 1801 else if (VT == MVT::i128) 1802 LC = RTLIB::UREM_I128; 1803 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 1804 1805 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1806 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false), Lo, Hi); 1807} 1808 1809void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 1810 SDValue &Lo, SDValue &Hi) { 1811 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1812 SDValue Op = N->getOperand(0); 1813 if (Op.getValueType().bitsLE(NVT)) { 1814 // The low part is zero extension of the input (degenerates to a copy). 1815 Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, N->getOperand(0)); 1816 Hi = DAG.getConstant(0, NVT); // The high part is just a zero. 1817 } else { 1818 // For example, extension of an i48 to an i64. The operand type necessarily 1819 // promotes to the result type, so will end up being expanded too. 1820 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 1821 "Only know how to promote this result!"); 1822 SDValue Res = GetPromotedInteger(Op); 1823 assert(Res.getValueType() == N->getValueType(0) && 1824 "Operand over promoted?"); 1825 // Split the promoted operand. This will simplify when it is expanded. 1826 SplitInteger(Res, Lo, Hi); 1827 unsigned ExcessBits = 1828 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 1829 Hi = DAG.getZeroExtendInReg(Hi, MVT::getIntegerVT(ExcessBits)); 1830 } 1831} 1832 1833 1834//===----------------------------------------------------------------------===// 1835// Integer Operand Expansion 1836//===----------------------------------------------------------------------===// 1837 1838/// ExpandIntegerOperand - This method is called when the specified operand of 1839/// the specified node is found to need expansion. At this point, all of the 1840/// result types of the node are known to be legal, but other operands of the 1841/// node may need promotion or expansion as well as the specified one. 1842bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 1843 DEBUG(cerr << "Expand integer operand: "; N->dump(&DAG); cerr << "\n"); 1844 SDValue Res = SDValue(); 1845 1846 if (TLI.getOperationAction(N->getOpcode(), N->getOperand(OpNo).getValueType()) 1847 == TargetLowering::Custom) 1848 Res = TLI.LowerOperation(SDValue(N, 0), DAG); 1849 1850 if (Res.getNode() == 0) { 1851 switch (N->getOpcode()) { 1852 default: 1853 #ifndef NDEBUG 1854 cerr << "ExpandIntegerOperand Op #" << OpNo << ": "; 1855 N->dump(&DAG); cerr << "\n"; 1856 #endif 1857 assert(0 && "Do not know how to expand this operator's operand!"); 1858 abort(); 1859 1860 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1861 case ISD::BIT_CONVERT: Res = ExpandOp_BIT_CONVERT(N); break; 1862 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1863 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 1864 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 1865 1866 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 1867 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 1868 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 1869 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 1870 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); 1871 break; 1872 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 1873 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 1874 } 1875 } 1876 1877 // If the result is null, the sub-method took care of registering results etc. 1878 if (!Res.getNode()) return false; 1879 1880 // If the result is N, the sub-method updated N in place. Tell the legalizer 1881 // core about this. 1882 if (Res.getNode() == N) 1883 return true; 1884 1885 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1886 "Invalid operand expansion"); 1887 1888 ReplaceValueWith(SDValue(N, 0), Res); 1889 return false; 1890} 1891 1892/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 1893/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1894void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 1895 SDValue &NewRHS, 1896 ISD::CondCode &CCCode) { 1897 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 1898 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 1899 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 1900 1901 MVT VT = NewLHS.getValueType(); 1902 1903 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 1904 if (RHSLo == RHSHi) { 1905 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 1906 if (RHSCST->isAllOnesValue()) { 1907 // Equality comparison to -1. 1908 NewLHS = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi); 1909 NewRHS = RHSLo; 1910 return; 1911 } 1912 } 1913 } 1914 1915 NewLHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo); 1916 NewRHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi); 1917 NewLHS = DAG.getNode(ISD::OR, NewLHS.getValueType(), NewLHS, NewRHS); 1918 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 1919 return; 1920 } 1921 1922 // If this is a comparison of the sign bit, just look at the top part. 1923 // X > -1, x < 0 1924 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 1925 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 1926 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 1927 NewLHS = LHSHi; 1928 NewRHS = RHSHi; 1929 return; 1930 } 1931 1932 // FIXME: This generated code sucks. 1933 ISD::CondCode LowCC; 1934 switch (CCCode) { 1935 default: assert(0 && "Unknown integer setcc!"); 1936 case ISD::SETLT: 1937 case ISD::SETULT: LowCC = ISD::SETULT; break; 1938 case ISD::SETGT: 1939 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 1940 case ISD::SETLE: 1941 case ISD::SETULE: LowCC = ISD::SETULE; break; 1942 case ISD::SETGE: 1943 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 1944 } 1945 1946 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 1947 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 1948 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 1949 1950 // NOTE: on targets without efficient SELECT of bools, we can always use 1951 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 1952 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL); 1953 SDValue Tmp1, Tmp2; 1954 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), 1955 LHSLo, RHSLo, LowCC, false, DagCombineInfo); 1956 if (!Tmp1.getNode()) 1957 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo.getValueType()), 1958 LHSLo, RHSLo, LowCC); 1959 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 1960 LHSHi, RHSHi, CCCode, false, DagCombineInfo); 1961 if (!Tmp2.getNode()) 1962 Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHSHi.getValueType()), 1963 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 1964 1965 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 1966 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 1967 if ((Tmp1C && Tmp1C->isNullValue()) || 1968 (Tmp2C && Tmp2C->isNullValue() && 1969 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 1970 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 1971 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 1972 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 1973 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 1974 // low part is known false, returns high part. 1975 // For LE / GE, if high part is known false, ignore the low part. 1976 // For LT / GT, if high part is known true, ignore the low part. 1977 NewLHS = Tmp2; 1978 NewRHS = SDValue(); 1979 return; 1980 } 1981 1982 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 1983 LHSHi, RHSHi, ISD::SETEQ, false, DagCombineInfo); 1984 if (!NewLHS.getNode()) 1985 NewLHS = DAG.getSetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 1986 LHSHi, RHSHi, ISD::SETEQ); 1987 NewLHS = DAG.getNode(ISD::SELECT, Tmp1.getValueType(), 1988 NewLHS, Tmp1, Tmp2); 1989 NewRHS = SDValue(); 1990} 1991 1992SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 1993 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1994 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1995 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 1996 1997 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1998 // against zero to select between true and false values. 1999 if (NewRHS.getNode() == 0) { 2000 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2001 CCCode = ISD::SETNE; 2002 } 2003 2004 // Update N to have the operands specified. 2005 return DAG.UpdateNodeOperands(SDValue(N, 0), N->getOperand(0), 2006 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2007 N->getOperand(4)); 2008} 2009 2010SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 2011 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2012 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2013 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 2014 2015 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2016 // against zero to select between true and false values. 2017 if (NewRHS.getNode() == 0) { 2018 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2019 CCCode = ISD::SETNE; 2020 } 2021 2022 // Update N to have the operands specified. 2023 return DAG.UpdateNodeOperands(SDValue(N, 0), NewLHS, NewRHS, 2024 N->getOperand(2), N->getOperand(3), 2025 DAG.getCondCode(CCCode)); 2026} 2027 2028SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 2029 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2030 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2031 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 2032 2033 // If ExpandSetCCOperands returned a scalar, use it. 2034 if (NewRHS.getNode() == 0) { 2035 assert(NewLHS.getValueType() == N->getValueType(0) && 2036 "Unexpected setcc expansion!"); 2037 return NewLHS; 2038 } 2039 2040 // Otherwise, update N to have the operands specified. 2041 return DAG.UpdateNodeOperands(SDValue(N, 0), NewLHS, NewRHS, 2042 DAG.getCondCode(CCCode)); 2043} 2044 2045SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 2046 SDValue Op = N->getOperand(0); 2047 MVT DstVT = N->getValueType(0); 2048 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 2049 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2050 "Don't know how to expand this SINT_TO_FP!"); 2051 return MakeLibCall(LC, DstVT, &Op, 1, true); 2052} 2053 2054SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 2055 if (ISD::isNormalStore(N)) 2056 return ExpandOp_NormalStore(N, OpNo); 2057 2058 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2059 assert(OpNo == 1 && "Can only expand the stored value so far"); 2060 2061 MVT VT = N->getOperand(1).getValueType(); 2062 MVT NVT = TLI.getTypeToTransformTo(VT); 2063 SDValue Ch = N->getChain(); 2064 SDValue Ptr = N->getBasePtr(); 2065 int SVOffset = N->getSrcValueOffset(); 2066 unsigned Alignment = N->getAlignment(); 2067 bool isVolatile = N->isVolatile(); 2068 SDValue Lo, Hi; 2069 2070 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2071 2072 if (N->getMemoryVT().bitsLE(NVT)) { 2073 GetExpandedInteger(N->getValue(), Lo, Hi); 2074 return DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 2075 N->getMemoryVT(), isVolatile, Alignment); 2076 } else if (TLI.isLittleEndian()) { 2077 // Little-endian - low bits are at low addresses. 2078 GetExpandedInteger(N->getValue(), Lo, Hi); 2079 2080 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 2081 isVolatile, Alignment); 2082 2083 unsigned ExcessBits = 2084 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2085 MVT NEVT = MVT::getIntegerVT(ExcessBits); 2086 2087 // Increment the pointer to the other half. 2088 unsigned IncrementSize = NVT.getSizeInBits()/8; 2089 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 2090 DAG.getIntPtrConstant(IncrementSize)); 2091 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(), 2092 SVOffset+IncrementSize, NEVT, 2093 isVolatile, MinAlign(Alignment, IncrementSize)); 2094 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 2095 } else { 2096 // Big-endian - high bits are at low addresses. Favor aligned stores at 2097 // the cost of some bit-fiddling. 2098 GetExpandedInteger(N->getValue(), Lo, Hi); 2099 2100 MVT EVT = N->getMemoryVT(); 2101 unsigned EBytes = EVT.getStoreSizeInBits()/8; 2102 unsigned IncrementSize = NVT.getSizeInBits()/8; 2103 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2104 MVT HiVT = MVT::getIntegerVT(EVT.getSizeInBits() - ExcessBits); 2105 2106 if (ExcessBits < NVT.getSizeInBits()) { 2107 // Transfer high bits from the top of Lo to the bottom of Hi. 2108 Hi = DAG.getNode(ISD::SHL, NVT, Hi, 2109 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 2110 TLI.getShiftAmountTy())); 2111 Hi = DAG.getNode(ISD::OR, NVT, Hi, 2112 DAG.getNode(ISD::SRL, NVT, Lo, 2113 DAG.getConstant(ExcessBits, 2114 TLI.getShiftAmountTy()))); 2115 } 2116 2117 // Store both the high bits and maybe some of the low bits. 2118 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(), 2119 SVOffset, HiVT, isVolatile, Alignment); 2120 2121 // Increment the pointer to the other half. 2122 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 2123 DAG.getIntPtrConstant(IncrementSize)); 2124 // Store the lowest ExcessBits bits in the second half. 2125 Lo = DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), 2126 SVOffset+IncrementSize, 2127 MVT::getIntegerVT(ExcessBits), 2128 isVolatile, MinAlign(Alignment, IncrementSize)); 2129 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 2130 } 2131} 2132 2133SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 2134 SDValue InL, InH; 2135 GetExpandedInteger(N->getOperand(0), InL, InH); 2136 // Just truncate the low part of the source. 2137 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), InL); 2138} 2139 2140SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 2141 SDValue Op = N->getOperand(0); 2142 MVT SrcVT = Op.getValueType(); 2143 MVT DstVT = N->getValueType(0); 2144 2145 if (TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 2146 // Do a signed conversion then adjust the result. 2147 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, DstVT, Op); 2148 SignedConv = TLI.LowerOperation(SignedConv, DAG); 2149 2150 // The result of the signed conversion needs adjusting if the 'sign bit' of 2151 // the incoming integer was set. To handle this, we dynamically test to see 2152 // if it is set, and, if so, add a fudge factor. 2153 2154 const uint64_t F32TwoE32 = 0x4F800000ULL; 2155 const uint64_t F32TwoE64 = 0x5F800000ULL; 2156 const uint64_t F32TwoE128 = 0x7F800000ULL; 2157 2158 APInt FF(32, 0); 2159 if (SrcVT == MVT::i32) 2160 FF = APInt(32, F32TwoE32); 2161 else if (SrcVT == MVT::i64) 2162 FF = APInt(32, F32TwoE64); 2163 else if (SrcVT == MVT::i128) 2164 FF = APInt(32, F32TwoE128); 2165 else 2166 assert(false && "Unsupported UINT_TO_FP!"); 2167 2168 // Check whether the sign bit is set. 2169 SDValue Lo, Hi; 2170 GetExpandedInteger(Op, Lo, Hi); 2171 SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi.getValueType()), 2172 Hi, DAG.getConstant(0, Hi.getValueType()), 2173 ISD::SETLT); 2174 2175 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 2176 SDValue FudgePtr = DAG.getConstantPool(ConstantInt::get(FF.zext(64)), 2177 TLI.getPointerTy()); 2178 2179 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 2180 SDValue Zero = DAG.getIntPtrConstant(0); 2181 SDValue Four = DAG.getIntPtrConstant(4); 2182 if (TLI.isBigEndian()) std::swap(Zero, Four); 2183 SDValue Offset = DAG.getNode(ISD::SELECT, Zero.getValueType(), SignSet, 2184 Zero, Four); 2185 unsigned Alignment = 2186 1 << cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 2187 FudgePtr = DAG.getNode(ISD::ADD, TLI.getPointerTy(), FudgePtr, Offset); 2188 Alignment = std::min(Alignment, 4u); 2189 2190 // Load the value out, extending it from f32 to the destination float type. 2191 // FIXME: Avoid the extend by constructing the right constant pool? 2192 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, DstVT, DAG.getEntryNode(), 2193 FudgePtr, NULL, 0, MVT::f32, 2194 false, Alignment); 2195 return DAG.getNode(ISD::FADD, DstVT, SignedConv, Fudge); 2196 } 2197 2198 // Otherwise, use a libcall. 2199 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 2200 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2201 "Don't know how to expand this UINT_TO_FP!"); 2202 return MakeLibCall(LC, DstVT, &Op, 1, true); 2203} 2204