LegalizeIntegerTypes.cpp revision 126d90770bdb17e6925b2fe26de99aa079b7b9b3
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22#include "llvm/Constants.h" 23using namespace llvm; 24 25//===----------------------------------------------------------------------===// 26// Integer Result Promotion 27//===----------------------------------------------------------------------===// 28 29/// PromoteIntegerResult - This method is called when a result of a node is 30/// found to be in need of promotion to a larger type. At this point, the node 31/// may also have invalid operands or may have other results that need 32/// expansion, we just know that (at least) one result needs promotion. 33void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 34 DEBUG(cerr << "Promote integer result: "; N->dump(&DAG); cerr << "\n"); 35 SDOperand Result = SDOperand(); 36 37 // See if the target wants to custom expand this node. 38 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(ResNo)) == 39 TargetLowering::Custom) { 40 // If the target wants to, allow it to lower this itself. 41 if (SDNode *P = TLI.ReplaceNodeResults(N, DAG)) { 42 // Everything that once used N now uses P. We are guaranteed that the 43 // result value types of N and the result value types of P match. 44 ReplaceNodeWith(N, P); 45 return; 46 } 47 } 48 49 switch (N->getOpcode()) { 50 default: 51#ifndef NDEBUG 52 cerr << "PromoteIntegerResult #" << ResNo << ": "; 53 N->dump(&DAG); cerr << "\n"; 54#endif 55 assert(0 && "Do not know how to promote this operator!"); 56 abort(); 57 case ISD::UNDEF: Result = PromoteIntRes_UNDEF(N); break; 58 case ISD::Constant: Result = PromoteIntRes_Constant(N); break; 59 60 case ISD::TRUNCATE: Result = PromoteIntRes_TRUNCATE(N); break; 61 case ISD::SIGN_EXTEND: 62 case ISD::ZERO_EXTEND: 63 case ISD::ANY_EXTEND: Result = PromoteIntRes_INT_EXTEND(N); break; 64 case ISD::FP_ROUND: Result = PromoteIntRes_FP_ROUND(N); break; 65 case ISD::FP_TO_SINT: 66 case ISD::FP_TO_UINT: Result = PromoteIntRes_FP_TO_XINT(N); break; 67 case ISD::SETCC: Result = PromoteIntRes_SETCC(N); break; 68 case ISD::LOAD: Result = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 69 case ISD::BUILD_PAIR: Result = PromoteIntRes_BUILD_PAIR(N); break; 70 case ISD::BIT_CONVERT: Result = PromoteIntRes_BIT_CONVERT(N); break; 71 72 case ISD::AND: 73 case ISD::OR: 74 case ISD::XOR: 75 case ISD::ADD: 76 case ISD::SUB: 77 case ISD::MUL: Result = PromoteIntRes_SimpleIntBinOp(N); break; 78 79 case ISD::SDIV: 80 case ISD::SREM: Result = PromoteIntRes_SDIV(N); break; 81 82 case ISD::UDIV: 83 case ISD::UREM: Result = PromoteIntRes_UDIV(N); break; 84 85 case ISD::SHL: Result = PromoteIntRes_SHL(N); break; 86 case ISD::SRA: Result = PromoteIntRes_SRA(N); break; 87 case ISD::SRL: Result = PromoteIntRes_SRL(N); break; 88 89 case ISD::SELECT: Result = PromoteIntRes_SELECT(N); break; 90 case ISD::SELECT_CC: Result = PromoteIntRes_SELECT_CC(N); break; 91 92 case ISD::CTLZ: Result = PromoteIntRes_CTLZ(N); break; 93 case ISD::CTPOP: Result = PromoteIntRes_CTPOP(N); break; 94 case ISD::CTTZ: Result = PromoteIntRes_CTTZ(N); break; 95 96 case ISD::EXTRACT_VECTOR_ELT: 97 Result = PromoteIntRes_EXTRACT_VECTOR_ELT(N); 98 break; 99 100 case ISD::VAARG : Result = PromoteIntRes_VAARG(N); break; 101 } 102 103 // If Result is null, the sub-method took care of registering the result. 104 if (Result.Val) 105 SetPromotedInteger(SDOperand(N, ResNo), Result); 106} 107 108SDOperand DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 109 return DAG.getNode(ISD::UNDEF, TLI.getTypeToTransformTo(N->getValueType(0))); 110} 111 112SDOperand DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 113 MVT VT = N->getValueType(0); 114 // Zero extend things like i1, sign extend everything else. It shouldn't 115 // matter in theory which one we pick, but this tends to give better code? 116 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 117 SDOperand Result = DAG.getNode(Opc, TLI.getTypeToTransformTo(VT), 118 SDOperand(N, 0)); 119 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 120 return Result; 121} 122 123SDOperand DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 124 SDOperand Res; 125 126 switch (getTypeAction(N->getOperand(0).getValueType())) { 127 default: assert(0 && "Unknown type action!"); 128 case Legal: 129 case ExpandInteger: 130 Res = N->getOperand(0); 131 break; 132 case PromoteInteger: 133 Res = GetPromotedInteger(N->getOperand(0)); 134 break; 135 } 136 137 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 138 assert(Res.getValueType().getSizeInBits() >= NVT.getSizeInBits() && 139 "Truncation doesn't make sense!"); 140 if (Res.getValueType() == NVT) 141 return Res; 142 143 // Truncate to NVT instead of VT 144 return DAG.getNode(ISD::TRUNCATE, NVT, Res); 145} 146 147SDOperand DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 148 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 149 150 if (getTypeAction(N->getOperand(0).getValueType()) == PromoteInteger) { 151 SDOperand Res = GetPromotedInteger(N->getOperand(0)); 152 assert(Res.getValueType().getSizeInBits() <= NVT.getSizeInBits() && 153 "Extension doesn't make sense!"); 154 155 // If the result and operand types are the same after promotion, simplify 156 // to an in-register extension. 157 if (NVT == Res.getValueType()) { 158 // The high bits are not guaranteed to be anything. Insert an extend. 159 if (N->getOpcode() == ISD::SIGN_EXTEND) 160 return DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Res, 161 DAG.getValueType(N->getOperand(0).getValueType())); 162 if (N->getOpcode() == ISD::ZERO_EXTEND) 163 return DAG.getZeroExtendInReg(Res, N->getOperand(0).getValueType()); 164 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 165 return Res; 166 } 167 } 168 169 // Otherwise, just extend the original operand all the way to the larger type. 170 return DAG.getNode(N->getOpcode(), NVT, N->getOperand(0)); 171} 172 173SDOperand DAGTypeLegalizer::PromoteIntRes_FP_ROUND(SDNode *N) { 174 // NOTE: Assumes input is legal. 175 if (N->getConstantOperandVal(1) == 0) 176 return DAG.getNode(ISD::FP_ROUND_INREG, N->getOperand(0).getValueType(), 177 N->getOperand(0), DAG.getValueType(N->getValueType(0))); 178 // If the precision discard isn't needed, just return the operand unrounded. 179 return N->getOperand(0); 180} 181 182SDOperand DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 183 unsigned NewOpc = N->getOpcode(); 184 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 185 186 // If we're promoting a UINT to a larger size, check to see if the new node 187 // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since 188 // we can use that instead. This allows us to generate better code for 189 // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not 190 // legal, such as PowerPC. 191 if (N->getOpcode() == ISD::FP_TO_UINT) { 192 if (!TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 193 (TLI.isOperationLegal(ISD::FP_TO_SINT, NVT) || 194 TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)) 195 NewOpc = ISD::FP_TO_SINT; 196 } 197 198 return DAG.getNode(NewOpc, NVT, N->getOperand(0)); 199} 200 201SDOperand DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 202 assert(isTypeLegal(TLI.getSetCCResultType(N->getOperand(0))) 203 && "SetCC type is not legal??"); 204 return DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(N->getOperand(0)), 205 N->getOperand(0), N->getOperand(1), N->getOperand(2)); 206} 207 208SDOperand DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 209 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 210 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 211 ISD::LoadExtType ExtType = 212 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 213 SDOperand Res = DAG.getExtLoad(ExtType, NVT, N->getChain(), N->getBasePtr(), 214 N->getSrcValue(), N->getSrcValueOffset(), 215 N->getMemoryVT(), N->isVolatile(), 216 N->getAlignment()); 217 218 // Legalized the chain result - switch anything that used the old chain to 219 // use the new one. 220 ReplaceValueWith(SDOperand(N, 1), Res.getValue(1)); 221 return Res; 222} 223 224SDOperand DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 225 // The pair element type may be legal, or may not promote to the same type as 226 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 227 return DAG.getNode(ISD::ANY_EXTEND, 228 TLI.getTypeToTransformTo(N->getValueType(0)), 229 JoinIntegers(N->getOperand(0), N->getOperand(1))); 230} 231 232SDOperand DAGTypeLegalizer::PromoteIntRes_BIT_CONVERT(SDNode *N) { 233 SDOperand InOp = N->getOperand(0); 234 MVT InVT = InOp.getValueType(); 235 MVT NInVT = TLI.getTypeToTransformTo(InVT); 236 MVT OutVT = TLI.getTypeToTransformTo(N->getValueType(0)); 237 238 switch (getTypeAction(InVT)) { 239 default: 240 assert(false && "Unknown type action!"); 241 break; 242 case Legal: 243 break; 244 case PromoteInteger: 245 if (OutVT.getSizeInBits() == NInVT.getSizeInBits()) 246 // The input promotes to the same size. Convert the promoted value. 247 return DAG.getNode(ISD::BIT_CONVERT, OutVT, GetPromotedInteger(InOp)); 248 break; 249 case SoftenFloat: 250 // Promote the integer operand by hand. 251 return DAG.getNode(ISD::ANY_EXTEND, OutVT, GetSoftenedFloat(InOp)); 252 case ExpandInteger: 253 case ExpandFloat: 254 break; 255 case Scalarize: 256 // Convert the element to an integer and promote it by hand. 257 return DAG.getNode(ISD::ANY_EXTEND, OutVT, 258 BitConvertToInteger(GetScalarizedVector(InOp))); 259 case Split: 260 // For example, i32 = BIT_CONVERT v2i16 on alpha. Convert the split 261 // pieces of the input into integers and reassemble in the final type. 262 SDOperand Lo, Hi; 263 GetSplitVector(N->getOperand(0), Lo, Hi); 264 Lo = BitConvertToInteger(Lo); 265 Hi = BitConvertToInteger(Hi); 266 267 if (TLI.isBigEndian()) 268 std::swap(Lo, Hi); 269 270 InOp = DAG.getNode(ISD::ANY_EXTEND, 271 MVT::getIntegerVT(OutVT.getSizeInBits()), 272 JoinIntegers(Lo, Hi)); 273 return DAG.getNode(ISD::BIT_CONVERT, OutVT, InOp); 274 } 275 276 // Otherwise, lower the bit-convert to a store/load from the stack, then 277 // promote the load. 278 SDOperand Op = CreateStackStoreLoad(InOp, N->getValueType(0)); 279 return PromoteIntRes_LOAD(cast<LoadSDNode>(Op.Val)); 280} 281 282SDOperand DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 283 // The input may have strange things in the top bits of the registers, but 284 // these operations don't care. They may have weird bits going out, but 285 // that too is okay if they are integer operations. 286 SDOperand LHS = GetPromotedInteger(N->getOperand(0)); 287 SDOperand RHS = GetPromotedInteger(N->getOperand(1)); 288 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 289} 290 291SDOperand DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) { 292 // Sign extend the input. 293 SDOperand LHS = GetPromotedInteger(N->getOperand(0)); 294 SDOperand RHS = GetPromotedInteger(N->getOperand(1)); 295 MVT VT = N->getValueType(0); 296 LHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, LHS.getValueType(), LHS, 297 DAG.getValueType(VT)); 298 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, RHS.getValueType(), RHS, 299 DAG.getValueType(VT)); 300 301 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 302} 303 304SDOperand DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { 305 // Zero extend the input. 306 SDOperand LHS = GetPromotedInteger(N->getOperand(0)); 307 SDOperand RHS = GetPromotedInteger(N->getOperand(1)); 308 MVT VT = N->getValueType(0); 309 LHS = DAG.getZeroExtendInReg(LHS, VT); 310 RHS = DAG.getZeroExtendInReg(RHS, VT); 311 312 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 313} 314 315SDOperand DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 316 return DAG.getNode(ISD::SHL, TLI.getTypeToTransformTo(N->getValueType(0)), 317 GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); 318} 319 320SDOperand DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 321 // The input value must be properly sign extended. 322 MVT VT = N->getValueType(0); 323 MVT NVT = TLI.getTypeToTransformTo(VT); 324 SDOperand Res = GetPromotedInteger(N->getOperand(0)); 325 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Res, DAG.getValueType(VT)); 326 return DAG.getNode(ISD::SRA, NVT, Res, N->getOperand(1)); 327} 328 329SDOperand DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 330 // The input value must be properly zero extended. 331 MVT VT = N->getValueType(0); 332 MVT NVT = TLI.getTypeToTransformTo(VT); 333 SDOperand Res = ZExtPromotedInteger(N->getOperand(0)); 334 return DAG.getNode(ISD::SRL, NVT, Res, N->getOperand(1)); 335} 336 337SDOperand DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 338 SDOperand LHS = GetPromotedInteger(N->getOperand(1)); 339 SDOperand RHS = GetPromotedInteger(N->getOperand(2)); 340 return DAG.getNode(ISD::SELECT, LHS.getValueType(), N->getOperand(0),LHS,RHS); 341} 342 343SDOperand DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 344 SDOperand LHS = GetPromotedInteger(N->getOperand(2)); 345 SDOperand RHS = GetPromotedInteger(N->getOperand(3)); 346 return DAG.getNode(ISD::SELECT_CC, LHS.getValueType(), N->getOperand(0), 347 N->getOperand(1), LHS, RHS, N->getOperand(4)); 348} 349 350SDOperand DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 351 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 352 MVT OVT = N->getValueType(0); 353 MVT NVT = Op.getValueType(); 354 // Zero extend to the promoted type and do the count there. 355 Op = DAG.getNode(ISD::CTLZ, NVT, DAG.getZeroExtendInReg(Op, OVT)); 356 // Subtract off the extra leading bits in the bigger type. 357 return DAG.getNode(ISD::SUB, NVT, Op, 358 DAG.getConstant(NVT.getSizeInBits() - 359 OVT.getSizeInBits(), NVT)); 360} 361 362SDOperand DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 363 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 364 MVT OVT = N->getValueType(0); 365 MVT NVT = Op.getValueType(); 366 // Zero extend to the promoted type and do the count there. 367 return DAG.getNode(ISD::CTPOP, NVT, DAG.getZeroExtendInReg(Op, OVT)); 368} 369 370SDOperand DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 371 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 372 MVT OVT = N->getValueType(0); 373 MVT NVT = Op.getValueType(); 374 // The count is the same in the promoted type except if the original 375 // value was zero. This can be handled by setting the bit just off 376 // the top of the original type. 377 Op = DAG.getNode(ISD::OR, NVT, Op, 378 // FIXME: Do this using an APINT constant. 379 DAG.getConstant(1UL << OVT.getSizeInBits(), NVT)); 380 return DAG.getNode(ISD::CTTZ, NVT, Op); 381} 382 383SDOperand DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 384 MVT OldVT = N->getValueType(0); 385 SDOperand OldVec = N->getOperand(0); 386 unsigned OldElts = OldVec.getValueType().getVectorNumElements(); 387 388 if (OldElts == 1) { 389 assert(!isTypeLegal(OldVec.getValueType()) && 390 "Legal one-element vector of a type needing promotion!"); 391 // It is tempting to follow GetScalarizedVector by a call to 392 // GetPromotedInteger, but this would be wrong because the 393 // scalarized value may not yet have been processed. 394 return DAG.getNode(ISD::ANY_EXTEND, TLI.getTypeToTransformTo(OldVT), 395 GetScalarizedVector(OldVec)); 396 } 397 398 // Convert to a vector half as long with an element type of twice the width, 399 // for example <4 x i16> -> <2 x i32>. 400 assert(!(OldElts & 1) && "Odd length vectors not supported!"); 401 MVT NewVT = MVT::getIntegerVT(2 * OldVT.getSizeInBits()); 402 assert(OldVT.isSimple() && NewVT.isSimple()); 403 404 SDOperand NewVec = DAG.getNode(ISD::BIT_CONVERT, 405 MVT::getVectorVT(NewVT, OldElts / 2), 406 OldVec); 407 408 // Extract the element at OldIdx / 2 from the new vector. 409 SDOperand OldIdx = N->getOperand(1); 410 SDOperand NewIdx = DAG.getNode(ISD::SRL, OldIdx.getValueType(), OldIdx, 411 DAG.getConstant(1, TLI.getShiftAmountTy())); 412 SDOperand Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, NewIdx); 413 414 // Select the appropriate half of the element: Lo if OldIdx was even, 415 // Hi if it was odd. 416 SDOperand Lo = Elt; 417 SDOperand Hi = DAG.getNode(ISD::SRL, NewVT, Elt, 418 DAG.getConstant(OldVT.getSizeInBits(), 419 TLI.getShiftAmountTy())); 420 if (TLI.isBigEndian()) 421 std::swap(Lo, Hi); 422 423 SDOperand Odd = DAG.getNode(ISD::AND, OldIdx.getValueType(), OldIdx, 424 DAG.getConstant(1, TLI.getShiftAmountTy())); 425 return DAG.getNode(ISD::SELECT, NewVT, Odd, Hi, Lo); 426} 427 428SDOperand DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 429 SDOperand Chain = N->getOperand(0); // Get the chain. 430 SDOperand Ptr = N->getOperand(1); // Get the pointer. 431 MVT VT = N->getValueType(0); 432 433 const Value *V = cast<SrcValueSDNode>(N->getOperand(2))->getValue(); 434 SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Chain, Ptr, V, 0); 435 436 // Increment the arg pointer, VAList, to the next vaarg 437 // FIXME: should the ABI size be used for the increment? Think of 438 // x86 long double (10 bytes long, but aligned on 4 or 8 bytes) or 439 // integers of unusual size (such MVT::i1, which gives an increment 440 // of zero here!). 441 unsigned Increment = VT.getSizeInBits() / 8; 442 SDOperand Tmp = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, 443 DAG.getConstant(Increment, TLI.getPointerTy())); 444 445 // Store the incremented VAList to the pointer. 446 Tmp = DAG.getStore(VAList.getValue(1), Tmp, Ptr, V, 0); 447 448 // Load the actual argument out of the arg pointer VAList. 449 Tmp = DAG.getExtLoad(ISD::EXTLOAD, TLI.getTypeToTransformTo(VT), Tmp, 450 VAList, NULL, 0, VT); 451 452 // Legalized the chain result - switch anything that used the old chain to 453 // use the new one. 454 ReplaceValueWith(SDOperand(N, 1), Tmp.getValue(1)); 455 return Tmp; 456} 457 458//===----------------------------------------------------------------------===// 459// Integer Operand Promotion 460//===----------------------------------------------------------------------===// 461 462/// PromoteIntegerOperand - This method is called when the specified operand of 463/// the specified node is found to need promotion. At this point, all of the 464/// result types of the node are known to be legal, but other operands of the 465/// node may need promotion or expansion as well as the specified one. 466bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 467 DEBUG(cerr << "Promote integer operand: "; N->dump(&DAG); cerr << "\n"); 468 SDOperand Res(0, 0); 469 470 if (TLI.getOperationAction(N->getOpcode(), N->getOperand(OpNo).getValueType()) 471 == TargetLowering::Custom) 472 Res = TLI.LowerOperation(SDOperand(N, OpNo), DAG); 473 474 if (Res.Val == 0) { 475 switch (N->getOpcode()) { 476 default: 477 #ifndef NDEBUG 478 cerr << "PromoteIntegerOperand Op #" << OpNo << ": "; 479 N->dump(&DAG); cerr << "\n"; 480 #endif 481 assert(0 && "Do not know how to promote this operator's operand!"); 482 abort(); 483 484 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 485 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 486 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 487 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 488 case ISD::FP_EXTEND: Res = PromoteIntOp_FP_EXTEND(N); break; 489 case ISD::FP_ROUND: Res = PromoteIntOp_FP_ROUND(N); break; 490 case ISD::SINT_TO_FP: 491 case ISD::UINT_TO_FP: Res = PromoteIntOp_INT_TO_FP(N); break; 492 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 493 494 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 495 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 496 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 497 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 498 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 499 500 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 501 OpNo); break; 502 503 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 504 case ISD::INSERT_VECTOR_ELT: 505 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); 506 break; 507 508 case ISD::MEMBARRIER: Res = PromoteIntOp_MEMBARRIER(N); break; 509 } 510 } 511 512 // If the result is null, the sub-method took care of registering results etc. 513 if (!Res.Val) return false; 514 // If the result is N, the sub-method updated N in place. 515 if (Res.Val == N) { 516 // Mark N as new and remark N and its operands. This allows us to correctly 517 // revisit N if it needs another step of promotion and allows us to visit 518 // any new operands to N. 519 ReanalyzeNode(N); 520 return true; 521 } 522 523 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 524 "Invalid operand expansion"); 525 526 ReplaceValueWith(SDOperand(N, 0), Res); 527 return false; 528} 529 530SDOperand DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 531 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 532 return DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 533} 534 535SDOperand DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 536 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 537 Op = DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 538 return DAG.getZeroExtendInReg(Op, N->getOperand(0).getValueType()); 539} 540 541SDOperand DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 542 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 543 Op = DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op); 544 return DAG.getNode(ISD::SIGN_EXTEND_INREG, Op.getValueType(), 545 Op, DAG.getValueType(N->getOperand(0).getValueType())); 546} 547 548SDOperand DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 549 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 550 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), Op); 551} 552 553SDOperand DAGTypeLegalizer::PromoteIntOp_FP_EXTEND(SDNode *N) { 554 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 555 return DAG.getNode(ISD::FP_EXTEND, N->getValueType(0), Op); 556} 557 558SDOperand DAGTypeLegalizer::PromoteIntOp_FP_ROUND(SDNode *N) { 559 SDOperand Op = GetPromotedInteger(N->getOperand(0)); 560 return DAG.getNode(ISD::FP_ROUND, N->getValueType(0), Op, 561 DAG.getIntPtrConstant(0)); 562} 563 564SDOperand DAGTypeLegalizer::PromoteIntOp_INT_TO_FP(SDNode *N) { 565 SDOperand In = GetPromotedInteger(N->getOperand(0)); 566 MVT OpVT = N->getOperand(0).getValueType(); 567 if (N->getOpcode() == ISD::UINT_TO_FP) 568 In = DAG.getZeroExtendInReg(In, OpVT); 569 else 570 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, In.getValueType(), 571 In, DAG.getValueType(OpVT)); 572 573 return DAG.UpdateNodeOperands(SDOperand(N, 0), In); 574} 575 576SDOperand DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 577 // Since the result type is legal, the operands must promote to it. 578 MVT OVT = N->getOperand(0).getValueType(); 579 SDOperand Lo = GetPromotedInteger(N->getOperand(0)); 580 SDOperand Hi = GetPromotedInteger(N->getOperand(1)); 581 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 582 583 Lo = DAG.getZeroExtendInReg(Lo, OVT); 584 Hi = DAG.getNode(ISD::SHL, N->getValueType(0), Hi, 585 DAG.getConstant(OVT.getSizeInBits(), 586 TLI.getShiftAmountTy())); 587 return DAG.getNode(ISD::OR, N->getValueType(0), Lo, Hi); 588} 589 590SDOperand DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 591 assert(OpNo == 0 && "Only know how to promote condition"); 592 SDOperand Cond = GetPromotedInteger(N->getOperand(0)); // Promote condition. 593 594 // The top bits of the promoted condition are not necessarily zero, ensure 595 // that the value is properly zero extended. 596 unsigned BitWidth = Cond.getValueSizeInBits(); 597 if (!DAG.MaskedValueIsZero(Cond, 598 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 599 Cond = DAG.getZeroExtendInReg(Cond, MVT::i1); 600 601 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 602 return DAG.UpdateNodeOperands(SDOperand(N, 0), Cond, N->getOperand(1), 603 N->getOperand(2)); 604} 605 606SDOperand DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 607 assert(OpNo == 1 && "only know how to promote condition"); 608 SDOperand Cond = GetPromotedInteger(N->getOperand(1)); // Promote condition. 609 610 // The top bits of the promoted condition are not necessarily zero, ensure 611 // that the value is properly zero extended. 612 unsigned BitWidth = Cond.getValueSizeInBits(); 613 if (!DAG.MaskedValueIsZero(Cond, 614 APInt::getHighBitsSet(BitWidth, BitWidth-1))) 615 Cond = DAG.getZeroExtendInReg(Cond, MVT::i1); 616 617 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 618 return DAG.UpdateNodeOperands(SDOperand(N, 0), N->getOperand(0), Cond, 619 N->getOperand(2)); 620} 621 622SDOperand DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 623 assert(OpNo == 2 && "Don't know how to promote this operand!"); 624 625 SDOperand LHS = N->getOperand(2); 626 SDOperand RHS = N->getOperand(3); 627 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 628 629 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 630 // legal types. 631 return DAG.UpdateNodeOperands(SDOperand(N, 0), N->getOperand(0), 632 N->getOperand(1), LHS, RHS, N->getOperand(4)); 633} 634 635SDOperand DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 636 assert(OpNo == 0 && "Don't know how to promote this operand!"); 637 638 SDOperand LHS = N->getOperand(0); 639 SDOperand RHS = N->getOperand(1); 640 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 641 642 // The CC (#4) and the possible return values (#2 and #3) have legal types. 643 return DAG.UpdateNodeOperands(SDOperand(N, 0), LHS, RHS, N->getOperand(2), 644 N->getOperand(3), N->getOperand(4)); 645} 646 647SDOperand DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 648 assert(OpNo == 0 && "Don't know how to promote this operand!"); 649 650 SDOperand LHS = N->getOperand(0); 651 SDOperand RHS = N->getOperand(1); 652 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 653 654 // The CC (#2) is always legal. 655 return DAG.UpdateNodeOperands(SDOperand(N, 0), LHS, RHS, N->getOperand(2)); 656} 657 658/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 659/// shared among BR_CC, SELECT_CC, and SETCC handlers. 660void DAGTypeLegalizer::PromoteSetCCOperands(SDOperand &NewLHS,SDOperand &NewRHS, 661 ISD::CondCode CCCode) { 662 MVT VT = NewLHS.getValueType(); 663 664 // Get the promoted values. 665 NewLHS = GetPromotedInteger(NewLHS); 666 NewRHS = GetPromotedInteger(NewRHS); 667 668 // Otherwise, we have to insert explicit sign or zero extends. Note 669 // that we could insert sign extends for ALL conditions, but zero extend 670 // is cheaper on many machines (an AND instead of two shifts), so prefer 671 // it. 672 switch (CCCode) { 673 default: assert(0 && "Unknown integer comparison!"); 674 case ISD::SETEQ: 675 case ISD::SETNE: 676 case ISD::SETUGE: 677 case ISD::SETUGT: 678 case ISD::SETULE: 679 case ISD::SETULT: 680 // ALL of these operations will work if we either sign or zero extend 681 // the operands (including the unsigned comparisons!). Zero extend is 682 // usually a simpler/cheaper operation, so prefer it. 683 NewLHS = DAG.getZeroExtendInReg(NewLHS, VT); 684 NewRHS = DAG.getZeroExtendInReg(NewRHS, VT); 685 break; 686 case ISD::SETGE: 687 case ISD::SETGT: 688 case ISD::SETLT: 689 case ISD::SETLE: 690 NewLHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, NewLHS.getValueType(), NewLHS, 691 DAG.getValueType(VT)); 692 NewRHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, NewRHS.getValueType(), NewRHS, 693 DAG.getValueType(VT)); 694 break; 695 } 696} 697 698SDOperand DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 699 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 700 SDOperand Ch = N->getChain(), Ptr = N->getBasePtr(); 701 int SVOffset = N->getSrcValueOffset(); 702 unsigned Alignment = N->getAlignment(); 703 bool isVolatile = N->isVolatile(); 704 705 SDOperand Val = GetPromotedInteger(N->getValue()); // Get promoted value. 706 707 assert(!N->isTruncatingStore() && "Cannot promote this store operand!"); 708 709 // Truncate the value and store the result. 710 return DAG.getTruncStore(Ch, Val, Ptr, N->getSrcValue(), 711 SVOffset, N->getMemoryVT(), 712 isVolatile, Alignment); 713} 714 715SDOperand DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 716 // The vector type is legal but the element type is not. This implies 717 // that the vector is a power-of-two in length and that the element 718 // type does not have a strange size (eg: it is not i1). 719 MVT VecVT = N->getValueType(0); 720 unsigned NumElts = VecVT.getVectorNumElements(); 721 assert(!(NumElts & 1) && "Legal vector of one illegal element?"); 722 723 // Build a vector of half the length out of elements of twice the bitwidth. 724 // For example <4 x i16> -> <2 x i32>. 725 MVT OldVT = N->getOperand(0).getValueType(); 726 MVT NewVT = MVT::getIntegerVT(2 * OldVT.getSizeInBits()); 727 assert(OldVT.isSimple() && NewVT.isSimple()); 728 729 std::vector<SDOperand> NewElts; 730 NewElts.reserve(NumElts/2); 731 732 for (unsigned i = 0; i < NumElts; i += 2) { 733 // Combine two successive elements into one promoted element. 734 SDOperand Lo = N->getOperand(i); 735 SDOperand Hi = N->getOperand(i+1); 736 if (TLI.isBigEndian()) 737 std::swap(Lo, Hi); 738 NewElts.push_back(JoinIntegers(Lo, Hi)); 739 } 740 741 SDOperand NewVec = DAG.getNode(ISD::BUILD_VECTOR, 742 MVT::getVectorVT(NewVT, NewElts.size()), 743 &NewElts[0], NewElts.size()); 744 745 // Convert the new vector to the old vector type. 746 return DAG.getNode(ISD::BIT_CONVERT, VecVT, NewVec); 747} 748 749SDOperand DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 750 unsigned OpNo) { 751 if (OpNo == 1) { 752 // Promote the inserted value. This is valid because the type does not 753 // have to match the vector element type. 754 755 // Check that any extra bits introduced will be truncated away. 756 assert(N->getOperand(1).getValueType().getSizeInBits() >= 757 N->getValueType(0).getVectorElementType().getSizeInBits() && 758 "Type of inserted value narrower than vector element type!"); 759 return DAG.UpdateNodeOperands(SDOperand(N, 0), N->getOperand(0), 760 GetPromotedInteger(N->getOperand(1)), 761 N->getOperand(2)); 762 } 763 764 assert(OpNo == 2 && "Different operand and result vector types?"); 765 766 // Promote the index. 767 SDOperand Idx = N->getOperand(2); 768 Idx = DAG.getZeroExtendInReg(GetPromotedInteger(Idx), Idx.getValueType()); 769 return DAG.UpdateNodeOperands(SDOperand(N, 0), N->getOperand(0), 770 N->getOperand(1), Idx); 771} 772 773SDOperand DAGTypeLegalizer::PromoteIntOp_MEMBARRIER(SDNode *N) { 774 SDOperand NewOps[6]; 775 NewOps[0] = N->getOperand(0); 776 for (unsigned i = 1; i < array_lengthof(NewOps); ++i) { 777 SDOperand Flag = GetPromotedInteger(N->getOperand(i)); 778 NewOps[i] = DAG.getZeroExtendInReg(Flag, MVT::i1); 779 } 780 return DAG.UpdateNodeOperands(SDOperand (N, 0), NewOps, 781 array_lengthof(NewOps)); 782} 783 784 785//===----------------------------------------------------------------------===// 786// Integer Result Expansion 787//===----------------------------------------------------------------------===// 788 789/// ExpandIntegerResult - This method is called when the specified result of the 790/// specified node is found to need expansion. At this point, the node may also 791/// have invalid operands or may have other results that need promotion, we just 792/// know that (at least) one result needs expansion. 793void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 794 DEBUG(cerr << "Expand integer result: "; N->dump(&DAG); cerr << "\n"); 795 SDOperand Lo, Hi; 796 Lo = Hi = SDOperand(); 797 798 // See if the target wants to custom expand this node. 799 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(ResNo)) == 800 TargetLowering::Custom) { 801 // If the target wants to, allow it to lower this itself. 802 if (SDNode *P = TLI.ReplaceNodeResults(N, DAG)) { 803 // Everything that once used N now uses P. We are guaranteed that the 804 // result value types of N and the result value types of P match. 805 ReplaceNodeWith(N, P); 806 return; 807 } 808 } 809 810 switch (N->getOpcode()) { 811 default: 812#ifndef NDEBUG 813 cerr << "ExpandIntegerResult #" << ResNo << ": "; 814 N->dump(&DAG); cerr << "\n"; 815#endif 816 assert(0 && "Do not know how to expand the result of this operator!"); 817 abort(); 818 819 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 820 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 821 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 822 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 823 824 case ISD::BIT_CONVERT: ExpandRes_BIT_CONVERT(N, Lo, Hi); break; 825 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 826 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 827 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 828 829 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 830 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 831 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 832 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 833 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 834 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 835 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 836 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 837 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 838 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 839 840 case ISD::AND: 841 case ISD::OR: 842 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 843 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 844 case ISD::ADD: 845 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 846 case ISD::ADDC: 847 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 848 case ISD::ADDE: 849 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 850 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 851 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 852 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 853 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 854 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 855 case ISD::SHL: 856 case ISD::SRA: 857 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 858 859 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 860 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 861 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 862 } 863 864 // If Lo/Hi is null, the sub-method took care of registering results etc. 865 if (Lo.Val) 866 SetExpandedInteger(SDOperand(N, ResNo), Lo, Hi); 867} 868 869void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 870 SDOperand &Lo, SDOperand &Hi) { 871 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 872 unsigned NBitWidth = NVT.getSizeInBits(); 873 const APInt &Cst = cast<ConstantSDNode>(N)->getAPIntValue(); 874 Lo = DAG.getConstant(APInt(Cst).trunc(NBitWidth), NVT); 875 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT); 876} 877 878void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 879 SDOperand &Lo, SDOperand &Hi) { 880 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 881 SDOperand Op = N->getOperand(0); 882 if (Op.getValueType().bitsLE(NVT)) { 883 // The low part is any extension of the input (which degenerates to a copy). 884 Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Op); 885 Hi = DAG.getNode(ISD::UNDEF, NVT); // The high part is undefined. 886 } else { 887 // For example, extension of an i48 to an i64. The operand type necessarily 888 // promotes to the result type, so will end up being expanded too. 889 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 890 "Only know how to promote this result!"); 891 SDOperand Res = GetPromotedInteger(Op); 892 assert(Res.getValueType() == N->getValueType(0) && 893 "Operand over promoted?"); 894 // Split the promoted operand. This will simplify when it is expanded. 895 SplitInteger(Res, Lo, Hi); 896 } 897} 898 899void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 900 SDOperand &Lo, SDOperand &Hi) { 901 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 902 SDOperand Op = N->getOperand(0); 903 if (Op.getValueType().bitsLE(NVT)) { 904 // The low part is zero extension of the input (which degenerates to a copy). 905 Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, N->getOperand(0)); 906 Hi = DAG.getConstant(0, NVT); // The high part is just a zero. 907 } else { 908 // For example, extension of an i48 to an i64. The operand type necessarily 909 // promotes to the result type, so will end up being expanded too. 910 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 911 "Only know how to promote this result!"); 912 SDOperand Res = GetPromotedInteger(Op); 913 assert(Res.getValueType() == N->getValueType(0) && 914 "Operand over promoted?"); 915 // Split the promoted operand. This will simplify when it is expanded. 916 SplitInteger(Res, Lo, Hi); 917 unsigned ExcessBits = 918 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 919 Hi = DAG.getZeroExtendInReg(Hi, MVT::getIntegerVT(ExcessBits)); 920 } 921} 922 923void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 924 SDOperand &Lo, SDOperand &Hi) { 925 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 926 SDOperand Op = N->getOperand(0); 927 if (Op.getValueType().bitsLE(NVT)) { 928 // The low part is sign extension of the input (which degenerates to a copy). 929 Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, N->getOperand(0)); 930 // The high part is obtained by SRA'ing all but one of the bits of low part. 931 unsigned LoSize = NVT.getSizeInBits(); 932 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 933 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 934 } else { 935 // For example, extension of an i48 to an i64. The operand type necessarily 936 // promotes to the result type, so will end up being expanded too. 937 assert(getTypeAction(Op.getValueType()) == PromoteInteger && 938 "Only know how to promote this result!"); 939 SDOperand Res = GetPromotedInteger(Op); 940 assert(Res.getValueType() == N->getValueType(0) && 941 "Operand over promoted?"); 942 // Split the promoted operand. This will simplify when it is expanded. 943 SplitInteger(Res, Lo, Hi); 944 unsigned ExcessBits = 945 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 946 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi, 947 DAG.getValueType(MVT::getIntegerVT(ExcessBits))); 948 } 949} 950 951void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 952 SDOperand &Lo, SDOperand &Hi) { 953 GetExpandedInteger(N->getOperand(0), Lo, Hi); 954 MVT NVT = Lo.getValueType(); 955 MVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 956 unsigned NVTBits = NVT.getSizeInBits(); 957 unsigned EVTBits = EVT.getSizeInBits(); 958 959 if (NVTBits < EVTBits) { 960 Hi = DAG.getNode(ISD::AssertZext, NVT, Hi, 961 DAG.getValueType(MVT::getIntegerVT(EVTBits - NVTBits))); 962 } else { 963 Lo = DAG.getNode(ISD::AssertZext, NVT, Lo, DAG.getValueType(EVT)); 964 // The high part must be zero, make it explicit. 965 Hi = DAG.getConstant(0, NVT); 966 } 967} 968 969void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 970 SDOperand &Lo, SDOperand &Hi) { 971 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 972 Lo = DAG.getNode(ISD::TRUNCATE, NVT, N->getOperand(0)); 973 Hi = DAG.getNode(ISD::SRL, N->getOperand(0).getValueType(), N->getOperand(0), 974 DAG.getConstant(NVT.getSizeInBits(), 975 TLI.getShiftAmountTy())); 976 Hi = DAG.getNode(ISD::TRUNCATE, NVT, Hi); 977} 978 979void DAGTypeLegalizer:: 980ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDOperand &Lo, SDOperand &Hi) { 981 GetExpandedInteger(N->getOperand(0), Lo, Hi); 982 MVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 983 984 if (EVT.bitsLE(Lo.getValueType())) { 985 // sext_inreg the low part if needed. 986 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, Lo.getValueType(), Lo, 987 N->getOperand(1)); 988 989 // The high part gets the sign extension from the lo-part. This handles 990 // things like sextinreg V:i64 from i8. 991 Hi = DAG.getNode(ISD::SRA, Hi.getValueType(), Lo, 992 DAG.getConstant(Hi.getValueType().getSizeInBits()-1, 993 TLI.getShiftAmountTy())); 994 } else { 995 // For example, extension of an i48 to an i64. Leave the low part alone, 996 // sext_inreg the high part. 997 unsigned ExcessBits = 998 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 999 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi, 1000 DAG.getValueType(MVT::getIntegerVT(ExcessBits))); 1001 } 1002} 1003 1004void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDOperand &Lo, 1005 SDOperand &Hi) { 1006 MVT VT = N->getValueType(0); 1007 SDOperand Op = N->getOperand(0); 1008 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1009 if (VT == MVT::i64) { 1010 if (Op.getValueType() == MVT::f32) 1011 LC = RTLIB::FPTOSINT_F32_I64; 1012 else if (Op.getValueType() == MVT::f64) 1013 LC = RTLIB::FPTOSINT_F64_I64; 1014 else if (Op.getValueType() == MVT::f80) 1015 LC = RTLIB::FPTOSINT_F80_I64; 1016 else if (Op.getValueType() == MVT::ppcf128) 1017 LC = RTLIB::FPTOSINT_PPCF128_I64; 1018 } else if (VT == MVT::i128) { 1019 if (Op.getValueType() == MVT::f32) 1020 LC = RTLIB::FPTOSINT_F32_I128; 1021 else if (Op.getValueType() == MVT::f64) 1022 LC = RTLIB::FPTOSINT_F64_I128; 1023 else if (Op.getValueType() == MVT::f80) 1024 LC = RTLIB::FPTOSINT_F80_I128; 1025 else if (Op.getValueType() == MVT::ppcf128) 1026 LC = RTLIB::FPTOSINT_PPCF128_I128; 1027 } else { 1028 assert(0 && "Unexpected fp-to-sint conversion!"); 1029 } 1030 SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*sign irrelevant*/), Lo, Hi); 1031} 1032 1033void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDOperand &Lo, 1034 SDOperand &Hi) { 1035 MVT VT = N->getValueType(0); 1036 SDOperand Op = N->getOperand(0); 1037 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1038 if (VT == MVT::i64) { 1039 if (Op.getValueType() == MVT::f32) 1040 LC = RTLIB::FPTOUINT_F32_I64; 1041 else if (Op.getValueType() == MVT::f64) 1042 LC = RTLIB::FPTOUINT_F64_I64; 1043 else if (Op.getValueType() == MVT::f80) 1044 LC = RTLIB::FPTOUINT_F80_I64; 1045 else if (Op.getValueType() == MVT::ppcf128) 1046 LC = RTLIB::FPTOUINT_PPCF128_I64; 1047 } else if (VT == MVT::i128) { 1048 if (Op.getValueType() == MVT::f32) 1049 LC = RTLIB::FPTOUINT_F32_I128; 1050 else if (Op.getValueType() == MVT::f64) 1051 LC = RTLIB::FPTOUINT_F64_I128; 1052 else if (Op.getValueType() == MVT::f80) 1053 LC = RTLIB::FPTOUINT_F80_I128; 1054 else if (Op.getValueType() == MVT::ppcf128) 1055 LC = RTLIB::FPTOUINT_PPCF128_I128; 1056 } else { 1057 assert(0 && "Unexpected fp-to-uint conversion!"); 1058 } 1059 SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*sign irrelevant*/), Lo, Hi); 1060} 1061 1062void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 1063 SDOperand &Lo, SDOperand &Hi) { 1064 if (ISD::isNormalLoad(N)) { 1065 ExpandRes_NormalLoad(N, Lo, Hi); 1066 return; 1067 } 1068 1069 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1070 1071 MVT VT = N->getValueType(0); 1072 MVT NVT = TLI.getTypeToTransformTo(VT); 1073 SDOperand Ch = N->getChain(); // Legalize the chain. 1074 SDOperand Ptr = N->getBasePtr(); // Legalize the pointer. 1075 ISD::LoadExtType ExtType = N->getExtensionType(); 1076 int SVOffset = N->getSrcValueOffset(); 1077 unsigned Alignment = N->getAlignment(); 1078 bool isVolatile = N->isVolatile(); 1079 1080 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1081 1082 if (N->getMemoryVT().bitsLE(NVT)) { 1083 MVT EVT = N->getMemoryVT(); 1084 1085 Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, EVT, 1086 isVolatile, Alignment); 1087 1088 // Remember the chain. 1089 Ch = Lo.getValue(1); 1090 1091 if (ExtType == ISD::SEXTLOAD) { 1092 // The high part is obtained by SRA'ing all but one of the bits of the 1093 // lo part. 1094 unsigned LoSize = Lo.getValueType().getSizeInBits(); 1095 Hi = DAG.getNode(ISD::SRA, NVT, Lo, 1096 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); 1097 } else if (ExtType == ISD::ZEXTLOAD) { 1098 // The high part is just a zero. 1099 Hi = DAG.getConstant(0, NVT); 1100 } else { 1101 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 1102 // The high part is undefined. 1103 Hi = DAG.getNode(ISD::UNDEF, NVT); 1104 } 1105 } else if (TLI.isLittleEndian()) { 1106 // Little-endian - low bits are at low addresses. 1107 Lo = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset, 1108 isVolatile, Alignment); 1109 1110 unsigned ExcessBits = 1111 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1112 MVT NEVT = MVT::getIntegerVT(ExcessBits); 1113 1114 // Increment the pointer to the other half. 1115 unsigned IncrementSize = NVT.getSizeInBits()/8; 1116 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 1117 DAG.getIntPtrConstant(IncrementSize)); 1118 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), 1119 SVOffset+IncrementSize, NEVT, 1120 isVolatile, MinAlign(Alignment, IncrementSize)); 1121 1122 // Build a factor node to remember that this load is independent of the 1123 // other one. 1124 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 1125 Hi.getValue(1)); 1126 } else { 1127 // Big-endian - high bits are at low addresses. Favor aligned loads at 1128 // the cost of some bit-fiddling. 1129 MVT EVT = N->getMemoryVT(); 1130 unsigned EBytes = EVT.getStoreSizeInBits()/8; 1131 unsigned IncrementSize = NVT.getSizeInBits()/8; 1132 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1133 1134 // Load both the high bits and maybe some of the low bits. 1135 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, 1136 MVT::getIntegerVT(EVT.getSizeInBits() - ExcessBits), 1137 isVolatile, Alignment); 1138 1139 // Increment the pointer to the other half. 1140 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 1141 DAG.getIntPtrConstant(IncrementSize)); 1142 // Load the rest of the low bits. 1143 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, NVT, Ch, Ptr, N->getSrcValue(), 1144 SVOffset+IncrementSize, 1145 MVT::getIntegerVT(ExcessBits), 1146 isVolatile, MinAlign(Alignment, IncrementSize)); 1147 1148 // Build a factor node to remember that this load is independent of the 1149 // other one. 1150 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 1151 Hi.getValue(1)); 1152 1153 if (ExcessBits < NVT.getSizeInBits()) { 1154 // Transfer low bits from the bottom of Hi to the top of Lo. 1155 Lo = DAG.getNode(ISD::OR, NVT, Lo, 1156 DAG.getNode(ISD::SHL, NVT, Hi, 1157 DAG.getConstant(ExcessBits, 1158 TLI.getShiftAmountTy()))); 1159 // Move high bits to the right position in Hi. 1160 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, NVT, Hi, 1161 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 1162 TLI.getShiftAmountTy())); 1163 } 1164 } 1165 1166 // Legalized the chain result - switch anything that used the old chain to 1167 // use the new one. 1168 ReplaceValueWith(SDOperand(N, 1), Ch); 1169} 1170 1171void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 1172 SDOperand &Lo, SDOperand &Hi) { 1173 SDOperand LL, LH, RL, RH; 1174 GetExpandedInteger(N->getOperand(0), LL, LH); 1175 GetExpandedInteger(N->getOperand(1), RL, RH); 1176 Lo = DAG.getNode(N->getOpcode(), LL.getValueType(), LL, RL); 1177 Hi = DAG.getNode(N->getOpcode(), LL.getValueType(), LH, RH); 1178} 1179 1180void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1181 SDOperand &Lo, SDOperand &Hi) { 1182 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1183 Lo = DAG.getNode(ISD::BSWAP, Lo.getValueType(), Lo); 1184 Hi = DAG.getNode(ISD::BSWAP, Hi.getValueType(), Hi); 1185} 1186 1187void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1188 SDOperand &Lo, SDOperand &Hi) { 1189 // Expand the subcomponents. 1190 SDOperand LHSL, LHSH, RHSL, RHSH; 1191 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1192 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1193 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1194 SDOperand LoOps[2] = { LHSL, RHSL }; 1195 SDOperand HiOps[3] = { LHSH, RHSH }; 1196 1197 if (N->getOpcode() == ISD::ADD) { 1198 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 1199 HiOps[2] = Lo.getValue(1); 1200 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 1201 } else { 1202 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); 1203 HiOps[2] = Lo.getValue(1); 1204 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); 1205 } 1206} 1207 1208void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1209 SDOperand &Lo, SDOperand &Hi) { 1210 // Expand the subcomponents. 1211 SDOperand LHSL, LHSH, RHSL, RHSH; 1212 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1213 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1214 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1215 SDOperand LoOps[2] = { LHSL, RHSL }; 1216 SDOperand HiOps[3] = { LHSH, RHSH }; 1217 1218 if (N->getOpcode() == ISD::ADDC) { 1219 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); 1220 HiOps[2] = Lo.getValue(1); 1221 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); 1222 } else { 1223 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); 1224 HiOps[2] = Lo.getValue(1); 1225 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); 1226 } 1227 1228 // Legalized the flag result - switch anything that used the old flag to 1229 // use the new one. 1230 ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1)); 1231} 1232 1233void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1234 SDOperand &Lo, SDOperand &Hi) { 1235 // Expand the subcomponents. 1236 SDOperand LHSL, LHSH, RHSL, RHSH; 1237 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1238 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1239 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); 1240 SDOperand LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1241 SDOperand HiOps[3] = { LHSH, RHSH }; 1242 1243 Lo = DAG.getNode(N->getOpcode(), VTList, LoOps, 3); 1244 HiOps[2] = Lo.getValue(1); 1245 Hi = DAG.getNode(N->getOpcode(), VTList, HiOps, 3); 1246 1247 // Legalized the flag result - switch anything that used the old flag to 1248 // use the new one. 1249 ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1)); 1250} 1251 1252void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 1253 SDOperand &Lo, SDOperand &Hi) { 1254 MVT VT = N->getValueType(0); 1255 MVT NVT = TLI.getTypeToTransformTo(VT); 1256 1257 bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT); 1258 bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT); 1259 bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT); 1260 bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT); 1261 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 1262 SDOperand LL, LH, RL, RH; 1263 GetExpandedInteger(N->getOperand(0), LL, LH); 1264 GetExpandedInteger(N->getOperand(1), RL, RH); 1265 unsigned OuterBitSize = VT.getSizeInBits(); 1266 unsigned InnerBitSize = NVT.getSizeInBits(); 1267 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0)); 1268 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1)); 1269 1270 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 1271 if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) && 1272 DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) { 1273 // The inputs are both zero-extended. 1274 if (HasUMUL_LOHI) { 1275 // We can emit a umul_lohi. 1276 Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); 1277 Hi = SDOperand(Lo.Val, 1); 1278 return; 1279 } 1280 if (HasMULHU) { 1281 // We can emit a mulhu+mul. 1282 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1283 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); 1284 return; 1285 } 1286 } 1287 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 1288 // The input values are both sign-extended. 1289 if (HasSMUL_LOHI) { 1290 // We can emit a smul_lohi. 1291 Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); 1292 Hi = SDOperand(Lo.Val, 1); 1293 return; 1294 } 1295 if (HasMULHS) { 1296 // We can emit a mulhs+mul. 1297 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1298 Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL); 1299 return; 1300 } 1301 } 1302 if (HasUMUL_LOHI) { 1303 // Lo,Hi = umul LHS, RHS. 1304 SDOperand UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, 1305 DAG.getVTList(NVT, NVT), LL, RL); 1306 Lo = UMulLOHI; 1307 Hi = UMulLOHI.getValue(1); 1308 RH = DAG.getNode(ISD::MUL, NVT, LL, RH); 1309 LH = DAG.getNode(ISD::MUL, NVT, LH, RL); 1310 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); 1311 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH); 1312 return; 1313 } 1314 if (HasMULHU) { 1315 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); 1316 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); 1317 RH = DAG.getNode(ISD::MUL, NVT, LL, RH); 1318 LH = DAG.getNode(ISD::MUL, NVT, LH, RL); 1319 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); 1320 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH); 1321 return; 1322 } 1323 } 1324 1325 // If nothing else, we can make a libcall. 1326 RTLIB::Libcall LC; 1327 switch (VT.getSimpleVT()) { 1328 default: 1329 assert(false && "Unsupported MUL!"); 1330 case MVT::i64: 1331 LC = RTLIB::MUL_I64; 1332 break; 1333 } 1334 1335 SDOperand Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1336 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*sign irrelevant*/), Lo, Hi); 1337} 1338 1339void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 1340 SDOperand &Lo, SDOperand &Hi) { 1341 assert(N->getValueType(0) == MVT::i64 && "Unsupported sdiv!"); 1342 SDOperand Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1343 SplitInteger(MakeLibCall(RTLIB::SDIV_I64, N->getValueType(0), Ops, 2, true), 1344 Lo, Hi); 1345} 1346 1347void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 1348 SDOperand &Lo, SDOperand &Hi) { 1349 assert(N->getValueType(0) == MVT::i64 && "Unsupported srem!"); 1350 SDOperand Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1351 SplitInteger(MakeLibCall(RTLIB::SREM_I64, N->getValueType(0), Ops, 2, true), 1352 Lo, Hi); 1353} 1354 1355void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 1356 SDOperand &Lo, SDOperand &Hi) { 1357 assert(N->getValueType(0) == MVT::i64 && "Unsupported udiv!"); 1358 SDOperand Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1359 SplitInteger(MakeLibCall(RTLIB::UDIV_I64, N->getValueType(0), Ops, 2, false), 1360 Lo, Hi); 1361} 1362 1363void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 1364 SDOperand &Lo, SDOperand &Hi) { 1365 assert(N->getValueType(0) == MVT::i64 && "Unsupported urem!"); 1366 SDOperand Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1367 SplitInteger(MakeLibCall(RTLIB::UREM_I64, N->getValueType(0), Ops, 2, false), 1368 Lo, Hi); 1369} 1370 1371void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 1372 SDOperand &Lo, SDOperand &Hi) { 1373 MVT VT = N->getValueType(0); 1374 1375 // If we can emit an efficient shift operation, do so now. Check to see if 1376 // the RHS is a constant. 1377 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 1378 return ExpandShiftByConstant(N, CN->getValue(), Lo, Hi); 1379 1380 // If we can determine that the high bit of the shift is zero or one, even if 1381 // the low bits are variable, emit this shift in an optimized form. 1382 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 1383 return; 1384 1385 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 1386 unsigned PartsOpc; 1387 if (N->getOpcode() == ISD::SHL) { 1388 PartsOpc = ISD::SHL_PARTS; 1389 } else if (N->getOpcode() == ISD::SRL) { 1390 PartsOpc = ISD::SRL_PARTS; 1391 } else { 1392 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1393 PartsOpc = ISD::SRA_PARTS; 1394 } 1395 1396 // Next check to see if the target supports this SHL_PARTS operation or if it 1397 // will custom expand it. 1398 MVT NVT = TLI.getTypeToTransformTo(VT); 1399 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 1400 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 1401 Action == TargetLowering::Custom) { 1402 // Expand the subcomponents. 1403 SDOperand LHSL, LHSH; 1404 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1405 1406 SDOperand Ops[] = { LHSL, LHSH, N->getOperand(1) }; 1407 MVT VT = LHSL.getValueType(); 1408 Lo = DAG.getNode(PartsOpc, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3); 1409 Hi = Lo.getValue(1); 1410 return; 1411 } 1412 1413 // Otherwise, emit a libcall. 1414 assert(VT == MVT::i64 && "Unsupported shift!"); 1415 1416 RTLIB::Libcall LC; 1417 bool isSigned; 1418 if (N->getOpcode() == ISD::SHL) { 1419 LC = RTLIB::SHL_I64; 1420 isSigned = false; /*sign irrelevant*/ 1421 } else if (N->getOpcode() == ISD::SRL) { 1422 LC = RTLIB::SRL_I64; 1423 isSigned = false; 1424 } else { 1425 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1426 LC = RTLIB::SRA_I64; 1427 isSigned = true; 1428 } 1429 1430 SDOperand Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1431 SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned), Lo, Hi); 1432} 1433 1434void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1435 SDOperand &Lo, SDOperand &Hi) { 1436 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1437 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1438 MVT NVT = Lo.getValueType(); 1439 1440 SDOperand HiNotZero = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi, 1441 DAG.getConstant(0, NVT), ISD::SETNE); 1442 1443 SDOperand LoLZ = DAG.getNode(ISD::CTLZ, NVT, Lo); 1444 SDOperand HiLZ = DAG.getNode(ISD::CTLZ, NVT, Hi); 1445 1446 Lo = DAG.getNode(ISD::SELECT, NVT, HiNotZero, HiLZ, 1447 DAG.getNode(ISD::ADD, NVT, LoLZ, 1448 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1449 Hi = DAG.getConstant(0, NVT); 1450} 1451 1452void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1453 SDOperand &Lo, SDOperand &Hi) { 1454 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1455 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1456 MVT NVT = Lo.getValueType(); 1457 Lo = DAG.getNode(ISD::ADD, NVT, DAG.getNode(ISD::CTPOP, NVT, Lo), 1458 DAG.getNode(ISD::CTPOP, NVT, Hi)); 1459 Hi = DAG.getConstant(0, NVT); 1460} 1461 1462void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1463 SDOperand &Lo, SDOperand &Hi) { 1464 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1465 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1466 MVT NVT = Lo.getValueType(); 1467 1468 SDOperand LoNotZero = DAG.getSetCC(TLI.getSetCCResultType(Lo), Lo, 1469 DAG.getConstant(0, NVT), ISD::SETNE); 1470 1471 SDOperand LoLZ = DAG.getNode(ISD::CTTZ, NVT, Lo); 1472 SDOperand HiLZ = DAG.getNode(ISD::CTTZ, NVT, Hi); 1473 1474 Lo = DAG.getNode(ISD::SELECT, NVT, LoNotZero, LoLZ, 1475 DAG.getNode(ISD::ADD, NVT, HiLZ, 1476 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1477 Hi = DAG.getConstant(0, NVT); 1478} 1479 1480/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded, 1481/// and the shift amount is a constant 'Amt'. Expand the operation. 1482void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 1483 SDOperand &Lo, SDOperand &Hi) { 1484 // Expand the incoming operand to be shifted, so that we have its parts 1485 SDOperand InL, InH; 1486 GetExpandedInteger(N->getOperand(0), InL, InH); 1487 1488 MVT NVT = InL.getValueType(); 1489 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1490 unsigned NVTBits = NVT.getSizeInBits(); 1491 MVT ShTy = N->getOperand(1).getValueType(); 1492 1493 if (N->getOpcode() == ISD::SHL) { 1494 if (Amt > VTBits) { 1495 Lo = Hi = DAG.getConstant(0, NVT); 1496 } else if (Amt > NVTBits) { 1497 Lo = DAG.getConstant(0, NVT); 1498 Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt-NVTBits,ShTy)); 1499 } else if (Amt == NVTBits) { 1500 Lo = DAG.getConstant(0, NVT); 1501 Hi = InL; 1502 } else { 1503 Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt, ShTy)); 1504 Hi = DAG.getNode(ISD::OR, NVT, 1505 DAG.getNode(ISD::SHL, NVT, InH, 1506 DAG.getConstant(Amt, ShTy)), 1507 DAG.getNode(ISD::SRL, NVT, InL, 1508 DAG.getConstant(NVTBits-Amt, ShTy))); 1509 } 1510 return; 1511 } 1512 1513 if (N->getOpcode() == ISD::SRL) { 1514 if (Amt > VTBits) { 1515 Lo = DAG.getConstant(0, NVT); 1516 Hi = DAG.getConstant(0, NVT); 1517 } else if (Amt > NVTBits) { 1518 Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy)); 1519 Hi = DAG.getConstant(0, NVT); 1520 } else if (Amt == NVTBits) { 1521 Lo = InH; 1522 Hi = DAG.getConstant(0, NVT); 1523 } else { 1524 Lo = DAG.getNode(ISD::OR, NVT, 1525 DAG.getNode(ISD::SRL, NVT, InL, 1526 DAG.getConstant(Amt, ShTy)), 1527 DAG.getNode(ISD::SHL, NVT, InH, 1528 DAG.getConstant(NVTBits-Amt, ShTy))); 1529 Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1530 } 1531 return; 1532 } 1533 1534 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1535 if (Amt > VTBits) { 1536 Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH, 1537 DAG.getConstant(NVTBits-1, ShTy)); 1538 } else if (Amt > NVTBits) { 1539 Lo = DAG.getNode(ISD::SRA, NVT, InH, 1540 DAG.getConstant(Amt-NVTBits, ShTy)); 1541 Hi = DAG.getNode(ISD::SRA, NVT, InH, 1542 DAG.getConstant(NVTBits-1, ShTy)); 1543 } else if (Amt == NVTBits) { 1544 Lo = InH; 1545 Hi = DAG.getNode(ISD::SRA, NVT, InH, 1546 DAG.getConstant(NVTBits-1, ShTy)); 1547 } else { 1548 Lo = DAG.getNode(ISD::OR, NVT, 1549 DAG.getNode(ISD::SRL, NVT, InL, 1550 DAG.getConstant(Amt, ShTy)), 1551 DAG.getNode(ISD::SHL, NVT, InH, 1552 DAG.getConstant(NVTBits-Amt, ShTy))); 1553 Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Amt, ShTy)); 1554 } 1555} 1556 1557/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1558/// this shift based on knowledge of the high bit of the shift amount. If we 1559/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1560/// shift amount. 1561bool DAGTypeLegalizer:: 1562ExpandShiftWithKnownAmountBit(SDNode *N, SDOperand &Lo, SDOperand &Hi) { 1563 SDOperand Amt = N->getOperand(1); 1564 MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1565 MVT ShTy = Amt.getValueType(); 1566 unsigned ShBits = ShTy.getSizeInBits(); 1567 unsigned NVTBits = NVT.getSizeInBits(); 1568 assert(isPowerOf2_32(NVTBits) && 1569 "Expanded integer type size not a power of two!"); 1570 1571 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1572 APInt KnownZero, KnownOne; 1573 DAG.ComputeMaskedBits(N->getOperand(1), HighBitMask, KnownZero, KnownOne); 1574 1575 // If we don't know anything about the high bits, exit. 1576 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1577 return false; 1578 1579 // Get the incoming operand to be shifted. 1580 SDOperand InL, InH; 1581 GetExpandedInteger(N->getOperand(0), InL, InH); 1582 1583 // If we know that any of the high bits of the shift amount are one, then we 1584 // can do this as a couple of simple shifts. 1585 if (KnownOne.intersects(HighBitMask)) { 1586 // Mask out the high bit, which we know is set. 1587 Amt = DAG.getNode(ISD::AND, ShTy, Amt, 1588 DAG.getConstant(~HighBitMask, ShTy)); 1589 1590 switch (N->getOpcode()) { 1591 default: assert(0 && "Unknown shift"); 1592 case ISD::SHL: 1593 Lo = DAG.getConstant(0, NVT); // Low part is zero. 1594 Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part. 1595 return true; 1596 case ISD::SRL: 1597 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 1598 Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part. 1599 return true; 1600 case ISD::SRA: 1601 Hi = DAG.getNode(ISD::SRA, NVT, InH, // Sign extend high part. 1602 DAG.getConstant(NVTBits-1, ShTy)); 1603 Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part. 1604 return true; 1605 } 1606 } 1607 1608 // If we know that all of the high bits of the shift amount are zero, then we 1609 // can do this as a couple of simple shifts. 1610 if ((KnownZero & HighBitMask) == HighBitMask) { 1611 // Compute 32-amt. 1612 SDOperand Amt2 = DAG.getNode(ISD::SUB, ShTy, 1613 DAG.getConstant(NVTBits, ShTy), 1614 Amt); 1615 unsigned Op1, Op2; 1616 switch (N->getOpcode()) { 1617 default: assert(0 && "Unknown shift"); 1618 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1619 case ISD::SRL: 1620 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1621 } 1622 1623 Lo = DAG.getNode(N->getOpcode(), NVT, InL, Amt); 1624 Hi = DAG.getNode(ISD::OR, NVT, 1625 DAG.getNode(Op1, NVT, InH, Amt), 1626 DAG.getNode(Op2, NVT, InL, Amt2)); 1627 return true; 1628 } 1629 1630 return false; 1631} 1632 1633 1634//===----------------------------------------------------------------------===// 1635// Integer Operand Expansion 1636//===----------------------------------------------------------------------===// 1637 1638/// ExpandIntegerOperand - This method is called when the specified operand of 1639/// the specified node is found to need expansion. At this point, all of the 1640/// result types of the node are known to be legal, but other operands of the 1641/// node may need promotion or expansion as well as the specified one. 1642bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 1643 DEBUG(cerr << "Expand integer operand: "; N->dump(&DAG); cerr << "\n"); 1644 SDOperand Res(0, 0); 1645 1646 if (TLI.getOperationAction(N->getOpcode(), N->getOperand(OpNo).getValueType()) 1647 == TargetLowering::Custom) 1648 Res = TLI.LowerOperation(SDOperand(N, OpNo), DAG); 1649 1650 if (Res.Val == 0) { 1651 switch (N->getOpcode()) { 1652 default: 1653 #ifndef NDEBUG 1654 cerr << "ExpandIntegerOperand Op #" << OpNo << ": "; 1655 N->dump(&DAG); cerr << "\n"; 1656 #endif 1657 assert(0 && "Do not know how to expand this operator's operand!"); 1658 abort(); 1659 1660 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1661 case ISD::BIT_CONVERT: Res = ExpandOp_BIT_CONVERT(N); break; 1662 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1663 1664 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 1665 1666 case ISD::SINT_TO_FP: 1667 Res = ExpandIntOp_SINT_TO_FP(N->getOperand(0), N->getValueType(0)); 1668 break; 1669 case ISD::UINT_TO_FP: 1670 Res = ExpandIntOp_UINT_TO_FP(N->getOperand(0), N->getValueType(0)); 1671 break; 1672 1673 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 1674 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 1675 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 1676 1677 case ISD::STORE: 1678 Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); 1679 break; 1680 } 1681 } 1682 1683 // If the result is null, the sub-method took care of registering results etc. 1684 if (!Res.Val) return false; 1685 // If the result is N, the sub-method updated N in place. Check to see if any 1686 // operands are new, and if so, mark them. 1687 if (Res.Val == N) { 1688 // Mark N as new and remark N and its operands. This allows us to correctly 1689 // revisit N if it needs another step of expansion and allows us to visit 1690 // any new operands to N. 1691 ReanalyzeNode(N); 1692 return true; 1693 } 1694 1695 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1696 "Invalid operand expansion"); 1697 1698 ReplaceValueWith(SDOperand(N, 0), Res); 1699 return false; 1700} 1701 1702SDOperand DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 1703 SDOperand InL, InH; 1704 GetExpandedInteger(N->getOperand(0), InL, InH); 1705 // Just truncate the low part of the source. 1706 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), InL); 1707} 1708 1709SDOperand DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDOperand Source, 1710 MVT DestTy) { 1711 // We know the destination is legal, but that the input needs to be expanded. 1712 MVT SourceVT = Source.getValueType(); 1713 1714 // Check to see if the target has a custom way to lower this. If so, use it. 1715 switch (TLI.getOperationAction(ISD::SINT_TO_FP, SourceVT)) { 1716 default: assert(0 && "This action not implemented for this operation!"); 1717 case TargetLowering::Legal: 1718 case TargetLowering::Expand: 1719 break; // This case is handled below. 1720 case TargetLowering::Custom: 1721 SDOperand NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy, 1722 Source), DAG); 1723 if (NV.Val) return NV; 1724 break; // The target lowered this. 1725 } 1726 1727 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1728 if (SourceVT == MVT::i64) { 1729 if (DestTy == MVT::f32) 1730 LC = RTLIB::SINTTOFP_I64_F32; 1731 else { 1732 assert(DestTy == MVT::f64 && "Unknown fp value type!"); 1733 LC = RTLIB::SINTTOFP_I64_F64; 1734 } 1735 } else if (SourceVT == MVT::i128) { 1736 if (DestTy == MVT::f32) 1737 LC = RTLIB::SINTTOFP_I128_F32; 1738 else if (DestTy == MVT::f64) 1739 LC = RTLIB::SINTTOFP_I128_F64; 1740 else if (DestTy == MVT::f80) 1741 LC = RTLIB::SINTTOFP_I128_F80; 1742 else { 1743 assert(DestTy == MVT::ppcf128 && "Unknown fp value type!"); 1744 LC = RTLIB::SINTTOFP_I128_PPCF128; 1745 } 1746 } else { 1747 assert(0 && "Unknown int value type!"); 1748 } 1749 1750 assert(LC != RTLIB::UNKNOWN_LIBCALL && 1751 "Don't know how to expand this SINT_TO_FP!"); 1752 return MakeLibCall(LC, DestTy, &Source, 1, true); 1753} 1754 1755SDOperand DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDOperand Source, 1756 MVT DestTy) { 1757 // We know the destination is legal, but that the input needs to be expanded. 1758 assert(getTypeAction(Source.getValueType()) == ExpandInteger && 1759 "This is not an expansion!"); 1760 1761 // If this is unsigned, and not supported, first perform the conversion to 1762 // signed, then adjust the result if the sign bit is set. 1763 SDOperand SignedConv = ExpandIntOp_SINT_TO_FP(Source, DestTy); 1764 1765 // The 64-bit value loaded will be incorrectly if the 'sign bit' of the 1766 // incoming integer is set. To handle this, we dynamically test to see if 1767 // it is set, and, if so, add a fudge factor. 1768 SDOperand Lo, Hi; 1769 GetExpandedInteger(Source, Lo, Hi); 1770 1771 SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi, 1772 DAG.getConstant(0, Hi.getValueType()), 1773 ISD::SETLT); 1774 SDOperand Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 1775 SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(), 1776 SignSet, Four, Zero); 1777 uint64_t FF = 0x5f800000ULL; 1778 if (TLI.isLittleEndian()) FF <<= 32; 1779 Constant *FudgeFactor = ConstantInt::get((Type*)Type::Int64Ty, FF); 1780 1781 SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 1782 CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset); 1783 SDOperand FudgeInReg; 1784 if (DestTy == MVT::f32) 1785 FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0); 1786 else if (DestTy.bitsGT(MVT::f32)) 1787 // FIXME: Avoid the extend by construction the right constantpool? 1788 FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(), 1789 CPIdx, NULL, 0, MVT::f32); 1790 else 1791 assert(0 && "Unexpected conversion"); 1792 1793 return DAG.getNode(ISD::FADD, DestTy, SignedConv, FudgeInReg); 1794} 1795 1796SDOperand DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 1797 SDOperand NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1798 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1799 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 1800 1801 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1802 // against zero to select between true and false values. 1803 if (NewRHS.Val == 0) { 1804 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 1805 CCCode = ISD::SETNE; 1806 } 1807 1808 // Update N to have the operands specified. 1809 return DAG.UpdateNodeOperands(SDOperand(N, 0), N->getOperand(0), 1810 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1811 N->getOperand(4)); 1812} 1813 1814SDOperand DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 1815 SDOperand NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1816 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1817 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 1818 1819 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1820 // against zero to select between true and false values. 1821 if (NewRHS.Val == 0) { 1822 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 1823 CCCode = ISD::SETNE; 1824 } 1825 1826 // Update N to have the operands specified. 1827 return DAG.UpdateNodeOperands(SDOperand(N, 0), NewLHS, NewRHS, 1828 N->getOperand(2), N->getOperand(3), 1829 DAG.getCondCode(CCCode)); 1830} 1831 1832SDOperand DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 1833 SDOperand NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1834 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 1835 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode); 1836 1837 // If ExpandSetCCOperands returned a scalar, use it. 1838 if (NewRHS.Val == 0) { 1839 assert(NewLHS.getValueType() == N->getValueType(0) && 1840 "Unexpected setcc expansion!"); 1841 return NewLHS; 1842 } 1843 1844 // Otherwise, update N to have the operands specified. 1845 return DAG.UpdateNodeOperands(SDOperand(N, 0), NewLHS, NewRHS, 1846 DAG.getCondCode(CCCode)); 1847} 1848 1849/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 1850/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1851void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDOperand &NewLHS, 1852 SDOperand &NewRHS, 1853 ISD::CondCode &CCCode) { 1854 SDOperand LHSLo, LHSHi, RHSLo, RHSHi; 1855 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 1856 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 1857 1858 MVT VT = NewLHS.getValueType(); 1859 1860 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 1861 if (RHSLo == RHSHi) { 1862 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 1863 if (RHSCST->isAllOnesValue()) { 1864 // Equality comparison to -1. 1865 NewLHS = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi); 1866 NewRHS = RHSLo; 1867 return; 1868 } 1869 } 1870 } 1871 1872 NewLHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo); 1873 NewRHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi); 1874 NewLHS = DAG.getNode(ISD::OR, NewLHS.getValueType(), NewLHS, NewRHS); 1875 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 1876 return; 1877 } 1878 1879 // If this is a comparison of the sign bit, just look at the top part. 1880 // X > -1, x < 0 1881 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 1882 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 1883 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 1884 NewLHS = LHSHi; 1885 NewRHS = RHSHi; 1886 return; 1887 } 1888 1889 // FIXME: This generated code sucks. 1890 ISD::CondCode LowCC; 1891 switch (CCCode) { 1892 default: assert(0 && "Unknown integer setcc!"); 1893 case ISD::SETLT: 1894 case ISD::SETULT: LowCC = ISD::SETULT; break; 1895 case ISD::SETGT: 1896 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 1897 case ISD::SETLE: 1898 case ISD::SETULE: LowCC = ISD::SETULE; break; 1899 case ISD::SETGE: 1900 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 1901 } 1902 1903 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 1904 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 1905 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 1906 1907 // NOTE: on targets without efficient SELECT of bools, we can always use 1908 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 1909 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL); 1910 SDOperand Tmp1, Tmp2; 1911 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC, 1912 false, DagCombineInfo); 1913 if (!Tmp1.Val) 1914 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC); 1915 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 1916 CCCode, false, DagCombineInfo); 1917 if (!Tmp2.Val) 1918 Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 1919 DAG.getCondCode(CCCode)); 1920 1921 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.Val); 1922 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.Val); 1923 if ((Tmp1C && Tmp1C->isNullValue()) || 1924 (Tmp2C && Tmp2C->isNullValue() && 1925 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 1926 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 1927 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 1928 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 1929 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 1930 // low part is known false, returns high part. 1931 // For LE / GE, if high part is known false, ignore the low part. 1932 // For LT / GT, if high part is known true, ignore the low part. 1933 NewLHS = Tmp2; 1934 NewRHS = SDOperand(); 1935 return; 1936 } 1937 1938 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 1939 ISD::SETEQ, false, DagCombineInfo); 1940 if (!NewLHS.Val) 1941 NewLHS = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, 1942 ISD::SETEQ); 1943 NewLHS = DAG.getNode(ISD::SELECT, Tmp1.getValueType(), 1944 NewLHS, Tmp1, Tmp2); 1945 NewRHS = SDOperand(); 1946} 1947 1948SDOperand DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 1949 if (ISD::isNormalStore(N)) 1950 return ExpandOp_NormalStore(N, OpNo); 1951 1952 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1953 assert(OpNo == 1 && "Can only expand the stored value so far"); 1954 1955 MVT VT = N->getOperand(1).getValueType(); 1956 MVT NVT = TLI.getTypeToTransformTo(VT); 1957 SDOperand Ch = N->getChain(); 1958 SDOperand Ptr = N->getBasePtr(); 1959 int SVOffset = N->getSrcValueOffset(); 1960 unsigned Alignment = N->getAlignment(); 1961 bool isVolatile = N->isVolatile(); 1962 SDOperand Lo, Hi; 1963 1964 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1965 1966 if (N->getMemoryVT().bitsLE(NVT)) { 1967 GetExpandedInteger(N->getValue(), Lo, Hi); 1968 return DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 1969 N->getMemoryVT(), isVolatile, Alignment); 1970 } else if (TLI.isLittleEndian()) { 1971 // Little-endian - low bits are at low addresses. 1972 GetExpandedInteger(N->getValue(), Lo, Hi); 1973 1974 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 1975 isVolatile, Alignment); 1976 1977 unsigned ExcessBits = 1978 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1979 MVT NEVT = MVT::getIntegerVT(ExcessBits); 1980 1981 // Increment the pointer to the other half. 1982 unsigned IncrementSize = NVT.getSizeInBits()/8; 1983 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 1984 DAG.getIntPtrConstant(IncrementSize)); 1985 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(), 1986 SVOffset+IncrementSize, NEVT, 1987 isVolatile, MinAlign(Alignment, IncrementSize)); 1988 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 1989 } else { 1990 // Big-endian - high bits are at low addresses. Favor aligned stores at 1991 // the cost of some bit-fiddling. 1992 GetExpandedInteger(N->getValue(), Lo, Hi); 1993 1994 MVT EVT = N->getMemoryVT(); 1995 unsigned EBytes = EVT.getStoreSizeInBits()/8; 1996 unsigned IncrementSize = NVT.getSizeInBits()/8; 1997 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1998 MVT HiVT = MVT::getIntegerVT(EVT.getSizeInBits() - ExcessBits); 1999 2000 if (ExcessBits < NVT.getSizeInBits()) { 2001 // Transfer high bits from the top of Lo to the bottom of Hi. 2002 Hi = DAG.getNode(ISD::SHL, NVT, Hi, 2003 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 2004 TLI.getShiftAmountTy())); 2005 Hi = DAG.getNode(ISD::OR, NVT, Hi, 2006 DAG.getNode(ISD::SRL, NVT, Lo, 2007 DAG.getConstant(ExcessBits, 2008 TLI.getShiftAmountTy()))); 2009 } 2010 2011 // Store both the high bits and maybe some of the low bits. 2012 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(), 2013 SVOffset, HiVT, isVolatile, Alignment); 2014 2015 // Increment the pointer to the other half. 2016 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 2017 DAG.getIntPtrConstant(IncrementSize)); 2018 // Store the lowest ExcessBits bits in the second half. 2019 Lo = DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), 2020 SVOffset+IncrementSize, 2021 MVT::getIntegerVT(ExcessBits), 2022 isVolatile, MinAlign(Alignment, IncrementSize)); 2023 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 2024 } 2025} 2026