SelectionDAGBuilder.cpp revision dce4a407a24b04eebc6a376f8e62b41aaa7b071f
1//===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This implements routines for translating from LLVM IR into SelectionDAG IR. 11// 12//===----------------------------------------------------------------------===// 13 14#include "SelectionDAGBuilder.h" 15#include "SDNodeDbgValue.h" 16#include "llvm/ADT/BitVector.h" 17#include "llvm/ADT/Optional.h" 18#include "llvm/ADT/SmallSet.h" 19#include "llvm/Analysis/AliasAnalysis.h" 20#include "llvm/Analysis/BranchProbabilityInfo.h" 21#include "llvm/Analysis/ConstantFolding.h" 22#include "llvm/Analysis/ValueTracking.h" 23#include "llvm/CodeGen/Analysis.h" 24#include "llvm/CodeGen/FastISel.h" 25#include "llvm/CodeGen/FunctionLoweringInfo.h" 26#include "llvm/CodeGen/GCMetadata.h" 27#include "llvm/CodeGen/GCStrategy.h" 28#include "llvm/CodeGen/MachineFrameInfo.h" 29#include "llvm/CodeGen/MachineFunction.h" 30#include "llvm/CodeGen/MachineInstrBuilder.h" 31#include "llvm/CodeGen/MachineJumpTableInfo.h" 32#include "llvm/CodeGen/MachineModuleInfo.h" 33#include "llvm/CodeGen/MachineRegisterInfo.h" 34#include "llvm/CodeGen/SelectionDAG.h" 35#include "llvm/CodeGen/StackMaps.h" 36#include "llvm/IR/CallingConv.h" 37#include "llvm/IR/Constants.h" 38#include "llvm/IR/DataLayout.h" 39#include "llvm/IR/DebugInfo.h" 40#include "llvm/IR/DerivedTypes.h" 41#include "llvm/IR/Function.h" 42#include "llvm/IR/GlobalVariable.h" 43#include "llvm/IR/InlineAsm.h" 44#include "llvm/IR/Instructions.h" 45#include "llvm/IR/IntrinsicInst.h" 46#include "llvm/IR/Intrinsics.h" 47#include "llvm/IR/LLVMContext.h" 48#include "llvm/IR/Module.h" 49#include "llvm/Support/CommandLine.h" 50#include "llvm/Support/Debug.h" 51#include "llvm/Support/ErrorHandling.h" 52#include "llvm/Support/MathExtras.h" 53#include "llvm/Support/raw_ostream.h" 54#include "llvm/Target/TargetFrameLowering.h" 55#include "llvm/Target/TargetInstrInfo.h" 56#include "llvm/Target/TargetIntrinsicInfo.h" 57#include "llvm/Target/TargetLibraryInfo.h" 58#include "llvm/Target/TargetLowering.h" 59#include "llvm/Target/TargetOptions.h" 60#include "llvm/Target/TargetSelectionDAGInfo.h" 61#include <algorithm> 62using namespace llvm; 63 64#define DEBUG_TYPE "isel" 65 66/// LimitFloatPrecision - Generate low-precision inline sequences for 67/// some float libcalls (6, 8 or 12 bits). 68static unsigned LimitFloatPrecision; 69 70static cl::opt<unsigned, true> 71LimitFPPrecision("limit-float-precision", 72 cl::desc("Generate low-precision inline sequences " 73 "for some float libcalls"), 74 cl::location(LimitFloatPrecision), 75 cl::init(0)); 76 77// Limit the width of DAG chains. This is important in general to prevent 78// prevent DAG-based analysis from blowing up. For example, alias analysis and 79// load clustering may not complete in reasonable time. It is difficult to 80// recognize and avoid this situation within each individual analysis, and 81// future analyses are likely to have the same behavior. Limiting DAG width is 82// the safe approach, and will be especially important with global DAGs. 83// 84// MaxParallelChains default is arbitrarily high to avoid affecting 85// optimization, but could be lowered to improve compile time. Any ld-ld-st-st 86// sequence over this should have been converted to llvm.memcpy by the 87// frontend. It easy to induce this behavior with .ll code such as: 88// %buffer = alloca [4096 x i8] 89// %data = load [4096 x i8]* %argPtr 90// store [4096 x i8] %data, [4096 x i8]* %buffer 91static const unsigned MaxParallelChains = 64; 92 93static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL, 94 const SDValue *Parts, unsigned NumParts, 95 MVT PartVT, EVT ValueVT, const Value *V); 96 97/// getCopyFromParts - Create a value that contains the specified legal parts 98/// combined into the value they represent. If the parts combine to a type 99/// larger then ValueVT then AssertOp can be used to specify whether the extra 100/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT 101/// (ISD::AssertSext). 102static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL, 103 const SDValue *Parts, 104 unsigned NumParts, MVT PartVT, EVT ValueVT, 105 const Value *V, 106 ISD::NodeType AssertOp = ISD::DELETED_NODE) { 107 if (ValueVT.isVector()) 108 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, 109 PartVT, ValueVT, V); 110 111 assert(NumParts > 0 && "No parts to assemble!"); 112 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 113 SDValue Val = Parts[0]; 114 115 if (NumParts > 1) { 116 // Assemble the value from multiple parts. 117 if (ValueVT.isInteger()) { 118 unsigned PartBits = PartVT.getSizeInBits(); 119 unsigned ValueBits = ValueVT.getSizeInBits(); 120 121 // Assemble the power of 2 part. 122 unsigned RoundParts = NumParts & (NumParts - 1) ? 123 1 << Log2_32(NumParts) : NumParts; 124 unsigned RoundBits = PartBits * RoundParts; 125 EVT RoundVT = RoundBits == ValueBits ? 126 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); 127 SDValue Lo, Hi; 128 129 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); 130 131 if (RoundParts > 2) { 132 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, 133 PartVT, HalfVT, V); 134 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, 135 RoundParts / 2, PartVT, HalfVT, V); 136 } else { 137 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); 138 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); 139 } 140 141 if (TLI.isBigEndian()) 142 std::swap(Lo, Hi); 143 144 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); 145 146 if (RoundParts < NumParts) { 147 // Assemble the trailing non-power-of-2 part. 148 unsigned OddParts = NumParts - RoundParts; 149 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); 150 Hi = getCopyFromParts(DAG, DL, 151 Parts + RoundParts, OddParts, PartVT, OddVT, V); 152 153 // Combine the round and odd parts. 154 Lo = Val; 155 if (TLI.isBigEndian()) 156 std::swap(Lo, Hi); 157 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 158 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); 159 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi, 160 DAG.getConstant(Lo.getValueType().getSizeInBits(), 161 TLI.getPointerTy())); 162 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); 163 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); 164 } 165 } else if (PartVT.isFloatingPoint()) { 166 // FP split into multiple FP parts (for ppcf128) 167 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && 168 "Unexpected split"); 169 SDValue Lo, Hi; 170 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); 171 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); 172 if (TLI.isBigEndian()) 173 std::swap(Lo, Hi); 174 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); 175 } else { 176 // FP split into integer parts (soft fp) 177 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() && 178 !PartVT.isVector() && "Unexpected split"); 179 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); 180 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V); 181 } 182 } 183 184 // There is now one part, held in Val. Correct it to match ValueVT. 185 EVT PartEVT = Val.getValueType(); 186 187 if (PartEVT == ValueVT) 188 return Val; 189 190 if (PartEVT.isInteger() && ValueVT.isInteger()) { 191 if (ValueVT.bitsLT(PartEVT)) { 192 // For a truncate, see if we have any information to 193 // indicate whether the truncated bits will always be 194 // zero or sign-extension. 195 if (AssertOp != ISD::DELETED_NODE) 196 Val = DAG.getNode(AssertOp, DL, PartEVT, Val, 197 DAG.getValueType(ValueVT)); 198 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 199 } 200 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); 201 } 202 203 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { 204 // FP_ROUND's are always exact here. 205 if (ValueVT.bitsLT(Val.getValueType())) 206 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, 207 DAG.getTargetConstant(1, TLI.getPointerTy())); 208 209 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); 210 } 211 212 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) 213 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 214 215 llvm_unreachable("Unknown mismatch!"); 216} 217 218static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, 219 const Twine &ErrMsg) { 220 const Instruction *I = dyn_cast_or_null<Instruction>(V); 221 if (!V) 222 return Ctx.emitError(ErrMsg); 223 224 const char *AsmError = ", possible invalid constraint for vector type"; 225 if (const CallInst *CI = dyn_cast<CallInst>(I)) 226 if (isa<InlineAsm>(CI->getCalledValue())) 227 return Ctx.emitError(I, ErrMsg + AsmError); 228 229 return Ctx.emitError(I, ErrMsg); 230} 231 232/// getCopyFromPartsVector - Create a value that contains the specified legal 233/// parts combined into the value they represent. If the parts combine to a 234/// type larger then ValueVT then AssertOp can be used to specify whether the 235/// extra bits are known to be zero (ISD::AssertZext) or sign extended from 236/// ValueVT (ISD::AssertSext). 237static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL, 238 const SDValue *Parts, unsigned NumParts, 239 MVT PartVT, EVT ValueVT, const Value *V) { 240 assert(ValueVT.isVector() && "Not a vector value"); 241 assert(NumParts > 0 && "No parts to assemble!"); 242 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 243 SDValue Val = Parts[0]; 244 245 // Handle a multi-element vector. 246 if (NumParts > 1) { 247 EVT IntermediateVT; 248 MVT RegisterVT; 249 unsigned NumIntermediates; 250 unsigned NumRegs = 251 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, 252 NumIntermediates, RegisterVT); 253 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); 254 NumParts = NumRegs; // Silence a compiler warning. 255 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); 256 assert(RegisterVT == Parts[0].getSimpleValueType() && 257 "Part type doesn't match part!"); 258 259 // Assemble the parts into intermediate operands. 260 SmallVector<SDValue, 8> Ops(NumIntermediates); 261 if (NumIntermediates == NumParts) { 262 // If the register was not expanded, truncate or copy the value, 263 // as appropriate. 264 for (unsigned i = 0; i != NumParts; ++i) 265 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, 266 PartVT, IntermediateVT, V); 267 } else if (NumParts > 0) { 268 // If the intermediate type was expanded, build the intermediate 269 // operands from the parts. 270 assert(NumParts % NumIntermediates == 0 && 271 "Must expand into a divisible number of parts!"); 272 unsigned Factor = NumParts / NumIntermediates; 273 for (unsigned i = 0; i != NumIntermediates; ++i) 274 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, 275 PartVT, IntermediateVT, V); 276 } 277 278 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the 279 // intermediate operands. 280 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS 281 : ISD::BUILD_VECTOR, 282 DL, ValueVT, Ops); 283 } 284 285 // There is now one part, held in Val. Correct it to match ValueVT. 286 EVT PartEVT = Val.getValueType(); 287 288 if (PartEVT == ValueVT) 289 return Val; 290 291 if (PartEVT.isVector()) { 292 // If the element type of the source/dest vectors are the same, but the 293 // parts vector has more elements than the value vector, then we have a 294 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the 295 // elements we want. 296 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) { 297 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && 298 "Cannot narrow, it would be a lossy transformation"); 299 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, 300 DAG.getConstant(0, TLI.getVectorIdxTy())); 301 } 302 303 // Vector/Vector bitcast. 304 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) 305 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 306 307 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && 308 "Cannot handle this kind of promotion"); 309 // Promoted vector extract 310 bool Smaller = ValueVT.bitsLE(PartEVT); 311 return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 312 DL, ValueVT, Val); 313 314 } 315 316 // Trivial bitcast if the types are the same size and the destination 317 // vector type is legal. 318 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && 319 TLI.isTypeLegal(ValueVT)) 320 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 321 322 // Handle cases such as i8 -> <1 x i1> 323 if (ValueVT.getVectorNumElements() != 1) { 324 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V, 325 "non-trivial scalar-to-vector conversion"); 326 return DAG.getUNDEF(ValueVT); 327 } 328 329 if (ValueVT.getVectorNumElements() == 1 && 330 ValueVT.getVectorElementType() != PartEVT) { 331 bool Smaller = ValueVT.bitsLE(PartEVT); 332 Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 333 DL, ValueVT.getScalarType(), Val); 334 } 335 336 return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val); 337} 338 339static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl, 340 SDValue Val, SDValue *Parts, unsigned NumParts, 341 MVT PartVT, const Value *V); 342 343/// getCopyToParts - Create a series of nodes that contain the specified value 344/// split into legal parts. If the parts contain more bits than Val, then, for 345/// integers, ExtendKind can be used to specify how to generate the extra bits. 346static void getCopyToParts(SelectionDAG &DAG, SDLoc DL, 347 SDValue Val, SDValue *Parts, unsigned NumParts, 348 MVT PartVT, const Value *V, 349 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { 350 EVT ValueVT = Val.getValueType(); 351 352 // Handle the vector case separately. 353 if (ValueVT.isVector()) 354 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V); 355 356 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 357 unsigned PartBits = PartVT.getSizeInBits(); 358 unsigned OrigNumParts = NumParts; 359 assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!"); 360 361 if (NumParts == 0) 362 return; 363 364 assert(!ValueVT.isVector() && "Vector case handled elsewhere"); 365 EVT PartEVT = PartVT; 366 if (PartEVT == ValueVT) { 367 assert(NumParts == 1 && "No-op copy with multiple parts!"); 368 Parts[0] = Val; 369 return; 370 } 371 372 if (NumParts * PartBits > ValueVT.getSizeInBits()) { 373 // If the parts cover more bits than the value has, promote the value. 374 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { 375 assert(NumParts == 1 && "Do not know what to promote to!"); 376 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); 377 } else { 378 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && 379 ValueVT.isInteger() && 380 "Unknown mismatch!"); 381 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 382 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); 383 if (PartVT == MVT::x86mmx) 384 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 385 } 386 } else if (PartBits == ValueVT.getSizeInBits()) { 387 // Different types of the same size. 388 assert(NumParts == 1 && PartEVT != ValueVT); 389 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 390 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { 391 // If the parts cover less bits than value has, truncate the value. 392 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && 393 ValueVT.isInteger() && 394 "Unknown mismatch!"); 395 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 396 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 397 if (PartVT == MVT::x86mmx) 398 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 399 } 400 401 // The value may have changed - recompute ValueVT. 402 ValueVT = Val.getValueType(); 403 assert(NumParts * PartBits == ValueVT.getSizeInBits() && 404 "Failed to tile the value with PartVT!"); 405 406 if (NumParts == 1) { 407 if (PartEVT != ValueVT) 408 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V, 409 "scalar-to-vector conversion failed"); 410 411 Parts[0] = Val; 412 return; 413 } 414 415 // Expand the value into multiple parts. 416 if (NumParts & (NumParts - 1)) { 417 // The number of parts is not a power of 2. Split off and copy the tail. 418 assert(PartVT.isInteger() && ValueVT.isInteger() && 419 "Do not know what to expand to!"); 420 unsigned RoundParts = 1 << Log2_32(NumParts); 421 unsigned RoundBits = RoundParts * PartBits; 422 unsigned OddParts = NumParts - RoundParts; 423 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, 424 DAG.getIntPtrConstant(RoundBits)); 425 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V); 426 427 if (TLI.isBigEndian()) 428 // The odd parts were reversed by getCopyToParts - unreverse them. 429 std::reverse(Parts + RoundParts, Parts + NumParts); 430 431 NumParts = RoundParts; 432 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 433 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 434 } 435 436 // The number of parts is a power of 2. Repeatedly bisect the value using 437 // EXTRACT_ELEMENT. 438 Parts[0] = DAG.getNode(ISD::BITCAST, DL, 439 EVT::getIntegerVT(*DAG.getContext(), 440 ValueVT.getSizeInBits()), 441 Val); 442 443 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { 444 for (unsigned i = 0; i < NumParts; i += StepSize) { 445 unsigned ThisBits = StepSize * PartBits / 2; 446 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); 447 SDValue &Part0 = Parts[i]; 448 SDValue &Part1 = Parts[i+StepSize/2]; 449 450 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, 451 ThisVT, Part0, DAG.getIntPtrConstant(1)); 452 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, 453 ThisVT, Part0, DAG.getIntPtrConstant(0)); 454 455 if (ThisBits == PartBits && ThisVT != PartVT) { 456 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); 457 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); 458 } 459 } 460 } 461 462 if (TLI.isBigEndian()) 463 std::reverse(Parts, Parts + OrigNumParts); 464} 465 466 467/// getCopyToPartsVector - Create a series of nodes that contain the specified 468/// value split into legal parts. 469static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL, 470 SDValue Val, SDValue *Parts, unsigned NumParts, 471 MVT PartVT, const Value *V) { 472 EVT ValueVT = Val.getValueType(); 473 assert(ValueVT.isVector() && "Not a vector"); 474 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 475 476 if (NumParts == 1) { 477 EVT PartEVT = PartVT; 478 if (PartEVT == ValueVT) { 479 // Nothing to do. 480 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { 481 // Bitconvert vector->vector case. 482 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 483 } else if (PartVT.isVector() && 484 PartEVT.getVectorElementType() == ValueVT.getVectorElementType() && 485 PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) { 486 EVT ElementVT = PartVT.getVectorElementType(); 487 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in 488 // undef elements. 489 SmallVector<SDValue, 16> Ops; 490 for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i) 491 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 492 ElementVT, Val, DAG.getConstant(i, 493 TLI.getVectorIdxTy()))); 494 495 for (unsigned i = ValueVT.getVectorNumElements(), 496 e = PartVT.getVectorNumElements(); i != e; ++i) 497 Ops.push_back(DAG.getUNDEF(ElementVT)); 498 499 Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops); 500 501 // FIXME: Use CONCAT for 2x -> 4x. 502 503 //SDValue UndefElts = DAG.getUNDEF(VectorTy); 504 //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts); 505 } else if (PartVT.isVector() && 506 PartEVT.getVectorElementType().bitsGE( 507 ValueVT.getVectorElementType()) && 508 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) { 509 510 // Promoted vector extract 511 bool Smaller = PartEVT.bitsLE(ValueVT); 512 Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 513 DL, PartVT, Val); 514 } else{ 515 // Vector -> scalar conversion. 516 assert(ValueVT.getVectorNumElements() == 1 && 517 "Only trivial vector-to-scalar conversions should get here!"); 518 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 519 PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy())); 520 521 bool Smaller = ValueVT.bitsLE(PartVT); 522 Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 523 DL, PartVT, Val); 524 } 525 526 Parts[0] = Val; 527 return; 528 } 529 530 // Handle a multi-element vector. 531 EVT IntermediateVT; 532 MVT RegisterVT; 533 unsigned NumIntermediates; 534 unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, 535 IntermediateVT, 536 NumIntermediates, RegisterVT); 537 unsigned NumElements = ValueVT.getVectorNumElements(); 538 539 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); 540 NumParts = NumRegs; // Silence a compiler warning. 541 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); 542 543 // Split the vector into intermediate operands. 544 SmallVector<SDValue, 8> Ops(NumIntermediates); 545 for (unsigned i = 0; i != NumIntermediates; ++i) { 546 if (IntermediateVT.isVector()) 547 Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, 548 IntermediateVT, Val, 549 DAG.getConstant(i * (NumElements / NumIntermediates), 550 TLI.getVectorIdxTy())); 551 else 552 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 553 IntermediateVT, Val, 554 DAG.getConstant(i, TLI.getVectorIdxTy())); 555 } 556 557 // Split the intermediate operands into legal parts. 558 if (NumParts == NumIntermediates) { 559 // If the register was not expanded, promote or copy the value, 560 // as appropriate. 561 for (unsigned i = 0; i != NumParts; ++i) 562 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V); 563 } else if (NumParts > 0) { 564 // If the intermediate type was expanded, split each the value into 565 // legal parts. 566 assert(NumParts % NumIntermediates == 0 && 567 "Must expand into a divisible number of parts!"); 568 unsigned Factor = NumParts / NumIntermediates; 569 for (unsigned i = 0; i != NumIntermediates; ++i) 570 getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V); 571 } 572} 573 574namespace { 575 /// RegsForValue - This struct represents the registers (physical or virtual) 576 /// that a particular set of values is assigned, and the type information 577 /// about the value. The most common situation is to represent one value at a 578 /// time, but struct or array values are handled element-wise as multiple 579 /// values. The splitting of aggregates is performed recursively, so that we 580 /// never have aggregate-typed registers. The values at this point do not 581 /// necessarily have legal types, so each value may require one or more 582 /// registers of some legal type. 583 /// 584 struct RegsForValue { 585 /// ValueVTs - The value types of the values, which may not be legal, and 586 /// may need be promoted or synthesized from one or more registers. 587 /// 588 SmallVector<EVT, 4> ValueVTs; 589 590 /// RegVTs - The value types of the registers. This is the same size as 591 /// ValueVTs and it records, for each value, what the type of the assigned 592 /// register or registers are. (Individual values are never synthesized 593 /// from more than one type of register.) 594 /// 595 /// With virtual registers, the contents of RegVTs is redundant with TLI's 596 /// getRegisterType member function, however when with physical registers 597 /// it is necessary to have a separate record of the types. 598 /// 599 SmallVector<MVT, 4> RegVTs; 600 601 /// Regs - This list holds the registers assigned to the values. 602 /// Each legal or promoted value requires one register, and each 603 /// expanded value requires multiple registers. 604 /// 605 SmallVector<unsigned, 4> Regs; 606 607 RegsForValue() {} 608 609 RegsForValue(const SmallVector<unsigned, 4> ®s, 610 MVT regvt, EVT valuevt) 611 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {} 612 613 RegsForValue(LLVMContext &Context, const TargetLowering &tli, 614 unsigned Reg, Type *Ty) { 615 ComputeValueVTs(tli, Ty, ValueVTs); 616 617 for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { 618 EVT ValueVT = ValueVTs[Value]; 619 unsigned NumRegs = tli.getNumRegisters(Context, ValueVT); 620 MVT RegisterVT = tli.getRegisterType(Context, ValueVT); 621 for (unsigned i = 0; i != NumRegs; ++i) 622 Regs.push_back(Reg + i); 623 RegVTs.push_back(RegisterVT); 624 Reg += NumRegs; 625 } 626 } 627 628 /// append - Add the specified values to this one. 629 void append(const RegsForValue &RHS) { 630 ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end()); 631 RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end()); 632 Regs.append(RHS.Regs.begin(), RHS.Regs.end()); 633 } 634 635 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from 636 /// this value and returns the result as a ValueVTs value. This uses 637 /// Chain/Flag as the input and updates them for the output Chain/Flag. 638 /// If the Flag pointer is NULL, no flag is used. 639 SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, 640 SDLoc dl, 641 SDValue &Chain, SDValue *Flag, 642 const Value *V = nullptr) const; 643 644 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the 645 /// specified value into the registers specified by this object. This uses 646 /// Chain/Flag as the input and updates them for the output Chain/Flag. 647 /// If the Flag pointer is NULL, no flag is used. 648 void getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, 649 SDValue &Chain, SDValue *Flag, const Value *V) const; 650 651 /// AddInlineAsmOperands - Add this value to the specified inlineasm node 652 /// operand list. This adds the code marker, matching input operand index 653 /// (if applicable), and includes the number of values added into it. 654 void AddInlineAsmOperands(unsigned Kind, 655 bool HasMatching, unsigned MatchingIdx, 656 SelectionDAG &DAG, 657 std::vector<SDValue> &Ops) const; 658 }; 659} 660 661/// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from 662/// this value and returns the result as a ValueVT value. This uses 663/// Chain/Flag as the input and updates them for the output Chain/Flag. 664/// If the Flag pointer is NULL, no flag is used. 665SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, 666 FunctionLoweringInfo &FuncInfo, 667 SDLoc dl, 668 SDValue &Chain, SDValue *Flag, 669 const Value *V) const { 670 // A Value with type {} or [0 x %t] needs no registers. 671 if (ValueVTs.empty()) 672 return SDValue(); 673 674 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 675 676 // Assemble the legal parts into the final values. 677 SmallVector<SDValue, 4> Values(ValueVTs.size()); 678 SmallVector<SDValue, 8> Parts; 679 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { 680 // Copy the legal parts from the registers. 681 EVT ValueVT = ValueVTs[Value]; 682 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT); 683 MVT RegisterVT = RegVTs[Value]; 684 685 Parts.resize(NumRegs); 686 for (unsigned i = 0; i != NumRegs; ++i) { 687 SDValue P; 688 if (!Flag) { 689 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); 690 } else { 691 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); 692 *Flag = P.getValue(2); 693 } 694 695 Chain = P.getValue(1); 696 Parts[i] = P; 697 698 // If the source register was virtual and if we know something about it, 699 // add an assert node. 700 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) || 701 !RegisterVT.isInteger() || RegisterVT.isVector()) 702 continue; 703 704 const FunctionLoweringInfo::LiveOutInfo *LOI = 705 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); 706 if (!LOI) 707 continue; 708 709 unsigned RegSize = RegisterVT.getSizeInBits(); 710 unsigned NumSignBits = LOI->NumSignBits; 711 unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes(); 712 713 if (NumZeroBits == RegSize) { 714 // The current value is a zero. 715 // Explicitly express that as it would be easier for 716 // optimizations to kick in. 717 Parts[i] = DAG.getConstant(0, RegisterVT); 718 continue; 719 } 720 721 // FIXME: We capture more information than the dag can represent. For 722 // now, just use the tightest assertzext/assertsext possible. 723 bool isSExt = true; 724 EVT FromVT(MVT::Other); 725 if (NumSignBits == RegSize) 726 isSExt = true, FromVT = MVT::i1; // ASSERT SEXT 1 727 else if (NumZeroBits >= RegSize-1) 728 isSExt = false, FromVT = MVT::i1; // ASSERT ZEXT 1 729 else if (NumSignBits > RegSize-8) 730 isSExt = true, FromVT = MVT::i8; // ASSERT SEXT 8 731 else if (NumZeroBits >= RegSize-8) 732 isSExt = false, FromVT = MVT::i8; // ASSERT ZEXT 8 733 else if (NumSignBits > RegSize-16) 734 isSExt = true, FromVT = MVT::i16; // ASSERT SEXT 16 735 else if (NumZeroBits >= RegSize-16) 736 isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16 737 else if (NumSignBits > RegSize-32) 738 isSExt = true, FromVT = MVT::i32; // ASSERT SEXT 32 739 else if (NumZeroBits >= RegSize-32) 740 isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32 741 else 742 continue; 743 744 // Add an assertion node. 745 assert(FromVT != MVT::Other); 746 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, 747 RegisterVT, P, DAG.getValueType(FromVT)); 748 } 749 750 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), 751 NumRegs, RegisterVT, ValueVT, V); 752 Part += NumRegs; 753 Parts.clear(); 754 } 755 756 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values); 757} 758 759/// getCopyToRegs - Emit a series of CopyToReg nodes that copies the 760/// specified value into the registers specified by this object. This uses 761/// Chain/Flag as the input and updates them for the output Chain/Flag. 762/// If the Flag pointer is NULL, no flag is used. 763void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, 764 SDValue &Chain, SDValue *Flag, 765 const Value *V) const { 766 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 767 768 // Get the list of the values's legal parts. 769 unsigned NumRegs = Regs.size(); 770 SmallVector<SDValue, 8> Parts(NumRegs); 771 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { 772 EVT ValueVT = ValueVTs[Value]; 773 unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT); 774 MVT RegisterVT = RegVTs[Value]; 775 ISD::NodeType ExtendKind = 776 TLI.isZExtFree(Val, RegisterVT)? ISD::ZERO_EXTEND: ISD::ANY_EXTEND; 777 778 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), 779 &Parts[Part], NumParts, RegisterVT, V, ExtendKind); 780 Part += NumParts; 781 } 782 783 // Copy the parts into the registers. 784 SmallVector<SDValue, 8> Chains(NumRegs); 785 for (unsigned i = 0; i != NumRegs; ++i) { 786 SDValue Part; 787 if (!Flag) { 788 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); 789 } else { 790 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); 791 *Flag = Part.getValue(1); 792 } 793 794 Chains[i] = Part.getValue(0); 795 } 796 797 if (NumRegs == 1 || Flag) 798 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is 799 // flagged to it. That is the CopyToReg nodes and the user are considered 800 // a single scheduling unit. If we create a TokenFactor and return it as 801 // chain, then the TokenFactor is both a predecessor (operand) of the 802 // user as well as a successor (the TF operands are flagged to the user). 803 // c1, f1 = CopyToReg 804 // c2, f2 = CopyToReg 805 // c3 = TokenFactor c1, c2 806 // ... 807 // = op c3, ..., f2 808 Chain = Chains[NumRegs-1]; 809 else 810 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 811} 812 813/// AddInlineAsmOperands - Add this value to the specified inlineasm node 814/// operand list. This adds the code marker and includes the number of 815/// values added into it. 816void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, 817 unsigned MatchingIdx, 818 SelectionDAG &DAG, 819 std::vector<SDValue> &Ops) const { 820 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 821 822 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); 823 if (HasMatching) 824 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); 825 else if (!Regs.empty() && 826 TargetRegisterInfo::isVirtualRegister(Regs.front())) { 827 // Put the register class of the virtual registers in the flag word. That 828 // way, later passes can recompute register class constraints for inline 829 // assembly as well as normal instructions. 830 // Don't do this for tied operands that can use the regclass information 831 // from the def. 832 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 833 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); 834 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); 835 } 836 837 SDValue Res = DAG.getTargetConstant(Flag, MVT::i32); 838 Ops.push_back(Res); 839 840 unsigned SP = TLI.getStackPointerRegisterToSaveRestore(); 841 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { 842 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]); 843 MVT RegisterVT = RegVTs[Value]; 844 for (unsigned i = 0; i != NumRegs; ++i) { 845 assert(Reg < Regs.size() && "Mismatch in # registers expected"); 846 unsigned TheReg = Regs[Reg++]; 847 Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); 848 849 if (TheReg == SP && Code == InlineAsm::Kind_Clobber) { 850 // If we clobbered the stack pointer, MFI should know about it. 851 assert(DAG.getMachineFunction().getFrameInfo()-> 852 hasInlineAsmWithSPAdjust()); 853 } 854 } 855 } 856} 857 858void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa, 859 const TargetLibraryInfo *li) { 860 AA = &aa; 861 GFI = gfi; 862 LibInfo = li; 863 DL = DAG.getTarget().getDataLayout(); 864 Context = DAG.getContext(); 865 LPadToCallSiteMap.clear(); 866} 867 868/// clear - Clear out the current SelectionDAG and the associated 869/// state and prepare this SelectionDAGBuilder object to be used 870/// for a new block. This doesn't clear out information about 871/// additional blocks that are needed to complete switch lowering 872/// or PHI node updating; that information is cleared out as it is 873/// consumed. 874void SelectionDAGBuilder::clear() { 875 NodeMap.clear(); 876 UnusedArgNodeMap.clear(); 877 PendingLoads.clear(); 878 PendingExports.clear(); 879 CurInst = nullptr; 880 HasTailCall = false; 881 SDNodeOrder = LowestSDNodeOrder; 882} 883 884/// clearDanglingDebugInfo - Clear the dangling debug information 885/// map. This function is separated from the clear so that debug 886/// information that is dangling in a basic block can be properly 887/// resolved in a different basic block. This allows the 888/// SelectionDAG to resolve dangling debug information attached 889/// to PHI nodes. 890void SelectionDAGBuilder::clearDanglingDebugInfo() { 891 DanglingDebugInfoMap.clear(); 892} 893 894/// getRoot - Return the current virtual root of the Selection DAG, 895/// flushing any PendingLoad items. This must be done before emitting 896/// a store or any other node that may need to be ordered after any 897/// prior load instructions. 898/// 899SDValue SelectionDAGBuilder::getRoot() { 900 if (PendingLoads.empty()) 901 return DAG.getRoot(); 902 903 if (PendingLoads.size() == 1) { 904 SDValue Root = PendingLoads[0]; 905 DAG.setRoot(Root); 906 PendingLoads.clear(); 907 return Root; 908 } 909 910 // Otherwise, we have to make a token factor node. 911 SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 912 PendingLoads); 913 PendingLoads.clear(); 914 DAG.setRoot(Root); 915 return Root; 916} 917 918/// getControlRoot - Similar to getRoot, but instead of flushing all the 919/// PendingLoad items, flush all the PendingExports items. It is necessary 920/// to do this before emitting a terminator instruction. 921/// 922SDValue SelectionDAGBuilder::getControlRoot() { 923 SDValue Root = DAG.getRoot(); 924 925 if (PendingExports.empty()) 926 return Root; 927 928 // Turn all of the CopyToReg chains into one factored node. 929 if (Root.getOpcode() != ISD::EntryToken) { 930 unsigned i = 0, e = PendingExports.size(); 931 for (; i != e; ++i) { 932 assert(PendingExports[i].getNode()->getNumOperands() > 1); 933 if (PendingExports[i].getNode()->getOperand(0) == Root) 934 break; // Don't add the root if we already indirectly depend on it. 935 } 936 937 if (i == e) 938 PendingExports.push_back(Root); 939 } 940 941 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 942 PendingExports); 943 PendingExports.clear(); 944 DAG.setRoot(Root); 945 return Root; 946} 947 948void SelectionDAGBuilder::visit(const Instruction &I) { 949 // Set up outgoing PHI node register values before emitting the terminator. 950 if (isa<TerminatorInst>(&I)) 951 HandlePHINodesInSuccessorBlocks(I.getParent()); 952 953 ++SDNodeOrder; 954 955 CurInst = &I; 956 957 visit(I.getOpcode(), I); 958 959 if (!isa<TerminatorInst>(&I) && !HasTailCall) 960 CopyToExportRegsIfNeeded(&I); 961 962 CurInst = nullptr; 963} 964 965void SelectionDAGBuilder::visitPHI(const PHINode &) { 966 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!"); 967} 968 969void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { 970 // Note: this doesn't use InstVisitor, because it has to work with 971 // ConstantExpr's in addition to instructions. 972 switch (Opcode) { 973 default: llvm_unreachable("Unknown instruction type encountered!"); 974 // Build the switch statement using the Instruction.def file. 975#define HANDLE_INST(NUM, OPCODE, CLASS) \ 976 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; 977#include "llvm/IR/Instruction.def" 978 } 979} 980 981// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, 982// generate the debug data structures now that we've seen its definition. 983void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, 984 SDValue Val) { 985 DanglingDebugInfo &DDI = DanglingDebugInfoMap[V]; 986 if (DDI.getDI()) { 987 const DbgValueInst *DI = DDI.getDI(); 988 DebugLoc dl = DDI.getdl(); 989 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); 990 MDNode *Variable = DI->getVariable(); 991 uint64_t Offset = DI->getOffset(); 992 // A dbg.value for an alloca is always indirect. 993 bool IsIndirect = isa<AllocaInst>(V) || Offset != 0; 994 SDDbgValue *SDV; 995 if (Val.getNode()) { 996 if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, Val)) { 997 SDV = DAG.getDbgValue(Variable, Val.getNode(), 998 Val.getResNo(), IsIndirect, 999 Offset, dl, DbgSDNodeOrder); 1000 DAG.AddDbgValue(SDV, Val.getNode(), false); 1001 } 1002 } else 1003 DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); 1004 DanglingDebugInfoMap[V] = DanglingDebugInfo(); 1005 } 1006} 1007 1008/// getValue - Return an SDValue for the given Value. 1009SDValue SelectionDAGBuilder::getValue(const Value *V) { 1010 // If we already have an SDValue for this value, use it. It's important 1011 // to do this first, so that we don't create a CopyFromReg if we already 1012 // have a regular SDValue. 1013 SDValue &N = NodeMap[V]; 1014 if (N.getNode()) return N; 1015 1016 // If there's a virtual register allocated and initialized for this 1017 // value, use it. 1018 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V); 1019 if (It != FuncInfo.ValueMap.end()) { 1020 unsigned InReg = It->second; 1021 RegsForValue RFV(*DAG.getContext(), *TM.getTargetLowering(), 1022 InReg, V->getType()); 1023 SDValue Chain = DAG.getEntryNode(); 1024 N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); 1025 resolveDanglingDebugInfo(V, N); 1026 return N; 1027 } 1028 1029 // Otherwise create a new SDValue and remember it. 1030 SDValue Val = getValueImpl(V); 1031 NodeMap[V] = Val; 1032 resolveDanglingDebugInfo(V, Val); 1033 return Val; 1034} 1035 1036/// getNonRegisterValue - Return an SDValue for the given Value, but 1037/// don't look in FuncInfo.ValueMap for a virtual register. 1038SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) { 1039 // If we already have an SDValue for this value, use it. 1040 SDValue &N = NodeMap[V]; 1041 if (N.getNode()) return N; 1042 1043 // Otherwise create a new SDValue and remember it. 1044 SDValue Val = getValueImpl(V); 1045 NodeMap[V] = Val; 1046 resolveDanglingDebugInfo(V, Val); 1047 return Val; 1048} 1049 1050/// getValueImpl - Helper function for getValue and getNonRegisterValue. 1051/// Create an SDValue for the given value. 1052SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { 1053 const TargetLowering *TLI = TM.getTargetLowering(); 1054 1055 if (const Constant *C = dyn_cast<Constant>(V)) { 1056 EVT VT = TLI->getValueType(V->getType(), true); 1057 1058 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) 1059 return DAG.getConstant(*CI, VT); 1060 1061 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) 1062 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); 1063 1064 if (isa<ConstantPointerNull>(C)) { 1065 unsigned AS = V->getType()->getPointerAddressSpace(); 1066 return DAG.getConstant(0, TLI->getPointerTy(AS)); 1067 } 1068 1069 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) 1070 return DAG.getConstantFP(*CFP, VT); 1071 1072 if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) 1073 return DAG.getUNDEF(VT); 1074 1075 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 1076 visit(CE->getOpcode(), *CE); 1077 SDValue N1 = NodeMap[V]; 1078 assert(N1.getNode() && "visit didn't populate the NodeMap!"); 1079 return N1; 1080 } 1081 1082 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { 1083 SmallVector<SDValue, 4> Constants; 1084 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end(); 1085 OI != OE; ++OI) { 1086 SDNode *Val = getValue(*OI).getNode(); 1087 // If the operand is an empty aggregate, there are no values. 1088 if (!Val) continue; 1089 // Add each leaf value from the operand to the Constants list 1090 // to form a flattened list of all the values. 1091 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) 1092 Constants.push_back(SDValue(Val, i)); 1093 } 1094 1095 return DAG.getMergeValues(Constants, getCurSDLoc()); 1096 } 1097 1098 if (const ConstantDataSequential *CDS = 1099 dyn_cast<ConstantDataSequential>(C)) { 1100 SmallVector<SDValue, 4> Ops; 1101 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) { 1102 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); 1103 // Add each leaf value from the operand to the Constants list 1104 // to form a flattened list of all the values. 1105 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) 1106 Ops.push_back(SDValue(Val, i)); 1107 } 1108 1109 if (isa<ArrayType>(CDS->getType())) 1110 return DAG.getMergeValues(Ops, getCurSDLoc()); 1111 return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), 1112 VT, Ops); 1113 } 1114 1115 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) { 1116 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && 1117 "Unknown struct or array constant!"); 1118 1119 SmallVector<EVT, 4> ValueVTs; 1120 ComputeValueVTs(*TLI, C->getType(), ValueVTs); 1121 unsigned NumElts = ValueVTs.size(); 1122 if (NumElts == 0) 1123 return SDValue(); // empty struct 1124 SmallVector<SDValue, 4> Constants(NumElts); 1125 for (unsigned i = 0; i != NumElts; ++i) { 1126 EVT EltVT = ValueVTs[i]; 1127 if (isa<UndefValue>(C)) 1128 Constants[i] = DAG.getUNDEF(EltVT); 1129 else if (EltVT.isFloatingPoint()) 1130 Constants[i] = DAG.getConstantFP(0, EltVT); 1131 else 1132 Constants[i] = DAG.getConstant(0, EltVT); 1133 } 1134 1135 return DAG.getMergeValues(Constants, getCurSDLoc()); 1136 } 1137 1138 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) 1139 return DAG.getBlockAddress(BA, VT); 1140 1141 VectorType *VecTy = cast<VectorType>(V->getType()); 1142 unsigned NumElements = VecTy->getNumElements(); 1143 1144 // Now that we know the number and type of the elements, get that number of 1145 // elements into the Ops array based on what kind of constant it is. 1146 SmallVector<SDValue, 16> Ops; 1147 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) { 1148 for (unsigned i = 0; i != NumElements; ++i) 1149 Ops.push_back(getValue(CV->getOperand(i))); 1150 } else { 1151 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!"); 1152 EVT EltVT = TLI->getValueType(VecTy->getElementType()); 1153 1154 SDValue Op; 1155 if (EltVT.isFloatingPoint()) 1156 Op = DAG.getConstantFP(0, EltVT); 1157 else 1158 Op = DAG.getConstant(0, EltVT); 1159 Ops.assign(NumElements, Op); 1160 } 1161 1162 // Create a BUILD_VECTOR node. 1163 return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops); 1164 } 1165 1166 // If this is a static alloca, generate it as the frameindex instead of 1167 // computation. 1168 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { 1169 DenseMap<const AllocaInst*, int>::iterator SI = 1170 FuncInfo.StaticAllocaMap.find(AI); 1171 if (SI != FuncInfo.StaticAllocaMap.end()) 1172 return DAG.getFrameIndex(SI->second, TLI->getPointerTy()); 1173 } 1174 1175 // If this is an instruction which fast-isel has deferred, select it now. 1176 if (const Instruction *Inst = dyn_cast<Instruction>(V)) { 1177 unsigned InReg = FuncInfo.InitializeRegForValue(Inst); 1178 RegsForValue RFV(*DAG.getContext(), *TLI, InReg, Inst->getType()); 1179 SDValue Chain = DAG.getEntryNode(); 1180 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); 1181 } 1182 1183 llvm_unreachable("Can't get register for value!"); 1184} 1185 1186void SelectionDAGBuilder::visitRet(const ReturnInst &I) { 1187 const TargetLowering *TLI = TM.getTargetLowering(); 1188 SDValue Chain = getControlRoot(); 1189 SmallVector<ISD::OutputArg, 8> Outs; 1190 SmallVector<SDValue, 8> OutVals; 1191 1192 if (!FuncInfo.CanLowerReturn) { 1193 unsigned DemoteReg = FuncInfo.DemoteRegister; 1194 const Function *F = I.getParent()->getParent(); 1195 1196 // Emit a store of the return value through the virtual register. 1197 // Leave Outs empty so that LowerReturn won't try to load return 1198 // registers the usual way. 1199 SmallVector<EVT, 1> PtrValueVTs; 1200 ComputeValueVTs(*TLI, PointerType::getUnqual(F->getReturnType()), 1201 PtrValueVTs); 1202 1203 SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]); 1204 SDValue RetOp = getValue(I.getOperand(0)); 1205 1206 SmallVector<EVT, 4> ValueVTs; 1207 SmallVector<uint64_t, 4> Offsets; 1208 ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets); 1209 unsigned NumValues = ValueVTs.size(); 1210 1211 SmallVector<SDValue, 4> Chains(NumValues); 1212 for (unsigned i = 0; i != NumValues; ++i) { 1213 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), 1214 RetPtr.getValueType(), RetPtr, 1215 DAG.getIntPtrConstant(Offsets[i])); 1216 Chains[i] = 1217 DAG.getStore(Chain, getCurSDLoc(), 1218 SDValue(RetOp.getNode(), RetOp.getResNo() + i), 1219 // FIXME: better loc info would be nice. 1220 Add, MachinePointerInfo(), false, false, 0); 1221 } 1222 1223 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 1224 MVT::Other, Chains); 1225 } else if (I.getNumOperands() != 0) { 1226 SmallVector<EVT, 4> ValueVTs; 1227 ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs); 1228 unsigned NumValues = ValueVTs.size(); 1229 if (NumValues) { 1230 SDValue RetOp = getValue(I.getOperand(0)); 1231 for (unsigned j = 0, f = NumValues; j != f; ++j) { 1232 EVT VT = ValueVTs[j]; 1233 1234 ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 1235 1236 const Function *F = I.getParent()->getParent(); 1237 if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1238 Attribute::SExt)) 1239 ExtendKind = ISD::SIGN_EXTEND; 1240 else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1241 Attribute::ZExt)) 1242 ExtendKind = ISD::ZERO_EXTEND; 1243 1244 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) 1245 VT = TLI->getTypeForExtArgOrReturn(VT.getSimpleVT(), ExtendKind); 1246 1247 unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), VT); 1248 MVT PartVT = TLI->getRegisterType(*DAG.getContext(), VT); 1249 SmallVector<SDValue, 4> Parts(NumParts); 1250 getCopyToParts(DAG, getCurSDLoc(), 1251 SDValue(RetOp.getNode(), RetOp.getResNo() + j), 1252 &Parts[0], NumParts, PartVT, &I, ExtendKind); 1253 1254 // 'inreg' on function refers to return value 1255 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 1256 if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1257 Attribute::InReg)) 1258 Flags.setInReg(); 1259 1260 // Propagate extension type if any 1261 if (ExtendKind == ISD::SIGN_EXTEND) 1262 Flags.setSExt(); 1263 else if (ExtendKind == ISD::ZERO_EXTEND) 1264 Flags.setZExt(); 1265 1266 for (unsigned i = 0; i < NumParts; ++i) { 1267 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(), 1268 VT, /*isfixed=*/true, 0, 0)); 1269 OutVals.push_back(Parts[i]); 1270 } 1271 } 1272 } 1273 } 1274 1275 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 1276 CallingConv::ID CallConv = 1277 DAG.getMachineFunction().getFunction()->getCallingConv(); 1278 Chain = TM.getTargetLowering()->LowerReturn(Chain, CallConv, isVarArg, 1279 Outs, OutVals, getCurSDLoc(), 1280 DAG); 1281 1282 // Verify that the target's LowerReturn behaved as expected. 1283 assert(Chain.getNode() && Chain.getValueType() == MVT::Other && 1284 "LowerReturn didn't return a valid chain!"); 1285 1286 // Update the DAG with the new chain value resulting from return lowering. 1287 DAG.setRoot(Chain); 1288} 1289 1290/// CopyToExportRegsIfNeeded - If the given value has virtual registers 1291/// created for it, emit nodes to copy the value into the virtual 1292/// registers. 1293void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { 1294 // Skip empty types 1295 if (V->getType()->isEmptyTy()) 1296 return; 1297 1298 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); 1299 if (VMI != FuncInfo.ValueMap.end()) { 1300 assert(!V->use_empty() && "Unused value assigned virtual registers!"); 1301 CopyValueToVirtualRegister(V, VMI->second); 1302 } 1303} 1304 1305/// ExportFromCurrentBlock - If this condition isn't known to be exported from 1306/// the current basic block, add it to ValueMap now so that we'll get a 1307/// CopyTo/FromReg. 1308void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { 1309 // No need to export constants. 1310 if (!isa<Instruction>(V) && !isa<Argument>(V)) return; 1311 1312 // Already exported? 1313 if (FuncInfo.isExportedInst(V)) return; 1314 1315 unsigned Reg = FuncInfo.InitializeRegForValue(V); 1316 CopyValueToVirtualRegister(V, Reg); 1317} 1318 1319bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, 1320 const BasicBlock *FromBB) { 1321 // The operands of the setcc have to be in this block. We don't know 1322 // how to export them from some other block. 1323 if (const Instruction *VI = dyn_cast<Instruction>(V)) { 1324 // Can export from current BB. 1325 if (VI->getParent() == FromBB) 1326 return true; 1327 1328 // Is already exported, noop. 1329 return FuncInfo.isExportedInst(V); 1330 } 1331 1332 // If this is an argument, we can export it if the BB is the entry block or 1333 // if it is already exported. 1334 if (isa<Argument>(V)) { 1335 if (FromBB == &FromBB->getParent()->getEntryBlock()) 1336 return true; 1337 1338 // Otherwise, can only export this if it is already exported. 1339 return FuncInfo.isExportedInst(V); 1340 } 1341 1342 // Otherwise, constants can always be exported. 1343 return true; 1344} 1345 1346/// Return branch probability calculated by BranchProbabilityInfo for IR blocks. 1347uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src, 1348 const MachineBasicBlock *Dst) const { 1349 BranchProbabilityInfo *BPI = FuncInfo.BPI; 1350 if (!BPI) 1351 return 0; 1352 const BasicBlock *SrcBB = Src->getBasicBlock(); 1353 const BasicBlock *DstBB = Dst->getBasicBlock(); 1354 return BPI->getEdgeWeight(SrcBB, DstBB); 1355} 1356 1357void SelectionDAGBuilder:: 1358addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst, 1359 uint32_t Weight /* = 0 */) { 1360 if (!Weight) 1361 Weight = getEdgeWeight(Src, Dst); 1362 Src->addSuccessor(Dst, Weight); 1363} 1364 1365 1366static bool InBlock(const Value *V, const BasicBlock *BB) { 1367 if (const Instruction *I = dyn_cast<Instruction>(V)) 1368 return I->getParent() == BB; 1369 return true; 1370} 1371 1372/// EmitBranchForMergedCondition - Helper method for FindMergedConditions. 1373/// This function emits a branch and is used at the leaves of an OR or an 1374/// AND operator tree. 1375/// 1376void 1377SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, 1378 MachineBasicBlock *TBB, 1379 MachineBasicBlock *FBB, 1380 MachineBasicBlock *CurBB, 1381 MachineBasicBlock *SwitchBB, 1382 uint32_t TWeight, 1383 uint32_t FWeight) { 1384 const BasicBlock *BB = CurBB->getBasicBlock(); 1385 1386 // If the leaf of the tree is a comparison, merge the condition into 1387 // the caseblock. 1388 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { 1389 // The operands of the cmp have to be in this block. We don't know 1390 // how to export them from some other block. If this is the first block 1391 // of the sequence, no exporting is needed. 1392 if (CurBB == SwitchBB || 1393 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && 1394 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { 1395 ISD::CondCode Condition; 1396 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { 1397 Condition = getICmpCondCode(IC->getPredicate()); 1398 } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) { 1399 Condition = getFCmpCondCode(FC->getPredicate()); 1400 if (TM.Options.NoNaNsFPMath) 1401 Condition = getFCmpCodeWithoutNaN(Condition); 1402 } else { 1403 Condition = ISD::SETEQ; // silence warning. 1404 llvm_unreachable("Unknown compare instruction"); 1405 } 1406 1407 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr, 1408 TBB, FBB, CurBB, TWeight, FWeight); 1409 SwitchCases.push_back(CB); 1410 return; 1411 } 1412 } 1413 1414 // Create a CaseBlock record representing this branch. 1415 CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()), 1416 nullptr, TBB, FBB, CurBB, TWeight, FWeight); 1417 SwitchCases.push_back(CB); 1418} 1419 1420/// Scale down both weights to fit into uint32_t. 1421static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) { 1422 uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse; 1423 uint32_t Scale = (NewMax / UINT32_MAX) + 1; 1424 NewTrue = NewTrue / Scale; 1425 NewFalse = NewFalse / Scale; 1426} 1427 1428/// FindMergedConditions - If Cond is an expression like 1429void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, 1430 MachineBasicBlock *TBB, 1431 MachineBasicBlock *FBB, 1432 MachineBasicBlock *CurBB, 1433 MachineBasicBlock *SwitchBB, 1434 unsigned Opc, uint32_t TWeight, 1435 uint32_t FWeight) { 1436 // If this node is not part of the or/and tree, emit it as a branch. 1437 const Instruction *BOp = dyn_cast<Instruction>(Cond); 1438 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) || 1439 (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() || 1440 BOp->getParent() != CurBB->getBasicBlock() || 1441 !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) || 1442 !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) { 1443 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, 1444 TWeight, FWeight); 1445 return; 1446 } 1447 1448 // Create TmpBB after CurBB. 1449 MachineFunction::iterator BBI = CurBB; 1450 MachineFunction &MF = DAG.getMachineFunction(); 1451 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); 1452 CurBB->getParent()->insert(++BBI, TmpBB); 1453 1454 if (Opc == Instruction::Or) { 1455 // Codegen X | Y as: 1456 // BB1: 1457 // jmp_if_X TBB 1458 // jmp TmpBB 1459 // TmpBB: 1460 // jmp_if_Y TBB 1461 // jmp FBB 1462 // 1463 1464 // We have flexibility in setting Prob for BB1 and Prob for TmpBB. 1465 // The requirement is that 1466 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) 1467 // = TrueProb for orignal BB. 1468 // Assuming the orignal weights are A and B, one choice is to set BB1's 1469 // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice 1470 // assumes that 1471 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. 1472 // Another choice is to assume TrueProb for BB1 equals to TrueProb for 1473 // TmpBB, but the math is more complicated. 1474 1475 uint64_t NewTrueWeight = TWeight; 1476 uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight; 1477 ScaleWeights(NewTrueWeight, NewFalseWeight); 1478 // Emit the LHS condition. 1479 FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc, 1480 NewTrueWeight, NewFalseWeight); 1481 1482 NewTrueWeight = TWeight; 1483 NewFalseWeight = 2 * (uint64_t)FWeight; 1484 ScaleWeights(NewTrueWeight, NewFalseWeight); 1485 // Emit the RHS condition into TmpBB. 1486 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc, 1487 NewTrueWeight, NewFalseWeight); 1488 } else { 1489 assert(Opc == Instruction::And && "Unknown merge op!"); 1490 // Codegen X & Y as: 1491 // BB1: 1492 // jmp_if_X TmpBB 1493 // jmp FBB 1494 // TmpBB: 1495 // jmp_if_Y TBB 1496 // jmp FBB 1497 // 1498 // This requires creation of TmpBB after CurBB. 1499 1500 // We have flexibility in setting Prob for BB1 and Prob for TmpBB. 1501 // The requirement is that 1502 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) 1503 // = FalseProb for orignal BB. 1504 // Assuming the orignal weights are A and B, one choice is to set BB1's 1505 // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice 1506 // assumes that 1507 // FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB. 1508 1509 uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight; 1510 uint64_t NewFalseWeight = FWeight; 1511 ScaleWeights(NewTrueWeight, NewFalseWeight); 1512 // Emit the LHS condition. 1513 FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc, 1514 NewTrueWeight, NewFalseWeight); 1515 1516 NewTrueWeight = 2 * (uint64_t)TWeight; 1517 NewFalseWeight = FWeight; 1518 ScaleWeights(NewTrueWeight, NewFalseWeight); 1519 // Emit the RHS condition into TmpBB. 1520 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc, 1521 NewTrueWeight, NewFalseWeight); 1522 } 1523} 1524 1525/// If the set of cases should be emitted as a series of branches, return true. 1526/// If we should emit this as a bunch of and/or'd together conditions, return 1527/// false. 1528bool 1529SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { 1530 if (Cases.size() != 2) return true; 1531 1532 // If this is two comparisons of the same values or'd or and'd together, they 1533 // will get folded into a single comparison, so don't emit two blocks. 1534 if ((Cases[0].CmpLHS == Cases[1].CmpLHS && 1535 Cases[0].CmpRHS == Cases[1].CmpRHS) || 1536 (Cases[0].CmpRHS == Cases[1].CmpLHS && 1537 Cases[0].CmpLHS == Cases[1].CmpRHS)) { 1538 return false; 1539 } 1540 1541 // Handle: (X != null) | (Y != null) --> (X|Y) != 0 1542 // Handle: (X == null) & (Y == null) --> (X|Y) == 0 1543 if (Cases[0].CmpRHS == Cases[1].CmpRHS && 1544 Cases[0].CC == Cases[1].CC && 1545 isa<Constant>(Cases[0].CmpRHS) && 1546 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { 1547 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) 1548 return false; 1549 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB) 1550 return false; 1551 } 1552 1553 return true; 1554} 1555 1556void SelectionDAGBuilder::visitBr(const BranchInst &I) { 1557 MachineBasicBlock *BrMBB = FuncInfo.MBB; 1558 1559 // Update machine-CFG edges. 1560 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; 1561 1562 // Figure out which block is immediately after the current one. 1563 MachineBasicBlock *NextBlock = nullptr; 1564 MachineFunction::iterator BBI = BrMBB; 1565 if (++BBI != FuncInfo.MF->end()) 1566 NextBlock = BBI; 1567 1568 if (I.isUnconditional()) { 1569 // Update machine-CFG edges. 1570 BrMBB->addSuccessor(Succ0MBB); 1571 1572 // If this is not a fall-through branch or optimizations are switched off, 1573 // emit the branch. 1574 if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None) 1575 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 1576 MVT::Other, getControlRoot(), 1577 DAG.getBasicBlock(Succ0MBB))); 1578 1579 return; 1580 } 1581 1582 // If this condition is one of the special cases we handle, do special stuff 1583 // now. 1584 const Value *CondVal = I.getCondition(); 1585 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; 1586 1587 // If this is a series of conditions that are or'd or and'd together, emit 1588 // this as a sequence of branches instead of setcc's with and/or operations. 1589 // As long as jumps are not expensive, this should improve performance. 1590 // For example, instead of something like: 1591 // cmp A, B 1592 // C = seteq 1593 // cmp D, E 1594 // F = setle 1595 // or C, F 1596 // jnz foo 1597 // Emit: 1598 // cmp A, B 1599 // je foo 1600 // cmp D, E 1601 // jle foo 1602 // 1603 if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) { 1604 if (!TM.getTargetLowering()->isJumpExpensive() && 1605 BOp->hasOneUse() && 1606 (BOp->getOpcode() == Instruction::And || 1607 BOp->getOpcode() == Instruction::Or)) { 1608 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, 1609 BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB), 1610 getEdgeWeight(BrMBB, Succ1MBB)); 1611 // If the compares in later blocks need to use values not currently 1612 // exported from this block, export them now. This block should always 1613 // be the first entry. 1614 assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"); 1615 1616 // Allow some cases to be rejected. 1617 if (ShouldEmitAsBranches(SwitchCases)) { 1618 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) { 1619 ExportFromCurrentBlock(SwitchCases[i].CmpLHS); 1620 ExportFromCurrentBlock(SwitchCases[i].CmpRHS); 1621 } 1622 1623 // Emit the branch for this block. 1624 visitSwitchCase(SwitchCases[0], BrMBB); 1625 SwitchCases.erase(SwitchCases.begin()); 1626 return; 1627 } 1628 1629 // Okay, we decided not to do this, remove any inserted MBB's and clear 1630 // SwitchCases. 1631 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) 1632 FuncInfo.MF->erase(SwitchCases[i].ThisBB); 1633 1634 SwitchCases.clear(); 1635 } 1636 } 1637 1638 // Create a CaseBlock record representing this branch. 1639 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), 1640 nullptr, Succ0MBB, Succ1MBB, BrMBB); 1641 1642 // Use visitSwitchCase to actually insert the fast branch sequence for this 1643 // cond branch. 1644 visitSwitchCase(CB, BrMBB); 1645} 1646 1647/// visitSwitchCase - Emits the necessary code to represent a single node in 1648/// the binary search tree resulting from lowering a switch instruction. 1649void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, 1650 MachineBasicBlock *SwitchBB) { 1651 SDValue Cond; 1652 SDValue CondLHS = getValue(CB.CmpLHS); 1653 SDLoc dl = getCurSDLoc(); 1654 1655 // Build the setcc now. 1656 if (!CB.CmpMHS) { 1657 // Fold "(X == true)" to X and "(X == false)" to !X to 1658 // handle common cases produced by branch lowering. 1659 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && 1660 CB.CC == ISD::SETEQ) 1661 Cond = CondLHS; 1662 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && 1663 CB.CC == ISD::SETEQ) { 1664 SDValue True = DAG.getConstant(1, CondLHS.getValueType()); 1665 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); 1666 } else 1667 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC); 1668 } else { 1669 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now"); 1670 1671 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); 1672 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); 1673 1674 SDValue CmpOp = getValue(CB.CmpMHS); 1675 EVT VT = CmpOp.getValueType(); 1676 1677 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { 1678 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT), 1679 ISD::SETLE); 1680 } else { 1681 SDValue SUB = DAG.getNode(ISD::SUB, dl, 1682 VT, CmpOp, DAG.getConstant(Low, VT)); 1683 Cond = DAG.getSetCC(dl, MVT::i1, SUB, 1684 DAG.getConstant(High-Low, VT), ISD::SETULE); 1685 } 1686 } 1687 1688 // Update successor info 1689 addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight); 1690 // TrueBB and FalseBB are always different unless the incoming IR is 1691 // degenerate. This only happens when running llc on weird IR. 1692 if (CB.TrueBB != CB.FalseBB) 1693 addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight); 1694 1695 // Set NextBlock to be the MBB immediately after the current one, if any. 1696 // This is used to avoid emitting unnecessary branches to the next block. 1697 MachineBasicBlock *NextBlock = nullptr; 1698 MachineFunction::iterator BBI = SwitchBB; 1699 if (++BBI != FuncInfo.MF->end()) 1700 NextBlock = BBI; 1701 1702 // If the lhs block is the next block, invert the condition so that we can 1703 // fall through to the lhs instead of the rhs block. 1704 if (CB.TrueBB == NextBlock) { 1705 std::swap(CB.TrueBB, CB.FalseBB); 1706 SDValue True = DAG.getConstant(1, Cond.getValueType()); 1707 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); 1708 } 1709 1710 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, 1711 MVT::Other, getControlRoot(), Cond, 1712 DAG.getBasicBlock(CB.TrueBB)); 1713 1714 // Insert the false branch. Do this even if it's a fall through branch, 1715 // this makes it easier to do DAG optimizations which require inverting 1716 // the branch condition. 1717 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, 1718 DAG.getBasicBlock(CB.FalseBB)); 1719 1720 DAG.setRoot(BrCond); 1721} 1722 1723/// visitJumpTable - Emit JumpTable node in the current MBB 1724void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) { 1725 // Emit the code for the jump table 1726 assert(JT.Reg != -1U && "Should lower JT Header first!"); 1727 EVT PTy = TM.getTargetLowering()->getPointerTy(); 1728 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), 1729 JT.Reg, PTy); 1730 SDValue Table = DAG.getJumpTable(JT.JTI, PTy); 1731 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(), 1732 MVT::Other, Index.getValue(1), 1733 Table, Index); 1734 DAG.setRoot(BrJumpTable); 1735} 1736 1737/// visitJumpTableHeader - This function emits necessary code to produce index 1738/// in the JumpTable from switch case. 1739void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, 1740 JumpTableHeader &JTH, 1741 MachineBasicBlock *SwitchBB) { 1742 // Subtract the lowest switch case value from the value being switched on and 1743 // conditional branch to default mbb if the result is greater than the 1744 // difference between smallest and largest cases. 1745 SDValue SwitchOp = getValue(JTH.SValue); 1746 EVT VT = SwitchOp.getValueType(); 1747 SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp, 1748 DAG.getConstant(JTH.First, VT)); 1749 1750 // The SDNode we just created, which holds the value being switched on minus 1751 // the smallest case value, needs to be copied to a virtual register so it 1752 // can be used as an index into the jump table in a subsequent basic block. 1753 // This value may be smaller or larger than the target's pointer type, and 1754 // therefore require extension or truncating. 1755 const TargetLowering *TLI = TM.getTargetLowering(); 1756 SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI->getPointerTy()); 1757 1758 unsigned JumpTableReg = FuncInfo.CreateReg(TLI->getPointerTy()); 1759 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(), 1760 JumpTableReg, SwitchOp); 1761 JT.Reg = JumpTableReg; 1762 1763 // Emit the range check for the jump table, and branch to the default block 1764 // for the switch statement if the value being switched on exceeds the largest 1765 // case in the switch. 1766 SDValue CMP = DAG.getSetCC(getCurSDLoc(), 1767 TLI->getSetCCResultType(*DAG.getContext(), 1768 Sub.getValueType()), 1769 Sub, 1770 DAG.getConstant(JTH.Last - JTH.First,VT), 1771 ISD::SETUGT); 1772 1773 // Set NextBlock to be the MBB immediately after the current one, if any. 1774 // This is used to avoid emitting unnecessary branches to the next block. 1775 MachineBasicBlock *NextBlock = nullptr; 1776 MachineFunction::iterator BBI = SwitchBB; 1777 1778 if (++BBI != FuncInfo.MF->end()) 1779 NextBlock = BBI; 1780 1781 SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1782 MVT::Other, CopyTo, CMP, 1783 DAG.getBasicBlock(JT.Default)); 1784 1785 if (JT.MBB != NextBlock) 1786 BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond, 1787 DAG.getBasicBlock(JT.MBB)); 1788 1789 DAG.setRoot(BrCond); 1790} 1791 1792/// Codegen a new tail for a stack protector check ParentMBB which has had its 1793/// tail spliced into a stack protector check success bb. 1794/// 1795/// For a high level explanation of how this fits into the stack protector 1796/// generation see the comment on the declaration of class 1797/// StackProtectorDescriptor. 1798void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, 1799 MachineBasicBlock *ParentBB) { 1800 1801 // First create the loads to the guard/stack slot for the comparison. 1802 const TargetLowering *TLI = TM.getTargetLowering(); 1803 EVT PtrTy = TLI->getPointerTy(); 1804 1805 MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo(); 1806 int FI = MFI->getStackProtectorIndex(); 1807 1808 const Value *IRGuard = SPD.getGuard(); 1809 SDValue GuardPtr = getValue(IRGuard); 1810 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); 1811 1812 unsigned Align = 1813 TLI->getDataLayout()->getPrefTypeAlignment(IRGuard->getType()); 1814 SDValue Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(), 1815 GuardPtr, MachinePointerInfo(IRGuard, 0), 1816 true, false, false, Align); 1817 1818 SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(), 1819 StackSlotPtr, 1820 MachinePointerInfo::getFixedStack(FI), 1821 true, false, false, Align); 1822 1823 // Perform the comparison via a subtract/getsetcc. 1824 EVT VT = Guard.getValueType(); 1825 SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot); 1826 1827 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), 1828 TLI->getSetCCResultType(*DAG.getContext(), 1829 Sub.getValueType()), 1830 Sub, DAG.getConstant(0, VT), 1831 ISD::SETNE); 1832 1833 // If the sub is not 0, then we know the guard/stackslot do not equal, so 1834 // branch to failure MBB. 1835 SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1836 MVT::Other, StackSlot.getOperand(0), 1837 Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); 1838 // Otherwise branch to success MBB. 1839 SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(), 1840 MVT::Other, BrCond, 1841 DAG.getBasicBlock(SPD.getSuccessMBB())); 1842 1843 DAG.setRoot(Br); 1844} 1845 1846/// Codegen the failure basic block for a stack protector check. 1847/// 1848/// A failure stack protector machine basic block consists simply of a call to 1849/// __stack_chk_fail(). 1850/// 1851/// For a high level explanation of how this fits into the stack protector 1852/// generation see the comment on the declaration of class 1853/// StackProtectorDescriptor. 1854void 1855SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { 1856 const TargetLowering *TLI = TM.getTargetLowering(); 1857 SDValue Chain = TLI->makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, 1858 MVT::isVoid, nullptr, 0, false, 1859 getCurSDLoc(), false, false).second; 1860 DAG.setRoot(Chain); 1861} 1862 1863/// visitBitTestHeader - This function emits necessary code to produce value 1864/// suitable for "bit tests" 1865void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, 1866 MachineBasicBlock *SwitchBB) { 1867 // Subtract the minimum value 1868 SDValue SwitchOp = getValue(B.SValue); 1869 EVT VT = SwitchOp.getValueType(); 1870 SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp, 1871 DAG.getConstant(B.First, VT)); 1872 1873 // Check range 1874 const TargetLowering *TLI = TM.getTargetLowering(); 1875 SDValue RangeCmp = DAG.getSetCC(getCurSDLoc(), 1876 TLI->getSetCCResultType(*DAG.getContext(), 1877 Sub.getValueType()), 1878 Sub, DAG.getConstant(B.Range, VT), 1879 ISD::SETUGT); 1880 1881 // Determine the type of the test operands. 1882 bool UsePtrType = false; 1883 if (!TLI->isTypeLegal(VT)) 1884 UsePtrType = true; 1885 else { 1886 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) 1887 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { 1888 // Switch table case range are encoded into series of masks. 1889 // Just use pointer type, it's guaranteed to fit. 1890 UsePtrType = true; 1891 break; 1892 } 1893 } 1894 if (UsePtrType) { 1895 VT = TLI->getPointerTy(); 1896 Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT); 1897 } 1898 1899 B.RegVT = VT.getSimpleVT(); 1900 B.Reg = FuncInfo.CreateReg(B.RegVT); 1901 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(), 1902 B.Reg, Sub); 1903 1904 // Set NextBlock to be the MBB immediately after the current one, if any. 1905 // This is used to avoid emitting unnecessary branches to the next block. 1906 MachineBasicBlock *NextBlock = nullptr; 1907 MachineFunction::iterator BBI = SwitchBB; 1908 if (++BBI != FuncInfo.MF->end()) 1909 NextBlock = BBI; 1910 1911 MachineBasicBlock* MBB = B.Cases[0].ThisBB; 1912 1913 addSuccessorWithWeight(SwitchBB, B.Default); 1914 addSuccessorWithWeight(SwitchBB, MBB); 1915 1916 SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1917 MVT::Other, CopyTo, RangeCmp, 1918 DAG.getBasicBlock(B.Default)); 1919 1920 if (MBB != NextBlock) 1921 BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo, 1922 DAG.getBasicBlock(MBB)); 1923 1924 DAG.setRoot(BrRange); 1925} 1926 1927/// visitBitTestCase - this function produces one "bit test" 1928void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, 1929 MachineBasicBlock* NextMBB, 1930 uint32_t BranchWeightToNext, 1931 unsigned Reg, 1932 BitTestCase &B, 1933 MachineBasicBlock *SwitchBB) { 1934 MVT VT = BB.RegVT; 1935 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), 1936 Reg, VT); 1937 SDValue Cmp; 1938 unsigned PopCount = CountPopulation_64(B.Mask); 1939 const TargetLowering *TLI = TM.getTargetLowering(); 1940 if (PopCount == 1) { 1941 // Testing for a single bit; just compare the shift count with what it 1942 // would need to be to shift a 1 bit in that position. 1943 Cmp = DAG.getSetCC(getCurSDLoc(), 1944 TLI->getSetCCResultType(*DAG.getContext(), VT), 1945 ShiftOp, 1946 DAG.getConstant(countTrailingZeros(B.Mask), VT), 1947 ISD::SETEQ); 1948 } else if (PopCount == BB.Range) { 1949 // There is only one zero bit in the range, test for it directly. 1950 Cmp = DAG.getSetCC(getCurSDLoc(), 1951 TLI->getSetCCResultType(*DAG.getContext(), VT), 1952 ShiftOp, 1953 DAG.getConstant(CountTrailingOnes_64(B.Mask), VT), 1954 ISD::SETNE); 1955 } else { 1956 // Make desired shift 1957 SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT, 1958 DAG.getConstant(1, VT), ShiftOp); 1959 1960 // Emit bit tests and jumps 1961 SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(), 1962 VT, SwitchVal, DAG.getConstant(B.Mask, VT)); 1963 Cmp = DAG.getSetCC(getCurSDLoc(), 1964 TLI->getSetCCResultType(*DAG.getContext(), VT), 1965 AndOp, DAG.getConstant(0, VT), 1966 ISD::SETNE); 1967 } 1968 1969 // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight. 1970 addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight); 1971 // The branch weight from SwitchBB to NextMBB is BranchWeightToNext. 1972 addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext); 1973 1974 SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1975 MVT::Other, getControlRoot(), 1976 Cmp, DAG.getBasicBlock(B.TargetBB)); 1977 1978 // Set NextBlock to be the MBB immediately after the current one, if any. 1979 // This is used to avoid emitting unnecessary branches to the next block. 1980 MachineBasicBlock *NextBlock = nullptr; 1981 MachineFunction::iterator BBI = SwitchBB; 1982 if (++BBI != FuncInfo.MF->end()) 1983 NextBlock = BBI; 1984 1985 if (NextMBB != NextBlock) 1986 BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd, 1987 DAG.getBasicBlock(NextMBB)); 1988 1989 DAG.setRoot(BrAnd); 1990} 1991 1992void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { 1993 MachineBasicBlock *InvokeMBB = FuncInfo.MBB; 1994 1995 // Retrieve successors. 1996 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; 1997 MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)]; 1998 1999 const Value *Callee(I.getCalledValue()); 2000 const Function *Fn = dyn_cast<Function>(Callee); 2001 if (isa<InlineAsm>(Callee)) 2002 visitInlineAsm(&I); 2003 else if (Fn && Fn->isIntrinsic()) { 2004 assert(Fn->getIntrinsicID() == Intrinsic::donothing); 2005 // Ignore invokes to @llvm.donothing: jump directly to the next BB. 2006 } else 2007 LowerCallTo(&I, getValue(Callee), false, LandingPad); 2008 2009 // If the value of the invoke is used outside of its defining block, make it 2010 // available as a virtual register. 2011 CopyToExportRegsIfNeeded(&I); 2012 2013 // Update successor info 2014 addSuccessorWithWeight(InvokeMBB, Return); 2015 addSuccessorWithWeight(InvokeMBB, LandingPad); 2016 2017 // Drop into normal successor. 2018 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 2019 MVT::Other, getControlRoot(), 2020 DAG.getBasicBlock(Return))); 2021} 2022 2023void SelectionDAGBuilder::visitResume(const ResumeInst &RI) { 2024 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!"); 2025} 2026 2027void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { 2028 assert(FuncInfo.MBB->isLandingPad() && 2029 "Call to landingpad not in landing pad!"); 2030 2031 MachineBasicBlock *MBB = FuncInfo.MBB; 2032 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 2033 AddLandingPadInfo(LP, MMI, MBB); 2034 2035 // If there aren't registers to copy the values into (e.g., during SjLj 2036 // exceptions), then don't bother to create these DAG nodes. 2037 const TargetLowering *TLI = TM.getTargetLowering(); 2038 if (TLI->getExceptionPointerRegister() == 0 && 2039 TLI->getExceptionSelectorRegister() == 0) 2040 return; 2041 2042 SmallVector<EVT, 2> ValueVTs; 2043 ComputeValueVTs(*TLI, LP.getType(), ValueVTs); 2044 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported"); 2045 2046 // Get the two live-in registers as SDValues. The physregs have already been 2047 // copied into virtual registers. 2048 SDValue Ops[2]; 2049 Ops[0] = DAG.getZExtOrTrunc( 2050 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), 2051 FuncInfo.ExceptionPointerVirtReg, TLI->getPointerTy()), 2052 getCurSDLoc(), ValueVTs[0]); 2053 Ops[1] = DAG.getZExtOrTrunc( 2054 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), 2055 FuncInfo.ExceptionSelectorVirtReg, TLI->getPointerTy()), 2056 getCurSDLoc(), ValueVTs[1]); 2057 2058 // Merge into one. 2059 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 2060 DAG.getVTList(ValueVTs), Ops); 2061 setValue(&LP, Res); 2062} 2063 2064/// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for 2065/// small case ranges). 2066bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, 2067 CaseRecVector& WorkList, 2068 const Value* SV, 2069 MachineBasicBlock *Default, 2070 MachineBasicBlock *SwitchBB) { 2071 // Size is the number of Cases represented by this range. 2072 size_t Size = CR.Range.second - CR.Range.first; 2073 if (Size > 3) 2074 return false; 2075 2076 // Get the MachineFunction which holds the current MBB. This is used when 2077 // inserting any additional MBBs necessary to represent the switch. 2078 MachineFunction *CurMF = FuncInfo.MF; 2079 2080 // Figure out which block is immediately after the current one. 2081 MachineBasicBlock *NextBlock = nullptr; 2082 MachineFunction::iterator BBI = CR.CaseBB; 2083 2084 if (++BBI != FuncInfo.MF->end()) 2085 NextBlock = BBI; 2086 2087 BranchProbabilityInfo *BPI = FuncInfo.BPI; 2088 // If any two of the cases has the same destination, and if one value 2089 // is the same as the other, but has one bit unset that the other has set, 2090 // use bit manipulation to do two compares at once. For example: 2091 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)" 2092 // TODO: This could be extended to merge any 2 cases in switches with 3 cases. 2093 // TODO: Handle cases where CR.CaseBB != SwitchBB. 2094 if (Size == 2 && CR.CaseBB == SwitchBB) { 2095 Case &Small = *CR.Range.first; 2096 Case &Big = *(CR.Range.second-1); 2097 2098 if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) { 2099 const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue(); 2100 const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue(); 2101 2102 // Check that there is only one bit different. 2103 if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 && 2104 (SmallValue | BigValue) == BigValue) { 2105 // Isolate the common bit. 2106 APInt CommonBit = BigValue & ~SmallValue; 2107 assert((SmallValue | CommonBit) == BigValue && 2108 CommonBit.countPopulation() == 1 && "Not a common bit?"); 2109 2110 SDValue CondLHS = getValue(SV); 2111 EVT VT = CondLHS.getValueType(); 2112 SDLoc DL = getCurSDLoc(); 2113 2114 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS, 2115 DAG.getConstant(CommonBit, VT)); 2116 SDValue Cond = DAG.getSetCC(DL, MVT::i1, 2117 Or, DAG.getConstant(BigValue, VT), 2118 ISD::SETEQ); 2119 2120 // Update successor info. 2121 // Both Small and Big will jump to Small.BB, so we sum up the weights. 2122 addSuccessorWithWeight(SwitchBB, Small.BB, 2123 Small.ExtraWeight + Big.ExtraWeight); 2124 addSuccessorWithWeight(SwitchBB, Default, 2125 // The default destination is the first successor in IR. 2126 BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0); 2127 2128 // Insert the true branch. 2129 SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other, 2130 getControlRoot(), Cond, 2131 DAG.getBasicBlock(Small.BB)); 2132 2133 // Insert the false branch. 2134 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond, 2135 DAG.getBasicBlock(Default)); 2136 2137 DAG.setRoot(BrCond); 2138 return true; 2139 } 2140 } 2141 } 2142 2143 // Order cases by weight so the most likely case will be checked first. 2144 uint32_t UnhandledWeights = 0; 2145 if (BPI) { 2146 for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) { 2147 uint32_t IWeight = I->ExtraWeight; 2148 UnhandledWeights += IWeight; 2149 for (CaseItr J = CR.Range.first; J < I; ++J) { 2150 uint32_t JWeight = J->ExtraWeight; 2151 if (IWeight > JWeight) 2152 std::swap(*I, *J); 2153 } 2154 } 2155 } 2156 // Rearrange the case blocks so that the last one falls through if possible. 2157 Case &BackCase = *(CR.Range.second-1); 2158 if (Size > 1 && 2159 NextBlock && Default != NextBlock && BackCase.BB != NextBlock) { 2160 // The last case block won't fall through into 'NextBlock' if we emit the 2161 // branches in this order. See if rearranging a case value would help. 2162 // We start at the bottom as it's the case with the least weight. 2163 for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I) 2164 if (I->BB == NextBlock) { 2165 std::swap(*I, BackCase); 2166 break; 2167 } 2168 } 2169 2170 // Create a CaseBlock record representing a conditional branch to 2171 // the Case's target mbb if the value being switched on SV is equal 2172 // to C. 2173 MachineBasicBlock *CurBlock = CR.CaseBB; 2174 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { 2175 MachineBasicBlock *FallThrough; 2176 if (I != E-1) { 2177 FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock()); 2178 CurMF->insert(BBI, FallThrough); 2179 2180 // Put SV in a virtual register to make it available from the new blocks. 2181 ExportFromCurrentBlock(SV); 2182 } else { 2183 // If the last case doesn't match, go to the default block. 2184 FallThrough = Default; 2185 } 2186 2187 const Value *RHS, *LHS, *MHS; 2188 ISD::CondCode CC; 2189 if (I->High == I->Low) { 2190 // This is just small small case range :) containing exactly 1 case 2191 CC = ISD::SETEQ; 2192 LHS = SV; RHS = I->High; MHS = nullptr; 2193 } else { 2194 CC = ISD::SETLE; 2195 LHS = I->Low; MHS = SV; RHS = I->High; 2196 } 2197 2198 // The false weight should be sum of all un-handled cases. 2199 UnhandledWeights -= I->ExtraWeight; 2200 CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough, 2201 /* me */ CurBlock, 2202 /* trueweight */ I->ExtraWeight, 2203 /* falseweight */ UnhandledWeights); 2204 2205 // If emitting the first comparison, just call visitSwitchCase to emit the 2206 // code into the current block. Otherwise, push the CaseBlock onto the 2207 // vector to be later processed by SDISel, and insert the node's MBB 2208 // before the next MBB. 2209 if (CurBlock == SwitchBB) 2210 visitSwitchCase(CB, SwitchBB); 2211 else 2212 SwitchCases.push_back(CB); 2213 2214 CurBlock = FallThrough; 2215 } 2216 2217 return true; 2218} 2219 2220static inline bool areJTsAllowed(const TargetLowering &TLI) { 2221 return TLI.supportJumpTables() && 2222 (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) || 2223 TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other)); 2224} 2225 2226static APInt ComputeRange(const APInt &First, const APInt &Last) { 2227 uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1; 2228 APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth); 2229 return (LastExt - FirstExt + 1ULL); 2230} 2231 2232/// handleJTSwitchCase - Emit jumptable for current switch case range 2233bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR, 2234 CaseRecVector &WorkList, 2235 const Value *SV, 2236 MachineBasicBlock *Default, 2237 MachineBasicBlock *SwitchBB) { 2238 Case& FrontCase = *CR.Range.first; 2239 Case& BackCase = *(CR.Range.second-1); 2240 2241 const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue(); 2242 const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); 2243 2244 APInt TSize(First.getBitWidth(), 0); 2245 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) 2246 TSize += I->size(); 2247 2248 const TargetLowering *TLI = TM.getTargetLowering(); 2249 if (!areJTsAllowed(*TLI) || TSize.ult(TLI->getMinimumJumpTableEntries())) 2250 return false; 2251 2252 APInt Range = ComputeRange(First, Last); 2253 // The density is TSize / Range. Require at least 40%. 2254 // It should not be possible for IntTSize to saturate for sane code, but make 2255 // sure we handle Range saturation correctly. 2256 uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10); 2257 uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10); 2258 if (IntTSize * 10 < IntRange * 4) 2259 return false; 2260 2261 DEBUG(dbgs() << "Lowering jump table\n" 2262 << "First entry: " << First << ". Last entry: " << Last << '\n' 2263 << "Range: " << Range << ". Size: " << TSize << ".\n\n"); 2264 2265 // Get the MachineFunction which holds the current MBB. This is used when 2266 // inserting any additional MBBs necessary to represent the switch. 2267 MachineFunction *CurMF = FuncInfo.MF; 2268 2269 // Figure out which block is immediately after the current one. 2270 MachineFunction::iterator BBI = CR.CaseBB; 2271 ++BBI; 2272 2273 const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2274 2275 // Create a new basic block to hold the code for loading the address 2276 // of the jump table, and jumping to it. Update successor information; 2277 // we will either branch to the default case for the switch, or the jump 2278 // table. 2279 MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2280 CurMF->insert(BBI, JumpTableBB); 2281 2282 addSuccessorWithWeight(CR.CaseBB, Default); 2283 addSuccessorWithWeight(CR.CaseBB, JumpTableBB); 2284 2285 // Build a vector of destination BBs, corresponding to each target 2286 // of the jump table. If the value of the jump table slot corresponds to 2287 // a case statement, push the case's BB onto the vector, otherwise, push 2288 // the default BB. 2289 std::vector<MachineBasicBlock*> DestBBs; 2290 APInt TEI = First; 2291 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) { 2292 const APInt &Low = cast<ConstantInt>(I->Low)->getValue(); 2293 const APInt &High = cast<ConstantInt>(I->High)->getValue(); 2294 2295 if (Low.sle(TEI) && TEI.sle(High)) { 2296 DestBBs.push_back(I->BB); 2297 if (TEI==High) 2298 ++I; 2299 } else { 2300 DestBBs.push_back(Default); 2301 } 2302 } 2303 2304 // Calculate weight for each unique destination in CR. 2305 DenseMap<MachineBasicBlock*, uint32_t> DestWeights; 2306 if (FuncInfo.BPI) 2307 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { 2308 DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr = 2309 DestWeights.find(I->BB); 2310 if (Itr != DestWeights.end()) 2311 Itr->second += I->ExtraWeight; 2312 else 2313 DestWeights[I->BB] = I->ExtraWeight; 2314 } 2315 2316 // Update successor info. Add one edge to each unique successor. 2317 BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs()); 2318 for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(), 2319 E = DestBBs.end(); I != E; ++I) { 2320 if (!SuccsHandled[(*I)->getNumber()]) { 2321 SuccsHandled[(*I)->getNumber()] = true; 2322 DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr = 2323 DestWeights.find(*I); 2324 addSuccessorWithWeight(JumpTableBB, *I, 2325 Itr != DestWeights.end() ? Itr->second : 0); 2326 } 2327 } 2328 2329 // Create a jump table index for this jump table. 2330 unsigned JTEncoding = TLI->getJumpTableEncoding(); 2331 unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding) 2332 ->createJumpTableIndex(DestBBs); 2333 2334 // Set the jump table information so that we can codegen it as a second 2335 // MachineBasicBlock 2336 JumpTable JT(-1U, JTI, JumpTableBB, Default); 2337 JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB)); 2338 if (CR.CaseBB == SwitchBB) 2339 visitJumpTableHeader(JT, JTH, SwitchBB); 2340 2341 JTCases.push_back(JumpTableBlock(JTH, JT)); 2342 return true; 2343} 2344 2345/// handleBTSplitSwitchCase - emit comparison and split binary search tree into 2346/// 2 subtrees. 2347bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, 2348 CaseRecVector& WorkList, 2349 const Value* SV, 2350 MachineBasicBlock* Default, 2351 MachineBasicBlock* SwitchBB) { 2352 // Get the MachineFunction which holds the current MBB. This is used when 2353 // inserting any additional MBBs necessary to represent the switch. 2354 MachineFunction *CurMF = FuncInfo.MF; 2355 2356 // Figure out which block is immediately after the current one. 2357 MachineFunction::iterator BBI = CR.CaseBB; 2358 ++BBI; 2359 2360 Case& FrontCase = *CR.Range.first; 2361 Case& BackCase = *(CR.Range.second-1); 2362 const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2363 2364 // Size is the number of Cases represented by this range. 2365 unsigned Size = CR.Range.second - CR.Range.first; 2366 2367 const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue(); 2368 const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); 2369 double FMetric = 0; 2370 CaseItr Pivot = CR.Range.first + Size/2; 2371 2372 // Select optimal pivot, maximizing sum density of LHS and RHS. This will 2373 // (heuristically) allow us to emit JumpTable's later. 2374 APInt TSize(First.getBitWidth(), 0); 2375 for (CaseItr I = CR.Range.first, E = CR.Range.second; 2376 I!=E; ++I) 2377 TSize += I->size(); 2378 2379 APInt LSize = FrontCase.size(); 2380 APInt RSize = TSize-LSize; 2381 DEBUG(dbgs() << "Selecting best pivot: \n" 2382 << "First: " << First << ", Last: " << Last <<'\n' 2383 << "LSize: " << LSize << ", RSize: " << RSize << '\n'); 2384 for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second; 2385 J!=E; ++I, ++J) { 2386 const APInt &LEnd = cast<ConstantInt>(I->High)->getValue(); 2387 const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue(); 2388 APInt Range = ComputeRange(LEnd, RBegin); 2389 assert((Range - 2ULL).isNonNegative() && 2390 "Invalid case distance"); 2391 // Use volatile double here to avoid excess precision issues on some hosts, 2392 // e.g. that use 80-bit X87 registers. 2393 volatile double LDensity = 2394 (double)LSize.roundToDouble() / 2395 (LEnd - First + 1ULL).roundToDouble(); 2396 volatile double RDensity = 2397 (double)RSize.roundToDouble() / 2398 (Last - RBegin + 1ULL).roundToDouble(); 2399 volatile double Metric = Range.logBase2()*(LDensity+RDensity); 2400 // Should always split in some non-trivial place 2401 DEBUG(dbgs() <<"=>Step\n" 2402 << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n' 2403 << "LDensity: " << LDensity 2404 << ", RDensity: " << RDensity << '\n' 2405 << "Metric: " << Metric << '\n'); 2406 if (FMetric < Metric) { 2407 Pivot = J; 2408 FMetric = Metric; 2409 DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n'); 2410 } 2411 2412 LSize += J->size(); 2413 RSize -= J->size(); 2414 } 2415 2416 const TargetLowering *TLI = TM.getTargetLowering(); 2417 if (areJTsAllowed(*TLI)) { 2418 // If our case is dense we *really* should handle it earlier! 2419 assert((FMetric > 0) && "Should handle dense range earlier!"); 2420 } else { 2421 Pivot = CR.Range.first + Size/2; 2422 } 2423 2424 CaseRange LHSR(CR.Range.first, Pivot); 2425 CaseRange RHSR(Pivot, CR.Range.second); 2426 const Constant *C = Pivot->Low; 2427 MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr; 2428 2429 // We know that we branch to the LHS if the Value being switched on is 2430 // less than the Pivot value, C. We use this to optimize our binary 2431 // tree a bit, by recognizing that if SV is greater than or equal to the 2432 // LHS's Case Value, and that Case Value is exactly one less than the 2433 // Pivot's Value, then we can branch directly to the LHS's Target, 2434 // rather than creating a leaf node for it. 2435 if ((LHSR.second - LHSR.first) == 1 && 2436 LHSR.first->High == CR.GE && 2437 cast<ConstantInt>(C)->getValue() == 2438 (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) { 2439 TrueBB = LHSR.first->BB; 2440 } else { 2441 TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2442 CurMF->insert(BBI, TrueBB); 2443 WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR)); 2444 2445 // Put SV in a virtual register to make it available from the new blocks. 2446 ExportFromCurrentBlock(SV); 2447 } 2448 2449 // Similar to the optimization above, if the Value being switched on is 2450 // known to be less than the Constant CR.LT, and the current Case Value 2451 // is CR.LT - 1, then we can branch directly to the target block for 2452 // the current Case Value, rather than emitting a RHS leaf node for it. 2453 if ((RHSR.second - RHSR.first) == 1 && CR.LT && 2454 cast<ConstantInt>(RHSR.first->Low)->getValue() == 2455 (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) { 2456 FalseBB = RHSR.first->BB; 2457 } else { 2458 FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2459 CurMF->insert(BBI, FalseBB); 2460 WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR)); 2461 2462 // Put SV in a virtual register to make it available from the new blocks. 2463 ExportFromCurrentBlock(SV); 2464 } 2465 2466 // Create a CaseBlock record representing a conditional branch to 2467 // the LHS node if the value being switched on SV is less than C. 2468 // Otherwise, branch to LHS. 2469 CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB); 2470 2471 if (CR.CaseBB == SwitchBB) 2472 visitSwitchCase(CB, SwitchBB); 2473 else 2474 SwitchCases.push_back(CB); 2475 2476 return true; 2477} 2478 2479/// handleBitTestsSwitchCase - if current case range has few destination and 2480/// range span less, than machine word bitwidth, encode case range into series 2481/// of masks and emit bit tests with these masks. 2482bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, 2483 CaseRecVector& WorkList, 2484 const Value* SV, 2485 MachineBasicBlock* Default, 2486 MachineBasicBlock* SwitchBB) { 2487 const TargetLowering *TLI = TM.getTargetLowering(); 2488 EVT PTy = TLI->getPointerTy(); 2489 unsigned IntPtrBits = PTy.getSizeInBits(); 2490 2491 Case& FrontCase = *CR.Range.first; 2492 Case& BackCase = *(CR.Range.second-1); 2493 2494 // Get the MachineFunction which holds the current MBB. This is used when 2495 // inserting any additional MBBs necessary to represent the switch. 2496 MachineFunction *CurMF = FuncInfo.MF; 2497 2498 // If target does not have legal shift left, do not emit bit tests at all. 2499 if (!TLI->isOperationLegal(ISD::SHL, PTy)) 2500 return false; 2501 2502 size_t numCmps = 0; 2503 for (CaseItr I = CR.Range.first, E = CR.Range.second; 2504 I!=E; ++I) { 2505 // Single case counts one, case range - two. 2506 numCmps += (I->Low == I->High ? 1 : 2); 2507 } 2508 2509 // Count unique destinations 2510 SmallSet<MachineBasicBlock*, 4> Dests; 2511 for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) { 2512 Dests.insert(I->BB); 2513 if (Dests.size() > 3) 2514 // Don't bother the code below, if there are too much unique destinations 2515 return false; 2516 } 2517 DEBUG(dbgs() << "Total number of unique destinations: " 2518 << Dests.size() << '\n' 2519 << "Total number of comparisons: " << numCmps << '\n'); 2520 2521 // Compute span of values. 2522 const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue(); 2523 const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue(); 2524 APInt cmpRange = maxValue - minValue; 2525 2526 DEBUG(dbgs() << "Compare range: " << cmpRange << '\n' 2527 << "Low bound: " << minValue << '\n' 2528 << "High bound: " << maxValue << '\n'); 2529 2530 if (cmpRange.uge(IntPtrBits) || 2531 (!(Dests.size() == 1 && numCmps >= 3) && 2532 !(Dests.size() == 2 && numCmps >= 5) && 2533 !(Dests.size() >= 3 && numCmps >= 6))) 2534 return false; 2535 2536 DEBUG(dbgs() << "Emitting bit tests\n"); 2537 APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth()); 2538 2539 // Optimize the case where all the case values fit in a 2540 // word without having to subtract minValue. In this case, 2541 // we can optimize away the subtraction. 2542 if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) { 2543 cmpRange = maxValue; 2544 } else { 2545 lowBound = minValue; 2546 } 2547 2548 CaseBitsVector CasesBits; 2549 unsigned i, count = 0; 2550 2551 for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) { 2552 MachineBasicBlock* Dest = I->BB; 2553 for (i = 0; i < count; ++i) 2554 if (Dest == CasesBits[i].BB) 2555 break; 2556 2557 if (i == count) { 2558 assert((count < 3) && "Too much destinations to test!"); 2559 CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/)); 2560 count++; 2561 } 2562 2563 const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue(); 2564 const APInt& highValue = cast<ConstantInt>(I->High)->getValue(); 2565 2566 uint64_t lo = (lowValue - lowBound).getZExtValue(); 2567 uint64_t hi = (highValue - lowBound).getZExtValue(); 2568 CasesBits[i].ExtraWeight += I->ExtraWeight; 2569 2570 for (uint64_t j = lo; j <= hi; j++) { 2571 CasesBits[i].Mask |= 1ULL << j; 2572 CasesBits[i].Bits++; 2573 } 2574 2575 } 2576 std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp()); 2577 2578 BitTestInfo BTC; 2579 2580 // Figure out which block is immediately after the current one. 2581 MachineFunction::iterator BBI = CR.CaseBB; 2582 ++BBI; 2583 2584 const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2585 2586 DEBUG(dbgs() << "Cases:\n"); 2587 for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) { 2588 DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask 2589 << ", Bits: " << CasesBits[i].Bits 2590 << ", BB: " << CasesBits[i].BB << '\n'); 2591 2592 MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2593 CurMF->insert(BBI, CaseBB); 2594 BTC.push_back(BitTestCase(CasesBits[i].Mask, 2595 CaseBB, 2596 CasesBits[i].BB, CasesBits[i].ExtraWeight)); 2597 2598 // Put SV in a virtual register to make it available from the new blocks. 2599 ExportFromCurrentBlock(SV); 2600 } 2601 2602 BitTestBlock BTB(lowBound, cmpRange, SV, 2603 -1U, MVT::Other, (CR.CaseBB == SwitchBB), 2604 CR.CaseBB, Default, BTC); 2605 2606 if (CR.CaseBB == SwitchBB) 2607 visitBitTestHeader(BTB, SwitchBB); 2608 2609 BitTestCases.push_back(BTB); 2610 2611 return true; 2612} 2613 2614/// Clusterify - Transform simple list of Cases into list of CaseRange's 2615size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases, 2616 const SwitchInst& SI) { 2617 size_t numCmps = 0; 2618 2619 BranchProbabilityInfo *BPI = FuncInfo.BPI; 2620 // Start with "simple" cases 2621 for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end(); 2622 i != e; ++i) { 2623 const BasicBlock *SuccBB = i.getCaseSuccessor(); 2624 MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB]; 2625 2626 uint32_t ExtraWeight = 2627 BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0; 2628 2629 Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(), 2630 SMBB, ExtraWeight)); 2631 } 2632 std::sort(Cases.begin(), Cases.end(), CaseCmp()); 2633 2634 // Merge case into clusters 2635 if (Cases.size() >= 2) 2636 // Must recompute end() each iteration because it may be 2637 // invalidated by erase if we hold on to it 2638 for (CaseItr I = Cases.begin(), J = std::next(Cases.begin()); 2639 J != Cases.end(); ) { 2640 const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue(); 2641 const APInt& currentValue = cast<ConstantInt>(I->High)->getValue(); 2642 MachineBasicBlock* nextBB = J->BB; 2643 MachineBasicBlock* currentBB = I->BB; 2644 2645 // If the two neighboring cases go to the same destination, merge them 2646 // into a single case. 2647 if ((nextValue - currentValue == 1) && (currentBB == nextBB)) { 2648 I->High = J->High; 2649 I->ExtraWeight += J->ExtraWeight; 2650 J = Cases.erase(J); 2651 } else { 2652 I = J++; 2653 } 2654 } 2655 2656 for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) { 2657 if (I->Low != I->High) 2658 // A range counts double, since it requires two compares. 2659 ++numCmps; 2660 } 2661 2662 return numCmps; 2663} 2664 2665void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, 2666 MachineBasicBlock *Last) { 2667 // Update JTCases. 2668 for (unsigned i = 0, e = JTCases.size(); i != e; ++i) 2669 if (JTCases[i].first.HeaderBB == First) 2670 JTCases[i].first.HeaderBB = Last; 2671 2672 // Update BitTestCases. 2673 for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) 2674 if (BitTestCases[i].Parent == First) 2675 BitTestCases[i].Parent = Last; 2676} 2677 2678void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { 2679 MachineBasicBlock *SwitchMBB = FuncInfo.MBB; 2680 2681 // Figure out which block is immediately after the current one. 2682 MachineBasicBlock *NextBlock = nullptr; 2683 MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()]; 2684 2685 // If there is only the default destination, branch to it if it is not the 2686 // next basic block. Otherwise, just fall through. 2687 if (!SI.getNumCases()) { 2688 // Update machine-CFG edges. 2689 2690 // If this is not a fall-through branch, emit the branch. 2691 SwitchMBB->addSuccessor(Default); 2692 if (Default != NextBlock) 2693 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 2694 MVT::Other, getControlRoot(), 2695 DAG.getBasicBlock(Default))); 2696 2697 return; 2698 } 2699 2700 // If there are any non-default case statements, create a vector of Cases 2701 // representing each one, and sort the vector so that we can efficiently 2702 // create a binary search tree from them. 2703 CaseVector Cases; 2704 size_t numCmps = Clusterify(Cases, SI); 2705 DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size() 2706 << ". Total compares: " << numCmps << '\n'); 2707 (void)numCmps; 2708 2709 // Get the Value to be switched on and default basic blocks, which will be 2710 // inserted into CaseBlock records, representing basic blocks in the binary 2711 // search tree. 2712 const Value *SV = SI.getCondition(); 2713 2714 // Push the initial CaseRec onto the worklist 2715 CaseRecVector WorkList; 2716 WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr, 2717 CaseRange(Cases.begin(),Cases.end()))); 2718 2719 while (!WorkList.empty()) { 2720 // Grab a record representing a case range to process off the worklist 2721 CaseRec CR = WorkList.back(); 2722 WorkList.pop_back(); 2723 2724 if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB)) 2725 continue; 2726 2727 // If the range has few cases (two or less) emit a series of specific 2728 // tests. 2729 if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB)) 2730 continue; 2731 2732 // If the switch has more than N blocks, and is at least 40% dense, and the 2733 // target supports indirect branches, then emit a jump table rather than 2734 // lowering the switch to a binary tree of conditional branches. 2735 // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries(). 2736 if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB)) 2737 continue; 2738 2739 // Emit binary tree. We need to pick a pivot, and push left and right ranges 2740 // onto the worklist. Leafs are handled via handleSmallSwitchRange() call. 2741 handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB); 2742 } 2743} 2744 2745void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { 2746 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB; 2747 2748 // Update machine-CFG edges with unique successors. 2749 SmallSet<BasicBlock*, 32> Done; 2750 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) { 2751 BasicBlock *BB = I.getSuccessor(i); 2752 bool Inserted = Done.insert(BB); 2753 if (!Inserted) 2754 continue; 2755 2756 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB]; 2757 addSuccessorWithWeight(IndirectBrMBB, Succ); 2758 } 2759 2760 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), 2761 MVT::Other, getControlRoot(), 2762 getValue(I.getAddress()))); 2763} 2764 2765void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) { 2766 if (DAG.getTarget().Options.TrapUnreachable) 2767 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot())); 2768} 2769 2770void SelectionDAGBuilder::visitFSub(const User &I) { 2771 // -0.0 - X --> fneg 2772 Type *Ty = I.getType(); 2773 if (isa<Constant>(I.getOperand(0)) && 2774 I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) { 2775 SDValue Op2 = getValue(I.getOperand(1)); 2776 setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(), 2777 Op2.getValueType(), Op2)); 2778 return; 2779 } 2780 2781 visitBinary(I, ISD::FSUB); 2782} 2783 2784void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) { 2785 SDValue Op1 = getValue(I.getOperand(0)); 2786 SDValue Op2 = getValue(I.getOperand(1)); 2787 setValue(&I, DAG.getNode(OpCode, getCurSDLoc(), 2788 Op1.getValueType(), Op1, Op2)); 2789} 2790 2791void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { 2792 SDValue Op1 = getValue(I.getOperand(0)); 2793 SDValue Op2 = getValue(I.getOperand(1)); 2794 2795 EVT ShiftTy = TM.getTargetLowering()->getShiftAmountTy(Op2.getValueType()); 2796 2797 // Coerce the shift amount to the right type if we can. 2798 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) { 2799 unsigned ShiftSize = ShiftTy.getSizeInBits(); 2800 unsigned Op2Size = Op2.getValueType().getSizeInBits(); 2801 SDLoc DL = getCurSDLoc(); 2802 2803 // If the operand is smaller than the shift count type, promote it. 2804 if (ShiftSize > Op2Size) 2805 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2); 2806 2807 // If the operand is larger than the shift count type but the shift 2808 // count type has enough bits to represent any shift value, truncate 2809 // it now. This is a common case and it exposes the truncate to 2810 // optimization early. 2811 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits())) 2812 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2); 2813 // Otherwise we'll need to temporarily settle for some other convenient 2814 // type. Type legalization will make adjustments once the shiftee is split. 2815 else 2816 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32); 2817 } 2818 2819 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), 2820 Op1.getValueType(), Op1, Op2)); 2821} 2822 2823void SelectionDAGBuilder::visitSDiv(const User &I) { 2824 SDValue Op1 = getValue(I.getOperand(0)); 2825 SDValue Op2 = getValue(I.getOperand(1)); 2826 2827 // Turn exact SDivs into multiplications. 2828 // FIXME: This should be in DAGCombiner, but it doesn't have access to the 2829 // exact bit. 2830 if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() && 2831 !isa<ConstantSDNode>(Op1) && 2832 isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue()) 2833 setValue(&I, TM.getTargetLowering()->BuildExactSDIV(Op1, Op2, 2834 getCurSDLoc(), DAG)); 2835 else 2836 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), 2837 Op1, Op2)); 2838} 2839 2840void SelectionDAGBuilder::visitICmp(const User &I) { 2841 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; 2842 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I)) 2843 predicate = IC->getPredicate(); 2844 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) 2845 predicate = ICmpInst::Predicate(IC->getPredicate()); 2846 SDValue Op1 = getValue(I.getOperand(0)); 2847 SDValue Op2 = getValue(I.getOperand(1)); 2848 ISD::CondCode Opcode = getICmpCondCode(predicate); 2849 2850 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2851 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); 2852} 2853 2854void SelectionDAGBuilder::visitFCmp(const User &I) { 2855 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; 2856 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I)) 2857 predicate = FC->getPredicate(); 2858 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) 2859 predicate = FCmpInst::Predicate(FC->getPredicate()); 2860 SDValue Op1 = getValue(I.getOperand(0)); 2861 SDValue Op2 = getValue(I.getOperand(1)); 2862 ISD::CondCode Condition = getFCmpCondCode(predicate); 2863 if (TM.Options.NoNaNsFPMath) 2864 Condition = getFCmpCodeWithoutNaN(Condition); 2865 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2866 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition)); 2867} 2868 2869void SelectionDAGBuilder::visitSelect(const User &I) { 2870 SmallVector<EVT, 4> ValueVTs; 2871 ComputeValueVTs(*TM.getTargetLowering(), I.getType(), ValueVTs); 2872 unsigned NumValues = ValueVTs.size(); 2873 if (NumValues == 0) return; 2874 2875 SmallVector<SDValue, 4> Values(NumValues); 2876 SDValue Cond = getValue(I.getOperand(0)); 2877 SDValue TrueVal = getValue(I.getOperand(1)); 2878 SDValue FalseVal = getValue(I.getOperand(2)); 2879 ISD::NodeType OpCode = Cond.getValueType().isVector() ? 2880 ISD::VSELECT : ISD::SELECT; 2881 2882 for (unsigned i = 0; i != NumValues; ++i) 2883 Values[i] = DAG.getNode(OpCode, getCurSDLoc(), 2884 TrueVal.getNode()->getValueType(TrueVal.getResNo()+i), 2885 Cond, 2886 SDValue(TrueVal.getNode(), 2887 TrueVal.getResNo() + i), 2888 SDValue(FalseVal.getNode(), 2889 FalseVal.getResNo() + i)); 2890 2891 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 2892 DAG.getVTList(ValueVTs), Values)); 2893} 2894 2895void SelectionDAGBuilder::visitTrunc(const User &I) { 2896 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). 2897 SDValue N = getValue(I.getOperand(0)); 2898 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2899 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N)); 2900} 2901 2902void SelectionDAGBuilder::visitZExt(const User &I) { 2903 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 2904 // ZExt also can't be a cast to bool for same reason. So, nothing much to do 2905 SDValue N = getValue(I.getOperand(0)); 2906 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2907 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N)); 2908} 2909 2910void SelectionDAGBuilder::visitSExt(const User &I) { 2911 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 2912 // SExt also can't be a cast to bool for same reason. So, nothing much to do 2913 SDValue N = getValue(I.getOperand(0)); 2914 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2915 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); 2916} 2917 2918void SelectionDAGBuilder::visitFPTrunc(const User &I) { 2919 // FPTrunc is never a no-op cast, no need to check 2920 SDValue N = getValue(I.getOperand(0)); 2921 const TargetLowering *TLI = TM.getTargetLowering(); 2922 EVT DestVT = TLI->getValueType(I.getType()); 2923 setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(), 2924 DestVT, N, 2925 DAG.getTargetConstant(0, TLI->getPointerTy()))); 2926} 2927 2928void SelectionDAGBuilder::visitFPExt(const User &I) { 2929 // FPExt is never a no-op cast, no need to check 2930 SDValue N = getValue(I.getOperand(0)); 2931 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2932 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N)); 2933} 2934 2935void SelectionDAGBuilder::visitFPToUI(const User &I) { 2936 // FPToUI is never a no-op cast, no need to check 2937 SDValue N = getValue(I.getOperand(0)); 2938 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2939 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N)); 2940} 2941 2942void SelectionDAGBuilder::visitFPToSI(const User &I) { 2943 // FPToSI is never a no-op cast, no need to check 2944 SDValue N = getValue(I.getOperand(0)); 2945 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2946 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N)); 2947} 2948 2949void SelectionDAGBuilder::visitUIToFP(const User &I) { 2950 // UIToFP is never a no-op cast, no need to check 2951 SDValue N = getValue(I.getOperand(0)); 2952 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2953 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N)); 2954} 2955 2956void SelectionDAGBuilder::visitSIToFP(const User &I) { 2957 // SIToFP is never a no-op cast, no need to check 2958 SDValue N = getValue(I.getOperand(0)); 2959 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2960 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N)); 2961} 2962 2963void SelectionDAGBuilder::visitPtrToInt(const User &I) { 2964 // What to do depends on the size of the integer and the size of the pointer. 2965 // We can either truncate, zero extend, or no-op, accordingly. 2966 SDValue N = getValue(I.getOperand(0)); 2967 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2968 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT)); 2969} 2970 2971void SelectionDAGBuilder::visitIntToPtr(const User &I) { 2972 // What to do depends on the size of the integer and the size of the pointer. 2973 // We can either truncate, zero extend, or no-op, accordingly. 2974 SDValue N = getValue(I.getOperand(0)); 2975 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2976 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT)); 2977} 2978 2979void SelectionDAGBuilder::visitBitCast(const User &I) { 2980 SDValue N = getValue(I.getOperand(0)); 2981 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2982 2983 // BitCast assures us that source and destination are the same size so this is 2984 // either a BITCAST or a no-op. 2985 if (DestVT != N.getValueType()) 2986 setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(), 2987 DestVT, N)); // convert types. 2988 // Check if the original LLVM IR Operand was a ConstantInt, because getValue() 2989 // might fold any kind of constant expression to an integer constant and that 2990 // is not what we are looking for. Only regcognize a bitcast of a genuine 2991 // constant integer as an opaque constant. 2992 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0))) 2993 setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false, 2994 /*isOpaque*/true)); 2995 else 2996 setValue(&I, N); // noop cast. 2997} 2998 2999void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) { 3000 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3001 const Value *SV = I.getOperand(0); 3002 SDValue N = getValue(SV); 3003 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 3004 3005 unsigned SrcAS = SV->getType()->getPointerAddressSpace(); 3006 unsigned DestAS = I.getType()->getPointerAddressSpace(); 3007 3008 if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS)) 3009 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS); 3010 3011 setValue(&I, N); 3012} 3013 3014void SelectionDAGBuilder::visitInsertElement(const User &I) { 3015 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3016 SDValue InVec = getValue(I.getOperand(0)); 3017 SDValue InVal = getValue(I.getOperand(1)); 3018 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), 3019 getCurSDLoc(), TLI.getVectorIdxTy()); 3020 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(), 3021 TM.getTargetLowering()->getValueType(I.getType()), 3022 InVec, InVal, InIdx)); 3023} 3024 3025void SelectionDAGBuilder::visitExtractElement(const User &I) { 3026 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3027 SDValue InVec = getValue(I.getOperand(0)); 3028 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), 3029 getCurSDLoc(), TLI.getVectorIdxTy()); 3030 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), 3031 TM.getTargetLowering()->getValueType(I.getType()), 3032 InVec, InIdx)); 3033} 3034 3035// Utility for visitShuffleVector - Return true if every element in Mask, 3036// beginning from position Pos and ending in Pos+Size, falls within the 3037// specified sequential range [L, L+Pos). or is undef. 3038static bool isSequentialInRange(const SmallVectorImpl<int> &Mask, 3039 unsigned Pos, unsigned Size, int Low) { 3040 for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low) 3041 if (Mask[i] >= 0 && Mask[i] != Low) 3042 return false; 3043 return true; 3044} 3045 3046void SelectionDAGBuilder::visitShuffleVector(const User &I) { 3047 SDValue Src1 = getValue(I.getOperand(0)); 3048 SDValue Src2 = getValue(I.getOperand(1)); 3049 3050 SmallVector<int, 8> Mask; 3051 ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask); 3052 unsigned MaskNumElts = Mask.size(); 3053 3054 const TargetLowering *TLI = TM.getTargetLowering(); 3055 EVT VT = TLI->getValueType(I.getType()); 3056 EVT SrcVT = Src1.getValueType(); 3057 unsigned SrcNumElts = SrcVT.getVectorNumElements(); 3058 3059 if (SrcNumElts == MaskNumElts) { 3060 setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3061 &Mask[0])); 3062 return; 3063 } 3064 3065 // Normalize the shuffle vector since mask and vector length don't match. 3066 if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) { 3067 // Mask is longer than the source vectors and is a multiple of the source 3068 // vectors. We can use concatenate vector to make the mask and vectors 3069 // lengths match. 3070 if (SrcNumElts*2 == MaskNumElts) { 3071 // First check for Src1 in low and Src2 in high 3072 if (isSequentialInRange(Mask, 0, SrcNumElts, 0) && 3073 isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) { 3074 // The shuffle is concatenating two vectors together. 3075 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(), 3076 VT, Src1, Src2)); 3077 return; 3078 } 3079 // Then check for Src2 in low and Src1 in high 3080 if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) && 3081 isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) { 3082 // The shuffle is concatenating two vectors together. 3083 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(), 3084 VT, Src2, Src1)); 3085 return; 3086 } 3087 } 3088 3089 // Pad both vectors with undefs to make them the same length as the mask. 3090 unsigned NumConcat = MaskNumElts / SrcNumElts; 3091 bool Src1U = Src1.getOpcode() == ISD::UNDEF; 3092 bool Src2U = Src2.getOpcode() == ISD::UNDEF; 3093 SDValue UndefVal = DAG.getUNDEF(SrcVT); 3094 3095 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal); 3096 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal); 3097 MOps1[0] = Src1; 3098 MOps2[0] = Src2; 3099 3100 Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, 3101 getCurSDLoc(), VT, MOps1); 3102 Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, 3103 getCurSDLoc(), VT, MOps2); 3104 3105 // Readjust mask for new input vector length. 3106 SmallVector<int, 8> MappedOps; 3107 for (unsigned i = 0; i != MaskNumElts; ++i) { 3108 int Idx = Mask[i]; 3109 if (Idx >= (int)SrcNumElts) 3110 Idx -= SrcNumElts - MaskNumElts; 3111 MappedOps.push_back(Idx); 3112 } 3113 3114 setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3115 &MappedOps[0])); 3116 return; 3117 } 3118 3119 if (SrcNumElts > MaskNumElts) { 3120 // Analyze the access pattern of the vector to see if we can extract 3121 // two subvectors and do the shuffle. The analysis is done by calculating 3122 // the range of elements the mask access on both vectors. 3123 int MinRange[2] = { static_cast<int>(SrcNumElts), 3124 static_cast<int>(SrcNumElts)}; 3125 int MaxRange[2] = {-1, -1}; 3126 3127 for (unsigned i = 0; i != MaskNumElts; ++i) { 3128 int Idx = Mask[i]; 3129 unsigned Input = 0; 3130 if (Idx < 0) 3131 continue; 3132 3133 if (Idx >= (int)SrcNumElts) { 3134 Input = 1; 3135 Idx -= SrcNumElts; 3136 } 3137 if (Idx > MaxRange[Input]) 3138 MaxRange[Input] = Idx; 3139 if (Idx < MinRange[Input]) 3140 MinRange[Input] = Idx; 3141 } 3142 3143 // Check if the access is smaller than the vector size and can we find 3144 // a reasonable extract index. 3145 int RangeUse[2] = { -1, -1 }; // 0 = Unused, 1 = Extract, -1 = Can not 3146 // Extract. 3147 int StartIdx[2]; // StartIdx to extract from 3148 for (unsigned Input = 0; Input < 2; ++Input) { 3149 if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) { 3150 RangeUse[Input] = 0; // Unused 3151 StartIdx[Input] = 0; 3152 continue; 3153 } 3154 3155 // Find a good start index that is a multiple of the mask length. Then 3156 // see if the rest of the elements are in range. 3157 StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts; 3158 if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts && 3159 StartIdx[Input] + MaskNumElts <= SrcNumElts) 3160 RangeUse[Input] = 1; // Extract from a multiple of the mask length. 3161 } 3162 3163 if (RangeUse[0] == 0 && RangeUse[1] == 0) { 3164 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. 3165 return; 3166 } 3167 if (RangeUse[0] >= 0 && RangeUse[1] >= 0) { 3168 // Extract appropriate subvector and generate a vector shuffle 3169 for (unsigned Input = 0; Input < 2; ++Input) { 3170 SDValue &Src = Input == 0 ? Src1 : Src2; 3171 if (RangeUse[Input] == 0) 3172 Src = DAG.getUNDEF(VT); 3173 else 3174 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT, 3175 Src, DAG.getConstant(StartIdx[Input], 3176 TLI->getVectorIdxTy())); 3177 } 3178 3179 // Calculate new mask. 3180 SmallVector<int, 8> MappedOps; 3181 for (unsigned i = 0; i != MaskNumElts; ++i) { 3182 int Idx = Mask[i]; 3183 if (Idx >= 0) { 3184 if (Idx < (int)SrcNumElts) 3185 Idx -= StartIdx[0]; 3186 else 3187 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts; 3188 } 3189 MappedOps.push_back(Idx); 3190 } 3191 3192 setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3193 &MappedOps[0])); 3194 return; 3195 } 3196 } 3197 3198 // We can't use either concat vectors or extract subvectors so fall back to 3199 // replacing the shuffle with extract and build vector. 3200 // to insert and build vector. 3201 EVT EltVT = VT.getVectorElementType(); 3202 EVT IdxVT = TLI->getVectorIdxTy(); 3203 SmallVector<SDValue,8> Ops; 3204 for (unsigned i = 0; i != MaskNumElts; ++i) { 3205 int Idx = Mask[i]; 3206 SDValue Res; 3207 3208 if (Idx < 0) { 3209 Res = DAG.getUNDEF(EltVT); 3210 } else { 3211 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2; 3212 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts; 3213 3214 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), 3215 EltVT, Src, DAG.getConstant(Idx, IdxVT)); 3216 } 3217 3218 Ops.push_back(Res); 3219 } 3220 3221 setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops)); 3222} 3223 3224void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { 3225 const Value *Op0 = I.getOperand(0); 3226 const Value *Op1 = I.getOperand(1); 3227 Type *AggTy = I.getType(); 3228 Type *ValTy = Op1->getType(); 3229 bool IntoUndef = isa<UndefValue>(Op0); 3230 bool FromUndef = isa<UndefValue>(Op1); 3231 3232 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); 3233 3234 const TargetLowering *TLI = TM.getTargetLowering(); 3235 SmallVector<EVT, 4> AggValueVTs; 3236 ComputeValueVTs(*TLI, AggTy, AggValueVTs); 3237 SmallVector<EVT, 4> ValValueVTs; 3238 ComputeValueVTs(*TLI, ValTy, ValValueVTs); 3239 3240 unsigned NumAggValues = AggValueVTs.size(); 3241 unsigned NumValValues = ValValueVTs.size(); 3242 SmallVector<SDValue, 4> Values(NumAggValues); 3243 3244 SDValue Agg = getValue(Op0); 3245 unsigned i = 0; 3246 // Copy the beginning value(s) from the original aggregate. 3247 for (; i != LinearIndex; ++i) 3248 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3249 SDValue(Agg.getNode(), Agg.getResNo() + i); 3250 // Copy values from the inserted value(s). 3251 if (NumValValues) { 3252 SDValue Val = getValue(Op1); 3253 for (; i != LinearIndex + NumValValues; ++i) 3254 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3255 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); 3256 } 3257 // Copy remaining value(s) from the original aggregate. 3258 for (; i != NumAggValues; ++i) 3259 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3260 SDValue(Agg.getNode(), Agg.getResNo() + i); 3261 3262 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3263 DAG.getVTList(AggValueVTs), Values)); 3264} 3265 3266void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { 3267 const Value *Op0 = I.getOperand(0); 3268 Type *AggTy = Op0->getType(); 3269 Type *ValTy = I.getType(); 3270 bool OutOfUndef = isa<UndefValue>(Op0); 3271 3272 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); 3273 3274 const TargetLowering *TLI = TM.getTargetLowering(); 3275 SmallVector<EVT, 4> ValValueVTs; 3276 ComputeValueVTs(*TLI, ValTy, ValValueVTs); 3277 3278 unsigned NumValValues = ValValueVTs.size(); 3279 3280 // Ignore a extractvalue that produces an empty object 3281 if (!NumValValues) { 3282 setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); 3283 return; 3284 } 3285 3286 SmallVector<SDValue, 4> Values(NumValValues); 3287 3288 SDValue Agg = getValue(Op0); 3289 // Copy out the selected value(s). 3290 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) 3291 Values[i - LinearIndex] = 3292 OutOfUndef ? 3293 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : 3294 SDValue(Agg.getNode(), Agg.getResNo() + i); 3295 3296 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3297 DAG.getVTList(ValValueVTs), Values)); 3298} 3299 3300void SelectionDAGBuilder::visitGetElementPtr(const User &I) { 3301 Value *Op0 = I.getOperand(0); 3302 // Note that the pointer operand may be a vector of pointers. Take the scalar 3303 // element which holds a pointer. 3304 Type *Ty = Op0->getType()->getScalarType(); 3305 unsigned AS = Ty->getPointerAddressSpace(); 3306 SDValue N = getValue(Op0); 3307 3308 for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end(); 3309 OI != E; ++OI) { 3310 const Value *Idx = *OI; 3311 if (StructType *StTy = dyn_cast<StructType>(Ty)) { 3312 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue(); 3313 if (Field) { 3314 // N = N + Offset 3315 uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field); 3316 N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, 3317 DAG.getConstant(Offset, N.getValueType())); 3318 } 3319 3320 Ty = StTy->getElementType(Field); 3321 } else { 3322 Ty = cast<SequentialType>(Ty)->getElementType(); 3323 3324 // If this is a constant subscript, handle it quickly. 3325 const TargetLowering *TLI = TM.getTargetLowering(); 3326 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { 3327 if (CI->isZero()) continue; 3328 uint64_t Offs = 3329 DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue(); 3330 SDValue OffsVal; 3331 EVT PTy = TLI->getPointerTy(AS); 3332 unsigned PtrBits = PTy.getSizeInBits(); 3333 if (PtrBits < 64) 3334 OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy, 3335 DAG.getConstant(Offs, MVT::i64)); 3336 else 3337 OffsVal = DAG.getConstant(Offs, PTy); 3338 3339 N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, 3340 OffsVal); 3341 continue; 3342 } 3343 3344 // N = N + Idx * ElementSize; 3345 APInt ElementSize = APInt(TLI->getPointerSizeInBits(AS), 3346 DL->getTypeAllocSize(Ty)); 3347 SDValue IdxN = getValue(Idx); 3348 3349 // If the index is smaller or larger than intptr_t, truncate or extend 3350 // it. 3351 IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType()); 3352 3353 // If this is a multiply by a power of two, turn it into a shl 3354 // immediately. This is a very common case. 3355 if (ElementSize != 1) { 3356 if (ElementSize.isPowerOf2()) { 3357 unsigned Amt = ElementSize.logBase2(); 3358 IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(), 3359 N.getValueType(), IdxN, 3360 DAG.getConstant(Amt, IdxN.getValueType())); 3361 } else { 3362 SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType()); 3363 IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(), 3364 N.getValueType(), IdxN, Scale); 3365 } 3366 } 3367 3368 N = DAG.getNode(ISD::ADD, getCurSDLoc(), 3369 N.getValueType(), N, IdxN); 3370 } 3371 } 3372 3373 setValue(&I, N); 3374} 3375 3376void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { 3377 // If this is a fixed sized alloca in the entry block of the function, 3378 // allocate it statically on the stack. 3379 if (FuncInfo.StaticAllocaMap.count(&I)) 3380 return; // getValue will auto-populate this. 3381 3382 Type *Ty = I.getAllocatedType(); 3383 const TargetLowering *TLI = TM.getTargetLowering(); 3384 uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty); 3385 unsigned Align = 3386 std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), 3387 I.getAlignment()); 3388 3389 SDValue AllocSize = getValue(I.getArraySize()); 3390 3391 EVT IntPtr = TLI->getPointerTy(); 3392 if (AllocSize.getValueType() != IntPtr) 3393 AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr); 3394 3395 AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr, 3396 AllocSize, 3397 DAG.getConstant(TySize, IntPtr)); 3398 3399 // Handle alignment. If the requested alignment is less than or equal to 3400 // the stack alignment, ignore it. If the size is greater than or equal to 3401 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. 3402 unsigned StackAlign = TM.getFrameLowering()->getStackAlignment(); 3403 if (Align <= StackAlign) 3404 Align = 0; 3405 3406 // Round the size of the allocation up to the stack alignment size 3407 // by add SA-1 to the size. 3408 AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(), 3409 AllocSize.getValueType(), AllocSize, 3410 DAG.getIntPtrConstant(StackAlign-1)); 3411 3412 // Mask out the low bits for alignment purposes. 3413 AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(), 3414 AllocSize.getValueType(), AllocSize, 3415 DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); 3416 3417 SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; 3418 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); 3419 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops); 3420 setValue(&I, DSA); 3421 DAG.setRoot(DSA.getValue(1)); 3422 3423 assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects()); 3424} 3425 3426void SelectionDAGBuilder::visitLoad(const LoadInst &I) { 3427 if (I.isAtomic()) 3428 return visitAtomicLoad(I); 3429 3430 const Value *SV = I.getOperand(0); 3431 SDValue Ptr = getValue(SV); 3432 3433 Type *Ty = I.getType(); 3434 3435 bool isVolatile = I.isVolatile(); 3436 bool isNonTemporal = I.getMetadata("nontemporal") != nullptr; 3437 bool isInvariant = I.getMetadata("invariant.load") != nullptr; 3438 unsigned Alignment = I.getAlignment(); 3439 const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa); 3440 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); 3441 3442 SmallVector<EVT, 4> ValueVTs; 3443 SmallVector<uint64_t, 4> Offsets; 3444 ComputeValueVTs(*TM.getTargetLowering(), Ty, ValueVTs, &Offsets); 3445 unsigned NumValues = ValueVTs.size(); 3446 if (NumValues == 0) 3447 return; 3448 3449 SDValue Root; 3450 bool ConstantMemory = false; 3451 if (isVolatile || NumValues > MaxParallelChains) 3452 // Serialize volatile loads with other side effects. 3453 Root = getRoot(); 3454 else if (AA->pointsToConstantMemory( 3455 AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), TBAAInfo))) { 3456 // Do not serialize (non-volatile) loads of constant memory with anything. 3457 Root = DAG.getEntryNode(); 3458 ConstantMemory = true; 3459 } else { 3460 // Do not serialize non-volatile loads against each other. 3461 Root = DAG.getRoot(); 3462 } 3463 3464 const TargetLowering *TLI = TM.getTargetLowering(); 3465 if (isVolatile) 3466 Root = TLI->prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG); 3467 3468 SmallVector<SDValue, 4> Values(NumValues); 3469 SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains), 3470 NumValues)); 3471 EVT PtrVT = Ptr.getValueType(); 3472 unsigned ChainI = 0; 3473 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { 3474 // Serializing loads here may result in excessive register pressure, and 3475 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling 3476 // could recover a bit by hoisting nodes upward in the chain by recognizing 3477 // they are side-effect free or do not alias. The optimizer should really 3478 // avoid this case by converting large object/array copies to llvm.memcpy 3479 // (MaxParallelChains should always remain as failsafe). 3480 if (ChainI == MaxParallelChains) { 3481 assert(PendingLoads.empty() && "PendingLoads must be serialized first"); 3482 SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 3483 makeArrayRef(Chains.data(), ChainI)); 3484 Root = Chain; 3485 ChainI = 0; 3486 } 3487 SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(), 3488 PtrVT, Ptr, 3489 DAG.getConstant(Offsets[i], PtrVT)); 3490 SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root, 3491 A, MachinePointerInfo(SV, Offsets[i]), isVolatile, 3492 isNonTemporal, isInvariant, Alignment, TBAAInfo, 3493 Ranges); 3494 3495 Values[i] = L; 3496 Chains[ChainI] = L.getValue(1); 3497 } 3498 3499 if (!ConstantMemory) { 3500 SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 3501 makeArrayRef(Chains.data(), ChainI)); 3502 if (isVolatile) 3503 DAG.setRoot(Chain); 3504 else 3505 PendingLoads.push_back(Chain); 3506 } 3507 3508 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3509 DAG.getVTList(ValueVTs), Values)); 3510} 3511 3512void SelectionDAGBuilder::visitStore(const StoreInst &I) { 3513 if (I.isAtomic()) 3514 return visitAtomicStore(I); 3515 3516 const Value *SrcV = I.getOperand(0); 3517 const Value *PtrV = I.getOperand(1); 3518 3519 SmallVector<EVT, 4> ValueVTs; 3520 SmallVector<uint64_t, 4> Offsets; 3521 ComputeValueVTs(*TM.getTargetLowering(), SrcV->getType(), ValueVTs, &Offsets); 3522 unsigned NumValues = ValueVTs.size(); 3523 if (NumValues == 0) 3524 return; 3525 3526 // Get the lowered operands. Note that we do this after 3527 // checking if NumResults is zero, because with zero results 3528 // the operands won't have values in the map. 3529 SDValue Src = getValue(SrcV); 3530 SDValue Ptr = getValue(PtrV); 3531 3532 SDValue Root = getRoot(); 3533 SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains), 3534 NumValues)); 3535 EVT PtrVT = Ptr.getValueType(); 3536 bool isVolatile = I.isVolatile(); 3537 bool isNonTemporal = I.getMetadata("nontemporal") != nullptr; 3538 unsigned Alignment = I.getAlignment(); 3539 const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa); 3540 3541 unsigned ChainI = 0; 3542 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { 3543 // See visitLoad comments. 3544 if (ChainI == MaxParallelChains) { 3545 SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 3546 makeArrayRef(Chains.data(), ChainI)); 3547 Root = Chain; 3548 ChainI = 0; 3549 } 3550 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr, 3551 DAG.getConstant(Offsets[i], PtrVT)); 3552 SDValue St = DAG.getStore(Root, getCurSDLoc(), 3553 SDValue(Src.getNode(), Src.getResNo() + i), 3554 Add, MachinePointerInfo(PtrV, Offsets[i]), 3555 isVolatile, isNonTemporal, Alignment, TBAAInfo); 3556 Chains[ChainI] = St; 3557 } 3558 3559 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 3560 makeArrayRef(Chains.data(), ChainI)); 3561 DAG.setRoot(StoreNode); 3562} 3563 3564static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order, 3565 SynchronizationScope Scope, 3566 bool Before, SDLoc dl, 3567 SelectionDAG &DAG, 3568 const TargetLowering &TLI) { 3569 // Fence, if necessary 3570 if (Before) { 3571 if (Order == AcquireRelease || Order == SequentiallyConsistent) 3572 Order = Release; 3573 else if (Order == Acquire || Order == Monotonic) 3574 return Chain; 3575 } else { 3576 if (Order == AcquireRelease) 3577 Order = Acquire; 3578 else if (Order == Release || Order == Monotonic) 3579 return Chain; 3580 } 3581 SDValue Ops[3]; 3582 Ops[0] = Chain; 3583 Ops[1] = DAG.getConstant(Order, TLI.getPointerTy()); 3584 Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy()); 3585 return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops); 3586} 3587 3588void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { 3589 SDLoc dl = getCurSDLoc(); 3590 AtomicOrdering SuccessOrder = I.getSuccessOrdering(); 3591 AtomicOrdering FailureOrder = I.getFailureOrdering(); 3592 SynchronizationScope Scope = I.getSynchScope(); 3593 3594 SDValue InChain = getRoot(); 3595 3596 const TargetLowering *TLI = TM.getTargetLowering(); 3597 if (TLI->getInsertFencesForAtomic()) 3598 InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl, 3599 DAG, *TLI); 3600 3601 SDValue L = 3602 DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, 3603 getValue(I.getCompareOperand()).getSimpleValueType(), 3604 InChain, 3605 getValue(I.getPointerOperand()), 3606 getValue(I.getCompareOperand()), 3607 getValue(I.getNewValOperand()), 3608 MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */, 3609 TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder, 3610 TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder, 3611 Scope); 3612 3613 SDValue OutChain = L.getValue(1); 3614 3615 if (TLI->getInsertFencesForAtomic()) 3616 OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl, 3617 DAG, *TLI); 3618 3619 setValue(&I, L); 3620 DAG.setRoot(OutChain); 3621} 3622 3623void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { 3624 SDLoc dl = getCurSDLoc(); 3625 ISD::NodeType NT; 3626 switch (I.getOperation()) { 3627 default: llvm_unreachable("Unknown atomicrmw operation"); 3628 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; 3629 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; 3630 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; 3631 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; 3632 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; 3633 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; 3634 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; 3635 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; 3636 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; 3637 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; 3638 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; 3639 } 3640 AtomicOrdering Order = I.getOrdering(); 3641 SynchronizationScope Scope = I.getSynchScope(); 3642 3643 SDValue InChain = getRoot(); 3644 3645 const TargetLowering *TLI = TM.getTargetLowering(); 3646 if (TLI->getInsertFencesForAtomic()) 3647 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, 3648 DAG, *TLI); 3649 3650 SDValue L = 3651 DAG.getAtomic(NT, dl, 3652 getValue(I.getValOperand()).getSimpleValueType(), 3653 InChain, 3654 getValue(I.getPointerOperand()), 3655 getValue(I.getValOperand()), 3656 I.getPointerOperand(), 0 /* Alignment */, 3657 TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3658 Scope); 3659 3660 SDValue OutChain = L.getValue(1); 3661 3662 if (TLI->getInsertFencesForAtomic()) 3663 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3664 DAG, *TLI); 3665 3666 setValue(&I, L); 3667 DAG.setRoot(OutChain); 3668} 3669 3670void SelectionDAGBuilder::visitFence(const FenceInst &I) { 3671 SDLoc dl = getCurSDLoc(); 3672 const TargetLowering *TLI = TM.getTargetLowering(); 3673 SDValue Ops[3]; 3674 Ops[0] = getRoot(); 3675 Ops[1] = DAG.getConstant(I.getOrdering(), TLI->getPointerTy()); 3676 Ops[2] = DAG.getConstant(I.getSynchScope(), TLI->getPointerTy()); 3677 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops)); 3678} 3679 3680void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { 3681 SDLoc dl = getCurSDLoc(); 3682 AtomicOrdering Order = I.getOrdering(); 3683 SynchronizationScope Scope = I.getSynchScope(); 3684 3685 SDValue InChain = getRoot(); 3686 3687 const TargetLowering *TLI = TM.getTargetLowering(); 3688 EVT VT = TLI->getValueType(I.getType()); 3689 3690 if (I.getAlignment() < VT.getSizeInBits() / 8) 3691 report_fatal_error("Cannot generate unaligned atomic load"); 3692 3693 MachineMemOperand *MMO = 3694 DAG.getMachineFunction(). 3695 getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), 3696 MachineMemOperand::MOVolatile | 3697 MachineMemOperand::MOLoad, 3698 VT.getStoreSize(), 3699 I.getAlignment() ? I.getAlignment() : 3700 DAG.getEVTAlignment(VT)); 3701 3702 InChain = TLI->prepareVolatileOrAtomicLoad(InChain, dl, DAG); 3703 SDValue L = 3704 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, 3705 getValue(I.getPointerOperand()), MMO, 3706 TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3707 Scope); 3708 3709 SDValue OutChain = L.getValue(1); 3710 3711 if (TLI->getInsertFencesForAtomic()) 3712 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3713 DAG, *TLI); 3714 3715 setValue(&I, L); 3716 DAG.setRoot(OutChain); 3717} 3718 3719void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { 3720 SDLoc dl = getCurSDLoc(); 3721 3722 AtomicOrdering Order = I.getOrdering(); 3723 SynchronizationScope Scope = I.getSynchScope(); 3724 3725 SDValue InChain = getRoot(); 3726 3727 const TargetLowering *TLI = TM.getTargetLowering(); 3728 EVT VT = TLI->getValueType(I.getValueOperand()->getType()); 3729 3730 if (I.getAlignment() < VT.getSizeInBits() / 8) 3731 report_fatal_error("Cannot generate unaligned atomic store"); 3732 3733 if (TLI->getInsertFencesForAtomic()) 3734 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, 3735 DAG, *TLI); 3736 3737 SDValue OutChain = 3738 DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, 3739 InChain, 3740 getValue(I.getPointerOperand()), 3741 getValue(I.getValueOperand()), 3742 I.getPointerOperand(), I.getAlignment(), 3743 TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3744 Scope); 3745 3746 if (TLI->getInsertFencesForAtomic()) 3747 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3748 DAG, *TLI); 3749 3750 DAG.setRoot(OutChain); 3751} 3752 3753/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC 3754/// node. 3755void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, 3756 unsigned Intrinsic) { 3757 bool HasChain = !I.doesNotAccessMemory(); 3758 bool OnlyLoad = HasChain && I.onlyReadsMemory(); 3759 3760 // Build the operand list. 3761 SmallVector<SDValue, 8> Ops; 3762 if (HasChain) { // If this intrinsic has side-effects, chainify it. 3763 if (OnlyLoad) { 3764 // We don't need to serialize loads against other loads. 3765 Ops.push_back(DAG.getRoot()); 3766 } else { 3767 Ops.push_back(getRoot()); 3768 } 3769 } 3770 3771 // Info is set by getTgtMemInstrinsic 3772 TargetLowering::IntrinsicInfo Info; 3773 const TargetLowering *TLI = TM.getTargetLowering(); 3774 bool IsTgtIntrinsic = TLI->getTgtMemIntrinsic(Info, I, Intrinsic); 3775 3776 // Add the intrinsic ID as an integer operand if it's not a target intrinsic. 3777 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID || 3778 Info.opc == ISD::INTRINSIC_W_CHAIN) 3779 Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI->getPointerTy())); 3780 3781 // Add all operands of the call to the operand list. 3782 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) { 3783 SDValue Op = getValue(I.getArgOperand(i)); 3784 Ops.push_back(Op); 3785 } 3786 3787 SmallVector<EVT, 4> ValueVTs; 3788 ComputeValueVTs(*TLI, I.getType(), ValueVTs); 3789 3790 if (HasChain) 3791 ValueVTs.push_back(MVT::Other); 3792 3793 SDVTList VTs = DAG.getVTList(ValueVTs); 3794 3795 // Create the node. 3796 SDValue Result; 3797 if (IsTgtIntrinsic) { 3798 // This is target intrinsic that touches memory 3799 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), 3800 VTs, Ops, Info.memVT, 3801 MachinePointerInfo(Info.ptrVal, Info.offset), 3802 Info.align, Info.vol, 3803 Info.readMem, Info.writeMem); 3804 } else if (!HasChain) { 3805 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); 3806 } else if (!I.getType()->isVoidTy()) { 3807 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops); 3808 } else { 3809 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops); 3810 } 3811 3812 if (HasChain) { 3813 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); 3814 if (OnlyLoad) 3815 PendingLoads.push_back(Chain); 3816 else 3817 DAG.setRoot(Chain); 3818 } 3819 3820 if (!I.getType()->isVoidTy()) { 3821 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) { 3822 EVT VT = TLI->getValueType(PTy); 3823 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result); 3824 } 3825 3826 setValue(&I, Result); 3827 } 3828} 3829 3830/// GetSignificand - Get the significand and build it into a floating-point 3831/// number with exponent of 1: 3832/// 3833/// Op = (Op & 0x007fffff) | 0x3f800000; 3834/// 3835/// where Op is the hexadecimal representation of floating point value. 3836static SDValue 3837GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) { 3838 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, 3839 DAG.getConstant(0x007fffff, MVT::i32)); 3840 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, 3841 DAG.getConstant(0x3f800000, MVT::i32)); 3842 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2); 3843} 3844 3845/// GetExponent - Get the exponent: 3846/// 3847/// (float)(int)(((Op & 0x7f800000) >> 23) - 127); 3848/// 3849/// where Op is the hexadecimal representation of floating point value. 3850static SDValue 3851GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, 3852 SDLoc dl) { 3853 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, 3854 DAG.getConstant(0x7f800000, MVT::i32)); 3855 SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0, 3856 DAG.getConstant(23, TLI.getPointerTy())); 3857 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, 3858 DAG.getConstant(127, MVT::i32)); 3859 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); 3860} 3861 3862/// getF32Constant - Get 32-bit floating point constant. 3863static SDValue 3864getF32Constant(SelectionDAG &DAG, unsigned Flt) { 3865 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)), 3866 MVT::f32); 3867} 3868 3869/// expandExp - Lower an exp intrinsic. Handles the special sequences for 3870/// limited-precision mode. 3871static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG, 3872 const TargetLowering &TLI) { 3873 if (Op.getValueType() == MVT::f32 && 3874 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 3875 3876 // Put the exponent in the right bit position for later addition to the 3877 // final result: 3878 // 3879 // #define LOG2OFe 1.4426950f 3880 // IntegerPartOfX = ((int32_t)(X * LOG2OFe)); 3881 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, 3882 getF32Constant(DAG, 0x3fb8aa3b)); 3883 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); 3884 3885 // FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX; 3886 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 3887 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); 3888 3889 // IntegerPartOfX <<= 23; 3890 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 3891 DAG.getConstant(23, TLI.getPointerTy())); 3892 3893 SDValue TwoToFracPartOfX; 3894 if (LimitFloatPrecision <= 6) { 3895 // For floating-point precision of 6: 3896 // 3897 // TwoToFractionalPartOfX = 3898 // 0.997535578f + 3899 // (0.735607626f + 0.252464424f * x) * x; 3900 // 3901 // error 0.0144103317, which is 6 bits 3902 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3903 getF32Constant(DAG, 0x3e814304)); 3904 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3905 getF32Constant(DAG, 0x3f3c50c8)); 3906 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3907 TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3908 getF32Constant(DAG, 0x3f7f5e7e)); 3909 } else if (LimitFloatPrecision <= 12) { 3910 // For floating-point precision of 12: 3911 // 3912 // TwoToFractionalPartOfX = 3913 // 0.999892986f + 3914 // (0.696457318f + 3915 // (0.224338339f + 0.792043434e-1f * x) * x) * x; 3916 // 3917 // 0.000107046256 error, which is 13 to 14 bits 3918 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3919 getF32Constant(DAG, 0x3da235e3)); 3920 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3921 getF32Constant(DAG, 0x3e65b8f3)); 3922 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3923 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3924 getF32Constant(DAG, 0x3f324b07)); 3925 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3926 TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 3927 getF32Constant(DAG, 0x3f7ff8fd)); 3928 } else { // LimitFloatPrecision <= 18 3929 // For floating-point precision of 18: 3930 // 3931 // TwoToFractionalPartOfX = 3932 // 0.999999982f + 3933 // (0.693148872f + 3934 // (0.240227044f + 3935 // (0.554906021e-1f + 3936 // (0.961591928e-2f + 3937 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 3938 // 3939 // error 2.47208000*10^(-7), which is better than 18 bits 3940 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3941 getF32Constant(DAG, 0x3924b03e)); 3942 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3943 getF32Constant(DAG, 0x3ab24b87)); 3944 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3945 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3946 getF32Constant(DAG, 0x3c1d8c17)); 3947 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3948 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 3949 getF32Constant(DAG, 0x3d634a1d)); 3950 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 3951 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 3952 getF32Constant(DAG, 0x3e75fe14)); 3953 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 3954 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 3955 getF32Constant(DAG, 0x3f317234)); 3956 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 3957 TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 3958 getF32Constant(DAG, 0x3f800000)); 3959 } 3960 3961 // Add the exponent into the result in integer domain. 3962 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX); 3963 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 3964 DAG.getNode(ISD::ADD, dl, MVT::i32, 3965 t13, IntegerPartOfX)); 3966 } 3967 3968 // No special expansion. 3969 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op); 3970} 3971 3972/// expandLog - Lower a log intrinsic. Handles the special sequences for 3973/// limited-precision mode. 3974static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG, 3975 const TargetLowering &TLI) { 3976 if (Op.getValueType() == MVT::f32 && 3977 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 3978 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 3979 3980 // Scale the exponent by log(2) [0.69314718f]. 3981 SDValue Exp = GetExponent(DAG, Op1, TLI, dl); 3982 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, 3983 getF32Constant(DAG, 0x3f317218)); 3984 3985 // Get the significand and build it into a floating-point number with 3986 // exponent of 1. 3987 SDValue X = GetSignificand(DAG, Op1, dl); 3988 3989 SDValue LogOfMantissa; 3990 if (LimitFloatPrecision <= 6) { 3991 // For floating-point precision of 6: 3992 // 3993 // LogofMantissa = 3994 // -1.1609546f + 3995 // (1.4034025f - 0.23903021f * x) * x; 3996 // 3997 // error 0.0034276066, which is better than 8 bits 3998 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3999 getF32Constant(DAG, 0xbe74c456)); 4000 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4001 getF32Constant(DAG, 0x3fb3a2b1)); 4002 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4003 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4004 getF32Constant(DAG, 0x3f949a29)); 4005 } else if (LimitFloatPrecision <= 12) { 4006 // For floating-point precision of 12: 4007 // 4008 // LogOfMantissa = 4009 // -1.7417939f + 4010 // (2.8212026f + 4011 // (-1.4699568f + 4012 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x; 4013 // 4014 // error 0.000061011436, which is 14 bits 4015 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4016 getF32Constant(DAG, 0xbd67b6d6)); 4017 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4018 getF32Constant(DAG, 0x3ee4f4b8)); 4019 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4020 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4021 getF32Constant(DAG, 0x3fbc278b)); 4022 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4023 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4024 getF32Constant(DAG, 0x40348e95)); 4025 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4026 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4027 getF32Constant(DAG, 0x3fdef31a)); 4028 } else { // LimitFloatPrecision <= 18 4029 // For floating-point precision of 18: 4030 // 4031 // LogOfMantissa = 4032 // -2.1072184f + 4033 // (4.2372794f + 4034 // (-3.7029485f + 4035 // (2.2781945f + 4036 // (-0.87823314f + 4037 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x; 4038 // 4039 // error 0.0000023660568, which is better than 18 bits 4040 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4041 getF32Constant(DAG, 0xbc91e5ac)); 4042 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4043 getF32Constant(DAG, 0x3e4350aa)); 4044 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4045 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4046 getF32Constant(DAG, 0x3f60d3e3)); 4047 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4048 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4049 getF32Constant(DAG, 0x4011cdf0)); 4050 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4051 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4052 getF32Constant(DAG, 0x406cfd1c)); 4053 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4054 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4055 getF32Constant(DAG, 0x408797cb)); 4056 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4057 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, 4058 getF32Constant(DAG, 0x4006dcab)); 4059 } 4060 4061 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); 4062 } 4063 4064 // No special expansion. 4065 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op); 4066} 4067 4068/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for 4069/// limited-precision mode. 4070static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4071 const TargetLowering &TLI) { 4072 if (Op.getValueType() == MVT::f32 && 4073 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4074 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 4075 4076 // Get the exponent. 4077 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); 4078 4079 // Get the significand and build it into a floating-point number with 4080 // exponent of 1. 4081 SDValue X = GetSignificand(DAG, Op1, dl); 4082 4083 // Different possible minimax approximations of significand in 4084 // floating-point for various degrees of accuracy over [1,2]. 4085 SDValue Log2ofMantissa; 4086 if (LimitFloatPrecision <= 6) { 4087 // For floating-point precision of 6: 4088 // 4089 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x; 4090 // 4091 // error 0.0049451742, which is more than 7 bits 4092 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4093 getF32Constant(DAG, 0xbeb08fe0)); 4094 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4095 getF32Constant(DAG, 0x40019463)); 4096 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4097 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4098 getF32Constant(DAG, 0x3fd6633d)); 4099 } else if (LimitFloatPrecision <= 12) { 4100 // For floating-point precision of 12: 4101 // 4102 // Log2ofMantissa = 4103 // -2.51285454f + 4104 // (4.07009056f + 4105 // (-2.12067489f + 4106 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x; 4107 // 4108 // error 0.0000876136000, which is better than 13 bits 4109 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4110 getF32Constant(DAG, 0xbda7262e)); 4111 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4112 getF32Constant(DAG, 0x3f25280b)); 4113 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4114 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4115 getF32Constant(DAG, 0x4007b923)); 4116 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4117 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4118 getF32Constant(DAG, 0x40823e2f)); 4119 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4120 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4121 getF32Constant(DAG, 0x4020d29c)); 4122 } else { // LimitFloatPrecision <= 18 4123 // For floating-point precision of 18: 4124 // 4125 // Log2ofMantissa = 4126 // -3.0400495f + 4127 // (6.1129976f + 4128 // (-5.3420409f + 4129 // (3.2865683f + 4130 // (-1.2669343f + 4131 // (0.27515199f - 4132 // 0.25691327e-1f * x) * x) * x) * x) * x) * x; 4133 // 4134 // error 0.0000018516, which is better than 18 bits 4135 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4136 getF32Constant(DAG, 0xbcd2769e)); 4137 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4138 getF32Constant(DAG, 0x3e8ce0b9)); 4139 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4140 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4141 getF32Constant(DAG, 0x3fa22ae7)); 4142 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4143 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4144 getF32Constant(DAG, 0x40525723)); 4145 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4146 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4147 getF32Constant(DAG, 0x40aaf200)); 4148 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4149 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4150 getF32Constant(DAG, 0x40c39dad)); 4151 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4152 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, 4153 getF32Constant(DAG, 0x4042902c)); 4154 } 4155 4156 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); 4157 } 4158 4159 // No special expansion. 4160 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op); 4161} 4162 4163/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for 4164/// limited-precision mode. 4165static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4166 const TargetLowering &TLI) { 4167 if (Op.getValueType() == MVT::f32 && 4168 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4169 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 4170 4171 // Scale the exponent by log10(2) [0.30102999f]. 4172 SDValue Exp = GetExponent(DAG, Op1, TLI, dl); 4173 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, 4174 getF32Constant(DAG, 0x3e9a209a)); 4175 4176 // Get the significand and build it into a floating-point number with 4177 // exponent of 1. 4178 SDValue X = GetSignificand(DAG, Op1, dl); 4179 4180 SDValue Log10ofMantissa; 4181 if (LimitFloatPrecision <= 6) { 4182 // For floating-point precision of 6: 4183 // 4184 // Log10ofMantissa = 4185 // -0.50419619f + 4186 // (0.60948995f - 0.10380950f * x) * x; 4187 // 4188 // error 0.0014886165, which is 6 bits 4189 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4190 getF32Constant(DAG, 0xbdd49a13)); 4191 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4192 getF32Constant(DAG, 0x3f1c0789)); 4193 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4194 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4195 getF32Constant(DAG, 0x3f011300)); 4196 } else if (LimitFloatPrecision <= 12) { 4197 // For floating-point precision of 12: 4198 // 4199 // Log10ofMantissa = 4200 // -0.64831180f + 4201 // (0.91751397f + 4202 // (-0.31664806f + 0.47637168e-1f * x) * x) * x; 4203 // 4204 // error 0.00019228036, which is better than 12 bits 4205 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4206 getF32Constant(DAG, 0x3d431f31)); 4207 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, 4208 getF32Constant(DAG, 0x3ea21fb2)); 4209 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4210 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4211 getF32Constant(DAG, 0x3f6ae232)); 4212 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4213 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, 4214 getF32Constant(DAG, 0x3f25f7c3)); 4215 } else { // LimitFloatPrecision <= 18 4216 // For floating-point precision of 18: 4217 // 4218 // Log10ofMantissa = 4219 // -0.84299375f + 4220 // (1.5327582f + 4221 // (-1.0688956f + 4222 // (0.49102474f + 4223 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x; 4224 // 4225 // error 0.0000037995730, which is better than 18 bits 4226 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4227 getF32Constant(DAG, 0x3c5d51ce)); 4228 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, 4229 getF32Constant(DAG, 0x3e00685a)); 4230 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4231 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4232 getF32Constant(DAG, 0x3efb6798)); 4233 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4234 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, 4235 getF32Constant(DAG, 0x3f88d192)); 4236 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4237 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4238 getF32Constant(DAG, 0x3fc4316c)); 4239 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4240 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, 4241 getF32Constant(DAG, 0x3f57ce70)); 4242 } 4243 4244 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); 4245 } 4246 4247 // No special expansion. 4248 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op); 4249} 4250 4251/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for 4252/// limited-precision mode. 4253static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4254 const TargetLowering &TLI) { 4255 if (Op.getValueType() == MVT::f32 && 4256 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4257 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op); 4258 4259 // FractionalPartOfX = x - (float)IntegerPartOfX; 4260 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 4261 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1); 4262 4263 // IntegerPartOfX <<= 23; 4264 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 4265 DAG.getConstant(23, TLI.getPointerTy())); 4266 4267 SDValue TwoToFractionalPartOfX; 4268 if (LimitFloatPrecision <= 6) { 4269 // For floating-point precision of 6: 4270 // 4271 // TwoToFractionalPartOfX = 4272 // 0.997535578f + 4273 // (0.735607626f + 0.252464424f * x) * x; 4274 // 4275 // error 0.0144103317, which is 6 bits 4276 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4277 getF32Constant(DAG, 0x3e814304)); 4278 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4279 getF32Constant(DAG, 0x3f3c50c8)); 4280 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4281 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4282 getF32Constant(DAG, 0x3f7f5e7e)); 4283 } else if (LimitFloatPrecision <= 12) { 4284 // For floating-point precision of 12: 4285 // 4286 // TwoToFractionalPartOfX = 4287 // 0.999892986f + 4288 // (0.696457318f + 4289 // (0.224338339f + 0.792043434e-1f * x) * x) * x; 4290 // 4291 // error 0.000107046256, which is 13 to 14 bits 4292 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4293 getF32Constant(DAG, 0x3da235e3)); 4294 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4295 getF32Constant(DAG, 0x3e65b8f3)); 4296 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4297 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4298 getF32Constant(DAG, 0x3f324b07)); 4299 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4300 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4301 getF32Constant(DAG, 0x3f7ff8fd)); 4302 } else { // LimitFloatPrecision <= 18 4303 // For floating-point precision of 18: 4304 // 4305 // TwoToFractionalPartOfX = 4306 // 0.999999982f + 4307 // (0.693148872f + 4308 // (0.240227044f + 4309 // (0.554906021e-1f + 4310 // (0.961591928e-2f + 4311 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 4312 // error 2.47208000*10^(-7), which is better than 18 bits 4313 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4314 getF32Constant(DAG, 0x3924b03e)); 4315 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4316 getF32Constant(DAG, 0x3ab24b87)); 4317 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4318 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4319 getF32Constant(DAG, 0x3c1d8c17)); 4320 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4321 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4322 getF32Constant(DAG, 0x3d634a1d)); 4323 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4324 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4325 getF32Constant(DAG, 0x3e75fe14)); 4326 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4327 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 4328 getF32Constant(DAG, 0x3f317234)); 4329 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 4330 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 4331 getF32Constant(DAG, 0x3f800000)); 4332 } 4333 4334 // Add the exponent into the result in integer domain. 4335 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, 4336 TwoToFractionalPartOfX); 4337 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 4338 DAG.getNode(ISD::ADD, dl, MVT::i32, 4339 t13, IntegerPartOfX)); 4340 } 4341 4342 // No special expansion. 4343 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op); 4344} 4345 4346/// visitPow - Lower a pow intrinsic. Handles the special sequences for 4347/// limited-precision mode with x == 10.0f. 4348static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS, 4349 SelectionDAG &DAG, const TargetLowering &TLI) { 4350 bool IsExp10 = false; 4351 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 && 4352 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4353 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) { 4354 APFloat Ten(10.0f); 4355 IsExp10 = LHSC->isExactlyValue(Ten); 4356 } 4357 } 4358 4359 if (IsExp10) { 4360 // Put the exponent in the right bit position for later addition to the 4361 // final result: 4362 // 4363 // #define LOG2OF10 3.3219281f 4364 // IntegerPartOfX = (int32_t)(x * LOG2OF10); 4365 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, 4366 getF32Constant(DAG, 0x40549a78)); 4367 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); 4368 4369 // FractionalPartOfX = x - (float)IntegerPartOfX; 4370 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 4371 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); 4372 4373 // IntegerPartOfX <<= 23; 4374 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 4375 DAG.getConstant(23, TLI.getPointerTy())); 4376 4377 SDValue TwoToFractionalPartOfX; 4378 if (LimitFloatPrecision <= 6) { 4379 // For floating-point precision of 6: 4380 // 4381 // twoToFractionalPartOfX = 4382 // 0.997535578f + 4383 // (0.735607626f + 0.252464424f * x) * x; 4384 // 4385 // error 0.0144103317, which is 6 bits 4386 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4387 getF32Constant(DAG, 0x3e814304)); 4388 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4389 getF32Constant(DAG, 0x3f3c50c8)); 4390 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4391 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4392 getF32Constant(DAG, 0x3f7f5e7e)); 4393 } else if (LimitFloatPrecision <= 12) { 4394 // For floating-point precision of 12: 4395 // 4396 // TwoToFractionalPartOfX = 4397 // 0.999892986f + 4398 // (0.696457318f + 4399 // (0.224338339f + 0.792043434e-1f * x) * x) * x; 4400 // 4401 // error 0.000107046256, which is 13 to 14 bits 4402 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4403 getF32Constant(DAG, 0x3da235e3)); 4404 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4405 getF32Constant(DAG, 0x3e65b8f3)); 4406 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4407 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4408 getF32Constant(DAG, 0x3f324b07)); 4409 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4410 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4411 getF32Constant(DAG, 0x3f7ff8fd)); 4412 } else { // LimitFloatPrecision <= 18 4413 // For floating-point precision of 18: 4414 // 4415 // TwoToFractionalPartOfX = 4416 // 0.999999982f + 4417 // (0.693148872f + 4418 // (0.240227044f + 4419 // (0.554906021e-1f + 4420 // (0.961591928e-2f + 4421 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 4422 // error 2.47208000*10^(-7), which is better than 18 bits 4423 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4424 getF32Constant(DAG, 0x3924b03e)); 4425 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4426 getF32Constant(DAG, 0x3ab24b87)); 4427 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4428 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4429 getF32Constant(DAG, 0x3c1d8c17)); 4430 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4431 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4432 getF32Constant(DAG, 0x3d634a1d)); 4433 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4434 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4435 getF32Constant(DAG, 0x3e75fe14)); 4436 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4437 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 4438 getF32Constant(DAG, 0x3f317234)); 4439 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 4440 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 4441 getF32Constant(DAG, 0x3f800000)); 4442 } 4443 4444 SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX); 4445 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 4446 DAG.getNode(ISD::ADD, dl, MVT::i32, 4447 t13, IntegerPartOfX)); 4448 } 4449 4450 // No special expansion. 4451 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS); 4452} 4453 4454 4455/// ExpandPowI - Expand a llvm.powi intrinsic. 4456static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS, 4457 SelectionDAG &DAG) { 4458 // If RHS is a constant, we can expand this out to a multiplication tree, 4459 // otherwise we end up lowering to a call to __powidf2 (for example). When 4460 // optimizing for size, we only want to do this if the expansion would produce 4461 // a small number of multiplies, otherwise we do the full expansion. 4462 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 4463 // Get the exponent as a positive value. 4464 unsigned Val = RHSC->getSExtValue(); 4465 if ((int)Val < 0) Val = -Val; 4466 4467 // powi(x, 0) -> 1.0 4468 if (Val == 0) 4469 return DAG.getConstantFP(1.0, LHS.getValueType()); 4470 4471 const Function *F = DAG.getMachineFunction().getFunction(); 4472 if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, 4473 Attribute::OptimizeForSize) || 4474 // If optimizing for size, don't insert too many multiplies. This 4475 // inserts up to 5 multiplies. 4476 CountPopulation_32(Val)+Log2_32(Val) < 7) { 4477 // We use the simple binary decomposition method to generate the multiply 4478 // sequence. There are more optimal ways to do this (for example, 4479 // powi(x,15) generates one more multiply than it should), but this has 4480 // the benefit of being both really simple and much better than a libcall. 4481 SDValue Res; // Logically starts equal to 1.0 4482 SDValue CurSquare = LHS; 4483 while (Val) { 4484 if (Val & 1) { 4485 if (Res.getNode()) 4486 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare); 4487 else 4488 Res = CurSquare; // 1.0*CurSquare. 4489 } 4490 4491 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), 4492 CurSquare, CurSquare); 4493 Val >>= 1; 4494 } 4495 4496 // If the original was negative, invert the result, producing 1/(x*x*x). 4497 if (RHSC->getSExtValue() < 0) 4498 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), 4499 DAG.getConstantFP(1.0, LHS.getValueType()), Res); 4500 return Res; 4501 } 4502 } 4503 4504 // Otherwise, expand to a libcall. 4505 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); 4506} 4507 4508// getTruncatedArgReg - Find underlying register used for an truncated 4509// argument. 4510static unsigned getTruncatedArgReg(const SDValue &N) { 4511 if (N.getOpcode() != ISD::TRUNCATE) 4512 return 0; 4513 4514 const SDValue &Ext = N.getOperand(0); 4515 if (Ext.getOpcode() == ISD::AssertZext || 4516 Ext.getOpcode() == ISD::AssertSext) { 4517 const SDValue &CFR = Ext.getOperand(0); 4518 if (CFR.getOpcode() == ISD::CopyFromReg) 4519 return cast<RegisterSDNode>(CFR.getOperand(1))->getReg(); 4520 if (CFR.getOpcode() == ISD::TRUNCATE) 4521 return getTruncatedArgReg(CFR); 4522 } 4523 return 0; 4524} 4525 4526/// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function 4527/// argument, create the corresponding DBG_VALUE machine instruction for it now. 4528/// At the end of instruction selection, they will be inserted to the entry BB. 4529bool 4530SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable, 4531 int64_t Offset, bool IsIndirect, 4532 const SDValue &N) { 4533 const Argument *Arg = dyn_cast<Argument>(V); 4534 if (!Arg) 4535 return false; 4536 4537 MachineFunction &MF = DAG.getMachineFunction(); 4538 const TargetInstrInfo *TII = DAG.getTarget().getInstrInfo(); 4539 4540 // Ignore inlined function arguments here. 4541 DIVariable DV(Variable); 4542 if (DV.isInlinedFnArgument(MF.getFunction())) 4543 return false; 4544 4545 Optional<MachineOperand> Op; 4546 // Some arguments' frame index is recorded during argument lowering. 4547 if (int FI = FuncInfo.getArgumentFrameIndex(Arg)) 4548 Op = MachineOperand::CreateFI(FI); 4549 4550 if (!Op && N.getNode()) { 4551 unsigned Reg; 4552 if (N.getOpcode() == ISD::CopyFromReg) 4553 Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg(); 4554 else 4555 Reg = getTruncatedArgReg(N); 4556 if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) { 4557 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 4558 unsigned PR = RegInfo.getLiveInPhysReg(Reg); 4559 if (PR) 4560 Reg = PR; 4561 } 4562 if (Reg) 4563 Op = MachineOperand::CreateReg(Reg, false); 4564 } 4565 4566 if (!Op) { 4567 // Check if ValueMap has reg number. 4568 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); 4569 if (VMI != FuncInfo.ValueMap.end()) 4570 Op = MachineOperand::CreateReg(VMI->second, false); 4571 } 4572 4573 if (!Op && N.getNode()) 4574 // Check if frame index is available. 4575 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode())) 4576 if (FrameIndexSDNode *FINode = 4577 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) 4578 Op = MachineOperand::CreateFI(FINode->getIndex()); 4579 4580 if (!Op) 4581 return false; 4582 4583 if (Op->isReg()) 4584 FuncInfo.ArgDbgValues.push_back(BuildMI(MF, getCurDebugLoc(), 4585 TII->get(TargetOpcode::DBG_VALUE), 4586 IsIndirect, 4587 Op->getReg(), Offset, Variable)); 4588 else 4589 FuncInfo.ArgDbgValues.push_back( 4590 BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE)) 4591 .addOperand(*Op).addImm(Offset).addMetadata(Variable)); 4592 4593 return true; 4594} 4595 4596// VisualStudio defines setjmp as _setjmp 4597#if defined(_MSC_VER) && defined(setjmp) && \ 4598 !defined(setjmp_undefined_for_msvc) 4599# pragma push_macro("setjmp") 4600# undef setjmp 4601# define setjmp_undefined_for_msvc 4602#endif 4603 4604/// visitIntrinsicCall - Lower the call to the specified intrinsic function. If 4605/// we want to emit this as a call to a named external function, return the name 4606/// otherwise lower it and return null. 4607const char * 4608SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { 4609 const TargetLowering *TLI = TM.getTargetLowering(); 4610 SDLoc sdl = getCurSDLoc(); 4611 DebugLoc dl = getCurDebugLoc(); 4612 SDValue Res; 4613 4614 switch (Intrinsic) { 4615 default: 4616 // By default, turn this into a target intrinsic node. 4617 visitTargetIntrinsic(I, Intrinsic); 4618 return nullptr; 4619 case Intrinsic::vastart: visitVAStart(I); return nullptr; 4620 case Intrinsic::vaend: visitVAEnd(I); return nullptr; 4621 case Intrinsic::vacopy: visitVACopy(I); return nullptr; 4622 case Intrinsic::returnaddress: 4623 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI->getPointerTy(), 4624 getValue(I.getArgOperand(0)))); 4625 return nullptr; 4626 case Intrinsic::frameaddress: 4627 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI->getPointerTy(), 4628 getValue(I.getArgOperand(0)))); 4629 return nullptr; 4630 case Intrinsic::read_register: { 4631 Value *Reg = I.getArgOperand(0); 4632 SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg)); 4633 EVT VT = TM.getTargetLowering()->getValueType(I.getType()); 4634 setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName)); 4635 return nullptr; 4636 } 4637 case Intrinsic::write_register: { 4638 Value *Reg = I.getArgOperand(0); 4639 Value *RegValue = I.getArgOperand(1); 4640 SDValue Chain = getValue(RegValue).getOperand(0); 4641 SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg)); 4642 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain, 4643 RegName, getValue(RegValue))); 4644 return nullptr; 4645 } 4646 case Intrinsic::setjmp: 4647 return &"_setjmp"[!TLI->usesUnderscoreSetJmp()]; 4648 case Intrinsic::longjmp: 4649 return &"_longjmp"[!TLI->usesUnderscoreLongJmp()]; 4650 case Intrinsic::memcpy: { 4651 // Assert for address < 256 since we support only user defined address 4652 // spaces. 4653 assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4654 < 256 && 4655 cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace() 4656 < 256 && 4657 "Unknown address space"); 4658 SDValue Op1 = getValue(I.getArgOperand(0)); 4659 SDValue Op2 = getValue(I.getArgOperand(1)); 4660 SDValue Op3 = getValue(I.getArgOperand(2)); 4661 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4662 if (!Align) 4663 Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment. 4664 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4665 DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false, 4666 MachinePointerInfo(I.getArgOperand(0)), 4667 MachinePointerInfo(I.getArgOperand(1)))); 4668 return nullptr; 4669 } 4670 case Intrinsic::memset: { 4671 // Assert for address < 256 since we support only user defined address 4672 // spaces. 4673 assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4674 < 256 && 4675 "Unknown address space"); 4676 SDValue Op1 = getValue(I.getArgOperand(0)); 4677 SDValue Op2 = getValue(I.getArgOperand(1)); 4678 SDValue Op3 = getValue(I.getArgOperand(2)); 4679 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4680 if (!Align) 4681 Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment. 4682 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4683 DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, 4684 MachinePointerInfo(I.getArgOperand(0)))); 4685 return nullptr; 4686 } 4687 case Intrinsic::memmove: { 4688 // Assert for address < 256 since we support only user defined address 4689 // spaces. 4690 assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4691 < 256 && 4692 cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace() 4693 < 256 && 4694 "Unknown address space"); 4695 SDValue Op1 = getValue(I.getArgOperand(0)); 4696 SDValue Op2 = getValue(I.getArgOperand(1)); 4697 SDValue Op3 = getValue(I.getArgOperand(2)); 4698 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4699 if (!Align) 4700 Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment. 4701 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4702 DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, 4703 MachinePointerInfo(I.getArgOperand(0)), 4704 MachinePointerInfo(I.getArgOperand(1)))); 4705 return nullptr; 4706 } 4707 case Intrinsic::dbg_declare: { 4708 const DbgDeclareInst &DI = cast<DbgDeclareInst>(I); 4709 MDNode *Variable = DI.getVariable(); 4710 const Value *Address = DI.getAddress(); 4711 DIVariable DIVar(Variable); 4712 assert((!DIVar || DIVar.isVariable()) && 4713 "Variable in DbgDeclareInst should be either null or a DIVariable."); 4714 if (!Address || !DIVar) { 4715 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4716 return nullptr; 4717 } 4718 4719 // Check if address has undef value. 4720 if (isa<UndefValue>(Address) || 4721 (Address->use_empty() && !isa<Argument>(Address))) { 4722 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4723 return nullptr; 4724 } 4725 4726 SDValue &N = NodeMap[Address]; 4727 if (!N.getNode() && isa<Argument>(Address)) 4728 // Check unused arguments map. 4729 N = UnusedArgNodeMap[Address]; 4730 SDDbgValue *SDV; 4731 if (N.getNode()) { 4732 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) 4733 Address = BCI->getOperand(0); 4734 // Parameters are handled specially. 4735 bool isParameter = 4736 (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable || 4737 isa<Argument>(Address)); 4738 4739 const AllocaInst *AI = dyn_cast<AllocaInst>(Address); 4740 4741 if (isParameter && !AI) { 4742 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode()); 4743 if (FINode) 4744 // Byval parameter. We have a frame index at this point. 4745 SDV = DAG.getFrameIndexDbgValue(Variable, FINode->getIndex(), 4746 0, dl, SDNodeOrder); 4747 else { 4748 // Address is an argument, so try to emit its dbg value using 4749 // virtual register info from the FuncInfo.ValueMap. 4750 EmitFuncArgumentDbgValue(Address, Variable, 0, false, N); 4751 return nullptr; 4752 } 4753 } else if (AI) 4754 SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(), 4755 true, 0, dl, SDNodeOrder); 4756 else { 4757 // Can't do anything with other non-AI cases yet. 4758 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4759 DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t"); 4760 DEBUG(Address->dump()); 4761 return nullptr; 4762 } 4763 DAG.AddDbgValue(SDV, N.getNode(), isParameter); 4764 } else { 4765 // If Address is an argument then try to emit its dbg value using 4766 // virtual register info from the FuncInfo.ValueMap. 4767 if (!EmitFuncArgumentDbgValue(Address, Variable, 0, false, N)) { 4768 // If variable is pinned by a alloca in dominating bb then 4769 // use StaticAllocaMap. 4770 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) { 4771 if (AI->getParent() != DI.getParent()) { 4772 DenseMap<const AllocaInst*, int>::iterator SI = 4773 FuncInfo.StaticAllocaMap.find(AI); 4774 if (SI != FuncInfo.StaticAllocaMap.end()) { 4775 SDV = DAG.getFrameIndexDbgValue(Variable, SI->second, 4776 0, dl, SDNodeOrder); 4777 DAG.AddDbgValue(SDV, nullptr, false); 4778 return nullptr; 4779 } 4780 } 4781 } 4782 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4783 } 4784 } 4785 return nullptr; 4786 } 4787 case Intrinsic::dbg_value: { 4788 const DbgValueInst &DI = cast<DbgValueInst>(I); 4789 DIVariable DIVar(DI.getVariable()); 4790 assert((!DIVar || DIVar.isVariable()) && 4791 "Variable in DbgValueInst should be either null or a DIVariable."); 4792 if (!DIVar) 4793 return nullptr; 4794 4795 MDNode *Variable = DI.getVariable(); 4796 uint64_t Offset = DI.getOffset(); 4797 const Value *V = DI.getValue(); 4798 if (!V) 4799 return nullptr; 4800 4801 SDDbgValue *SDV; 4802 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) { 4803 SDV = DAG.getConstantDbgValue(Variable, V, Offset, dl, SDNodeOrder); 4804 DAG.AddDbgValue(SDV, nullptr, false); 4805 } else { 4806 // Do not use getValue() in here; we don't want to generate code at 4807 // this point if it hasn't been done yet. 4808 SDValue N = NodeMap[V]; 4809 if (!N.getNode() && isa<Argument>(V)) 4810 // Check unused arguments map. 4811 N = UnusedArgNodeMap[V]; 4812 if (N.getNode()) { 4813 // A dbg.value for an alloca is always indirect. 4814 bool IsIndirect = isa<AllocaInst>(V) || Offset != 0; 4815 if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, N)) { 4816 SDV = DAG.getDbgValue(Variable, N.getNode(), 4817 N.getResNo(), IsIndirect, 4818 Offset, dl, SDNodeOrder); 4819 DAG.AddDbgValue(SDV, N.getNode(), false); 4820 } 4821 } else if (!V->use_empty() ) { 4822 // Do not call getValue(V) yet, as we don't want to generate code. 4823 // Remember it for later. 4824 DanglingDebugInfo DDI(&DI, dl, SDNodeOrder); 4825 DanglingDebugInfoMap[V] = DDI; 4826 } else { 4827 // We may expand this to cover more cases. One case where we have no 4828 // data available is an unreferenced parameter. 4829 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4830 } 4831 } 4832 4833 // Build a debug info table entry. 4834 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V)) 4835 V = BCI->getOperand(0); 4836 const AllocaInst *AI = dyn_cast<AllocaInst>(V); 4837 // Don't handle byval struct arguments or VLAs, for example. 4838 if (!AI) { 4839 DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n"); 4840 DEBUG(dbgs() << " Last seen at:\n " << *V << "\n"); 4841 return nullptr; 4842 } 4843 DenseMap<const AllocaInst*, int>::iterator SI = 4844 FuncInfo.StaticAllocaMap.find(AI); 4845 if (SI == FuncInfo.StaticAllocaMap.end()) 4846 return nullptr; // VLAs. 4847 return nullptr; 4848 } 4849 4850 case Intrinsic::eh_typeid_for: { 4851 // Find the type id for the given typeinfo. 4852 GlobalVariable *GV = ExtractTypeInfo(I.getArgOperand(0)); 4853 unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV); 4854 Res = DAG.getConstant(TypeID, MVT::i32); 4855 setValue(&I, Res); 4856 return nullptr; 4857 } 4858 4859 case Intrinsic::eh_return_i32: 4860 case Intrinsic::eh_return_i64: 4861 DAG.getMachineFunction().getMMI().setCallsEHReturn(true); 4862 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, 4863 MVT::Other, 4864 getControlRoot(), 4865 getValue(I.getArgOperand(0)), 4866 getValue(I.getArgOperand(1)))); 4867 return nullptr; 4868 case Intrinsic::eh_unwind_init: 4869 DAG.getMachineFunction().getMMI().setCallsUnwindInit(true); 4870 return nullptr; 4871 case Intrinsic::eh_dwarf_cfa: { 4872 SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl, 4873 TLI->getPointerTy()); 4874 SDValue Offset = DAG.getNode(ISD::ADD, sdl, 4875 CfaArg.getValueType(), 4876 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl, 4877 CfaArg.getValueType()), 4878 CfaArg); 4879 SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, 4880 TLI->getPointerTy(), 4881 DAG.getConstant(0, TLI->getPointerTy())); 4882 setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(), 4883 FA, Offset)); 4884 return nullptr; 4885 } 4886 case Intrinsic::eh_sjlj_callsite: { 4887 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 4888 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0)); 4889 assert(CI && "Non-constant call site value in eh.sjlj.callsite!"); 4890 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!"); 4891 4892 MMI.setCurrentCallSite(CI->getZExtValue()); 4893 return nullptr; 4894 } 4895 case Intrinsic::eh_sjlj_functioncontext: { 4896 // Get and store the index of the function context. 4897 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 4898 AllocaInst *FnCtx = 4899 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); 4900 int FI = FuncInfo.StaticAllocaMap[FnCtx]; 4901 MFI->setFunctionContextIndex(FI); 4902 return nullptr; 4903 } 4904 case Intrinsic::eh_sjlj_setjmp: { 4905 SDValue Ops[2]; 4906 Ops[0] = getRoot(); 4907 Ops[1] = getValue(I.getArgOperand(0)); 4908 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl, 4909 DAG.getVTList(MVT::i32, MVT::Other), Ops); 4910 setValue(&I, Op.getValue(0)); 4911 DAG.setRoot(Op.getValue(1)); 4912 return nullptr; 4913 } 4914 case Intrinsic::eh_sjlj_longjmp: { 4915 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, 4916 getRoot(), getValue(I.getArgOperand(0)))); 4917 return nullptr; 4918 } 4919 4920 case Intrinsic::x86_mmx_pslli_w: 4921 case Intrinsic::x86_mmx_pslli_d: 4922 case Intrinsic::x86_mmx_pslli_q: 4923 case Intrinsic::x86_mmx_psrli_w: 4924 case Intrinsic::x86_mmx_psrli_d: 4925 case Intrinsic::x86_mmx_psrli_q: 4926 case Intrinsic::x86_mmx_psrai_w: 4927 case Intrinsic::x86_mmx_psrai_d: { 4928 SDValue ShAmt = getValue(I.getArgOperand(1)); 4929 if (isa<ConstantSDNode>(ShAmt)) { 4930 visitTargetIntrinsic(I, Intrinsic); 4931 return nullptr; 4932 } 4933 unsigned NewIntrinsic = 0; 4934 EVT ShAmtVT = MVT::v2i32; 4935 switch (Intrinsic) { 4936 case Intrinsic::x86_mmx_pslli_w: 4937 NewIntrinsic = Intrinsic::x86_mmx_psll_w; 4938 break; 4939 case Intrinsic::x86_mmx_pslli_d: 4940 NewIntrinsic = Intrinsic::x86_mmx_psll_d; 4941 break; 4942 case Intrinsic::x86_mmx_pslli_q: 4943 NewIntrinsic = Intrinsic::x86_mmx_psll_q; 4944 break; 4945 case Intrinsic::x86_mmx_psrli_w: 4946 NewIntrinsic = Intrinsic::x86_mmx_psrl_w; 4947 break; 4948 case Intrinsic::x86_mmx_psrli_d: 4949 NewIntrinsic = Intrinsic::x86_mmx_psrl_d; 4950 break; 4951 case Intrinsic::x86_mmx_psrli_q: 4952 NewIntrinsic = Intrinsic::x86_mmx_psrl_q; 4953 break; 4954 case Intrinsic::x86_mmx_psrai_w: 4955 NewIntrinsic = Intrinsic::x86_mmx_psra_w; 4956 break; 4957 case Intrinsic::x86_mmx_psrai_d: 4958 NewIntrinsic = Intrinsic::x86_mmx_psra_d; 4959 break; 4960 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 4961 } 4962 4963 // The vector shift intrinsics with scalars uses 32b shift amounts but 4964 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits 4965 // to be zero. 4966 // We must do this early because v2i32 is not a legal type. 4967 SDValue ShOps[2]; 4968 ShOps[0] = ShAmt; 4969 ShOps[1] = DAG.getConstant(0, MVT::i32); 4970 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps); 4971 EVT DestVT = TLI->getValueType(I.getType()); 4972 ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt); 4973 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT, 4974 DAG.getConstant(NewIntrinsic, MVT::i32), 4975 getValue(I.getArgOperand(0)), ShAmt); 4976 setValue(&I, Res); 4977 return nullptr; 4978 } 4979 case Intrinsic::x86_avx_vinsertf128_pd_256: 4980 case Intrinsic::x86_avx_vinsertf128_ps_256: 4981 case Intrinsic::x86_avx_vinsertf128_si_256: 4982 case Intrinsic::x86_avx2_vinserti128: { 4983 EVT DestVT = TLI->getValueType(I.getType()); 4984 EVT ElVT = TLI->getValueType(I.getArgOperand(1)->getType()); 4985 uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) * 4986 ElVT.getVectorNumElements(); 4987 Res = DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT, 4988 getValue(I.getArgOperand(0)), 4989 getValue(I.getArgOperand(1)), 4990 DAG.getConstant(Idx, TLI->getVectorIdxTy())); 4991 setValue(&I, Res); 4992 return nullptr; 4993 } 4994 case Intrinsic::x86_avx_vextractf128_pd_256: 4995 case Intrinsic::x86_avx_vextractf128_ps_256: 4996 case Intrinsic::x86_avx_vextractf128_si_256: 4997 case Intrinsic::x86_avx2_vextracti128: { 4998 EVT DestVT = TLI->getValueType(I.getType()); 4999 uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) * 5000 DestVT.getVectorNumElements(); 5001 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT, 5002 getValue(I.getArgOperand(0)), 5003 DAG.getConstant(Idx, TLI->getVectorIdxTy())); 5004 setValue(&I, Res); 5005 return nullptr; 5006 } 5007 case Intrinsic::convertff: 5008 case Intrinsic::convertfsi: 5009 case Intrinsic::convertfui: 5010 case Intrinsic::convertsif: 5011 case Intrinsic::convertuif: 5012 case Intrinsic::convertss: 5013 case Intrinsic::convertsu: 5014 case Intrinsic::convertus: 5015 case Intrinsic::convertuu: { 5016 ISD::CvtCode Code = ISD::CVT_INVALID; 5017 switch (Intrinsic) { 5018 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 5019 case Intrinsic::convertff: Code = ISD::CVT_FF; break; 5020 case Intrinsic::convertfsi: Code = ISD::CVT_FS; break; 5021 case Intrinsic::convertfui: Code = ISD::CVT_FU; break; 5022 case Intrinsic::convertsif: Code = ISD::CVT_SF; break; 5023 case Intrinsic::convertuif: Code = ISD::CVT_UF; break; 5024 case Intrinsic::convertss: Code = ISD::CVT_SS; break; 5025 case Intrinsic::convertsu: Code = ISD::CVT_SU; break; 5026 case Intrinsic::convertus: Code = ISD::CVT_US; break; 5027 case Intrinsic::convertuu: Code = ISD::CVT_UU; break; 5028 } 5029 EVT DestVT = TLI->getValueType(I.getType()); 5030 const Value *Op1 = I.getArgOperand(0); 5031 Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1), 5032 DAG.getValueType(DestVT), 5033 DAG.getValueType(getValue(Op1).getValueType()), 5034 getValue(I.getArgOperand(1)), 5035 getValue(I.getArgOperand(2)), 5036 Code); 5037 setValue(&I, Res); 5038 return nullptr; 5039 } 5040 case Intrinsic::powi: 5041 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)), 5042 getValue(I.getArgOperand(1)), DAG)); 5043 return nullptr; 5044 case Intrinsic::log: 5045 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 5046 return nullptr; 5047 case Intrinsic::log2: 5048 setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 5049 return nullptr; 5050 case Intrinsic::log10: 5051 setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 5052 return nullptr; 5053 case Intrinsic::exp: 5054 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 5055 return nullptr; 5056 case Intrinsic::exp2: 5057 setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 5058 return nullptr; 5059 case Intrinsic::pow: 5060 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)), 5061 getValue(I.getArgOperand(1)), DAG, *TLI)); 5062 return nullptr; 5063 case Intrinsic::sqrt: 5064 case Intrinsic::fabs: 5065 case Intrinsic::sin: 5066 case Intrinsic::cos: 5067 case Intrinsic::floor: 5068 case Intrinsic::ceil: 5069 case Intrinsic::trunc: 5070 case Intrinsic::rint: 5071 case Intrinsic::nearbyint: 5072 case Intrinsic::round: { 5073 unsigned Opcode; 5074 switch (Intrinsic) { 5075 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 5076 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; 5077 case Intrinsic::fabs: Opcode = ISD::FABS; break; 5078 case Intrinsic::sin: Opcode = ISD::FSIN; break; 5079 case Intrinsic::cos: Opcode = ISD::FCOS; break; 5080 case Intrinsic::floor: Opcode = ISD::FFLOOR; break; 5081 case Intrinsic::ceil: Opcode = ISD::FCEIL; break; 5082 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; 5083 case Intrinsic::rint: Opcode = ISD::FRINT; break; 5084 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; 5085 case Intrinsic::round: Opcode = ISD::FROUND; break; 5086 } 5087 5088 setValue(&I, DAG.getNode(Opcode, sdl, 5089 getValue(I.getArgOperand(0)).getValueType(), 5090 getValue(I.getArgOperand(0)))); 5091 return nullptr; 5092 } 5093 case Intrinsic::copysign: 5094 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, 5095 getValue(I.getArgOperand(0)).getValueType(), 5096 getValue(I.getArgOperand(0)), 5097 getValue(I.getArgOperand(1)))); 5098 return nullptr; 5099 case Intrinsic::fma: 5100 setValue(&I, DAG.getNode(ISD::FMA, sdl, 5101 getValue(I.getArgOperand(0)).getValueType(), 5102 getValue(I.getArgOperand(0)), 5103 getValue(I.getArgOperand(1)), 5104 getValue(I.getArgOperand(2)))); 5105 return nullptr; 5106 case Intrinsic::fmuladd: { 5107 EVT VT = TLI->getValueType(I.getType()); 5108 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && 5109 TLI->isFMAFasterThanFMulAndFAdd(VT)) { 5110 setValue(&I, DAG.getNode(ISD::FMA, sdl, 5111 getValue(I.getArgOperand(0)).getValueType(), 5112 getValue(I.getArgOperand(0)), 5113 getValue(I.getArgOperand(1)), 5114 getValue(I.getArgOperand(2)))); 5115 } else { 5116 SDValue Mul = DAG.getNode(ISD::FMUL, sdl, 5117 getValue(I.getArgOperand(0)).getValueType(), 5118 getValue(I.getArgOperand(0)), 5119 getValue(I.getArgOperand(1))); 5120 SDValue Add = DAG.getNode(ISD::FADD, sdl, 5121 getValue(I.getArgOperand(0)).getValueType(), 5122 Mul, 5123 getValue(I.getArgOperand(2))); 5124 setValue(&I, Add); 5125 } 5126 return nullptr; 5127 } 5128 case Intrinsic::convert_to_fp16: 5129 setValue(&I, DAG.getNode(ISD::FP32_TO_FP16, sdl, 5130 MVT::i16, getValue(I.getArgOperand(0)))); 5131 return nullptr; 5132 case Intrinsic::convert_from_fp16: 5133 setValue(&I, DAG.getNode(ISD::FP16_TO_FP32, sdl, 5134 MVT::f32, getValue(I.getArgOperand(0)))); 5135 return nullptr; 5136 case Intrinsic::pcmarker: { 5137 SDValue Tmp = getValue(I.getArgOperand(0)); 5138 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); 5139 return nullptr; 5140 } 5141 case Intrinsic::readcyclecounter: { 5142 SDValue Op = getRoot(); 5143 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl, 5144 DAG.getVTList(MVT::i64, MVT::Other), Op); 5145 setValue(&I, Res); 5146 DAG.setRoot(Res.getValue(1)); 5147 return nullptr; 5148 } 5149 case Intrinsic::bswap: 5150 setValue(&I, DAG.getNode(ISD::BSWAP, sdl, 5151 getValue(I.getArgOperand(0)).getValueType(), 5152 getValue(I.getArgOperand(0)))); 5153 return nullptr; 5154 case Intrinsic::cttz: { 5155 SDValue Arg = getValue(I.getArgOperand(0)); 5156 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); 5157 EVT Ty = Arg.getValueType(); 5158 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, 5159 sdl, Ty, Arg)); 5160 return nullptr; 5161 } 5162 case Intrinsic::ctlz: { 5163 SDValue Arg = getValue(I.getArgOperand(0)); 5164 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); 5165 EVT Ty = Arg.getValueType(); 5166 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, 5167 sdl, Ty, Arg)); 5168 return nullptr; 5169 } 5170 case Intrinsic::ctpop: { 5171 SDValue Arg = getValue(I.getArgOperand(0)); 5172 EVT Ty = Arg.getValueType(); 5173 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); 5174 return nullptr; 5175 } 5176 case Intrinsic::stacksave: { 5177 SDValue Op = getRoot(); 5178 Res = DAG.getNode(ISD::STACKSAVE, sdl, 5179 DAG.getVTList(TLI->getPointerTy(), MVT::Other), Op); 5180 setValue(&I, Res); 5181 DAG.setRoot(Res.getValue(1)); 5182 return nullptr; 5183 } 5184 case Intrinsic::stackrestore: { 5185 Res = getValue(I.getArgOperand(0)); 5186 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); 5187 return nullptr; 5188 } 5189 case Intrinsic::stackprotector: { 5190 // Emit code into the DAG to store the stack guard onto the stack. 5191 MachineFunction &MF = DAG.getMachineFunction(); 5192 MachineFrameInfo *MFI = MF.getFrameInfo(); 5193 EVT PtrTy = TLI->getPointerTy(); 5194 5195 SDValue Src = getValue(I.getArgOperand(0)); // The guard's value. 5196 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1)); 5197 5198 int FI = FuncInfo.StaticAllocaMap[Slot]; 5199 MFI->setStackProtectorIndex(FI); 5200 5201 SDValue FIN = DAG.getFrameIndex(FI, PtrTy); 5202 5203 // Store the stack protector onto the stack. 5204 Res = DAG.getStore(getRoot(), sdl, Src, FIN, 5205 MachinePointerInfo::getFixedStack(FI), 5206 true, false, 0); 5207 setValue(&I, Res); 5208 DAG.setRoot(Res); 5209 return nullptr; 5210 } 5211 case Intrinsic::objectsize: { 5212 // If we don't know by now, we're never going to know. 5213 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1)); 5214 5215 assert(CI && "Non-constant type in __builtin_object_size?"); 5216 5217 SDValue Arg = getValue(I.getCalledValue()); 5218 EVT Ty = Arg.getValueType(); 5219 5220 if (CI->isZero()) 5221 Res = DAG.getConstant(-1ULL, Ty); 5222 else 5223 Res = DAG.getConstant(0, Ty); 5224 5225 setValue(&I, Res); 5226 return nullptr; 5227 } 5228 case Intrinsic::annotation: 5229 case Intrinsic::ptr_annotation: 5230 // Drop the intrinsic, but forward the value 5231 setValue(&I, getValue(I.getOperand(0))); 5232 return nullptr; 5233 case Intrinsic::var_annotation: 5234 // Discard annotate attributes 5235 return nullptr; 5236 5237 case Intrinsic::init_trampoline: { 5238 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts()); 5239 5240 SDValue Ops[6]; 5241 Ops[0] = getRoot(); 5242 Ops[1] = getValue(I.getArgOperand(0)); 5243 Ops[2] = getValue(I.getArgOperand(1)); 5244 Ops[3] = getValue(I.getArgOperand(2)); 5245 Ops[4] = DAG.getSrcValue(I.getArgOperand(0)); 5246 Ops[5] = DAG.getSrcValue(F); 5247 5248 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops); 5249 5250 DAG.setRoot(Res); 5251 return nullptr; 5252 } 5253 case Intrinsic::adjust_trampoline: { 5254 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, 5255 TLI->getPointerTy(), 5256 getValue(I.getArgOperand(0)))); 5257 return nullptr; 5258 } 5259 case Intrinsic::gcroot: 5260 if (GFI) { 5261 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts(); 5262 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1)); 5263 5264 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode()); 5265 GFI->addStackRoot(FI->getIndex(), TypeMap); 5266 } 5267 return nullptr; 5268 case Intrinsic::gcread: 5269 case Intrinsic::gcwrite: 5270 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); 5271 case Intrinsic::flt_rounds: 5272 setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32)); 5273 return nullptr; 5274 5275 case Intrinsic::expect: { 5276 // Just replace __builtin_expect(exp, c) with EXP. 5277 setValue(&I, getValue(I.getArgOperand(0))); 5278 return nullptr; 5279 } 5280 5281 case Intrinsic::debugtrap: 5282 case Intrinsic::trap: { 5283 StringRef TrapFuncName = TM.Options.getTrapFunctionName(); 5284 if (TrapFuncName.empty()) { 5285 ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ? 5286 ISD::TRAP : ISD::DEBUGTRAP; 5287 DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot())); 5288 return nullptr; 5289 } 5290 TargetLowering::ArgListTy Args; 5291 5292 TargetLowering::CallLoweringInfo CLI(DAG); 5293 CLI.setDebugLoc(sdl).setChain(getRoot()) 5294 .setCallee(CallingConv::C, I.getType(), 5295 DAG.getExternalSymbol(TrapFuncName.data(), TLI->getPointerTy()), 5296 &Args, 0); 5297 5298 std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI); 5299 DAG.setRoot(Result.second); 5300 return nullptr; 5301 } 5302 5303 case Intrinsic::uadd_with_overflow: 5304 case Intrinsic::sadd_with_overflow: 5305 case Intrinsic::usub_with_overflow: 5306 case Intrinsic::ssub_with_overflow: 5307 case Intrinsic::umul_with_overflow: 5308 case Intrinsic::smul_with_overflow: { 5309 ISD::NodeType Op; 5310 switch (Intrinsic) { 5311 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 5312 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break; 5313 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break; 5314 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break; 5315 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break; 5316 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break; 5317 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break; 5318 } 5319 SDValue Op1 = getValue(I.getArgOperand(0)); 5320 SDValue Op2 = getValue(I.getArgOperand(1)); 5321 5322 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1); 5323 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); 5324 return nullptr; 5325 } 5326 case Intrinsic::prefetch: { 5327 SDValue Ops[5]; 5328 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); 5329 Ops[0] = getRoot(); 5330 Ops[1] = getValue(I.getArgOperand(0)); 5331 Ops[2] = getValue(I.getArgOperand(1)); 5332 Ops[3] = getValue(I.getArgOperand(2)); 5333 Ops[4] = getValue(I.getArgOperand(3)); 5334 DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl, 5335 DAG.getVTList(MVT::Other), Ops, 5336 EVT::getIntegerVT(*Context, 8), 5337 MachinePointerInfo(I.getArgOperand(0)), 5338 0, /* align */ 5339 false, /* volatile */ 5340 rw==0, /* read */ 5341 rw==1)); /* write */ 5342 return nullptr; 5343 } 5344 case Intrinsic::lifetime_start: 5345 case Intrinsic::lifetime_end: { 5346 bool IsStart = (Intrinsic == Intrinsic::lifetime_start); 5347 // Stack coloring is not enabled in O0, discard region information. 5348 if (TM.getOptLevel() == CodeGenOpt::None) 5349 return nullptr; 5350 5351 SmallVector<Value *, 4> Allocas; 5352 GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL); 5353 5354 for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(), 5355 E = Allocas.end(); Object != E; ++Object) { 5356 AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object); 5357 5358 // Could not find an Alloca. 5359 if (!LifetimeObject) 5360 continue; 5361 5362 int FI = FuncInfo.StaticAllocaMap[LifetimeObject]; 5363 5364 SDValue Ops[2]; 5365 Ops[0] = getRoot(); 5366 Ops[1] = DAG.getFrameIndex(FI, TLI->getPointerTy(), true); 5367 unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END); 5368 5369 Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops); 5370 DAG.setRoot(Res); 5371 } 5372 return nullptr; 5373 } 5374 case Intrinsic::invariant_start: 5375 // Discard region information. 5376 setValue(&I, DAG.getUNDEF(TLI->getPointerTy())); 5377 return nullptr; 5378 case Intrinsic::invariant_end: 5379 // Discard region information. 5380 return nullptr; 5381 case Intrinsic::stackprotectorcheck: { 5382 // Do not actually emit anything for this basic block. Instead we initialize 5383 // the stack protector descriptor and export the guard variable so we can 5384 // access it in FinishBasicBlock. 5385 const BasicBlock *BB = I.getParent(); 5386 SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I); 5387 ExportFromCurrentBlock(SPDescriptor.getGuard()); 5388 5389 // Flush our exports since we are going to process a terminator. 5390 (void)getControlRoot(); 5391 return nullptr; 5392 } 5393 case Intrinsic::clear_cache: 5394 return TLI->getClearCacheBuiltinName(); 5395 case Intrinsic::donothing: 5396 // ignore 5397 return nullptr; 5398 case Intrinsic::experimental_stackmap: { 5399 visitStackmap(I); 5400 return nullptr; 5401 } 5402 case Intrinsic::experimental_patchpoint_void: 5403 case Intrinsic::experimental_patchpoint_i64: { 5404 visitPatchpoint(I); 5405 return nullptr; 5406 } 5407 } 5408} 5409 5410void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, 5411 bool isTailCall, 5412 MachineBasicBlock *LandingPad) { 5413 PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType()); 5414 FunctionType *FTy = cast<FunctionType>(PT->getElementType()); 5415 Type *RetTy = FTy->getReturnType(); 5416 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 5417 MCSymbol *BeginLabel = nullptr; 5418 5419 TargetLowering::ArgListTy Args; 5420 TargetLowering::ArgListEntry Entry; 5421 Args.reserve(CS.arg_size()); 5422 5423 // Check whether the function can return without sret-demotion. 5424 SmallVector<ISD::OutputArg, 4> Outs; 5425 const TargetLowering *TLI = TM.getTargetLowering(); 5426 GetReturnInfo(RetTy, CS.getAttributes(), Outs, *TLI); 5427 5428 bool CanLowerReturn = TLI->CanLowerReturn(CS.getCallingConv(), 5429 DAG.getMachineFunction(), 5430 FTy->isVarArg(), Outs, 5431 FTy->getContext()); 5432 5433 SDValue DemoteStackSlot; 5434 int DemoteStackIdx = -100; 5435 5436 if (!CanLowerReturn) { 5437 assert(!CS.hasInAllocaArgument() && 5438 "sret demotion is incompatible with inalloca"); 5439 uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize( 5440 FTy->getReturnType()); 5441 unsigned Align = TLI->getDataLayout()->getPrefTypeAlignment( 5442 FTy->getReturnType()); 5443 MachineFunction &MF = DAG.getMachineFunction(); 5444 DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); 5445 Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType()); 5446 5447 DemoteStackSlot = DAG.getFrameIndex(DemoteStackIdx, TLI->getPointerTy()); 5448 Entry.Node = DemoteStackSlot; 5449 Entry.Ty = StackSlotPtrType; 5450 Entry.isSExt = false; 5451 Entry.isZExt = false; 5452 Entry.isInReg = false; 5453 Entry.isSRet = true; 5454 Entry.isNest = false; 5455 Entry.isByVal = false; 5456 Entry.isReturned = false; 5457 Entry.Alignment = Align; 5458 Args.push_back(Entry); 5459 RetTy = Type::getVoidTy(FTy->getContext()); 5460 } 5461 5462 for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); 5463 i != e; ++i) { 5464 const Value *V = *i; 5465 5466 // Skip empty types 5467 if (V->getType()->isEmptyTy()) 5468 continue; 5469 5470 SDValue ArgNode = getValue(V); 5471 Entry.Node = ArgNode; Entry.Ty = V->getType(); 5472 5473 // Skip the first return-type Attribute to get to params. 5474 Entry.setAttributes(&CS, i - CS.arg_begin() + 1); 5475 Args.push_back(Entry); 5476 } 5477 5478 if (LandingPad) { 5479 // Insert a label before the invoke call to mark the try range. This can be 5480 // used to detect deletion of the invoke via the MachineModuleInfo. 5481 BeginLabel = MMI.getContext().CreateTempSymbol(); 5482 5483 // For SjLj, keep track of which landing pads go with which invokes 5484 // so as to maintain the ordering of pads in the LSDA. 5485 unsigned CallSiteIndex = MMI.getCurrentCallSite(); 5486 if (CallSiteIndex) { 5487 MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex); 5488 LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex); 5489 5490 // Now that the call site is handled, stop tracking it. 5491 MMI.setCurrentCallSite(0); 5492 } 5493 5494 // Both PendingLoads and PendingExports must be flushed here; 5495 // this call might not return. 5496 (void)getRoot(); 5497 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel)); 5498 } 5499 5500 // Check if target-independent constraints permit a tail call here. 5501 // Target-dependent constraints are checked within TLI->LowerCallTo. 5502 if (isTailCall && !isInTailCallPosition(CS, *TLI)) 5503 isTailCall = false; 5504 5505 TargetLowering::CallLoweringInfo CLI(DAG); 5506 CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot()) 5507 .setCallee(RetTy, FTy, Callee, &Args, CS).setTailCall(isTailCall); 5508 5509 std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI); 5510 assert((isTailCall || Result.second.getNode()) && 5511 "Non-null chain expected with non-tail call!"); 5512 assert((Result.second.getNode() || !Result.first.getNode()) && 5513 "Null value expected with tail call!"); 5514 if (Result.first.getNode()) { 5515 setValue(CS.getInstruction(), Result.first); 5516 } else if (!CanLowerReturn && Result.second.getNode()) { 5517 // The instruction result is the result of loading from the 5518 // hidden sret parameter. 5519 SmallVector<EVT, 1> PVTs; 5520 Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType()); 5521 5522 ComputeValueVTs(*TLI, PtrRetTy, PVTs); 5523 assert(PVTs.size() == 1 && "Pointers should fit in one register"); 5524 EVT PtrVT = PVTs[0]; 5525 5526 SmallVector<EVT, 4> RetTys; 5527 SmallVector<uint64_t, 4> Offsets; 5528 RetTy = FTy->getReturnType(); 5529 ComputeValueVTs(*TLI, RetTy, RetTys, &Offsets); 5530 5531 unsigned NumValues = RetTys.size(); 5532 SmallVector<SDValue, 4> Values(NumValues); 5533 SmallVector<SDValue, 4> Chains(NumValues); 5534 5535 for (unsigned i = 0; i < NumValues; ++i) { 5536 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, 5537 DemoteStackSlot, 5538 DAG.getConstant(Offsets[i], PtrVT)); 5539 SDValue L = DAG.getLoad(RetTys[i], getCurSDLoc(), Result.second, Add, 5540 MachinePointerInfo::getFixedStack(DemoteStackIdx, Offsets[i]), 5541 false, false, false, 1); 5542 Values[i] = L; 5543 Chains[i] = L.getValue(1); 5544 } 5545 5546 SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 5547 MVT::Other, Chains); 5548 PendingLoads.push_back(Chain); 5549 5550 setValue(CS.getInstruction(), 5551 DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 5552 DAG.getVTList(RetTys), Values)); 5553 } 5554 5555 if (!Result.second.getNode()) { 5556 // As a special case, a null chain means that a tail call has been emitted 5557 // and the DAG root is already updated. 5558 HasTailCall = true; 5559 5560 // Since there's no actual continuation from this block, nothing can be 5561 // relying on us setting vregs for them. 5562 PendingExports.clear(); 5563 } else { 5564 DAG.setRoot(Result.second); 5565 } 5566 5567 if (LandingPad) { 5568 // Insert a label at the end of the invoke call to mark the try range. This 5569 // can be used to detect deletion of the invoke via the MachineModuleInfo. 5570 MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol(); 5571 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel)); 5572 5573 // Inform MachineModuleInfo of range. 5574 MMI.addInvoke(LandingPad, BeginLabel, EndLabel); 5575 } 5576} 5577 5578/// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the 5579/// value is equal or not-equal to zero. 5580static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) { 5581 for (const User *U : V->users()) { 5582 if (const ICmpInst *IC = dyn_cast<ICmpInst>(U)) 5583 if (IC->isEquality()) 5584 if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1))) 5585 if (C->isNullValue()) 5586 continue; 5587 // Unknown instruction. 5588 return false; 5589 } 5590 return true; 5591} 5592 5593static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, 5594 Type *LoadTy, 5595 SelectionDAGBuilder &Builder) { 5596 5597 // Check to see if this load can be trivially constant folded, e.g. if the 5598 // input is from a string literal. 5599 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { 5600 // Cast pointer to the type we really want to load. 5601 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput), 5602 PointerType::getUnqual(LoadTy)); 5603 5604 if (const Constant *LoadCst = 5605 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput), 5606 Builder.DL)) 5607 return Builder.getValue(LoadCst); 5608 } 5609 5610 // Otherwise, we have to emit the load. If the pointer is to unfoldable but 5611 // still constant memory, the input chain can be the entry node. 5612 SDValue Root; 5613 bool ConstantMemory = false; 5614 5615 // Do not serialize (non-volatile) loads of constant memory with anything. 5616 if (Builder.AA->pointsToConstantMemory(PtrVal)) { 5617 Root = Builder.DAG.getEntryNode(); 5618 ConstantMemory = true; 5619 } else { 5620 // Do not serialize non-volatile loads against each other. 5621 Root = Builder.DAG.getRoot(); 5622 } 5623 5624 SDValue Ptr = Builder.getValue(PtrVal); 5625 SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, 5626 Ptr, MachinePointerInfo(PtrVal), 5627 false /*volatile*/, 5628 false /*nontemporal*/, 5629 false /*isinvariant*/, 1 /* align=1 */); 5630 5631 if (!ConstantMemory) 5632 Builder.PendingLoads.push_back(LoadVal.getValue(1)); 5633 return LoadVal; 5634} 5635 5636/// processIntegerCallValue - Record the value for an instruction that 5637/// produces an integer result, converting the type where necessary. 5638void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I, 5639 SDValue Value, 5640 bool IsSigned) { 5641 EVT VT = TM.getTargetLowering()->getValueType(I.getType(), true); 5642 if (IsSigned) 5643 Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT); 5644 else 5645 Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT); 5646 setValue(&I, Value); 5647} 5648 5649/// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form. 5650/// If so, return true and lower it, otherwise return false and it will be 5651/// lowered like a normal call. 5652bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) { 5653 // Verify that the prototype makes sense. int memcmp(void*,void*,size_t) 5654 if (I.getNumArgOperands() != 3) 5655 return false; 5656 5657 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1); 5658 if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() || 5659 !I.getArgOperand(2)->getType()->isIntegerTy() || 5660 !I.getType()->isIntegerTy()) 5661 return false; 5662 5663 const Value *Size = I.getArgOperand(2); 5664 const ConstantInt *CSize = dyn_cast<ConstantInt>(Size); 5665 if (CSize && CSize->getZExtValue() == 0) { 5666 EVT CallVT = TM.getTargetLowering()->getValueType(I.getType(), true); 5667 setValue(&I, DAG.getConstant(0, CallVT)); 5668 return true; 5669 } 5670 5671 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5672 std::pair<SDValue, SDValue> Res = 5673 TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(), 5674 getValue(LHS), getValue(RHS), getValue(Size), 5675 MachinePointerInfo(LHS), 5676 MachinePointerInfo(RHS)); 5677 if (Res.first.getNode()) { 5678 processIntegerCallValue(I, Res.first, true); 5679 PendingLoads.push_back(Res.second); 5680 return true; 5681 } 5682 5683 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 5684 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 5685 if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) { 5686 bool ActuallyDoIt = true; 5687 MVT LoadVT; 5688 Type *LoadTy; 5689 switch (CSize->getZExtValue()) { 5690 default: 5691 LoadVT = MVT::Other; 5692 LoadTy = nullptr; 5693 ActuallyDoIt = false; 5694 break; 5695 case 2: 5696 LoadVT = MVT::i16; 5697 LoadTy = Type::getInt16Ty(CSize->getContext()); 5698 break; 5699 case 4: 5700 LoadVT = MVT::i32; 5701 LoadTy = Type::getInt32Ty(CSize->getContext()); 5702 break; 5703 case 8: 5704 LoadVT = MVT::i64; 5705 LoadTy = Type::getInt64Ty(CSize->getContext()); 5706 break; 5707 /* 5708 case 16: 5709 LoadVT = MVT::v4i32; 5710 LoadTy = Type::getInt32Ty(CSize->getContext()); 5711 LoadTy = VectorType::get(LoadTy, 4); 5712 break; 5713 */ 5714 } 5715 5716 // This turns into unaligned loads. We only do this if the target natively 5717 // supports the MVT we'll be loading or if it is small enough (<= 4) that 5718 // we'll only produce a small number of byte loads. 5719 5720 // Require that we can find a legal MVT, and only do this if the target 5721 // supports unaligned loads of that type. Expanding into byte loads would 5722 // bloat the code. 5723 const TargetLowering *TLI = TM.getTargetLowering(); 5724 if (ActuallyDoIt && CSize->getZExtValue() > 4) { 5725 unsigned DstAS = LHS->getType()->getPointerAddressSpace(); 5726 unsigned SrcAS = RHS->getType()->getPointerAddressSpace(); 5727 // TODO: Handle 5 byte compare as 4-byte + 1 byte. 5728 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. 5729 if (!TLI->isTypeLegal(LoadVT) || 5730 !TLI->allowsUnalignedMemoryAccesses(LoadVT, SrcAS) || 5731 !TLI->allowsUnalignedMemoryAccesses(LoadVT, DstAS)) 5732 ActuallyDoIt = false; 5733 } 5734 5735 if (ActuallyDoIt) { 5736 SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this); 5737 SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this); 5738 5739 SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal, 5740 ISD::SETNE); 5741 processIntegerCallValue(I, Res, false); 5742 return true; 5743 } 5744 } 5745 5746 5747 return false; 5748} 5749 5750/// visitMemChrCall -- See if we can lower a memchr call into an optimized 5751/// form. If so, return true and lower it, otherwise return false and it 5752/// will be lowered like a normal call. 5753bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { 5754 // Verify that the prototype makes sense. void *memchr(void *, int, size_t) 5755 if (I.getNumArgOperands() != 3) 5756 return false; 5757 5758 const Value *Src = I.getArgOperand(0); 5759 const Value *Char = I.getArgOperand(1); 5760 const Value *Length = I.getArgOperand(2); 5761 if (!Src->getType()->isPointerTy() || 5762 !Char->getType()->isIntegerTy() || 5763 !Length->getType()->isIntegerTy() || 5764 !I.getType()->isPointerTy()) 5765 return false; 5766 5767 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5768 std::pair<SDValue, SDValue> Res = 5769 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(), 5770 getValue(Src), getValue(Char), getValue(Length), 5771 MachinePointerInfo(Src)); 5772 if (Res.first.getNode()) { 5773 setValue(&I, Res.first); 5774 PendingLoads.push_back(Res.second); 5775 return true; 5776 } 5777 5778 return false; 5779} 5780 5781/// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an 5782/// optimized form. If so, return true and lower it, otherwise return false 5783/// and it will be lowered like a normal call. 5784bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { 5785 // Verify that the prototype makes sense. char *strcpy(char *, char *) 5786 if (I.getNumArgOperands() != 2) 5787 return false; 5788 5789 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5790 if (!Arg0->getType()->isPointerTy() || 5791 !Arg1->getType()->isPointerTy() || 5792 !I.getType()->isPointerTy()) 5793 return false; 5794 5795 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5796 std::pair<SDValue, SDValue> Res = 5797 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(), 5798 getValue(Arg0), getValue(Arg1), 5799 MachinePointerInfo(Arg0), 5800 MachinePointerInfo(Arg1), isStpcpy); 5801 if (Res.first.getNode()) { 5802 setValue(&I, Res.first); 5803 DAG.setRoot(Res.second); 5804 return true; 5805 } 5806 5807 return false; 5808} 5809 5810/// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form. 5811/// If so, return true and lower it, otherwise return false and it will be 5812/// lowered like a normal call. 5813bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { 5814 // Verify that the prototype makes sense. int strcmp(void*,void*) 5815 if (I.getNumArgOperands() != 2) 5816 return false; 5817 5818 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5819 if (!Arg0->getType()->isPointerTy() || 5820 !Arg1->getType()->isPointerTy() || 5821 !I.getType()->isIntegerTy()) 5822 return false; 5823 5824 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5825 std::pair<SDValue, SDValue> Res = 5826 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(), 5827 getValue(Arg0), getValue(Arg1), 5828 MachinePointerInfo(Arg0), 5829 MachinePointerInfo(Arg1)); 5830 if (Res.first.getNode()) { 5831 processIntegerCallValue(I, Res.first, true); 5832 PendingLoads.push_back(Res.second); 5833 return true; 5834 } 5835 5836 return false; 5837} 5838 5839/// visitStrLenCall -- See if we can lower a strlen call into an optimized 5840/// form. If so, return true and lower it, otherwise return false and it 5841/// will be lowered like a normal call. 5842bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { 5843 // Verify that the prototype makes sense. size_t strlen(char *) 5844 if (I.getNumArgOperands() != 1) 5845 return false; 5846 5847 const Value *Arg0 = I.getArgOperand(0); 5848 if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy()) 5849 return false; 5850 5851 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5852 std::pair<SDValue, SDValue> Res = 5853 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(), 5854 getValue(Arg0), MachinePointerInfo(Arg0)); 5855 if (Res.first.getNode()) { 5856 processIntegerCallValue(I, Res.first, false); 5857 PendingLoads.push_back(Res.second); 5858 return true; 5859 } 5860 5861 return false; 5862} 5863 5864/// visitStrNLenCall -- See if we can lower a strnlen call into an optimized 5865/// form. If so, return true and lower it, otherwise return false and it 5866/// will be lowered like a normal call. 5867bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { 5868 // Verify that the prototype makes sense. size_t strnlen(char *, size_t) 5869 if (I.getNumArgOperands() != 2) 5870 return false; 5871 5872 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5873 if (!Arg0->getType()->isPointerTy() || 5874 !Arg1->getType()->isIntegerTy() || 5875 !I.getType()->isIntegerTy()) 5876 return false; 5877 5878 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5879 std::pair<SDValue, SDValue> Res = 5880 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(), 5881 getValue(Arg0), getValue(Arg1), 5882 MachinePointerInfo(Arg0)); 5883 if (Res.first.getNode()) { 5884 processIntegerCallValue(I, Res.first, false); 5885 PendingLoads.push_back(Res.second); 5886 return true; 5887 } 5888 5889 return false; 5890} 5891 5892/// visitUnaryFloatCall - If a call instruction is a unary floating-point 5893/// operation (as expected), translate it to an SDNode with the specified opcode 5894/// and return true. 5895bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, 5896 unsigned Opcode) { 5897 // Sanity check that it really is a unary floating-point call. 5898 if (I.getNumArgOperands() != 1 || 5899 !I.getArgOperand(0)->getType()->isFloatingPointTy() || 5900 I.getType() != I.getArgOperand(0)->getType() || 5901 !I.onlyReadsMemory()) 5902 return false; 5903 5904 SDValue Tmp = getValue(I.getArgOperand(0)); 5905 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp)); 5906 return true; 5907} 5908 5909void SelectionDAGBuilder::visitCall(const CallInst &I) { 5910 // Handle inline assembly differently. 5911 if (isa<InlineAsm>(I.getCalledValue())) { 5912 visitInlineAsm(&I); 5913 return; 5914 } 5915 5916 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 5917 ComputeUsesVAFloatArgument(I, &MMI); 5918 5919 const char *RenameFn = nullptr; 5920 if (Function *F = I.getCalledFunction()) { 5921 if (F->isDeclaration()) { 5922 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) { 5923 if (unsigned IID = II->getIntrinsicID(F)) { 5924 RenameFn = visitIntrinsicCall(I, IID); 5925 if (!RenameFn) 5926 return; 5927 } 5928 } 5929 if (unsigned IID = F->getIntrinsicID()) { 5930 RenameFn = visitIntrinsicCall(I, IID); 5931 if (!RenameFn) 5932 return; 5933 } 5934 } 5935 5936 // Check for well-known libc/libm calls. If the function is internal, it 5937 // can't be a library call. 5938 LibFunc::Func Func; 5939 if (!F->hasLocalLinkage() && F->hasName() && 5940 LibInfo->getLibFunc(F->getName(), Func) && 5941 LibInfo->hasOptimizedCodeGen(Func)) { 5942 switch (Func) { 5943 default: break; 5944 case LibFunc::copysign: 5945 case LibFunc::copysignf: 5946 case LibFunc::copysignl: 5947 if (I.getNumArgOperands() == 2 && // Basic sanity checks. 5948 I.getArgOperand(0)->getType()->isFloatingPointTy() && 5949 I.getType() == I.getArgOperand(0)->getType() && 5950 I.getType() == I.getArgOperand(1)->getType() && 5951 I.onlyReadsMemory()) { 5952 SDValue LHS = getValue(I.getArgOperand(0)); 5953 SDValue RHS = getValue(I.getArgOperand(1)); 5954 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(), 5955 LHS.getValueType(), LHS, RHS)); 5956 return; 5957 } 5958 break; 5959 case LibFunc::fabs: 5960 case LibFunc::fabsf: 5961 case LibFunc::fabsl: 5962 if (visitUnaryFloatCall(I, ISD::FABS)) 5963 return; 5964 break; 5965 case LibFunc::sin: 5966 case LibFunc::sinf: 5967 case LibFunc::sinl: 5968 if (visitUnaryFloatCall(I, ISD::FSIN)) 5969 return; 5970 break; 5971 case LibFunc::cos: 5972 case LibFunc::cosf: 5973 case LibFunc::cosl: 5974 if (visitUnaryFloatCall(I, ISD::FCOS)) 5975 return; 5976 break; 5977 case LibFunc::sqrt: 5978 case LibFunc::sqrtf: 5979 case LibFunc::sqrtl: 5980 case LibFunc::sqrt_finite: 5981 case LibFunc::sqrtf_finite: 5982 case LibFunc::sqrtl_finite: 5983 if (visitUnaryFloatCall(I, ISD::FSQRT)) 5984 return; 5985 break; 5986 case LibFunc::floor: 5987 case LibFunc::floorf: 5988 case LibFunc::floorl: 5989 if (visitUnaryFloatCall(I, ISD::FFLOOR)) 5990 return; 5991 break; 5992 case LibFunc::nearbyint: 5993 case LibFunc::nearbyintf: 5994 case LibFunc::nearbyintl: 5995 if (visitUnaryFloatCall(I, ISD::FNEARBYINT)) 5996 return; 5997 break; 5998 case LibFunc::ceil: 5999 case LibFunc::ceilf: 6000 case LibFunc::ceill: 6001 if (visitUnaryFloatCall(I, ISD::FCEIL)) 6002 return; 6003 break; 6004 case LibFunc::rint: 6005 case LibFunc::rintf: 6006 case LibFunc::rintl: 6007 if (visitUnaryFloatCall(I, ISD::FRINT)) 6008 return; 6009 break; 6010 case LibFunc::round: 6011 case LibFunc::roundf: 6012 case LibFunc::roundl: 6013 if (visitUnaryFloatCall(I, ISD::FROUND)) 6014 return; 6015 break; 6016 case LibFunc::trunc: 6017 case LibFunc::truncf: 6018 case LibFunc::truncl: 6019 if (visitUnaryFloatCall(I, ISD::FTRUNC)) 6020 return; 6021 break; 6022 case LibFunc::log2: 6023 case LibFunc::log2f: 6024 case LibFunc::log2l: 6025 if (visitUnaryFloatCall(I, ISD::FLOG2)) 6026 return; 6027 break; 6028 case LibFunc::exp2: 6029 case LibFunc::exp2f: 6030 case LibFunc::exp2l: 6031 if (visitUnaryFloatCall(I, ISD::FEXP2)) 6032 return; 6033 break; 6034 case LibFunc::memcmp: 6035 if (visitMemCmpCall(I)) 6036 return; 6037 break; 6038 case LibFunc::memchr: 6039 if (visitMemChrCall(I)) 6040 return; 6041 break; 6042 case LibFunc::strcpy: 6043 if (visitStrCpyCall(I, false)) 6044 return; 6045 break; 6046 case LibFunc::stpcpy: 6047 if (visitStrCpyCall(I, true)) 6048 return; 6049 break; 6050 case LibFunc::strcmp: 6051 if (visitStrCmpCall(I)) 6052 return; 6053 break; 6054 case LibFunc::strlen: 6055 if (visitStrLenCall(I)) 6056 return; 6057 break; 6058 case LibFunc::strnlen: 6059 if (visitStrNLenCall(I)) 6060 return; 6061 break; 6062 } 6063 } 6064 } 6065 6066 SDValue Callee; 6067 if (!RenameFn) 6068 Callee = getValue(I.getCalledValue()); 6069 else 6070 Callee = DAG.getExternalSymbol(RenameFn, 6071 TM.getTargetLowering()->getPointerTy()); 6072 6073 // Check if we can potentially perform a tail call. More detailed checking is 6074 // be done within LowerCallTo, after more information about the call is known. 6075 LowerCallTo(&I, Callee, I.isTailCall()); 6076} 6077 6078namespace { 6079 6080/// AsmOperandInfo - This contains information for each constraint that we are 6081/// lowering. 6082class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { 6083public: 6084 /// CallOperand - If this is the result output operand or a clobber 6085 /// this is null, otherwise it is the incoming operand to the CallInst. 6086 /// This gets modified as the asm is processed. 6087 SDValue CallOperand; 6088 6089 /// AssignedRegs - If this is a register or register class operand, this 6090 /// contains the set of register corresponding to the operand. 6091 RegsForValue AssignedRegs; 6092 6093 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info) 6094 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) { 6095 } 6096 6097 /// getCallOperandValEVT - Return the EVT of the Value* that this operand 6098 /// corresponds to. If there is no Value* for this operand, it returns 6099 /// MVT::Other. 6100 EVT getCallOperandValEVT(LLVMContext &Context, 6101 const TargetLowering &TLI, 6102 const DataLayout *DL) const { 6103 if (!CallOperandVal) return MVT::Other; 6104 6105 if (isa<BasicBlock>(CallOperandVal)) 6106 return TLI.getPointerTy(); 6107 6108 llvm::Type *OpTy = CallOperandVal->getType(); 6109 6110 // FIXME: code duplicated from TargetLowering::ParseConstraints(). 6111 // If this is an indirect operand, the operand is a pointer to the 6112 // accessed type. 6113 if (isIndirect) { 6114 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); 6115 if (!PtrTy) 6116 report_fatal_error("Indirect operand for inline asm not a pointer!"); 6117 OpTy = PtrTy->getElementType(); 6118 } 6119 6120 // Look for vector wrapped in a struct. e.g. { <16 x i8> }. 6121 if (StructType *STy = dyn_cast<StructType>(OpTy)) 6122 if (STy->getNumElements() == 1) 6123 OpTy = STy->getElementType(0); 6124 6125 // If OpTy is not a single value, it may be a struct/union that we 6126 // can tile with integers. 6127 if (!OpTy->isSingleValueType() && OpTy->isSized()) { 6128 unsigned BitSize = DL->getTypeSizeInBits(OpTy); 6129 switch (BitSize) { 6130 default: break; 6131 case 1: 6132 case 8: 6133 case 16: 6134 case 32: 6135 case 64: 6136 case 128: 6137 OpTy = IntegerType::get(Context, BitSize); 6138 break; 6139 } 6140 } 6141 6142 return TLI.getValueType(OpTy, true); 6143 } 6144}; 6145 6146typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector; 6147 6148} // end anonymous namespace 6149 6150/// GetRegistersForValue - Assign registers (virtual or physical) for the 6151/// specified operand. We prefer to assign virtual registers, to allow the 6152/// register allocator to handle the assignment process. However, if the asm 6153/// uses features that we can't model on machineinstrs, we have SDISel do the 6154/// allocation. This produces generally horrible, but correct, code. 6155/// 6156/// OpInfo describes the operand. 6157/// 6158static void GetRegistersForValue(SelectionDAG &DAG, 6159 const TargetLowering &TLI, 6160 SDLoc DL, 6161 SDISelAsmOperandInfo &OpInfo) { 6162 LLVMContext &Context = *DAG.getContext(); 6163 6164 MachineFunction &MF = DAG.getMachineFunction(); 6165 SmallVector<unsigned, 4> Regs; 6166 6167 // If this is a constraint for a single physreg, or a constraint for a 6168 // register class, find it. 6169 std::pair<unsigned, const TargetRegisterClass*> PhysReg = 6170 TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode, 6171 OpInfo.ConstraintVT); 6172 6173 unsigned NumRegs = 1; 6174 if (OpInfo.ConstraintVT != MVT::Other) { 6175 // If this is a FP input in an integer register (or visa versa) insert a bit 6176 // cast of the input value. More generally, handle any case where the input 6177 // value disagrees with the register class we plan to stick this in. 6178 if (OpInfo.Type == InlineAsm::isInput && 6179 PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) { 6180 // Try to convert to the first EVT that the reg class contains. If the 6181 // types are identical size, use a bitcast to convert (e.g. two differing 6182 // vector types). 6183 MVT RegVT = *PhysReg.second->vt_begin(); 6184 if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) { 6185 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL, 6186 RegVT, OpInfo.CallOperand); 6187 OpInfo.ConstraintVT = RegVT; 6188 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) { 6189 // If the input is a FP value and we want it in FP registers, do a 6190 // bitcast to the corresponding integer type. This turns an f64 value 6191 // into i64, which can be passed with two i32 values on a 32-bit 6192 // machine. 6193 RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits()); 6194 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL, 6195 RegVT, OpInfo.CallOperand); 6196 OpInfo.ConstraintVT = RegVT; 6197 } 6198 } 6199 6200 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT); 6201 } 6202 6203 MVT RegVT; 6204 EVT ValueVT = OpInfo.ConstraintVT; 6205 6206 // If this is a constraint for a specific physical register, like {r17}, 6207 // assign it now. 6208 if (unsigned AssignedReg = PhysReg.first) { 6209 const TargetRegisterClass *RC = PhysReg.second; 6210 if (OpInfo.ConstraintVT == MVT::Other) 6211 ValueVT = *RC->vt_begin(); 6212 6213 // Get the actual register value type. This is important, because the user 6214 // may have asked for (e.g.) the AX register in i32 type. We need to 6215 // remember that AX is actually i16 to get the right extension. 6216 RegVT = *RC->vt_begin(); 6217 6218 // This is a explicit reference to a physical register. 6219 Regs.push_back(AssignedReg); 6220 6221 // If this is an expanded reference, add the rest of the regs to Regs. 6222 if (NumRegs != 1) { 6223 TargetRegisterClass::iterator I = RC->begin(); 6224 for (; *I != AssignedReg; ++I) 6225 assert(I != RC->end() && "Didn't find reg!"); 6226 6227 // Already added the first reg. 6228 --NumRegs; ++I; 6229 for (; NumRegs; --NumRegs, ++I) { 6230 assert(I != RC->end() && "Ran out of registers to allocate!"); 6231 Regs.push_back(*I); 6232 } 6233 } 6234 6235 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); 6236 return; 6237 } 6238 6239 // Otherwise, if this was a reference to an LLVM register class, create vregs 6240 // for this reference. 6241 if (const TargetRegisterClass *RC = PhysReg.second) { 6242 RegVT = *RC->vt_begin(); 6243 if (OpInfo.ConstraintVT == MVT::Other) 6244 ValueVT = RegVT; 6245 6246 // Create the appropriate number of virtual registers. 6247 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 6248 for (; NumRegs; --NumRegs) 6249 Regs.push_back(RegInfo.createVirtualRegister(RC)); 6250 6251 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); 6252 return; 6253 } 6254 6255 // Otherwise, we couldn't allocate enough registers for this. 6256} 6257 6258/// visitInlineAsm - Handle a call to an InlineAsm object. 6259/// 6260void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { 6261 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue()); 6262 6263 /// ConstraintOperands - Information about all of the constraints. 6264 SDISelAsmOperandInfoVector ConstraintOperands; 6265 6266 const TargetLowering *TLI = TM.getTargetLowering(); 6267 TargetLowering::AsmOperandInfoVector 6268 TargetConstraints = TLI->ParseConstraints(CS); 6269 6270 bool hasMemory = false; 6271 6272 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. 6273 unsigned ResNo = 0; // ResNo - The result number of the next output. 6274 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 6275 ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i])); 6276 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); 6277 6278 MVT OpVT = MVT::Other; 6279 6280 // Compute the value type for each operand. 6281 switch (OpInfo.Type) { 6282 case InlineAsm::isOutput: 6283 // Indirect outputs just consume an argument. 6284 if (OpInfo.isIndirect) { 6285 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++)); 6286 break; 6287 } 6288 6289 // The return value of the call is this value. As such, there is no 6290 // corresponding argument. 6291 assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); 6292 if (StructType *STy = dyn_cast<StructType>(CS.getType())) { 6293 OpVT = TLI->getSimpleValueType(STy->getElementType(ResNo)); 6294 } else { 6295 assert(ResNo == 0 && "Asm only has one result!"); 6296 OpVT = TLI->getSimpleValueType(CS.getType()); 6297 } 6298 ++ResNo; 6299 break; 6300 case InlineAsm::isInput: 6301 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++)); 6302 break; 6303 case InlineAsm::isClobber: 6304 // Nothing to do. 6305 break; 6306 } 6307 6308 // If this is an input or an indirect output, process the call argument. 6309 // BasicBlocks are labels, currently appearing only in asm's. 6310 if (OpInfo.CallOperandVal) { 6311 if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) { 6312 OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); 6313 } else { 6314 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); 6315 } 6316 6317 OpVT = OpInfo.getCallOperandValEVT(*DAG.getContext(), *TLI, DL). 6318 getSimpleVT(); 6319 } 6320 6321 OpInfo.ConstraintVT = OpVT; 6322 6323 // Indirect operand accesses access memory. 6324 if (OpInfo.isIndirect) 6325 hasMemory = true; 6326 else { 6327 for (unsigned j = 0, ee = OpInfo.Codes.size(); j != ee; ++j) { 6328 TargetLowering::ConstraintType 6329 CType = TLI->getConstraintType(OpInfo.Codes[j]); 6330 if (CType == TargetLowering::C_Memory) { 6331 hasMemory = true; 6332 break; 6333 } 6334 } 6335 } 6336 } 6337 6338 SDValue Chain, Flag; 6339 6340 // We won't need to flush pending loads if this asm doesn't touch 6341 // memory and is nonvolatile. 6342 if (hasMemory || IA->hasSideEffects()) 6343 Chain = getRoot(); 6344 else 6345 Chain = DAG.getRoot(); 6346 6347 // Second pass over the constraints: compute which constraint option to use 6348 // and assign registers to constraints that want a specific physreg. 6349 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6350 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6351 6352 // If this is an output operand with a matching input operand, look up the 6353 // matching input. If their types mismatch, e.g. one is an integer, the 6354 // other is floating point, or their sizes are different, flag it as an 6355 // error. 6356 if (OpInfo.hasMatchingInput()) { 6357 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput]; 6358 6359 if (OpInfo.ConstraintVT != Input.ConstraintVT) { 6360 std::pair<unsigned, const TargetRegisterClass*> MatchRC = 6361 TLI->getRegForInlineAsmConstraint(OpInfo.ConstraintCode, 6362 OpInfo.ConstraintVT); 6363 std::pair<unsigned, const TargetRegisterClass*> InputRC = 6364 TLI->getRegForInlineAsmConstraint(Input.ConstraintCode, 6365 Input.ConstraintVT); 6366 if ((OpInfo.ConstraintVT.isInteger() != 6367 Input.ConstraintVT.isInteger()) || 6368 (MatchRC.second != InputRC.second)) { 6369 report_fatal_error("Unsupported asm: input constraint" 6370 " with a matching output constraint of" 6371 " incompatible type!"); 6372 } 6373 Input.ConstraintVT = OpInfo.ConstraintVT; 6374 } 6375 } 6376 6377 // Compute the constraint code and ConstraintType to use. 6378 TLI->ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); 6379 6380 if (OpInfo.ConstraintType == TargetLowering::C_Memory && 6381 OpInfo.Type == InlineAsm::isClobber) 6382 continue; 6383 6384 // If this is a memory input, and if the operand is not indirect, do what we 6385 // need to to provide an address for the memory input. 6386 if (OpInfo.ConstraintType == TargetLowering::C_Memory && 6387 !OpInfo.isIndirect) { 6388 assert((OpInfo.isMultipleAlternative || 6389 (OpInfo.Type == InlineAsm::isInput)) && 6390 "Can only indirectify direct input operands!"); 6391 6392 // Memory operands really want the address of the value. If we don't have 6393 // an indirect input, put it in the constpool if we can, otherwise spill 6394 // it to a stack slot. 6395 // TODO: This isn't quite right. We need to handle these according to 6396 // the addressing mode that the constraint wants. Also, this may take 6397 // an additional register for the computation and we don't want that 6398 // either. 6399 6400 // If the operand is a float, integer, or vector constant, spill to a 6401 // constant pool entry to get its address. 6402 const Value *OpVal = OpInfo.CallOperandVal; 6403 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) || 6404 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) { 6405 OpInfo.CallOperand = DAG.getConstantPool(cast<Constant>(OpVal), 6406 TLI->getPointerTy()); 6407 } else { 6408 // Otherwise, create a stack slot and emit a store to it before the 6409 // asm. 6410 Type *Ty = OpVal->getType(); 6411 uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty); 6412 unsigned Align = TLI->getDataLayout()->getPrefTypeAlignment(Ty); 6413 MachineFunction &MF = DAG.getMachineFunction(); 6414 int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); 6415 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI->getPointerTy()); 6416 Chain = DAG.getStore(Chain, getCurSDLoc(), 6417 OpInfo.CallOperand, StackSlot, 6418 MachinePointerInfo::getFixedStack(SSFI), 6419 false, false, 0); 6420 OpInfo.CallOperand = StackSlot; 6421 } 6422 6423 // There is no longer a Value* corresponding to this operand. 6424 OpInfo.CallOperandVal = nullptr; 6425 6426 // It is now an indirect operand. 6427 OpInfo.isIndirect = true; 6428 } 6429 6430 // If this constraint is for a specific register, allocate it before 6431 // anything else. 6432 if (OpInfo.ConstraintType == TargetLowering::C_Register) 6433 GetRegistersForValue(DAG, *TLI, getCurSDLoc(), OpInfo); 6434 } 6435 6436 // Second pass - Loop over all of the operands, assigning virtual or physregs 6437 // to register class operands. 6438 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6439 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6440 6441 // C_Register operands have already been allocated, Other/Memory don't need 6442 // to be. 6443 if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass) 6444 GetRegistersForValue(DAG, *TLI, getCurSDLoc(), OpInfo); 6445 } 6446 6447 // AsmNodeOperands - The operands for the ISD::INLINEASM node. 6448 std::vector<SDValue> AsmNodeOperands; 6449 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain 6450 AsmNodeOperands.push_back( 6451 DAG.getTargetExternalSymbol(IA->getAsmString().c_str(), 6452 TLI->getPointerTy())); 6453 6454 // If we have a !srcloc metadata node associated with it, we want to attach 6455 // this to the ultimately generated inline asm machineinstr. To do this, we 6456 // pass in the third operand as this (potentially null) inline asm MDNode. 6457 const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc"); 6458 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc)); 6459 6460 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore 6461 // bits as operand 3. 6462 unsigned ExtraInfo = 0; 6463 if (IA->hasSideEffects()) 6464 ExtraInfo |= InlineAsm::Extra_HasSideEffects; 6465 if (IA->isAlignStack()) 6466 ExtraInfo |= InlineAsm::Extra_IsAlignStack; 6467 // Set the asm dialect. 6468 ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect; 6469 6470 // Determine if this InlineAsm MayLoad or MayStore based on the constraints. 6471 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 6472 TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 6473 6474 // Compute the constraint code and ConstraintType to use. 6475 TLI->ComputeConstraintToUse(OpInfo, SDValue()); 6476 6477 // Ideally, we would only check against memory constraints. However, the 6478 // meaning of an other constraint can be target-specific and we can't easily 6479 // reason about it. Therefore, be conservative and set MayLoad/MayStore 6480 // for other constriants as well. 6481 if (OpInfo.ConstraintType == TargetLowering::C_Memory || 6482 OpInfo.ConstraintType == TargetLowering::C_Other) { 6483 if (OpInfo.Type == InlineAsm::isInput) 6484 ExtraInfo |= InlineAsm::Extra_MayLoad; 6485 else if (OpInfo.Type == InlineAsm::isOutput) 6486 ExtraInfo |= InlineAsm::Extra_MayStore; 6487 else if (OpInfo.Type == InlineAsm::isClobber) 6488 ExtraInfo |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore); 6489 } 6490 } 6491 6492 AsmNodeOperands.push_back(DAG.getTargetConstant(ExtraInfo, 6493 TLI->getPointerTy())); 6494 6495 // Loop over all of the inputs, copying the operand values into the 6496 // appropriate registers and processing the output regs. 6497 RegsForValue RetValRegs; 6498 6499 // IndirectStoresToEmit - The set of stores to emit after the inline asm node. 6500 std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit; 6501 6502 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6503 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6504 6505 switch (OpInfo.Type) { 6506 case InlineAsm::isOutput: { 6507 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass && 6508 OpInfo.ConstraintType != TargetLowering::C_Register) { 6509 // Memory output, or 'other' output (e.g. 'X' constraint). 6510 assert(OpInfo.isIndirect && "Memory output must be indirect operand"); 6511 6512 // Add information to the INLINEASM node to know about this output. 6513 unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 6514 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, 6515 TLI->getPointerTy())); 6516 AsmNodeOperands.push_back(OpInfo.CallOperand); 6517 break; 6518 } 6519 6520 // Otherwise, this is a register or register class output. 6521 6522 // Copy the output from the appropriate register. Find a register that 6523 // we can use. 6524 if (OpInfo.AssignedRegs.Regs.empty()) { 6525 LLVMContext &Ctx = *DAG.getContext(); 6526 Ctx.emitError(CS.getInstruction(), 6527 "couldn't allocate output register for constraint '" + 6528 Twine(OpInfo.ConstraintCode) + "'"); 6529 return; 6530 } 6531 6532 // If this is an indirect operand, store through the pointer after the 6533 // asm. 6534 if (OpInfo.isIndirect) { 6535 IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs, 6536 OpInfo.CallOperandVal)); 6537 } else { 6538 // This is the result value of the call. 6539 assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); 6540 // Concatenate this output onto the outputs list. 6541 RetValRegs.append(OpInfo.AssignedRegs); 6542 } 6543 6544 // Add information to the INLINEASM node to know that this register is 6545 // set. 6546 OpInfo.AssignedRegs 6547 .AddInlineAsmOperands(OpInfo.isEarlyClobber 6548 ? InlineAsm::Kind_RegDefEarlyClobber 6549 : InlineAsm::Kind_RegDef, 6550 false, 0, DAG, AsmNodeOperands); 6551 break; 6552 } 6553 case InlineAsm::isInput: { 6554 SDValue InOperandVal = OpInfo.CallOperand; 6555 6556 if (OpInfo.isMatchingInputConstraint()) { // Matching constraint? 6557 // If this is required to match an output register we have already set, 6558 // just use its register. 6559 unsigned OperandNo = OpInfo.getMatchedOperand(); 6560 6561 // Scan until we find the definition we already emitted of this operand. 6562 // When we find it, create a RegsForValue operand. 6563 unsigned CurOp = InlineAsm::Op_FirstOperand; 6564 for (; OperandNo; --OperandNo) { 6565 // Advance to the next operand. 6566 unsigned OpFlag = 6567 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); 6568 assert((InlineAsm::isRegDefKind(OpFlag) || 6569 InlineAsm::isRegDefEarlyClobberKind(OpFlag) || 6570 InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"); 6571 CurOp += InlineAsm::getNumOperandRegisters(OpFlag)+1; 6572 } 6573 6574 unsigned OpFlag = 6575 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); 6576 if (InlineAsm::isRegDefKind(OpFlag) || 6577 InlineAsm::isRegDefEarlyClobberKind(OpFlag)) { 6578 // Add (OpFlag&0xffff)>>3 registers to MatchedRegs. 6579 if (OpInfo.isIndirect) { 6580 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c 6581 LLVMContext &Ctx = *DAG.getContext(); 6582 Ctx.emitError(CS.getInstruction(), "inline asm not supported yet:" 6583 " don't know how to handle tied " 6584 "indirect register inputs"); 6585 return; 6586 } 6587 6588 RegsForValue MatchedRegs; 6589 MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType()); 6590 MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType(); 6591 MatchedRegs.RegVTs.push_back(RegVT); 6592 MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo(); 6593 for (unsigned i = 0, e = InlineAsm::getNumOperandRegisters(OpFlag); 6594 i != e; ++i) { 6595 if (const TargetRegisterClass *RC = TLI->getRegClassFor(RegVT)) 6596 MatchedRegs.Regs.push_back(RegInfo.createVirtualRegister(RC)); 6597 else { 6598 LLVMContext &Ctx = *DAG.getContext(); 6599 Ctx.emitError(CS.getInstruction(), 6600 "inline asm error: This value" 6601 " type register class is not natively supported!"); 6602 return; 6603 } 6604 } 6605 // Use the produced MatchedRegs object to 6606 MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurSDLoc(), 6607 Chain, &Flag, CS.getInstruction()); 6608 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, 6609 true, OpInfo.getMatchedOperand(), 6610 DAG, AsmNodeOperands); 6611 break; 6612 } 6613 6614 assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!"); 6615 assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 && 6616 "Unexpected number of operands"); 6617 // Add information to the INLINEASM node to know about this input. 6618 // See InlineAsm.h isUseOperandTiedToDef. 6619 OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag, 6620 OpInfo.getMatchedOperand()); 6621 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlag, 6622 TLI->getPointerTy())); 6623 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); 6624 break; 6625 } 6626 6627 // Treat indirect 'X' constraint as memory. 6628 if (OpInfo.ConstraintType == TargetLowering::C_Other && 6629 OpInfo.isIndirect) 6630 OpInfo.ConstraintType = TargetLowering::C_Memory; 6631 6632 if (OpInfo.ConstraintType == TargetLowering::C_Other) { 6633 std::vector<SDValue> Ops; 6634 TLI->LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode, 6635 Ops, DAG); 6636 if (Ops.empty()) { 6637 LLVMContext &Ctx = *DAG.getContext(); 6638 Ctx.emitError(CS.getInstruction(), 6639 "invalid operand for inline asm constraint '" + 6640 Twine(OpInfo.ConstraintCode) + "'"); 6641 return; 6642 } 6643 6644 // Add information to the INLINEASM node to know about this input. 6645 unsigned ResOpType = 6646 InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size()); 6647 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, 6648 TLI->getPointerTy())); 6649 AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end()); 6650 break; 6651 } 6652 6653 if (OpInfo.ConstraintType == TargetLowering::C_Memory) { 6654 assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!"); 6655 assert(InOperandVal.getValueType() == TLI->getPointerTy() && 6656 "Memory operands expect pointer values"); 6657 6658 // Add information to the INLINEASM node to know about this input. 6659 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 6660 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, 6661 TLI->getPointerTy())); 6662 AsmNodeOperands.push_back(InOperandVal); 6663 break; 6664 } 6665 6666 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass || 6667 OpInfo.ConstraintType == TargetLowering::C_Register) && 6668 "Unknown constraint type!"); 6669 6670 // TODO: Support this. 6671 if (OpInfo.isIndirect) { 6672 LLVMContext &Ctx = *DAG.getContext(); 6673 Ctx.emitError(CS.getInstruction(), 6674 "Don't know how to handle indirect register inputs yet " 6675 "for constraint '" + 6676 Twine(OpInfo.ConstraintCode) + "'"); 6677 return; 6678 } 6679 6680 // Copy the input into the appropriate registers. 6681 if (OpInfo.AssignedRegs.Regs.empty()) { 6682 LLVMContext &Ctx = *DAG.getContext(); 6683 Ctx.emitError(CS.getInstruction(), 6684 "couldn't allocate input reg for constraint '" + 6685 Twine(OpInfo.ConstraintCode) + "'"); 6686 return; 6687 } 6688 6689 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurSDLoc(), 6690 Chain, &Flag, CS.getInstruction()); 6691 6692 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0, 6693 DAG, AsmNodeOperands); 6694 break; 6695 } 6696 case InlineAsm::isClobber: { 6697 // Add the clobbered value to the operand list, so that the register 6698 // allocator is aware that the physreg got clobbered. 6699 if (!OpInfo.AssignedRegs.Regs.empty()) 6700 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber, 6701 false, 0, DAG, 6702 AsmNodeOperands); 6703 break; 6704 } 6705 } 6706 } 6707 6708 // Finish up input operands. Set the input chain and add the flag last. 6709 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; 6710 if (Flag.getNode()) AsmNodeOperands.push_back(Flag); 6711 6712 Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(), 6713 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); 6714 Flag = Chain.getValue(1); 6715 6716 // If this asm returns a register value, copy the result from that register 6717 // and set it as the value of the call. 6718 if (!RetValRegs.Regs.empty()) { 6719 SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), 6720 Chain, &Flag, CS.getInstruction()); 6721 6722 // FIXME: Why don't we do this for inline asms with MRVs? 6723 if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) { 6724 EVT ResultType = TLI->getValueType(CS.getType()); 6725 6726 // If any of the results of the inline asm is a vector, it may have the 6727 // wrong width/num elts. This can happen for register classes that can 6728 // contain multiple different value types. The preg or vreg allocated may 6729 // not have the same VT as was expected. Convert it to the right type 6730 // with bit_convert. 6731 if (ResultType != Val.getValueType() && Val.getValueType().isVector()) { 6732 Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(), 6733 ResultType, Val); 6734 6735 } else if (ResultType != Val.getValueType() && 6736 ResultType.isInteger() && Val.getValueType().isInteger()) { 6737 // If a result value was tied to an input value, the computed result may 6738 // have a wider width than the expected result. Extract the relevant 6739 // portion. 6740 Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val); 6741 } 6742 6743 assert(ResultType == Val.getValueType() && "Asm result value mismatch!"); 6744 } 6745 6746 setValue(CS.getInstruction(), Val); 6747 // Don't need to use this as a chain in this case. 6748 if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty()) 6749 return; 6750 } 6751 6752 std::vector<std::pair<SDValue, const Value *> > StoresToEmit; 6753 6754 // Process indirect outputs, first output all of the flagged copies out of 6755 // physregs. 6756 for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { 6757 RegsForValue &OutRegs = IndirectStoresToEmit[i].first; 6758 const Value *Ptr = IndirectStoresToEmit[i].second; 6759 SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), 6760 Chain, &Flag, IA); 6761 StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); 6762 } 6763 6764 // Emit the non-flagged stores from the physregs. 6765 SmallVector<SDValue, 8> OutChains; 6766 for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) { 6767 SDValue Val = DAG.getStore(Chain, getCurSDLoc(), 6768 StoresToEmit[i].first, 6769 getValue(StoresToEmit[i].second), 6770 MachinePointerInfo(StoresToEmit[i].second), 6771 false, false, 0); 6772 OutChains.push_back(Val); 6773 } 6774 6775 if (!OutChains.empty()) 6776 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains); 6777 6778 DAG.setRoot(Chain); 6779} 6780 6781void SelectionDAGBuilder::visitVAStart(const CallInst &I) { 6782 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), 6783 MVT::Other, getRoot(), 6784 getValue(I.getArgOperand(0)), 6785 DAG.getSrcValue(I.getArgOperand(0)))); 6786} 6787 6788void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { 6789 const TargetLowering *TLI = TM.getTargetLowering(); 6790 const DataLayout &DL = *TLI->getDataLayout(); 6791 SDValue V = DAG.getVAArg(TLI->getValueType(I.getType()), getCurSDLoc(), 6792 getRoot(), getValue(I.getOperand(0)), 6793 DAG.getSrcValue(I.getOperand(0)), 6794 DL.getABITypeAlignment(I.getType())); 6795 setValue(&I, V); 6796 DAG.setRoot(V.getValue(1)); 6797} 6798 6799void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { 6800 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), 6801 MVT::Other, getRoot(), 6802 getValue(I.getArgOperand(0)), 6803 DAG.getSrcValue(I.getArgOperand(0)))); 6804} 6805 6806void SelectionDAGBuilder::visitVACopy(const CallInst &I) { 6807 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), 6808 MVT::Other, getRoot(), 6809 getValue(I.getArgOperand(0)), 6810 getValue(I.getArgOperand(1)), 6811 DAG.getSrcValue(I.getArgOperand(0)), 6812 DAG.getSrcValue(I.getArgOperand(1)))); 6813} 6814 6815/// \brief Lower an argument list according to the target calling convention. 6816/// 6817/// \return A tuple of <return-value, token-chain> 6818/// 6819/// This is a helper for lowering intrinsics that follow a target calling 6820/// convention or require stack pointer adjustment. Only a subset of the 6821/// intrinsic's operands need to participate in the calling convention. 6822std::pair<SDValue, SDValue> 6823SelectionDAGBuilder::LowerCallOperands(const CallInst &CI, unsigned ArgIdx, 6824 unsigned NumArgs, SDValue Callee, 6825 bool useVoidTy) { 6826 TargetLowering::ArgListTy Args; 6827 Args.reserve(NumArgs); 6828 6829 // Populate the argument list. 6830 // Attributes for args start at offset 1, after the return attribute. 6831 ImmutableCallSite CS(&CI); 6832 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1; 6833 ArgI != ArgE; ++ArgI) { 6834 const Value *V = CI.getOperand(ArgI); 6835 6836 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); 6837 6838 TargetLowering::ArgListEntry Entry; 6839 Entry.Node = getValue(V); 6840 Entry.Ty = V->getType(); 6841 Entry.setAttributes(&CS, AttrI); 6842 Args.push_back(Entry); 6843 } 6844 6845 Type *retTy = useVoidTy ? Type::getVoidTy(*DAG.getContext()) : CI.getType(); 6846 TargetLowering::CallLoweringInfo CLI(DAG); 6847 CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot()) 6848 .setCallee(CI.getCallingConv(), retTy, Callee, &Args, NumArgs) 6849 .setDiscardResult(!CI.use_empty()); 6850 6851 const TargetLowering *TLI = TM.getTargetLowering(); 6852 return TLI->LowerCallTo(CLI); 6853} 6854 6855/// \brief Add a stack map intrinsic call's live variable operands to a stackmap 6856/// or patchpoint target node's operand list. 6857/// 6858/// Constants are converted to TargetConstants purely as an optimization to 6859/// avoid constant materialization and register allocation. 6860/// 6861/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not 6862/// generate addess computation nodes, and so ExpandISelPseudo can convert the 6863/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids 6864/// address materialization and register allocation, but may also be required 6865/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an 6866/// alloca in the entry block, then the runtime may assume that the alloca's 6867/// StackMap location can be read immediately after compilation and that the 6868/// location is valid at any point during execution (this is similar to the 6869/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were 6870/// only available in a register, then the runtime would need to trap when 6871/// execution reaches the StackMap in order to read the alloca's location. 6872static void addStackMapLiveVars(const CallInst &CI, unsigned StartIdx, 6873 SmallVectorImpl<SDValue> &Ops, 6874 SelectionDAGBuilder &Builder) { 6875 for (unsigned i = StartIdx, e = CI.getNumArgOperands(); i != e; ++i) { 6876 SDValue OpVal = Builder.getValue(CI.getArgOperand(i)); 6877 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) { 6878 Ops.push_back( 6879 Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64)); 6880 Ops.push_back( 6881 Builder.DAG.getTargetConstant(C->getSExtValue(), MVT::i64)); 6882 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) { 6883 const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo(); 6884 Ops.push_back( 6885 Builder.DAG.getTargetFrameIndex(FI->getIndex(), TLI.getPointerTy())); 6886 } else 6887 Ops.push_back(OpVal); 6888 } 6889} 6890 6891/// \brief Lower llvm.experimental.stackmap directly to its target opcode. 6892void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { 6893 // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>, 6894 // [live variables...]) 6895 6896 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value."); 6897 6898 SDValue Chain, InFlag, Callee, NullPtr; 6899 SmallVector<SDValue, 32> Ops; 6900 6901 SDLoc DL = getCurSDLoc(); 6902 Callee = getValue(CI.getCalledValue()); 6903 NullPtr = DAG.getIntPtrConstant(0, true); 6904 6905 // The stackmap intrinsic only records the live variables (the arguemnts 6906 // passed to it) and emits NOPS (if requested). Unlike the patchpoint 6907 // intrinsic, this won't be lowered to a function call. This means we don't 6908 // have to worry about calling conventions and target specific lowering code. 6909 // Instead we perform the call lowering right here. 6910 // 6911 // chain, flag = CALLSEQ_START(chain, 0) 6912 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag) 6913 // chain, flag = CALLSEQ_END(chain, 0, 0, flag) 6914 // 6915 Chain = DAG.getCALLSEQ_START(getRoot(), NullPtr, DL); 6916 InFlag = Chain.getValue(1); 6917 6918 // Add the <id> and <numBytes> constants. 6919 SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos)); 6920 Ops.push_back(DAG.getTargetConstant( 6921 cast<ConstantSDNode>(IDVal)->getZExtValue(), MVT::i64)); 6922 SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos)); 6923 Ops.push_back(DAG.getTargetConstant( 6924 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), MVT::i32)); 6925 6926 // Push live variables for the stack map. 6927 addStackMapLiveVars(CI, 2, Ops, *this); 6928 6929 // We are not pushing any register mask info here on the operands list, 6930 // because the stackmap doesn't clobber anything. 6931 6932 // Push the chain and the glue flag. 6933 Ops.push_back(Chain); 6934 Ops.push_back(InFlag); 6935 6936 // Create the STACKMAP node. 6937 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 6938 SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops); 6939 Chain = SDValue(SM, 0); 6940 InFlag = Chain.getValue(1); 6941 6942 Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL); 6943 6944 // Stackmaps don't generate values, so nothing goes into the NodeMap. 6945 6946 // Set the root to the target-lowered call chain. 6947 DAG.setRoot(Chain); 6948 6949 // Inform the Frame Information that we have a stackmap in this function. 6950 FuncInfo.MF->getFrameInfo()->setHasStackMap(); 6951} 6952 6953/// \brief Lower llvm.experimental.patchpoint directly to its target opcode. 6954void SelectionDAGBuilder::visitPatchpoint(const CallInst &CI) { 6955 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>, 6956 // i32 <numBytes>, 6957 // i8* <target>, 6958 // i32 <numArgs>, 6959 // [Args...], 6960 // [live variables...]) 6961 6962 CallingConv::ID CC = CI.getCallingConv(); 6963 bool isAnyRegCC = CC == CallingConv::AnyReg; 6964 bool hasDef = !CI.getType()->isVoidTy(); 6965 SDValue Callee = getValue(CI.getOperand(2)); // <target> 6966 6967 // Get the real number of arguments participating in the call <numArgs> 6968 SDValue NArgVal = getValue(CI.getArgOperand(PatchPointOpers::NArgPos)); 6969 unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue(); 6970 6971 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> 6972 // Intrinsics include all meta-operands up to but not including CC. 6973 unsigned NumMetaOpers = PatchPointOpers::CCPos; 6974 assert(CI.getNumArgOperands() >= NumMetaOpers + NumArgs && 6975 "Not enough arguments provided to the patchpoint intrinsic"); 6976 6977 // For AnyRegCC the arguments are lowered later on manually. 6978 unsigned NumCallArgs = isAnyRegCC ? 0 : NumArgs; 6979 std::pair<SDValue, SDValue> Result = 6980 LowerCallOperands(CI, NumMetaOpers, NumCallArgs, Callee, isAnyRegCC); 6981 6982 // Set the root to the target-lowered call chain. 6983 SDValue Chain = Result.second; 6984 DAG.setRoot(Chain); 6985 6986 SDNode *CallEnd = Chain.getNode(); 6987 if (hasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) 6988 CallEnd = CallEnd->getOperand(0).getNode(); 6989 6990 /// Get a call instruction from the call sequence chain. 6991 /// Tail calls are not allowed. 6992 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && 6993 "Expected a callseq node."); 6994 SDNode *Call = CallEnd->getOperand(0).getNode(); 6995 bool hasGlue = Call->getGluedNode(); 6996 6997 // Replace the target specific call node with the patchable intrinsic. 6998 SmallVector<SDValue, 8> Ops; 6999 7000 // Add the <id> and <numBytes> constants. 7001 SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos)); 7002 Ops.push_back(DAG.getTargetConstant( 7003 cast<ConstantSDNode>(IDVal)->getZExtValue(), MVT::i64)); 7004 SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos)); 7005 Ops.push_back(DAG.getTargetConstant( 7006 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), MVT::i32)); 7007 7008 // Assume that the Callee is a constant address. 7009 // FIXME: handle function symbols in the future. 7010 Ops.push_back( 7011 DAG.getIntPtrConstant(cast<ConstantSDNode>(Callee)->getZExtValue(), 7012 /*isTarget=*/true)); 7013 7014 // Adjust <numArgs> to account for any arguments that have been passed on the 7015 // stack instead. 7016 // Call Node: Chain, Target, {Args}, RegMask, [Glue] 7017 unsigned NumCallRegArgs = Call->getNumOperands() - (hasGlue ? 4 : 3); 7018 NumCallRegArgs = isAnyRegCC ? NumArgs : NumCallRegArgs; 7019 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, MVT::i32)); 7020 7021 // Add the calling convention 7022 Ops.push_back(DAG.getTargetConstant((unsigned)CC, MVT::i32)); 7023 7024 // Add the arguments we omitted previously. The register allocator should 7025 // place these in any free register. 7026 if (isAnyRegCC) 7027 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) 7028 Ops.push_back(getValue(CI.getArgOperand(i))); 7029 7030 // Push the arguments from the call instruction up to the register mask. 7031 SDNode::op_iterator e = hasGlue ? Call->op_end()-2 : Call->op_end()-1; 7032 for (SDNode::op_iterator i = Call->op_begin()+2; i != e; ++i) 7033 Ops.push_back(*i); 7034 7035 // Push live variables for the stack map. 7036 addStackMapLiveVars(CI, NumMetaOpers + NumArgs, Ops, *this); 7037 7038 // Push the register mask info. 7039 if (hasGlue) 7040 Ops.push_back(*(Call->op_end()-2)); 7041 else 7042 Ops.push_back(*(Call->op_end()-1)); 7043 7044 // Push the chain (this is originally the first operand of the call, but 7045 // becomes now the last or second to last operand). 7046 Ops.push_back(*(Call->op_begin())); 7047 7048 // Push the glue flag (last operand). 7049 if (hasGlue) 7050 Ops.push_back(*(Call->op_end()-1)); 7051 7052 SDVTList NodeTys; 7053 if (isAnyRegCC && hasDef) { 7054 // Create the return types based on the intrinsic definition 7055 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7056 SmallVector<EVT, 3> ValueVTs; 7057 ComputeValueVTs(TLI, CI.getType(), ValueVTs); 7058 assert(ValueVTs.size() == 1 && "Expected only one return value type."); 7059 7060 // There is always a chain and a glue type at the end 7061 ValueVTs.push_back(MVT::Other); 7062 ValueVTs.push_back(MVT::Glue); 7063 NodeTys = DAG.getVTList(ValueVTs); 7064 } else 7065 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 7066 7067 // Replace the target specific call node with a PATCHPOINT node. 7068 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT, 7069 getCurSDLoc(), NodeTys, Ops); 7070 7071 // Update the NodeMap. 7072 if (hasDef) { 7073 if (isAnyRegCC) 7074 setValue(&CI, SDValue(MN, 0)); 7075 else 7076 setValue(&CI, Result.first); 7077 } 7078 7079 // Fixup the consumers of the intrinsic. The chain and glue may be used in the 7080 // call sequence. Furthermore the location of the chain and glue can change 7081 // when the AnyReg calling convention is used and the intrinsic returns a 7082 // value. 7083 if (isAnyRegCC && hasDef) { 7084 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)}; 7085 SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)}; 7086 DAG.ReplaceAllUsesOfValuesWith(From, To, 2); 7087 } else 7088 DAG.ReplaceAllUsesWith(Call, MN); 7089 DAG.DeleteNode(Call); 7090 7091 // Inform the Frame Information that we have a patchpoint in this function. 7092 FuncInfo.MF->getFrameInfo()->setHasPatchPoint(); 7093} 7094 7095/// TargetLowering::LowerCallTo - This is the default LowerCallTo 7096/// implementation, which just calls LowerCall. 7097/// FIXME: When all targets are 7098/// migrated to using LowerCall, this hook should be integrated into SDISel. 7099std::pair<SDValue, SDValue> 7100TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { 7101 // Handle the incoming return values from the call. 7102 CLI.Ins.clear(); 7103 SmallVector<EVT, 4> RetTys; 7104 ComputeValueVTs(*this, CLI.RetTy, RetTys); 7105 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { 7106 EVT VT = RetTys[I]; 7107 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); 7108 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT); 7109 for (unsigned i = 0; i != NumRegs; ++i) { 7110 ISD::InputArg MyFlags; 7111 MyFlags.VT = RegisterVT; 7112 MyFlags.ArgVT = VT; 7113 MyFlags.Used = CLI.IsReturnValueUsed; 7114 if (CLI.RetSExt) 7115 MyFlags.Flags.setSExt(); 7116 if (CLI.RetZExt) 7117 MyFlags.Flags.setZExt(); 7118 if (CLI.IsInReg) 7119 MyFlags.Flags.setInReg(); 7120 CLI.Ins.push_back(MyFlags); 7121 } 7122 } 7123 7124 // Handle all of the outgoing arguments. 7125 CLI.Outs.clear(); 7126 CLI.OutVals.clear(); 7127 ArgListTy &Args = CLI.getArgs(); 7128 for (unsigned i = 0, e = Args.size(); i != e; ++i) { 7129 SmallVector<EVT, 4> ValueVTs; 7130 ComputeValueVTs(*this, Args[i].Ty, ValueVTs); 7131 Type *FinalType = Args[i].Ty; 7132 if (Args[i].isByVal) 7133 FinalType = cast<PointerType>(Args[i].Ty)->getElementType(); 7134 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters( 7135 FinalType, CLI.CallConv, CLI.IsVarArg); 7136 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues; 7137 ++Value) { 7138 EVT VT = ValueVTs[Value]; 7139 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext()); 7140 SDValue Op = SDValue(Args[i].Node.getNode(), 7141 Args[i].Node.getResNo() + Value); 7142 ISD::ArgFlagsTy Flags; 7143 unsigned OriginalAlignment = getDataLayout()->getABITypeAlignment(ArgTy); 7144 7145 if (Args[i].isZExt) 7146 Flags.setZExt(); 7147 if (Args[i].isSExt) 7148 Flags.setSExt(); 7149 if (Args[i].isInReg) 7150 Flags.setInReg(); 7151 if (Args[i].isSRet) 7152 Flags.setSRet(); 7153 if (Args[i].isByVal) 7154 Flags.setByVal(); 7155 if (Args[i].isInAlloca) { 7156 Flags.setInAlloca(); 7157 // Set the byval flag for CCAssignFn callbacks that don't know about 7158 // inalloca. This way we can know how many bytes we should've allocated 7159 // and how many bytes a callee cleanup function will pop. If we port 7160 // inalloca to more targets, we'll have to add custom inalloca handling 7161 // in the various CC lowering callbacks. 7162 Flags.setByVal(); 7163 } 7164 if (Args[i].isByVal || Args[i].isInAlloca) { 7165 PointerType *Ty = cast<PointerType>(Args[i].Ty); 7166 Type *ElementTy = Ty->getElementType(); 7167 Flags.setByValSize(getDataLayout()->getTypeAllocSize(ElementTy)); 7168 // For ByVal, alignment should come from FE. BE will guess if this 7169 // info is not there but there are cases it cannot get right. 7170 unsigned FrameAlign; 7171 if (Args[i].Alignment) 7172 FrameAlign = Args[i].Alignment; 7173 else 7174 FrameAlign = getByValTypeAlignment(ElementTy); 7175 Flags.setByValAlign(FrameAlign); 7176 } 7177 if (Args[i].isNest) 7178 Flags.setNest(); 7179 if (NeedsRegBlock) 7180 Flags.setInConsecutiveRegs(); 7181 Flags.setOrigAlign(OriginalAlignment); 7182 7183 MVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT); 7184 unsigned NumParts = getNumRegisters(CLI.RetTy->getContext(), VT); 7185 SmallVector<SDValue, 4> Parts(NumParts); 7186 ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 7187 7188 if (Args[i].isSExt) 7189 ExtendKind = ISD::SIGN_EXTEND; 7190 else if (Args[i].isZExt) 7191 ExtendKind = ISD::ZERO_EXTEND; 7192 7193 // Conservatively only handle 'returned' on non-vectors for now 7194 if (Args[i].isReturned && !Op.getValueType().isVector()) { 7195 assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues && 7196 "unexpected use of 'returned'"); 7197 // Before passing 'returned' to the target lowering code, ensure that 7198 // either the register MVT and the actual EVT are the same size or that 7199 // the return value and argument are extended in the same way; in these 7200 // cases it's safe to pass the argument register value unchanged as the 7201 // return register value (although it's at the target's option whether 7202 // to do so) 7203 // TODO: allow code generation to take advantage of partially preserved 7204 // registers rather than clobbering the entire register when the 7205 // parameter extension method is not compatible with the return 7206 // extension method 7207 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) || 7208 (ExtendKind != ISD::ANY_EXTEND && 7209 CLI.RetSExt == Args[i].isSExt && CLI.RetZExt == Args[i].isZExt)) 7210 Flags.setReturned(); 7211 } 7212 7213 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, 7214 CLI.CS ? CLI.CS->getInstruction() : nullptr, ExtendKind); 7215 7216 for (unsigned j = 0; j != NumParts; ++j) { 7217 // if it isn't first piece, alignment must be 1 7218 ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT, 7219 i < CLI.NumFixedArgs, 7220 i, j*Parts[j].getValueType().getStoreSize()); 7221 if (NumParts > 1 && j == 0) 7222 MyFlags.Flags.setSplit(); 7223 else if (j != 0) 7224 MyFlags.Flags.setOrigAlign(1); 7225 7226 // Only mark the end at the last register of the last value. 7227 if (NeedsRegBlock && Value == NumValues - 1 && j == NumParts - 1) 7228 MyFlags.Flags.setInConsecutiveRegsLast(); 7229 7230 CLI.Outs.push_back(MyFlags); 7231 CLI.OutVals.push_back(Parts[j]); 7232 } 7233 } 7234 } 7235 7236 SmallVector<SDValue, 4> InVals; 7237 CLI.Chain = LowerCall(CLI, InVals); 7238 7239 // Verify that the target's LowerCall behaved as expected. 7240 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && 7241 "LowerCall didn't return a valid chain!"); 7242 assert((!CLI.IsTailCall || InVals.empty()) && 7243 "LowerCall emitted a return value for a tail call!"); 7244 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && 7245 "LowerCall didn't emit the correct number of values!"); 7246 7247 // For a tail call, the return value is merely live-out and there aren't 7248 // any nodes in the DAG representing it. Return a special value to 7249 // indicate that a tail call has been emitted and no more Instructions 7250 // should be processed in the current block. 7251 if (CLI.IsTailCall) { 7252 CLI.DAG.setRoot(CLI.Chain); 7253 return std::make_pair(SDValue(), SDValue()); 7254 } 7255 7256 DEBUG(for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) { 7257 assert(InVals[i].getNode() && 7258 "LowerCall emitted a null value!"); 7259 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && 7260 "LowerCall emitted a value with the wrong type!"); 7261 }); 7262 7263 // Collect the legal value parts into potentially illegal values 7264 // that correspond to the original function's return values. 7265 ISD::NodeType AssertOp = ISD::DELETED_NODE; 7266 if (CLI.RetSExt) 7267 AssertOp = ISD::AssertSext; 7268 else if (CLI.RetZExt) 7269 AssertOp = ISD::AssertZext; 7270 SmallVector<SDValue, 4> ReturnValues; 7271 unsigned CurReg = 0; 7272 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { 7273 EVT VT = RetTys[I]; 7274 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); 7275 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT); 7276 7277 ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg], 7278 NumRegs, RegisterVT, VT, nullptr, 7279 AssertOp)); 7280 CurReg += NumRegs; 7281 } 7282 7283 // For a function returning void, there is no return value. We can't create 7284 // such a node, so we just return a null return value in that case. In 7285 // that case, nothing will actually look at the value. 7286 if (ReturnValues.empty()) 7287 return std::make_pair(SDValue(), CLI.Chain); 7288 7289 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL, 7290 CLI.DAG.getVTList(RetTys), ReturnValues); 7291 return std::make_pair(Res, CLI.Chain); 7292} 7293 7294void TargetLowering::LowerOperationWrapper(SDNode *N, 7295 SmallVectorImpl<SDValue> &Results, 7296 SelectionDAG &DAG) const { 7297 SDValue Res = LowerOperation(SDValue(N, 0), DAG); 7298 if (Res.getNode()) 7299 Results.push_back(Res); 7300} 7301 7302SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 7303 llvm_unreachable("LowerOperation not implemented for this target!"); 7304} 7305 7306void 7307SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) { 7308 SDValue Op = getNonRegisterValue(V); 7309 assert((Op.getOpcode() != ISD::CopyFromReg || 7310 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && 7311 "Copy from a reg to the same reg!"); 7312 assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg"); 7313 7314 const TargetLowering *TLI = TM.getTargetLowering(); 7315 RegsForValue RFV(V->getContext(), *TLI, Reg, V->getType()); 7316 SDValue Chain = DAG.getEntryNode(); 7317 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V); 7318 PendingExports.push_back(Chain); 7319} 7320 7321#include "llvm/CodeGen/SelectionDAGISel.h" 7322 7323/// isOnlyUsedInEntryBlock - If the specified argument is only used in the 7324/// entry block, return true. This includes arguments used by switches, since 7325/// the switch may expand into multiple basic blocks. 7326static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { 7327 // With FastISel active, we may be splitting blocks, so force creation 7328 // of virtual registers for all non-dead arguments. 7329 if (FastISel) 7330 return A->use_empty(); 7331 7332 const BasicBlock *Entry = A->getParent()->begin(); 7333 for (const User *U : A->users()) 7334 if (cast<Instruction>(U)->getParent() != Entry || isa<SwitchInst>(U)) 7335 return false; // Use not in entry block. 7336 7337 return true; 7338} 7339 7340void SelectionDAGISel::LowerArguments(const Function &F) { 7341 SelectionDAG &DAG = SDB->DAG; 7342 SDLoc dl = SDB->getCurSDLoc(); 7343 const TargetLowering *TLI = getTargetLowering(); 7344 const DataLayout *DL = TLI->getDataLayout(); 7345 SmallVector<ISD::InputArg, 16> Ins; 7346 7347 if (!FuncInfo->CanLowerReturn) { 7348 // Put in an sret pointer parameter before all the other parameters. 7349 SmallVector<EVT, 1> ValueVTs; 7350 ComputeValueVTs(*getTargetLowering(), 7351 PointerType::getUnqual(F.getReturnType()), ValueVTs); 7352 7353 // NOTE: Assuming that a pointer will never break down to more than one VT 7354 // or one register. 7355 ISD::ArgFlagsTy Flags; 7356 Flags.setSRet(); 7357 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]); 7358 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true, 0, 0); 7359 Ins.push_back(RetArg); 7360 } 7361 7362 // Set up the incoming argument description vector. 7363 unsigned Idx = 1; 7364 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); 7365 I != E; ++I, ++Idx) { 7366 SmallVector<EVT, 4> ValueVTs; 7367 ComputeValueVTs(*TLI, I->getType(), ValueVTs); 7368 bool isArgValueUsed = !I->use_empty(); 7369 unsigned PartBase = 0; 7370 Type *FinalType = I->getType(); 7371 if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal)) 7372 FinalType = cast<PointerType>(FinalType)->getElementType(); 7373 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( 7374 FinalType, F.getCallingConv(), F.isVarArg()); 7375 for (unsigned Value = 0, NumValues = ValueVTs.size(); 7376 Value != NumValues; ++Value) { 7377 EVT VT = ValueVTs[Value]; 7378 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); 7379 ISD::ArgFlagsTy Flags; 7380 unsigned OriginalAlignment = DL->getABITypeAlignment(ArgTy); 7381 7382 if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt)) 7383 Flags.setZExt(); 7384 if (F.getAttributes().hasAttribute(Idx, Attribute::SExt)) 7385 Flags.setSExt(); 7386 if (F.getAttributes().hasAttribute(Idx, Attribute::InReg)) 7387 Flags.setInReg(); 7388 if (F.getAttributes().hasAttribute(Idx, Attribute::StructRet)) 7389 Flags.setSRet(); 7390 if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal)) 7391 Flags.setByVal(); 7392 if (F.getAttributes().hasAttribute(Idx, Attribute::InAlloca)) { 7393 Flags.setInAlloca(); 7394 // Set the byval flag for CCAssignFn callbacks that don't know about 7395 // inalloca. This way we can know how many bytes we should've allocated 7396 // and how many bytes a callee cleanup function will pop. If we port 7397 // inalloca to more targets, we'll have to add custom inalloca handling 7398 // in the various CC lowering callbacks. 7399 Flags.setByVal(); 7400 } 7401 if (Flags.isByVal() || Flags.isInAlloca()) { 7402 PointerType *Ty = cast<PointerType>(I->getType()); 7403 Type *ElementTy = Ty->getElementType(); 7404 Flags.setByValSize(DL->getTypeAllocSize(ElementTy)); 7405 // For ByVal, alignment should be passed from FE. BE will guess if 7406 // this info is not there but there are cases it cannot get right. 7407 unsigned FrameAlign; 7408 if (F.getParamAlignment(Idx)) 7409 FrameAlign = F.getParamAlignment(Idx); 7410 else 7411 FrameAlign = TLI->getByValTypeAlignment(ElementTy); 7412 Flags.setByValAlign(FrameAlign); 7413 } 7414 if (F.getAttributes().hasAttribute(Idx, Attribute::Nest)) 7415 Flags.setNest(); 7416 if (NeedsRegBlock) 7417 Flags.setInConsecutiveRegs(); 7418 Flags.setOrigAlign(OriginalAlignment); 7419 7420 MVT RegisterVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7421 unsigned NumRegs = TLI->getNumRegisters(*CurDAG->getContext(), VT); 7422 for (unsigned i = 0; i != NumRegs; ++i) { 7423 ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed, 7424 Idx-1, PartBase+i*RegisterVT.getStoreSize()); 7425 if (NumRegs > 1 && i == 0) 7426 MyFlags.Flags.setSplit(); 7427 // if it isn't first piece, alignment must be 1 7428 else if (i > 0) 7429 MyFlags.Flags.setOrigAlign(1); 7430 7431 // Only mark the end at the last register of the last value. 7432 if (NeedsRegBlock && Value == NumValues - 1 && i == NumRegs - 1) 7433 MyFlags.Flags.setInConsecutiveRegsLast(); 7434 7435 Ins.push_back(MyFlags); 7436 } 7437 PartBase += VT.getStoreSize(); 7438 } 7439 } 7440 7441 // Call the target to set up the argument values. 7442 SmallVector<SDValue, 8> InVals; 7443 SDValue NewRoot = TLI->LowerFormalArguments(DAG.getRoot(), F.getCallingConv(), 7444 F.isVarArg(), Ins, 7445 dl, DAG, InVals); 7446 7447 // Verify that the target's LowerFormalArguments behaved as expected. 7448 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && 7449 "LowerFormalArguments didn't return a valid chain!"); 7450 assert(InVals.size() == Ins.size() && 7451 "LowerFormalArguments didn't emit the correct number of values!"); 7452 DEBUG({ 7453 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 7454 assert(InVals[i].getNode() && 7455 "LowerFormalArguments emitted a null value!"); 7456 assert(EVT(Ins[i].VT) == InVals[i].getValueType() && 7457 "LowerFormalArguments emitted a value with the wrong type!"); 7458 } 7459 }); 7460 7461 // Update the DAG with the new chain value resulting from argument lowering. 7462 DAG.setRoot(NewRoot); 7463 7464 // Set up the argument values. 7465 unsigned i = 0; 7466 Idx = 1; 7467 if (!FuncInfo->CanLowerReturn) { 7468 // Create a virtual register for the sret pointer, and put in a copy 7469 // from the sret argument into it. 7470 SmallVector<EVT, 1> ValueVTs; 7471 ComputeValueVTs(*TLI, PointerType::getUnqual(F.getReturnType()), ValueVTs); 7472 MVT VT = ValueVTs[0].getSimpleVT(); 7473 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7474 ISD::NodeType AssertOp = ISD::DELETED_NODE; 7475 SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, 7476 RegVT, VT, nullptr, AssertOp); 7477 7478 MachineFunction& MF = SDB->DAG.getMachineFunction(); 7479 MachineRegisterInfo& RegInfo = MF.getRegInfo(); 7480 unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT)); 7481 FuncInfo->DemoteRegister = SRetReg; 7482 NewRoot = SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), 7483 SRetReg, ArgValue); 7484 DAG.setRoot(NewRoot); 7485 7486 // i indexes lowered arguments. Bump it past the hidden sret argument. 7487 // Idx indexes LLVM arguments. Don't touch it. 7488 ++i; 7489 } 7490 7491 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; 7492 ++I, ++Idx) { 7493 SmallVector<SDValue, 4> ArgValues; 7494 SmallVector<EVT, 4> ValueVTs; 7495 ComputeValueVTs(*TLI, I->getType(), ValueVTs); 7496 unsigned NumValues = ValueVTs.size(); 7497 7498 // If this argument is unused then remember its value. It is used to generate 7499 // debugging information. 7500 if (I->use_empty() && NumValues) { 7501 SDB->setUnusedArgValue(I, InVals[i]); 7502 7503 // Also remember any frame index for use in FastISel. 7504 if (FrameIndexSDNode *FI = 7505 dyn_cast<FrameIndexSDNode>(InVals[i].getNode())) 7506 FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7507 } 7508 7509 for (unsigned Val = 0; Val != NumValues; ++Val) { 7510 EVT VT = ValueVTs[Val]; 7511 MVT PartVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7512 unsigned NumParts = TLI->getNumRegisters(*CurDAG->getContext(), VT); 7513 7514 if (!I->use_empty()) { 7515 ISD::NodeType AssertOp = ISD::DELETED_NODE; 7516 if (F.getAttributes().hasAttribute(Idx, Attribute::SExt)) 7517 AssertOp = ISD::AssertSext; 7518 else if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt)) 7519 AssertOp = ISD::AssertZext; 7520 7521 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], 7522 NumParts, PartVT, VT, 7523 nullptr, AssertOp)); 7524 } 7525 7526 i += NumParts; 7527 } 7528 7529 // We don't need to do anything else for unused arguments. 7530 if (ArgValues.empty()) 7531 continue; 7532 7533 // Note down frame index. 7534 if (FrameIndexSDNode *FI = 7535 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) 7536 FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7537 7538 SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues), 7539 SDB->getCurSDLoc()); 7540 7541 SDB->setValue(I, Res); 7542 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { 7543 if (LoadSDNode *LNode = 7544 dyn_cast<LoadSDNode>(Res.getOperand(0).getNode())) 7545 if (FrameIndexSDNode *FI = 7546 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) 7547 FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7548 } 7549 7550 // If this argument is live outside of the entry block, insert a copy from 7551 // wherever we got it to the vreg that other BB's will reference it as. 7552 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) { 7553 // If we can, though, try to skip creating an unnecessary vreg. 7554 // FIXME: This isn't very clean... it would be nice to make this more 7555 // general. It's also subtly incompatible with the hacks FastISel 7556 // uses with vregs. 7557 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); 7558 if (TargetRegisterInfo::isVirtualRegister(Reg)) { 7559 FuncInfo->ValueMap[I] = Reg; 7560 continue; 7561 } 7562 } 7563 if (!isOnlyUsedInEntryBlock(I, TM.Options.EnableFastISel)) { 7564 FuncInfo->InitializeRegForValue(I); 7565 SDB->CopyToExportRegsIfNeeded(I); 7566 } 7567 } 7568 7569 assert(i == InVals.size() && "Argument register count mismatch!"); 7570 7571 // Finally, if the target has anything special to do, allow it to do so. 7572 // FIXME: this should insert code into the DAG! 7573 EmitFunctionEntryCode(); 7574} 7575 7576/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to 7577/// ensure constants are generated when needed. Remember the virtual registers 7578/// that need to be added to the Machine PHI nodes as input. We cannot just 7579/// directly add them, because expansion might result in multiple MBB's for one 7580/// BB. As such, the start of the BB might correspond to a different MBB than 7581/// the end. 7582/// 7583void 7584SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { 7585 const TerminatorInst *TI = LLVMBB->getTerminator(); 7586 7587 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; 7588 7589 // Check successor nodes' PHI nodes that expect a constant to be available 7590 // from this block. 7591 for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { 7592 const BasicBlock *SuccBB = TI->getSuccessor(succ); 7593 if (!isa<PHINode>(SuccBB->begin())) continue; 7594 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB]; 7595 7596 // If this terminator has multiple identical successors (common for 7597 // switches), only handle each succ once. 7598 if (!SuccsHandled.insert(SuccMBB)) continue; 7599 7600 MachineBasicBlock::iterator MBBI = SuccMBB->begin(); 7601 7602 // At this point we know that there is a 1-1 correspondence between LLVM PHI 7603 // nodes and Machine PHI nodes, but the incoming operands have not been 7604 // emitted yet. 7605 for (BasicBlock::const_iterator I = SuccBB->begin(); 7606 const PHINode *PN = dyn_cast<PHINode>(I); ++I) { 7607 // Ignore dead phi's. 7608 if (PN->use_empty()) continue; 7609 7610 // Skip empty types 7611 if (PN->getType()->isEmptyTy()) 7612 continue; 7613 7614 unsigned Reg; 7615 const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); 7616 7617 if (const Constant *C = dyn_cast<Constant>(PHIOp)) { 7618 unsigned &RegOut = ConstantsOut[C]; 7619 if (RegOut == 0) { 7620 RegOut = FuncInfo.CreateRegs(C->getType()); 7621 CopyValueToVirtualRegister(C, RegOut); 7622 } 7623 Reg = RegOut; 7624 } else { 7625 DenseMap<const Value *, unsigned>::iterator I = 7626 FuncInfo.ValueMap.find(PHIOp); 7627 if (I != FuncInfo.ValueMap.end()) 7628 Reg = I->second; 7629 else { 7630 assert(isa<AllocaInst>(PHIOp) && 7631 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && 7632 "Didn't codegen value into a register!??"); 7633 Reg = FuncInfo.CreateRegs(PHIOp->getType()); 7634 CopyValueToVirtualRegister(PHIOp, Reg); 7635 } 7636 } 7637 7638 // Remember that this register needs to added to the machine PHI node as 7639 // the input for this MBB. 7640 SmallVector<EVT, 4> ValueVTs; 7641 const TargetLowering *TLI = TM.getTargetLowering(); 7642 ComputeValueVTs(*TLI, PN->getType(), ValueVTs); 7643 for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { 7644 EVT VT = ValueVTs[vti]; 7645 unsigned NumRegisters = TLI->getNumRegisters(*DAG.getContext(), VT); 7646 for (unsigned i = 0, e = NumRegisters; i != e; ++i) 7647 FuncInfo.PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); 7648 Reg += NumRegisters; 7649 } 7650 } 7651 } 7652 7653 ConstantsOut.clear(); 7654} 7655 7656/// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB 7657/// is 0. 7658MachineBasicBlock * 7659SelectionDAGBuilder::StackProtectorDescriptor:: 7660AddSuccessorMBB(const BasicBlock *BB, 7661 MachineBasicBlock *ParentMBB, 7662 MachineBasicBlock *SuccMBB) { 7663 // If SuccBB has not been created yet, create it. 7664 if (!SuccMBB) { 7665 MachineFunction *MF = ParentMBB->getParent(); 7666 MachineFunction::iterator BBI = ParentMBB; 7667 SuccMBB = MF->CreateMachineBasicBlock(BB); 7668 MF->insert(++BBI, SuccMBB); 7669 } 7670 // Add it as a successor of ParentMBB. 7671 ParentMBB->addSuccessor(SuccMBB); 7672 return SuccMBB; 7673} 7674