IntrinsicLowering.cpp revision a7235ea7245028a0723e8ab7fd011386b3900777
1//===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the IntrinsicLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/Constants.h" 15#include "llvm/DerivedTypes.h" 16#include "llvm/Module.h" 17#include "llvm/Type.h" 18#include "llvm/CodeGen/IntrinsicLowering.h" 19#include "llvm/Support/IRBuilder.h" 20#include "llvm/Support/ErrorHandling.h" 21#include "llvm/Target/TargetData.h" 22#include "llvm/ADT/SmallVector.h" 23using namespace llvm; 24 25template <class ArgIt> 26static void EnsureFunctionExists(Module &M, const char *Name, 27 ArgIt ArgBegin, ArgIt ArgEnd, 28 const Type *RetTy) { 29 // Insert a correctly-typed definition now. 30 std::vector<const Type *> ParamTys; 31 for (ArgIt I = ArgBegin; I != ArgEnd; ++I) 32 ParamTys.push_back(I->getType()); 33 M.getOrInsertFunction(Name, FunctionType::get(RetTy, ParamTys, false)); 34} 35 36static void EnsureFPIntrinsicsExist(Module &M, Function *Fn, 37 const char *FName, 38 const char *DName, const char *LDName) { 39 // Insert definitions for all the floating point types. 40 switch((int)Fn->arg_begin()->getType()->getTypeID()) { 41 case Type::FloatTyID: 42 EnsureFunctionExists(M, FName, Fn->arg_begin(), Fn->arg_end(), 43 Type::FloatTy); 44 break; 45 case Type::DoubleTyID: 46 EnsureFunctionExists(M, DName, Fn->arg_begin(), Fn->arg_end(), 47 Type::DoubleTy); 48 break; 49 case Type::X86_FP80TyID: 50 case Type::FP128TyID: 51 case Type::PPC_FP128TyID: 52 EnsureFunctionExists(M, LDName, Fn->arg_begin(), Fn->arg_end(), 53 Fn->arg_begin()->getType()); 54 break; 55 } 56} 57 58/// ReplaceCallWith - This function is used when we want to lower an intrinsic 59/// call to a call of an external function. This handles hard cases such as 60/// when there was already a prototype for the external function, and if that 61/// prototype doesn't match the arguments we expect to pass in. 62template <class ArgIt> 63static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI, 64 ArgIt ArgBegin, ArgIt ArgEnd, 65 const Type *RetTy) { 66 // If we haven't already looked up this function, check to see if the 67 // program already contains a function with this name. 68 Module *M = CI->getParent()->getParent()->getParent(); 69 // Get or insert the definition now. 70 std::vector<const Type *> ParamTys; 71 for (ArgIt I = ArgBegin; I != ArgEnd; ++I) 72 ParamTys.push_back((*I)->getType()); 73 Constant* FCache = M->getOrInsertFunction(NewFn, 74 FunctionType::get(RetTy, ParamTys, false)); 75 76 IRBuilder<> Builder(CI->getParent(), CI); 77 SmallVector<Value *, 8> Args(ArgBegin, ArgEnd); 78 CallInst *NewCI = Builder.CreateCall(FCache, Args.begin(), Args.end()); 79 NewCI->setName(CI->getName()); 80 if (!CI->use_empty()) 81 CI->replaceAllUsesWith(NewCI); 82 return NewCI; 83} 84 85void IntrinsicLowering::AddPrototypes(Module &M) { 86 for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) 87 if (I->isDeclaration() && !I->use_empty()) 88 switch (I->getIntrinsicID()) { 89 default: break; 90 case Intrinsic::setjmp: 91 EnsureFunctionExists(M, "setjmp", I->arg_begin(), I->arg_end(), 92 Type::Int32Ty); 93 break; 94 case Intrinsic::longjmp: 95 EnsureFunctionExists(M, "longjmp", I->arg_begin(), I->arg_end(), 96 Type::VoidTy); 97 break; 98 case Intrinsic::siglongjmp: 99 EnsureFunctionExists(M, "abort", I->arg_end(), I->arg_end(), 100 Type::VoidTy); 101 break; 102 case Intrinsic::memcpy: 103 M.getOrInsertFunction("memcpy", PointerType::getUnqual(Type::Int8Ty), 104 PointerType::getUnqual(Type::Int8Ty), 105 PointerType::getUnqual(Type::Int8Ty), 106 TD.getIntPtrType(), (Type *)0); 107 break; 108 case Intrinsic::memmove: 109 M.getOrInsertFunction("memmove", PointerType::getUnqual(Type::Int8Ty), 110 PointerType::getUnqual(Type::Int8Ty), 111 PointerType::getUnqual(Type::Int8Ty), 112 TD.getIntPtrType(), (Type *)0); 113 break; 114 case Intrinsic::memset: 115 M.getOrInsertFunction("memset", PointerType::getUnqual(Type::Int8Ty), 116 PointerType::getUnqual(Type::Int8Ty), 117 Type::Int32Ty, 118 TD.getIntPtrType(), (Type *)0); 119 break; 120 case Intrinsic::sqrt: 121 EnsureFPIntrinsicsExist(M, I, "sqrtf", "sqrt", "sqrtl"); 122 break; 123 case Intrinsic::sin: 124 EnsureFPIntrinsicsExist(M, I, "sinf", "sin", "sinl"); 125 break; 126 case Intrinsic::cos: 127 EnsureFPIntrinsicsExist(M, I, "cosf", "cos", "cosl"); 128 break; 129 case Intrinsic::pow: 130 EnsureFPIntrinsicsExist(M, I, "powf", "pow", "powl"); 131 break; 132 case Intrinsic::log: 133 EnsureFPIntrinsicsExist(M, I, "logf", "log", "logl"); 134 break; 135 case Intrinsic::log2: 136 EnsureFPIntrinsicsExist(M, I, "log2f", "log2", "log2l"); 137 break; 138 case Intrinsic::log10: 139 EnsureFPIntrinsicsExist(M, I, "log10f", "log10", "log10l"); 140 break; 141 case Intrinsic::exp: 142 EnsureFPIntrinsicsExist(M, I, "expf", "exp", "expl"); 143 break; 144 case Intrinsic::exp2: 145 EnsureFPIntrinsicsExist(M, I, "exp2f", "exp2", "exp2l"); 146 break; 147 } 148} 149 150/// LowerBSWAP - Emit the code to lower bswap of V before the specified 151/// instruction IP. 152static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) { 153 assert(V->getType()->isInteger() && "Can't bswap a non-integer type!"); 154 155 unsigned BitSize = V->getType()->getPrimitiveSizeInBits(); 156 157 IRBuilder<> Builder(IP->getParent(), IP); 158 159 switch(BitSize) { 160 default: llvm_unreachable("Unhandled type size of value to byteswap!"); 161 case 16: { 162 Value *Tmp1 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8), 163 "bswap.2"); 164 Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8), 165 "bswap.1"); 166 V = Builder.CreateOr(Tmp1, Tmp2, "bswap.i16"); 167 break; 168 } 169 case 32: { 170 Value *Tmp4 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24), 171 "bswap.4"); 172 Value *Tmp3 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8), 173 "bswap.3"); 174 Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8), 175 "bswap.2"); 176 Value *Tmp1 = Builder.CreateLShr(V,ConstantInt::get(V->getType(), 24), 177 "bswap.1"); 178 Tmp3 = Builder.CreateAnd(Tmp3, 179 ConstantInt::get(Type::Int32Ty, 0xFF0000), 180 "bswap.and3"); 181 Tmp2 = Builder.CreateAnd(Tmp2, 182 ConstantInt::get(Type::Int32Ty, 0xFF00), 183 "bswap.and2"); 184 Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or1"); 185 Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or2"); 186 V = Builder.CreateOr(Tmp4, Tmp2, "bswap.i32"); 187 break; 188 } 189 case 64: { 190 Value *Tmp8 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 56), 191 "bswap.8"); 192 Value *Tmp7 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 40), 193 "bswap.7"); 194 Value *Tmp6 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24), 195 "bswap.6"); 196 Value *Tmp5 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8), 197 "bswap.5"); 198 Value* Tmp4 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8), 199 "bswap.4"); 200 Value* Tmp3 = Builder.CreateLShr(V, 201 ConstantInt::get(V->getType(), 24), 202 "bswap.3"); 203 Value* Tmp2 = Builder.CreateLShr(V, 204 ConstantInt::get(V->getType(), 40), 205 "bswap.2"); 206 Value* Tmp1 = Builder.CreateLShr(V, 207 ConstantInt::get(V->getType(), 56), 208 "bswap.1"); 209 Tmp7 = Builder.CreateAnd(Tmp7, 210 ConstantInt::get(Type::Int64Ty, 211 0xFF000000000000ULL), 212 "bswap.and7"); 213 Tmp6 = Builder.CreateAnd(Tmp6, 214 ConstantInt::get(Type::Int64Ty, 215 0xFF0000000000ULL), 216 "bswap.and6"); 217 Tmp5 = Builder.CreateAnd(Tmp5, 218 ConstantInt::get(Type::Int64Ty, 0xFF00000000ULL), 219 "bswap.and5"); 220 Tmp4 = Builder.CreateAnd(Tmp4, 221 ConstantInt::get(Type::Int64Ty, 0xFF000000ULL), 222 "bswap.and4"); 223 Tmp3 = Builder.CreateAnd(Tmp3, 224 ConstantInt::get(Type::Int64Ty, 0xFF0000ULL), 225 "bswap.and3"); 226 Tmp2 = Builder.CreateAnd(Tmp2, 227 ConstantInt::get(Type::Int64Ty, 0xFF00ULL), 228 "bswap.and2"); 229 Tmp8 = Builder.CreateOr(Tmp8, Tmp7, "bswap.or1"); 230 Tmp6 = Builder.CreateOr(Tmp6, Tmp5, "bswap.or2"); 231 Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or3"); 232 Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or4"); 233 Tmp8 = Builder.CreateOr(Tmp8, Tmp6, "bswap.or5"); 234 Tmp4 = Builder.CreateOr(Tmp4, Tmp2, "bswap.or6"); 235 V = Builder.CreateOr(Tmp8, Tmp4, "bswap.i64"); 236 break; 237 } 238 } 239 return V; 240} 241 242/// LowerCTPOP - Emit the code to lower ctpop of V before the specified 243/// instruction IP. 244static Value *LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP) { 245 assert(V->getType()->isInteger() && "Can't ctpop a non-integer type!"); 246 247 static const uint64_t MaskValues[6] = { 248 0x5555555555555555ULL, 0x3333333333333333ULL, 249 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL, 250 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL 251 }; 252 253 IRBuilder<> Builder(IP->getParent(), IP); 254 255 unsigned BitSize = V->getType()->getPrimitiveSizeInBits(); 256 unsigned WordSize = (BitSize + 63) / 64; 257 Value *Count = ConstantInt::get(V->getType(), 0); 258 259 for (unsigned n = 0; n < WordSize; ++n) { 260 Value *PartValue = V; 261 for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize); 262 i <<= 1, ++ct) { 263 Value *MaskCst = ConstantInt::get(V->getType(), MaskValues[ct]); 264 Value *LHS = Builder.CreateAnd(PartValue, MaskCst, "cppop.and1"); 265 Value *VShift = Builder.CreateLShr(PartValue, 266 ConstantInt::get(V->getType(), i), 267 "ctpop.sh"); 268 Value *RHS = Builder.CreateAnd(VShift, MaskCst, "cppop.and2"); 269 PartValue = Builder.CreateAdd(LHS, RHS, "ctpop.step"); 270 } 271 Count = Builder.CreateAdd(PartValue, Count, "ctpop.part"); 272 if (BitSize > 64) { 273 V = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 64), 274 "ctpop.part.sh"); 275 BitSize -= 64; 276 } 277 } 278 279 return Count; 280} 281 282/// LowerCTLZ - Emit the code to lower ctlz of V before the specified 283/// instruction IP. 284static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) { 285 286 IRBuilder<> Builder(IP->getParent(), IP); 287 288 unsigned BitSize = V->getType()->getPrimitiveSizeInBits(); 289 for (unsigned i = 1; i < BitSize; i <<= 1) { 290 Value *ShVal = ConstantInt::get(V->getType(), i); 291 ShVal = Builder.CreateLShr(V, ShVal, "ctlz.sh"); 292 V = Builder.CreateOr(V, ShVal, "ctlz.step"); 293 } 294 295 V = Builder.CreateNot(V); 296 return LowerCTPOP(Context, V, IP); 297} 298 299static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname, 300 const char *Dname, 301 const char *LDname) { 302 switch (CI->getOperand(1)->getType()->getTypeID()) { 303 default: llvm_unreachable("Invalid type in intrinsic"); 304 case Type::FloatTyID: 305 ReplaceCallWith(Fname, CI, CI->op_begin() + 1, CI->op_end(), 306 Type::FloatTy); 307 break; 308 case Type::DoubleTyID: 309 ReplaceCallWith(Dname, CI, CI->op_begin() + 1, CI->op_end(), 310 Type::DoubleTy); 311 break; 312 case Type::X86_FP80TyID: 313 case Type::FP128TyID: 314 case Type::PPC_FP128TyID: 315 ReplaceCallWith(LDname, CI, CI->op_begin() + 1, CI->op_end(), 316 CI->getOperand(1)->getType()); 317 break; 318 } 319} 320 321void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) { 322 IRBuilder<> Builder(CI->getParent(), CI); 323 LLVMContext &Context = CI->getContext(); 324 325 Function *Callee = CI->getCalledFunction(); 326 assert(Callee && "Cannot lower an indirect call!"); 327 328 switch (Callee->getIntrinsicID()) { 329 case Intrinsic::not_intrinsic: 330 llvm_report_error("Cannot lower a call to a non-intrinsic function '"+ 331 Callee->getName() + "'!"); 332 default: 333 llvm_report_error("Code generator does not support intrinsic function '"+ 334 Callee->getName()+"'!"); 335 336 // The setjmp/longjmp intrinsics should only exist in the code if it was 337 // never optimized (ie, right out of the CFE), or if it has been hacked on 338 // by the lowerinvoke pass. In both cases, the right thing to do is to 339 // convert the call to an explicit setjmp or longjmp call. 340 case Intrinsic::setjmp: { 341 Value *V = ReplaceCallWith("setjmp", CI, CI->op_begin() + 1, CI->op_end(), 342 Type::Int32Ty); 343 if (CI->getType() != Type::VoidTy) 344 CI->replaceAllUsesWith(V); 345 break; 346 } 347 case Intrinsic::sigsetjmp: 348 if (CI->getType() != Type::VoidTy) 349 CI->replaceAllUsesWith(Constant::getNullValue(CI->getType())); 350 break; 351 352 case Intrinsic::longjmp: { 353 ReplaceCallWith("longjmp", CI, CI->op_begin() + 1, CI->op_end(), 354 Type::VoidTy); 355 break; 356 } 357 358 case Intrinsic::siglongjmp: { 359 // Insert the call to abort 360 ReplaceCallWith("abort", CI, CI->op_end(), CI->op_end(), 361 Type::VoidTy); 362 break; 363 } 364 case Intrinsic::ctpop: 365 CI->replaceAllUsesWith(LowerCTPOP(Context, CI->getOperand(1), CI)); 366 break; 367 368 case Intrinsic::bswap: 369 CI->replaceAllUsesWith(LowerBSWAP(Context, CI->getOperand(1), CI)); 370 break; 371 372 case Intrinsic::ctlz: 373 CI->replaceAllUsesWith(LowerCTLZ(Context, CI->getOperand(1), CI)); 374 break; 375 376 case Intrinsic::cttz: { 377 // cttz(x) -> ctpop(~X & (X-1)) 378 Value *Src = CI->getOperand(1); 379 Value *NotSrc = Builder.CreateNot(Src); 380 NotSrc->setName(Src->getName() + ".not"); 381 Value *SrcM1 = ConstantInt::get(Src->getType(), 1); 382 SrcM1 = Builder.CreateSub(Src, SrcM1); 383 Src = LowerCTPOP(Context, Builder.CreateAnd(NotSrc, SrcM1), CI); 384 CI->replaceAllUsesWith(Src); 385 break; 386 } 387 388 case Intrinsic::stacksave: 389 case Intrinsic::stackrestore: { 390 if (!Warned) 391 cerr << "WARNING: this target does not support the llvm.stack" 392 << (Callee->getIntrinsicID() == Intrinsic::stacksave ? 393 "save" : "restore") << " intrinsic.\n"; 394 Warned = true; 395 if (Callee->getIntrinsicID() == Intrinsic::stacksave) 396 CI->replaceAllUsesWith(Constant::getNullValue(CI->getType())); 397 break; 398 } 399 400 case Intrinsic::returnaddress: 401 case Intrinsic::frameaddress: 402 cerr << "WARNING: this target does not support the llvm." 403 << (Callee->getIntrinsicID() == Intrinsic::returnaddress ? 404 "return" : "frame") << "address intrinsic.\n"; 405 CI->replaceAllUsesWith(ConstantPointerNull::get( 406 cast<PointerType>(CI->getType()))); 407 break; 408 409 case Intrinsic::prefetch: 410 break; // Simply strip out prefetches on unsupported architectures 411 412 case Intrinsic::pcmarker: 413 break; // Simply strip out pcmarker on unsupported architectures 414 case Intrinsic::readcyclecounter: { 415 cerr << "WARNING: this target does not support the llvm.readcyclecoun" 416 << "ter intrinsic. It is being lowered to a constant 0\n"; 417 CI->replaceAllUsesWith(ConstantInt::get(Type::Int64Ty, 0)); 418 break; 419 } 420 421 case Intrinsic::dbg_stoppoint: 422 case Intrinsic::dbg_region_start: 423 case Intrinsic::dbg_region_end: 424 case Intrinsic::dbg_func_start: 425 case Intrinsic::dbg_declare: 426 break; // Simply strip out debugging intrinsics 427 428 case Intrinsic::eh_exception: 429 case Intrinsic::eh_selector_i32: 430 case Intrinsic::eh_selector_i64: 431 CI->replaceAllUsesWith(Constant::getNullValue(CI->getType())); 432 break; 433 434 case Intrinsic::eh_typeid_for_i32: 435 case Intrinsic::eh_typeid_for_i64: 436 // Return something different to eh_selector. 437 CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1)); 438 break; 439 440 case Intrinsic::var_annotation: 441 break; // Strip out annotate intrinsic 442 443 case Intrinsic::memcpy: { 444 const IntegerType *IntPtr = TD.getIntPtrType(); 445 Value *Size = Builder.CreateIntCast(CI->getOperand(3), IntPtr, 446 /* isSigned */ false); 447 Value *Ops[3]; 448 Ops[0] = CI->getOperand(1); 449 Ops[1] = CI->getOperand(2); 450 Ops[2] = Size; 451 ReplaceCallWith("memcpy", CI, Ops, Ops+3, CI->getOperand(1)->getType()); 452 break; 453 } 454 case Intrinsic::memmove: { 455 const IntegerType *IntPtr = TD.getIntPtrType(); 456 Value *Size = Builder.CreateIntCast(CI->getOperand(3), IntPtr, 457 /* isSigned */ false); 458 Value *Ops[3]; 459 Ops[0] = CI->getOperand(1); 460 Ops[1] = CI->getOperand(2); 461 Ops[2] = Size; 462 ReplaceCallWith("memmove", CI, Ops, Ops+3, CI->getOperand(1)->getType()); 463 break; 464 } 465 case Intrinsic::memset: { 466 const IntegerType *IntPtr = TD.getIntPtrType(); 467 Value *Size = Builder.CreateIntCast(CI->getOperand(3), IntPtr, 468 /* isSigned */ false); 469 Value *Ops[3]; 470 Ops[0] = CI->getOperand(1); 471 // Extend the amount to i32. 472 Ops[1] = Builder.CreateIntCast(CI->getOperand(2), Type::Int32Ty, 473 /* isSigned */ false); 474 Ops[2] = Size; 475 ReplaceCallWith("memset", CI, Ops, Ops+3, CI->getOperand(1)->getType()); 476 break; 477 } 478 case Intrinsic::sqrt: { 479 ReplaceFPIntrinsicWithCall(CI, "sqrtf", "sqrt", "sqrtl"); 480 break; 481 } 482 case Intrinsic::log: { 483 ReplaceFPIntrinsicWithCall(CI, "logf", "log", "logl"); 484 break; 485 } 486 case Intrinsic::log2: { 487 ReplaceFPIntrinsicWithCall(CI, "log2f", "log2", "log2l"); 488 break; 489 } 490 case Intrinsic::log10: { 491 ReplaceFPIntrinsicWithCall(CI, "log10f", "log10", "log10l"); 492 break; 493 } 494 case Intrinsic::exp: { 495 ReplaceFPIntrinsicWithCall(CI, "expf", "exp", "expl"); 496 break; 497 } 498 case Intrinsic::exp2: { 499 ReplaceFPIntrinsicWithCall(CI, "exp2f", "exp2", "exp2l"); 500 break; 501 } 502 case Intrinsic::pow: { 503 ReplaceFPIntrinsicWithCall(CI, "powf", "pow", "powl"); 504 break; 505 } 506 case Intrinsic::flt_rounds: 507 // Lower to "round to the nearest" 508 if (CI->getType() != Type::VoidTy) 509 CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1)); 510 break; 511 } 512 513 assert(CI->use_empty() && 514 "Lowering should have eliminated any uses of the intrinsic call!"); 515 CI->eraseFromParent(); 516} 517