IntrinsicLowering.cpp revision 35b4707edb32008b1a7976dcbf3920b1160fc1c6
1//===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the IntrinsicLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/Constants.h" 15#include "llvm/DerivedTypes.h" 16#include "llvm/Module.h" 17#include "llvm/Type.h" 18#include "llvm/CodeGen/IntrinsicLowering.h" 19#include "llvm/Support/IRBuilder.h" 20#include "llvm/Support/ErrorHandling.h" 21#include "llvm/Target/TargetData.h" 22#include "llvm/ADT/SmallVector.h" 23using namespace llvm; 24 25template <class ArgIt> 26static void EnsureFunctionExists(Module &M, const char *Name, 27 ArgIt ArgBegin, ArgIt ArgEnd, 28 const Type *RetTy) { 29 // Insert a correctly-typed definition now. 30 std::vector<const Type *> ParamTys; 31 for (ArgIt I = ArgBegin; I != ArgEnd; ++I) 32 ParamTys.push_back(I->getType()); 33 M.getOrInsertFunction(Name, FunctionType::get(RetTy, ParamTys, false)); 34} 35 36static void EnsureFPIntrinsicsExist(Module &M, Function *Fn, 37 const char *FName, 38 const char *DName, const char *LDName) { 39 // Insert definitions for all the floating point types. 40 switch((int)Fn->arg_begin()->getType()->getTypeID()) { 41 case Type::FloatTyID: 42 EnsureFunctionExists(M, FName, Fn->arg_begin(), Fn->arg_end(), 43 Type::getFloatTy(M.getContext())); 44 break; 45 case Type::DoubleTyID: 46 EnsureFunctionExists(M, DName, Fn->arg_begin(), Fn->arg_end(), 47 Type::getDoubleTy(M.getContext())); 48 break; 49 case Type::X86_FP80TyID: 50 case Type::FP128TyID: 51 case Type::PPC_FP128TyID: 52 EnsureFunctionExists(M, LDName, Fn->arg_begin(), Fn->arg_end(), 53 Fn->arg_begin()->getType()); 54 break; 55 } 56} 57 58/// ReplaceCallWith - This function is used when we want to lower an intrinsic 59/// call to a call of an external function. This handles hard cases such as 60/// when there was already a prototype for the external function, and if that 61/// prototype doesn't match the arguments we expect to pass in. 62template <class ArgIt> 63static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI, 64 ArgIt ArgBegin, ArgIt ArgEnd, 65 const Type *RetTy) { 66 // If we haven't already looked up this function, check to see if the 67 // program already contains a function with this name. 68 Module *M = CI->getParent()->getParent()->getParent(); 69 // Get or insert the definition now. 70 std::vector<const Type *> ParamTys; 71 for (ArgIt I = ArgBegin; I != ArgEnd; ++I) 72 ParamTys.push_back((*I)->getType()); 73 Constant* FCache = M->getOrInsertFunction(NewFn, 74 FunctionType::get(RetTy, ParamTys, false)); 75 76 IRBuilder<> Builder(CI->getParent(), CI); 77 SmallVector<Value *, 8> Args(ArgBegin, ArgEnd); 78 CallInst *NewCI = Builder.CreateCall(FCache, Args.begin(), Args.end()); 79 NewCI->setName(CI->getName()); 80 if (!CI->use_empty()) 81 CI->replaceAllUsesWith(NewCI); 82 return NewCI; 83} 84 85void IntrinsicLowering::AddPrototypes(Module &M) { 86 LLVMContext &Context = M.getContext(); 87 for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) 88 if (I->isDeclaration() && !I->use_empty()) 89 switch (I->getIntrinsicID()) { 90 default: break; 91 case Intrinsic::setjmp: 92 EnsureFunctionExists(M, "setjmp", I->arg_begin(), I->arg_end(), 93 Type::getInt32Ty(M.getContext())); 94 break; 95 case Intrinsic::longjmp: 96 EnsureFunctionExists(M, "longjmp", I->arg_begin(), I->arg_end(), 97 Type::getVoidTy(M.getContext())); 98 break; 99 case Intrinsic::siglongjmp: 100 EnsureFunctionExists(M, "abort", I->arg_end(), I->arg_end(), 101 Type::getVoidTy(M.getContext())); 102 break; 103 case Intrinsic::memcpy: 104 M.getOrInsertFunction("memcpy", 105 PointerType::getUnqual(Type::getInt8Ty(Context)), 106 PointerType::getUnqual(Type::getInt8Ty(Context)), 107 PointerType::getUnqual(Type::getInt8Ty(Context)), 108 TD.getIntPtrType(Context), (Type *)0); 109 break; 110 case Intrinsic::memmove: 111 M.getOrInsertFunction("memmove", 112 PointerType::getUnqual(Type::getInt8Ty(Context)), 113 PointerType::getUnqual(Type::getInt8Ty(Context)), 114 PointerType::getUnqual(Type::getInt8Ty(Context)), 115 TD.getIntPtrType(Context), (Type *)0); 116 break; 117 case Intrinsic::memset: 118 M.getOrInsertFunction("memset", 119 PointerType::getUnqual(Type::getInt8Ty(Context)), 120 PointerType::getUnqual(Type::getInt8Ty(Context)), 121 Type::getInt32Ty(M.getContext()), 122 TD.getIntPtrType(Context), (Type *)0); 123 break; 124 case Intrinsic::sqrt: 125 EnsureFPIntrinsicsExist(M, I, "sqrtf", "sqrt", "sqrtl"); 126 break; 127 case Intrinsic::sin: 128 EnsureFPIntrinsicsExist(M, I, "sinf", "sin", "sinl"); 129 break; 130 case Intrinsic::cos: 131 EnsureFPIntrinsicsExist(M, I, "cosf", "cos", "cosl"); 132 break; 133 case Intrinsic::pow: 134 EnsureFPIntrinsicsExist(M, I, "powf", "pow", "powl"); 135 break; 136 case Intrinsic::log: 137 EnsureFPIntrinsicsExist(M, I, "logf", "log", "logl"); 138 break; 139 case Intrinsic::log2: 140 EnsureFPIntrinsicsExist(M, I, "log2f", "log2", "log2l"); 141 break; 142 case Intrinsic::log10: 143 EnsureFPIntrinsicsExist(M, I, "log10f", "log10", "log10l"); 144 break; 145 case Intrinsic::exp: 146 EnsureFPIntrinsicsExist(M, I, "expf", "exp", "expl"); 147 break; 148 case Intrinsic::exp2: 149 EnsureFPIntrinsicsExist(M, I, "exp2f", "exp2", "exp2l"); 150 break; 151 } 152} 153 154/// LowerBSWAP - Emit the code to lower bswap of V before the specified 155/// instruction IP. 156static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) { 157 assert(V->getType()->isInteger() && "Can't bswap a non-integer type!"); 158 159 unsigned BitSize = V->getType()->getPrimitiveSizeInBits(); 160 161 IRBuilder<> Builder(IP->getParent(), IP); 162 163 switch(BitSize) { 164 default: llvm_unreachable("Unhandled type size of value to byteswap!"); 165 case 16: { 166 Value *Tmp1 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8), 167 "bswap.2"); 168 Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8), 169 "bswap.1"); 170 V = Builder.CreateOr(Tmp1, Tmp2, "bswap.i16"); 171 break; 172 } 173 case 32: { 174 Value *Tmp4 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24), 175 "bswap.4"); 176 Value *Tmp3 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8), 177 "bswap.3"); 178 Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8), 179 "bswap.2"); 180 Value *Tmp1 = Builder.CreateLShr(V,ConstantInt::get(V->getType(), 24), 181 "bswap.1"); 182 Tmp3 = Builder.CreateAnd(Tmp3, 183 ConstantInt::get(Type::getInt32Ty(Context), 0xFF0000), 184 "bswap.and3"); 185 Tmp2 = Builder.CreateAnd(Tmp2, 186 ConstantInt::get(Type::getInt32Ty(Context), 0xFF00), 187 "bswap.and2"); 188 Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or1"); 189 Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or2"); 190 V = Builder.CreateOr(Tmp4, Tmp2, "bswap.i32"); 191 break; 192 } 193 case 64: { 194 Value *Tmp8 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 56), 195 "bswap.8"); 196 Value *Tmp7 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 40), 197 "bswap.7"); 198 Value *Tmp6 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24), 199 "bswap.6"); 200 Value *Tmp5 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8), 201 "bswap.5"); 202 Value* Tmp4 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8), 203 "bswap.4"); 204 Value* Tmp3 = Builder.CreateLShr(V, 205 ConstantInt::get(V->getType(), 24), 206 "bswap.3"); 207 Value* Tmp2 = Builder.CreateLShr(V, 208 ConstantInt::get(V->getType(), 40), 209 "bswap.2"); 210 Value* Tmp1 = Builder.CreateLShr(V, 211 ConstantInt::get(V->getType(), 56), 212 "bswap.1"); 213 Tmp7 = Builder.CreateAnd(Tmp7, 214 ConstantInt::get(Type::getInt64Ty(Context), 215 0xFF000000000000ULL), 216 "bswap.and7"); 217 Tmp6 = Builder.CreateAnd(Tmp6, 218 ConstantInt::get(Type::getInt64Ty(Context), 219 0xFF0000000000ULL), 220 "bswap.and6"); 221 Tmp5 = Builder.CreateAnd(Tmp5, 222 ConstantInt::get(Type::getInt64Ty(Context), 223 0xFF00000000ULL), 224 "bswap.and5"); 225 Tmp4 = Builder.CreateAnd(Tmp4, 226 ConstantInt::get(Type::getInt64Ty(Context), 227 0xFF000000ULL), 228 "bswap.and4"); 229 Tmp3 = Builder.CreateAnd(Tmp3, 230 ConstantInt::get(Type::getInt64Ty(Context), 231 0xFF0000ULL), 232 "bswap.and3"); 233 Tmp2 = Builder.CreateAnd(Tmp2, 234 ConstantInt::get(Type::getInt64Ty(Context), 235 0xFF00ULL), 236 "bswap.and2"); 237 Tmp8 = Builder.CreateOr(Tmp8, Tmp7, "bswap.or1"); 238 Tmp6 = Builder.CreateOr(Tmp6, Tmp5, "bswap.or2"); 239 Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or3"); 240 Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or4"); 241 Tmp8 = Builder.CreateOr(Tmp8, Tmp6, "bswap.or5"); 242 Tmp4 = Builder.CreateOr(Tmp4, Tmp2, "bswap.or6"); 243 V = Builder.CreateOr(Tmp8, Tmp4, "bswap.i64"); 244 break; 245 } 246 } 247 return V; 248} 249 250/// LowerCTPOP - Emit the code to lower ctpop of V before the specified 251/// instruction IP. 252static Value *LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP) { 253 assert(V->getType()->isInteger() && "Can't ctpop a non-integer type!"); 254 255 static const uint64_t MaskValues[6] = { 256 0x5555555555555555ULL, 0x3333333333333333ULL, 257 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL, 258 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL 259 }; 260 261 IRBuilder<> Builder(IP->getParent(), IP); 262 263 unsigned BitSize = V->getType()->getPrimitiveSizeInBits(); 264 unsigned WordSize = (BitSize + 63) / 64; 265 Value *Count = ConstantInt::get(V->getType(), 0); 266 267 for (unsigned n = 0; n < WordSize; ++n) { 268 Value *PartValue = V; 269 for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize); 270 i <<= 1, ++ct) { 271 Value *MaskCst = ConstantInt::get(V->getType(), MaskValues[ct]); 272 Value *LHS = Builder.CreateAnd(PartValue, MaskCst, "cppop.and1"); 273 Value *VShift = Builder.CreateLShr(PartValue, 274 ConstantInt::get(V->getType(), i), 275 "ctpop.sh"); 276 Value *RHS = Builder.CreateAnd(VShift, MaskCst, "cppop.and2"); 277 PartValue = Builder.CreateAdd(LHS, RHS, "ctpop.step"); 278 } 279 Count = Builder.CreateAdd(PartValue, Count, "ctpop.part"); 280 if (BitSize > 64) { 281 V = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 64), 282 "ctpop.part.sh"); 283 BitSize -= 64; 284 } 285 } 286 287 return Count; 288} 289 290/// LowerCTLZ - Emit the code to lower ctlz of V before the specified 291/// instruction IP. 292static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) { 293 294 IRBuilder<> Builder(IP->getParent(), IP); 295 296 unsigned BitSize = V->getType()->getPrimitiveSizeInBits(); 297 for (unsigned i = 1; i < BitSize; i <<= 1) { 298 Value *ShVal = ConstantInt::get(V->getType(), i); 299 ShVal = Builder.CreateLShr(V, ShVal, "ctlz.sh"); 300 V = Builder.CreateOr(V, ShVal, "ctlz.step"); 301 } 302 303 V = Builder.CreateNot(V); 304 return LowerCTPOP(Context, V, IP); 305} 306 307static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname, 308 const char *Dname, 309 const char *LDname) { 310 switch (CI->getOperand(1)->getType()->getTypeID()) { 311 default: llvm_unreachable("Invalid type in intrinsic"); 312 case Type::FloatTyID: 313 ReplaceCallWith(Fname, CI, CI->op_begin() + 1, CI->op_end(), 314 Type::getFloatTy(CI->getContext())); 315 break; 316 case Type::DoubleTyID: 317 ReplaceCallWith(Dname, CI, CI->op_begin() + 1, CI->op_end(), 318 Type::getDoubleTy(CI->getContext())); 319 break; 320 case Type::X86_FP80TyID: 321 case Type::FP128TyID: 322 case Type::PPC_FP128TyID: 323 ReplaceCallWith(LDname, CI, CI->op_begin() + 1, CI->op_end(), 324 CI->getOperand(1)->getType()); 325 break; 326 } 327} 328 329void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) { 330 IRBuilder<> Builder(CI->getParent(), CI); 331 LLVMContext &Context = CI->getContext(); 332 333 Function *Callee = CI->getCalledFunction(); 334 assert(Callee && "Cannot lower an indirect call!"); 335 336 switch (Callee->getIntrinsicID()) { 337 case Intrinsic::not_intrinsic: 338 llvm_report_error("Cannot lower a call to a non-intrinsic function '"+ 339 Callee->getName() + "'!"); 340 default: 341 llvm_report_error("Code generator does not support intrinsic function '"+ 342 Callee->getName()+"'!"); 343 344 // The setjmp/longjmp intrinsics should only exist in the code if it was 345 // never optimized (ie, right out of the CFE), or if it has been hacked on 346 // by the lowerinvoke pass. In both cases, the right thing to do is to 347 // convert the call to an explicit setjmp or longjmp call. 348 case Intrinsic::setjmp: { 349 Value *V = ReplaceCallWith("setjmp", CI, CI->op_begin() + 1, CI->op_end(), 350 Type::getInt32Ty(Context)); 351 if (CI->getType() != Type::getVoidTy(Context)) 352 CI->replaceAllUsesWith(V); 353 break; 354 } 355 case Intrinsic::sigsetjmp: 356 if (CI->getType() != Type::getVoidTy(Context)) 357 CI->replaceAllUsesWith(Constant::getNullValue(CI->getType())); 358 break; 359 360 case Intrinsic::longjmp: { 361 ReplaceCallWith("longjmp", CI, CI->op_begin() + 1, CI->op_end(), 362 Type::getVoidTy(Context)); 363 break; 364 } 365 366 case Intrinsic::siglongjmp: { 367 // Insert the call to abort 368 ReplaceCallWith("abort", CI, CI->op_end(), CI->op_end(), 369 Type::getVoidTy(Context)); 370 break; 371 } 372 case Intrinsic::ctpop: 373 CI->replaceAllUsesWith(LowerCTPOP(Context, CI->getOperand(1), CI)); 374 break; 375 376 case Intrinsic::bswap: 377 CI->replaceAllUsesWith(LowerBSWAP(Context, CI->getOperand(1), CI)); 378 break; 379 380 case Intrinsic::ctlz: 381 CI->replaceAllUsesWith(LowerCTLZ(Context, CI->getOperand(1), CI)); 382 break; 383 384 case Intrinsic::cttz: { 385 // cttz(x) -> ctpop(~X & (X-1)) 386 Value *Src = CI->getOperand(1); 387 Value *NotSrc = Builder.CreateNot(Src); 388 NotSrc->setName(Src->getName() + ".not"); 389 Value *SrcM1 = ConstantInt::get(Src->getType(), 1); 390 SrcM1 = Builder.CreateSub(Src, SrcM1); 391 Src = LowerCTPOP(Context, Builder.CreateAnd(NotSrc, SrcM1), CI); 392 CI->replaceAllUsesWith(Src); 393 break; 394 } 395 396 case Intrinsic::stacksave: 397 case Intrinsic::stackrestore: { 398 if (!Warned) 399 cerr << "WARNING: this target does not support the llvm.stack" 400 << (Callee->getIntrinsicID() == Intrinsic::stacksave ? 401 "save" : "restore") << " intrinsic.\n"; 402 Warned = true; 403 if (Callee->getIntrinsicID() == Intrinsic::stacksave) 404 CI->replaceAllUsesWith(Constant::getNullValue(CI->getType())); 405 break; 406 } 407 408 case Intrinsic::returnaddress: 409 case Intrinsic::frameaddress: 410 cerr << "WARNING: this target does not support the llvm." 411 << (Callee->getIntrinsicID() == Intrinsic::returnaddress ? 412 "return" : "frame") << "address intrinsic.\n"; 413 CI->replaceAllUsesWith(ConstantPointerNull::get( 414 cast<PointerType>(CI->getType()))); 415 break; 416 417 case Intrinsic::prefetch: 418 break; // Simply strip out prefetches on unsupported architectures 419 420 case Intrinsic::pcmarker: 421 break; // Simply strip out pcmarker on unsupported architectures 422 case Intrinsic::readcyclecounter: { 423 cerr << "WARNING: this target does not support the llvm.readcyclecoun" 424 << "ter intrinsic. It is being lowered to a constant 0\n"; 425 CI->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context), 0)); 426 break; 427 } 428 429 case Intrinsic::dbg_stoppoint: 430 case Intrinsic::dbg_region_start: 431 case Intrinsic::dbg_region_end: 432 case Intrinsic::dbg_func_start: 433 case Intrinsic::dbg_declare: 434 break; // Simply strip out debugging intrinsics 435 436 case Intrinsic::eh_exception: 437 case Intrinsic::eh_selector_i32: 438 case Intrinsic::eh_selector_i64: 439 CI->replaceAllUsesWith(Constant::getNullValue(CI->getType())); 440 break; 441 442 case Intrinsic::eh_typeid_for_i32: 443 case Intrinsic::eh_typeid_for_i64: 444 // Return something different to eh_selector. 445 CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1)); 446 break; 447 448 case Intrinsic::var_annotation: 449 break; // Strip out annotate intrinsic 450 451 case Intrinsic::memcpy: { 452 const IntegerType *IntPtr = TD.getIntPtrType(Context); 453 Value *Size = Builder.CreateIntCast(CI->getOperand(3), IntPtr, 454 /* isSigned */ false); 455 Value *Ops[3]; 456 Ops[0] = CI->getOperand(1); 457 Ops[1] = CI->getOperand(2); 458 Ops[2] = Size; 459 ReplaceCallWith("memcpy", CI, Ops, Ops+3, CI->getOperand(1)->getType()); 460 break; 461 } 462 case Intrinsic::memmove: { 463 const IntegerType *IntPtr = TD.getIntPtrType(Context); 464 Value *Size = Builder.CreateIntCast(CI->getOperand(3), IntPtr, 465 /* isSigned */ false); 466 Value *Ops[3]; 467 Ops[0] = CI->getOperand(1); 468 Ops[1] = CI->getOperand(2); 469 Ops[2] = Size; 470 ReplaceCallWith("memmove", CI, Ops, Ops+3, CI->getOperand(1)->getType()); 471 break; 472 } 473 case Intrinsic::memset: { 474 const IntegerType *IntPtr = TD.getIntPtrType(Context); 475 Value *Size = Builder.CreateIntCast(CI->getOperand(3), IntPtr, 476 /* isSigned */ false); 477 Value *Ops[3]; 478 Ops[0] = CI->getOperand(1); 479 // Extend the amount to i32. 480 Ops[1] = Builder.CreateIntCast(CI->getOperand(2), Type::getInt32Ty(Context), 481 /* isSigned */ false); 482 Ops[2] = Size; 483 ReplaceCallWith("memset", CI, Ops, Ops+3, CI->getOperand(1)->getType()); 484 break; 485 } 486 case Intrinsic::sqrt: { 487 ReplaceFPIntrinsicWithCall(CI, "sqrtf", "sqrt", "sqrtl"); 488 break; 489 } 490 case Intrinsic::log: { 491 ReplaceFPIntrinsicWithCall(CI, "logf", "log", "logl"); 492 break; 493 } 494 case Intrinsic::log2: { 495 ReplaceFPIntrinsicWithCall(CI, "log2f", "log2", "log2l"); 496 break; 497 } 498 case Intrinsic::log10: { 499 ReplaceFPIntrinsicWithCall(CI, "log10f", "log10", "log10l"); 500 break; 501 } 502 case Intrinsic::exp: { 503 ReplaceFPIntrinsicWithCall(CI, "expf", "exp", "expl"); 504 break; 505 } 506 case Intrinsic::exp2: { 507 ReplaceFPIntrinsicWithCall(CI, "exp2f", "exp2", "exp2l"); 508 break; 509 } 510 case Intrinsic::pow: { 511 ReplaceFPIntrinsicWithCall(CI, "powf", "pow", "powl"); 512 break; 513 } 514 case Intrinsic::flt_rounds: 515 // Lower to "round to the nearest" 516 if (CI->getType() != Type::getVoidTy(Context)) 517 CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1)); 518 break; 519 } 520 521 assert(CI->use_empty() && 522 "Lowering should have eliminated any uses of the intrinsic call!"); 523 CI->eraseFromParent(); 524} 525