JITEmitter.cpp revision 95da605e15a6f108b551ecc6772823ea53de3007
1//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a MachineCodeEmitter object that is used by the JIT to 11// write machine code to memory and remember where relocatable values are. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "jit" 16#include "JIT.h" 17#include "JITDebugRegisterer.h" 18#include "JITDwarfEmitter.h" 19#include "llvm/ADT/OwningPtr.h" 20#include "llvm/Constants.h" 21#include "llvm/Module.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/Analysis/DebugInfo.h" 24#include "llvm/CodeGen/JITCodeEmitter.h" 25#include "llvm/CodeGen/MachineFunction.h" 26#include "llvm/CodeGen/MachineConstantPool.h" 27#include "llvm/CodeGen/MachineJumpTableInfo.h" 28#include "llvm/CodeGen/MachineModuleInfo.h" 29#include "llvm/CodeGen/MachineRelocation.h" 30#include "llvm/ExecutionEngine/GenericValue.h" 31#include "llvm/ExecutionEngine/JITEventListener.h" 32#include "llvm/ExecutionEngine/JITMemoryManager.h" 33#include "llvm/CodeGen/MachineCodeInfo.h" 34#include "llvm/Target/TargetData.h" 35#include "llvm/Target/TargetJITInfo.h" 36#include "llvm/Target/TargetMachine.h" 37#include "llvm/Target/TargetOptions.h" 38#include "llvm/Support/Debug.h" 39#include "llvm/Support/ErrorHandling.h" 40#include "llvm/Support/ManagedStatic.h" 41#include "llvm/Support/MutexGuard.h" 42#include "llvm/Support/ValueHandle.h" 43#include "llvm/Support/raw_ostream.h" 44#include "llvm/System/Disassembler.h" 45#include "llvm/System/Memory.h" 46#include "llvm/Target/TargetInstrInfo.h" 47#include "llvm/ADT/DenseMap.h" 48#include "llvm/ADT/SmallPtrSet.h" 49#include "llvm/ADT/SmallVector.h" 50#include "llvm/ADT/Statistic.h" 51#include "llvm/ADT/ValueMap.h" 52#include <algorithm> 53#ifndef NDEBUG 54#include <iomanip> 55#endif 56using namespace llvm; 57 58STATISTIC(NumBytes, "Number of bytes of machine code compiled"); 59STATISTIC(NumRelos, "Number of relocations applied"); 60STATISTIC(NumRetries, "Number of retries with more memory"); 61 62 63// A declaration may stop being a declaration once it's fully read from bitcode. 64// This function returns true if F is fully read and is still a declaration. 65static bool isNonGhostDeclaration(const Function *F) { 66 return F->isDeclaration() && !F->isMaterializable(); 67} 68 69//===----------------------------------------------------------------------===// 70// JIT lazy compilation code. 71// 72namespace { 73 class JITEmitter; 74 class JITResolverState; 75 76 template<typename ValueTy> 77 struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> { 78 typedef JITResolverState *ExtraData; 79 static void onRAUW(JITResolverState *, Value *Old, Value *New) { 80 assert(false && "The JIT doesn't know how to handle a" 81 " RAUW on a value it has emitted."); 82 } 83 }; 84 85 struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> { 86 typedef JITResolverState *ExtraData; 87 static void onDelete(JITResolverState *JRS, Function *F); 88 }; 89 90 class JITResolverState { 91 public: 92 typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> > 93 FunctionToLazyStubMapTy; 94 typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy; 95 typedef ValueMap<Function *, SmallPtrSet<void*, 1>, 96 CallSiteValueMapConfig> FunctionToCallSitesMapTy; 97 typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; 98 private: 99 /// FunctionToLazyStubMap - Keep track of the lazy stub created for a 100 /// particular function so that we can reuse them if necessary. 101 FunctionToLazyStubMapTy FunctionToLazyStubMap; 102 103 /// CallSiteToFunctionMap - Keep track of the function that each lazy call 104 /// site corresponds to, and vice versa. 105 CallSiteToFunctionMapTy CallSiteToFunctionMap; 106 FunctionToCallSitesMapTy FunctionToCallSitesMap; 107 108 /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a 109 /// particular GlobalVariable so that we can reuse them if necessary. 110 GlobalToIndirectSymMapTy GlobalToIndirectSymMap; 111 112 /// Instance of the JIT this ResolverState serves. 113 JIT *TheJIT; 114 115 public: 116 JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), 117 FunctionToCallSitesMap(this), 118 TheJIT(jit) {} 119 120 FunctionToLazyStubMapTy& getFunctionToLazyStubMap( 121 const MutexGuard& locked) { 122 assert(locked.holds(TheJIT->lock)); 123 return FunctionToLazyStubMap; 124 } 125 126 GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& locked) { 127 assert(locked.holds(TheJIT->lock)); 128 return GlobalToIndirectSymMap; 129 } 130 131 pair<void *, Function *> LookupFunctionFromCallSite( 132 const MutexGuard &locked, void *CallSite) const { 133 assert(locked.holds(TheJIT->lock)); 134 135 // The address given to us for the stub may not be exactly right, it might be 136 // a little bit after the stub. As such, use upper_bound to find it. 137 CallSiteToFunctionMapTy::const_iterator I = 138 CallSiteToFunctionMap.upper_bound(CallSite); 139 assert(I != CallSiteToFunctionMap.begin() && 140 "This is not a known call site!"); 141 --I; 142 return *I; 143 } 144 145 void AddCallSite(const MutexGuard &locked, void *CallSite, Function *F) { 146 assert(locked.holds(TheJIT->lock)); 147 148 bool Inserted = CallSiteToFunctionMap.insert( 149 std::make_pair(CallSite, F)).second; 150 (void)Inserted; 151 assert(Inserted && "Pair was already in CallSiteToFunctionMap"); 152 FunctionToCallSitesMap[F].insert(CallSite); 153 } 154 155 // Returns the Function of the stub if a stub was erased, or NULL if there 156 // was no stub. This function uses the call-site->function map to find a 157 // relevant function, but asserts that only stubs and not other call sites 158 // will be passed in. 159 Function *EraseStub(const MutexGuard &locked, void *Stub); 160 161 void EraseAllCallSitesFor(const MutexGuard &locked, Function *F) { 162 assert(locked.holds(TheJIT->lock)); 163 EraseAllCallSitesForPrelocked(F); 164 } 165 void EraseAllCallSitesForPrelocked(Function *F); 166 167 // Erases _all_ call sites regardless of their function. This is used to 168 // unregister the stub addresses from the StubToResolverMap in 169 // ~JITResolver(). 170 void EraseAllCallSitesPrelocked(); 171 }; 172 173 /// JITResolver - Keep track of, and resolve, call sites for functions that 174 /// have not yet been compiled. 175 class JITResolver { 176 typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; 177 typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; 178 typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; 179 180 /// LazyResolverFn - The target lazy resolver function that we actually 181 /// rewrite instructions to use. 182 TargetJITInfo::LazyResolverFn LazyResolverFn; 183 184 JITResolverState state; 185 186 /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap 187 /// for external functions. TODO: Of course, external functions don't need 188 /// a lazy stub. It's actually here to make it more likely that far calls 189 /// succeed, but no single stub can guarantee that. I'll remove this in a 190 /// subsequent checkin when I actually fix far calls. 191 std::map<void*, void*> ExternalFnToStubMap; 192 193 /// revGOTMap - map addresses to indexes in the GOT 194 std::map<void*, unsigned> revGOTMap; 195 unsigned nextGOTIndex; 196 197 JITEmitter &JE; 198 199 /// Instance of JIT corresponding to this Resolver. 200 JIT *TheJIT; 201 202 public: 203 explicit JITResolver(JIT &jit, JITEmitter &je) 204 : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { 205 LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); 206 } 207 208 ~JITResolver(); 209 210 /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's 211 /// lazy-compilation stub if it has already been created. 212 void *getLazyFunctionStubIfAvailable(Function *F); 213 214 /// getLazyFunctionStub - This returns a pointer to a function's 215 /// lazy-compilation stub, creating one on demand as needed. 216 void *getLazyFunctionStub(Function *F); 217 218 /// getExternalFunctionStub - Return a stub for the function at the 219 /// specified address, created lazily on demand. 220 void *getExternalFunctionStub(void *FnAddr); 221 222 /// getGlobalValueIndirectSym - Return an indirect symbol containing the 223 /// specified GV address. 224 void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); 225 226 void getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs, 227 SmallVectorImpl<void*> &Ptrs); 228 229 /// getGOTIndexForAddress - Return a new or existing index in the GOT for 230 /// an address. This function only manages slots, it does not manage the 231 /// contents of the slots or the memory associated with the GOT. 232 unsigned getGOTIndexForAddr(void *addr); 233 234 /// JITCompilerFn - This function is called to resolve a stub to a compiled 235 /// address. If the LLVM Function corresponding to the stub has not yet 236 /// been compiled, this function compiles it first. 237 static void *JITCompilerFn(void *Stub); 238 }; 239 240 class StubToResolverMapTy { 241 /// Map a stub address to a specific instance of a JITResolver so that 242 /// lazily-compiled functions can find the right resolver to use. 243 /// 244 /// Guarded by Lock. 245 std::map<void*, JITResolver*> Map; 246 247 /// Guards Map from concurrent accesses. 248 mutable sys::Mutex Lock; 249 250 public: 251 /// Registers a Stub to be resolved by Resolver. 252 void RegisterStubResolver(void *Stub, JITResolver *Resolver) { 253 MutexGuard guard(Lock); 254 Map.insert(std::make_pair(Stub, Resolver)); 255 } 256 /// Unregisters the Stub when it's invalidated. 257 void UnregisterStubResolver(void *Stub) { 258 MutexGuard guard(Lock); 259 Map.erase(Stub); 260 } 261 /// Returns the JITResolver instance that owns the Stub. 262 JITResolver *getResolverFromStub(void *Stub) const { 263 MutexGuard guard(Lock); 264 // The address given to us for the stub may not be exactly right, it might 265 // be a little bit after the stub. As such, use upper_bound to find it. 266 // This is the same trick as in LookupFunctionFromCallSite from 267 // JITResolverState. 268 std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub); 269 assert(I != Map.begin() && "This is not a known stub!"); 270 --I; 271 return I->second; 272 } 273 /// True if any stubs refer to the given resolver. Only used in an assert(). 274 /// O(N) 275 bool ResolverHasStubs(JITResolver* Resolver) const { 276 MutexGuard guard(Lock); 277 for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(), 278 E = Map.end(); I != E; ++I) { 279 if (I->second == Resolver) 280 return true; 281 } 282 return false; 283 } 284 }; 285 /// This needs to be static so that a lazy call stub can access it with no 286 /// context except the address of the stub. 287 ManagedStatic<StubToResolverMapTy> StubToResolverMap; 288 289 /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is 290 /// used to output functions to memory for execution. 291 class JITEmitter : public JITCodeEmitter { 292 JITMemoryManager *MemMgr; 293 294 // When outputting a function stub in the context of some other function, we 295 // save BufferBegin/BufferEnd/CurBufferPtr here. 296 uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 297 298 // When reattempting to JIT a function after running out of space, we store 299 // the estimated size of the function we're trying to JIT here, so we can 300 // ask the memory manager for at least this much space. When we 301 // successfully emit the function, we reset this back to zero. 302 uintptr_t SizeEstimate; 303 304 /// Relocations - These are the relocations that the function needs, as 305 /// emitted. 306 std::vector<MachineRelocation> Relocations; 307 308 /// MBBLocations - This vector is a mapping from MBB ID's to their address. 309 /// It is filled in by the StartMachineBasicBlock callback and queried by 310 /// the getMachineBasicBlockAddress callback. 311 std::vector<uintptr_t> MBBLocations; 312 313 /// ConstantPool - The constant pool for the current function. 314 /// 315 MachineConstantPool *ConstantPool; 316 317 /// ConstantPoolBase - A pointer to the first entry in the constant pool. 318 /// 319 void *ConstantPoolBase; 320 321 /// ConstPoolAddresses - Addresses of individual constant pool entries. 322 /// 323 SmallVector<uintptr_t, 8> ConstPoolAddresses; 324 325 /// JumpTable - The jump tables for the current function. 326 /// 327 MachineJumpTableInfo *JumpTable; 328 329 /// JumpTableBase - A pointer to the first entry in the jump table. 330 /// 331 void *JumpTableBase; 332 333 /// Resolver - This contains info about the currently resolved functions. 334 JITResolver Resolver; 335 336 /// DE - The dwarf emitter for the jit. 337 OwningPtr<JITDwarfEmitter> DE; 338 339 /// DR - The debug registerer for the jit. 340 OwningPtr<JITDebugRegisterer> DR; 341 342 /// LabelLocations - This vector is a mapping from Label ID's to their 343 /// address. 344 std::vector<uintptr_t> LabelLocations; 345 346 /// MMI - Machine module info for exception informations 347 MachineModuleInfo* MMI; 348 349 // CurFn - The llvm function being emitted. Only valid during 350 // finishFunction(). 351 const Function *CurFn; 352 353 /// Information about emitted code, which is passed to the 354 /// JITEventListeners. This is reset in startFunction and used in 355 /// finishFunction. 356 JITEvent_EmittedFunctionDetails EmissionDetails; 357 358 struct EmittedCode { 359 void *FunctionBody; // Beginning of the function's allocation. 360 void *Code; // The address the function's code actually starts at. 361 void *ExceptionTable; 362 EmittedCode() : FunctionBody(0), Code(0), ExceptionTable(0) {} 363 }; 364 struct EmittedFunctionConfig : public ValueMapConfig<const Function*> { 365 typedef JITEmitter *ExtraData; 366 static void onDelete(JITEmitter *, const Function*); 367 static void onRAUW(JITEmitter *, const Function*, const Function*); 368 }; 369 ValueMap<const Function *, EmittedCode, 370 EmittedFunctionConfig> EmittedFunctions; 371 372 DILocation PrevDLT; 373 374 /// Instance of the JIT 375 JIT *TheJIT; 376 377 public: 378 JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) 379 : SizeEstimate(0), Resolver(jit, *this), MMI(0), CurFn(0), 380 EmittedFunctions(this), PrevDLT(NULL), TheJIT(&jit) { 381 MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); 382 if (jit.getJITInfo().needsGOT()) { 383 MemMgr->AllocateGOT(); 384 DEBUG(dbgs() << "JIT is managing a GOT\n"); 385 } 386 387 if (DwarfExceptionHandling || JITEmitDebugInfo) { 388 DE.reset(new JITDwarfEmitter(jit)); 389 } 390 if (JITEmitDebugInfo) { 391 DR.reset(new JITDebugRegisterer(TM)); 392 } 393 } 394 ~JITEmitter() { 395 delete MemMgr; 396 } 397 398 /// classof - Methods for support type inquiry through isa, cast, and 399 /// dyn_cast: 400 /// 401 static inline bool classof(const JITEmitter*) { return true; } 402 static inline bool classof(const MachineCodeEmitter*) { return true; } 403 404 JITResolver &getJITResolver() { return Resolver; } 405 406 virtual void startFunction(MachineFunction &F); 407 virtual bool finishFunction(MachineFunction &F); 408 409 void emitConstantPool(MachineConstantPool *MCP); 410 void initJumpTableInfo(MachineJumpTableInfo *MJTI); 411 void emitJumpTableInfo(MachineJumpTableInfo *MJTI); 412 413 void startGVStub(const GlobalValue* GV, 414 unsigned StubSize, unsigned Alignment = 1); 415 void startGVStub(void *Buffer, unsigned StubSize); 416 void finishGVStub(); 417 virtual void *allocIndirectGV(const GlobalValue *GV, 418 const uint8_t *Buffer, size_t Size, 419 unsigned Alignment); 420 421 /// allocateSpace - Reserves space in the current block if any, or 422 /// allocate a new one of the given size. 423 virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); 424 425 /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, 426 /// this method does not allocate memory in the current output buffer, 427 /// because a global may live longer than the current function. 428 virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment); 429 430 virtual void addRelocation(const MachineRelocation &MR) { 431 Relocations.push_back(MR); 432 } 433 434 virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { 435 if (MBBLocations.size() <= (unsigned)MBB->getNumber()) 436 MBBLocations.resize((MBB->getNumber()+1)*2); 437 MBBLocations[MBB->getNumber()] = getCurrentPCValue(); 438 DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" 439 << (void*) getCurrentPCValue() << "]\n"); 440 } 441 442 virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; 443 virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; 444 445 virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const { 446 assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 447 MBBLocations[MBB->getNumber()] && "MBB not emitted!"); 448 return MBBLocations[MBB->getNumber()]; 449 } 450 451 /// retryWithMoreMemory - Log a retry and deallocate all memory for the 452 /// given function. Increase the minimum allocation size so that we get 453 /// more memory next time. 454 void retryWithMoreMemory(MachineFunction &F); 455 456 /// deallocateMemForFunction - Deallocate all memory for the specified 457 /// function body. 458 void deallocateMemForFunction(const Function *F); 459 460 virtual void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn); 461 462 virtual void emitLabel(uint64_t LabelID) { 463 if (LabelLocations.size() <= LabelID) 464 LabelLocations.resize((LabelID+1)*2); 465 LabelLocations[LabelID] = getCurrentPCValue(); 466 } 467 468 virtual uintptr_t getLabelAddress(uint64_t LabelID) const { 469 assert(LabelLocations.size() > (unsigned)LabelID && 470 LabelLocations[LabelID] && "Label not emitted!"); 471 return LabelLocations[LabelID]; 472 } 473 474 virtual void setModuleInfo(MachineModuleInfo* Info) { 475 MMI = Info; 476 if (DE.get()) DE->setModuleInfo(Info); 477 } 478 479 void setMemoryExecutable() { 480 MemMgr->setMemoryExecutable(); 481 } 482 483 JITMemoryManager *getMemMgr() const { return MemMgr; } 484 485 private: 486 void *getPointerToGlobal(GlobalValue *GV, void *Reference, 487 bool MayNeedFarStub); 488 void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); 489 unsigned addSizeOfGlobal(const GlobalVariable *GV, unsigned Size); 490 unsigned addSizeOfGlobalsInConstantVal( 491 const Constant *C, unsigned Size, 492 SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 493 SmallVectorImpl<const GlobalVariable*> &Worklist); 494 unsigned addSizeOfGlobalsInInitializer( 495 const Constant *Init, unsigned Size, 496 SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 497 SmallVectorImpl<const GlobalVariable*> &Worklist); 498 unsigned GetSizeOfGlobalsInBytes(MachineFunction &MF); 499 }; 500} 501 502void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { 503 JRS->EraseAllCallSitesForPrelocked(F); 504} 505 506Function *JITResolverState::EraseStub(const MutexGuard &locked, void *Stub) { 507 CallSiteToFunctionMapTy::iterator C2F_I = 508 CallSiteToFunctionMap.find(Stub); 509 if (C2F_I == CallSiteToFunctionMap.end()) { 510 // Not a stub. 511 return NULL; 512 } 513 514 StubToResolverMap->UnregisterStubResolver(Stub); 515 516 Function *const F = C2F_I->second; 517#ifndef NDEBUG 518 void *RealStub = FunctionToLazyStubMap.lookup(F); 519 assert(RealStub == Stub && 520 "Call-site that wasn't a stub passed in to EraseStub"); 521#endif 522 FunctionToLazyStubMap.erase(F); 523 CallSiteToFunctionMap.erase(C2F_I); 524 525 // Remove the stub from the function->call-sites map, and remove the whole 526 // entry from the map if that was the last call site. 527 FunctionToCallSitesMapTy::iterator F2C_I = FunctionToCallSitesMap.find(F); 528 assert(F2C_I != FunctionToCallSitesMap.end() && 529 "FunctionToCallSitesMap broken"); 530 bool Erased = F2C_I->second.erase(Stub); 531 (void)Erased; 532 assert(Erased && "FunctionToCallSitesMap broken"); 533 if (F2C_I->second.empty()) 534 FunctionToCallSitesMap.erase(F2C_I); 535 536 return F; 537} 538 539void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { 540 FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); 541 if (F2C == FunctionToCallSitesMap.end()) 542 return; 543 StubToResolverMapTy &S2RMap = *StubToResolverMap; 544 for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(), 545 E = F2C->second.end(); I != E; ++I) { 546 S2RMap.UnregisterStubResolver(*I); 547 bool Erased = CallSiteToFunctionMap.erase(*I); 548 (void)Erased; 549 assert(Erased && "Missing call site->function mapping"); 550 } 551 FunctionToCallSitesMap.erase(F2C); 552} 553 554void JITResolverState::EraseAllCallSitesPrelocked() { 555 StubToResolverMapTy &S2RMap = *StubToResolverMap; 556 for (CallSiteToFunctionMapTy::const_iterator 557 I = CallSiteToFunctionMap.begin(), 558 E = CallSiteToFunctionMap.end(); I != E; ++I) { 559 S2RMap.UnregisterStubResolver(I->first); 560 } 561 CallSiteToFunctionMap.clear(); 562 FunctionToCallSitesMap.clear(); 563} 564 565JITResolver::~JITResolver() { 566 // No need to lock because we're in the destructor, and state isn't shared. 567 state.EraseAllCallSitesPrelocked(); 568 assert(!StubToResolverMap->ResolverHasStubs(this) && 569 "Resolver destroyed with stubs still alive."); 570} 571 572/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub 573/// if it has already been created. 574void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { 575 MutexGuard locked(TheJIT->lock); 576 577 // If we already have a stub for this function, recycle it. 578 return state.getFunctionToLazyStubMap(locked).lookup(F); 579} 580 581/// getFunctionStub - This returns a pointer to a function stub, creating 582/// one on demand as needed. 583void *JITResolver::getLazyFunctionStub(Function *F) { 584 MutexGuard locked(TheJIT->lock); 585 586 // If we already have a lazy stub for this function, recycle it. 587 void *&Stub = state.getFunctionToLazyStubMap(locked)[F]; 588 if (Stub) return Stub; 589 590 // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we 591 // must resolve the symbol now. 592 void *Actual = TheJIT->isCompilingLazily() 593 ? (void *)(intptr_t)LazyResolverFn : (void *)0; 594 595 // If this is an external declaration, attempt to resolve the address now 596 // to place in the stub. 597 if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { 598 Actual = TheJIT->getPointerToFunction(F); 599 600 // If we resolved the symbol to a null address (eg. a weak external) 601 // don't emit a stub. Return a null pointer to the application. 602 if (!Actual) return 0; 603 } 604 605 TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 606 JE.startGVStub(F, SL.Size, SL.Alignment); 607 // Codegen a new stub, calling the lazy resolver or the actual address of the 608 // external function, if it was resolved. 609 Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); 610 JE.finishGVStub(); 611 612 if (Actual != (void*)(intptr_t)LazyResolverFn) { 613 // If we are getting the stub for an external function, we really want the 614 // address of the stub in the GlobalAddressMap for the JIT, not the address 615 // of the external function. 616 TheJIT->updateGlobalMapping(F, Stub); 617 } 618 619 DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" 620 << F->getName() << "'\n"); 621 622 if (TheJIT->isCompilingLazily()) { 623 // Register this JITResolver as the one corresponding to this call site so 624 // JITCompilerFn will be able to find it. 625 StubToResolverMap->RegisterStubResolver(Stub, this); 626 627 // Finally, keep track of the stub-to-Function mapping so that the 628 // JITCompilerFn knows which function to compile! 629 state.AddCallSite(locked, Stub, F); 630 } else if (!Actual) { 631 // If we are JIT'ing non-lazily but need to call a function that does not 632 // exist yet, add it to the JIT's work list so that we can fill in the 633 // stub address later. 634 assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && 635 "'Actual' should have been set above."); 636 TheJIT->addPendingFunction(F); 637 } 638 639 return Stub; 640} 641 642/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified 643/// GV address. 644void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { 645 MutexGuard locked(TheJIT->lock); 646 647 // If we already have a stub for this global variable, recycle it. 648 void *&IndirectSym = state.getGlobalToIndirectSymMap(locked)[GV]; 649 if (IndirectSym) return IndirectSym; 650 651 // Otherwise, codegen a new indirect symbol. 652 IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, 653 JE); 654 655 DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym 656 << "] for GV '" << GV->getName() << "'\n"); 657 658 return IndirectSym; 659} 660 661/// getExternalFunctionStub - Return a stub for the function at the 662/// specified address, created lazily on demand. 663void *JITResolver::getExternalFunctionStub(void *FnAddr) { 664 // If we already have a stub for this function, recycle it. 665 void *&Stub = ExternalFnToStubMap[FnAddr]; 666 if (Stub) return Stub; 667 668 TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 669 JE.startGVStub(0, SL.Size, SL.Alignment); 670 Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, JE); 671 JE.finishGVStub(); 672 673 DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub 674 << "] for external function at '" << FnAddr << "'\n"); 675 return Stub; 676} 677 678unsigned JITResolver::getGOTIndexForAddr(void* addr) { 679 unsigned idx = revGOTMap[addr]; 680 if (!idx) { 681 idx = ++nextGOTIndex; 682 revGOTMap[addr] = idx; 683 DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" 684 << addr << "]\n"); 685 } 686 return idx; 687} 688 689void JITResolver::getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs, 690 SmallVectorImpl<void*> &Ptrs) { 691 MutexGuard locked(TheJIT->lock); 692 693 const FunctionToLazyStubMapTy &FM = state.getFunctionToLazyStubMap(locked); 694 GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); 695 696 for (FunctionToLazyStubMapTy::const_iterator i = FM.begin(), e = FM.end(); 697 i != e; ++i){ 698 Function *F = i->first; 699 if (F->isDeclaration() && F->hasExternalLinkage()) { 700 GVs.push_back(i->first); 701 Ptrs.push_back(i->second); 702 } 703 } 704 for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); 705 i != e; ++i) { 706 GVs.push_back(i->first); 707 Ptrs.push_back(i->second); 708 } 709} 710 711/// JITCompilerFn - This function is called when a lazy compilation stub has 712/// been entered. It looks up which function this stub corresponds to, compiles 713/// it if necessary, then returns the resultant function pointer. 714void *JITResolver::JITCompilerFn(void *Stub) { 715 JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); 716 assert(JR && "Unable to find the corresponding JITResolver to the call site"); 717 718 Function* F = 0; 719 void* ActualPtr = 0; 720 721 { 722 // Only lock for getting the Function. The call getPointerToFunction made 723 // in this function might trigger function materializing, which requires 724 // JIT lock to be unlocked. 725 MutexGuard locked(JR->TheJIT->lock); 726 727 // The address given to us for the stub may not be exactly right, it might 728 // be a little bit after the stub. As such, use upper_bound to find it. 729 pair<void*, Function*> I = 730 JR->state.LookupFunctionFromCallSite(locked, Stub); 731 F = I.second; 732 ActualPtr = I.first; 733 } 734 735 // If we have already code generated the function, just return the address. 736 void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); 737 738 if (!Result) { 739 // Otherwise we don't have it, do lazy compilation now. 740 741 // If lazy compilation is disabled, emit a useful error message and abort. 742 if (!JR->TheJIT->isCompilingLazily()) { 743 llvm_report_error("LLVM JIT requested to do lazy compilation of function '" 744 + F->getName() + "' when lazy compiles are disabled!"); 745 } 746 747 DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() 748 << "' In stub ptr = " << Stub << " actual ptr = " 749 << ActualPtr << "\n"); 750 751 Result = JR->TheJIT->getPointerToFunction(F); 752 } 753 754 // Reacquire the lock to update the GOT map. 755 MutexGuard locked(JR->TheJIT->lock); 756 757 // We might like to remove the call site from the CallSiteToFunction map, but 758 // we can't do that! Multiple threads could be stuck, waiting to acquire the 759 // lock above. As soon as the 1st function finishes compiling the function, 760 // the next one will be released, and needs to be able to find the function it 761 // needs to call. 762 763 // FIXME: We could rewrite all references to this stub if we knew them. 764 765 // What we will do is set the compiled function address to map to the 766 // same GOT entry as the stub so that later clients may update the GOT 767 // if they see it still using the stub address. 768 // Note: this is done so the Resolver doesn't have to manage GOT memory 769 // Do this without allocating map space if the target isn't using a GOT 770 if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) 771 JR->revGOTMap[Result] = JR->revGOTMap[Stub]; 772 773 return Result; 774} 775 776//===----------------------------------------------------------------------===// 777// JITEmitter code. 778// 779void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, 780 bool MayNeedFarStub) { 781 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) 782 return TheJIT->getOrEmitGlobalVariable(GV); 783 784 if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 785 return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false)); 786 787 // If we have already compiled the function, return a pointer to its body. 788 Function *F = cast<Function>(V); 789 790 void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); 791 if (FnStub) { 792 // Return the function stub if it's already created. We do this first so 793 // that we're returning the same address for the function as any previous 794 // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be 795 // close enough to call. 796 return FnStub; 797 } 798 799 // If we know the target can handle arbitrary-distance calls, try to 800 // return a direct pointer. 801 if (!MayNeedFarStub) { 802 // If we have code, go ahead and return that. 803 void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); 804 if (ResultPtr) return ResultPtr; 805 806 // If this is an external function pointer, we can force the JIT to 807 // 'compile' it, which really just adds it to the map. 808 if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) 809 return TheJIT->getPointerToFunction(F); 810 } 811 812 // Otherwise, we may need a to emit a stub, and, conservatively, we always do 813 // so. Note that it's possible to return null from getLazyFunctionStub in the 814 // case of a weak extern that fails to resolve. 815 return Resolver.getLazyFunctionStub(F); 816} 817 818void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { 819 // Make sure GV is emitted first, and create a stub containing the fully 820 // resolved address. 821 void *GVAddress = getPointerToGlobal(V, Reference, false); 822 void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); 823 return StubAddr; 824} 825 826void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { 827 if (!DL.isUnknown()) { 828 DILocation CurDLT = EmissionDetails.MF->getDILocation(DL); 829 830 if (BeforePrintingInsn) { 831 if (CurDLT.getScope().getNode() != 0 832 && PrevDLT.getNode() != CurDLT.getNode()) { 833 JITEvent_EmittedFunctionDetails::LineStart NextLine; 834 NextLine.Address = getCurrentPCValue(); 835 NextLine.Loc = DL; 836 EmissionDetails.LineStarts.push_back(NextLine); 837 } 838 839 PrevDLT = CurDLT; 840 } 841 } 842} 843 844static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, 845 const TargetData *TD) { 846 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 847 if (Constants.empty()) return 0; 848 849 unsigned Size = 0; 850 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 851 MachineConstantPoolEntry CPE = Constants[i]; 852 unsigned AlignMask = CPE.getAlignment() - 1; 853 Size = (Size + AlignMask) & ~AlignMask; 854 const Type *Ty = CPE.getType(); 855 Size += TD->getTypeAllocSize(Ty); 856 } 857 return Size; 858} 859 860static unsigned GetJumpTableSizeInBytes(MachineJumpTableInfo *MJTI, JIT *jit) { 861 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 862 if (JT.empty()) return 0; 863 864 unsigned NumEntries = 0; 865 for (unsigned i = 0, e = JT.size(); i != e; ++i) 866 NumEntries += JT[i].MBBs.size(); 867 868 return NumEntries * MJTI->getEntrySize(*jit->getTargetData()); 869} 870 871static uintptr_t RoundUpToAlign(uintptr_t Size, unsigned Alignment) { 872 if (Alignment == 0) Alignment = 1; 873 // Since we do not know where the buffer will be allocated, be pessimistic. 874 return Size + Alignment; 875} 876 877/// addSizeOfGlobal - add the size of the global (plus any alignment padding) 878/// into the running total Size. 879 880unsigned JITEmitter::addSizeOfGlobal(const GlobalVariable *GV, unsigned Size) { 881 const Type *ElTy = GV->getType()->getElementType(); 882 size_t GVSize = (size_t)TheJIT->getTargetData()->getTypeAllocSize(ElTy); 883 size_t GVAlign = 884 (size_t)TheJIT->getTargetData()->getPreferredAlignment(GV); 885 DEBUG(dbgs() << "JIT: Adding in size " << GVSize << " alignment " << GVAlign); 886 DEBUG(GV->dump()); 887 // Assume code section ends with worst possible alignment, so first 888 // variable needs maximal padding. 889 if (Size==0) 890 Size = 1; 891 Size = ((Size+GVAlign-1)/GVAlign)*GVAlign; 892 Size += GVSize; 893 return Size; 894} 895 896/// addSizeOfGlobalsInConstantVal - find any globals that we haven't seen yet 897/// but are referenced from the constant; put them in SeenGlobals and the 898/// Worklist, and add their size into the running total Size. 899 900unsigned JITEmitter::addSizeOfGlobalsInConstantVal( 901 const Constant *C, 902 unsigned Size, 903 SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 904 SmallVectorImpl<const GlobalVariable*> &Worklist) { 905 // If its undefined, return the garbage. 906 if (isa<UndefValue>(C)) 907 return Size; 908 909 // If the value is a ConstantExpr 910 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 911 Constant *Op0 = CE->getOperand(0); 912 switch (CE->getOpcode()) { 913 case Instruction::GetElementPtr: 914 case Instruction::Trunc: 915 case Instruction::ZExt: 916 case Instruction::SExt: 917 case Instruction::FPTrunc: 918 case Instruction::FPExt: 919 case Instruction::UIToFP: 920 case Instruction::SIToFP: 921 case Instruction::FPToUI: 922 case Instruction::FPToSI: 923 case Instruction::PtrToInt: 924 case Instruction::IntToPtr: 925 case Instruction::BitCast: { 926 Size = addSizeOfGlobalsInConstantVal(Op0, Size, SeenGlobals, Worklist); 927 break; 928 } 929 case Instruction::Add: 930 case Instruction::FAdd: 931 case Instruction::Sub: 932 case Instruction::FSub: 933 case Instruction::Mul: 934 case Instruction::FMul: 935 case Instruction::UDiv: 936 case Instruction::SDiv: 937 case Instruction::URem: 938 case Instruction::SRem: 939 case Instruction::And: 940 case Instruction::Or: 941 case Instruction::Xor: { 942 Size = addSizeOfGlobalsInConstantVal(Op0, Size, SeenGlobals, Worklist); 943 Size = addSizeOfGlobalsInConstantVal(CE->getOperand(1), Size, 944 SeenGlobals, Worklist); 945 break; 946 } 947 default: { 948 std::string msg; 949 raw_string_ostream Msg(msg); 950 Msg << "ConstantExpr not handled: " << *CE; 951 llvm_report_error(Msg.str()); 952 } 953 } 954 } 955 956 if (C->getType()->getTypeID() == Type::PointerTyID) 957 if (const GlobalVariable* GV = dyn_cast<GlobalVariable>(C)) 958 if (SeenGlobals.insert(GV)) { 959 Worklist.push_back(GV); 960 Size = addSizeOfGlobal(GV, Size); 961 } 962 963 return Size; 964} 965 966/// addSizeOfGLobalsInInitializer - handle any globals that we haven't seen yet 967/// but are referenced from the given initializer. 968 969unsigned JITEmitter::addSizeOfGlobalsInInitializer( 970 const Constant *Init, 971 unsigned Size, 972 SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 973 SmallVectorImpl<const GlobalVariable*> &Worklist) { 974 if (!isa<UndefValue>(Init) && 975 !isa<ConstantVector>(Init) && 976 !isa<ConstantAggregateZero>(Init) && 977 !isa<ConstantArray>(Init) && 978 !isa<ConstantStruct>(Init) && 979 Init->getType()->isFirstClassType()) 980 Size = addSizeOfGlobalsInConstantVal(Init, Size, SeenGlobals, Worklist); 981 return Size; 982} 983 984/// GetSizeOfGlobalsInBytes - walk the code for the function, looking for 985/// globals; then walk the initializers of those globals looking for more. 986/// If their size has not been considered yet, add it into the running total 987/// Size. 988 989unsigned JITEmitter::GetSizeOfGlobalsInBytes(MachineFunction &MF) { 990 unsigned Size = 0; 991 SmallPtrSet<const GlobalVariable*, 8> SeenGlobals; 992 993 for (MachineFunction::iterator MBB = MF.begin(), E = MF.end(); 994 MBB != E; ++MBB) { 995 for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end(); 996 I != E; ++I) { 997 const TargetInstrDesc &Desc = I->getDesc(); 998 const MachineInstr &MI = *I; 999 unsigned NumOps = Desc.getNumOperands(); 1000 for (unsigned CurOp = 0; CurOp < NumOps; CurOp++) { 1001 const MachineOperand &MO = MI.getOperand(CurOp); 1002 if (MO.isGlobal()) { 1003 GlobalValue* V = MO.getGlobal(); 1004 const GlobalVariable *GV = dyn_cast<const GlobalVariable>(V); 1005 if (!GV) 1006 continue; 1007 // If seen in previous function, it will have an entry here. 1008 if (TheJIT->getPointerToGlobalIfAvailable(GV)) 1009 continue; 1010 // If seen earlier in this function, it will have an entry here. 1011 // FIXME: it should be possible to combine these tables, by 1012 // assuming the addresses of the new globals in this module 1013 // start at 0 (or something) and adjusting them after codegen 1014 // complete. Another possibility is to grab a marker bit in GV. 1015 if (SeenGlobals.insert(GV)) 1016 // A variable as yet unseen. Add in its size. 1017 Size = addSizeOfGlobal(GV, Size); 1018 } 1019 } 1020 } 1021 } 1022 DEBUG(dbgs() << "JIT: About to look through initializers\n"); 1023 // Look for more globals that are referenced only from initializers. 1024 SmallVector<const GlobalVariable*, 8> Worklist( 1025 SeenGlobals.begin(), SeenGlobals.end()); 1026 while (!Worklist.empty()) { 1027 const GlobalVariable* GV = Worklist.back(); 1028 Worklist.pop_back(); 1029 if (GV->hasInitializer()) 1030 Size = addSizeOfGlobalsInInitializer(GV->getInitializer(), Size, 1031 SeenGlobals, Worklist); 1032 } 1033 1034 return Size; 1035} 1036 1037void JITEmitter::startFunction(MachineFunction &F) { 1038 DEBUG(dbgs() << "JIT: Starting CodeGen of Function " 1039 << F.getFunction()->getName() << "\n"); 1040 1041 uintptr_t ActualSize = 0; 1042 // Set the memory writable, if it's not already 1043 MemMgr->setMemoryWritable(); 1044 if (MemMgr->NeedsExactSize()) { 1045 DEBUG(dbgs() << "JIT: ExactSize\n"); 1046 const TargetInstrInfo* TII = F.getTarget().getInstrInfo(); 1047 MachineConstantPool *MCP = F.getConstantPool(); 1048 1049 // Ensure the constant pool/jump table info is at least 4-byte aligned. 1050 ActualSize = RoundUpToAlign(ActualSize, 16); 1051 1052 // Add the alignment of the constant pool 1053 ActualSize = RoundUpToAlign(ActualSize, MCP->getConstantPoolAlignment()); 1054 1055 // Add the constant pool size 1056 ActualSize += GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 1057 1058 if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) { 1059 // Add the aligment of the jump table info 1060 ActualSize = RoundUpToAlign(ActualSize, 1061 MJTI->getEntryAlignment(*TheJIT->getTargetData())); 1062 1063 // Add the jump table size 1064 ActualSize += GetJumpTableSizeInBytes(MJTI, TheJIT); 1065 } 1066 1067 // Add the alignment for the function 1068 ActualSize = RoundUpToAlign(ActualSize, 1069 std::max(F.getFunction()->getAlignment(), 8U)); 1070 1071 // Add the function size 1072 ActualSize += TII->GetFunctionSizeInBytes(F); 1073 1074 DEBUG(dbgs() << "JIT: ActualSize before globals " << ActualSize << "\n"); 1075 // Add the size of the globals that will be allocated after this function. 1076 // These are all the ones referenced from this function that were not 1077 // previously allocated. 1078 ActualSize += GetSizeOfGlobalsInBytes(F); 1079 DEBUG(dbgs() << "JIT: ActualSize after globals " << ActualSize << "\n"); 1080 } else if (SizeEstimate > 0) { 1081 // SizeEstimate will be non-zero on reallocation attempts. 1082 ActualSize = SizeEstimate; 1083 } 1084 1085 BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), 1086 ActualSize); 1087 BufferEnd = BufferBegin+ActualSize; 1088 EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; 1089 1090 // Ensure the constant pool/jump table info is at least 4-byte aligned. 1091 emitAlignment(16); 1092 1093 emitConstantPool(F.getConstantPool()); 1094 if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 1095 initJumpTableInfo(MJTI); 1096 1097 // About to start emitting the machine code for the function. 1098 emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); 1099 TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); 1100 EmittedFunctions[F.getFunction()].Code = CurBufferPtr; 1101 1102 MBBLocations.clear(); 1103 1104 EmissionDetails.MF = &F; 1105 EmissionDetails.LineStarts.clear(); 1106} 1107 1108bool JITEmitter::finishFunction(MachineFunction &F) { 1109 if (CurBufferPtr == BufferEnd) { 1110 // We must call endFunctionBody before retrying, because 1111 // deallocateMemForFunction requires it. 1112 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 1113 retryWithMoreMemory(F); 1114 return true; 1115 } 1116 1117 if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 1118 emitJumpTableInfo(MJTI); 1119 1120 // FnStart is the start of the text, not the start of the constant pool and 1121 // other per-function data. 1122 uint8_t *FnStart = 1123 (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); 1124 1125 // FnEnd is the end of the function's machine code. 1126 uint8_t *FnEnd = CurBufferPtr; 1127 1128 if (!Relocations.empty()) { 1129 CurFn = F.getFunction(); 1130 NumRelos += Relocations.size(); 1131 1132 // Resolve the relocations to concrete pointers. 1133 for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { 1134 MachineRelocation &MR = Relocations[i]; 1135 void *ResultPtr = 0; 1136 if (!MR.letTargetResolve()) { 1137 if (MR.isExternalSymbol()) { 1138 ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), 1139 false); 1140 DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" 1141 << ResultPtr << "]\n"); 1142 1143 // If the target REALLY wants a stub for this function, emit it now. 1144 if (MR.mayNeedFarStub()) { 1145 ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); 1146 } 1147 } else if (MR.isGlobalValue()) { 1148 ResultPtr = getPointerToGlobal(MR.getGlobalValue(), 1149 BufferBegin+MR.getMachineCodeOffset(), 1150 MR.mayNeedFarStub()); 1151 } else if (MR.isIndirectSymbol()) { 1152 ResultPtr = getPointerToGVIndirectSym( 1153 MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); 1154 } else if (MR.isBasicBlock()) { 1155 ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); 1156 } else if (MR.isConstantPoolIndex()) { 1157 ResultPtr = (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); 1158 } else { 1159 assert(MR.isJumpTableIndex()); 1160 ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); 1161 } 1162 1163 MR.setResultPointer(ResultPtr); 1164 } 1165 1166 // if we are managing the GOT and the relocation wants an index, 1167 // give it one 1168 if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { 1169 unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); 1170 MR.setGOTIndex(idx); 1171 if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { 1172 DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr 1173 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 1174 << "\n"); 1175 ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; 1176 } 1177 } 1178 } 1179 1180 CurFn = 0; 1181 TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], 1182 Relocations.size(), MemMgr->getGOTBase()); 1183 } 1184 1185 // Update the GOT entry for F to point to the new code. 1186 if (MemMgr->isManagingGOT()) { 1187 unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); 1188 if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { 1189 DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin 1190 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 1191 << "\n"); 1192 ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; 1193 } 1194 } 1195 1196 // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for 1197 // global variables that were referenced in the relocations. 1198 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 1199 1200 if (CurBufferPtr == BufferEnd) { 1201 retryWithMoreMemory(F); 1202 return true; 1203 } else { 1204 // Now that we've succeeded in emitting the function, reset the 1205 // SizeEstimate back down to zero. 1206 SizeEstimate = 0; 1207 } 1208 1209 BufferBegin = CurBufferPtr = 0; 1210 NumBytes += FnEnd-FnStart; 1211 1212 // Invalidate the icache if necessary. 1213 sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); 1214 1215 TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, 1216 EmissionDetails); 1217 1218 DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart 1219 << "] Function: " << F.getFunction()->getName() 1220 << ": " << (FnEnd-FnStart) << " bytes of text, " 1221 << Relocations.size() << " relocations\n"); 1222 1223 Relocations.clear(); 1224 ConstPoolAddresses.clear(); 1225 1226 // Mark code region readable and executable if it's not so already. 1227 MemMgr->setMemoryExecutable(); 1228 1229 DEBUG( 1230 if (sys::hasDisassembler()) { 1231 dbgs() << "JIT: Disassembled code:\n"; 1232 dbgs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, 1233 (uintptr_t)FnStart); 1234 } else { 1235 dbgs() << "JIT: Binary code:\n"; 1236 uint8_t* q = FnStart; 1237 for (int i = 0; q < FnEnd; q += 4, ++i) { 1238 if (i == 4) 1239 i = 0; 1240 if (i == 0) 1241 dbgs() << "JIT: " << (long)(q - FnStart) << ": "; 1242 bool Done = false; 1243 for (int j = 3; j >= 0; --j) { 1244 if (q + j >= FnEnd) 1245 Done = true; 1246 else 1247 dbgs() << (unsigned short)q[j]; 1248 } 1249 if (Done) 1250 break; 1251 dbgs() << ' '; 1252 if (i == 3) 1253 dbgs() << '\n'; 1254 } 1255 dbgs()<< '\n'; 1256 } 1257 ); 1258 1259 if (DwarfExceptionHandling || JITEmitDebugInfo) { 1260 uintptr_t ActualSize = 0; 1261 SavedBufferBegin = BufferBegin; 1262 SavedBufferEnd = BufferEnd; 1263 SavedCurBufferPtr = CurBufferPtr; 1264 1265 if (MemMgr->NeedsExactSize()) { 1266 ActualSize = DE->GetDwarfTableSizeInBytes(F, *this, FnStart, FnEnd); 1267 } 1268 1269 BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(), 1270 ActualSize); 1271 BufferEnd = BufferBegin+ActualSize; 1272 EmittedFunctions[F.getFunction()].ExceptionTable = BufferBegin; 1273 uint8_t *EhStart; 1274 uint8_t *FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd, 1275 EhStart); 1276 MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, 1277 FrameRegister); 1278 uint8_t *EhEnd = CurBufferPtr; 1279 BufferBegin = SavedBufferBegin; 1280 BufferEnd = SavedBufferEnd; 1281 CurBufferPtr = SavedCurBufferPtr; 1282 1283 if (DwarfExceptionHandling) { 1284 TheJIT->RegisterTable(FrameRegister); 1285 } 1286 1287 if (JITEmitDebugInfo) { 1288 DebugInfo I; 1289 I.FnStart = FnStart; 1290 I.FnEnd = FnEnd; 1291 I.EhStart = EhStart; 1292 I.EhEnd = EhEnd; 1293 DR->RegisterFunction(F.getFunction(), I); 1294 } 1295 } 1296 1297 if (MMI) 1298 MMI->EndFunction(); 1299 1300 return false; 1301} 1302 1303void JITEmitter::retryWithMoreMemory(MachineFunction &F) { 1304 DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); 1305 Relocations.clear(); // Clear the old relocations or we'll reapply them. 1306 ConstPoolAddresses.clear(); 1307 ++NumRetries; 1308 deallocateMemForFunction(F.getFunction()); 1309 // Try again with at least twice as much free space. 1310 SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); 1311} 1312 1313/// deallocateMemForFunction - Deallocate all memory for the specified 1314/// function body. Also drop any references the function has to stubs. 1315/// May be called while the Function is being destroyed inside ~Value(). 1316void JITEmitter::deallocateMemForFunction(const Function *F) { 1317 ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator 1318 Emitted = EmittedFunctions.find(F); 1319 if (Emitted != EmittedFunctions.end()) { 1320 MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); 1321 MemMgr->deallocateExceptionTable(Emitted->second.ExceptionTable); 1322 TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); 1323 1324 EmittedFunctions.erase(Emitted); 1325 } 1326 1327 // TODO: Do we need to unregister exception handling information from libgcc 1328 // here? 1329 1330 if (JITEmitDebugInfo) { 1331 DR->UnregisterFunction(F); 1332 } 1333} 1334 1335 1336void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { 1337 if (BufferBegin) 1338 return JITCodeEmitter::allocateSpace(Size, Alignment); 1339 1340 // create a new memory block if there is no active one. 1341 // care must be taken so that BufferBegin is invalidated when a 1342 // block is trimmed 1343 BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); 1344 BufferEnd = BufferBegin+Size; 1345 return CurBufferPtr; 1346} 1347 1348void* JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { 1349 // Delegate this call through the memory manager. 1350 return MemMgr->allocateGlobal(Size, Alignment); 1351} 1352 1353void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { 1354 if (TheJIT->getJITInfo().hasCustomConstantPool()) 1355 return; 1356 1357 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 1358 if (Constants.empty()) return; 1359 1360 unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 1361 unsigned Align = MCP->getConstantPoolAlignment(); 1362 ConstantPoolBase = allocateSpace(Size, Align); 1363 ConstantPool = MCP; 1364 1365 if (ConstantPoolBase == 0) return; // Buffer overflow. 1366 1367 DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase 1368 << "] (size: " << Size << ", alignment: " << Align << ")\n"); 1369 1370 // Initialize the memory for all of the constant pool entries. 1371 unsigned Offset = 0; 1372 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 1373 MachineConstantPoolEntry CPE = Constants[i]; 1374 unsigned AlignMask = CPE.getAlignment() - 1; 1375 Offset = (Offset + AlignMask) & ~AlignMask; 1376 1377 uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; 1378 ConstPoolAddresses.push_back(CAddr); 1379 if (CPE.isMachineConstantPoolEntry()) { 1380 // FIXME: add support to lower machine constant pool values into bytes! 1381 llvm_report_error("Initialize memory with machine specific constant pool" 1382 "entry has not been implemented!"); 1383 } 1384 TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); 1385 DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; 1386 dbgs().write_hex(CAddr) << "]\n"); 1387 1388 const Type *Ty = CPE.Val.ConstVal->getType(); 1389 Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty); 1390 } 1391} 1392 1393void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { 1394 if (TheJIT->getJITInfo().hasCustomJumpTables()) 1395 return; 1396 if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) 1397 return; 1398 1399 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1400 if (JT.empty()) return; 1401 1402 unsigned NumEntries = 0; 1403 for (unsigned i = 0, e = JT.size(); i != e; ++i) 1404 NumEntries += JT[i].MBBs.size(); 1405 1406 unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getTargetData()); 1407 1408 // Just allocate space for all the jump tables now. We will fix up the actual 1409 // MBB entries in the tables after we emit the code for each block, since then 1410 // we will know the final locations of the MBBs in memory. 1411 JumpTable = MJTI; 1412 JumpTableBase = allocateSpace(NumEntries * EntrySize, 1413 MJTI->getEntryAlignment(*TheJIT->getTargetData())); 1414} 1415 1416void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { 1417 if (TheJIT->getJITInfo().hasCustomJumpTables()) 1418 return; 1419 1420 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1421 if (JT.empty() || JumpTableBase == 0) return; 1422 1423 1424 switch (MJTI->getEntryKind()) { 1425 case MachineJumpTableInfo::EK_Inline: 1426 return; 1427 case MachineJumpTableInfo::EK_BlockAddress: { 1428 // EK_BlockAddress - Each entry is a plain address of block, e.g.: 1429 // .word LBB123 1430 assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == sizeof(void*) && 1431 "Cross JIT'ing?"); 1432 1433 // For each jump table, map each target in the jump table to the address of 1434 // an emitted MachineBasicBlock. 1435 intptr_t *SlotPtr = (intptr_t*)JumpTableBase; 1436 1437 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1438 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1439 // Store the address of the basic block for this jump table slot in the 1440 // memory we allocated for the jump table in 'initJumpTableInfo' 1441 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 1442 *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); 1443 } 1444 break; 1445 } 1446 1447 case MachineJumpTableInfo::EK_Custom32: 1448 case MachineJumpTableInfo::EK_GPRel32BlockAddress: 1449 case MachineJumpTableInfo::EK_LabelDifference32: { 1450 assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == 4&&"Cross JIT'ing?"); 1451 // For each jump table, place the offset from the beginning of the table 1452 // to the target address. 1453 int *SlotPtr = (int*)JumpTableBase; 1454 1455 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1456 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1457 // Store the offset of the basic block for this jump table slot in the 1458 // memory we allocated for the jump table in 'initJumpTableInfo' 1459 uintptr_t Base = (uintptr_t)SlotPtr; 1460 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { 1461 uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); 1462 /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. 1463 *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); 1464 } 1465 } 1466 break; 1467 } 1468 } 1469} 1470 1471void JITEmitter::startGVStub(const GlobalValue* GV, 1472 unsigned StubSize, unsigned Alignment) { 1473 SavedBufferBegin = BufferBegin; 1474 SavedBufferEnd = BufferEnd; 1475 SavedCurBufferPtr = CurBufferPtr; 1476 1477 BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); 1478 BufferEnd = BufferBegin+StubSize+1; 1479} 1480 1481void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { 1482 SavedBufferBegin = BufferBegin; 1483 SavedBufferEnd = BufferEnd; 1484 SavedCurBufferPtr = CurBufferPtr; 1485 1486 BufferBegin = CurBufferPtr = (uint8_t *)Buffer; 1487 BufferEnd = BufferBegin+StubSize+1; 1488} 1489 1490void JITEmitter::finishGVStub() { 1491 assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); 1492 NumBytes += getCurrentPCOffset(); 1493 BufferBegin = SavedBufferBegin; 1494 BufferEnd = SavedBufferEnd; 1495 CurBufferPtr = SavedCurBufferPtr; 1496} 1497 1498void *JITEmitter::allocIndirectGV(const GlobalValue *GV, 1499 const uint8_t *Buffer, size_t Size, 1500 unsigned Alignment) { 1501 uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); 1502 memcpy(IndGV, Buffer, Size); 1503 return IndGV; 1504} 1505 1506// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry 1507// in the constant pool that was last emitted with the 'emitConstantPool' 1508// method. 1509// 1510uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { 1511 assert(ConstantNum < ConstantPool->getConstants().size() && 1512 "Invalid ConstantPoolIndex!"); 1513 return ConstPoolAddresses[ConstantNum]; 1514} 1515 1516// getJumpTableEntryAddress - Return the address of the JumpTable with index 1517// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' 1518// 1519uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { 1520 const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); 1521 assert(Index < JT.size() && "Invalid jump table index!"); 1522 1523 unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getTargetData()); 1524 1525 unsigned Offset = 0; 1526 for (unsigned i = 0; i < Index; ++i) 1527 Offset += JT[i].MBBs.size(); 1528 1529 Offset *= EntrySize; 1530 1531 return (uintptr_t)((char *)JumpTableBase + Offset); 1532} 1533 1534void JITEmitter::EmittedFunctionConfig::onDelete( 1535 JITEmitter *Emitter, const Function *F) { 1536 Emitter->deallocateMemForFunction(F); 1537} 1538void JITEmitter::EmittedFunctionConfig::onRAUW( 1539 JITEmitter *, const Function*, const Function*) { 1540 llvm_unreachable("The JIT doesn't know how to handle a" 1541 " RAUW on a value it has emitted."); 1542} 1543 1544 1545//===----------------------------------------------------------------------===// 1546// Public interface to this file 1547//===----------------------------------------------------------------------===// 1548 1549JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, 1550 TargetMachine &tm) { 1551 return new JITEmitter(jit, JMM, tm); 1552} 1553 1554// getPointerToFunctionOrStub - If the specified function has been 1555// code-gen'd, return a pointer to the function. If not, compile it, or use 1556// a stub to implement lazy compilation if available. 1557// 1558void *JIT::getPointerToFunctionOrStub(Function *F) { 1559 // If we have already code generated the function, just return the address. 1560 if (void *Addr = getPointerToGlobalIfAvailable(F)) 1561 return Addr; 1562 1563 // Get a stub if the target supports it. 1564 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1565 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 1566 return JE->getJITResolver().getLazyFunctionStub(F); 1567} 1568 1569void JIT::updateFunctionStub(Function *F) { 1570 // Get the empty stub we generated earlier. 1571 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1572 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 1573 void *Stub = JE->getJITResolver().getLazyFunctionStub(F); 1574 void *Addr = getPointerToGlobalIfAvailable(F); 1575 assert(Addr != Stub && "Function must have non-stub address to be updated."); 1576 1577 // Tell the target jit info to rewrite the stub at the specified address, 1578 // rather than creating a new one. 1579 TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); 1580 JE->startGVStub(Stub, layout.Size); 1581 getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); 1582 JE->finishGVStub(); 1583} 1584 1585/// freeMachineCodeForFunction - release machine code memory for given Function. 1586/// 1587void JIT::freeMachineCodeForFunction(Function *F) { 1588 // Delete translation for this from the ExecutionEngine, so it will get 1589 // retranslated next time it is used. 1590 updateGlobalMapping(F, 0); 1591 1592 // Free the actual memory for the function body and related stuff. 1593 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1594 cast<JITEmitter>(JCE)->deallocateMemForFunction(F); 1595} 1596