JITEmitter.cpp revision 75361b69f3f327842b9dad69fa7f28ae3b688412
1//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a MachineCodeEmitter object that is used by the JIT to 11// write machine code to memory and remember where relocatable values are. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "jit" 16#include "JIT.h" 17#include "JITDebugRegisterer.h" 18#include "JITDwarfEmitter.h" 19#include "llvm/ADT/OwningPtr.h" 20#include "llvm/Constants.h" 21#include "llvm/Module.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/Analysis/DebugInfo.h" 24#include "llvm/CodeGen/JITCodeEmitter.h" 25#include "llvm/CodeGen/MachineFunction.h" 26#include "llvm/CodeGen/MachineConstantPool.h" 27#include "llvm/CodeGen/MachineJumpTableInfo.h" 28#include "llvm/CodeGen/MachineModuleInfo.h" 29#include "llvm/CodeGen/MachineRelocation.h" 30#include "llvm/ExecutionEngine/GenericValue.h" 31#include "llvm/ExecutionEngine/JITEventListener.h" 32#include "llvm/ExecutionEngine/JITMemoryManager.h" 33#include "llvm/CodeGen/MachineCodeInfo.h" 34#include "llvm/Target/TargetData.h" 35#include "llvm/Target/TargetJITInfo.h" 36#include "llvm/Target/TargetMachine.h" 37#include "llvm/Target/TargetOptions.h" 38#include "llvm/Support/Debug.h" 39#include "llvm/Support/ErrorHandling.h" 40#include "llvm/Support/ManagedStatic.h" 41#include "llvm/Support/MutexGuard.h" 42#include "llvm/Support/ValueHandle.h" 43#include "llvm/Support/raw_ostream.h" 44#include "llvm/System/Disassembler.h" 45#include "llvm/System/Memory.h" 46#include "llvm/Target/TargetInstrInfo.h" 47#include "llvm/ADT/DenseMap.h" 48#include "llvm/ADT/SmallPtrSet.h" 49#include "llvm/ADT/SmallVector.h" 50#include "llvm/ADT/Statistic.h" 51#include "llvm/ADT/ValueMap.h" 52#include <algorithm> 53#ifndef NDEBUG 54#include <iomanip> 55#endif 56using namespace llvm; 57 58STATISTIC(NumBytes, "Number of bytes of machine code compiled"); 59STATISTIC(NumRelos, "Number of relocations applied"); 60STATISTIC(NumRetries, "Number of retries with more memory"); 61 62 63// A declaration may stop being a declaration once it's fully read from bitcode. 64// This function returns true if F is fully read and is still a declaration. 65static bool isNonGhostDeclaration(const Function *F) { 66 return F->isDeclaration() && !F->isMaterializable(); 67} 68 69//===----------------------------------------------------------------------===// 70// JIT lazy compilation code. 71// 72namespace { 73 class JITEmitter; 74 class JITResolverState; 75 76 template<typename ValueTy> 77 struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> { 78 typedef JITResolverState *ExtraData; 79 static void onRAUW(JITResolverState *, Value *Old, Value *New) { 80 assert(false && "The JIT doesn't know how to handle a" 81 " RAUW on a value it has emitted."); 82 } 83 }; 84 85 struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> { 86 typedef JITResolverState *ExtraData; 87 static void onDelete(JITResolverState *JRS, Function *F); 88 }; 89 90 class JITResolverState { 91 public: 92 typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> > 93 FunctionToLazyStubMapTy; 94 typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy; 95 typedef ValueMap<Function *, SmallPtrSet<void*, 1>, 96 CallSiteValueMapConfig> FunctionToCallSitesMapTy; 97 typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; 98 private: 99 /// FunctionToLazyStubMap - Keep track of the lazy stub created for a 100 /// particular function so that we can reuse them if necessary. 101 FunctionToLazyStubMapTy FunctionToLazyStubMap; 102 103 /// CallSiteToFunctionMap - Keep track of the function that each lazy call 104 /// site corresponds to, and vice versa. 105 CallSiteToFunctionMapTy CallSiteToFunctionMap; 106 FunctionToCallSitesMapTy FunctionToCallSitesMap; 107 108 /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a 109 /// particular GlobalVariable so that we can reuse them if necessary. 110 GlobalToIndirectSymMapTy GlobalToIndirectSymMap; 111 112 /// Instance of the JIT this ResolverState serves. 113 JIT *TheJIT; 114 115 public: 116 JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), 117 FunctionToCallSitesMap(this), 118 TheJIT(jit) {} 119 120 FunctionToLazyStubMapTy& getFunctionToLazyStubMap( 121 const MutexGuard& locked) { 122 assert(locked.holds(TheJIT->lock)); 123 return FunctionToLazyStubMap; 124 } 125 126 GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& locked) { 127 assert(locked.holds(TheJIT->lock)); 128 return GlobalToIndirectSymMap; 129 } 130 131 pair<void *, Function *> LookupFunctionFromCallSite( 132 const MutexGuard &locked, void *CallSite) const { 133 assert(locked.holds(TheJIT->lock)); 134 135 // The address given to us for the stub may not be exactly right, it might be 136 // a little bit after the stub. As such, use upper_bound to find it. 137 CallSiteToFunctionMapTy::const_iterator I = 138 CallSiteToFunctionMap.upper_bound(CallSite); 139 assert(I != CallSiteToFunctionMap.begin() && 140 "This is not a known call site!"); 141 --I; 142 return *I; 143 } 144 145 void AddCallSite(const MutexGuard &locked, void *CallSite, Function *F) { 146 assert(locked.holds(TheJIT->lock)); 147 148 bool Inserted = CallSiteToFunctionMap.insert( 149 std::make_pair(CallSite, F)).second; 150 (void)Inserted; 151 assert(Inserted && "Pair was already in CallSiteToFunctionMap"); 152 FunctionToCallSitesMap[F].insert(CallSite); 153 } 154 155 // Returns the Function of the stub if a stub was erased, or NULL if there 156 // was no stub. This function uses the call-site->function map to find a 157 // relevant function, but asserts that only stubs and not other call sites 158 // will be passed in. 159 Function *EraseStub(const MutexGuard &locked, void *Stub); 160 161 void EraseAllCallSitesFor(const MutexGuard &locked, Function *F) { 162 assert(locked.holds(TheJIT->lock)); 163 EraseAllCallSitesForPrelocked(F); 164 } 165 void EraseAllCallSitesForPrelocked(Function *F); 166 167 // Erases _all_ call sites regardless of their function. This is used to 168 // unregister the stub addresses from the StubToResolverMap in 169 // ~JITResolver(). 170 void EraseAllCallSitesPrelocked(); 171 }; 172 173 /// JITResolver - Keep track of, and resolve, call sites for functions that 174 /// have not yet been compiled. 175 class JITResolver { 176 typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; 177 typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; 178 typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; 179 180 /// LazyResolverFn - The target lazy resolver function that we actually 181 /// rewrite instructions to use. 182 TargetJITInfo::LazyResolverFn LazyResolverFn; 183 184 JITResolverState state; 185 186 /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap 187 /// for external functions. TODO: Of course, external functions don't need 188 /// a lazy stub. It's actually here to make it more likely that far calls 189 /// succeed, but no single stub can guarantee that. I'll remove this in a 190 /// subsequent checkin when I actually fix far calls. 191 std::map<void*, void*> ExternalFnToStubMap; 192 193 /// revGOTMap - map addresses to indexes in the GOT 194 std::map<void*, unsigned> revGOTMap; 195 unsigned nextGOTIndex; 196 197 JITEmitter &JE; 198 199 /// Instance of JIT corresponding to this Resolver. 200 JIT *TheJIT; 201 202 public: 203 explicit JITResolver(JIT &jit, JITEmitter &je) 204 : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { 205 LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); 206 } 207 208 ~JITResolver(); 209 210 /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's 211 /// lazy-compilation stub if it has already been created. 212 void *getLazyFunctionStubIfAvailable(Function *F); 213 214 /// getLazyFunctionStub - This returns a pointer to a function's 215 /// lazy-compilation stub, creating one on demand as needed. 216 void *getLazyFunctionStub(Function *F); 217 218 /// getExternalFunctionStub - Return a stub for the function at the 219 /// specified address, created lazily on demand. 220 void *getExternalFunctionStub(void *FnAddr); 221 222 /// getGlobalValueIndirectSym - Return an indirect symbol containing the 223 /// specified GV address. 224 void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); 225 226 void getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs, 227 SmallVectorImpl<void*> &Ptrs); 228 229 /// getGOTIndexForAddress - Return a new or existing index in the GOT for 230 /// an address. This function only manages slots, it does not manage the 231 /// contents of the slots or the memory associated with the GOT. 232 unsigned getGOTIndexForAddr(void *addr); 233 234 /// JITCompilerFn - This function is called to resolve a stub to a compiled 235 /// address. If the LLVM Function corresponding to the stub has not yet 236 /// been compiled, this function compiles it first. 237 static void *JITCompilerFn(void *Stub); 238 }; 239 240 class StubToResolverMapTy { 241 /// Map a stub address to a specific instance of a JITResolver so that 242 /// lazily-compiled functions can find the right resolver to use. 243 /// 244 /// Guarded by Lock. 245 std::map<void*, JITResolver*> Map; 246 247 /// Guards Map from concurrent accesses. 248 mutable sys::Mutex Lock; 249 250 public: 251 /// Registers a Stub to be resolved by Resolver. 252 void RegisterStubResolver(void *Stub, JITResolver *Resolver) { 253 MutexGuard guard(Lock); 254 Map.insert(std::make_pair(Stub, Resolver)); 255 } 256 /// Unregisters the Stub when it's invalidated. 257 void UnregisterStubResolver(void *Stub) { 258 MutexGuard guard(Lock); 259 Map.erase(Stub); 260 } 261 /// Returns the JITResolver instance that owns the Stub. 262 JITResolver *getResolverFromStub(void *Stub) const { 263 MutexGuard guard(Lock); 264 // The address given to us for the stub may not be exactly right, it might 265 // be a little bit after the stub. As such, use upper_bound to find it. 266 // This is the same trick as in LookupFunctionFromCallSite from 267 // JITResolverState. 268 std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub); 269 assert(I != Map.begin() && "This is not a known stub!"); 270 --I; 271 return I->second; 272 } 273 /// True if any stubs refer to the given resolver. Only used in an assert(). 274 /// O(N) 275 bool ResolverHasStubs(JITResolver* Resolver) const { 276 MutexGuard guard(Lock); 277 for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(), 278 E = Map.end(); I != E; ++I) { 279 if (I->second == Resolver) 280 return true; 281 } 282 return false; 283 } 284 }; 285 /// This needs to be static so that a lazy call stub can access it with no 286 /// context except the address of the stub. 287 ManagedStatic<StubToResolverMapTy> StubToResolverMap; 288 289 /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is 290 /// used to output functions to memory for execution. 291 class JITEmitter : public JITCodeEmitter { 292 JITMemoryManager *MemMgr; 293 294 // When outputting a function stub in the context of some other function, we 295 // save BufferBegin/BufferEnd/CurBufferPtr here. 296 uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 297 298 // When reattempting to JIT a function after running out of space, we store 299 // the estimated size of the function we're trying to JIT here, so we can 300 // ask the memory manager for at least this much space. When we 301 // successfully emit the function, we reset this back to zero. 302 uintptr_t SizeEstimate; 303 304 /// Relocations - These are the relocations that the function needs, as 305 /// emitted. 306 std::vector<MachineRelocation> Relocations; 307 308 /// MBBLocations - This vector is a mapping from MBB ID's to their address. 309 /// It is filled in by the StartMachineBasicBlock callback and queried by 310 /// the getMachineBasicBlockAddress callback. 311 std::vector<uintptr_t> MBBLocations; 312 313 /// ConstantPool - The constant pool for the current function. 314 /// 315 MachineConstantPool *ConstantPool; 316 317 /// ConstantPoolBase - A pointer to the first entry in the constant pool. 318 /// 319 void *ConstantPoolBase; 320 321 /// ConstPoolAddresses - Addresses of individual constant pool entries. 322 /// 323 SmallVector<uintptr_t, 8> ConstPoolAddresses; 324 325 /// JumpTable - The jump tables for the current function. 326 /// 327 MachineJumpTableInfo *JumpTable; 328 329 /// JumpTableBase - A pointer to the first entry in the jump table. 330 /// 331 void *JumpTableBase; 332 333 /// Resolver - This contains info about the currently resolved functions. 334 JITResolver Resolver; 335 336 /// DE - The dwarf emitter for the jit. 337 OwningPtr<JITDwarfEmitter> DE; 338 339 /// DR - The debug registerer for the jit. 340 OwningPtr<JITDebugRegisterer> DR; 341 342 /// LabelLocations - This vector is a mapping from Label ID's to their 343 /// address. 344 DenseMap<MCSymbol*, uintptr_t> LabelLocations; 345 346 /// MMI - Machine module info for exception informations 347 MachineModuleInfo* MMI; 348 349 // CurFn - The llvm function being emitted. Only valid during 350 // finishFunction(). 351 const Function *CurFn; 352 353 /// Information about emitted code, which is passed to the 354 /// JITEventListeners. This is reset in startFunction and used in 355 /// finishFunction. 356 JITEvent_EmittedFunctionDetails EmissionDetails; 357 358 struct EmittedCode { 359 void *FunctionBody; // Beginning of the function's allocation. 360 void *Code; // The address the function's code actually starts at. 361 void *ExceptionTable; 362 EmittedCode() : FunctionBody(0), Code(0), ExceptionTable(0) {} 363 }; 364 struct EmittedFunctionConfig : public ValueMapConfig<const Function*> { 365 typedef JITEmitter *ExtraData; 366 static void onDelete(JITEmitter *, const Function*); 367 static void onRAUW(JITEmitter *, const Function*, const Function*); 368 }; 369 ValueMap<const Function *, EmittedCode, 370 EmittedFunctionConfig> EmittedFunctions; 371 372 DILocation PrevDLT; 373 374 /// Instance of the JIT 375 JIT *TheJIT; 376 377 public: 378 JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) 379 : SizeEstimate(0), Resolver(jit, *this), MMI(0), CurFn(0), 380 EmittedFunctions(this), PrevDLT(NULL), TheJIT(&jit) { 381 MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); 382 if (jit.getJITInfo().needsGOT()) { 383 MemMgr->AllocateGOT(); 384 DEBUG(dbgs() << "JIT is managing a GOT\n"); 385 } 386 387 if (DwarfExceptionHandling || JITEmitDebugInfo) { 388 DE.reset(new JITDwarfEmitter(jit)); 389 } 390 if (JITEmitDebugInfo) { 391 DR.reset(new JITDebugRegisterer(TM)); 392 } 393 } 394 ~JITEmitter() { 395 delete MemMgr; 396 } 397 398 /// classof - Methods for support type inquiry through isa, cast, and 399 /// dyn_cast: 400 /// 401 static inline bool classof(const JITEmitter*) { return true; } 402 static inline bool classof(const MachineCodeEmitter*) { return true; } 403 404 JITResolver &getJITResolver() { return Resolver; } 405 406 virtual void startFunction(MachineFunction &F); 407 virtual bool finishFunction(MachineFunction &F); 408 409 void emitConstantPool(MachineConstantPool *MCP); 410 void initJumpTableInfo(MachineJumpTableInfo *MJTI); 411 void emitJumpTableInfo(MachineJumpTableInfo *MJTI); 412 413 void startGVStub(const GlobalValue* GV, 414 unsigned StubSize, unsigned Alignment = 1); 415 void startGVStub(void *Buffer, unsigned StubSize); 416 void finishGVStub(); 417 virtual void *allocIndirectGV(const GlobalValue *GV, 418 const uint8_t *Buffer, size_t Size, 419 unsigned Alignment); 420 421 /// allocateSpace - Reserves space in the current block if any, or 422 /// allocate a new one of the given size. 423 virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); 424 425 /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, 426 /// this method does not allocate memory in the current output buffer, 427 /// because a global may live longer than the current function. 428 virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment); 429 430 virtual void addRelocation(const MachineRelocation &MR) { 431 Relocations.push_back(MR); 432 } 433 434 virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { 435 if (MBBLocations.size() <= (unsigned)MBB->getNumber()) 436 MBBLocations.resize((MBB->getNumber()+1)*2); 437 MBBLocations[MBB->getNumber()] = getCurrentPCValue(); 438 DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" 439 << (void*) getCurrentPCValue() << "]\n"); 440 } 441 442 virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; 443 virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; 444 445 virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const { 446 assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 447 MBBLocations[MBB->getNumber()] && "MBB not emitted!"); 448 return MBBLocations[MBB->getNumber()]; 449 } 450 451 /// retryWithMoreMemory - Log a retry and deallocate all memory for the 452 /// given function. Increase the minimum allocation size so that we get 453 /// more memory next time. 454 void retryWithMoreMemory(MachineFunction &F); 455 456 /// deallocateMemForFunction - Deallocate all memory for the specified 457 /// function body. 458 void deallocateMemForFunction(const Function *F); 459 460 virtual void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn); 461 462 virtual void emitLabel(MCSymbol *Label) { 463 LabelLocations[Label] = getCurrentPCValue(); 464 } 465 466 virtual uintptr_t getLabelAddress(MCSymbol *Label) const { 467 assert(LabelLocations.count(Label) && "Label not emitted!"); 468 return LabelLocations.find(Label)->second; 469 } 470 471 virtual void setModuleInfo(MachineModuleInfo* Info) { 472 MMI = Info; 473 if (DE.get()) DE->setModuleInfo(Info); 474 } 475 476 void setMemoryExecutable() { 477 MemMgr->setMemoryExecutable(); 478 } 479 480 JITMemoryManager *getMemMgr() const { return MemMgr; } 481 482 private: 483 void *getPointerToGlobal(GlobalValue *GV, void *Reference, 484 bool MayNeedFarStub); 485 void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); 486 unsigned addSizeOfGlobal(const GlobalVariable *GV, unsigned Size); 487 unsigned addSizeOfGlobalsInConstantVal( 488 const Constant *C, unsigned Size, 489 SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 490 SmallVectorImpl<const GlobalVariable*> &Worklist); 491 unsigned addSizeOfGlobalsInInitializer( 492 const Constant *Init, unsigned Size, 493 SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 494 SmallVectorImpl<const GlobalVariable*> &Worklist); 495 unsigned GetSizeOfGlobalsInBytes(MachineFunction &MF); 496 }; 497} 498 499void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { 500 JRS->EraseAllCallSitesForPrelocked(F); 501} 502 503Function *JITResolverState::EraseStub(const MutexGuard &locked, void *Stub) { 504 CallSiteToFunctionMapTy::iterator C2F_I = 505 CallSiteToFunctionMap.find(Stub); 506 if (C2F_I == CallSiteToFunctionMap.end()) { 507 // Not a stub. 508 return NULL; 509 } 510 511 StubToResolverMap->UnregisterStubResolver(Stub); 512 513 Function *const F = C2F_I->second; 514#ifndef NDEBUG 515 void *RealStub = FunctionToLazyStubMap.lookup(F); 516 assert(RealStub == Stub && 517 "Call-site that wasn't a stub passed in to EraseStub"); 518#endif 519 FunctionToLazyStubMap.erase(F); 520 CallSiteToFunctionMap.erase(C2F_I); 521 522 // Remove the stub from the function->call-sites map, and remove the whole 523 // entry from the map if that was the last call site. 524 FunctionToCallSitesMapTy::iterator F2C_I = FunctionToCallSitesMap.find(F); 525 assert(F2C_I != FunctionToCallSitesMap.end() && 526 "FunctionToCallSitesMap broken"); 527 bool Erased = F2C_I->second.erase(Stub); 528 (void)Erased; 529 assert(Erased && "FunctionToCallSitesMap broken"); 530 if (F2C_I->second.empty()) 531 FunctionToCallSitesMap.erase(F2C_I); 532 533 return F; 534} 535 536void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { 537 FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); 538 if (F2C == FunctionToCallSitesMap.end()) 539 return; 540 StubToResolverMapTy &S2RMap = *StubToResolverMap; 541 for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(), 542 E = F2C->second.end(); I != E; ++I) { 543 S2RMap.UnregisterStubResolver(*I); 544 bool Erased = CallSiteToFunctionMap.erase(*I); 545 (void)Erased; 546 assert(Erased && "Missing call site->function mapping"); 547 } 548 FunctionToCallSitesMap.erase(F2C); 549} 550 551void JITResolverState::EraseAllCallSitesPrelocked() { 552 StubToResolverMapTy &S2RMap = *StubToResolverMap; 553 for (CallSiteToFunctionMapTy::const_iterator 554 I = CallSiteToFunctionMap.begin(), 555 E = CallSiteToFunctionMap.end(); I != E; ++I) { 556 S2RMap.UnregisterStubResolver(I->first); 557 } 558 CallSiteToFunctionMap.clear(); 559 FunctionToCallSitesMap.clear(); 560} 561 562JITResolver::~JITResolver() { 563 // No need to lock because we're in the destructor, and state isn't shared. 564 state.EraseAllCallSitesPrelocked(); 565 assert(!StubToResolverMap->ResolverHasStubs(this) && 566 "Resolver destroyed with stubs still alive."); 567} 568 569/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub 570/// if it has already been created. 571void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { 572 MutexGuard locked(TheJIT->lock); 573 574 // If we already have a stub for this function, recycle it. 575 return state.getFunctionToLazyStubMap(locked).lookup(F); 576} 577 578/// getFunctionStub - This returns a pointer to a function stub, creating 579/// one on demand as needed. 580void *JITResolver::getLazyFunctionStub(Function *F) { 581 MutexGuard locked(TheJIT->lock); 582 583 // If we already have a lazy stub for this function, recycle it. 584 void *&Stub = state.getFunctionToLazyStubMap(locked)[F]; 585 if (Stub) return Stub; 586 587 // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we 588 // must resolve the symbol now. 589 void *Actual = TheJIT->isCompilingLazily() 590 ? (void *)(intptr_t)LazyResolverFn : (void *)0; 591 592 // If this is an external declaration, attempt to resolve the address now 593 // to place in the stub. 594 if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { 595 Actual = TheJIT->getPointerToFunction(F); 596 597 // If we resolved the symbol to a null address (eg. a weak external) 598 // don't emit a stub. Return a null pointer to the application. 599 if (!Actual) return 0; 600 } 601 602 TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 603 JE.startGVStub(F, SL.Size, SL.Alignment); 604 // Codegen a new stub, calling the lazy resolver or the actual address of the 605 // external function, if it was resolved. 606 Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); 607 JE.finishGVStub(); 608 609 if (Actual != (void*)(intptr_t)LazyResolverFn) { 610 // If we are getting the stub for an external function, we really want the 611 // address of the stub in the GlobalAddressMap for the JIT, not the address 612 // of the external function. 613 TheJIT->updateGlobalMapping(F, Stub); 614 } 615 616 DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" 617 << F->getName() << "'\n"); 618 619 if (TheJIT->isCompilingLazily()) { 620 // Register this JITResolver as the one corresponding to this call site so 621 // JITCompilerFn will be able to find it. 622 StubToResolverMap->RegisterStubResolver(Stub, this); 623 624 // Finally, keep track of the stub-to-Function mapping so that the 625 // JITCompilerFn knows which function to compile! 626 state.AddCallSite(locked, Stub, F); 627 } else if (!Actual) { 628 // If we are JIT'ing non-lazily but need to call a function that does not 629 // exist yet, add it to the JIT's work list so that we can fill in the 630 // stub address later. 631 assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && 632 "'Actual' should have been set above."); 633 TheJIT->addPendingFunction(F); 634 } 635 636 return Stub; 637} 638 639/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified 640/// GV address. 641void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { 642 MutexGuard locked(TheJIT->lock); 643 644 // If we already have a stub for this global variable, recycle it. 645 void *&IndirectSym = state.getGlobalToIndirectSymMap(locked)[GV]; 646 if (IndirectSym) return IndirectSym; 647 648 // Otherwise, codegen a new indirect symbol. 649 IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, 650 JE); 651 652 DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym 653 << "] for GV '" << GV->getName() << "'\n"); 654 655 return IndirectSym; 656} 657 658/// getExternalFunctionStub - Return a stub for the function at the 659/// specified address, created lazily on demand. 660void *JITResolver::getExternalFunctionStub(void *FnAddr) { 661 // If we already have a stub for this function, recycle it. 662 void *&Stub = ExternalFnToStubMap[FnAddr]; 663 if (Stub) return Stub; 664 665 TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 666 JE.startGVStub(0, SL.Size, SL.Alignment); 667 Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, JE); 668 JE.finishGVStub(); 669 670 DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub 671 << "] for external function at '" << FnAddr << "'\n"); 672 return Stub; 673} 674 675unsigned JITResolver::getGOTIndexForAddr(void* addr) { 676 unsigned idx = revGOTMap[addr]; 677 if (!idx) { 678 idx = ++nextGOTIndex; 679 revGOTMap[addr] = idx; 680 DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" 681 << addr << "]\n"); 682 } 683 return idx; 684} 685 686void JITResolver::getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs, 687 SmallVectorImpl<void*> &Ptrs) { 688 MutexGuard locked(TheJIT->lock); 689 690 const FunctionToLazyStubMapTy &FM = state.getFunctionToLazyStubMap(locked); 691 GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); 692 693 for (FunctionToLazyStubMapTy::const_iterator i = FM.begin(), e = FM.end(); 694 i != e; ++i){ 695 Function *F = i->first; 696 if (F->isDeclaration() && F->hasExternalLinkage()) { 697 GVs.push_back(i->first); 698 Ptrs.push_back(i->second); 699 } 700 } 701 for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); 702 i != e; ++i) { 703 GVs.push_back(i->first); 704 Ptrs.push_back(i->second); 705 } 706} 707 708/// JITCompilerFn - This function is called when a lazy compilation stub has 709/// been entered. It looks up which function this stub corresponds to, compiles 710/// it if necessary, then returns the resultant function pointer. 711void *JITResolver::JITCompilerFn(void *Stub) { 712 JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); 713 assert(JR && "Unable to find the corresponding JITResolver to the call site"); 714 715 Function* F = 0; 716 void* ActualPtr = 0; 717 718 { 719 // Only lock for getting the Function. The call getPointerToFunction made 720 // in this function might trigger function materializing, which requires 721 // JIT lock to be unlocked. 722 MutexGuard locked(JR->TheJIT->lock); 723 724 // The address given to us for the stub may not be exactly right, it might 725 // be a little bit after the stub. As such, use upper_bound to find it. 726 pair<void*, Function*> I = 727 JR->state.LookupFunctionFromCallSite(locked, Stub); 728 F = I.second; 729 ActualPtr = I.first; 730 } 731 732 // If we have already code generated the function, just return the address. 733 void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); 734 735 if (!Result) { 736 // Otherwise we don't have it, do lazy compilation now. 737 738 // If lazy compilation is disabled, emit a useful error message and abort. 739 if (!JR->TheJIT->isCompilingLazily()) { 740 report_fatal_error("LLVM JIT requested to do lazy compilation of function '" 741 + F->getName() + "' when lazy compiles are disabled!"); 742 } 743 744 DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() 745 << "' In stub ptr = " << Stub << " actual ptr = " 746 << ActualPtr << "\n"); 747 748 Result = JR->TheJIT->getPointerToFunction(F); 749 } 750 751 // Reacquire the lock to update the GOT map. 752 MutexGuard locked(JR->TheJIT->lock); 753 754 // We might like to remove the call site from the CallSiteToFunction map, but 755 // we can't do that! Multiple threads could be stuck, waiting to acquire the 756 // lock above. As soon as the 1st function finishes compiling the function, 757 // the next one will be released, and needs to be able to find the function it 758 // needs to call. 759 760 // FIXME: We could rewrite all references to this stub if we knew them. 761 762 // What we will do is set the compiled function address to map to the 763 // same GOT entry as the stub so that later clients may update the GOT 764 // if they see it still using the stub address. 765 // Note: this is done so the Resolver doesn't have to manage GOT memory 766 // Do this without allocating map space if the target isn't using a GOT 767 if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) 768 JR->revGOTMap[Result] = JR->revGOTMap[Stub]; 769 770 return Result; 771} 772 773//===----------------------------------------------------------------------===// 774// JITEmitter code. 775// 776void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, 777 bool MayNeedFarStub) { 778 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) 779 return TheJIT->getOrEmitGlobalVariable(GV); 780 781 if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 782 return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false)); 783 784 // If we have already compiled the function, return a pointer to its body. 785 Function *F = cast<Function>(V); 786 787 void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); 788 if (FnStub) { 789 // Return the function stub if it's already created. We do this first so 790 // that we're returning the same address for the function as any previous 791 // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be 792 // close enough to call. 793 return FnStub; 794 } 795 796 // If we know the target can handle arbitrary-distance calls, try to 797 // return a direct pointer. 798 if (!MayNeedFarStub) { 799 // If we have code, go ahead and return that. 800 void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); 801 if (ResultPtr) return ResultPtr; 802 803 // If this is an external function pointer, we can force the JIT to 804 // 'compile' it, which really just adds it to the map. 805 if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) 806 return TheJIT->getPointerToFunction(F); 807 } 808 809 // Otherwise, we may need a to emit a stub, and, conservatively, we always do 810 // so. Note that it's possible to return null from getLazyFunctionStub in the 811 // case of a weak extern that fails to resolve. 812 return Resolver.getLazyFunctionStub(F); 813} 814 815void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { 816 // Make sure GV is emitted first, and create a stub containing the fully 817 // resolved address. 818 void *GVAddress = getPointerToGlobal(V, Reference, false); 819 void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); 820 return StubAddr; 821} 822 823void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { 824 if (DL.isUnknown()) return; 825 if (!BeforePrintingInsn) return; 826 827 // FIXME: This is horribly inefficient. 828 DILocation CurDLT(DL.getAsMDNode( 829 EmissionDetails.MF->getFunction()->getContext())); 830 831 if (CurDLT.getScope().getNode() != 0 && PrevDLT.getNode() !=CurDLT.getNode()){ 832 JITEvent_EmittedFunctionDetails::LineStart NextLine; 833 NextLine.Address = getCurrentPCValue(); 834 NextLine.Loc = DL; 835 EmissionDetails.LineStarts.push_back(NextLine); 836 } 837 838 PrevDLT = CurDLT; 839} 840 841static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, 842 const TargetData *TD) { 843 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 844 if (Constants.empty()) return 0; 845 846 unsigned Size = 0; 847 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 848 MachineConstantPoolEntry CPE = Constants[i]; 849 unsigned AlignMask = CPE.getAlignment() - 1; 850 Size = (Size + AlignMask) & ~AlignMask; 851 const Type *Ty = CPE.getType(); 852 Size += TD->getTypeAllocSize(Ty); 853 } 854 return Size; 855} 856 857static unsigned GetJumpTableSizeInBytes(MachineJumpTableInfo *MJTI, JIT *jit) { 858 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 859 if (JT.empty()) return 0; 860 861 unsigned NumEntries = 0; 862 for (unsigned i = 0, e = JT.size(); i != e; ++i) 863 NumEntries += JT[i].MBBs.size(); 864 865 return NumEntries * MJTI->getEntrySize(*jit->getTargetData()); 866} 867 868static uintptr_t RoundUpToAlign(uintptr_t Size, unsigned Alignment) { 869 if (Alignment == 0) Alignment = 1; 870 // Since we do not know where the buffer will be allocated, be pessimistic. 871 return Size + Alignment; 872} 873 874/// addSizeOfGlobal - add the size of the global (plus any alignment padding) 875/// into the running total Size. 876 877unsigned JITEmitter::addSizeOfGlobal(const GlobalVariable *GV, unsigned Size) { 878 const Type *ElTy = GV->getType()->getElementType(); 879 size_t GVSize = (size_t)TheJIT->getTargetData()->getTypeAllocSize(ElTy); 880 size_t GVAlign = 881 (size_t)TheJIT->getTargetData()->getPreferredAlignment(GV); 882 DEBUG(dbgs() << "JIT: Adding in size " << GVSize << " alignment " << GVAlign); 883 DEBUG(GV->dump()); 884 // Assume code section ends with worst possible alignment, so first 885 // variable needs maximal padding. 886 if (Size==0) 887 Size = 1; 888 Size = ((Size+GVAlign-1)/GVAlign)*GVAlign; 889 Size += GVSize; 890 return Size; 891} 892 893/// addSizeOfGlobalsInConstantVal - find any globals that we haven't seen yet 894/// but are referenced from the constant; put them in SeenGlobals and the 895/// Worklist, and add their size into the running total Size. 896 897unsigned JITEmitter::addSizeOfGlobalsInConstantVal( 898 const Constant *C, 899 unsigned Size, 900 SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 901 SmallVectorImpl<const GlobalVariable*> &Worklist) { 902 // If its undefined, return the garbage. 903 if (isa<UndefValue>(C)) 904 return Size; 905 906 // If the value is a ConstantExpr 907 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 908 Constant *Op0 = CE->getOperand(0); 909 switch (CE->getOpcode()) { 910 case Instruction::GetElementPtr: 911 case Instruction::Trunc: 912 case Instruction::ZExt: 913 case Instruction::SExt: 914 case Instruction::FPTrunc: 915 case Instruction::FPExt: 916 case Instruction::UIToFP: 917 case Instruction::SIToFP: 918 case Instruction::FPToUI: 919 case Instruction::FPToSI: 920 case Instruction::PtrToInt: 921 case Instruction::IntToPtr: 922 case Instruction::BitCast: { 923 Size = addSizeOfGlobalsInConstantVal(Op0, Size, SeenGlobals, Worklist); 924 break; 925 } 926 case Instruction::Add: 927 case Instruction::FAdd: 928 case Instruction::Sub: 929 case Instruction::FSub: 930 case Instruction::Mul: 931 case Instruction::FMul: 932 case Instruction::UDiv: 933 case Instruction::SDiv: 934 case Instruction::URem: 935 case Instruction::SRem: 936 case Instruction::And: 937 case Instruction::Or: 938 case Instruction::Xor: { 939 Size = addSizeOfGlobalsInConstantVal(Op0, Size, SeenGlobals, Worklist); 940 Size = addSizeOfGlobalsInConstantVal(CE->getOperand(1), Size, 941 SeenGlobals, Worklist); 942 break; 943 } 944 default: { 945 std::string msg; 946 raw_string_ostream Msg(msg); 947 Msg << "ConstantExpr not handled: " << *CE; 948 report_fatal_error(Msg.str()); 949 } 950 } 951 } 952 953 if (C->getType()->getTypeID() == Type::PointerTyID) 954 if (const GlobalVariable* GV = dyn_cast<GlobalVariable>(C)) 955 if (SeenGlobals.insert(GV)) { 956 Worklist.push_back(GV); 957 Size = addSizeOfGlobal(GV, Size); 958 } 959 960 return Size; 961} 962 963/// addSizeOfGLobalsInInitializer - handle any globals that we haven't seen yet 964/// but are referenced from the given initializer. 965 966unsigned JITEmitter::addSizeOfGlobalsInInitializer( 967 const Constant *Init, 968 unsigned Size, 969 SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 970 SmallVectorImpl<const GlobalVariable*> &Worklist) { 971 if (!isa<UndefValue>(Init) && 972 !isa<ConstantVector>(Init) && 973 !isa<ConstantAggregateZero>(Init) && 974 !isa<ConstantArray>(Init) && 975 !isa<ConstantStruct>(Init) && 976 Init->getType()->isFirstClassType()) 977 Size = addSizeOfGlobalsInConstantVal(Init, Size, SeenGlobals, Worklist); 978 return Size; 979} 980 981/// GetSizeOfGlobalsInBytes - walk the code for the function, looking for 982/// globals; then walk the initializers of those globals looking for more. 983/// If their size has not been considered yet, add it into the running total 984/// Size. 985 986unsigned JITEmitter::GetSizeOfGlobalsInBytes(MachineFunction &MF) { 987 unsigned Size = 0; 988 SmallPtrSet<const GlobalVariable*, 8> SeenGlobals; 989 990 for (MachineFunction::iterator MBB = MF.begin(), E = MF.end(); 991 MBB != E; ++MBB) { 992 for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end(); 993 I != E; ++I) { 994 const TargetInstrDesc &Desc = I->getDesc(); 995 const MachineInstr &MI = *I; 996 unsigned NumOps = Desc.getNumOperands(); 997 for (unsigned CurOp = 0; CurOp < NumOps; CurOp++) { 998 const MachineOperand &MO = MI.getOperand(CurOp); 999 if (MO.isGlobal()) { 1000 GlobalValue* V = MO.getGlobal(); 1001 const GlobalVariable *GV = dyn_cast<const GlobalVariable>(V); 1002 if (!GV) 1003 continue; 1004 // If seen in previous function, it will have an entry here. 1005 if (TheJIT->getPointerToGlobalIfAvailable(GV)) 1006 continue; 1007 // If seen earlier in this function, it will have an entry here. 1008 // FIXME: it should be possible to combine these tables, by 1009 // assuming the addresses of the new globals in this module 1010 // start at 0 (or something) and adjusting them after codegen 1011 // complete. Another possibility is to grab a marker bit in GV. 1012 if (SeenGlobals.insert(GV)) 1013 // A variable as yet unseen. Add in its size. 1014 Size = addSizeOfGlobal(GV, Size); 1015 } 1016 } 1017 } 1018 } 1019 DEBUG(dbgs() << "JIT: About to look through initializers\n"); 1020 // Look for more globals that are referenced only from initializers. 1021 SmallVector<const GlobalVariable*, 8> Worklist( 1022 SeenGlobals.begin(), SeenGlobals.end()); 1023 while (!Worklist.empty()) { 1024 const GlobalVariable* GV = Worklist.back(); 1025 Worklist.pop_back(); 1026 if (GV->hasInitializer()) 1027 Size = addSizeOfGlobalsInInitializer(GV->getInitializer(), Size, 1028 SeenGlobals, Worklist); 1029 } 1030 1031 return Size; 1032} 1033 1034void JITEmitter::startFunction(MachineFunction &F) { 1035 DEBUG(dbgs() << "JIT: Starting CodeGen of Function " 1036 << F.getFunction()->getName() << "\n"); 1037 1038 uintptr_t ActualSize = 0; 1039 // Set the memory writable, if it's not already 1040 MemMgr->setMemoryWritable(); 1041 if (MemMgr->NeedsExactSize()) { 1042 DEBUG(dbgs() << "JIT: ExactSize\n"); 1043 const TargetInstrInfo* TII = F.getTarget().getInstrInfo(); 1044 MachineConstantPool *MCP = F.getConstantPool(); 1045 1046 // Ensure the constant pool/jump table info is at least 4-byte aligned. 1047 ActualSize = RoundUpToAlign(ActualSize, 16); 1048 1049 // Add the alignment of the constant pool 1050 ActualSize = RoundUpToAlign(ActualSize, MCP->getConstantPoolAlignment()); 1051 1052 // Add the constant pool size 1053 ActualSize += GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 1054 1055 if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) { 1056 // Add the aligment of the jump table info 1057 ActualSize = RoundUpToAlign(ActualSize, 1058 MJTI->getEntryAlignment(*TheJIT->getTargetData())); 1059 1060 // Add the jump table size 1061 ActualSize += GetJumpTableSizeInBytes(MJTI, TheJIT); 1062 } 1063 1064 // Add the alignment for the function 1065 ActualSize = RoundUpToAlign(ActualSize, 1066 std::max(F.getFunction()->getAlignment(), 8U)); 1067 1068 // Add the function size 1069 ActualSize += TII->GetFunctionSizeInBytes(F); 1070 1071 DEBUG(dbgs() << "JIT: ActualSize before globals " << ActualSize << "\n"); 1072 // Add the size of the globals that will be allocated after this function. 1073 // These are all the ones referenced from this function that were not 1074 // previously allocated. 1075 ActualSize += GetSizeOfGlobalsInBytes(F); 1076 DEBUG(dbgs() << "JIT: ActualSize after globals " << ActualSize << "\n"); 1077 } else if (SizeEstimate > 0) { 1078 // SizeEstimate will be non-zero on reallocation attempts. 1079 ActualSize = SizeEstimate; 1080 } 1081 1082 BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), 1083 ActualSize); 1084 BufferEnd = BufferBegin+ActualSize; 1085 EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; 1086 1087 // Ensure the constant pool/jump table info is at least 4-byte aligned. 1088 emitAlignment(16); 1089 1090 emitConstantPool(F.getConstantPool()); 1091 if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 1092 initJumpTableInfo(MJTI); 1093 1094 // About to start emitting the machine code for the function. 1095 emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); 1096 TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); 1097 EmittedFunctions[F.getFunction()].Code = CurBufferPtr; 1098 1099 MBBLocations.clear(); 1100 1101 EmissionDetails.MF = &F; 1102 EmissionDetails.LineStarts.clear(); 1103} 1104 1105bool JITEmitter::finishFunction(MachineFunction &F) { 1106 if (CurBufferPtr == BufferEnd) { 1107 // We must call endFunctionBody before retrying, because 1108 // deallocateMemForFunction requires it. 1109 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 1110 retryWithMoreMemory(F); 1111 return true; 1112 } 1113 1114 if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 1115 emitJumpTableInfo(MJTI); 1116 1117 // FnStart is the start of the text, not the start of the constant pool and 1118 // other per-function data. 1119 uint8_t *FnStart = 1120 (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); 1121 1122 // FnEnd is the end of the function's machine code. 1123 uint8_t *FnEnd = CurBufferPtr; 1124 1125 if (!Relocations.empty()) { 1126 CurFn = F.getFunction(); 1127 NumRelos += Relocations.size(); 1128 1129 // Resolve the relocations to concrete pointers. 1130 for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { 1131 MachineRelocation &MR = Relocations[i]; 1132 void *ResultPtr = 0; 1133 if (!MR.letTargetResolve()) { 1134 if (MR.isExternalSymbol()) { 1135 ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), 1136 false); 1137 DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" 1138 << ResultPtr << "]\n"); 1139 1140 // If the target REALLY wants a stub for this function, emit it now. 1141 if (MR.mayNeedFarStub()) { 1142 ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); 1143 } 1144 } else if (MR.isGlobalValue()) { 1145 ResultPtr = getPointerToGlobal(MR.getGlobalValue(), 1146 BufferBegin+MR.getMachineCodeOffset(), 1147 MR.mayNeedFarStub()); 1148 } else if (MR.isIndirectSymbol()) { 1149 ResultPtr = getPointerToGVIndirectSym( 1150 MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); 1151 } else if (MR.isBasicBlock()) { 1152 ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); 1153 } else if (MR.isConstantPoolIndex()) { 1154 ResultPtr = (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); 1155 } else { 1156 assert(MR.isJumpTableIndex()); 1157 ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); 1158 } 1159 1160 MR.setResultPointer(ResultPtr); 1161 } 1162 1163 // if we are managing the GOT and the relocation wants an index, 1164 // give it one 1165 if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { 1166 unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); 1167 MR.setGOTIndex(idx); 1168 if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { 1169 DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr 1170 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 1171 << "\n"); 1172 ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; 1173 } 1174 } 1175 } 1176 1177 CurFn = 0; 1178 TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], 1179 Relocations.size(), MemMgr->getGOTBase()); 1180 } 1181 1182 // Update the GOT entry for F to point to the new code. 1183 if (MemMgr->isManagingGOT()) { 1184 unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); 1185 if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { 1186 DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin 1187 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 1188 << "\n"); 1189 ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; 1190 } 1191 } 1192 1193 // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for 1194 // global variables that were referenced in the relocations. 1195 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 1196 1197 if (CurBufferPtr == BufferEnd) { 1198 retryWithMoreMemory(F); 1199 return true; 1200 } else { 1201 // Now that we've succeeded in emitting the function, reset the 1202 // SizeEstimate back down to zero. 1203 SizeEstimate = 0; 1204 } 1205 1206 BufferBegin = CurBufferPtr = 0; 1207 NumBytes += FnEnd-FnStart; 1208 1209 // Invalidate the icache if necessary. 1210 sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); 1211 1212 TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, 1213 EmissionDetails); 1214 1215 DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart 1216 << "] Function: " << F.getFunction()->getName() 1217 << ": " << (FnEnd-FnStart) << " bytes of text, " 1218 << Relocations.size() << " relocations\n"); 1219 1220 Relocations.clear(); 1221 ConstPoolAddresses.clear(); 1222 1223 // Mark code region readable and executable if it's not so already. 1224 MemMgr->setMemoryExecutable(); 1225 1226 DEBUG( 1227 if (sys::hasDisassembler()) { 1228 dbgs() << "JIT: Disassembled code:\n"; 1229 dbgs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, 1230 (uintptr_t)FnStart); 1231 } else { 1232 dbgs() << "JIT: Binary code:\n"; 1233 uint8_t* q = FnStart; 1234 for (int i = 0; q < FnEnd; q += 4, ++i) { 1235 if (i == 4) 1236 i = 0; 1237 if (i == 0) 1238 dbgs() << "JIT: " << (long)(q - FnStart) << ": "; 1239 bool Done = false; 1240 for (int j = 3; j >= 0; --j) { 1241 if (q + j >= FnEnd) 1242 Done = true; 1243 else 1244 dbgs() << (unsigned short)q[j]; 1245 } 1246 if (Done) 1247 break; 1248 dbgs() << ' '; 1249 if (i == 3) 1250 dbgs() << '\n'; 1251 } 1252 dbgs()<< '\n'; 1253 } 1254 ); 1255 1256 if (DwarfExceptionHandling || JITEmitDebugInfo) { 1257 uintptr_t ActualSize = 0; 1258 SavedBufferBegin = BufferBegin; 1259 SavedBufferEnd = BufferEnd; 1260 SavedCurBufferPtr = CurBufferPtr; 1261 1262 if (MemMgr->NeedsExactSize()) { 1263 ActualSize = DE->GetDwarfTableSizeInBytes(F, *this, FnStart, FnEnd); 1264 } 1265 1266 BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(), 1267 ActualSize); 1268 BufferEnd = BufferBegin+ActualSize; 1269 EmittedFunctions[F.getFunction()].ExceptionTable = BufferBegin; 1270 uint8_t *EhStart; 1271 uint8_t *FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd, 1272 EhStart); 1273 MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, 1274 FrameRegister); 1275 uint8_t *EhEnd = CurBufferPtr; 1276 BufferBegin = SavedBufferBegin; 1277 BufferEnd = SavedBufferEnd; 1278 CurBufferPtr = SavedCurBufferPtr; 1279 1280 if (DwarfExceptionHandling) { 1281 TheJIT->RegisterTable(FrameRegister); 1282 } 1283 1284 if (JITEmitDebugInfo) { 1285 DebugInfo I; 1286 I.FnStart = FnStart; 1287 I.FnEnd = FnEnd; 1288 I.EhStart = EhStart; 1289 I.EhEnd = EhEnd; 1290 DR->RegisterFunction(F.getFunction(), I); 1291 } 1292 } 1293 1294 if (MMI) 1295 MMI->EndFunction(); 1296 1297 return false; 1298} 1299 1300void JITEmitter::retryWithMoreMemory(MachineFunction &F) { 1301 DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); 1302 Relocations.clear(); // Clear the old relocations or we'll reapply them. 1303 ConstPoolAddresses.clear(); 1304 ++NumRetries; 1305 deallocateMemForFunction(F.getFunction()); 1306 // Try again with at least twice as much free space. 1307 SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); 1308} 1309 1310/// deallocateMemForFunction - Deallocate all memory for the specified 1311/// function body. Also drop any references the function has to stubs. 1312/// May be called while the Function is being destroyed inside ~Value(). 1313void JITEmitter::deallocateMemForFunction(const Function *F) { 1314 ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator 1315 Emitted = EmittedFunctions.find(F); 1316 if (Emitted != EmittedFunctions.end()) { 1317 MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); 1318 MemMgr->deallocateExceptionTable(Emitted->second.ExceptionTable); 1319 TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); 1320 1321 EmittedFunctions.erase(Emitted); 1322 } 1323 1324 // TODO: Do we need to unregister exception handling information from libgcc 1325 // here? 1326 1327 if (JITEmitDebugInfo) { 1328 DR->UnregisterFunction(F); 1329 } 1330} 1331 1332 1333void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { 1334 if (BufferBegin) 1335 return JITCodeEmitter::allocateSpace(Size, Alignment); 1336 1337 // create a new memory block if there is no active one. 1338 // care must be taken so that BufferBegin is invalidated when a 1339 // block is trimmed 1340 BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); 1341 BufferEnd = BufferBegin+Size; 1342 return CurBufferPtr; 1343} 1344 1345void* JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { 1346 // Delegate this call through the memory manager. 1347 return MemMgr->allocateGlobal(Size, Alignment); 1348} 1349 1350void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { 1351 if (TheJIT->getJITInfo().hasCustomConstantPool()) 1352 return; 1353 1354 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 1355 if (Constants.empty()) return; 1356 1357 unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 1358 unsigned Align = MCP->getConstantPoolAlignment(); 1359 ConstantPoolBase = allocateSpace(Size, Align); 1360 ConstantPool = MCP; 1361 1362 if (ConstantPoolBase == 0) return; // Buffer overflow. 1363 1364 DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase 1365 << "] (size: " << Size << ", alignment: " << Align << ")\n"); 1366 1367 // Initialize the memory for all of the constant pool entries. 1368 unsigned Offset = 0; 1369 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 1370 MachineConstantPoolEntry CPE = Constants[i]; 1371 unsigned AlignMask = CPE.getAlignment() - 1; 1372 Offset = (Offset + AlignMask) & ~AlignMask; 1373 1374 uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; 1375 ConstPoolAddresses.push_back(CAddr); 1376 if (CPE.isMachineConstantPoolEntry()) { 1377 // FIXME: add support to lower machine constant pool values into bytes! 1378 report_fatal_error("Initialize memory with machine specific constant pool" 1379 "entry has not been implemented!"); 1380 } 1381 TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); 1382 DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; 1383 dbgs().write_hex(CAddr) << "]\n"); 1384 1385 const Type *Ty = CPE.Val.ConstVal->getType(); 1386 Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty); 1387 } 1388} 1389 1390void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { 1391 if (TheJIT->getJITInfo().hasCustomJumpTables()) 1392 return; 1393 if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) 1394 return; 1395 1396 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1397 if (JT.empty()) return; 1398 1399 unsigned NumEntries = 0; 1400 for (unsigned i = 0, e = JT.size(); i != e; ++i) 1401 NumEntries += JT[i].MBBs.size(); 1402 1403 unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getTargetData()); 1404 1405 // Just allocate space for all the jump tables now. We will fix up the actual 1406 // MBB entries in the tables after we emit the code for each block, since then 1407 // we will know the final locations of the MBBs in memory. 1408 JumpTable = MJTI; 1409 JumpTableBase = allocateSpace(NumEntries * EntrySize, 1410 MJTI->getEntryAlignment(*TheJIT->getTargetData())); 1411} 1412 1413void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { 1414 if (TheJIT->getJITInfo().hasCustomJumpTables()) 1415 return; 1416 1417 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1418 if (JT.empty() || JumpTableBase == 0) return; 1419 1420 1421 switch (MJTI->getEntryKind()) { 1422 case MachineJumpTableInfo::EK_Inline: 1423 return; 1424 case MachineJumpTableInfo::EK_BlockAddress: { 1425 // EK_BlockAddress - Each entry is a plain address of block, e.g.: 1426 // .word LBB123 1427 assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == sizeof(void*) && 1428 "Cross JIT'ing?"); 1429 1430 // For each jump table, map each target in the jump table to the address of 1431 // an emitted MachineBasicBlock. 1432 intptr_t *SlotPtr = (intptr_t*)JumpTableBase; 1433 1434 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1435 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1436 // Store the address of the basic block for this jump table slot in the 1437 // memory we allocated for the jump table in 'initJumpTableInfo' 1438 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 1439 *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); 1440 } 1441 break; 1442 } 1443 1444 case MachineJumpTableInfo::EK_Custom32: 1445 case MachineJumpTableInfo::EK_GPRel32BlockAddress: 1446 case MachineJumpTableInfo::EK_LabelDifference32: { 1447 assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == 4&&"Cross JIT'ing?"); 1448 // For each jump table, place the offset from the beginning of the table 1449 // to the target address. 1450 int *SlotPtr = (int*)JumpTableBase; 1451 1452 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1453 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1454 // Store the offset of the basic block for this jump table slot in the 1455 // memory we allocated for the jump table in 'initJumpTableInfo' 1456 uintptr_t Base = (uintptr_t)SlotPtr; 1457 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { 1458 uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); 1459 /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. 1460 *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); 1461 } 1462 } 1463 break; 1464 } 1465 } 1466} 1467 1468void JITEmitter::startGVStub(const GlobalValue* GV, 1469 unsigned StubSize, unsigned Alignment) { 1470 SavedBufferBegin = BufferBegin; 1471 SavedBufferEnd = BufferEnd; 1472 SavedCurBufferPtr = CurBufferPtr; 1473 1474 BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); 1475 BufferEnd = BufferBegin+StubSize+1; 1476} 1477 1478void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { 1479 SavedBufferBegin = BufferBegin; 1480 SavedBufferEnd = BufferEnd; 1481 SavedCurBufferPtr = CurBufferPtr; 1482 1483 BufferBegin = CurBufferPtr = (uint8_t *)Buffer; 1484 BufferEnd = BufferBegin+StubSize+1; 1485} 1486 1487void JITEmitter::finishGVStub() { 1488 assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); 1489 NumBytes += getCurrentPCOffset(); 1490 BufferBegin = SavedBufferBegin; 1491 BufferEnd = SavedBufferEnd; 1492 CurBufferPtr = SavedCurBufferPtr; 1493} 1494 1495void *JITEmitter::allocIndirectGV(const GlobalValue *GV, 1496 const uint8_t *Buffer, size_t Size, 1497 unsigned Alignment) { 1498 uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); 1499 memcpy(IndGV, Buffer, Size); 1500 return IndGV; 1501} 1502 1503// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry 1504// in the constant pool that was last emitted with the 'emitConstantPool' 1505// method. 1506// 1507uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { 1508 assert(ConstantNum < ConstantPool->getConstants().size() && 1509 "Invalid ConstantPoolIndex!"); 1510 return ConstPoolAddresses[ConstantNum]; 1511} 1512 1513// getJumpTableEntryAddress - Return the address of the JumpTable with index 1514// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' 1515// 1516uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { 1517 const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); 1518 assert(Index < JT.size() && "Invalid jump table index!"); 1519 1520 unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getTargetData()); 1521 1522 unsigned Offset = 0; 1523 for (unsigned i = 0; i < Index; ++i) 1524 Offset += JT[i].MBBs.size(); 1525 1526 Offset *= EntrySize; 1527 1528 return (uintptr_t)((char *)JumpTableBase + Offset); 1529} 1530 1531void JITEmitter::EmittedFunctionConfig::onDelete( 1532 JITEmitter *Emitter, const Function *F) { 1533 Emitter->deallocateMemForFunction(F); 1534} 1535void JITEmitter::EmittedFunctionConfig::onRAUW( 1536 JITEmitter *, const Function*, const Function*) { 1537 llvm_unreachable("The JIT doesn't know how to handle a" 1538 " RAUW on a value it has emitted."); 1539} 1540 1541 1542//===----------------------------------------------------------------------===// 1543// Public interface to this file 1544//===----------------------------------------------------------------------===// 1545 1546JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, 1547 TargetMachine &tm) { 1548 return new JITEmitter(jit, JMM, tm); 1549} 1550 1551// getPointerToFunctionOrStub - If the specified function has been 1552// code-gen'd, return a pointer to the function. If not, compile it, or use 1553// a stub to implement lazy compilation if available. 1554// 1555void *JIT::getPointerToFunctionOrStub(Function *F) { 1556 // If we have already code generated the function, just return the address. 1557 if (void *Addr = getPointerToGlobalIfAvailable(F)) 1558 return Addr; 1559 1560 // Get a stub if the target supports it. 1561 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1562 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 1563 return JE->getJITResolver().getLazyFunctionStub(F); 1564} 1565 1566void JIT::updateFunctionStub(Function *F) { 1567 // Get the empty stub we generated earlier. 1568 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1569 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 1570 void *Stub = JE->getJITResolver().getLazyFunctionStub(F); 1571 void *Addr = getPointerToGlobalIfAvailable(F); 1572 assert(Addr != Stub && "Function must have non-stub address to be updated."); 1573 1574 // Tell the target jit info to rewrite the stub at the specified address, 1575 // rather than creating a new one. 1576 TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); 1577 JE->startGVStub(Stub, layout.Size); 1578 getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); 1579 JE->finishGVStub(); 1580} 1581 1582/// freeMachineCodeForFunction - release machine code memory for given Function. 1583/// 1584void JIT::freeMachineCodeForFunction(Function *F) { 1585 // Delete translation for this from the ExecutionEngine, so it will get 1586 // retranslated next time it is used. 1587 updateGlobalMapping(F, 0); 1588 1589 // Free the actual memory for the function body and related stuff. 1590 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1591 cast<JITEmitter>(JCE)->deallocateMemForFunction(F); 1592} 1593