JITEmitter.cpp revision dc17ab2bf0c4d325b87ac8130004ab11f3f7106d
1//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a MachineCodeEmitter object that is used by the JIT to 11// write machine code to memory and remember where relocatable values are. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "jit" 16#include "JIT.h" 17#include "JITDwarfEmitter.h" 18#include "llvm/Constant.h" 19#include "llvm/Module.h" 20#include "llvm/Type.h" 21#include "llvm/CodeGen/MachineCodeEmitter.h" 22#include "llvm/CodeGen/MachineFunction.h" 23#include "llvm/CodeGen/MachineConstantPool.h" 24#include "llvm/CodeGen/MachineJumpTableInfo.h" 25#include "llvm/CodeGen/MachineModuleInfo.h" 26#include "llvm/CodeGen/MachineRelocation.h" 27#include "llvm/ExecutionEngine/JITMemoryManager.h" 28#include "llvm/Target/TargetData.h" 29#include "llvm/Target/TargetJITInfo.h" 30#include "llvm/Target/TargetMachine.h" 31#include "llvm/Target/TargetOptions.h" 32#include "llvm/Support/Debug.h" 33#include "llvm/Support/MutexGuard.h" 34#include "llvm/System/Disassembler.h" 35#include "llvm/Target/TargetInstrInfo.h" 36#include "llvm/ADT/Statistic.h" 37#include <algorithm> 38using namespace llvm; 39 40STATISTIC(NumBytes, "Number of bytes of machine code compiled"); 41STATISTIC(NumRelos, "Number of relocations applied"); 42static JIT *TheJIT = 0; 43 44 45//===----------------------------------------------------------------------===// 46// JIT lazy compilation code. 47// 48namespace { 49 class JITResolverState { 50 private: 51 /// FunctionToStubMap - Keep track of the stub created for a particular 52 /// function so that we can reuse them if necessary. 53 std::map<Function*, void*> FunctionToStubMap; 54 55 /// StubToFunctionMap - Keep track of the function that each stub 56 /// corresponds to. 57 std::map<void*, Function*> StubToFunctionMap; 58 59 /// GlobalToLazyPtrMap - Keep track of the lazy pointer created for a 60 /// particular GlobalVariable so that we can reuse them if necessary. 61 std::map<GlobalValue*, void*> GlobalToLazyPtrMap; 62 63 public: 64 std::map<Function*, void*>& getFunctionToStubMap(const MutexGuard& locked) { 65 assert(locked.holds(TheJIT->lock)); 66 return FunctionToStubMap; 67 } 68 69 std::map<void*, Function*>& getStubToFunctionMap(const MutexGuard& locked) { 70 assert(locked.holds(TheJIT->lock)); 71 return StubToFunctionMap; 72 } 73 74 std::map<GlobalValue*, void*>& 75 getGlobalToLazyPtrMap(const MutexGuard& locked) { 76 assert(locked.holds(TheJIT->lock)); 77 return GlobalToLazyPtrMap; 78 } 79 }; 80 81 /// JITResolver - Keep track of, and resolve, call sites for functions that 82 /// have not yet been compiled. 83 class JITResolver { 84 /// LazyResolverFn - The target lazy resolver function that we actually 85 /// rewrite instructions to use. 86 TargetJITInfo::LazyResolverFn LazyResolverFn; 87 88 JITResolverState state; 89 90 /// ExternalFnToStubMap - This is the equivalent of FunctionToStubMap for 91 /// external functions. 92 std::map<void*, void*> ExternalFnToStubMap; 93 94 //map addresses to indexes in the GOT 95 std::map<void*, unsigned> revGOTMap; 96 unsigned nextGOTIndex; 97 98 static JITResolver *TheJITResolver; 99 public: 100 explicit JITResolver(JIT &jit) : nextGOTIndex(0) { 101 TheJIT = &jit; 102 103 LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); 104 assert(TheJITResolver == 0 && "Multiple JIT resolvers?"); 105 TheJITResolver = this; 106 } 107 108 ~JITResolver() { 109 TheJITResolver = 0; 110 } 111 112 /// getFunctionStub - This returns a pointer to a function stub, creating 113 /// one on demand as needed. 114 void *getFunctionStub(Function *F); 115 116 /// getExternalFunctionStub - Return a stub for the function at the 117 /// specified address, created lazily on demand. 118 void *getExternalFunctionStub(void *FnAddr); 119 120 /// getGlobalValueLazyPtr - Return a lazy pointer containing the specified 121 /// GV address. 122 void *getGlobalValueLazyPtr(GlobalValue *V, void *GVAddress); 123 124 /// AddCallbackAtLocation - If the target is capable of rewriting an 125 /// instruction without the use of a stub, record the location of the use so 126 /// we know which function is being used at the location. 127 void *AddCallbackAtLocation(Function *F, void *Location) { 128 MutexGuard locked(TheJIT->lock); 129 /// Get the target-specific JIT resolver function. 130 state.getStubToFunctionMap(locked)[Location] = F; 131 return (void*)(intptr_t)LazyResolverFn; 132 } 133 134 /// getGOTIndexForAddress - Return a new or existing index in the GOT for 135 /// an address. This function only manages slots, it does not manage the 136 /// contents of the slots or the memory associated with the GOT. 137 unsigned getGOTIndexForAddr(void *addr); 138 139 /// JITCompilerFn - This function is called to resolve a stub to a compiled 140 /// address. If the LLVM Function corresponding to the stub has not yet 141 /// been compiled, this function compiles it first. 142 static void *JITCompilerFn(void *Stub); 143 }; 144} 145 146JITResolver *JITResolver::TheJITResolver = 0; 147 148#if (defined(__POWERPC__) || defined (__ppc__) || defined(_POWER)) && \ 149 defined(__APPLE__) 150extern "C" void sys_icache_invalidate(const void *Addr, size_t len); 151#endif 152 153/// synchronizeICache - On some targets, the JIT emitted code must be 154/// explicitly refetched to ensure correct execution. 155static void synchronizeICache(const void *Addr, size_t len) { 156#if (defined(__POWERPC__) || defined (__ppc__) || defined(_POWER)) && \ 157 defined(__APPLE__) 158 sys_icache_invalidate(Addr, len); 159#endif 160} 161 162/// getFunctionStub - This returns a pointer to a function stub, creating 163/// one on demand as needed. 164void *JITResolver::getFunctionStub(Function *F) { 165 MutexGuard locked(TheJIT->lock); 166 167 // If we already have a stub for this function, recycle it. 168 void *&Stub = state.getFunctionToStubMap(locked)[F]; 169 if (Stub) return Stub; 170 171 // Call the lazy resolver function unless we already KNOW it is an external 172 // function, in which case we just skip the lazy resolution step. 173 void *Actual = (void*)(intptr_t)LazyResolverFn; 174 if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode()) 175 Actual = TheJIT->getPointerToFunction(F); 176 177 // Otherwise, codegen a new stub. For now, the stub will call the lazy 178 // resolver function. 179 Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, 180 *TheJIT->getCodeEmitter()); 181 182 if (Actual != (void*)(intptr_t)LazyResolverFn) { 183 // If we are getting the stub for an external function, we really want the 184 // address of the stub in the GlobalAddressMap for the JIT, not the address 185 // of the external function. 186 TheJIT->updateGlobalMapping(F, Stub); 187 } 188 189 DOUT << "JIT: Stub emitted at [" << Stub << "] for function '" 190 << F->getName() << "'\n"; 191 192 // Finally, keep track of the stub-to-Function mapping so that the 193 // JITCompilerFn knows which function to compile! 194 state.getStubToFunctionMap(locked)[Stub] = F; 195 return Stub; 196} 197 198/// getGlobalValueLazyPtr - Return a lazy pointer containing the specified 199/// GV address. 200void *JITResolver::getGlobalValueLazyPtr(GlobalValue *GV, void *GVAddress) { 201 MutexGuard locked(TheJIT->lock); 202 203 // If we already have a stub for this global variable, recycle it. 204 void *&LazyPtr = state.getGlobalToLazyPtrMap(locked)[GV]; 205 if (LazyPtr) return LazyPtr; 206 207 // Otherwise, codegen a new lazy pointer. 208 LazyPtr = TheJIT->getJITInfo().emitGlobalValueLazyPtr(GV, GVAddress, 209 *TheJIT->getCodeEmitter()); 210 211 DOUT << "JIT: Stub emitted at [" << LazyPtr << "] for GV '" 212 << GV->getName() << "'\n"; 213 214 return LazyPtr; 215} 216 217/// getExternalFunctionStub - Return a stub for the function at the 218/// specified address, created lazily on demand. 219void *JITResolver::getExternalFunctionStub(void *FnAddr) { 220 // If we already have a stub for this function, recycle it. 221 void *&Stub = ExternalFnToStubMap[FnAddr]; 222 if (Stub) return Stub; 223 224 Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, 225 *TheJIT->getCodeEmitter()); 226 227 DOUT << "JIT: Stub emitted at [" << Stub 228 << "] for external function at '" << FnAddr << "'\n"; 229 return Stub; 230} 231 232unsigned JITResolver::getGOTIndexForAddr(void* addr) { 233 unsigned idx = revGOTMap[addr]; 234 if (!idx) { 235 idx = ++nextGOTIndex; 236 revGOTMap[addr] = idx; 237 DOUT << "Adding GOT entry " << idx << " for addr " << addr << "\n"; 238 } 239 return idx; 240} 241 242/// JITCompilerFn - This function is called when a lazy compilation stub has 243/// been entered. It looks up which function this stub corresponds to, compiles 244/// it if necessary, then returns the resultant function pointer. 245void *JITResolver::JITCompilerFn(void *Stub) { 246 JITResolver &JR = *TheJITResolver; 247 248 MutexGuard locked(TheJIT->lock); 249 250 // The address given to us for the stub may not be exactly right, it might be 251 // a little bit after the stub. As such, use upper_bound to find it. 252 std::map<void*, Function*>::iterator I = 253 JR.state.getStubToFunctionMap(locked).upper_bound(Stub); 254 assert(I != JR.state.getStubToFunctionMap(locked).begin() && 255 "This is not a known stub!"); 256 Function *F = (--I)->second; 257 258 // If we have already code generated the function, just return the address. 259 void *Result = TheJIT->getPointerToGlobalIfAvailable(F); 260 261 if (!Result) { 262 // Otherwise we don't have it, do lazy compilation now. 263 264 // If lazy compilation is disabled, emit a useful error message and abort. 265 if (TheJIT->isLazyCompilationDisabled()) { 266 cerr << "LLVM JIT requested to do lazy compilation of function '" 267 << F->getName() << "' when lazy compiles are disabled!\n"; 268 abort(); 269 } 270 271 // We might like to remove the stub from the StubToFunction map. 272 // We can't do that! Multiple threads could be stuck, waiting to acquire the 273 // lock above. As soon as the 1st function finishes compiling the function, 274 // the next one will be released, and needs to be able to find the function 275 // it needs to call. 276 //JR.state.getStubToFunctionMap(locked).erase(I); 277 278 DOUT << "JIT: Lazily resolving function '" << F->getName() 279 << "' In stub ptr = " << Stub << " actual ptr = " 280 << I->first << "\n"; 281 282 Result = TheJIT->getPointerToFunction(F); 283 } 284 285 // We don't need to reuse this stub in the future, as F is now compiled. 286 JR.state.getFunctionToStubMap(locked).erase(F); 287 288 // FIXME: We could rewrite all references to this stub if we knew them. 289 290 // What we will do is set the compiled function address to map to the 291 // same GOT entry as the stub so that later clients may update the GOT 292 // if they see it still using the stub address. 293 // Note: this is done so the Resolver doesn't have to manage GOT memory 294 // Do this without allocating map space if the target isn't using a GOT 295 if(JR.revGOTMap.find(Stub) != JR.revGOTMap.end()) 296 JR.revGOTMap[Result] = JR.revGOTMap[Stub]; 297 298 return Result; 299} 300 301//===----------------------------------------------------------------------===// 302// Function Index Support 303 304// On MacOS we generate an index of currently JIT'd functions so that 305// performance tools can determine a symbol name and accurate code range for a 306// PC value. Because performance tools are generally asynchronous, the code 307// below is written with the hope that it could be interrupted at any time and 308// have useful answers. However, we don't go crazy with atomic operations, we 309// just do a "reasonable effort". 310#ifdef __APPLE__ 311#define ENABLE_JIT_SYMBOL_TABLE 1 312#endif 313 314/// JitSymbolEntry - Each function that is JIT compiled results in one of these 315/// being added to an array of symbols. This indicates the name of the function 316/// as well as the address range it occupies. This allows the client to map 317/// from a PC value to the name of the function. 318struct JitSymbolEntry { 319 const char *FnName; // FnName - a strdup'd string. 320 void *FnStart; 321 intptr_t FnSize; 322}; 323 324 325struct JitSymbolTable { 326 /// NextPtr - This forms a linked list of JitSymbolTable entries. This 327 /// pointer is not used right now, but might be used in the future. Consider 328 /// it reserved for future use. 329 JitSymbolTable *NextPtr; 330 331 /// Symbols - This is an array of JitSymbolEntry entries. Only the first 332 /// 'NumSymbols' symbols are valid. 333 JitSymbolEntry *Symbols; 334 335 /// NumSymbols - This indicates the number entries in the Symbols array that 336 /// are valid. 337 unsigned NumSymbols; 338 339 /// NumAllocated - This indicates the amount of space we have in the Symbols 340 /// array. This is a private field that should not be read by external tools. 341 unsigned NumAllocated; 342}; 343 344#if ENABLE_JIT_SYMBOL_TABLE 345JitSymbolTable *__jitSymbolTable; 346#endif 347 348static void AddFunctionToSymbolTable(const char *FnName, 349 void *FnStart, intptr_t FnSize) { 350 assert(FnName != 0 && FnStart != 0 && "Bad symbol to add"); 351 JitSymbolTable **SymTabPtrPtr = 0; 352#if !ENABLE_JIT_SYMBOL_TABLE 353 return; 354#else 355 SymTabPtrPtr = &__jitSymbolTable; 356#endif 357 358 // If this is the first entry in the symbol table, add the JitSymbolTable 359 // index. 360 if (*SymTabPtrPtr == 0) { 361 JitSymbolTable *New = new JitSymbolTable(); 362 New->NextPtr = 0; 363 New->Symbols = 0; 364 New->NumSymbols = 0; 365 New->NumAllocated = 0; 366 *SymTabPtrPtr = New; 367 } 368 369 JitSymbolTable *SymTabPtr = *SymTabPtrPtr; 370 371 // If we have space in the table, reallocate the table. 372 if (SymTabPtr->NumSymbols >= SymTabPtr->NumAllocated) { 373 // If we don't have space, reallocate the table. 374 unsigned NewSize = std::max(64U, SymTabPtr->NumAllocated*2); 375 JitSymbolEntry *NewSymbols = new JitSymbolEntry[NewSize]; 376 JitSymbolEntry *OldSymbols = SymTabPtr->Symbols; 377 378 // Copy the old entries over. 379 memcpy(NewSymbols, OldSymbols, 380 SymTabPtr->NumSymbols*sizeof(OldSymbols[0])); 381 382 // Swap the new symbols in, delete the old ones. 383 SymTabPtr->Symbols = NewSymbols; 384 SymTabPtr->NumAllocated = NewSize; 385 delete [] OldSymbols; 386 } 387 388 // Otherwise, we have enough space, just tack it onto the end of the array. 389 JitSymbolEntry &Entry = SymTabPtr->Symbols[SymTabPtr->NumSymbols]; 390 Entry.FnName = strdup(FnName); 391 Entry.FnStart = FnStart; 392 Entry.FnSize = FnSize; 393 ++SymTabPtr->NumSymbols; 394} 395 396static void RemoveFunctionFromSymbolTable(void *FnStart) { 397 assert(FnStart && "Invalid function pointer"); 398 JitSymbolTable **SymTabPtrPtr = 0; 399#if !ENABLE_JIT_SYMBOL_TABLE 400 return; 401#else 402 SymTabPtrPtr = &__jitSymbolTable; 403#endif 404 405 JitSymbolTable *SymTabPtr = *SymTabPtrPtr; 406 JitSymbolEntry *Symbols = SymTabPtr->Symbols; 407 408 // Scan the table to find its index. The table is not sorted, so do a linear 409 // scan. 410 unsigned Index; 411 for (Index = 0; Symbols[Index].FnStart != FnStart; ++Index) 412 assert(Index != SymTabPtr->NumSymbols && "Didn't find function!"); 413 414 // Once we have an index, we know to nuke this entry, overwrite it with the 415 // entry at the end of the array, making the last entry redundant. 416 const char *OldName = Symbols[Index].FnName; 417 Symbols[Index] = Symbols[SymTabPtr->NumSymbols-1]; 418 free((void*)OldName); 419 420 // Drop the number of symbols in the table. 421 --SymTabPtr->NumSymbols; 422 423 // Finally, if we deleted the final symbol, deallocate the table itself. 424 if (SymTabPtr->NumSymbols == 0) 425 return; 426 427 *SymTabPtrPtr = 0; 428 delete [] Symbols; 429 delete SymTabPtr; 430} 431 432//===----------------------------------------------------------------------===// 433// JITEmitter code. 434// 435namespace { 436 /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is 437 /// used to output functions to memory for execution. 438 class JITEmitter : public MachineCodeEmitter { 439 JITMemoryManager *MemMgr; 440 441 // When outputting a function stub in the context of some other function, we 442 // save BufferBegin/BufferEnd/CurBufferPtr here. 443 unsigned char *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 444 445 /// Relocations - These are the relocations that the function needs, as 446 /// emitted. 447 std::vector<MachineRelocation> Relocations; 448 449 /// MBBLocations - This vector is a mapping from MBB ID's to their address. 450 /// It is filled in by the StartMachineBasicBlock callback and queried by 451 /// the getMachineBasicBlockAddress callback. 452 std::vector<intptr_t> MBBLocations; 453 454 /// ConstantPool - The constant pool for the current function. 455 /// 456 MachineConstantPool *ConstantPool; 457 458 /// ConstantPoolBase - A pointer to the first entry in the constant pool. 459 /// 460 void *ConstantPoolBase; 461 462 /// JumpTable - The jump tables for the current function. 463 /// 464 MachineJumpTableInfo *JumpTable; 465 466 /// JumpTableBase - A pointer to the first entry in the jump table. 467 /// 468 void *JumpTableBase; 469 470 /// Resolver - This contains info about the currently resolved functions. 471 JITResolver Resolver; 472 473 /// DE - The dwarf emitter for the jit. 474 JITDwarfEmitter *DE; 475 476 /// LabelLocations - This vector is a mapping from Label ID's to their 477 /// address. 478 std::vector<intptr_t> LabelLocations; 479 480 /// MMI - Machine module info for exception informations 481 MachineModuleInfo* MMI; 482 483 public: 484 JITEmitter(JIT &jit, JITMemoryManager *JMM) : Resolver(jit) { 485 MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); 486 if (jit.getJITInfo().needsGOT()) { 487 MemMgr->AllocateGOT(); 488 DOUT << "JIT is managing a GOT\n"; 489 } 490 491 if (ExceptionHandling) DE = new JITDwarfEmitter(jit); 492 } 493 ~JITEmitter() { 494 delete MemMgr; 495 if (ExceptionHandling) delete DE; 496 } 497 498 JITResolver &getJITResolver() { return Resolver; } 499 500 virtual void startFunction(MachineFunction &F); 501 virtual bool finishFunction(MachineFunction &F); 502 503 void emitConstantPool(MachineConstantPool *MCP); 504 void initJumpTableInfo(MachineJumpTableInfo *MJTI); 505 void emitJumpTableInfo(MachineJumpTableInfo *MJTI); 506 507 virtual void startFunctionStub(const GlobalValue* F, unsigned StubSize, 508 unsigned Alignment = 1); 509 virtual void* finishFunctionStub(const GlobalValue *F); 510 511 virtual void addRelocation(const MachineRelocation &MR) { 512 Relocations.push_back(MR); 513 } 514 515 virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { 516 if (MBBLocations.size() <= (unsigned)MBB->getNumber()) 517 MBBLocations.resize((MBB->getNumber()+1)*2); 518 MBBLocations[MBB->getNumber()] = getCurrentPCValue(); 519 } 520 521 virtual intptr_t getConstantPoolEntryAddress(unsigned Entry) const; 522 virtual intptr_t getJumpTableEntryAddress(unsigned Entry) const; 523 524 virtual intptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const { 525 assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 526 MBBLocations[MBB->getNumber()] && "MBB not emitted!"); 527 return MBBLocations[MBB->getNumber()]; 528 } 529 530 /// deallocateMemForFunction - Deallocate all memory for the specified 531 /// function body. 532 void deallocateMemForFunction(Function *F) { 533 MemMgr->deallocateMemForFunction(F); 534 } 535 536 virtual void emitLabel(uint64_t LabelID) { 537 if (LabelLocations.size() <= LabelID) 538 LabelLocations.resize((LabelID+1)*2); 539 LabelLocations[LabelID] = getCurrentPCValue(); 540 } 541 542 virtual intptr_t getLabelAddress(uint64_t LabelID) const { 543 assert(LabelLocations.size() > (unsigned)LabelID && 544 LabelLocations[LabelID] && "Label not emitted!"); 545 return LabelLocations[LabelID]; 546 } 547 548 virtual void setModuleInfo(MachineModuleInfo* Info) { 549 MMI = Info; 550 if (ExceptionHandling) DE->setModuleInfo(Info); 551 } 552 553 private: 554 void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub); 555 void *getPointerToGVLazyPtr(GlobalValue *V, void *Reference, 556 bool NoNeedStub); 557 }; 558} 559 560void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, 561 bool DoesntNeedStub) { 562 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) { 563 /// FIXME: If we straightened things out, this could actually emit the 564 /// global immediately instead of queuing it for codegen later! 565 return TheJIT->getOrEmitGlobalVariable(GV); 566 } 567 568 // If we have already compiled the function, return a pointer to its body. 569 Function *F = cast<Function>(V); 570 void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); 571 if (ResultPtr) return ResultPtr; 572 573 if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode()) { 574 // If this is an external function pointer, we can force the JIT to 575 // 'compile' it, which really just adds it to the map. 576 if (DoesntNeedStub) 577 return TheJIT->getPointerToFunction(F); 578 579 return Resolver.getFunctionStub(F); 580 } 581 582 // Okay, the function has not been compiled yet, if the target callback 583 // mechanism is capable of rewriting the instruction directly, prefer to do 584 // that instead of emitting a stub. 585 if (DoesntNeedStub) 586 return Resolver.AddCallbackAtLocation(F, Reference); 587 588 // Otherwise, we have to emit a lazy resolving stub. 589 return Resolver.getFunctionStub(F); 590} 591 592void *JITEmitter::getPointerToGVLazyPtr(GlobalValue *V, void *Reference, 593 bool DoesntNeedStub) { 594 // Make sure GV is emitted first. 595 // FIXME: For now, if the GV is an external function we force the JIT to 596 // compile it so the lazy pointer will contain the fully resolved address. 597 void *GVAddress = getPointerToGlobal(V, Reference, true); 598 return Resolver.getGlobalValueLazyPtr(V, GVAddress); 599} 600 601static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP) { 602 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 603 if (Constants.empty()) return 0; 604 605 MachineConstantPoolEntry CPE = Constants.back(); 606 unsigned Size = CPE.Offset; 607 const Type *Ty = CPE.isMachineConstantPoolEntry() 608 ? CPE.Val.MachineCPVal->getType() : CPE.Val.ConstVal->getType(); 609 Size += TheJIT->getTargetData()->getABITypeSize(Ty); 610 return Size; 611} 612 613static unsigned GetJumpTableSizeInBytes(MachineJumpTableInfo *MJTI) { 614 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 615 if (JT.empty()) return 0; 616 617 unsigned NumEntries = 0; 618 for (unsigned i = 0, e = JT.size(); i != e; ++i) 619 NumEntries += JT[i].MBBs.size(); 620 621 unsigned EntrySize = MJTI->getEntrySize(); 622 623 return NumEntries * EntrySize; 624} 625 626static void AddAlignment(uintptr_t& Size, unsigned Alignment) { 627 if (Alignment == 0) Alignment = 1; 628 Size = (Size + Alignment - 1) & (Alignment - 1); 629} 630 631void JITEmitter::startFunction(MachineFunction &F) { 632 uintptr_t ActualSize = 0; 633 if (MemMgr->RequiresSize()) { 634 const TargetInstrInfo* TII = F.getTarget().getInstrInfo(); 635 MachineJumpTableInfo *MJTI = F.getJumpTableInfo(); 636 MachineConstantPool *MCP = F.getConstantPool(); 637 638 // Ensure the constant pool/jump table info is at least 4-byte aligned. 639 AddAlignment(ActualSize, 16); 640 641 // Add the alignment of the constant pool 642 AddAlignment(ActualSize, 1 << MCP->getConstantPoolAlignment()); 643 644 // Add the constant pool size 645 ActualSize += GetConstantPoolSizeInBytes(MCP); 646 647 // Add the aligment of the jump table info 648 AddAlignment(ActualSize, MJTI->getAlignment()); 649 650 // Add the jump table size 651 ActualSize += GetJumpTableSizeInBytes(MJTI); 652 653 // Add the alignment for the function 654 AddAlignment(ActualSize, std::max(F.getFunction()->getAlignment(), 8U)); 655 656 // Add the function size 657 ActualSize += TII->GetFunctionSizeInBytes(F); 658 } 659 660 BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), 661 ActualSize); 662 BufferEnd = BufferBegin+ActualSize; 663 664 // Ensure the constant pool/jump table info is at least 4-byte aligned. 665 emitAlignment(16); 666 667 emitConstantPool(F.getConstantPool()); 668 initJumpTableInfo(F.getJumpTableInfo()); 669 670 // About to start emitting the machine code for the function. 671 emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); 672 TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); 673 674 MBBLocations.clear(); 675} 676 677bool JITEmitter::finishFunction(MachineFunction &F) { 678 if (CurBufferPtr == BufferEnd) { 679 // FIXME: Allocate more space, then try again. 680 cerr << "JIT: Ran out of space for generated machine code!\n"; 681 abort(); 682 } 683 684 emitJumpTableInfo(F.getJumpTableInfo()); 685 686 // FnStart is the start of the text, not the start of the constant pool and 687 // other per-function data. 688 unsigned char *FnStart = 689 (unsigned char *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); 690 unsigned char *FnEnd = CurBufferPtr; 691 692 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, FnEnd); 693 NumBytes += FnEnd-FnStart; 694 695 if (!Relocations.empty()) { 696 NumRelos += Relocations.size(); 697 698 // Resolve the relocations to concrete pointers. 699 for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { 700 MachineRelocation &MR = Relocations[i]; 701 void *ResultPtr; 702 if (MR.isString()) { 703 ResultPtr = TheJIT->getPointerToNamedFunction(MR.getString()); 704 705 // If the target REALLY wants a stub for this function, emit it now. 706 if (!MR.doesntNeedStub()) 707 ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); 708 } else if (MR.isGlobalValue()) { 709 ResultPtr = getPointerToGlobal(MR.getGlobalValue(), 710 BufferBegin+MR.getMachineCodeOffset(), 711 MR.doesntNeedStub()); 712 } else if (MR.isGlobalValueLazyPtr()) { 713 ResultPtr = getPointerToGVLazyPtr(MR.getGlobalValue(), 714 BufferBegin+MR.getMachineCodeOffset(), 715 MR.doesntNeedStub()); 716 } else if (MR.isBasicBlock()) { 717 ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); 718 } else if (MR.isConstantPoolIndex()) { 719 ResultPtr=(void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); 720 } else { 721 assert(MR.isJumpTableIndex()); 722 ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); 723 } 724 725 MR.setResultPointer(ResultPtr); 726 727 // if we are managing the GOT and the relocation wants an index, 728 // give it one 729 if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { 730 unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); 731 MR.setGOTIndex(idx); 732 if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { 733 DOUT << "GOT was out of date for " << ResultPtr 734 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 735 << "\n"; 736 ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; 737 } 738 } 739 } 740 741 TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], 742 Relocations.size(), MemMgr->getGOTBase()); 743 } 744 745 // Update the GOT entry for F to point to the new code. 746 if (MemMgr->isManagingGOT()) { 747 unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); 748 if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { 749 DOUT << "GOT was out of date for " << (void*)BufferBegin 750 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] << "\n"; 751 ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; 752 } 753 } 754 755 // Invalidate the icache if necessary. 756 synchronizeICache(FnStart, FnEnd-FnStart); 757 758 // Add it to the JIT symbol table if the host wants it. 759 AddFunctionToSymbolTable(F.getFunction()->getNameStart(), 760 FnStart, FnEnd-FnStart); 761 762 DOUT << "JIT: Finished CodeGen of [" << (void*)FnStart 763 << "] Function: " << F.getFunction()->getName() 764 << ": " << (FnEnd-FnStart) << " bytes of text, " 765 << Relocations.size() << " relocations\n"; 766 Relocations.clear(); 767 768#ifndef NDEBUG 769 if (sys::hasDisassembler()) 770 DOUT << "Disassembled code:\n" 771 << sys::disassembleBuffer(FnStart, FnEnd-FnStart, (uintptr_t)FnStart); 772#endif 773 if (ExceptionHandling) { 774 uintptr_t ActualSize = 0; 775 SavedBufferBegin = BufferBegin; 776 SavedBufferEnd = BufferEnd; 777 SavedCurBufferPtr = CurBufferPtr; 778 779 if (MemMgr->RequiresSize()) { 780 ActualSize = DE->GetDwarfTableSize(F, *this, FnStart, FnEnd); 781 } 782 783 BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(), 784 ActualSize); 785 BufferEnd = BufferBegin+ActualSize; 786 unsigned char* FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd); 787 MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, 788 FrameRegister); 789 BufferBegin = SavedBufferBegin; 790 BufferEnd = SavedBufferEnd; 791 CurBufferPtr = SavedCurBufferPtr; 792 793 TheJIT->RegisterTable(FrameRegister); 794 } 795 MMI->EndFunction(); 796 797 return false; 798} 799 800void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { 801 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 802 if (Constants.empty()) return; 803 804 MachineConstantPoolEntry CPE = Constants.back(); 805 unsigned Size = CPE.Offset; 806 const Type *Ty = CPE.isMachineConstantPoolEntry() 807 ? CPE.Val.MachineCPVal->getType() : CPE.Val.ConstVal->getType(); 808 Size += TheJIT->getTargetData()->getABITypeSize(Ty); 809 810 unsigned Align = 1 << MCP->getConstantPoolAlignment(); 811 ConstantPoolBase = allocateSpace(Size, Align); 812 ConstantPool = MCP; 813 814 if (ConstantPoolBase == 0) return; // Buffer overflow. 815 816 DOUT << "JIT: Emitted constant pool at [" << ConstantPoolBase 817 << "] (size: " << Size << ", alignment: " << Align << ")\n"; 818 819 // Initialize the memory for all of the constant pool entries. 820 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 821 void *CAddr = (char*)ConstantPoolBase+Constants[i].Offset; 822 if (Constants[i].isMachineConstantPoolEntry()) { 823 // FIXME: add support to lower machine constant pool values into bytes! 824 cerr << "Initialize memory with machine specific constant pool entry" 825 << " has not been implemented!\n"; 826 abort(); 827 } 828 TheJIT->InitializeMemory(Constants[i].Val.ConstVal, CAddr); 829 DOUT << "JIT: CP" << i << " at [" << CAddr << "]\n"; 830 } 831} 832 833void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { 834 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 835 if (JT.empty()) return; 836 837 unsigned NumEntries = 0; 838 for (unsigned i = 0, e = JT.size(); i != e; ++i) 839 NumEntries += JT[i].MBBs.size(); 840 841 unsigned EntrySize = MJTI->getEntrySize(); 842 843 // Just allocate space for all the jump tables now. We will fix up the actual 844 // MBB entries in the tables after we emit the code for each block, since then 845 // we will know the final locations of the MBBs in memory. 846 JumpTable = MJTI; 847 JumpTableBase = allocateSpace(NumEntries * EntrySize, MJTI->getAlignment()); 848} 849 850void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { 851 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 852 if (JT.empty() || JumpTableBase == 0) return; 853 854 if (TargetMachine::getRelocationModel() == Reloc::PIC_) { 855 assert(MJTI->getEntrySize() == 4 && "Cross JIT'ing?"); 856 // For each jump table, place the offset from the beginning of the table 857 // to the target address. 858 int *SlotPtr = (int*)JumpTableBase; 859 860 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 861 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 862 // Store the offset of the basic block for this jump table slot in the 863 // memory we allocated for the jump table in 'initJumpTableInfo' 864 intptr_t Base = (intptr_t)SlotPtr; 865 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { 866 intptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); 867 *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); 868 } 869 } 870 } else { 871 assert(MJTI->getEntrySize() == sizeof(void*) && "Cross JIT'ing?"); 872 873 // For each jump table, map each target in the jump table to the address of 874 // an emitted MachineBasicBlock. 875 intptr_t *SlotPtr = (intptr_t*)JumpTableBase; 876 877 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 878 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 879 // Store the address of the basic block for this jump table slot in the 880 // memory we allocated for the jump table in 'initJumpTableInfo' 881 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 882 *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); 883 } 884 } 885} 886 887void JITEmitter::startFunctionStub(const GlobalValue* F, unsigned StubSize, 888 unsigned Alignment) { 889 SavedBufferBegin = BufferBegin; 890 SavedBufferEnd = BufferEnd; 891 SavedCurBufferPtr = CurBufferPtr; 892 893 BufferBegin = CurBufferPtr = MemMgr->allocateStub(F, StubSize, Alignment); 894 BufferEnd = BufferBegin+StubSize+1; 895} 896 897void *JITEmitter::finishFunctionStub(const GlobalValue* F) { 898 NumBytes += getCurrentPCOffset(); 899 std::swap(SavedBufferBegin, BufferBegin); 900 BufferEnd = SavedBufferEnd; 901 CurBufferPtr = SavedCurBufferPtr; 902 return SavedBufferBegin; 903} 904 905// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry 906// in the constant pool that was last emitted with the 'emitConstantPool' 907// method. 908// 909intptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { 910 assert(ConstantNum < ConstantPool->getConstants().size() && 911 "Invalid ConstantPoolIndex!"); 912 return (intptr_t)ConstantPoolBase + 913 ConstantPool->getConstants()[ConstantNum].Offset; 914} 915 916// getJumpTableEntryAddress - Return the address of the JumpTable with index 917// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' 918// 919intptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { 920 const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); 921 assert(Index < JT.size() && "Invalid jump table index!"); 922 923 unsigned Offset = 0; 924 unsigned EntrySize = JumpTable->getEntrySize(); 925 926 for (unsigned i = 0; i < Index; ++i) 927 Offset += JT[i].MBBs.size(); 928 929 Offset *= EntrySize; 930 931 return (intptr_t)((char *)JumpTableBase + Offset); 932} 933 934//===----------------------------------------------------------------------===// 935// Public interface to this file 936//===----------------------------------------------------------------------===// 937 938MachineCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM) { 939 return new JITEmitter(jit, JMM); 940} 941 942// getPointerToNamedFunction - This function is used as a global wrapper to 943// JIT::getPointerToNamedFunction for the purpose of resolving symbols when 944// bugpoint is debugging the JIT. In that scenario, we are loading an .so and 945// need to resolve function(s) that are being mis-codegenerated, so we need to 946// resolve their addresses at runtime, and this is the way to do it. 947extern "C" { 948 void *getPointerToNamedFunction(const char *Name) { 949 if (Function *F = TheJIT->FindFunctionNamed(Name)) 950 return TheJIT->getPointerToFunction(F); 951 return TheJIT->getPointerToNamedFunction(Name); 952 } 953} 954 955// getPointerToFunctionOrStub - If the specified function has been 956// code-gen'd, return a pointer to the function. If not, compile it, or use 957// a stub to implement lazy compilation if available. 958// 959void *JIT::getPointerToFunctionOrStub(Function *F) { 960 // If we have already code generated the function, just return the address. 961 if (void *Addr = getPointerToGlobalIfAvailable(F)) 962 return Addr; 963 964 // Get a stub if the target supports it. 965 assert(dynamic_cast<JITEmitter*>(MCE) && "Unexpected MCE?"); 966 JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); 967 return JE->getJITResolver().getFunctionStub(F); 968} 969 970/// freeMachineCodeForFunction - release machine code memory for given Function. 971/// 972void JIT::freeMachineCodeForFunction(Function *F) { 973 974 // Delete translation for this from the ExecutionEngine, so it will get 975 // retranslated next time it is used. 976 void *OldPtr = updateGlobalMapping(F, 0); 977 978 if (OldPtr) 979 RemoveFunctionFromSymbolTable(OldPtr); 980 981 // Free the actual memory for the function body and related stuff. 982 assert(dynamic_cast<JITEmitter*>(MCE) && "Unexpected MCE?"); 983 static_cast<JITEmitter*>(MCE)->deallocateMemForFunction(F); 984} 985 986