JITEmitter.cpp revision 8cd4c3e6534a14566bf163301fd45bca34e655c1
1//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file was developed by the LLVM research group and is distributed under 6// the University of Illinois Open Source License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a MachineCodeEmitter object that is used by the JIT to 11// write machine code to memory and remember where relocatable values are. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "jit" 16#include "JIT.h" 17#include "llvm/Constant.h" 18#include "llvm/Module.h" 19#include "llvm/Type.h" 20#include "llvm/CodeGen/MachineCodeEmitter.h" 21#include "llvm/CodeGen/MachineFunction.h" 22#include "llvm/CodeGen/MachineConstantPool.h" 23#include "llvm/CodeGen/MachineJumpTableInfo.h" 24#include "llvm/CodeGen/MachineRelocation.h" 25#include "llvm/ExecutionEngine/GenericValue.h" 26#include "llvm/Target/TargetData.h" 27#include "llvm/Target/TargetJITInfo.h" 28#include "llvm/Target/TargetMachine.h" 29#include "llvm/Support/Debug.h" 30#include "llvm/Support/Disassembler.h" 31#include "llvm/Support/MutexGuard.h" 32#include "llvm/ADT/Statistic.h" 33#include "llvm/System/Memory.h" 34#include <algorithm> 35using namespace llvm; 36 37STATISTIC(NumBytes, "Number of bytes of machine code compiled"); 38STATISTIC(NumRelos, "Number of relocations applied"); 39static JIT *TheJIT = 0; 40 41//===----------------------------------------------------------------------===// 42// JITMemoryManager code. 43// 44namespace { 45 /// MemoryRangeHeader - For a range of memory, this is the header that we put 46 /// on the block of memory. It is carefully crafted to be one word of memory. 47 /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader 48 /// which starts with this. 49 struct FreeRangeHeader; 50 struct MemoryRangeHeader { 51 /// ThisAllocated - This is true if this block is currently allocated. If 52 /// not, this can be converted to a FreeRangeHeader. 53 intptr_t ThisAllocated : 1; 54 55 /// PrevAllocated - Keep track of whether the block immediately before us is 56 /// allocated. If not, the word immediately before this header is the size 57 /// of the previous block. 58 intptr_t PrevAllocated : 1; 59 60 /// BlockSize - This is the size in bytes of this memory block, 61 /// including this header. 62 uintptr_t BlockSize : (sizeof(intptr_t)*8 - 2); 63 64 65 /// getBlockAfter - Return the memory block immediately after this one. 66 /// 67 MemoryRangeHeader &getBlockAfter() const { 68 return *(MemoryRangeHeader*)((char*)this+BlockSize); 69 } 70 71 /// getFreeBlockBefore - If the block before this one is free, return it, 72 /// otherwise return null. 73 FreeRangeHeader *getFreeBlockBefore() const { 74 if (PrevAllocated) return 0; 75 intptr_t PrevSize = ((intptr_t *)this)[-1]; 76 return (FreeRangeHeader*)((char*)this-PrevSize); 77 } 78 79 /// FreeBlock - Turn an allocated block into a free block, adjusting 80 /// bits in the object headers, and adding an end of region memory block. 81 FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); 82 83 /// TrimAllocationToSize - If this allocated block is significantly larger 84 /// than NewSize, split it into two pieces (where the former is NewSize 85 /// bytes, including the header), and add the new block to the free list. 86 FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, 87 uint64_t NewSize); 88 }; 89 90 /// FreeRangeHeader - For a memory block that isn't already allocated, this 91 /// keeps track of the current block and has a pointer to the next free block. 92 /// Free blocks are kept on a circularly linked list. 93 struct FreeRangeHeader : public MemoryRangeHeader { 94 FreeRangeHeader *Prev; 95 FreeRangeHeader *Next; 96 97 /// getMinBlockSize - Get the minimum size for a memory block. Blocks 98 /// smaller than this size cannot be created. 99 static unsigned getMinBlockSize() { 100 return sizeof(FreeRangeHeader)+sizeof(intptr_t); 101 } 102 103 /// SetEndOfBlockSizeMarker - The word at the end of every free block is 104 /// known to be the size of the free block. Set it for this block. 105 void SetEndOfBlockSizeMarker() { 106 void *EndOfBlock = (char*)this + BlockSize; 107 ((intptr_t *)EndOfBlock)[-1] = BlockSize; 108 } 109 110 FreeRangeHeader *RemoveFromFreeList() { 111 assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); 112 Next->Prev = Prev; 113 return Prev->Next = Next; 114 } 115 116 void AddToFreeList(FreeRangeHeader *FreeList) { 117 Next = FreeList; 118 Prev = FreeList->Prev; 119 Prev->Next = this; 120 Next->Prev = this; 121 } 122 123 /// GrowBlock - The block after this block just got deallocated. Merge it 124 /// into the current block. 125 void GrowBlock(uintptr_t NewSize); 126 127 /// AllocateBlock - Mark this entire block allocated, updating freelists 128 /// etc. This returns a pointer to the circular free-list. 129 FreeRangeHeader *AllocateBlock(); 130 }; 131} 132 133 134/// AllocateBlock - Mark this entire block allocated, updating freelists 135/// etc. This returns a pointer to the circular free-list. 136FreeRangeHeader *FreeRangeHeader::AllocateBlock() { 137 assert(!ThisAllocated && !getBlockAfter().PrevAllocated && 138 "Cannot allocate an allocated block!"); 139 // Mark this block allocated. 140 ThisAllocated = 1; 141 getBlockAfter().PrevAllocated = 1; 142 143 // Remove it from the free list. 144 return RemoveFromFreeList(); 145} 146 147/// FreeBlock - Turn an allocated block into a free block, adjusting 148/// bits in the object headers, and adding an end of region memory block. 149/// If possible, coallesce this block with neighboring blocks. Return the 150/// FreeRangeHeader to allocate from. 151FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { 152 MemoryRangeHeader *FollowingBlock = &getBlockAfter(); 153 assert(ThisAllocated && "This block is already allocated!"); 154 assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); 155 156 FreeRangeHeader *FreeListToReturn = FreeList; 157 158 // If the block after this one is free, merge it into this block. 159 if (!FollowingBlock->ThisAllocated) { 160 FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; 161 // "FreeList" always needs to be a valid free block. If we're about to 162 // coallesce with it, update our notion of what the free list is. 163 if (&FollowingFreeBlock == FreeList) { 164 FreeList = FollowingFreeBlock.Next; 165 FreeListToReturn = 0; 166 assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); 167 } 168 FollowingFreeBlock.RemoveFromFreeList(); 169 170 // Include the following block into this one. 171 BlockSize += FollowingFreeBlock.BlockSize; 172 FollowingBlock = &FollowingFreeBlock.getBlockAfter(); 173 174 // Tell the block after the block we are coallescing that this block is 175 // allocated. 176 FollowingBlock->PrevAllocated = 1; 177 } 178 179 assert(FollowingBlock->ThisAllocated && "Missed coallescing?"); 180 181 if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { 182 PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); 183 return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; 184 } 185 186 // Otherwise, mark this block free. 187 FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; 188 FollowingBlock->PrevAllocated = 0; 189 FreeBlock.ThisAllocated = 0; 190 191 // Link this into the linked list of free blocks. 192 FreeBlock.AddToFreeList(FreeList); 193 194 // Add a marker at the end of the block, indicating the size of this free 195 // block. 196 FreeBlock.SetEndOfBlockSizeMarker(); 197 return FreeListToReturn ? FreeListToReturn : &FreeBlock; 198} 199 200/// GrowBlock - The block after this block just got deallocated. Merge it 201/// into the current block. 202void FreeRangeHeader::GrowBlock(uintptr_t NewSize) { 203 assert(NewSize > BlockSize && "Not growing block?"); 204 BlockSize = NewSize; 205 SetEndOfBlockSizeMarker(); 206 getBlockAfter().PrevAllocated = 0; 207} 208 209/// TrimAllocationToSize - If this allocated block is significantly larger 210/// than NewSize, split it into two pieces (where the former is NewSize 211/// bytes, including the header), and add the new block to the free list. 212FreeRangeHeader *MemoryRangeHeader:: 213TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { 214 assert(ThisAllocated && getBlockAfter().PrevAllocated && 215 "Cannot deallocate part of an allocated block!"); 216 217 // Round up size for alignment of header. 218 unsigned HeaderAlign = __alignof(FreeRangeHeader); 219 NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); 220 221 // Size is now the size of the block we will remove from the start of the 222 // current block. 223 assert(NewSize <= BlockSize && 224 "Allocating more space from this block than exists!"); 225 226 // If splitting this block will cause the remainder to be too small, do not 227 // split the block. 228 if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) 229 return FreeList; 230 231 // Otherwise, we splice the required number of bytes out of this block, form 232 // a new block immediately after it, then mark this block allocated. 233 MemoryRangeHeader &FormerNextBlock = getBlockAfter(); 234 235 // Change the size of this block. 236 BlockSize = NewSize; 237 238 // Get the new block we just sliced out and turn it into a free block. 239 FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); 240 NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; 241 NewNextBlock.ThisAllocated = 0; 242 NewNextBlock.PrevAllocated = 1; 243 NewNextBlock.SetEndOfBlockSizeMarker(); 244 FormerNextBlock.PrevAllocated = 0; 245 NewNextBlock.AddToFreeList(FreeList); 246 return &NewNextBlock; 247} 248 249 250namespace { 251 /// JITMemoryManager - Manage memory for the JIT code generation in a logical, 252 /// sane way. This splits a large block of MAP_NORESERVE'd memory into two 253 /// sections, one for function stubs, one for the functions themselves. We 254 /// have to do this because we may need to emit a function stub while in the 255 /// middle of emitting a function, and we don't know how large the function we 256 /// are emitting is. This never bothers to release the memory, because when 257 /// we are ready to destroy the JIT, the program exits. 258 class JITMemoryManager { 259 std::vector<sys::MemoryBlock> Blocks; // Memory blocks allocated by the JIT 260 FreeRangeHeader *FreeMemoryList; // Circular list of free blocks. 261 262 // When emitting code into a memory block, this is the block. 263 MemoryRangeHeader *CurBlock; 264 265 unsigned char *CurStubPtr, *StubBase; 266 unsigned char *GOTBase; // Target Specific reserved memory 267 268 // Centralize memory block allocation. 269 sys::MemoryBlock getNewMemoryBlock(unsigned size); 270 271 std::map<const Function*, MemoryRangeHeader*> FunctionBlocks; 272 public: 273 JITMemoryManager(bool useGOT); 274 ~JITMemoryManager(); 275 276 inline unsigned char *allocateStub(unsigned StubSize, unsigned Alignment); 277 278 /// startFunctionBody - When a function starts, allocate a block of free 279 /// executable memory, returning a pointer to it and its actual size. 280 unsigned char *startFunctionBody(uintptr_t &ActualSize) { 281 CurBlock = FreeMemoryList; 282 283 // Allocate the entire memory block. 284 FreeMemoryList = FreeMemoryList->AllocateBlock(); 285 ActualSize = CurBlock->BlockSize-sizeof(MemoryRangeHeader); 286 return (unsigned char *)(CurBlock+1); 287 } 288 289 /// endFunctionBody - The function F is now allocated, and takes the memory 290 /// in the range [FunctionStart,FunctionEnd). 291 void endFunctionBody(const Function *F, unsigned char *FunctionStart, 292 unsigned char *FunctionEnd) { 293 assert(FunctionEnd > FunctionStart); 294 assert(FunctionStart == (unsigned char *)(CurBlock+1) && 295 "Mismatched function start/end!"); 296 297 uintptr_t BlockSize = FunctionEnd - (unsigned char *)CurBlock; 298 FunctionBlocks[F] = CurBlock; 299 300 // Release the memory at the end of this block that isn't needed. 301 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 302 } 303 304 unsigned char *getGOTBase() const { 305 return GOTBase; 306 } 307 bool isManagingGOT() const { 308 return GOTBase != NULL; 309 } 310 311 /// deallocateMemForFunction - Deallocate all memory for the specified 312 /// function body. 313 void deallocateMemForFunction(const Function *F) { 314 std::map<const Function*, MemoryRangeHeader*>::iterator 315 I = FunctionBlocks.find(F); 316 if (I == FunctionBlocks.end()) return; 317 318 // Find the block that is allocated for this function. 319 MemoryRangeHeader *MemRange = I->second; 320 assert(MemRange->ThisAllocated && "Block isn't allocated!"); 321 322 // Fill the buffer with garbage! 323 DEBUG(memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange))); 324 325 // Free the memory. 326 FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); 327 328 // Finally, remove this entry from FunctionBlocks. 329 FunctionBlocks.erase(I); 330 } 331 }; 332} 333 334JITMemoryManager::JITMemoryManager(bool useGOT) { 335 // Allocate a 16M block of memory for functions. 336 sys::MemoryBlock MemBlock = getNewMemoryBlock(16 << 20); 337 338 unsigned char *MemBase = reinterpret_cast<unsigned char*>(MemBlock.base()); 339 340 // Allocate stubs backwards from the base, allocate functions forward 341 // from the base. 342 StubBase = MemBase; 343 CurStubPtr = MemBase + 512*1024; // Use 512k for stubs, working backwards. 344 345 // We set up the memory chunk with 4 mem regions, like this: 346 // [ START 347 // [ Free #0 ] -> Large space to allocate functions from. 348 // [ Allocated #1 ] -> Tiny space to separate regions. 349 // [ Free #2 ] -> Tiny space so there is always at least 1 free block. 350 // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. 351 // END ] 352 // 353 // The last three blocks are never deallocated or touched. 354 355 // Add MemoryRangeHeader to the end of the memory region, indicating that 356 // the space after the block of memory is allocated. This is block #3. 357 MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; 358 Mem3->ThisAllocated = 1; 359 Mem3->PrevAllocated = 0; 360 Mem3->BlockSize = 0; 361 362 /// Add a tiny free region so that the free list always has one entry. 363 FreeRangeHeader *Mem2 = 364 (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); 365 Mem2->ThisAllocated = 0; 366 Mem2->PrevAllocated = 1; 367 Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); 368 Mem2->SetEndOfBlockSizeMarker(); 369 Mem2->Prev = Mem2; // Mem2 *is* the free list for now. 370 Mem2->Next = Mem2; 371 372 /// Add a tiny allocated region so that Mem2 is never coallesced away. 373 MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; 374 Mem1->ThisAllocated = 1; 375 Mem1->PrevAllocated = 0; 376 Mem1->BlockSize = (char*)Mem2 - (char*)Mem1; 377 378 // Add a FreeRangeHeader to the start of the function body region, indicating 379 // that the space is free. Mark the previous block allocated so we never look 380 // at it. 381 FreeRangeHeader *Mem0 = (FreeRangeHeader*)CurStubPtr; 382 Mem0->ThisAllocated = 0; 383 Mem0->PrevAllocated = 1; 384 Mem0->BlockSize = (char*)Mem1-(char*)Mem0; 385 Mem0->SetEndOfBlockSizeMarker(); 386 Mem0->AddToFreeList(Mem2); 387 388 // Start out with the freelist pointing to Mem0. 389 FreeMemoryList = Mem0; 390 391 // Allocate the GOT. 392 GOTBase = NULL; 393 if (useGOT) GOTBase = new unsigned char[sizeof(void*) * 8192]; 394} 395 396JITMemoryManager::~JITMemoryManager() { 397 for (unsigned i = 0, e = Blocks.size(); i != e; ++i) 398 sys::Memory::ReleaseRWX(Blocks[i]); 399 400 delete[] GOTBase; 401 Blocks.clear(); 402} 403 404unsigned char *JITMemoryManager::allocateStub(unsigned StubSize, 405 unsigned Alignment) { 406 CurStubPtr -= StubSize; 407 CurStubPtr = (unsigned char*)(((intptr_t)CurStubPtr) & 408 ~(intptr_t)(Alignment-1)); 409 if (CurStubPtr < StubBase) { 410 // FIXME: allocate a new block 411 cerr << "JIT ran out of memory for function stubs!\n"; 412 abort(); 413 } 414 return CurStubPtr; 415} 416 417sys::MemoryBlock JITMemoryManager::getNewMemoryBlock(unsigned size) { 418 // Allocate a new block close to the last one. 419 const sys::MemoryBlock *BOld = Blocks.empty() ? 0 : &Blocks.front(); 420 std::string ErrMsg; 421 sys::MemoryBlock B = sys::Memory::AllocateRWX(size, BOld, &ErrMsg); 422 if (B.base() == 0) { 423 cerr << "Allocation failed when allocating new memory in the JIT\n"; 424 cerr << ErrMsg << "\n"; 425 abort(); 426 } 427 Blocks.push_back(B); 428 return B; 429} 430 431//===----------------------------------------------------------------------===// 432// JIT lazy compilation code. 433// 434namespace { 435 class JITResolverState { 436 private: 437 /// FunctionToStubMap - Keep track of the stub created for a particular 438 /// function so that we can reuse them if necessary. 439 std::map<Function*, void*> FunctionToStubMap; 440 441 /// StubToFunctionMap - Keep track of the function that each stub 442 /// corresponds to. 443 std::map<void*, Function*> StubToFunctionMap; 444 445 public: 446 std::map<Function*, void*>& getFunctionToStubMap(const MutexGuard& locked) { 447 assert(locked.holds(TheJIT->lock)); 448 return FunctionToStubMap; 449 } 450 451 std::map<void*, Function*>& getStubToFunctionMap(const MutexGuard& locked) { 452 assert(locked.holds(TheJIT->lock)); 453 return StubToFunctionMap; 454 } 455 }; 456 457 /// JITResolver - Keep track of, and resolve, call sites for functions that 458 /// have not yet been compiled. 459 class JITResolver { 460 /// MCE - The MachineCodeEmitter to use to emit stubs with. 461 MachineCodeEmitter &MCE; 462 463 /// LazyResolverFn - The target lazy resolver function that we actually 464 /// rewrite instructions to use. 465 TargetJITInfo::LazyResolverFn LazyResolverFn; 466 467 JITResolverState state; 468 469 /// ExternalFnToStubMap - This is the equivalent of FunctionToStubMap for 470 /// external functions. 471 std::map<void*, void*> ExternalFnToStubMap; 472 473 //map addresses to indexes in the GOT 474 std::map<void*, unsigned> revGOTMap; 475 unsigned nextGOTIndex; 476 477 public: 478 JITResolver(MachineCodeEmitter &mce) : MCE(mce), nextGOTIndex(0) { 479 LazyResolverFn = 480 TheJIT->getJITInfo().getLazyResolverFunction(JITCompilerFn); 481 } 482 483 /// getFunctionStub - This returns a pointer to a function stub, creating 484 /// one on demand as needed. 485 void *getFunctionStub(Function *F); 486 487 /// getExternalFunctionStub - Return a stub for the function at the 488 /// specified address, created lazily on demand. 489 void *getExternalFunctionStub(void *FnAddr); 490 491 /// AddCallbackAtLocation - If the target is capable of rewriting an 492 /// instruction without the use of a stub, record the location of the use so 493 /// we know which function is being used at the location. 494 void *AddCallbackAtLocation(Function *F, void *Location) { 495 MutexGuard locked(TheJIT->lock); 496 /// Get the target-specific JIT resolver function. 497 state.getStubToFunctionMap(locked)[Location] = F; 498 return (void*)(intptr_t)LazyResolverFn; 499 } 500 501 /// getGOTIndexForAddress - Return a new or existing index in the GOT for 502 /// and address. This function only manages slots, it does not manage the 503 /// contents of the slots or the memory associated with the GOT. 504 unsigned getGOTIndexForAddr(void* addr); 505 506 /// JITCompilerFn - This function is called to resolve a stub to a compiled 507 /// address. If the LLVM Function corresponding to the stub has not yet 508 /// been compiled, this function compiles it first. 509 static void *JITCompilerFn(void *Stub); 510 }; 511} 512 513/// getJITResolver - This function returns the one instance of the JIT resolver. 514/// 515static JITResolver &getJITResolver(MachineCodeEmitter *MCE = 0) { 516 static JITResolver TheJITResolver(*MCE); 517 return TheJITResolver; 518} 519 520#if (defined(__POWERPC__) || defined (__ppc__) || defined(_POWER)) && \ 521 defined(__APPLE__) 522extern "C" void sys_icache_invalidate(const void *Addr, size_t len); 523#endif 524 525/// synchronizeICache - On some targets, the JIT emitted code must be 526/// explicitly refetched to ensure correct execution. 527static void synchronizeICache(const void *Addr, size_t len) { 528#if (defined(__POWERPC__) || defined (__ppc__) || defined(_POWER)) && \ 529 defined(__APPLE__) 530 sys_icache_invalidate(Addr, len); 531#endif 532} 533 534/// getFunctionStub - This returns a pointer to a function stub, creating 535/// one on demand as needed. 536void *JITResolver::getFunctionStub(Function *F) { 537 MutexGuard locked(TheJIT->lock); 538 539 // If we already have a stub for this function, recycle it. 540 void *&Stub = state.getFunctionToStubMap(locked)[F]; 541 if (Stub) return Stub; 542 543 // Call the lazy resolver function unless we already KNOW it is an external 544 // function, in which case we just skip the lazy resolution step. 545 void *Actual = (void*)(intptr_t)LazyResolverFn; 546 if (F->isExternal() && !F->hasNotBeenReadFromBytecode()) 547 Actual = TheJIT->getPointerToFunction(F); 548 549 // Otherwise, codegen a new stub. For now, the stub will call the lazy 550 // resolver function. 551 Stub = TheJIT->getJITInfo().emitFunctionStub(Actual, MCE); 552 553 if (Actual != (void*)(intptr_t)LazyResolverFn) { 554 // If we are getting the stub for an external function, we really want the 555 // address of the stub in the GlobalAddressMap for the JIT, not the address 556 // of the external function. 557 TheJIT->updateGlobalMapping(F, Stub); 558 } 559 560 // Invalidate the icache if necessary. 561 synchronizeICache(Stub, MCE.getCurrentPCValue()-(intptr_t)Stub); 562 563 DOUT << "JIT: Stub emitted at [" << Stub << "] for function '" 564 << F->getName() << "'\n"; 565 566 // Finally, keep track of the stub-to-Function mapping so that the 567 // JITCompilerFn knows which function to compile! 568 state.getStubToFunctionMap(locked)[Stub] = F; 569 return Stub; 570} 571 572/// getExternalFunctionStub - Return a stub for the function at the 573/// specified address, created lazily on demand. 574void *JITResolver::getExternalFunctionStub(void *FnAddr) { 575 // If we already have a stub for this function, recycle it. 576 void *&Stub = ExternalFnToStubMap[FnAddr]; 577 if (Stub) return Stub; 578 579 Stub = TheJIT->getJITInfo().emitFunctionStub(FnAddr, MCE); 580 581 // Invalidate the icache if necessary. 582 synchronizeICache(Stub, MCE.getCurrentPCValue()-(intptr_t)Stub); 583 584 DOUT << "JIT: Stub emitted at [" << Stub 585 << "] for external function at '" << FnAddr << "'\n"; 586 return Stub; 587} 588 589unsigned JITResolver::getGOTIndexForAddr(void* addr) { 590 unsigned idx = revGOTMap[addr]; 591 if (!idx) { 592 idx = ++nextGOTIndex; 593 revGOTMap[addr] = idx; 594 DOUT << "Adding GOT entry " << idx 595 << " for addr " << addr << "\n"; 596 // ((void**)MemMgr.getGOTBase())[idx] = addr; 597 } 598 return idx; 599} 600 601/// JITCompilerFn - This function is called when a lazy compilation stub has 602/// been entered. It looks up which function this stub corresponds to, compiles 603/// it if necessary, then returns the resultant function pointer. 604void *JITResolver::JITCompilerFn(void *Stub) { 605 JITResolver &JR = getJITResolver(); 606 607 MutexGuard locked(TheJIT->lock); 608 609 // The address given to us for the stub may not be exactly right, it might be 610 // a little bit after the stub. As such, use upper_bound to find it. 611 std::map<void*, Function*>::iterator I = 612 JR.state.getStubToFunctionMap(locked).upper_bound(Stub); 613 assert(I != JR.state.getStubToFunctionMap(locked).begin() && 614 "This is not a known stub!"); 615 Function *F = (--I)->second; 616 617 // If disabled, emit a useful error message and abort. 618 if (TheJIT->isLazyCompilationDisabled()) { 619 cerr << "LLVM JIT requested to do lazy compilation of function '" 620 << F->getName() << "' when lazy compiles are disabled!\n"; 621 abort(); 622 } 623 624 // We might like to remove the stub from the StubToFunction map. 625 // We can't do that! Multiple threads could be stuck, waiting to acquire the 626 // lock above. As soon as the 1st function finishes compiling the function, 627 // the next one will be released, and needs to be able to find the function it 628 // needs to call. 629 //JR.state.getStubToFunctionMap(locked).erase(I); 630 631 DOUT << "JIT: Lazily resolving function '" << F->getName() 632 << "' In stub ptr = " << Stub << " actual ptr = " 633 << I->first << "\n"; 634 635 void *Result = TheJIT->getPointerToFunction(F); 636 637 // We don't need to reuse this stub in the future, as F is now compiled. 638 JR.state.getFunctionToStubMap(locked).erase(F); 639 640 // FIXME: We could rewrite all references to this stub if we knew them. 641 642 // What we will do is set the compiled function address to map to the 643 // same GOT entry as the stub so that later clients may update the GOT 644 // if they see it still using the stub address. 645 // Note: this is done so the Resolver doesn't have to manage GOT memory 646 // Do this without allocating map space if the target isn't using a GOT 647 if(JR.revGOTMap.find(Stub) != JR.revGOTMap.end()) 648 JR.revGOTMap[Result] = JR.revGOTMap[Stub]; 649 650 return Result; 651} 652 653 654//===----------------------------------------------------------------------===// 655// JITEmitter code. 656// 657namespace { 658 /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is 659 /// used to output functions to memory for execution. 660 class JITEmitter : public MachineCodeEmitter { 661 JITMemoryManager MemMgr; 662 663 // When outputting a function stub in the context of some other function, we 664 // save BufferBegin/BufferEnd/CurBufferPtr here. 665 unsigned char *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 666 667 /// Relocations - These are the relocations that the function needs, as 668 /// emitted. 669 std::vector<MachineRelocation> Relocations; 670 671 /// MBBLocations - This vector is a mapping from MBB ID's to their address. 672 /// It is filled in by the StartMachineBasicBlock callback and queried by 673 /// the getMachineBasicBlockAddress callback. 674 std::vector<intptr_t> MBBLocations; 675 676 /// ConstantPool - The constant pool for the current function. 677 /// 678 MachineConstantPool *ConstantPool; 679 680 /// ConstantPoolBase - A pointer to the first entry in the constant pool. 681 /// 682 void *ConstantPoolBase; 683 684 /// JumpTable - The jump tables for the current function. 685 /// 686 MachineJumpTableInfo *JumpTable; 687 688 /// JumpTableBase - A pointer to the first entry in the jump table. 689 /// 690 void *JumpTableBase; 691public: 692 JITEmitter(JIT &jit) : MemMgr(jit.getJITInfo().needsGOT()) { 693 TheJIT = &jit; 694 if (MemMgr.isManagingGOT()) DOUT << "JIT is managing a GOT\n"; 695 } 696 697 virtual void startFunction(MachineFunction &F); 698 virtual bool finishFunction(MachineFunction &F); 699 700 void emitConstantPool(MachineConstantPool *MCP); 701 void initJumpTableInfo(MachineJumpTableInfo *MJTI); 702 void emitJumpTableInfo(MachineJumpTableInfo *MJTI); 703 704 virtual void startFunctionStub(unsigned StubSize, unsigned Alignment = 1); 705 virtual void* finishFunctionStub(const Function *F); 706 707 virtual void addRelocation(const MachineRelocation &MR) { 708 Relocations.push_back(MR); 709 } 710 711 virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { 712 if (MBBLocations.size() <= (unsigned)MBB->getNumber()) 713 MBBLocations.resize((MBB->getNumber()+1)*2); 714 MBBLocations[MBB->getNumber()] = getCurrentPCValue(); 715 } 716 717 virtual intptr_t getConstantPoolEntryAddress(unsigned Entry) const; 718 virtual intptr_t getJumpTableEntryAddress(unsigned Entry) const; 719 720 virtual intptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const { 721 assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 722 MBBLocations[MBB->getNumber()] && "MBB not emitted!"); 723 return MBBLocations[MBB->getNumber()]; 724 } 725 726 /// deallocateMemForFunction - Deallocate all memory for the specified 727 /// function body. 728 void deallocateMemForFunction(Function *F) { 729 MemMgr.deallocateMemForFunction(F); 730 } 731 private: 732 void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub); 733 }; 734} 735 736void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, 737 bool DoesntNeedStub) { 738 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) { 739 /// FIXME: If we straightened things out, this could actually emit the 740 /// global immediately instead of queuing it for codegen later! 741 return TheJIT->getOrEmitGlobalVariable(GV); 742 } 743 744 // If we have already compiled the function, return a pointer to its body. 745 Function *F = cast<Function>(V); 746 void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); 747 if (ResultPtr) return ResultPtr; 748 749 if (F->isExternal() && !F->hasNotBeenReadFromBytecode()) { 750 // If this is an external function pointer, we can force the JIT to 751 // 'compile' it, which really just adds it to the map. 752 if (DoesntNeedStub) 753 return TheJIT->getPointerToFunction(F); 754 755 return getJITResolver(this).getFunctionStub(F); 756 } 757 758 // Okay, the function has not been compiled yet, if the target callback 759 // mechanism is capable of rewriting the instruction directly, prefer to do 760 // that instead of emitting a stub. 761 if (DoesntNeedStub) 762 return getJITResolver(this).AddCallbackAtLocation(F, Reference); 763 764 // Otherwise, we have to emit a lazy resolving stub. 765 return getJITResolver(this).getFunctionStub(F); 766} 767 768void JITEmitter::startFunction(MachineFunction &F) { 769 uintptr_t ActualSize; 770 BufferBegin = CurBufferPtr = MemMgr.startFunctionBody(ActualSize); 771 BufferEnd = BufferBegin+ActualSize; 772 773 // Ensure the constant pool/jump table info is at least 4-byte aligned. 774 emitAlignment(16); 775 776 emitConstantPool(F.getConstantPool()); 777 initJumpTableInfo(F.getJumpTableInfo()); 778 779 // About to start emitting the machine code for the function. 780 emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); 781 TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); 782 783 MBBLocations.clear(); 784} 785 786bool JITEmitter::finishFunction(MachineFunction &F) { 787 if (CurBufferPtr == BufferEnd) { 788 // FIXME: Allocate more space, then try again. 789 cerr << "JIT: Ran out of space for generated machine code!\n"; 790 abort(); 791 } 792 793 emitJumpTableInfo(F.getJumpTableInfo()); 794 795 // FnStart is the start of the text, not the start of the constant pool and 796 // other per-function data. 797 unsigned char *FnStart = 798 (unsigned char *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); 799 unsigned char *FnEnd = CurBufferPtr; 800 801 MemMgr.endFunctionBody(F.getFunction(), BufferBegin, FnEnd); 802 NumBytes += FnEnd-FnStart; 803 804 if (!Relocations.empty()) { 805 NumRelos += Relocations.size(); 806 807 // Resolve the relocations to concrete pointers. 808 for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { 809 MachineRelocation &MR = Relocations[i]; 810 void *ResultPtr; 811 if (MR.isString()) { 812 ResultPtr = TheJIT->getPointerToNamedFunction(MR.getString()); 813 814 // If the target REALLY wants a stub for this function, emit it now. 815 if (!MR.doesntNeedFunctionStub()) 816 ResultPtr = getJITResolver(this).getExternalFunctionStub(ResultPtr); 817 } else if (MR.isGlobalValue()) { 818 ResultPtr = getPointerToGlobal(MR.getGlobalValue(), 819 BufferBegin+MR.getMachineCodeOffset(), 820 MR.doesntNeedFunctionStub()); 821 } else if (MR.isBasicBlock()) { 822 ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); 823 } else if (MR.isConstantPoolIndex()) { 824 ResultPtr=(void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); 825 } else { 826 assert(MR.isJumpTableIndex()); 827 ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); 828 } 829 830 MR.setResultPointer(ResultPtr); 831 832 // if we are managing the GOT and the relocation wants an index, 833 // give it one 834 if (MemMgr.isManagingGOT() && MR.isGOTRelative()) { 835 unsigned idx = getJITResolver(this).getGOTIndexForAddr(ResultPtr); 836 MR.setGOTIndex(idx); 837 if (((void**)MemMgr.getGOTBase())[idx] != ResultPtr) { 838 DOUT << "GOT was out of date for " << ResultPtr 839 << " pointing at " << ((void**)MemMgr.getGOTBase())[idx] 840 << "\n"; 841 ((void**)MemMgr.getGOTBase())[idx] = ResultPtr; 842 } 843 } 844 } 845 846 TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], 847 Relocations.size(), MemMgr.getGOTBase()); 848 } 849 850 // Update the GOT entry for F to point to the new code. 851 if (MemMgr.isManagingGOT()) { 852 unsigned idx = getJITResolver(this).getGOTIndexForAddr((void*)BufferBegin); 853 if (((void**)MemMgr.getGOTBase())[idx] != (void*)BufferBegin) { 854 DOUT << "GOT was out of date for " << (void*)BufferBegin 855 << " pointing at " << ((void**)MemMgr.getGOTBase())[idx] << "\n"; 856 ((void**)MemMgr.getGOTBase())[idx] = (void*)BufferBegin; 857 } 858 } 859 860 // Invalidate the icache if necessary. 861 synchronizeICache(FnStart, FnEnd-FnStart); 862 863 DOUT << "JIT: Finished CodeGen of [" << (void*)FnStart 864 << "] Function: " << F.getFunction()->getName() 865 << ": " << (FnEnd-FnStart) << " bytes of text, " 866 << Relocations.size() << " relocations\n"; 867 Relocations.clear(); 868 869 DOUT << "Disassembled code:\n" 870#if defined(__i386__) 871 << disassembleBuffer(FnStart, FnEnd-FnStart, 872 Disassembler::X86_32, (uint32_t)FnStart); 873#elif defined(__amd64__) || defined(__x86_64__) 874 << disassembleBuffer(FnStart, FnEnd-FnStart, 875 Disassembler::X86_64, (uint32_t)FnStart); 876#else 877 << "N/A\n"; 878#endif 879 880 return false; 881} 882 883void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { 884 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 885 if (Constants.empty()) return; 886 887 MachineConstantPoolEntry CPE = Constants.back(); 888 unsigned Size = CPE.Offset; 889 const Type *Ty = CPE.isMachineConstantPoolEntry() 890 ? CPE.Val.MachineCPVal->getType() : CPE.Val.ConstVal->getType(); 891 Size += TheJIT->getTargetData()->getTypeSize(Ty); 892 893 ConstantPoolBase = allocateSpace(Size, 1 << MCP->getConstantPoolAlignment()); 894 ConstantPool = MCP; 895 896 if (ConstantPoolBase == 0) return; // Buffer overflow. 897 898 // Initialize the memory for all of the constant pool entries. 899 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 900 void *CAddr = (char*)ConstantPoolBase+Constants[i].Offset; 901 if (Constants[i].isMachineConstantPoolEntry()) { 902 // FIXME: add support to lower machine constant pool values into bytes! 903 cerr << "Initialize memory with machine specific constant pool entry" 904 << " has not been implemented!\n"; 905 abort(); 906 } 907 TheJIT->InitializeMemory(Constants[i].Val.ConstVal, CAddr); 908 } 909} 910 911void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { 912 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 913 if (JT.empty()) return; 914 915 unsigned NumEntries = 0; 916 for (unsigned i = 0, e = JT.size(); i != e; ++i) 917 NumEntries += JT[i].MBBs.size(); 918 919 unsigned EntrySize = MJTI->getEntrySize(); 920 921 // Just allocate space for all the jump tables now. We will fix up the actual 922 // MBB entries in the tables after we emit the code for each block, since then 923 // we will know the final locations of the MBBs in memory. 924 JumpTable = MJTI; 925 JumpTableBase = allocateSpace(NumEntries * EntrySize, MJTI->getAlignment()); 926} 927 928void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { 929 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 930 if (JT.empty() || JumpTableBase == 0) return; 931 932 if (TargetMachine::getRelocationModel() == Reloc::PIC_) { 933 assert(MJTI->getEntrySize() == 4 && "Cross JIT'ing?"); 934 // For each jump table, place the offset from the beginning of the table 935 // to the target address. 936 int *SlotPtr = (int*)JumpTableBase; 937 938 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 939 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 940 // Store the offset of the basic block for this jump table slot in the 941 // memory we allocated for the jump table in 'initJumpTableInfo' 942 intptr_t Base = (intptr_t)SlotPtr; 943 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 944 *SlotPtr++ = (intptr_t)getMachineBasicBlockAddress(MBBs[mi]) - Base; 945 } 946 } else { 947 assert(MJTI->getEntrySize() == sizeof(void*) && "Cross JIT'ing?"); 948 949 // For each jump table, map each target in the jump table to the address of 950 // an emitted MachineBasicBlock. 951 intptr_t *SlotPtr = (intptr_t*)JumpTableBase; 952 953 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 954 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 955 // Store the address of the basic block for this jump table slot in the 956 // memory we allocated for the jump table in 'initJumpTableInfo' 957 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 958 *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); 959 } 960 } 961} 962 963void JITEmitter::startFunctionStub(unsigned StubSize, unsigned Alignment) { 964 SavedBufferBegin = BufferBegin; 965 SavedBufferEnd = BufferEnd; 966 SavedCurBufferPtr = CurBufferPtr; 967 968 BufferBegin = CurBufferPtr = MemMgr.allocateStub(StubSize, Alignment); 969 BufferEnd = BufferBegin+StubSize+1; 970} 971 972void *JITEmitter::finishFunctionStub(const Function *F) { 973 NumBytes += getCurrentPCOffset(); 974 std::swap(SavedBufferBegin, BufferBegin); 975 BufferEnd = SavedBufferEnd; 976 CurBufferPtr = SavedCurBufferPtr; 977 return SavedBufferBegin; 978} 979 980// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry 981// in the constant pool that was last emitted with the 'emitConstantPool' 982// method. 983// 984intptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { 985 assert(ConstantNum < ConstantPool->getConstants().size() && 986 "Invalid ConstantPoolIndex!"); 987 return (intptr_t)ConstantPoolBase + 988 ConstantPool->getConstants()[ConstantNum].Offset; 989} 990 991// getJumpTableEntryAddress - Return the address of the JumpTable with index 992// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' 993// 994intptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { 995 const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); 996 assert(Index < JT.size() && "Invalid jump table index!"); 997 998 unsigned Offset = 0; 999 unsigned EntrySize = JumpTable->getEntrySize(); 1000 1001 for (unsigned i = 0; i < Index; ++i) 1002 Offset += JT[i].MBBs.size(); 1003 1004 Offset *= EntrySize; 1005 1006 return (intptr_t)((char *)JumpTableBase + Offset); 1007} 1008 1009//===----------------------------------------------------------------------===// 1010// Public interface to this file 1011//===----------------------------------------------------------------------===// 1012 1013MachineCodeEmitter *JIT::createEmitter(JIT &jit) { 1014 return new JITEmitter(jit); 1015} 1016 1017// getPointerToNamedFunction - This function is used as a global wrapper to 1018// JIT::getPointerToNamedFunction for the purpose of resolving symbols when 1019// bugpoint is debugging the JIT. In that scenario, we are loading an .so and 1020// need to resolve function(s) that are being mis-codegenerated, so we need to 1021// resolve their addresses at runtime, and this is the way to do it. 1022extern "C" { 1023 void *getPointerToNamedFunction(const char *Name) { 1024 if (Function *F = TheJIT->FindFunctionNamed(Name)) 1025 return TheJIT->getPointerToFunction(F); 1026 return TheJIT->getPointerToNamedFunction(Name); 1027 } 1028} 1029 1030// getPointerToFunctionOrStub - If the specified function has been 1031// code-gen'd, return a pointer to the function. If not, compile it, or use 1032// a stub to implement lazy compilation if available. 1033// 1034void *JIT::getPointerToFunctionOrStub(Function *F) { 1035 // If we have already code generated the function, just return the address. 1036 if (void *Addr = getPointerToGlobalIfAvailable(F)) 1037 return Addr; 1038 1039 // Get a stub if the target supports it 1040 return getJITResolver(MCE).getFunctionStub(F); 1041} 1042 1043/// freeMachineCodeForFunction - release machine code memory for given Function. 1044/// 1045void JIT::freeMachineCodeForFunction(Function *F) { 1046 // Delete translation for this from the ExecutionEngine, so it will get 1047 // retranslated next time it is used. 1048 updateGlobalMapping(F, 0); 1049 1050 // Free the actual memory for the function body and related stuff. 1051 assert(dynamic_cast<JITEmitter*>(MCE) && "Unexpected MCE?"); 1052 dynamic_cast<JITEmitter*>(MCE)->deallocateMemForFunction(F); 1053} 1054 1055