JITMemoryManager.cpp revision 7803ec3d458ec73372760b838469292cdf38fe67
1//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the DefaultJITMemoryManager class. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "jit" 15#include "llvm/ExecutionEngine/JITMemoryManager.h" 16#include "llvm/ADT/SmallPtrSet.h" 17#include "llvm/ADT/Statistic.h" 18#include "llvm/ADT/Twine.h" 19#include "llvm/GlobalValue.h" 20#include "llvm/Support/Allocator.h" 21#include "llvm/Support/Compiler.h" 22#include "llvm/Support/Debug.h" 23#include "llvm/Support/ErrorHandling.h" 24#include "llvm/Support/raw_ostream.h" 25#include "llvm/Support/Memory.h" 26#include <vector> 27#include <cassert> 28#include <climits> 29#include <cstring> 30using namespace llvm; 31 32STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); 33 34JITMemoryManager::~JITMemoryManager() {} 35 36//===----------------------------------------------------------------------===// 37// Memory Block Implementation. 38//===----------------------------------------------------------------------===// 39 40namespace { 41 /// MemoryRangeHeader - For a range of memory, this is the header that we put 42 /// on the block of memory. It is carefully crafted to be one word of memory. 43 /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader 44 /// which starts with this. 45 struct FreeRangeHeader; 46 struct MemoryRangeHeader { 47 /// ThisAllocated - This is true if this block is currently allocated. If 48 /// not, this can be converted to a FreeRangeHeader. 49 unsigned ThisAllocated : 1; 50 51 /// PrevAllocated - Keep track of whether the block immediately before us is 52 /// allocated. If not, the word immediately before this header is the size 53 /// of the previous block. 54 unsigned PrevAllocated : 1; 55 56 /// BlockSize - This is the size in bytes of this memory block, 57 /// including this header. 58 uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); 59 60 61 /// getBlockAfter - Return the memory block immediately after this one. 62 /// 63 MemoryRangeHeader &getBlockAfter() const { 64 return *(MemoryRangeHeader*)((char*)this+BlockSize); 65 } 66 67 /// getFreeBlockBefore - If the block before this one is free, return it, 68 /// otherwise return null. 69 FreeRangeHeader *getFreeBlockBefore() const { 70 if (PrevAllocated) return 0; 71 intptr_t PrevSize = ((intptr_t *)this)[-1]; 72 return (FreeRangeHeader*)((char*)this-PrevSize); 73 } 74 75 /// FreeBlock - Turn an allocated block into a free block, adjusting 76 /// bits in the object headers, and adding an end of region memory block. 77 FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); 78 79 /// TrimAllocationToSize - If this allocated block is significantly larger 80 /// than NewSize, split it into two pieces (where the former is NewSize 81 /// bytes, including the header), and add the new block to the free list. 82 FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, 83 uint64_t NewSize); 84 }; 85 86 /// FreeRangeHeader - For a memory block that isn't already allocated, this 87 /// keeps track of the current block and has a pointer to the next free block. 88 /// Free blocks are kept on a circularly linked list. 89 struct FreeRangeHeader : public MemoryRangeHeader { 90 FreeRangeHeader *Prev; 91 FreeRangeHeader *Next; 92 93 /// getMinBlockSize - Get the minimum size for a memory block. Blocks 94 /// smaller than this size cannot be created. 95 static unsigned getMinBlockSize() { 96 return sizeof(FreeRangeHeader)+sizeof(intptr_t); 97 } 98 99 /// SetEndOfBlockSizeMarker - The word at the end of every free block is 100 /// known to be the size of the free block. Set it for this block. 101 void SetEndOfBlockSizeMarker() { 102 void *EndOfBlock = (char*)this + BlockSize; 103 ((intptr_t *)EndOfBlock)[-1] = BlockSize; 104 } 105 106 FreeRangeHeader *RemoveFromFreeList() { 107 assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); 108 Next->Prev = Prev; 109 return Prev->Next = Next; 110 } 111 112 void AddToFreeList(FreeRangeHeader *FreeList) { 113 Next = FreeList; 114 Prev = FreeList->Prev; 115 Prev->Next = this; 116 Next->Prev = this; 117 } 118 119 /// GrowBlock - The block after this block just got deallocated. Merge it 120 /// into the current block. 121 void GrowBlock(uintptr_t NewSize); 122 123 /// AllocateBlock - Mark this entire block allocated, updating freelists 124 /// etc. This returns a pointer to the circular free-list. 125 FreeRangeHeader *AllocateBlock(); 126 }; 127} 128 129 130/// AllocateBlock - Mark this entire block allocated, updating freelists 131/// etc. This returns a pointer to the circular free-list. 132FreeRangeHeader *FreeRangeHeader::AllocateBlock() { 133 assert(!ThisAllocated && !getBlockAfter().PrevAllocated && 134 "Cannot allocate an allocated block!"); 135 // Mark this block allocated. 136 ThisAllocated = 1; 137 getBlockAfter().PrevAllocated = 1; 138 139 // Remove it from the free list. 140 return RemoveFromFreeList(); 141} 142 143/// FreeBlock - Turn an allocated block into a free block, adjusting 144/// bits in the object headers, and adding an end of region memory block. 145/// If possible, coalesce this block with neighboring blocks. Return the 146/// FreeRangeHeader to allocate from. 147FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { 148 MemoryRangeHeader *FollowingBlock = &getBlockAfter(); 149 assert(ThisAllocated && "This block is already free!"); 150 assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); 151 152 FreeRangeHeader *FreeListToReturn = FreeList; 153 154 // If the block after this one is free, merge it into this block. 155 if (!FollowingBlock->ThisAllocated) { 156 FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; 157 // "FreeList" always needs to be a valid free block. If we're about to 158 // coalesce with it, update our notion of what the free list is. 159 if (&FollowingFreeBlock == FreeList) { 160 FreeList = FollowingFreeBlock.Next; 161 FreeListToReturn = 0; 162 assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); 163 } 164 FollowingFreeBlock.RemoveFromFreeList(); 165 166 // Include the following block into this one. 167 BlockSize += FollowingFreeBlock.BlockSize; 168 FollowingBlock = &FollowingFreeBlock.getBlockAfter(); 169 170 // Tell the block after the block we are coalescing that this block is 171 // allocated. 172 FollowingBlock->PrevAllocated = 1; 173 } 174 175 assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); 176 177 if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { 178 PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); 179 return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; 180 } 181 182 // Otherwise, mark this block free. 183 FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; 184 FollowingBlock->PrevAllocated = 0; 185 FreeBlock.ThisAllocated = 0; 186 187 // Link this into the linked list of free blocks. 188 FreeBlock.AddToFreeList(FreeList); 189 190 // Add a marker at the end of the block, indicating the size of this free 191 // block. 192 FreeBlock.SetEndOfBlockSizeMarker(); 193 return FreeListToReturn ? FreeListToReturn : &FreeBlock; 194} 195 196/// GrowBlock - The block after this block just got deallocated. Merge it 197/// into the current block. 198void FreeRangeHeader::GrowBlock(uintptr_t NewSize) { 199 assert(NewSize > BlockSize && "Not growing block?"); 200 BlockSize = NewSize; 201 SetEndOfBlockSizeMarker(); 202 getBlockAfter().PrevAllocated = 0; 203} 204 205/// TrimAllocationToSize - If this allocated block is significantly larger 206/// than NewSize, split it into two pieces (where the former is NewSize 207/// bytes, including the header), and add the new block to the free list. 208FreeRangeHeader *MemoryRangeHeader:: 209TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { 210 assert(ThisAllocated && getBlockAfter().PrevAllocated && 211 "Cannot deallocate part of an allocated block!"); 212 213 // Don't allow blocks to be trimmed below minimum required size 214 NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); 215 216 // Round up size for alignment of header. 217 unsigned HeaderAlign = __alignof(FreeRangeHeader); 218 NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); 219 220 // Size is now the size of the block we will remove from the start of the 221 // current block. 222 assert(NewSize <= BlockSize && 223 "Allocating more space from this block than exists!"); 224 225 // If splitting this block will cause the remainder to be too small, do not 226 // split the block. 227 if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) 228 return FreeList; 229 230 // Otherwise, we splice the required number of bytes out of this block, form 231 // a new block immediately after it, then mark this block allocated. 232 MemoryRangeHeader &FormerNextBlock = getBlockAfter(); 233 234 // Change the size of this block. 235 BlockSize = NewSize; 236 237 // Get the new block we just sliced out and turn it into a free block. 238 FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); 239 NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; 240 NewNextBlock.ThisAllocated = 0; 241 NewNextBlock.PrevAllocated = 1; 242 NewNextBlock.SetEndOfBlockSizeMarker(); 243 FormerNextBlock.PrevAllocated = 0; 244 NewNextBlock.AddToFreeList(FreeList); 245 return &NewNextBlock; 246} 247 248//===----------------------------------------------------------------------===// 249// Memory Block Implementation. 250//===----------------------------------------------------------------------===// 251 252namespace { 253 254 class DefaultJITMemoryManager; 255 256 class JITSlabAllocator : public SlabAllocator { 257 DefaultJITMemoryManager &JMM; 258 public: 259 JITSlabAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } 260 virtual ~JITSlabAllocator() { } 261 virtual MemSlab *Allocate(size_t Size); 262 virtual void Deallocate(MemSlab *Slab); 263 }; 264 265 /// DefaultJITMemoryManager - Manage memory for the JIT code generation. 266 /// This splits a large block of MAP_NORESERVE'd memory into two 267 /// sections, one for function stubs, one for the functions themselves. We 268 /// have to do this because we may need to emit a function stub while in the 269 /// middle of emitting a function, and we don't know how large the function we 270 /// are emitting is. 271 class DefaultJITMemoryManager : public JITMemoryManager { 272 273 // Whether to poison freed memory. 274 bool PoisonMemory; 275 276 /// LastSlab - This points to the last slab allocated and is used as the 277 /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all 278 /// stubs, data, and code contiguously in memory. In general, however, this 279 /// is not possible because the NearBlock parameter is ignored on Windows 280 /// platforms and even on Unix it works on a best-effort pasis. 281 sys::MemoryBlock LastSlab; 282 283 // Memory slabs allocated by the JIT. We refer to them as slabs so we don't 284 // confuse them with the blocks of memory described above. 285 std::vector<sys::MemoryBlock> CodeSlabs; 286 JITSlabAllocator BumpSlabAllocator; 287 BumpPtrAllocator StubAllocator; 288 BumpPtrAllocator DataAllocator; 289 290 // Circular list of free blocks. 291 FreeRangeHeader *FreeMemoryList; 292 293 // When emitting code into a memory block, this is the block. 294 MemoryRangeHeader *CurBlock; 295 296 uint8_t *GOTBase; // Target Specific reserved memory 297 public: 298 DefaultJITMemoryManager(); 299 ~DefaultJITMemoryManager(); 300 301 /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the 302 /// last slab it allocated, so that subsequent allocations follow it. 303 sys::MemoryBlock allocateNewSlab(size_t size); 304 305 /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at 306 /// least this much unless more is requested. 307 static const size_t DefaultCodeSlabSize; 308 309 /// DefaultSlabSize - Allocate data into slabs of this size unless we get 310 /// an allocation above SizeThreshold. 311 static const size_t DefaultSlabSize; 312 313 /// DefaultSizeThreshold - For any allocation larger than this threshold, we 314 /// should allocate a separate slab. 315 static const size_t DefaultSizeThreshold; 316 317 /// getPointerToNamedFunction - This method returns the address of the 318 /// specified function by using the dlsym function call. As such it is only 319 /// useful for resolving library symbols, not code generated symbols. 320 /// 321 /// If AbortOnFailure is false and no function with the given name is 322 /// found, this function silently returns a null pointer. Otherwise, 323 /// it prints a message to stderr and aborts. 324 /// 325 virtual void *getPointerToNamedFunction(const std::string &Name, 326 bool AbortOnFailure = true); 327 328 void AllocateGOT(); 329 330 // Testing methods. 331 virtual bool CheckInvariants(std::string &ErrorStr); 332 size_t GetDefaultCodeSlabSize() { return DefaultCodeSlabSize; } 333 size_t GetDefaultDataSlabSize() { return DefaultSlabSize; } 334 size_t GetDefaultStubSlabSize() { return DefaultSlabSize; } 335 unsigned GetNumCodeSlabs() { return CodeSlabs.size(); } 336 unsigned GetNumDataSlabs() { return DataAllocator.GetNumSlabs(); } 337 unsigned GetNumStubSlabs() { return StubAllocator.GetNumSlabs(); } 338 339 /// startFunctionBody - When a function starts, allocate a block of free 340 /// executable memory, returning a pointer to it and its actual size. 341 uint8_t *startFunctionBody(const Function *F, uintptr_t &ActualSize) { 342 343 FreeRangeHeader* candidateBlock = FreeMemoryList; 344 FreeRangeHeader* head = FreeMemoryList; 345 FreeRangeHeader* iter = head->Next; 346 347 uintptr_t largest = candidateBlock->BlockSize; 348 349 // Search for the largest free block 350 while (iter != head) { 351 if (iter->BlockSize > largest) { 352 largest = iter->BlockSize; 353 candidateBlock = iter; 354 } 355 iter = iter->Next; 356 } 357 358 largest = largest - sizeof(MemoryRangeHeader); 359 360 // If this block isn't big enough for the allocation desired, allocate 361 // another block of memory and add it to the free list. 362 if (largest < ActualSize || 363 largest <= FreeRangeHeader::getMinBlockSize()) { 364 DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); 365 candidateBlock = allocateNewCodeSlab((size_t)ActualSize); 366 } 367 368 // Select this candidate block for allocation 369 CurBlock = candidateBlock; 370 371 // Allocate the entire memory block. 372 FreeMemoryList = candidateBlock->AllocateBlock(); 373 ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); 374 return (uint8_t *)(CurBlock + 1); 375 } 376 377 /// allocateNewCodeSlab - Helper method to allocate a new slab of code 378 /// memory from the OS and add it to the free list. Returns the new 379 /// FreeRangeHeader at the base of the slab. 380 FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { 381 // If the user needs at least MinSize free memory, then we account for 382 // two MemoryRangeHeaders: the one in the user's block, and the one at the 383 // end of the slab. 384 size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); 385 size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); 386 sys::MemoryBlock B = allocateNewSlab(SlabSize); 387 CodeSlabs.push_back(B); 388 char *MemBase = (char*)(B.base()); 389 390 // Put a tiny allocated block at the end of the memory chunk, so when 391 // FreeBlock calls getBlockAfter it doesn't fall off the end. 392 MemoryRangeHeader *EndBlock = 393 (MemoryRangeHeader*)(MemBase + B.size()) - 1; 394 EndBlock->ThisAllocated = 1; 395 EndBlock->PrevAllocated = 0; 396 EndBlock->BlockSize = sizeof(MemoryRangeHeader); 397 398 // Start out with a vast new block of free memory. 399 FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; 400 NewBlock->ThisAllocated = 0; 401 // Make sure getFreeBlockBefore doesn't look into unmapped memory. 402 NewBlock->PrevAllocated = 1; 403 NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; 404 NewBlock->SetEndOfBlockSizeMarker(); 405 NewBlock->AddToFreeList(FreeMemoryList); 406 407 assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && 408 "The block was too small!"); 409 return NewBlock; 410 } 411 412 /// endFunctionBody - The function F is now allocated, and takes the memory 413 /// in the range [FunctionStart,FunctionEnd). 414 void endFunctionBody(const Function *F, uint8_t *FunctionStart, 415 uint8_t *FunctionEnd) { 416 assert(FunctionEnd > FunctionStart); 417 assert(FunctionStart == (uint8_t *)(CurBlock+1) && 418 "Mismatched function start/end!"); 419 420 uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; 421 422 // Release the memory at the end of this block that isn't needed. 423 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 424 } 425 426 /// allocateSpace - Allocate a memory block of the given size. This method 427 /// cannot be called between calls to startFunctionBody and endFunctionBody. 428 uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) { 429 CurBlock = FreeMemoryList; 430 FreeMemoryList = FreeMemoryList->AllocateBlock(); 431 432 uint8_t *result = (uint8_t *)(CurBlock + 1); 433 434 if (Alignment == 0) Alignment = 1; 435 result = (uint8_t*)(((intptr_t)result+Alignment-1) & 436 ~(intptr_t)(Alignment-1)); 437 438 uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; 439 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 440 441 return result; 442 } 443 444 /// allocateStub - Allocate memory for a function stub. 445 uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, 446 unsigned Alignment) { 447 return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); 448 } 449 450 /// allocateGlobal - Allocate memory for a global. 451 uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) { 452 return (uint8_t*)DataAllocator.Allocate(Size, Alignment); 453 } 454 455 /// allocateCodeSection - Allocate memory for a code section. 456 uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, 457 unsigned SectionID) { 458 // FIXME: Alignement handling. 459 FreeRangeHeader* candidateBlock = FreeMemoryList; 460 FreeRangeHeader* head = FreeMemoryList; 461 FreeRangeHeader* iter = head->Next; 462 463 uintptr_t largest = candidateBlock->BlockSize; 464 465 // Search for the largest free block. 466 while (iter != head) { 467 if (iter->BlockSize > largest) { 468 largest = iter->BlockSize; 469 candidateBlock = iter; 470 } 471 iter = iter->Next; 472 } 473 474 largest = largest - sizeof(MemoryRangeHeader); 475 476 // If this block isn't big enough for the allocation desired, allocate 477 // another block of memory and add it to the free list. 478 if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) { 479 DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); 480 candidateBlock = allocateNewCodeSlab((size_t)Size); 481 } 482 483 // Select this candidate block for allocation 484 CurBlock = candidateBlock; 485 486 // Allocate the entire memory block. 487 FreeMemoryList = candidateBlock->AllocateBlock(); 488 // Release the memory at the end of this block that isn't needed. 489 FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size); 490 return (uint8_t *)(CurBlock + 1); 491 } 492 493 /// allocateDataSection - Allocate memory for a data section. 494 uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, 495 unsigned SectionID) { 496 return (uint8_t*)DataAllocator.Allocate(Size, Alignment); 497 } 498 499 /// startExceptionTable - Use startFunctionBody to allocate memory for the 500 /// function's exception table. 501 uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) { 502 return startFunctionBody(F, ActualSize); 503 } 504 505 /// endExceptionTable - The exception table of F is now allocated, 506 /// and takes the memory in the range [TableStart,TableEnd). 507 void endExceptionTable(const Function *F, uint8_t *TableStart, 508 uint8_t *TableEnd, uint8_t* FrameRegister) { 509 assert(TableEnd > TableStart); 510 assert(TableStart == (uint8_t *)(CurBlock+1) && 511 "Mismatched table start/end!"); 512 513 uintptr_t BlockSize = TableEnd - (uint8_t *)CurBlock; 514 515 // Release the memory at the end of this block that isn't needed. 516 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 517 } 518 519 uint8_t *getGOTBase() const { 520 return GOTBase; 521 } 522 523 void deallocateBlock(void *Block) { 524 // Find the block that is allocated for this function. 525 MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1; 526 assert(MemRange->ThisAllocated && "Block isn't allocated!"); 527 528 // Fill the buffer with garbage! 529 if (PoisonMemory) { 530 memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); 531 } 532 533 // Free the memory. 534 FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); 535 } 536 537 /// deallocateFunctionBody - Deallocate all memory for the specified 538 /// function body. 539 void deallocateFunctionBody(void *Body) { 540 if (Body) deallocateBlock(Body); 541 } 542 543 /// deallocateExceptionTable - Deallocate memory for the specified 544 /// exception table. 545 void deallocateExceptionTable(void *ET) { 546 if (ET) deallocateBlock(ET); 547 } 548 549 /// setMemoryWritable - When code generation is in progress, 550 /// the code pages may need permissions changed. 551 void setMemoryWritable() 552 { 553 for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 554 sys::Memory::setWritable(CodeSlabs[i]); 555 } 556 /// setMemoryExecutable - When code generation is done and we're ready to 557 /// start execution, the code pages may need permissions changed. 558 void setMemoryExecutable() 559 { 560 for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 561 sys::Memory::setExecutable(CodeSlabs[i]); 562 } 563 564 /// setPoisonMemory - Controls whether we write garbage over freed memory. 565 /// 566 void setPoisonMemory(bool poison) { 567 PoisonMemory = poison; 568 } 569 }; 570} 571 572MemSlab *JITSlabAllocator::Allocate(size_t Size) { 573 sys::MemoryBlock B = JMM.allocateNewSlab(Size); 574 MemSlab *Slab = (MemSlab*)B.base(); 575 Slab->Size = B.size(); 576 Slab->NextPtr = 0; 577 return Slab; 578} 579 580void JITSlabAllocator::Deallocate(MemSlab *Slab) { 581 sys::MemoryBlock B(Slab, Slab->Size); 582 sys::Memory::ReleaseRWX(B); 583} 584 585DefaultJITMemoryManager::DefaultJITMemoryManager() 586 : 587#ifdef NDEBUG 588 PoisonMemory(false), 589#else 590 PoisonMemory(true), 591#endif 592 LastSlab(0, 0), 593 BumpSlabAllocator(*this), 594 StubAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator), 595 DataAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator) { 596 597 // Allocate space for code. 598 sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); 599 CodeSlabs.push_back(MemBlock); 600 uint8_t *MemBase = (uint8_t*)MemBlock.base(); 601 602 // We set up the memory chunk with 4 mem regions, like this: 603 // [ START 604 // [ Free #0 ] -> Large space to allocate functions from. 605 // [ Allocated #1 ] -> Tiny space to separate regions. 606 // [ Free #2 ] -> Tiny space so there is always at least 1 free block. 607 // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. 608 // END ] 609 // 610 // The last three blocks are never deallocated or touched. 611 612 // Add MemoryRangeHeader to the end of the memory region, indicating that 613 // the space after the block of memory is allocated. This is block #3. 614 MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; 615 Mem3->ThisAllocated = 1; 616 Mem3->PrevAllocated = 0; 617 Mem3->BlockSize = sizeof(MemoryRangeHeader); 618 619 /// Add a tiny free region so that the free list always has one entry. 620 FreeRangeHeader *Mem2 = 621 (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); 622 Mem2->ThisAllocated = 0; 623 Mem2->PrevAllocated = 1; 624 Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); 625 Mem2->SetEndOfBlockSizeMarker(); 626 Mem2->Prev = Mem2; // Mem2 *is* the free list for now. 627 Mem2->Next = Mem2; 628 629 /// Add a tiny allocated region so that Mem2 is never coalesced away. 630 MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; 631 Mem1->ThisAllocated = 1; 632 Mem1->PrevAllocated = 0; 633 Mem1->BlockSize = sizeof(MemoryRangeHeader); 634 635 // Add a FreeRangeHeader to the start of the function body region, indicating 636 // that the space is free. Mark the previous block allocated so we never look 637 // at it. 638 FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; 639 Mem0->ThisAllocated = 0; 640 Mem0->PrevAllocated = 1; 641 Mem0->BlockSize = (char*)Mem1-(char*)Mem0; 642 Mem0->SetEndOfBlockSizeMarker(); 643 Mem0->AddToFreeList(Mem2); 644 645 // Start out with the freelist pointing to Mem0. 646 FreeMemoryList = Mem0; 647 648 GOTBase = NULL; 649} 650 651void DefaultJITMemoryManager::AllocateGOT() { 652 assert(GOTBase == 0 && "Cannot allocate the got multiple times"); 653 GOTBase = new uint8_t[sizeof(void*) * 8192]; 654 HasGOT = true; 655} 656 657DefaultJITMemoryManager::~DefaultJITMemoryManager() { 658 for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 659 sys::Memory::ReleaseRWX(CodeSlabs[i]); 660 661 delete[] GOTBase; 662} 663 664sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { 665 // Allocate a new block close to the last one. 666 std::string ErrMsg; 667 sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : 0; 668 sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); 669 if (B.base() == 0) { 670 report_fatal_error("Allocation failed when allocating new memory in the" 671 " JIT\n" + Twine(ErrMsg)); 672 } 673 LastSlab = B; 674 ++NumSlabs; 675 // Initialize the slab to garbage when debugging. 676 if (PoisonMemory) { 677 memset(B.base(), 0xCD, B.size()); 678 } 679 return B; 680} 681 682/// CheckInvariants - For testing only. Return "" if all internal invariants 683/// are preserved, and a helpful error message otherwise. For free and 684/// allocated blocks, make sure that adding BlockSize gives a valid block. 685/// For free blocks, make sure they're in the free list and that their end of 686/// block size marker is correct. This function should return an error before 687/// accessing bad memory. This function is defined here instead of in 688/// JITMemoryManagerTest.cpp so that we don't have to expose all of the 689/// implementation details of DefaultJITMemoryManager. 690bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { 691 raw_string_ostream Err(ErrorStr); 692 693 // Construct a the set of FreeRangeHeader pointers so we can query it 694 // efficiently. 695 llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; 696 FreeRangeHeader* FreeHead = FreeMemoryList; 697 FreeRangeHeader* FreeRange = FreeHead; 698 699 do { 700 // Check that the free range pointer is in the blocks we've allocated. 701 bool Found = false; 702 for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 703 E = CodeSlabs.end(); I != E && !Found; ++I) { 704 char *Start = (char*)I->base(); 705 char *End = Start + I->size(); 706 Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); 707 } 708 if (!Found) { 709 Err << "Corrupt free list; points to " << FreeRange; 710 return false; 711 } 712 713 if (FreeRange->Next->Prev != FreeRange) { 714 Err << "Next and Prev pointers do not match."; 715 return false; 716 } 717 718 // Otherwise, add it to the set. 719 FreeHdrSet.insert(FreeRange); 720 FreeRange = FreeRange->Next; 721 } while (FreeRange != FreeHead); 722 723 // Go over each block, and look at each MemoryRangeHeader. 724 for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 725 E = CodeSlabs.end(); I != E; ++I) { 726 char *Start = (char*)I->base(); 727 char *End = Start + I->size(); 728 729 // Check each memory range. 730 for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = NULL; 731 Start <= (char*)Hdr && (char*)Hdr < End; 732 Hdr = &Hdr->getBlockAfter()) { 733 if (Hdr->ThisAllocated == 0) { 734 // Check that this range is in the free list. 735 if (!FreeHdrSet.count(Hdr)) { 736 Err << "Found free header at " << Hdr << " that is not in free list."; 737 return false; 738 } 739 740 // Now make sure the size marker at the end of the block is correct. 741 uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; 742 if (!(Start <= (char*)Marker && (char*)Marker < End)) { 743 Err << "Block size in header points out of current MemoryBlock."; 744 return false; 745 } 746 if (Hdr->BlockSize != *Marker) { 747 Err << "End of block size marker (" << *Marker << ") " 748 << "and BlockSize (" << Hdr->BlockSize << ") don't match."; 749 return false; 750 } 751 } 752 753 if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { 754 Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " 755 << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; 756 return false; 757 } else if (!LastHdr && !Hdr->PrevAllocated) { 758 Err << "The first header should have PrevAllocated true."; 759 return false; 760 } 761 762 // Remember the last header. 763 LastHdr = Hdr; 764 } 765 } 766 767 // All invariants are preserved. 768 return true; 769} 770 771//===----------------------------------------------------------------------===// 772// getPointerToNamedFunction() implementation. 773//===----------------------------------------------------------------------===// 774#include "llvm/Support/ErrorHandling.h" 775#include "llvm/Support/DynamicLibrary.h" 776#include "llvm/Config/config.h" 777 778// AtExitHandlers - List of functions to call when the program exits, 779// registered with the atexit() library function. 780static std::vector<void (*)()> AtExitHandlers; 781 782/// runAtExitHandlers - Run any functions registered by the program's 783/// calls to atexit(3), which we intercept and store in 784/// AtExitHandlers. 785/// 786static void runAtExitHandlers() { 787 while (!AtExitHandlers.empty()) { 788 void (*Fn)() = AtExitHandlers.back(); 789 AtExitHandlers.pop_back(); 790 Fn(); 791 } 792} 793 794//===----------------------------------------------------------------------===// 795// Function stubs that are invoked instead of certain library calls 796//===----------------------------------------------------------------------===// 797 798// Force the following functions to be linked in to anything that uses the 799// JIT. This is a hack designed to work around the all-too-clever Glibc 800// strategy of making these functions work differently when inlined vs. when 801// not inlined, and hiding their real definitions in a separate archive file 802// that the dynamic linker can't see. For more info, search for 803// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274. 804#if defined(__linux__) 805#if defined(HAVE_SYS_STAT_H) 806#include <sys/stat.h> 807#endif 808#include <fcntl.h> 809#include <unistd.h> 810/* stat functions are redirecting to __xstat with a version number. On x86-64 811 * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat' 812 * available as an exported symbol, so we have to add it explicitly. 813 */ 814namespace { 815class StatSymbols { 816public: 817 StatSymbols() { 818 sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat); 819 sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat); 820 sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat); 821 sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64); 822 sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64); 823 sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64); 824 sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64); 825 sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64); 826 sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64); 827 sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit); 828 sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod); 829 } 830}; 831} 832static StatSymbols initStatSymbols; 833#endif // __linux__ 834 835// jit_exit - Used to intercept the "exit" library call. 836static void jit_exit(int Status) { 837 runAtExitHandlers(); // Run atexit handlers... 838 exit(Status); 839} 840 841// jit_atexit - Used to intercept the "atexit" library call. 842static int jit_atexit(void (*Fn)()) { 843 AtExitHandlers.push_back(Fn); // Take note of atexit handler... 844 return 0; // Always successful 845} 846 847static int jit_noop() { 848 return 0; 849} 850 851//===----------------------------------------------------------------------===// 852// 853/// getPointerToNamedFunction - This method returns the address of the specified 854/// function by using the dynamic loader interface. As such it is only useful 855/// for resolving library symbols, not code generated symbols. 856/// 857void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name, 858 bool AbortOnFailure) { 859 // Check to see if this is one of the functions we want to intercept. Note, 860 // we cast to intptr_t here to silence a -pedantic warning that complains 861 // about casting a function pointer to a normal pointer. 862 if (Name == "exit") return (void*)(intptr_t)&jit_exit; 863 if (Name == "atexit") return (void*)(intptr_t)&jit_atexit; 864 865 // We should not invoke parent's ctors/dtors from generated main()! 866 // On Mingw and Cygwin, the symbol __main is resolved to 867 // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors 868 // (and register wrong callee's dtors with atexit(3)). 869 // We expect ExecutionEngine::runStaticConstructorsDestructors() 870 // is called before ExecutionEngine::runFunctionAsMain() is called. 871 if (Name == "__main") return (void*)(intptr_t)&jit_noop; 872 873 const char *NameStr = Name.c_str(); 874 // If this is an asm specifier, skip the sentinal. 875 if (NameStr[0] == 1) ++NameStr; 876 877 // If it's an external function, look it up in the process image... 878 void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); 879 if (Ptr) return Ptr; 880 881 // If it wasn't found and if it starts with an underscore ('_') character, 882 // try again without the underscore. 883 if (NameStr[0] == '_') { 884 Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); 885 if (Ptr) return Ptr; 886 } 887 888 // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These 889 // are references to hidden visibility symbols that dlsym cannot resolve. 890 // If we have one of these, strip off $LDBLStub and try again. 891#if defined(__APPLE__) && defined(__ppc__) 892 if (Name.size() > 9 && Name[Name.size()-9] == '$' && 893 memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) { 894 // First try turning $LDBLStub into $LDBL128. If that fails, strip it off. 895 // This mirrors logic in libSystemStubs.a. 896 std::string Prefix = std::string(Name.begin(), Name.end()-9); 897 if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false)) 898 return Ptr; 899 if (void *Ptr = getPointerToNamedFunction(Prefix, false)) 900 return Ptr; 901 } 902#endif 903 904 if (AbortOnFailure) { 905 report_fatal_error("Program used external function '"+Name+ 906 "' which could not be resolved!"); 907 } 908 return 0; 909} 910 911 912 913JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { 914 return new DefaultJITMemoryManager(); 915} 916 917// Allocate memory for code in 512K slabs. 918const size_t DefaultJITMemoryManager::DefaultCodeSlabSize = 512 * 1024; 919 920// Allocate globals and stubs in slabs of 64K. (probably 16 pages) 921const size_t DefaultJITMemoryManager::DefaultSlabSize = 64 * 1024; 922 923// Waste at most 16K at the end of each bump slab. (probably 4 pages) 924const size_t DefaultJITMemoryManager::DefaultSizeThreshold = 16 * 1024; 925