JITMemoryManager.cpp revision 81ce3ed08c4df0c246b378c8972062d2f49f1ce9
18907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// 28907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 38907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// The LLVM Compiler Infrastructure 48907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 78907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 88907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 98907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// This file defines the DefaultJITMemoryManager class. 118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner#define DEBUG_TYPE "jit" 158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include "llvm/ExecutionEngine/JITMemoryManager.h" 1681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner#include "llvm/ADT/SmallPtrSet.h" 1781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner#include "llvm/ADT/Statistic.h" 1881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner#include "llvm/GlobalValue.h" 1981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner#include "llvm/Support/Allocator.h" 208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include "llvm/Support/Compiler.h" 2181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner#include "llvm/Support/Debug.h" 227d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin#include "llvm/Support/ErrorHandling.h" 2381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner#include "llvm/Support/raw_ostream.h" 248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include "llvm/System/Memory.h" 258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include <map> 268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include <vector> 273012ac63d35bc6dc450f97dbce3e2faaf8859198Chuck Rose III#include <cassert> 28de551f91d8816632a76a065084caab9fab6aacffDan Gohman#include <climits> 294520dd2b7b20af07d5a3e4d06d964a532044eb10Duncan Sands#include <cstdio> 30ae9f3a3b7c915f725aef5a7250e88eaeddda03c6Anton Korobeynikov#include <cstdlib> 31ae9f3a3b7c915f725aef5a7250e88eaeddda03c6Anton Korobeynikov#include <cstring> 328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnerusing namespace llvm; 338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 3481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid KlecknerSTATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); 358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerJITMemoryManager::~JITMemoryManager() {} 378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// Memory Block Implementation. 408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnernamespace { 438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// MemoryRangeHeader - For a range of memory, this is the header that we put 448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// on the block of memory. It is carefully crafted to be one word of memory. 458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader 468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// which starts with this. 478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct FreeRangeHeader; 488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct MemoryRangeHeader { 498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// ThisAllocated - This is true if this block is currently allocated. If 508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// not, this can be converted to a FreeRangeHeader. 518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned ThisAllocated : 1; 528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// PrevAllocated - Keep track of whether the block immediately before us is 548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// allocated. If not, the word immediately before this header is the size 558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// of the previous block. 568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned PrevAllocated : 1; 578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// BlockSize - This is the size in bytes of this memory block, 598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// including this header. 60de551f91d8816632a76a065084caab9fab6aacffDan Gohman uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); 618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 628907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getBlockAfter - Return the memory block immediately after this one. 648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// 658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader &getBlockAfter() const { 668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return *(MemoryRangeHeader*)((char*)this+BlockSize); 678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getFreeBlockBefore - If the block before this one is free, return it, 708907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// otherwise return null. 718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *getFreeBlockBefore() const { 728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (PrevAllocated) return 0; 738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner intptr_t PrevSize = ((intptr_t *)this)[-1]; 748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return (FreeRangeHeader*)((char*)this-PrevSize); 758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// FreeBlock - Turn an allocated block into a free block, adjusting 788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// bits in the object headers, and adding an end of region memory block. 798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); 808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// TrimAllocationToSize - If this allocated block is significantly larger 828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// than NewSize, split it into two pieces (where the former is NewSize 838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// bytes, including the header), and add the new block to the free list. 848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, 858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner uint64_t NewSize); 868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// FreeRangeHeader - For a memory block that isn't already allocated, this 898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// keeps track of the current block and has a pointer to the next free block. 908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Free blocks are kept on a circularly linked list. 918907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct FreeRangeHeader : public MemoryRangeHeader { 928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Prev; 938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Next; 948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getMinBlockSize - Get the minimum size for a memory block. Blocks 968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// smaller than this size cannot be created. 978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner static unsigned getMinBlockSize() { 988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return sizeof(FreeRangeHeader)+sizeof(intptr_t); 998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// SetEndOfBlockSizeMarker - The word at the end of every free block is 1028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// known to be the size of the free block. Set it for this block. 1038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void SetEndOfBlockSizeMarker() { 1048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void *EndOfBlock = (char*)this + BlockSize; 1058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ((intptr_t *)EndOfBlock)[-1] = BlockSize; 1068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *RemoveFromFreeList() { 1098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); 1108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next->Prev = Prev; 1118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return Prev->Next = Next; 1128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void AddToFreeList(FreeRangeHeader *FreeList) { 1158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next = FreeList; 1168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Prev = FreeList->Prev; 1178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Prev->Next = this; 1188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next->Prev = this; 1198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// GrowBlock - The block after this block just got deallocated. Merge it 1228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// into the current block. 1238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void GrowBlock(uintptr_t NewSize); 1248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// AllocateBlock - Mark this entire block allocated, updating freelists 1268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// etc. This returns a pointer to the circular free-list. 1278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *AllocateBlock(); 1288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 1298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// AllocateBlock - Mark this entire block allocated, updating freelists 1338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// etc. This returns a pointer to the circular free-list. 1348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *FreeRangeHeader::AllocateBlock() { 1358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(!ThisAllocated && !getBlockAfter().PrevAllocated && 1368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Cannot allocate an allocated block!"); 1378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Mark this block allocated. 1388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ThisAllocated = 1; 1398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner getBlockAfter().PrevAllocated = 1; 1408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Remove it from the free list. 1428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return RemoveFromFreeList(); 1438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// FreeBlock - Turn an allocated block into a free block, adjusting 1468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// bits in the object headers, and adding an end of region memory block. 1478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// If possible, coalesce this block with neighboring blocks. Return the 1488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// FreeRangeHeader to allocate from. 1498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { 1508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *FollowingBlock = &getBlockAfter(); 15181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner assert(ThisAllocated && "This block is already free!"); 1528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); 1538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *FreeListToReturn = FreeList; 1558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // If the block after this one is free, merge it into this block. 1578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (!FollowingBlock->ThisAllocated) { 1588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; 1598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // "FreeList" always needs to be a valid free block. If we're about to 1608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // coalesce with it, update our notion of what the free list is. 1618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (&FollowingFreeBlock == FreeList) { 1628907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeList = FollowingFreeBlock.Next; 1638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeListToReturn = 0; 1648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); 1658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingFreeBlock.RemoveFromFreeList(); 1678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Include the following block into this one. 1698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize += FollowingFreeBlock.BlockSize; 1708907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock = &FollowingFreeBlock.getBlockAfter(); 1718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Tell the block after the block we are coalescing that this block is 1738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // allocated. 1748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock->PrevAllocated = 1; 1758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); 1788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { 1808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); 1818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; 1828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Otherwise, mark this block free. 1858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; 1868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock->PrevAllocated = 0; 1878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.ThisAllocated = 0; 1888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Link this into the linked list of free blocks. 1908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.AddToFreeList(FreeList); 1918907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add a marker at the end of the block, indicating the size of this free 1938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // block. 1948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.SetEndOfBlockSizeMarker(); 1958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeListToReturn ? FreeListToReturn : &FreeBlock; 1968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// GrowBlock - The block after this block just got deallocated. Merge it 1998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// into the current block. 2008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnervoid FreeRangeHeader::GrowBlock(uintptr_t NewSize) { 2018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(NewSize > BlockSize && "Not growing block?"); 2028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize = NewSize; 2038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner SetEndOfBlockSizeMarker(); 2048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner getBlockAfter().PrevAllocated = 0; 2058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// TrimAllocationToSize - If this allocated block is significantly larger 2088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// than NewSize, split it into two pieces (where the former is NewSize 2098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// bytes, including the header), and add the new block to the free list. 2108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *MemoryRangeHeader:: 2118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerTrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { 2128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(ThisAllocated && getBlockAfter().PrevAllocated && 2138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Cannot deallocate part of an allocated block!"); 2148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 21560b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng // Don't allow blocks to be trimmed below minimum required size 21660b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); 21760b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng 2188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Round up size for alignment of header. 2198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned HeaderAlign = __alignof(FreeRangeHeader); 2208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); 2218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Size is now the size of the block we will remove from the start of the 2238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // current block. 2248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(NewSize <= BlockSize && 2258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Allocating more space from this block than exists!"); 2268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // If splitting this block will cause the remainder to be too small, do not 2288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // split the block. 2298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) 2308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeList; 2318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Otherwise, we splice the required number of bytes out of this block, form 2338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // a new block immediately after it, then mark this block allocated. 2348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader &FormerNextBlock = getBlockAfter(); 2358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Change the size of this block. 2378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize = NewSize; 2388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Get the new block we just sliced out and turn it into a free block. 2408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); 2418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; 2428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.ThisAllocated = 0; 2438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.PrevAllocated = 1; 2448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.SetEndOfBlockSizeMarker(); 2458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FormerNextBlock.PrevAllocated = 0; 2468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.AddToFreeList(FreeList); 2478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return &NewNextBlock; 2488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 2518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// Memory Block Implementation. 2528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 2538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 25481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Klecknernamespace { 25581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 25681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner class DefaultJITMemoryManager; 25781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 25881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner class JITSlabAllocator : public SlabAllocator { 25981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner DefaultJITMemoryManager &JMM; 26081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner public: 26181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner JITSlabAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } 26281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner virtual ~JITSlabAllocator() { } 26381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner virtual MemSlab *Allocate(size_t Size); 26481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner virtual void Deallocate(MemSlab *Slab); 26581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner }; 26681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 2678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// DefaultJITMemoryManager - Manage memory for the JIT code generation. 2688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// This splits a large block of MAP_NORESERVE'd memory into two 2698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// sections, one for function stubs, one for the functions themselves. We 2708907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// have to do this because we may need to emit a function stub while in the 2718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// middle of emitting a function, and we don't know how large the function we 2728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// are emitting is. 27381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner class DefaultJITMemoryManager : public JITMemoryManager { 27481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 27581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Whether to poison freed memory. 27681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner bool PoisonMemory; 27781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 27881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// LastSlab - This points to the last slab allocated and is used as the 27981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all 28081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// stubs, data, and code contiguously in memory. In general, however, this 28181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// is not possible because the NearBlock parameter is ignored on Windows 28281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// platforms and even on Unix it works on a best-effort pasis. 28381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::MemoryBlock LastSlab; 28481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 28581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Memory slabs allocated by the JIT. We refer to them as slabs so we don't 28681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // confuse them with the blocks of memory descibed above. 28781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner std::vector<sys::MemoryBlock> CodeSlabs; 28881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner JITSlabAllocator BumpSlabAllocator; 28981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner BumpPtrAllocator StubAllocator; 29081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner BumpPtrAllocator DataAllocator; 29181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 29281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Circular list of free blocks. 29381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner FreeRangeHeader *FreeMemoryList; 294489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 2958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // When emitting code into a memory block, this is the block. 2968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *CurBlock; 29781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 298186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *GOTBase; // Target Specific reserved memory 299186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void *DlsymTable; // Stub external symbol information 3008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 3018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner std::map<const Function*, MemoryRangeHeader*> FunctionBlocks; 302afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray std::map<const Function*, MemoryRangeHeader*> TableBlocks; 3038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner public: 3048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner DefaultJITMemoryManager(); 3058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ~DefaultJITMemoryManager(); 3068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 30781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the 30881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// last slab it allocated, so that subsequent allocations follow it. 30981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::MemoryBlock allocateNewSlab(size_t size); 31081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 31181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at 31281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// least this much unless more is requested. 31381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner static const size_t DefaultCodeSlabSize; 31481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 31581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// DefaultSlabSize - Allocate data into slabs of this size unless we get 31681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// an allocation above SizeThreshold. 31781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner static const size_t DefaultSlabSize; 31881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 31981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// DefaultSizeThreshold - For any allocation larger than this threshold, we 32081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// should allocate a separate slab. 32181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner static const size_t DefaultSizeThreshold; 32281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 3238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void AllocateGOT(); 324d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman void SetDlsymTable(void *); 32581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 32681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Testing methods. 32781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner virtual bool CheckInvariants(std::string &ErrorStr); 32881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner size_t GetDefaultCodeSlabSize() { return DefaultCodeSlabSize; } 32981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner size_t GetDefaultDataSlabSize() { return DefaultSlabSize; } 33081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner size_t GetDefaultStubSlabSize() { return DefaultSlabSize; } 33181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner unsigned GetNumCodeSlabs() { return CodeSlabs.size(); } 33281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner unsigned GetNumDataSlabs() { return DataAllocator.GetNumSlabs(); } 33381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner unsigned GetNumStubSlabs() { return StubAllocator.GetNumSlabs(); } 33481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 3358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// startFunctionBody - When a function starts, allocate a block of free 3368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// executable memory, returning a pointer to it and its actual size. 337186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *startFunctionBody(const Function *F, uintptr_t &ActualSize) { 33881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 33996c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* candidateBlock = FreeMemoryList; 34096c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* head = FreeMemoryList; 34196c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* iter = head->Next; 34296c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner 34396c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner uintptr_t largest = candidateBlock->BlockSize; 34481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 34596c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner // Search for the largest free block 34696c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner while (iter != head) { 34781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (iter->BlockSize > largest) { 34881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner largest = iter->BlockSize; 34981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner candidateBlock = iter; 35081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 35181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner iter = iter->Next; 35296c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner } 35381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 35481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // If this block isn't big enough for the allocation desired, allocate 35581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // another block of memory and add it to the free list. 35681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (largest - sizeof(MemoryRangeHeader) < ActualSize) { 35781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner DOUT << "JIT: Allocating another slab of memory for function."; 35881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner candidateBlock = allocateNewCodeSlab((size_t)ActualSize); 35981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 36081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 36196c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner // Select this candidate block for allocation 36296c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner CurBlock = candidateBlock; 36396c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner 3648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Allocate the entire memory block. 36596c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeMemoryList = candidateBlock->AllocateBlock(); 36681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); 36781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return (uint8_t *)(CurBlock + 1); 3688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 36981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 37081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// allocateNewCodeSlab - Helper method to allocate a new slab of code 37181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// memory from the OS and add it to the free list. Returns the new 37281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// FreeRangeHeader at the base of the slab. 37381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { 37481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // If the user needs at least MinSize free memory, then we account for 37581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // two MemoryRangeHeaders: the one in the user's block, and the one at the 37681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // end of the slab. 37781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); 37881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); 37981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::MemoryBlock B = allocateNewSlab(SlabSize); 38081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner CodeSlabs.push_back(B); 38181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner char *MemBase = (char*)(B.base()); 38281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 38381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Put a tiny allocated block at the end of the memory chunk, so when 38481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // FreeBlock calls getBlockAfter it doesn't fall off the end. 38581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner MemoryRangeHeader *EndBlock = 38681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner (MemoryRangeHeader*)(MemBase + B.size()) - 1; 38781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner EndBlock->ThisAllocated = 1; 38881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner EndBlock->PrevAllocated = 0; 38981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner EndBlock->BlockSize = sizeof(MemoryRangeHeader); 39081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 39181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Start out with a vast new block of free memory. 39281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; 39381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner NewBlock->ThisAllocated = 0; 39481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Make sure getFreeBlockBefore doesn't look into unmapped memory. 39581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner NewBlock->PrevAllocated = 1; 39681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; 39781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner NewBlock->SetEndOfBlockSizeMarker(); 39881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner NewBlock->AddToFreeList(FreeMemoryList); 39981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 40081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && 40181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner "The block was too small!"); 40281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return NewBlock; 40381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 40481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 4058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// endFunctionBody - The function F is now allocated, and takes the memory 4068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// in the range [FunctionStart,FunctionEnd). 407186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void endFunctionBody(const Function *F, uint8_t *FunctionStart, 408186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *FunctionEnd) { 4098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FunctionEnd > FunctionStart); 410186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes assert(FunctionStart == (uint8_t *)(CurBlock+1) && 4118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Mismatched function start/end!"); 412dd947ea3c5e020c33c58a31939561265b980a3adDale Johannesen 413186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; 4148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FunctionBlocks[F] = CurBlock; 4158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 4168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Release the memory at the end of this block that isn't needed. 4178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 4188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 419cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 42081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// allocateSpace - Allocate a memory block of the given size. This method 42181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// cannot be called between calls to startFunctionBody and endFunctionBody. 422186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) { 423cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes CurBlock = FreeMemoryList; 424cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes FreeMemoryList = FreeMemoryList->AllocateBlock(); 425cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 426489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin uint8_t *result = (uint8_t *)(CurBlock + 1); 427cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 428cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes if (Alignment == 0) Alignment = 1; 429186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes result = (uint8_t*)(((intptr_t)result+Alignment-1) & 430cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes ~(intptr_t)(Alignment-1)); 431cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 432186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; 433cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 434cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 435cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes return result; 436cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes } 437cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 43881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// allocateStub - Allocate memory for a function stub. 43981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, 44081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner unsigned Alignment) { 44181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); 44281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 443489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 44481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner /// allocateGlobal - Allocate memory for a global. 44581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) { 44681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return (uint8_t*)DataAllocator.Allocate(Size, Alignment); 447489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 448489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 449afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// startExceptionTable - Use startFunctionBody to allocate memory for the 450afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// function's exception table. 451186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) { 452afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray return startFunctionBody(F, ActualSize); 453afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray } 454afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 455afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// endExceptionTable - The exception table of F is now allocated, 456afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// and takes the memory in the range [TableStart,TableEnd). 457186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void endExceptionTable(const Function *F, uint8_t *TableStart, 458186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *TableEnd, uint8_t* FrameRegister) { 459afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray assert(TableEnd > TableStart); 460186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes assert(TableStart == (uint8_t *)(CurBlock+1) && 461afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray "Mismatched table start/end!"); 462afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 463186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = TableEnd - (uint8_t *)CurBlock; 464afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray TableBlocks[F] = CurBlock; 465afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 466afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Release the memory at the end of this block that isn't needed. 467afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 468afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray } 469afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 470186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *getGOTBase() const { 4718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return GOTBase; 4728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 4738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 474d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman void *getDlsymTable() const { 475d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman return DlsymTable; 476d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman } 477d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman 4788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// deallocateMemForFunction - Deallocate all memory for the specified 4798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// function body. 4808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void deallocateMemForFunction(const Function *F) { 4818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner std::map<const Function*, MemoryRangeHeader*>::iterator 4828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner I = FunctionBlocks.find(F); 4838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (I == FunctionBlocks.end()) return; 4848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 4858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Find the block that is allocated for this function. 4868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *MemRange = I->second; 4878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(MemRange->ThisAllocated && "Block isn't allocated!"); 488489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 4898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Fill the buffer with garbage! 490489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin if (PoisonMemory) { 491489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); 492489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 493489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 4948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Free the memory. 4958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); 4968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 4978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Finally, remove this entry from FunctionBlocks. 4988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FunctionBlocks.erase(I); 499afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 500afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray I = TableBlocks.find(F); 501afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray if (I == TableBlocks.end()) return; 502afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 503afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Find the block that is allocated for this function. 504afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray MemRange = I->second; 505afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray assert(MemRange->ThisAllocated && "Block isn't allocated!"); 506489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 507afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Fill the buffer with garbage! 508489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin if (PoisonMemory) { 509489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); 510489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 511489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 512afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Free the memory. 513afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); 514afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 515afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Finally, remove this entry from TableBlocks. 516afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray TableBlocks.erase(I); 5178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 518cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach 519cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// setMemoryWritable - When code generation is in progress, 520cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// the code pages may need permissions changed. 521cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach void setMemoryWritable(void) 522cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach { 52381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 52481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::Memory::setWritable(CodeSlabs[i]); 525cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach } 526cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// setMemoryExecutable - When code generation is done and we're ready to 527cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// start execution, the code pages may need permissions changed. 528cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach void setMemoryExecutable(void) 529cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach { 53081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 53181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::Memory::setExecutable(CodeSlabs[i]); 532cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach } 533489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 534489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin /// setPoisonMemory - Controls whether we write garbage over freed memory. 535489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin /// 536489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin void setPoisonMemory(bool poison) { 537489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin PoisonMemory = poison; 538489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 5398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 5408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 5418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 54281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid KlecknerMemSlab *JITSlabAllocator::Allocate(size_t Size) { 54381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::MemoryBlock B = JMM.allocateNewSlab(Size); 54481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner MemSlab *Slab = (MemSlab*)B.base(); 54581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Slab->Size = B.size(); 54681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Slab->NextPtr = 0; 54781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return Slab; 54881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner} 54981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 55081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Klecknervoid JITSlabAllocator::Deallocate(MemSlab *Slab) { 55181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::MemoryBlock B(Slab, Slab->Size); 55281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::Memory::ReleaseRWX(B); 55381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner} 55481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 55581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid KlecknerDefaultJITMemoryManager::DefaultJITMemoryManager() 55681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner : LastSlab(0, 0), 55781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner BumpSlabAllocator(*this), 55881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner StubAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator), 55981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner DataAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator) { 56081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 561489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin#ifdef NDEBUG 562489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin PoisonMemory = false; 563bc4707a2554ac04ba006bf70035e7bc7270236a9Evan Cheng#else 56481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner PoisonMemory = true; 565bc4707a2554ac04ba006bf70035e7bc7270236a9Evan Cheng#endif 5668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 56781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Allocate space for code. 56881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); 56981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner CodeSlabs.push_back(MemBlock); 57081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner uint8_t *MemBase = (uint8_t*)MemBlock.base(); 571489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // We set up the memory chunk with 4 mem regions, like this: 5738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ START 5748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Free #0 ] -> Large space to allocate functions from. 5758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Allocated #1 ] -> Tiny space to separate regions. 5768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Free #2 ] -> Tiny space so there is always at least 1 free block. 5778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. 5788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // END ] 5798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // 5808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // The last three blocks are never deallocated or touched. 5818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 5828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add MemoryRangeHeader to the end of the memory region, indicating that 5838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // the space after the block of memory is allocated. This is block #3. 5848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; 5858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem3->ThisAllocated = 1; 5868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem3->PrevAllocated = 0; 58781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Mem3->BlockSize = sizeof(MemoryRangeHeader); 5888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 5898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Add a tiny free region so that the free list always has one entry. 5908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Mem2 = 5918907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); 5928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->ThisAllocated = 0; 5938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->PrevAllocated = 1; 5948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); 5958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->SetEndOfBlockSizeMarker(); 5968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->Prev = Mem2; // Mem2 *is* the free list for now. 5978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->Next = Mem2; 5988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 5998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Add a tiny allocated region so that Mem2 is never coalesced away. 6008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; 6018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem1->ThisAllocated = 1; 6028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem1->PrevAllocated = 0; 60381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Mem1->BlockSize = sizeof(MemoryRangeHeader); 6048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add a FreeRangeHeader to the start of the function body region, indicating 6068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // that the space is free. Mark the previous block allocated so we never look 6078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // at it. 60881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; 6098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->ThisAllocated = 0; 6108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->PrevAllocated = 1; 6118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->BlockSize = (char*)Mem1-(char*)Mem0; 6128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->SetEndOfBlockSizeMarker(); 6138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->AddToFreeList(Mem2); 6148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Start out with the freelist pointing to Mem0. 6168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList = Mem0; 6178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner GOTBase = NULL; 619d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman DlsymTable = NULL; 6208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnervoid DefaultJITMemoryManager::AllocateGOT() { 6238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(GOTBase == 0 && "Cannot allocate the got multiple times"); 624186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes GOTBase = new uint8_t[sizeof(void*) * 8192]; 6258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner HasGOT = true; 6268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 628d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begemanvoid DefaultJITMemoryManager::SetDlsymTable(void *ptr) { 629d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman DlsymTable = ptr; 630d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman} 6318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerDefaultJITMemoryManager::~DefaultJITMemoryManager() { 63381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 63481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::Memory::ReleaseRWX(CodeSlabs[i]); 6358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 63681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner delete[] GOTBase; 6378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 63981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Klecknersys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { 6408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Allocate a new block close to the last one. 6418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner std::string ErrMsg; 64281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : 0; 64381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); 6448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (B.base() == 0) { 6457d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin llvm_report_error("Allocation failed when allocating new memory in the" 6467d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin " JIT\n" + ErrMsg); 6478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 64881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner LastSlab = B; 64981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner ++NumSlabs; 6508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return B; 6518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 65381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner/// CheckInvariants - For testing only. Return "" if all internal invariants 65481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner/// are preserved, and a helpful error message otherwise. For free and 65581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner/// allocated blocks, make sure that adding BlockSize gives a valid block. 65681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner/// For free blocks, make sure they're in the free list and that their end of 65781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner/// block size marker is correct. This function should return an error before 65881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner/// accessing bad memory. This function is defined here instead of in 65981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner/// JITMemoryManagerTest.cpp so that we don't have to expose all of the 66081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner/// implementation details of DefaultJITMemoryManager. 66181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Klecknerbool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { 66281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner raw_string_ostream Err(ErrorStr); 66381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 66481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Construct a the set of FreeRangeHeader pointers so we can query it 66581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // efficiently. 66681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; 66781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner FreeRangeHeader* FreeHead = FreeMemoryList; 66881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner FreeRangeHeader* FreeRange = FreeHead; 66981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 67081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner do { 67181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Check that the free range pointer is in the blocks we've allocated. 67281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner bool Found = false; 67381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 67481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner E = CodeSlabs.end(); I != E && !Found; ++I) { 67581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner char *Start = (char*)I->base(); 67681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner char *End = Start + I->size(); 67781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); 67881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 67981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (!Found) { 68081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Err << "Corrupt free list; points to " << FreeRange; 68181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return false; 68281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 68381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 68481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (FreeRange->Next->Prev != FreeRange) { 68581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Err << "Next and Prev pointers do not match."; 68681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return false; 68781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 68881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 68981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Otherwise, add it to the set. 69081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner FreeHdrSet.insert(FreeRange); 69181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner FreeRange = FreeRange->Next; 69281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } while (FreeRange != FreeHead); 69381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 69481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Go over each block, and look at each MemoryRangeHeader. 69581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 69681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner E = CodeSlabs.end(); I != E; ++I) { 69781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner char *Start = (char*)I->base(); 69881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner char *End = Start + I->size(); 69981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 70081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Check each memory range. 70181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = NULL; 70281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Start <= (char*)Hdr && (char*)Hdr < End; 70381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Hdr = &Hdr->getBlockAfter()) { 70481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (Hdr->ThisAllocated == 0) { 70581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Check that this range is in the free list. 70681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (!FreeHdrSet.count(Hdr)) { 70781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Err << "Found free header at " << Hdr << " that is not in free list."; 70881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return false; 70981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 71081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 71181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Now make sure the size marker at the end of the block is correct. 71281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; 71381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (!(Start <= (char*)Marker && (char*)Marker < End)) { 71481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Err << "Block size in header points out of current MemoryBlock."; 71581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return false; 71681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 71781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (Hdr->BlockSize != *Marker) { 71881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Err << "End of block size marker (" << *Marker << ") " 71981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner << "and BlockSize (" << Hdr->BlockSize << ") don't match."; 72081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return false; 72181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 72281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 72381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 72481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { 72581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " 72681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; 72781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return false; 72881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } else if (!LastHdr && !Hdr->PrevAllocated) { 72981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner Err << "The first header should have PrevAllocated true."; 73081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return false; 73181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 73281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 73381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // Remember the last header. 73481ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner LastHdr = Hdr; 73581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 73681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 73781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 73881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner // All invariants are preserved. 73981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner return true; 74081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner} 7418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 7428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerJITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { 7438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return new DefaultJITMemoryManager(); 7448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 74581ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 74681ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner// Allocate memory for code in 512K slabs. 74781ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Klecknerconst size_t DefaultJITMemoryManager::DefaultCodeSlabSize = 512 * 1024; 74881ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 74981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner// Allocate globals and stubs in slabs of 64K. (probably 16 pages) 75081ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Klecknerconst size_t DefaultJITMemoryManager::DefaultSlabSize = 64 * 1024; 75181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner 75281ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner// Waste at most 16K at the end of each bump slab. (probably 4 pages) 75381ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Klecknerconst size_t DefaultJITMemoryManager::DefaultSizeThreshold = 16 * 1024; 754