JITMemoryManager.cpp revision 01248e671100fbd6eac6bc3646096dc75ec885d1
18907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// 28907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 38907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// The LLVM Compiler Infrastructure 48907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 78907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 88907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 98907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// This file defines the DefaultJITMemoryManager class. 118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#define DEBUG_TYPE "jit" 154bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner#include "llvm/ExecutionEngine/JITMemoryManager.h" 1610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/ADT/SmallPtrSet.h" 1710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/ADT/Statistic.h" 1810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/GlobalValue.h" 1910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/Support/Allocator.h" 208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include "llvm/Support/Compiler.h" 2110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/Support/Debug.h" 227d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin#include "llvm/Support/ErrorHandling.h" 2310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/Support/raw_ostream.h" 248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include "llvm/System/Memory.h" 258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include <map> 268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include <vector> 273012ac63d35bc6dc450f97dbce3e2faaf8859198Chuck Rose III#include <cassert> 28de551f91d8816632a76a065084caab9fab6aacffDan Gohman#include <climits> 294520dd2b7b20af07d5a3e4d06d964a532044eb10Duncan Sands#include <cstdio> 30ae9f3a3b7c915f725aef5a7250e88eaeddda03c6Anton Korobeynikov#include <cstdlib> 31ae9f3a3b7c915f725aef5a7250e88eaeddda03c6Anton Korobeynikov#include <cstring> 328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnerusing namespace llvm; 338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 3410b4fc552f984dc978298d50c09c97c0764962fcReid KlecknerSTATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); 358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerJITMemoryManager::~JITMemoryManager() {} 378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// Memory Block Implementation. 408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnernamespace { 438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// MemoryRangeHeader - For a range of memory, this is the header that we put 448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// on the block of memory. It is carefully crafted to be one word of memory. 458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader 468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// which starts with this. 478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct FreeRangeHeader; 488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct MemoryRangeHeader { 498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// ThisAllocated - This is true if this block is currently allocated. If 508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// not, this can be converted to a FreeRangeHeader. 518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned ThisAllocated : 1; 528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// PrevAllocated - Keep track of whether the block immediately before us is 548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// allocated. If not, the word immediately before this header is the size 558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// of the previous block. 568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned PrevAllocated : 1; 578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// BlockSize - This is the size in bytes of this memory block, 598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// including this header. 60de551f91d8816632a76a065084caab9fab6aacffDan Gohman uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); 618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 628907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getBlockAfter - Return the memory block immediately after this one. 648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// 658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader &getBlockAfter() const { 668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return *(MemoryRangeHeader*)((char*)this+BlockSize); 678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getFreeBlockBefore - If the block before this one is free, return it, 708907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// otherwise return null. 718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *getFreeBlockBefore() const { 728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (PrevAllocated) return 0; 738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner intptr_t PrevSize = ((intptr_t *)this)[-1]; 748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return (FreeRangeHeader*)((char*)this-PrevSize); 758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// FreeBlock - Turn an allocated block into a free block, adjusting 788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// bits in the object headers, and adding an end of region memory block. 798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); 808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// TrimAllocationToSize - If this allocated block is significantly larger 828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// than NewSize, split it into two pieces (where the former is NewSize 838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// bytes, including the header), and add the new block to the free list. 848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, 858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner uint64_t NewSize); 868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// FreeRangeHeader - For a memory block that isn't already allocated, this 898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// keeps track of the current block and has a pointer to the next free block. 908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Free blocks are kept on a circularly linked list. 918907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct FreeRangeHeader : public MemoryRangeHeader { 928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Prev; 938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Next; 948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getMinBlockSize - Get the minimum size for a memory block. Blocks 968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// smaller than this size cannot be created. 978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner static unsigned getMinBlockSize() { 988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return sizeof(FreeRangeHeader)+sizeof(intptr_t); 998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// SetEndOfBlockSizeMarker - The word at the end of every free block is 1028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// known to be the size of the free block. Set it for this block. 1038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void SetEndOfBlockSizeMarker() { 1048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void *EndOfBlock = (char*)this + BlockSize; 1058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ((intptr_t *)EndOfBlock)[-1] = BlockSize; 1068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *RemoveFromFreeList() { 1098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); 1108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next->Prev = Prev; 1118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return Prev->Next = Next; 1128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void AddToFreeList(FreeRangeHeader *FreeList) { 1158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next = FreeList; 1168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Prev = FreeList->Prev; 1178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Prev->Next = this; 1188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next->Prev = this; 1198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// GrowBlock - The block after this block just got deallocated. Merge it 1228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// into the current block. 1238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void GrowBlock(uintptr_t NewSize); 1248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// AllocateBlock - Mark this entire block allocated, updating freelists 1268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// etc. This returns a pointer to the circular free-list. 1278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *AllocateBlock(); 1288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 1298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// AllocateBlock - Mark this entire block allocated, updating freelists 1338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// etc. This returns a pointer to the circular free-list. 1348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *FreeRangeHeader::AllocateBlock() { 1358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(!ThisAllocated && !getBlockAfter().PrevAllocated && 1368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Cannot allocate an allocated block!"); 1378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Mark this block allocated. 1388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ThisAllocated = 1; 1398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner getBlockAfter().PrevAllocated = 1; 1408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Remove it from the free list. 1428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return RemoveFromFreeList(); 1438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// FreeBlock - Turn an allocated block into a free block, adjusting 1468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// bits in the object headers, and adding an end of region memory block. 1478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// If possible, coalesce this block with neighboring blocks. Return the 1488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// FreeRangeHeader to allocate from. 1498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { 1508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *FollowingBlock = &getBlockAfter(); 15110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner assert(ThisAllocated && "This block is already free!"); 1528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); 1538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *FreeListToReturn = FreeList; 1558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // If the block after this one is free, merge it into this block. 1578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (!FollowingBlock->ThisAllocated) { 1588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; 1598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // "FreeList" always needs to be a valid free block. If we're about to 1608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // coalesce with it, update our notion of what the free list is. 1618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (&FollowingFreeBlock == FreeList) { 1628907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeList = FollowingFreeBlock.Next; 1638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeListToReturn = 0; 1648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); 1658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingFreeBlock.RemoveFromFreeList(); 1678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Include the following block into this one. 1698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize += FollowingFreeBlock.BlockSize; 1708907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock = &FollowingFreeBlock.getBlockAfter(); 1718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Tell the block after the block we are coalescing that this block is 1738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // allocated. 1748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock->PrevAllocated = 1; 1758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); 1788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { 1808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); 1818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; 1828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Otherwise, mark this block free. 1858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; 1868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock->PrevAllocated = 0; 1878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.ThisAllocated = 0; 1888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Link this into the linked list of free blocks. 1908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.AddToFreeList(FreeList); 1918907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add a marker at the end of the block, indicating the size of this free 1938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // block. 1948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.SetEndOfBlockSizeMarker(); 1958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeListToReturn ? FreeListToReturn : &FreeBlock; 1968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// GrowBlock - The block after this block just got deallocated. Merge it 1998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// into the current block. 2008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnervoid FreeRangeHeader::GrowBlock(uintptr_t NewSize) { 2018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(NewSize > BlockSize && "Not growing block?"); 2028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize = NewSize; 2038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner SetEndOfBlockSizeMarker(); 2048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner getBlockAfter().PrevAllocated = 0; 2058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// TrimAllocationToSize - If this allocated block is significantly larger 2088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// than NewSize, split it into two pieces (where the former is NewSize 2098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// bytes, including the header), and add the new block to the free list. 2108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *MemoryRangeHeader:: 2118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerTrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { 2128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(ThisAllocated && getBlockAfter().PrevAllocated && 2138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Cannot deallocate part of an allocated block!"); 2148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 21560b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng // Don't allow blocks to be trimmed below minimum required size 21660b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); 21760b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng 2188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Round up size for alignment of header. 2198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned HeaderAlign = __alignof(FreeRangeHeader); 2208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); 2218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Size is now the size of the block we will remove from the start of the 2238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // current block. 2248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(NewSize <= BlockSize && 2258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Allocating more space from this block than exists!"); 2268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // If splitting this block will cause the remainder to be too small, do not 2288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // split the block. 2298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) 2308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeList; 2318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Otherwise, we splice the required number of bytes out of this block, form 2338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // a new block immediately after it, then mark this block allocated. 2348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader &FormerNextBlock = getBlockAfter(); 2358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Change the size of this block. 2378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize = NewSize; 2388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Get the new block we just sliced out and turn it into a free block. 2408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); 2418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; 2428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.ThisAllocated = 0; 2438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.PrevAllocated = 1; 2448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.SetEndOfBlockSizeMarker(); 2458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FormerNextBlock.PrevAllocated = 0; 2468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.AddToFreeList(FreeList); 2478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return &NewNextBlock; 2488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 2518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// Memory Block Implementation. 2528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 2538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 25410b4fc552f984dc978298d50c09c97c0764962fcReid Klecknernamespace { 25510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 25610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner class DefaultJITMemoryManager; 25710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 25810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner class JITSlabAllocator : public SlabAllocator { 25910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner DefaultJITMemoryManager &JMM; 26010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner public: 26110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner JITSlabAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } 26210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner virtual ~JITSlabAllocator() { } 26310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner virtual MemSlab *Allocate(size_t Size); 26410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner virtual void Deallocate(MemSlab *Slab); 26510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner }; 26610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 2678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// DefaultJITMemoryManager - Manage memory for the JIT code generation. 2688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// This splits a large block of MAP_NORESERVE'd memory into two 2698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// sections, one for function stubs, one for the functions themselves. We 2708907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// have to do this because we may need to emit a function stub while in the 2718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// middle of emitting a function, and we don't know how large the function we 2728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// are emitting is. 27310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner class DefaultJITMemoryManager : public JITMemoryManager { 27410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 27510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Whether to poison freed memory. 27610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner bool PoisonMemory; 27710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 27810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// LastSlab - This points to the last slab allocated and is used as the 27910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all 28010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// stubs, data, and code contiguously in memory. In general, however, this 28110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// is not possible because the NearBlock parameter is ignored on Windows 28210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// platforms and even on Unix it works on a best-effort pasis. 28310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock LastSlab; 28410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 28510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Memory slabs allocated by the JIT. We refer to them as slabs so we don't 28610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // confuse them with the blocks of memory descibed above. 28710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner std::vector<sys::MemoryBlock> CodeSlabs; 28810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner JITSlabAllocator BumpSlabAllocator; 28910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner BumpPtrAllocator StubAllocator; 29010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner BumpPtrAllocator DataAllocator; 29110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 29210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Circular list of free blocks. 29310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *FreeMemoryList; 294489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 2958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // When emitting code into a memory block, this is the block. 2968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *CurBlock; 29710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 298186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *GOTBase; // Target Specific reserved memory 299186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void *DlsymTable; // Stub external symbol information 3008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 3018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner std::map<const Function*, MemoryRangeHeader*> FunctionBlocks; 302afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray std::map<const Function*, MemoryRangeHeader*> TableBlocks; 3038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner public: 3048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner DefaultJITMemoryManager(); 3058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ~DefaultJITMemoryManager(); 3068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 30710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the 30810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// last slab it allocated, so that subsequent allocations follow it. 30910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock allocateNewSlab(size_t size); 31010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 31110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at 31210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// least this much unless more is requested. 31310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner static const size_t DefaultCodeSlabSize; 31410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 31510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// DefaultSlabSize - Allocate data into slabs of this size unless we get 31610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// an allocation above SizeThreshold. 31710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner static const size_t DefaultSlabSize; 31810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 31910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// DefaultSizeThreshold - For any allocation larger than this threshold, we 32010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// should allocate a separate slab. 32110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner static const size_t DefaultSizeThreshold; 32210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 3238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void AllocateGOT(); 324d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman void SetDlsymTable(void *); 32510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 32610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Testing methods. 32710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner virtual bool CheckInvariants(std::string &ErrorStr); 32810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t GetDefaultCodeSlabSize() { return DefaultCodeSlabSize; } 32910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t GetDefaultDataSlabSize() { return DefaultSlabSize; } 33010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t GetDefaultStubSlabSize() { return DefaultSlabSize; } 33110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner unsigned GetNumCodeSlabs() { return CodeSlabs.size(); } 33210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner unsigned GetNumDataSlabs() { return DataAllocator.GetNumSlabs(); } 33310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner unsigned GetNumStubSlabs() { return StubAllocator.GetNumSlabs(); } 33410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 3358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// startFunctionBody - When a function starts, allocate a block of free 3368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// executable memory, returning a pointer to it and its actual size. 337186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *startFunctionBody(const Function *F, uintptr_t &ActualSize) { 33810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 33996c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* candidateBlock = FreeMemoryList; 34096c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* head = FreeMemoryList; 34196c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* iter = head->Next; 34296c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner 34396c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner uintptr_t largest = candidateBlock->BlockSize; 34410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 34596c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner // Search for the largest free block 34696c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner while (iter != head) { 34710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (iter->BlockSize > largest) { 34810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner largest = iter->BlockSize; 34910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner candidateBlock = iter; 35010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 35110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner iter = iter->Next; 35296c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner } 35310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 3541b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray largest = largest - sizeof(MemoryRangeHeader); 3551b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray 35610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // If this block isn't big enough for the allocation desired, allocate 35710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // another block of memory and add it to the free list. 3581b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray if (largest < ActualSize || 3591b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray largest <= FreeRangeHeader::getMinBlockSize()) { 36010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner DOUT << "JIT: Allocating another slab of memory for function."; 36110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner candidateBlock = allocateNewCodeSlab((size_t)ActualSize); 36210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 36310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 36496c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner // Select this candidate block for allocation 36596c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner CurBlock = candidateBlock; 36696c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner 3678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Allocate the entire memory block. 36896c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeMemoryList = candidateBlock->AllocateBlock(); 36910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); 37010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t *)(CurBlock + 1); 37181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 37210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 37310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateNewCodeSlab - Helper method to allocate a new slab of code 37410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// memory from the OS and add it to the free list. Returns the new 37510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// FreeRangeHeader at the base of the slab. 37610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { 37710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // If the user needs at least MinSize free memory, then we account for 37810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // two MemoryRangeHeaders: the one in the user's block, and the one at the 37910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // end of the slab. 38010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); 38110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); 38210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = allocateNewSlab(SlabSize); 38310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner CodeSlabs.push_back(B); 38410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *MemBase = (char*)(B.base()); 38510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 38610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Put a tiny allocated block at the end of the memory chunk, so when 38710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // FreeBlock calls getBlockAfter it doesn't fall off the end. 38810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner MemoryRangeHeader *EndBlock = 38910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner (MemoryRangeHeader*)(MemBase + B.size()) - 1; 39010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->ThisAllocated = 1; 39110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->PrevAllocated = 0; 39210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->BlockSize = sizeof(MemoryRangeHeader); 39310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 39410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Start out with a vast new block of free memory. 39510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; 39610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->ThisAllocated = 0; 39710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Make sure getFreeBlockBefore doesn't look into unmapped memory. 39810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->PrevAllocated = 1; 39910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; 40010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->SetEndOfBlockSizeMarker(); 40110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->AddToFreeList(FreeMemoryList); 40210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 40310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && 40410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner "The block was too small!"); 40510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return NewBlock; 40610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 40710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 4088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// endFunctionBody - The function F is now allocated, and takes the memory 4098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// in the range [FunctionStart,FunctionEnd). 410186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void endFunctionBody(const Function *F, uint8_t *FunctionStart, 411186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *FunctionEnd) { 4128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FunctionEnd > FunctionStart); 413186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes assert(FunctionStart == (uint8_t *)(CurBlock+1) && 4148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Mismatched function start/end!"); 415dd947ea3c5e020c33c58a31939561265b980a3adDale Johannesen 416186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; 4178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FunctionBlocks[F] = CurBlock; 4188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 4198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Release the memory at the end of this block that isn't needed. 4208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 4218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 422cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 42310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateSpace - Allocate a memory block of the given size. This method 42410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// cannot be called between calls to startFunctionBody and endFunctionBody. 425186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) { 426cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes CurBlock = FreeMemoryList; 427cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes FreeMemoryList = FreeMemoryList->AllocateBlock(); 428cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 429489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin uint8_t *result = (uint8_t *)(CurBlock + 1); 430cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 431cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes if (Alignment == 0) Alignment = 1; 432186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes result = (uint8_t*)(((intptr_t)result+Alignment-1) & 433cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes ~(intptr_t)(Alignment-1)); 434cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 435186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; 436cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 437cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 438cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes return result; 439cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes } 440cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 44110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateStub - Allocate memory for a function stub. 44210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, 44310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner unsigned Alignment) { 44410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); 44510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 4464bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner 44710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateGlobal - Allocate memory for a global. 44810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) { 44910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t*)DataAllocator.Allocate(Size, Alignment); 450489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 451489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 452afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// startExceptionTable - Use startFunctionBody to allocate memory for the 453afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// function's exception table. 454186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) { 455afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray return startFunctionBody(F, ActualSize); 456afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray } 457afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 458afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// endExceptionTable - The exception table of F is now allocated, 459afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// and takes the memory in the range [TableStart,TableEnd). 460186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void endExceptionTable(const Function *F, uint8_t *TableStart, 461186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *TableEnd, uint8_t* FrameRegister) { 462afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray assert(TableEnd > TableStart); 463186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes assert(TableStart == (uint8_t *)(CurBlock+1) && 464afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray "Mismatched table start/end!"); 465afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 466186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = TableEnd - (uint8_t *)CurBlock; 467afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray TableBlocks[F] = CurBlock; 468afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 469afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Release the memory at the end of this block that isn't needed. 470afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 471afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray } 472afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 473186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *getGOTBase() const { 4748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return GOTBase; 4758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 4768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 477d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman void *getDlsymTable() const { 478d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman return DlsymTable; 479d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman } 480d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman 4818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// deallocateMemForFunction - Deallocate all memory for the specified 4828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// function body. 4838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void deallocateMemForFunction(const Function *F) { 4848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner std::map<const Function*, MemoryRangeHeader*>::iterator 4858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner I = FunctionBlocks.find(F); 4868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (I == FunctionBlocks.end()) return; 4878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 4888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Find the block that is allocated for this function. 4898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *MemRange = I->second; 4908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(MemRange->ThisAllocated && "Block isn't allocated!"); 491489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 4928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Fill the buffer with garbage! 493489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin if (PoisonMemory) { 494489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); 495489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 496489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 4978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Free the memory. 4988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); 4998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 5008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Finally, remove this entry from FunctionBlocks. 5018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FunctionBlocks.erase(I); 502afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 503afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray I = TableBlocks.find(F); 504afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray if (I == TableBlocks.end()) return; 505afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 506afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Find the block that is allocated for this function. 507afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray MemRange = I->second; 508afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray assert(MemRange->ThisAllocated && "Block isn't allocated!"); 509489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 510afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Fill the buffer with garbage! 511489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin if (PoisonMemory) { 512489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); 513489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 514489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 515afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Free the memory. 516afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); 517afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 518afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Finally, remove this entry from TableBlocks. 519afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray TableBlocks.erase(I); 5208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 521cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach 522cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// setMemoryWritable - When code generation is in progress, 523cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// the code pages may need permissions changed. 524a9ad04191cb56c42944b17980b8b2bb2afe11ab2Dan Gohman void setMemoryWritable() 525cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach { 52610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 52710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::setWritable(CodeSlabs[i]); 528cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach } 529cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// setMemoryExecutable - When code generation is done and we're ready to 530cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// start execution, the code pages may need permissions changed. 531a9ad04191cb56c42944b17980b8b2bb2afe11ab2Dan Gohman void setMemoryExecutable() 532cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach { 53310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 53410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::setExecutable(CodeSlabs[i]); 535cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach } 536489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 537489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin /// setPoisonMemory - Controls whether we write garbage over freed memory. 538489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin /// 539489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin void setPoisonMemory(bool poison) { 540489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin PoisonMemory = poison; 541489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 5428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 5438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 5448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 54510b4fc552f984dc978298d50c09c97c0764962fcReid KlecknerMemSlab *JITSlabAllocator::Allocate(size_t Size) { 54610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = JMM.allocateNewSlab(Size); 54710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner MemSlab *Slab = (MemSlab*)B.base(); 54810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Slab->Size = B.size(); 54910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Slab->NextPtr = 0; 55010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return Slab; 55110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 55210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 55310b4fc552f984dc978298d50c09c97c0764962fcReid Klecknervoid JITSlabAllocator::Deallocate(MemSlab *Slab) { 55410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B(Slab, Slab->Size); 55510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::ReleaseRWX(B); 55610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 55710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 55810b4fc552f984dc978298d50c09c97c0764962fcReid KlecknerDefaultJITMemoryManager::DefaultJITMemoryManager() 55910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner : LastSlab(0, 0), 56010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner BumpSlabAllocator(*this), 56110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner StubAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator), 56210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner DataAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator) { 56310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 564489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin#ifdef NDEBUG 565489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin PoisonMemory = false; 566bc4707a2554ac04ba006bf70035e7bc7270236a9Evan Cheng#else 56710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner PoisonMemory = true; 568bc4707a2554ac04ba006bf70035e7bc7270236a9Evan Cheng#endif 5698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 57010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Allocate space for code. 57110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); 57210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner CodeSlabs.push_back(MemBlock); 57310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uint8_t *MemBase = (uint8_t*)MemBlock.base(); 574489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // We set up the memory chunk with 4 mem regions, like this: 5768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ START 5778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Free #0 ] -> Large space to allocate functions from. 5788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Allocated #1 ] -> Tiny space to separate regions. 5798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Free #2 ] -> Tiny space so there is always at least 1 free block. 5808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. 5818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // END ] 5828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // 5838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // The last three blocks are never deallocated or touched. 5848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 5858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add MemoryRangeHeader to the end of the memory region, indicating that 5868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // the space after the block of memory is allocated. This is block #3. 5878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; 5888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem3->ThisAllocated = 1; 5898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem3->PrevAllocated = 0; 59010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Mem3->BlockSize = sizeof(MemoryRangeHeader); 5918907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 5928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Add a tiny free region so that the free list always has one entry. 5938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Mem2 = 5948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); 5958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->ThisAllocated = 0; 5968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->PrevAllocated = 1; 5978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); 5988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->SetEndOfBlockSizeMarker(); 5998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->Prev = Mem2; // Mem2 *is* the free list for now. 6008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->Next = Mem2; 6018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Add a tiny allocated region so that Mem2 is never coalesced away. 6038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; 6048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem1->ThisAllocated = 1; 6058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem1->PrevAllocated = 0; 60610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Mem1->BlockSize = sizeof(MemoryRangeHeader); 6078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add a FreeRangeHeader to the start of the function body region, indicating 6098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // that the space is free. Mark the previous block allocated so we never look 6108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // at it. 61110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; 6128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->ThisAllocated = 0; 6138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->PrevAllocated = 1; 6148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->BlockSize = (char*)Mem1-(char*)Mem0; 6158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->SetEndOfBlockSizeMarker(); 6168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->AddToFreeList(Mem2); 6178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Start out with the freelist pointing to Mem0. 6198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList = Mem0; 6208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner GOTBase = NULL; 622d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman DlsymTable = NULL; 6238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnervoid DefaultJITMemoryManager::AllocateGOT() { 6268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(GOTBase == 0 && "Cannot allocate the got multiple times"); 627186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes GOTBase = new uint8_t[sizeof(void*) * 8192]; 6288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner HasGOT = true; 6298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 631d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begemanvoid DefaultJITMemoryManager::SetDlsymTable(void *ptr) { 632d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman DlsymTable = ptr; 633d6b7a242d345fd79a337afd384bb586c5619cfe7Nate Begeman} 6348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerDefaultJITMemoryManager::~DefaultJITMemoryManager() { 63610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 63710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::ReleaseRWX(CodeSlabs[i]); 6388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 63910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner delete[] GOTBase; 6404bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner} 6414bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner 64210b4fc552f984dc978298d50c09c97c0764962fcReid Klecknersys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { 6438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Allocate a new block close to the last one. 6448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner std::string ErrMsg; 64510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : 0; 64610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); 6478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (B.base() == 0) { 6487d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin llvm_report_error("Allocation failed when allocating new memory in the" 6497d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin " JIT\n" + ErrMsg); 6508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 65110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner LastSlab = B; 65210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner ++NumSlabs; 65301248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner // Initialize the slab to garbage when debugging. 65401248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner if (PoisonMemory) { 65501248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner memset(B.base(), 0xCD, B.size()); 65601248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner } 6578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return B; 6588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 66010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// CheckInvariants - For testing only. Return "" if all internal invariants 66110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// are preserved, and a helpful error message otherwise. For free and 66210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// allocated blocks, make sure that adding BlockSize gives a valid block. 66310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// For free blocks, make sure they're in the free list and that their end of 66410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// block size marker is correct. This function should return an error before 66510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// accessing bad memory. This function is defined here instead of in 66610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// JITMemoryManagerTest.cpp so that we don't have to expose all of the 66710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// implementation details of DefaultJITMemoryManager. 66810b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerbool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { 66910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner raw_string_ostream Err(ErrorStr); 67010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 67110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Construct a the set of FreeRangeHeader pointers so we can query it 67210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // efficiently. 67310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; 67410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader* FreeHead = FreeMemoryList; 67510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader* FreeRange = FreeHead; 67610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 67710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner do { 67810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check that the free range pointer is in the blocks we've allocated. 67910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner bool Found = false; 68010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 68110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner E = CodeSlabs.end(); I != E && !Found; ++I) { 68210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *Start = (char*)I->base(); 68310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *End = Start + I->size(); 68410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); 68510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 68610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!Found) { 68710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Corrupt free list; points to " << FreeRange; 68810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 68910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 69010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 69110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (FreeRange->Next->Prev != FreeRange) { 69210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Next and Prev pointers do not match."; 69310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 69410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 69510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 69610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Otherwise, add it to the set. 69710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeHdrSet.insert(FreeRange); 69810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRange = FreeRange->Next; 69910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } while (FreeRange != FreeHead); 70010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 70110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Go over each block, and look at each MemoryRangeHeader. 70210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 70310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner E = CodeSlabs.end(); I != E; ++I) { 70410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *Start = (char*)I->base(); 70510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *End = Start + I->size(); 70610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 70710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check each memory range. 70810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = NULL; 70910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Start <= (char*)Hdr && (char*)Hdr < End; 71010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Hdr = &Hdr->getBlockAfter()) { 71110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (Hdr->ThisAllocated == 0) { 71210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check that this range is in the free list. 71310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!FreeHdrSet.count(Hdr)) { 71410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Found free header at " << Hdr << " that is not in free list."; 71510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 71610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 71710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 71810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Now make sure the size marker at the end of the block is correct. 71910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; 72010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!(Start <= (char*)Marker && (char*)Marker < End)) { 72110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Block size in header points out of current MemoryBlock."; 72210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 72310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 72410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (Hdr->BlockSize != *Marker) { 72510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "End of block size marker (" << *Marker << ") " 72610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner << "and BlockSize (" << Hdr->BlockSize << ") don't match."; 72710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 72810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 72910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 73010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 73110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { 73210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " 73310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; 73410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 73510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } else if (!LastHdr && !Hdr->PrevAllocated) { 73610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "The first header should have PrevAllocated true."; 73710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 73810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 73910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 74010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Remember the last header. 74110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner LastHdr = Hdr; 74210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 74310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 74410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 74510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // All invariants are preserved. 74610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return true; 74710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 7488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 7498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerJITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { 7508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return new DefaultJITMemoryManager(); 7518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 75210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 75310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner// Allocate memory for code in 512K slabs. 75410b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerconst size_t DefaultJITMemoryManager::DefaultCodeSlabSize = 512 * 1024; 75510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 75610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner// Allocate globals and stubs in slabs of 64K. (probably 16 pages) 75710b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerconst size_t DefaultJITMemoryManager::DefaultSlabSize = 64 * 1024; 75810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 75910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner// Waste at most 16K at the end of each bump slab. (probably 4 pages) 76010b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerconst size_t DefaultJITMemoryManager::DefaultSizeThreshold = 16 * 1024; 761