18907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// 28907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 38907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// The LLVM Compiler Infrastructure 48907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 78907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 88907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 98907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// This file defines the DefaultJITMemoryManager class. 118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#define DEBUG_TYPE "jit" 154bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner#include "llvm/ExecutionEngine/JITMemoryManager.h" 1610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/ADT/SmallPtrSet.h" 1710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/ADT/Statistic.h" 181bd7335a17010bd4d8f86736cf73cac9f3fb80a5Benjamin Kramer#include "llvm/ADT/Twine.h" 1910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/GlobalValue.h" 2010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/Support/Allocator.h" 218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include "llvm/Support/Compiler.h" 2210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/Support/Debug.h" 237d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin#include "llvm/Support/ErrorHandling.h" 2410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/Support/raw_ostream.h" 251f6efa3996dd1929fbc129203ce5009b620e6969Michael J. Spencer#include "llvm/Support/Memory.h" 2630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include "llvm/Support/ErrorHandling.h" 2730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include "llvm/Support/DynamicLibrary.h" 2830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include "llvm/Config/config.h" 298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include <vector> 303012ac63d35bc6dc450f97dbce3e2faaf8859198Chuck Rose III#include <cassert> 31de551f91d8816632a76a065084caab9fab6aacffDan Gohman#include <climits> 32ae9f3a3b7c915f725aef5a7250e88eaeddda03c6Anton Korobeynikov#include <cstring> 3330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 3430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#if defined(__linux__) 3530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#if defined(HAVE_SYS_STAT_H) 3630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include <sys/stat.h> 3730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#endif 3830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include <fcntl.h> 3930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include <unistd.h> 4030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#endif 4130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnerusing namespace llvm; 438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 4410b4fc552f984dc978298d50c09c97c0764962fcReid KlecknerSTATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); 458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerJITMemoryManager::~JITMemoryManager() {} 478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// Memory Block Implementation. 508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnernamespace { 538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// MemoryRangeHeader - For a range of memory, this is the header that we put 548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// on the block of memory. It is carefully crafted to be one word of memory. 558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader 568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// which starts with this. 578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct FreeRangeHeader; 588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct MemoryRangeHeader { 598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// ThisAllocated - This is true if this block is currently allocated. If 608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// not, this can be converted to a FreeRangeHeader. 618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned ThisAllocated : 1; 625cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// PrevAllocated - Keep track of whether the block immediately before us is 648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// allocated. If not, the word immediately before this header is the size 658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// of the previous block. 668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned PrevAllocated : 1; 675cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// BlockSize - This is the size in bytes of this memory block, 698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// including this header. 70de551f91d8816632a76a065084caab9fab6aacffDan Gohman uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); 715cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getBlockAfter - Return the memory block immediately after this one. 748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// 758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader &getBlockAfter() const { 768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return *(MemoryRangeHeader*)((char*)this+BlockSize); 778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 785cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getFreeBlockBefore - If the block before this one is free, return it, 808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// otherwise return null. 818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *getFreeBlockBefore() const { 828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (PrevAllocated) return 0; 838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner intptr_t PrevSize = ((intptr_t *)this)[-1]; 848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return (FreeRangeHeader*)((char*)this-PrevSize); 858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 865cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// FreeBlock - Turn an allocated block into a free block, adjusting 888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// bits in the object headers, and adding an end of region memory block. 898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); 905cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 918907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// TrimAllocationToSize - If this allocated block is significantly larger 928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// than NewSize, split it into two pieces (where the former is NewSize 938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// bytes, including the header), and add the new block to the free list. 945cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, 958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner uint64_t NewSize); 968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// FreeRangeHeader - For a memory block that isn't already allocated, this 998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// keeps track of the current block and has a pointer to the next free block. 1008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Free blocks are kept on a circularly linked list. 1018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct FreeRangeHeader : public MemoryRangeHeader { 1028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Prev; 1038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Next; 1045cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getMinBlockSize - Get the minimum size for a memory block. Blocks 1068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// smaller than this size cannot be created. 1078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner static unsigned getMinBlockSize() { 1088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return sizeof(FreeRangeHeader)+sizeof(intptr_t); 1098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1105cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// SetEndOfBlockSizeMarker - The word at the end of every free block is 1128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// known to be the size of the free block. Set it for this block. 1138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void SetEndOfBlockSizeMarker() { 1148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void *EndOfBlock = (char*)this + BlockSize; 1158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ((intptr_t *)EndOfBlock)[-1] = BlockSize; 1168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *RemoveFromFreeList() { 1198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); 1208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next->Prev = Prev; 1218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return Prev->Next = Next; 1228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1235cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void AddToFreeList(FreeRangeHeader *FreeList) { 1258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next = FreeList; 1268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Prev = FreeList->Prev; 1278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Prev->Next = this; 1288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next->Prev = this; 1298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// GrowBlock - The block after this block just got deallocated. Merge it 1328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// into the current block. 1338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void GrowBlock(uintptr_t NewSize); 1345cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// AllocateBlock - Mark this entire block allocated, updating freelists 1368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// etc. This returns a pointer to the circular free-list. 1378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *AllocateBlock(); 1388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 1398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// AllocateBlock - Mark this entire block allocated, updating freelists 1438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// etc. This returns a pointer to the circular free-list. 1448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *FreeRangeHeader::AllocateBlock() { 1458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(!ThisAllocated && !getBlockAfter().PrevAllocated && 1468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Cannot allocate an allocated block!"); 1478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Mark this block allocated. 1488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ThisAllocated = 1; 1498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner getBlockAfter().PrevAllocated = 1; 1505cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Remove it from the free list. 1528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return RemoveFromFreeList(); 1538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// FreeBlock - Turn an allocated block into a free block, adjusting 1568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// bits in the object headers, and adding an end of region memory block. 1578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// If possible, coalesce this block with neighboring blocks. Return the 1588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// FreeRangeHeader to allocate from. 1598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { 1608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *FollowingBlock = &getBlockAfter(); 16110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner assert(ThisAllocated && "This block is already free!"); 1628907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); 1635cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *FreeListToReturn = FreeList; 1655cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // If the block after this one is free, merge it into this block. 1678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (!FollowingBlock->ThisAllocated) { 1688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; 1698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // "FreeList" always needs to be a valid free block. If we're about to 1708907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // coalesce with it, update our notion of what the free list is. 1718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (&FollowingFreeBlock == FreeList) { 1728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeList = FollowingFreeBlock.Next; 1738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeListToReturn = 0; 1748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); 1758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingFreeBlock.RemoveFromFreeList(); 1775cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Include the following block into this one. 1798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize += FollowingFreeBlock.BlockSize; 1808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock = &FollowingFreeBlock.getBlockAfter(); 1815cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Tell the block after the block we are coalescing that this block is 1838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // allocated. 1848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock->PrevAllocated = 1; 1858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1865cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); 1885cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { 1908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); 1918907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; 1928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Otherwise, mark this block free. 1958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; 1968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock->PrevAllocated = 0; 1978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.ThisAllocated = 0; 1988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Link this into the linked list of free blocks. 2008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.AddToFreeList(FreeList); 2018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add a marker at the end of the block, indicating the size of this free 2038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // block. 2048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.SetEndOfBlockSizeMarker(); 2058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeListToReturn ? FreeListToReturn : &FreeBlock; 2068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// GrowBlock - The block after this block just got deallocated. Merge it 2098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// into the current block. 2108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnervoid FreeRangeHeader::GrowBlock(uintptr_t NewSize) { 2118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(NewSize > BlockSize && "Not growing block?"); 2128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize = NewSize; 2138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner SetEndOfBlockSizeMarker(); 2148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner getBlockAfter().PrevAllocated = 0; 2158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// TrimAllocationToSize - If this allocated block is significantly larger 2188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// than NewSize, split it into two pieces (where the former is NewSize 2198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// bytes, including the header), and add the new block to the free list. 2208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *MemoryRangeHeader:: 2218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerTrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { 2228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(ThisAllocated && getBlockAfter().PrevAllocated && 2238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Cannot deallocate part of an allocated block!"); 2248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 22560b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng // Don't allow blocks to be trimmed below minimum required size 22660b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); 22760b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng 2288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Round up size for alignment of header. 2298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned HeaderAlign = __alignof(FreeRangeHeader); 2308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); 2315cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Size is now the size of the block we will remove from the start of the 2338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // current block. 2348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(NewSize <= BlockSize && 2358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Allocating more space from this block than exists!"); 2365cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // If splitting this block will cause the remainder to be too small, do not 2388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // split the block. 2398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) 2408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeList; 2415cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Otherwise, we splice the required number of bytes out of this block, form 2438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // a new block immediately after it, then mark this block allocated. 2448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader &FormerNextBlock = getBlockAfter(); 2455cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Change the size of this block. 2478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize = NewSize; 2485cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Get the new block we just sliced out and turn it into a free block. 2508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); 2518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; 2528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.ThisAllocated = 0; 2538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.PrevAllocated = 1; 2548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.SetEndOfBlockSizeMarker(); 2558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FormerNextBlock.PrevAllocated = 0; 2568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.AddToFreeList(FreeList); 2578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return &NewNextBlock; 2588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 2618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// Memory Block Implementation. 2628907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 2638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 26410b4fc552f984dc978298d50c09c97c0764962fcReid Klecknernamespace { 26510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 26610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner class DefaultJITMemoryManager; 26710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 26810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner class JITSlabAllocator : public SlabAllocator { 26910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner DefaultJITMemoryManager &JMM; 27010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner public: 27110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner JITSlabAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } 27210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner virtual ~JITSlabAllocator() { } 27310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner virtual MemSlab *Allocate(size_t Size); 27410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner virtual void Deallocate(MemSlab *Slab); 27510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner }; 27610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 2778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// DefaultJITMemoryManager - Manage memory for the JIT code generation. 2788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// This splits a large block of MAP_NORESERVE'd memory into two 2798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// sections, one for function stubs, one for the functions themselves. We 2808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// have to do this because we may need to emit a function stub while in the 2818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// middle of emitting a function, and we don't know how large the function we 2828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// are emitting is. 28310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner class DefaultJITMemoryManager : public JITMemoryManager { 28410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 28510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Whether to poison freed memory. 28610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner bool PoisonMemory; 28710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 28810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// LastSlab - This points to the last slab allocated and is used as the 28910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all 29010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// stubs, data, and code contiguously in memory. In general, however, this 29110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// is not possible because the NearBlock parameter is ignored on Windows 29210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// platforms and even on Unix it works on a best-effort pasis. 29310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock LastSlab; 29410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 29510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Memory slabs allocated by the JIT. We refer to them as slabs so we don't 2965cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher // confuse them with the blocks of memory described above. 29710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner std::vector<sys::MemoryBlock> CodeSlabs; 29810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner JITSlabAllocator BumpSlabAllocator; 29910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner BumpPtrAllocator StubAllocator; 30010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner BumpPtrAllocator DataAllocator; 30110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 30210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Circular list of free blocks. 30310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *FreeMemoryList; 304489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 3058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // When emitting code into a memory block, this is the block. 3068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *CurBlock; 30710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 308186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *GOTBase; // Target Specific reserved memory 3098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner public: 3108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner DefaultJITMemoryManager(); 3118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ~DefaultJITMemoryManager(); 3128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 31310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the 31410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// last slab it allocated, so that subsequent allocations follow it. 31510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock allocateNewSlab(size_t size); 31610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 31710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at 31810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// least this much unless more is requested. 31910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner static const size_t DefaultCodeSlabSize; 32010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 32110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// DefaultSlabSize - Allocate data into slabs of this size unless we get 32210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// an allocation above SizeThreshold. 32310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner static const size_t DefaultSlabSize; 32410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 32510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// DefaultSizeThreshold - For any allocation larger than this threshold, we 32610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// should allocate a separate slab. 32710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner static const size_t DefaultSizeThreshold; 32810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 32930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev /// getPointerToNamedFunction - This method returns the address of the 33030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev /// specified function by using the dlsym function call. 33130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev virtual void *getPointerToNamedFunction(const std::string &Name, 33230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev bool AbortOnFailure = true); 33330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 3348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void AllocateGOT(); 33510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 33610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Testing methods. 33710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner virtual bool CheckInvariants(std::string &ErrorStr); 33810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t GetDefaultCodeSlabSize() { return DefaultCodeSlabSize; } 33910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t GetDefaultDataSlabSize() { return DefaultSlabSize; } 34010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t GetDefaultStubSlabSize() { return DefaultSlabSize; } 34110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner unsigned GetNumCodeSlabs() { return CodeSlabs.size(); } 34210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner unsigned GetNumDataSlabs() { return DataAllocator.GetNumSlabs(); } 34310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner unsigned GetNumStubSlabs() { return StubAllocator.GetNumSlabs(); } 34410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 3458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// startFunctionBody - When a function starts, allocate a block of free 3468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// executable memory, returning a pointer to it and its actual size. 347186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *startFunctionBody(const Function *F, uintptr_t &ActualSize) { 34810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 34996c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* candidateBlock = FreeMemoryList; 35096c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* head = FreeMemoryList; 35196c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* iter = head->Next; 35296c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner 35396c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner uintptr_t largest = candidateBlock->BlockSize; 35410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 35596c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner // Search for the largest free block 35696c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner while (iter != head) { 35710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (iter->BlockSize > largest) { 35810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner largest = iter->BlockSize; 35910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner candidateBlock = iter; 36010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 36110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner iter = iter->Next; 36296c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner } 36310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 3641b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray largest = largest - sizeof(MemoryRangeHeader); 3655cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 36610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // If this block isn't big enough for the allocation desired, allocate 36710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // another block of memory and add it to the free list. 3681b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray if (largest < ActualSize || 3691b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray largest <= FreeRangeHeader::getMinBlockSize()) { 370bd8c8afe1b5439792d985da7c6e5949d8dc3f552David Greene DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); 37110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner candidateBlock = allocateNewCodeSlab((size_t)ActualSize); 37210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 37310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 37496c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner // Select this candidate block for allocation 37596c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner CurBlock = candidateBlock; 37696c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner 3778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Allocate the entire memory block. 37896c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeMemoryList = candidateBlock->AllocateBlock(); 37910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); 38010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t *)(CurBlock + 1); 38181ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 38210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 38310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateNewCodeSlab - Helper method to allocate a new slab of code 38410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// memory from the OS and add it to the free list. Returns the new 38510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// FreeRangeHeader at the base of the slab. 38610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { 38710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // If the user needs at least MinSize free memory, then we account for 38810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // two MemoryRangeHeaders: the one in the user's block, and the one at the 38910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // end of the slab. 39010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); 39110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); 39210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = allocateNewSlab(SlabSize); 39310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner CodeSlabs.push_back(B); 39410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *MemBase = (char*)(B.base()); 39510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 39610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Put a tiny allocated block at the end of the memory chunk, so when 39710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // FreeBlock calls getBlockAfter it doesn't fall off the end. 39810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner MemoryRangeHeader *EndBlock = 39910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner (MemoryRangeHeader*)(MemBase + B.size()) - 1; 40010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->ThisAllocated = 1; 40110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->PrevAllocated = 0; 40210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->BlockSize = sizeof(MemoryRangeHeader); 40310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 40410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Start out with a vast new block of free memory. 40510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; 40610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->ThisAllocated = 0; 40710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Make sure getFreeBlockBefore doesn't look into unmapped memory. 40810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->PrevAllocated = 1; 40910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; 41010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->SetEndOfBlockSizeMarker(); 41110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->AddToFreeList(FreeMemoryList); 41210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 41310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && 41410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner "The block was too small!"); 41510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return NewBlock; 41610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 41710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 4188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// endFunctionBody - The function F is now allocated, and takes the memory 4198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// in the range [FunctionStart,FunctionEnd). 420186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void endFunctionBody(const Function *F, uint8_t *FunctionStart, 421186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *FunctionEnd) { 4228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FunctionEnd > FunctionStart); 423186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes assert(FunctionStart == (uint8_t *)(CurBlock+1) && 4248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Mismatched function start/end!"); 425dd947ea3c5e020c33c58a31939561265b980a3adDale Johannesen 426186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; 4278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 4288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Release the memory at the end of this block that isn't needed. 4298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 4308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 431cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 43210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateSpace - Allocate a memory block of the given size. This method 43310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// cannot be called between calls to startFunctionBody and endFunctionBody. 434186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) { 435cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes CurBlock = FreeMemoryList; 436cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes FreeMemoryList = FreeMemoryList->AllocateBlock(); 437cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 438489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin uint8_t *result = (uint8_t *)(CurBlock + 1); 439cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 440cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes if (Alignment == 0) Alignment = 1; 441186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes result = (uint8_t*)(((intptr_t)result+Alignment-1) & 442cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes ~(intptr_t)(Alignment-1)); 443cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 444186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; 445cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 446cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 447cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes return result; 448cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes } 449cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 45010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateStub - Allocate memory for a function stub. 45110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, 45210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner unsigned Alignment) { 45310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); 45410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 4554bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner 45610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateGlobal - Allocate memory for a global. 45710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) { 45810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t*)DataAllocator.Allocate(Size, Alignment); 45961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 46061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 46161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach /// allocateCodeSection - Allocate memory for a code section. 46261425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, 46361425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach unsigned SectionID) { 464f92bbcd1ce5d7f1349d4ba7b3cdcd2c7c023f1b5Sean Callanan // Grow the required block size to account for the block header 465f92bbcd1ce5d7f1349d4ba7b3cdcd2c7c023f1b5Sean Callanan Size += sizeof(*CurBlock); 466f92bbcd1ce5d7f1349d4ba7b3cdcd2c7c023f1b5Sean Callanan 46761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // FIXME: Alignement handling. 46861425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeRangeHeader* candidateBlock = FreeMemoryList; 46961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeRangeHeader* head = FreeMemoryList; 47061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeRangeHeader* iter = head->Next; 47161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 47261425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach uintptr_t largest = candidateBlock->BlockSize; 47361425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 47461425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // Search for the largest free block. 47561425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach while (iter != head) { 47661425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach if (iter->BlockSize > largest) { 47761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach largest = iter->BlockSize; 47861425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach candidateBlock = iter; 47961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 48061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach iter = iter->Next; 48161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 48261425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 48361425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach largest = largest - sizeof(MemoryRangeHeader); 48461425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 48561425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // If this block isn't big enough for the allocation desired, allocate 48661425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // another block of memory and add it to the free list. 48761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) { 48861425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); 48961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach candidateBlock = allocateNewCodeSlab((size_t)Size); 49061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 49161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 49261425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // Select this candidate block for allocation 49361425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach CurBlock = candidateBlock; 49461425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 49561425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // Allocate the entire memory block. 49661425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeMemoryList = candidateBlock->AllocateBlock(); 49761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // Release the memory at the end of this block that isn't needed. 49861425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size); 49961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach return (uint8_t *)(CurBlock + 1); 50061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 50161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 50261425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach /// allocateDataSection - Allocate memory for a data section. 50361425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, 50461425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach unsigned SectionID) { 50561425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach return (uint8_t*)DataAllocator.Allocate(Size, Alignment); 506489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 507489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5085cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher /// startExceptionTable - Use startFunctionBody to allocate memory for the 509afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// function's exception table. 510186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) { 511afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray return startFunctionBody(F, ActualSize); 512afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray } 513afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 5145cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher /// endExceptionTable - The exception table of F is now allocated, 515afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray /// and takes the memory in the range [TableStart,TableEnd). 516186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void endExceptionTable(const Function *F, uint8_t *TableStart, 517186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *TableEnd, uint8_t* FrameRegister) { 518afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray assert(TableEnd > TableStart); 519186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes assert(TableStart == (uint8_t *)(CurBlock+1) && 520afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray "Mismatched table start/end!"); 5215cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 522186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = TableEnd - (uint8_t *)CurBlock; 523afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray 524afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray // Release the memory at the end of this block that isn't needed. 525afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 526afe6c2b001a924cd74bd0aacfed5984d9af004b0Nicolas Geoffray } 5275cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 528186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *getGOTBase() const { 5298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return GOTBase; 5308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 5315cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 5321e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin void deallocateBlock(void *Block) { 5338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Find the block that is allocated for this function. 5341e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1; 5358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(MemRange->ThisAllocated && "Block isn't allocated!"); 536489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Fill the buffer with garbage! 538489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin if (PoisonMemory) { 539489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); 540489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 541489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Free the memory. 5438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); 5441e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin } 545489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5461e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin /// deallocateFunctionBody - Deallocate all memory for the specified 5471e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin /// function body. 5481e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin void deallocateFunctionBody(void *Body) { 54971e0b7c560200a87ea49080f581153f9a2c9db72Nicolas Geoffray if (Body) deallocateBlock(Body); 5501e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin } 551489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5521e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin /// deallocateExceptionTable - Deallocate memory for the specified 5531e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin /// exception table. 5541e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin void deallocateExceptionTable(void *ET) { 55571e0b7c560200a87ea49080f581153f9a2c9db72Nicolas Geoffray if (ET) deallocateBlock(ET); 5568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 557cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach 558cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// setMemoryWritable - When code generation is in progress, 559cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// the code pages may need permissions changed. 560a9ad04191cb56c42944b17980b8b2bb2afe11ab2Dan Gohman void setMemoryWritable() 561cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach { 56210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 56310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::setWritable(CodeSlabs[i]); 564cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach } 565cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// setMemoryExecutable - When code generation is done and we're ready to 566cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// start execution, the code pages may need permissions changed. 567a9ad04191cb56c42944b17980b8b2bb2afe11ab2Dan Gohman void setMemoryExecutable() 568cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach { 56910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 57010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::setExecutable(CodeSlabs[i]); 571cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach } 572489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 573489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin /// setPoisonMemory - Controls whether we write garbage over freed memory. 574489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin /// 575489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin void setPoisonMemory(bool poison) { 576489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin PoisonMemory = poison; 577489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 5788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 5798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 5808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 58110b4fc552f984dc978298d50c09c97c0764962fcReid KlecknerMemSlab *JITSlabAllocator::Allocate(size_t Size) { 58210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = JMM.allocateNewSlab(Size); 58310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner MemSlab *Slab = (MemSlab*)B.base(); 58410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Slab->Size = B.size(); 58510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Slab->NextPtr = 0; 58610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return Slab; 58710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 58810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 58910b4fc552f984dc978298d50c09c97c0764962fcReid Klecknervoid JITSlabAllocator::Deallocate(MemSlab *Slab) { 59010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B(Slab, Slab->Size); 59110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::ReleaseRWX(B); 59210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 59310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 59410b4fc552f984dc978298d50c09c97c0764962fcReid KlecknerDefaultJITMemoryManager::DefaultJITMemoryManager() 5955b78d50cadf5df4948e8e5798a3e08e0ccf4cb06Dan Gohman : 596489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin#ifdef NDEBUG 5975b78d50cadf5df4948e8e5798a3e08e0ccf4cb06Dan Gohman PoisonMemory(false), 598bc4707a2554ac04ba006bf70035e7bc7270236a9Evan Cheng#else 5995b78d50cadf5df4948e8e5798a3e08e0ccf4cb06Dan Gohman PoisonMemory(true), 600bc4707a2554ac04ba006bf70035e7bc7270236a9Evan Cheng#endif 6015b78d50cadf5df4948e8e5798a3e08e0ccf4cb06Dan Gohman LastSlab(0, 0), 6025b78d50cadf5df4948e8e5798a3e08e0ccf4cb06Dan Gohman BumpSlabAllocator(*this), 6035b78d50cadf5df4948e8e5798a3e08e0ccf4cb06Dan Gohman StubAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator), 6045b78d50cadf5df4948e8e5798a3e08e0ccf4cb06Dan Gohman DataAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator) { 6058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 60610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Allocate space for code. 60710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); 60810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner CodeSlabs.push_back(MemBlock); 60910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uint8_t *MemBase = (uint8_t*)MemBlock.base(); 610489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 6118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // We set up the memory chunk with 4 mem regions, like this: 6128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ START 6138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Free #0 ] -> Large space to allocate functions from. 6148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Allocated #1 ] -> Tiny space to separate regions. 6158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Free #2 ] -> Tiny space so there is always at least 1 free block. 6168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. 6178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // END ] 6188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // 6198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // The last three blocks are never deallocated or touched. 6205cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 6218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add MemoryRangeHeader to the end of the memory region, indicating that 6228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // the space after the block of memory is allocated. This is block #3. 6238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; 6248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem3->ThisAllocated = 1; 6258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem3->PrevAllocated = 0; 62610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Mem3->BlockSize = sizeof(MemoryRangeHeader); 6275cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 6288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Add a tiny free region so that the free list always has one entry. 6295cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher FreeRangeHeader *Mem2 = 6308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); 6318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->ThisAllocated = 0; 6328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->PrevAllocated = 1; 6338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); 6348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->SetEndOfBlockSizeMarker(); 6358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->Prev = Mem2; // Mem2 *is* the free list for now. 6368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->Next = Mem2; 6378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Add a tiny allocated region so that Mem2 is never coalesced away. 6398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; 6408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem1->ThisAllocated = 1; 6418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem1->PrevAllocated = 0; 64210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Mem1->BlockSize = sizeof(MemoryRangeHeader); 6435cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 6448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add a FreeRangeHeader to the start of the function body region, indicating 6458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // that the space is free. Mark the previous block allocated so we never look 6468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // at it. 64710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; 6488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->ThisAllocated = 0; 6498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->PrevAllocated = 1; 6508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->BlockSize = (char*)Mem1-(char*)Mem0; 6518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->SetEndOfBlockSizeMarker(); 6528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->AddToFreeList(Mem2); 6535cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 6548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Start out with the freelist pointing to Mem0. 6558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList = Mem0; 6568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner GOTBase = NULL; 6588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnervoid DefaultJITMemoryManager::AllocateGOT() { 6618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(GOTBase == 0 && "Cannot allocate the got multiple times"); 662186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes GOTBase = new uint8_t[sizeof(void*) * 8192]; 6638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner HasGOT = true; 6648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerDefaultJITMemoryManager::~DefaultJITMemoryManager() { 66710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 66810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::ReleaseRWX(CodeSlabs[i]); 6698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 67010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner delete[] GOTBase; 6714bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner} 6724bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner 67310b4fc552f984dc978298d50c09c97c0764962fcReid Klecknersys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { 6748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Allocate a new block close to the last one. 6758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner std::string ErrMsg; 67610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : 0; 67710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); 6788907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (B.base() == 0) { 67975361b69f3f327842b9dad69fa7f28ae3b688412Chris Lattner report_fatal_error("Allocation failed when allocating new memory in the" 6801bd7335a17010bd4d8f86736cf73cac9f3fb80a5Benjamin Kramer " JIT\n" + Twine(ErrMsg)); 6818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 68210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner LastSlab = B; 68310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner ++NumSlabs; 68401248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner // Initialize the slab to garbage when debugging. 68501248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner if (PoisonMemory) { 68601248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner memset(B.base(), 0xCD, B.size()); 68701248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner } 6888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return B; 6898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 69110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// CheckInvariants - For testing only. Return "" if all internal invariants 69210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// are preserved, and a helpful error message otherwise. For free and 69310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// allocated blocks, make sure that adding BlockSize gives a valid block. 69410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// For free blocks, make sure they're in the free list and that their end of 69510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// block size marker is correct. This function should return an error before 69610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// accessing bad memory. This function is defined here instead of in 69710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// JITMemoryManagerTest.cpp so that we don't have to expose all of the 69810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// implementation details of DefaultJITMemoryManager. 69910b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerbool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { 70010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner raw_string_ostream Err(ErrorStr); 70110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 70210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Construct a the set of FreeRangeHeader pointers so we can query it 70310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // efficiently. 70410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; 70510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader* FreeHead = FreeMemoryList; 70610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader* FreeRange = FreeHead; 70710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 70810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner do { 70910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check that the free range pointer is in the blocks we've allocated. 71010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner bool Found = false; 71110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 71210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner E = CodeSlabs.end(); I != E && !Found; ++I) { 71310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *Start = (char*)I->base(); 71410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *End = Start + I->size(); 71510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); 71610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 71710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!Found) { 71810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Corrupt free list; points to " << FreeRange; 71910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 72010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 72110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 72210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (FreeRange->Next->Prev != FreeRange) { 72310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Next and Prev pointers do not match."; 72410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 72510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 72610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 72710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Otherwise, add it to the set. 72810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeHdrSet.insert(FreeRange); 72910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRange = FreeRange->Next; 73010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } while (FreeRange != FreeHead); 73110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 73210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Go over each block, and look at each MemoryRangeHeader. 73310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 73410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner E = CodeSlabs.end(); I != E; ++I) { 73510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *Start = (char*)I->base(); 73610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *End = Start + I->size(); 73710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 73810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check each memory range. 73910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = NULL; 74010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Start <= (char*)Hdr && (char*)Hdr < End; 74110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Hdr = &Hdr->getBlockAfter()) { 74210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (Hdr->ThisAllocated == 0) { 74310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check that this range is in the free list. 74410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!FreeHdrSet.count(Hdr)) { 74510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Found free header at " << Hdr << " that is not in free list."; 74610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 74710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 74810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 74910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Now make sure the size marker at the end of the block is correct. 75010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; 75110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!(Start <= (char*)Marker && (char*)Marker < End)) { 75210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Block size in header points out of current MemoryBlock."; 75310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 75410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 75510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (Hdr->BlockSize != *Marker) { 75610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "End of block size marker (" << *Marker << ") " 75710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner << "and BlockSize (" << Hdr->BlockSize << ") don't match."; 75810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 75910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 76010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 76110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 76210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { 76310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " 76410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; 76510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 76610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } else if (!LastHdr && !Hdr->PrevAllocated) { 76710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "The first header should have PrevAllocated true."; 76810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 76910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 77010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 77110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Remember the last header. 77210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner LastHdr = Hdr; 77310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 77410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 77510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 77610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // All invariants are preserved. 77710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return true; 77810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 7798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 78030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev//===----------------------------------------------------------------------===// 78130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// getPointerToNamedFunction() implementation. 78230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev//===----------------------------------------------------------------------===// 78330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 78430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// AtExitHandlers - List of functions to call when the program exits, 78530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// registered with the atexit() library function. 78630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic std::vector<void (*)()> AtExitHandlers; 78730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 78830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// runAtExitHandlers - Run any functions registered by the program's 78930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// calls to atexit(3), which we intercept and store in 79030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// AtExitHandlers. 79130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// 79230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic void runAtExitHandlers() { 79330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev while (!AtExitHandlers.empty()) { 79430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev void (*Fn)() = AtExitHandlers.back(); 79530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev AtExitHandlers.pop_back(); 79630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev Fn(); 79730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 79830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 79930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 80030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev//===----------------------------------------------------------------------===// 80130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// Function stubs that are invoked instead of certain library calls 80230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// 80330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// Force the following functions to be linked in to anything that uses the 80430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// JIT. This is a hack designed to work around the all-too-clever Glibc 80530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// strategy of making these functions work differently when inlined vs. when 80630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// not inlined, and hiding their real definitions in a separate archive file 80730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// that the dynamic linker can't see. For more info, search for 80830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274. 80930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#if defined(__linux__) 81030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/* stat functions are redirecting to __xstat with a version number. On x86-64 81130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat' 81230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev * available as an exported symbol, so we have to add it explicitly. 81330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev */ 81430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevnamespace { 81530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevclass StatSymbols { 81630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevpublic: 81730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev StatSymbols() { 81830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat); 81930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat); 82030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat); 82130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64); 82230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64); 82330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64); 82430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64); 82530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64); 82630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64); 82730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit); 82830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod); 82930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 83030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev}; 83130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 83230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic StatSymbols initStatSymbols; 83330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#endif // __linux__ 83430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 83530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// jit_exit - Used to intercept the "exit" library call. 83630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic void jit_exit(int Status) { 83730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev runAtExitHandlers(); // Run atexit handlers... 83830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev exit(Status); 83930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 84030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 84130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// jit_atexit - Used to intercept the "atexit" library call. 84230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic int jit_atexit(void (*Fn)()) { 84330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev AtExitHandlers.push_back(Fn); // Take note of atexit handler... 84430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return 0; // Always successful 84530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 84630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 84730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic int jit_noop() { 84830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return 0; 84930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 85030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 85130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev//===----------------------------------------------------------------------===// 85230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// 85330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// getPointerToNamedFunction - This method returns the address of the specified 85430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// function by using the dynamic loader interface. As such it is only useful 85530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// for resolving library symbols, not code generated symbols. 85630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// 85730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevvoid *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name, 8585fe019835c269ccbfe185276269bc53b3f9a7a86Eli Bendersky bool AbortOnFailure) { 85930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // Check to see if this is one of the functions we want to intercept. Note, 86030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // we cast to intptr_t here to silence a -pedantic warning that complains 86130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // about casting a function pointer to a normal pointer. 86230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Name == "exit") return (void*)(intptr_t)&jit_exit; 86330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Name == "atexit") return (void*)(intptr_t)&jit_atexit; 86430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 86530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // We should not invoke parent's ctors/dtors from generated main()! 86630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // On Mingw and Cygwin, the symbol __main is resolved to 86730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors 86830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // (and register wrong callee's dtors with atexit(3)). 86930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // We expect ExecutionEngine::runStaticConstructorsDestructors() 87030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // is called before ExecutionEngine::runFunctionAsMain() is called. 87130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Name == "__main") return (void*)(intptr_t)&jit_noop; 87230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 87330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev const char *NameStr = Name.c_str(); 87430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // If this is an asm specifier, skip the sentinal. 87530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (NameStr[0] == 1) ++NameStr; 87630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 87730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // If it's an external function, look it up in the process image... 87830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); 87930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Ptr) return Ptr; 88030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 88130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // If it wasn't found and if it starts with an underscore ('_') character, 88230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // try again without the underscore. 88330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (NameStr[0] == '_') { 88430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); 88530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Ptr) return Ptr; 88630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 88730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 88830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These 88930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // are references to hidden visibility symbols that dlsym cannot resolve. 89030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // If we have one of these, strip off $LDBLStub and try again. 89130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#if defined(__APPLE__) && defined(__ppc__) 89230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Name.size() > 9 && Name[Name.size()-9] == '$' && 89330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) { 89430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // First try turning $LDBLStub into $LDBL128. If that fails, strip it off. 89530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // This mirrors logic in libSystemStubs.a. 89630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev std::string Prefix = std::string(Name.begin(), Name.end()-9); 89730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false)) 89830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return Ptr; 89930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (void *Ptr = getPointerToNamedFunction(Prefix, false)) 90030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return Ptr; 90130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 90230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#endif 90330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 90430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (AbortOnFailure) { 90530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev report_fatal_error("Program used external function '"+Name+ 90630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev "' which could not be resolved!"); 90730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 90830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return 0; 90930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 91030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 91130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 91230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 9138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerJITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { 9148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return new DefaultJITMemoryManager(); 9158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 91610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 91710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner// Allocate memory for code in 512K slabs. 91810b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerconst size_t DefaultJITMemoryManager::DefaultCodeSlabSize = 512 * 1024; 91910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 92010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner// Allocate globals and stubs in slabs of 64K. (probably 16 pages) 92110b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerconst size_t DefaultJITMemoryManager::DefaultSlabSize = 64 * 1024; 92210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 92310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner// Waste at most 16K at the end of each bump slab. (probably 4 pages) 92410b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerconst size_t DefaultJITMemoryManager::DefaultSizeThreshold = 16 * 1024; 925