18907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// 28907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 38907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// The LLVM Compiler Infrastructure 48907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 78907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 88907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 98907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// This file defines the DefaultJITMemoryManager class. 118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// 128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 144bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner#include "llvm/ExecutionEngine/JITMemoryManager.h" 1510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/ADT/SmallPtrSet.h" 1610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/ADT/Statistic.h" 171bd7335a17010bd4d8f86736cf73cac9f3fb80a5Benjamin Kramer#include "llvm/ADT/Twine.h" 18d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Config/config.h" 190b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/GlobalValue.h" 2010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/Support/Allocator.h" 218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner#include "llvm/Support/Compiler.h" 2210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner#include "llvm/Support/Debug.h" 23d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Support/DynamicLibrary.h" 247d696d80409aad20bb5da0fc4eccab941dd371d4Torok Edwin#include "llvm/Support/ErrorHandling.h" 25d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Support/Memory.h" 26d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Support/raw_ostream.h" 273012ac63d35bc6dc450f97dbce3e2faaf8859198Chuck Rose III#include <cassert> 28de551f91d8816632a76a065084caab9fab6aacffDan Gohman#include <climits> 29ae9f3a3b7c915f725aef5a7250e88eaeddda03c6Anton Korobeynikov#include <cstring> 30d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include <vector> 3130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 3230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#if defined(__linux__) 3330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#if defined(HAVE_SYS_STAT_H) 3430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include <sys/stat.h> 3530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#endif 3630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include <fcntl.h> 3730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#include <unistd.h> 3830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#endif 3930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnerusing namespace llvm; 418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 42dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "jit" 43dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4410b4fc552f984dc978298d50c09c97c0764962fcReid KlecknerSTATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); 458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerJITMemoryManager::~JITMemoryManager() {} 478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// Memory Block Implementation. 508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnernamespace { 538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// MemoryRangeHeader - For a range of memory, this is the header that we put 548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// on the block of memory. It is carefully crafted to be one word of memory. 558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader 568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// which starts with this. 578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct FreeRangeHeader; 588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct MemoryRangeHeader { 598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// ThisAllocated - This is true if this block is currently allocated. If 608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// not, this can be converted to a FreeRangeHeader. 618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned ThisAllocated : 1; 625cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// PrevAllocated - Keep track of whether the block immediately before us is 648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// allocated. If not, the word immediately before this header is the size 658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// of the previous block. 668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned PrevAllocated : 1; 675cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// BlockSize - This is the size in bytes of this memory block, 698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// including this header. 70de551f91d8816632a76a065084caab9fab6aacffDan Gohman uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); 715cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getBlockAfter - Return the memory block immediately after this one. 748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// 758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader &getBlockAfter() const { 76fe1215ef935f182cdca28b4af655fa0bfa0f47e6David Greene return *reinterpret_cast<MemoryRangeHeader *>( 77fe1215ef935f182cdca28b4af655fa0bfa0f47e6David Greene reinterpret_cast<char*>( 78fe1215ef935f182cdca28b4af655fa0bfa0f47e6David Greene const_cast<MemoryRangeHeader *>(this))+BlockSize); 798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 805cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getFreeBlockBefore - If the block before this one is free, return it, 828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// otherwise return null. 838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *getFreeBlockBefore() const { 84dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (PrevAllocated) return nullptr; 85fe1215ef935f182cdca28b4af655fa0bfa0f47e6David Greene intptr_t PrevSize = reinterpret_cast<intptr_t *>( 86fe1215ef935f182cdca28b4af655fa0bfa0f47e6David Greene const_cast<MemoryRangeHeader *>(this))[-1]; 87fe1215ef935f182cdca28b4af655fa0bfa0f47e6David Greene return reinterpret_cast<FreeRangeHeader *>( 88fe1215ef935f182cdca28b4af655fa0bfa0f47e6David Greene reinterpret_cast<char*>( 89fe1215ef935f182cdca28b4af655fa0bfa0f47e6David Greene const_cast<MemoryRangeHeader *>(this))-PrevSize); 908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 915cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// FreeBlock - Turn an allocated block into a free block, adjusting 938907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// bits in the object headers, and adding an end of region memory block. 948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); 955cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// TrimAllocationToSize - If this allocated block is significantly larger 978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// than NewSize, split it into two pieces (where the former is NewSize 988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// bytes, including the header), and add the new block to the free list. 995cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, 1008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner uint64_t NewSize); 1018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 1028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// FreeRangeHeader - For a memory block that isn't already allocated, this 1048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// keeps track of the current block and has a pointer to the next free block. 1058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Free blocks are kept on a circularly linked list. 1068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner struct FreeRangeHeader : public MemoryRangeHeader { 1078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Prev; 1088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *Next; 1095cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// getMinBlockSize - Get the minimum size for a memory block. Blocks 1118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// smaller than this size cannot be created. 1128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner static unsigned getMinBlockSize() { 1138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return sizeof(FreeRangeHeader)+sizeof(intptr_t); 1148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1155cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// SetEndOfBlockSizeMarker - The word at the end of every free block is 1178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// known to be the size of the free block. Set it for this block. 1188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void SetEndOfBlockSizeMarker() { 1198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void *EndOfBlock = (char*)this + BlockSize; 1208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ((intptr_t *)EndOfBlock)[-1] = BlockSize; 1218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *RemoveFromFreeList() { 1248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); 1258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next->Prev = Prev; 1268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return Prev->Next = Next; 1278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1285cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void AddToFreeList(FreeRangeHeader *FreeList) { 1308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next = FreeList; 1318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Prev = FreeList->Prev; 1328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Prev->Next = this; 1338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Next->Prev = this; 1348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// GrowBlock - The block after this block just got deallocated. Merge it 1378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// into the current block. 1388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner void GrowBlock(uintptr_t NewSize); 1395cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// AllocateBlock - Mark this entire block allocated, updating freelists 1418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// etc. This returns a pointer to the circular free-list. 1428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *AllocateBlock(); 1438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 1448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1468907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// AllocateBlock - Mark this entire block allocated, updating freelists 1488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// etc. This returns a pointer to the circular free-list. 1498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *FreeRangeHeader::AllocateBlock() { 1508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(!ThisAllocated && !getBlockAfter().PrevAllocated && 1518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Cannot allocate an allocated block!"); 1528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Mark this block allocated. 1538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ThisAllocated = 1; 1548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner getBlockAfter().PrevAllocated = 1; 1555cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Remove it from the free list. 1578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return RemoveFromFreeList(); 1588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 1598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// FreeBlock - Turn an allocated block into a free block, adjusting 1618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// bits in the object headers, and adding an end of region memory block. 1628907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// If possible, coalesce this block with neighboring blocks. Return the 1638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// FreeRangeHeader to allocate from. 1648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { 1658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *FollowingBlock = &getBlockAfter(); 16610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner assert(ThisAllocated && "This block is already free!"); 1678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); 1685cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader *FreeListToReturn = FreeList; 1705cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // If the block after this one is free, merge it into this block. 1728907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (!FollowingBlock->ThisAllocated) { 1738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; 1748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // "FreeList" always needs to be a valid free block. If we're about to 1758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // coalesce with it, update our notion of what the free list is. 1768907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (&FollowingFreeBlock == FreeList) { 1778907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeList = FollowingFreeBlock.Next; 178dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FreeListToReturn = nullptr; 1798907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); 1808907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingFreeBlock.RemoveFromFreeList(); 1825cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Include the following block into this one. 1848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize += FollowingFreeBlock.BlockSize; 1858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock = &FollowingFreeBlock.getBlockAfter(); 1865cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1878907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Tell the block after the block we are coalescing that this block is 1888907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // allocated. 1898907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock->PrevAllocated = 1; 1908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1915cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1928907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); 1935cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 1948907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { 1958907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); 1968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; 1978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 1988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 1998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Otherwise, mark this block free. 2008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; 2018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FollowingBlock->PrevAllocated = 0; 2028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.ThisAllocated = 0; 2038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Link this into the linked list of free blocks. 2058907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.AddToFreeList(FreeList); 2068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add a marker at the end of the block, indicating the size of this free 2088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // block. 2098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeBlock.SetEndOfBlockSizeMarker(); 2108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeListToReturn ? FreeListToReturn : &FreeBlock; 2118907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2128907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// GrowBlock - The block after this block just got deallocated. Merge it 2148907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// into the current block. 2158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnervoid FreeRangeHeader::GrowBlock(uintptr_t NewSize) { 2168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(NewSize > BlockSize && "Not growing block?"); 2178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize = NewSize; 2188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner SetEndOfBlockSizeMarker(); 2198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner getBlockAfter().PrevAllocated = 0; 2208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// TrimAllocationToSize - If this allocated block is significantly larger 2238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// than NewSize, split it into two pieces (where the former is NewSize 2248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner/// bytes, including the header), and add the new block to the free list. 2258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerFreeRangeHeader *MemoryRangeHeader:: 2268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerTrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { 2278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(ThisAllocated && getBlockAfter().PrevAllocated && 2288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Cannot deallocate part of an allocated block!"); 2298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 23060b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng // Don't allow blocks to be trimmed below minimum required size 23160b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); 23260b75f462e1044a031fbd75a8dcb45cb3de29177Evan Cheng 2338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Round up size for alignment of header. 2348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner unsigned HeaderAlign = __alignof(FreeRangeHeader); 2358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); 2365cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Size is now the size of the block we will remove from the start of the 2388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // current block. 2398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(NewSize <= BlockSize && 2408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Allocating more space from this block than exists!"); 2415cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // If splitting this block will cause the remainder to be too small, do not 2438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // split the block. 2448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) 2458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return FreeList; 2465cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2478907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Otherwise, we splice the required number of bytes out of this block, form 2488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // a new block immediately after it, then mark this block allocated. 2498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader &FormerNextBlock = getBlockAfter(); 2505cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Change the size of this block. 2528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner BlockSize = NewSize; 2535cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 2548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Get the new block we just sliced out and turn it into a free block. 2558907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); 2568907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; 2578907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.ThisAllocated = 0; 2588907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.PrevAllocated = 1; 2598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.SetEndOfBlockSizeMarker(); 2608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FormerNextBlock.PrevAllocated = 0; 2618907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner NewNextBlock.AddToFreeList(FreeList); 2628907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return &NewNextBlock; 2638907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 2648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 2658907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 2668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner// Memory Block Implementation. 2678907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner//===----------------------------------------------------------------------===// 2688907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 26910b4fc552f984dc978298d50c09c97c0764962fcReid Klecknernamespace { 27010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 27110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner class DefaultJITMemoryManager; 27210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines class JITAllocator { 27410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner DefaultJITMemoryManager &JMM; 27510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner public: 276dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } 277dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines void *Allocate(size_t Size, size_t /*Alignment*/); 278dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines void Deallocate(void *Slab, size_t Size); 27910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner }; 28010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 2818907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// DefaultJITMemoryManager - Manage memory for the JIT code generation. 2828907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// This splits a large block of MAP_NORESERVE'd memory into two 2838907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// sections, one for function stubs, one for the functions themselves. We 2848907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// have to do this because we may need to emit a function stub while in the 2858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// middle of emitting a function, and we don't know how large the function we 2868907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// are emitting is. 28710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner class DefaultJITMemoryManager : public JITMemoryManager { 28836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 28936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at 29036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// least this much unless more is requested. Currently, in 512k slabs. 29136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static const size_t DefaultCodeSlabSize = 512 * 1024; 29236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 29336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably 29436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// 16 pages) unless we get an allocation above SizeThreshold. 29536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static const size_t DefaultSlabSize = 64 * 1024; 29610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 29736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// DefaultSizeThreshold - For any allocation larger than 16K (probably 29836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// 4 pages), we should allocate a separate slab to avoid wasted space at 29936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// the end of a normal slab. 30036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static const size_t DefaultSizeThreshold = 16 * 1024; 30136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 30236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines private: 30310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Whether to poison freed memory. 30410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner bool PoisonMemory; 30510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 30610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// LastSlab - This points to the last slab allocated and is used as the 30710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all 30810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// stubs, data, and code contiguously in memory. In general, however, this 30910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// is not possible because the NearBlock parameter is ignored on Windows 31010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// platforms and even on Unix it works on a best-effort pasis. 31110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock LastSlab; 31210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 31310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Memory slabs allocated by the JIT. We refer to them as slabs so we don't 3145cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher // confuse them with the blocks of memory described above. 31510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner std::vector<sys::MemoryBlock> CodeSlabs; 316dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize, 317dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DefaultSizeThreshold> StubAllocator; 318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize, 319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DefaultSizeThreshold> DataAllocator; 32010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 32110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Circular list of free blocks. 32210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *FreeMemoryList; 323489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 3248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // When emitting code into a memory block, this is the block. 3258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *CurBlock; 32610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 327186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uint8_t *GOTBase; // Target Specific reserved memory 3288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner public: 3298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner DefaultJITMemoryManager(); 3308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner ~DefaultJITMemoryManager(); 3318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 33210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the 33310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// last slab it allocated, so that subsequent allocations follow it. 33410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock allocateNewSlab(size_t size); 33510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 33630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev /// getPointerToNamedFunction - This method returns the address of the 33730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev /// specified function by using the dlsym function call. 33836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void *getPointerToNamedFunction(const std::string &Name, 33936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool AbortOnFailure = true) override; 34030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 34136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void AllocateGOT() override; 34210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 34310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Testing methods. 34436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool CheckInvariants(std::string &ErrorStr) override; 34536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; } 34636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; } 34736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; } 34836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); } 34936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); } 35036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); } 35110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 3528907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// startFunctionBody - When a function starts, allocate a block of free 3538907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// executable memory, returning a pointer to it and its actual size. 35436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint8_t *startFunctionBody(const Function *F, 35536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uintptr_t &ActualSize) override { 35610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 35796c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* candidateBlock = FreeMemoryList; 35896c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* head = FreeMemoryList; 35996c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeRangeHeader* iter = head->Next; 36096c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner 36196c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner uintptr_t largest = candidateBlock->BlockSize; 36210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 36396c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner // Search for the largest free block 36496c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner while (iter != head) { 36510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (iter->BlockSize > largest) { 36610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner largest = iter->BlockSize; 36710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner candidateBlock = iter; 36810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 36910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner iter = iter->Next; 37096c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner } 37110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 3721b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray largest = largest - sizeof(MemoryRangeHeader); 3735cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 37410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // If this block isn't big enough for the allocation desired, allocate 37510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // another block of memory and add it to the free list. 3761b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray if (largest < ActualSize || 3771b10d798d742860e677b13db13162ead8db647b1Nicolas Geoffray largest <= FreeRangeHeader::getMinBlockSize()) { 378bd8c8afe1b5439792d985da7c6e5949d8dc3f552David Greene DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); 37910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner candidateBlock = allocateNewCodeSlab((size_t)ActualSize); 38010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 38110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 38296c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner // Select this candidate block for allocation 38396c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner CurBlock = candidateBlock; 38496c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner 3858907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Allocate the entire memory block. 38696c96b46e962c801c610ba878751d8c0ee1359c9Chris Lattner FreeMemoryList = candidateBlock->AllocateBlock(); 38710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); 38810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t *)(CurBlock + 1); 38981ce3ed08c4df0c246b378c8972062d2f49f1ce9Reid Kleckner } 39010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 39110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateNewCodeSlab - Helper method to allocate a new slab of code 39210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// memory from the OS and add it to the free list. Returns the new 39310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// FreeRangeHeader at the base of the slab. 39410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { 39510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // If the user needs at least MinSize free memory, then we account for 39610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // two MemoryRangeHeaders: the one in the user's block, and the one at the 39710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // end of the slab. 39810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); 39910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); 40010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = allocateNewSlab(SlabSize); 40110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner CodeSlabs.push_back(B); 40210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *MemBase = (char*)(B.base()); 40310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 40410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Put a tiny allocated block at the end of the memory chunk, so when 40510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // FreeBlock calls getBlockAfter it doesn't fall off the end. 40610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner MemoryRangeHeader *EndBlock = 40710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner (MemoryRangeHeader*)(MemBase + B.size()) - 1; 40810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->ThisAllocated = 1; 40910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->PrevAllocated = 0; 41010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner EndBlock->BlockSize = sizeof(MemoryRangeHeader); 41110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 41210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Start out with a vast new block of free memory. 41310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; 41410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->ThisAllocated = 0; 41510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Make sure getFreeBlockBefore doesn't look into unmapped memory. 41610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->PrevAllocated = 1; 41710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; 41810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->SetEndOfBlockSizeMarker(); 41910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner NewBlock->AddToFreeList(FreeMemoryList); 42010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 42110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && 42210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner "The block was too small!"); 42310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return NewBlock; 42410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 42510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 4268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// endFunctionBody - The function F is now allocated, and takes the memory 4278907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// in the range [FunctionStart,FunctionEnd). 428186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes void endFunctionBody(const Function *F, uint8_t *FunctionStart, 42936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint8_t *FunctionEnd) override { 4308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(FunctionEnd > FunctionStart); 431186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes assert(FunctionStart == (uint8_t *)(CurBlock+1) && 4328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner "Mismatched function start/end!"); 433dd947ea3c5e020c33c58a31939561265b980a3adDale Johannesen 434186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; 4358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 4368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Release the memory at the end of this block that isn't needed. 4378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 4388907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 439cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 44010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateSpace - Allocate a memory block of the given size. This method 44110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// cannot be called between calls to startFunctionBody and endFunctionBody. 44236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override { 443cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes CurBlock = FreeMemoryList; 444cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes FreeMemoryList = FreeMemoryList->AllocateBlock(); 445cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 446489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin uint8_t *result = (uint8_t *)(CurBlock + 1); 447cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 448cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes if (Alignment == 0) Alignment = 1; 449186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes result = (uint8_t*)(((intptr_t)result+Alignment-1) & 450cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes ~(intptr_t)(Alignment-1)); 451cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 452186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; 453cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); 454cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 455cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes return result; 456cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes } 457cef7527a85d026aeb17a4dacca73c70c0ab5da40Nuno Lopes 45810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateStub - Allocate memory for a function stub. 45910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, 46036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Alignment) override { 46110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); 46210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 4634bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner 46410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner /// allocateGlobal - Allocate memory for a global. 46536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override { 46610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return (uint8_t*)DataAllocator.Allocate(Size, Alignment); 46761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 46861425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 46961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach /// allocateCodeSection - Allocate memory for a code section. 47061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, 47136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned SectionID, 47236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines StringRef SectionName) override { 473f92bbcd1ce5d7f1349d4ba7b3cdcd2c7c023f1b5Sean Callanan // Grow the required block size to account for the block header 474f92bbcd1ce5d7f1349d4ba7b3cdcd2c7c023f1b5Sean Callanan Size += sizeof(*CurBlock); 475f92bbcd1ce5d7f1349d4ba7b3cdcd2c7c023f1b5Sean Callanan 476e5cb25f860bb4ffe6908004e6a04c8d21b7d2a98Elena Demikhovsky // Alignment handling. 477e5cb25f860bb4ffe6908004e6a04c8d21b7d2a98Elena Demikhovsky if (!Alignment) 478e5cb25f860bb4ffe6908004e6a04c8d21b7d2a98Elena Demikhovsky Alignment = 16; 479e5cb25f860bb4ffe6908004e6a04c8d21b7d2a98Elena Demikhovsky Size += Alignment - 1; 480e5cb25f860bb4ffe6908004e6a04c8d21b7d2a98Elena Demikhovsky 48161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeRangeHeader* candidateBlock = FreeMemoryList; 48261425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeRangeHeader* head = FreeMemoryList; 48361425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeRangeHeader* iter = head->Next; 48461425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 48561425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach uintptr_t largest = candidateBlock->BlockSize; 48661425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 48761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // Search for the largest free block. 48861425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach while (iter != head) { 48961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach if (iter->BlockSize > largest) { 49061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach largest = iter->BlockSize; 49161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach candidateBlock = iter; 49261425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 49361425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach iter = iter->Next; 49461425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 49561425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 49661425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach largest = largest - sizeof(MemoryRangeHeader); 49761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 49861425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // If this block isn't big enough for the allocation desired, allocate 49961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // another block of memory and add it to the free list. 50061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) { 50161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); 50261425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach candidateBlock = allocateNewCodeSlab((size_t)Size); 50361425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 50461425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 50561425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // Select this candidate block for allocation 50661425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach CurBlock = candidateBlock; 50761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 50861425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // Allocate the entire memory block. 50961425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeMemoryList = candidateBlock->AllocateBlock(); 51061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach // Release the memory at the end of this block that isn't needed. 51161425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size); 512e5cb25f860bb4ffe6908004e6a04c8d21b7d2a98Elena Demikhovsky uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock); 513e5cb25f860bb4ffe6908004e6a04c8d21b7d2a98Elena Demikhovsky return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment); 51461425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach } 51561425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach 51661425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach /// allocateDataSection - Allocate memory for a data section. 51761425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, 5186eb43d295625cd2ff314c59b49d4fd11f3348cadFilip Pizlo unsigned SectionID, StringRef SectionName, 51936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IsReadOnly) override { 52061425c0a7f4e3608a85f7bbf254cd052a15b7446Jim Grosbach return (uint8_t*)DataAllocator.Allocate(Size, Alignment); 521489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 522489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 52336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool finalizeMemory(std::string *ErrMsg) override { 52453608a34ce3f0969e9fb01eaa983422761011e03Andrew Kaylor return false; 52553608a34ce3f0969e9fb01eaa983422761011e03Andrew Kaylor } 52653608a34ce3f0969e9fb01eaa983422761011e03Andrew Kaylor 52736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint8_t *getGOTBase() const override { 5288907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return GOTBase; 5298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 5305cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 5311e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin void deallocateBlock(void *Block) { 5328907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Find the block that is allocated for this function. 5331e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1; 5348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner assert(MemRange->ThisAllocated && "Block isn't allocated!"); 535489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Fill the buffer with garbage! 537489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin if (PoisonMemory) { 538489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); 539489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 540489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Free the memory. 5428907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); 5431e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin } 544489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5451e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin /// deallocateFunctionBody - Deallocate all memory for the specified 5461e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin /// function body. 54736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void deallocateFunctionBody(void *Body) override { 54871e0b7c560200a87ea49080f581153f9a2c9db72Nicolas Geoffray if (Body) deallocateBlock(Body); 5491e8613212286a8066001c8a3f516da89d250e05dJeffrey Yasskin } 550489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 551cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// setMemoryWritable - When code generation is in progress, 552cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// the code pages may need permissions changed. 55336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void setMemoryWritable() override { 55410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 55510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::setWritable(CodeSlabs[i]); 556cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach } 557cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// setMemoryExecutable - When code generation is done and we're ready to 558cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach /// start execution, the code pages may need permissions changed. 55936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void setMemoryExecutable() override { 56010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 56110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::setExecutable(CodeSlabs[i]); 562cce6c297c54b4c9c8615c77e97cd64e70812ea60Jim Grosbach } 563489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 564489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin /// setPoisonMemory - Controls whether we write garbage over freed memory. 565489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin /// 56636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void setPoisonMemory(bool poison) override { 567489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin PoisonMemory = poison; 568489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin } 5698907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner }; 5708907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 5718907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 572dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) { 57310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = JMM.allocateNewSlab(Size); 574dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return B.base(); 57510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 57610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 577dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid JITAllocator::Deallocate(void *Slab, size_t Size) { 578dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines sys::MemoryBlock B(Slab, Size); 57910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::ReleaseRWX(B); 58010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 58110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 58210b4fc552f984dc978298d50c09c97c0764962fcReid KlecknerDefaultJITMemoryManager::DefaultJITMemoryManager() 583dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : 584489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin#ifdef NDEBUG 585dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PoisonMemory(false), 586bc4707a2554ac04ba006bf70035e7bc7270236a9Evan Cheng#else 587dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PoisonMemory(true), 588bc4707a2554ac04ba006bf70035e7bc7270236a9Evan Cheng#endif 589dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) { 5908907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 59110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Allocate space for code. 59210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); 59310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner CodeSlabs.push_back(MemBlock); 59410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uint8_t *MemBase = (uint8_t*)MemBlock.base(); 595489393d7b92107cc3de17d8dbe7dd11ab7395fdcJeffrey Yasskin 5968907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // We set up the memory chunk with 4 mem regions, like this: 5978907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ START 5988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Free #0 ] -> Large space to allocate functions from. 5998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Allocated #1 ] -> Tiny space to separate regions. 6008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Free #2 ] -> Tiny space so there is always at least 1 free block. 6018907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. 6028907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // END ] 6038907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // 6048907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // The last three blocks are never deallocated or touched. 6055cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 6068907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add MemoryRangeHeader to the end of the memory region, indicating that 6078907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // the space after the block of memory is allocated. This is block #3. 6088907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; 6098907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem3->ThisAllocated = 1; 6108907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem3->PrevAllocated = 0; 61110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Mem3->BlockSize = sizeof(MemoryRangeHeader); 6125cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 6138907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Add a tiny free region so that the free list always has one entry. 6145cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher FreeRangeHeader *Mem2 = 6158907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); 6168907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->ThisAllocated = 0; 6178907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->PrevAllocated = 1; 6188907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); 6198907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->SetEndOfBlockSizeMarker(); 6208907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->Prev = Mem2; // Mem2 *is* the free list for now. 6218907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem2->Next = Mem2; 6228907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6238907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner /// Add a tiny allocated region so that Mem2 is never coalesced away. 6248907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; 6258907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem1->ThisAllocated = 1; 6268907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem1->PrevAllocated = 0; 62710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Mem1->BlockSize = sizeof(MemoryRangeHeader); 6285cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 6298907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Add a FreeRangeHeader to the start of the function body region, indicating 6308907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // that the space is free. Mark the previous block allocated so we never look 6318907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // at it. 63210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; 6338907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->ThisAllocated = 0; 6348907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->PrevAllocated = 1; 6358907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->BlockSize = (char*)Mem1-(char*)Mem0; 6368907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->SetEndOfBlockSizeMarker(); 6378907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner Mem0->AddToFreeList(Mem2); 6385cf0aedd0b8df5479a7df9dddbd029070eaae776Eric Christopher 6398907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Start out with the freelist pointing to Mem0. 6408907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner FreeMemoryList = Mem0; 6418907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines GOTBase = nullptr; 6438907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6448907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6458907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattnervoid DefaultJITMemoryManager::AllocateGOT() { 646dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(!GOTBase && "Cannot allocate the got multiple times"); 647186c670e15828327960d05a652ec43ae768c9b8eBruno Cardoso Lopes GOTBase = new uint8_t[sizeof(void*) * 8192]; 6488907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner HasGOT = true; 6498907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6508907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 6518907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerDefaultJITMemoryManager::~DefaultJITMemoryManager() { 65210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) 65310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::Memory::ReleaseRWX(CodeSlabs[i]); 6548907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 65510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner delete[] GOTBase; 6564bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner} 6574bf370698a456bcc96d26184785eb4f5fab396f2Reid Kleckner 65810b4fc552f984dc978298d50c09c97c0764962fcReid Klecknersys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { 6598907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner // Allocate a new block close to the last one. 6608907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner std::string ErrMsg; 661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr; 66210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); 663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!B.base()) { 66475361b69f3f327842b9dad69fa7f28ae3b688412Chris Lattner report_fatal_error("Allocation failed when allocating new memory in the" 6651bd7335a17010bd4d8f86736cf73cac9f3fb80a5Benjamin Kramer " JIT\n" + Twine(ErrMsg)); 6668907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner } 66710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner LastSlab = B; 66810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner ++NumSlabs; 66901248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner // Initialize the slab to garbage when debugging. 67001248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner if (PoisonMemory) { 67101248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner memset(B.base(), 0xCD, B.size()); 67201248e671100fbd6eac6bc3646096dc75ec885d1Reid Kleckner } 6738907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return B; 6748907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 6758907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 67610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// CheckInvariants - For testing only. Return "" if all internal invariants 67710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// are preserved, and a helpful error message otherwise. For free and 67810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// allocated blocks, make sure that adding BlockSize gives a valid block. 67910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// For free blocks, make sure they're in the free list and that their end of 68010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// block size marker is correct. This function should return an error before 68110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// accessing bad memory. This function is defined here instead of in 68210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// JITMemoryManagerTest.cpp so that we don't have to expose all of the 68310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner/// implementation details of DefaultJITMemoryManager. 68410b4fc552f984dc978298d50c09c97c0764962fcReid Klecknerbool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { 68510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner raw_string_ostream Err(ErrorStr); 68610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 68710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Construct a the set of FreeRangeHeader pointers so we can query it 68810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // efficiently. 68910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; 69010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader* FreeHead = FreeMemoryList; 69110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRangeHeader* FreeRange = FreeHead; 69210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 69310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner do { 69410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check that the free range pointer is in the blocks we've allocated. 69510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner bool Found = false; 69610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 69710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner E = CodeSlabs.end(); I != E && !Found; ++I) { 69810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *Start = (char*)I->base(); 69910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *End = Start + I->size(); 70010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); 70110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 70210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!Found) { 70310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Corrupt free list; points to " << FreeRange; 70410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 70510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 70610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 70710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (FreeRange->Next->Prev != FreeRange) { 70810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Next and Prev pointers do not match."; 70910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 71010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 71110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 71210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Otherwise, add it to the set. 71310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeHdrSet.insert(FreeRange); 71410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner FreeRange = FreeRange->Next; 71510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } while (FreeRange != FreeHead); 71610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 71710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Go over each block, and look at each MemoryRangeHeader. 71810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), 71910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner E = CodeSlabs.end(); I != E; ++I) { 72010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *Start = (char*)I->base(); 72110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner char *End = Start + I->size(); 72210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 72310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check each memory range. 724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr; 72510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Start <= (char*)Hdr && (char*)Hdr < End; 72610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Hdr = &Hdr->getBlockAfter()) { 72710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (Hdr->ThisAllocated == 0) { 72810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Check that this range is in the free list. 72910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!FreeHdrSet.count(Hdr)) { 73010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Found free header at " << Hdr << " that is not in free list."; 73110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 73210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 73310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 73410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Now make sure the size marker at the end of the block is correct. 73510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; 73610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (!(Start <= (char*)Marker && (char*)Marker < End)) { 73710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Block size in header points out of current MemoryBlock."; 73810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 73910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 74010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (Hdr->BlockSize != *Marker) { 74110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "End of block size marker (" << *Marker << ") " 74210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner << "and BlockSize (" << Hdr->BlockSize << ") don't match."; 74310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 74410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 74510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 74610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 74710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { 74810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " 74910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; 75010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 75110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } else if (!LastHdr && !Hdr->PrevAllocated) { 75210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner Err << "The first header should have PrevAllocated true."; 75310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return false; 75410b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 75510b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 75610b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // Remember the last header. 75710b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner LastHdr = Hdr; 75810b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 75910b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner } 76010b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 76110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner // All invariants are preserved. 76210b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner return true; 76310b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner} 7648907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner 76530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev//===----------------------------------------------------------------------===// 76630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// getPointerToNamedFunction() implementation. 76730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev//===----------------------------------------------------------------------===// 76830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 76930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// AtExitHandlers - List of functions to call when the program exits, 77030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// registered with the atexit() library function. 77130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic std::vector<void (*)()> AtExitHandlers; 77230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 77330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// runAtExitHandlers - Run any functions registered by the program's 77430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// calls to atexit(3), which we intercept and store in 77530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// AtExitHandlers. 77630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// 77730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic void runAtExitHandlers() { 77830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev while (!AtExitHandlers.empty()) { 77930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev void (*Fn)() = AtExitHandlers.back(); 78030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev AtExitHandlers.pop_back(); 78130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev Fn(); 78230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 78330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 78430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 78530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev//===----------------------------------------------------------------------===// 78630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// Function stubs that are invoked instead of certain library calls 78730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// 78830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// Force the following functions to be linked in to anything that uses the 78930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// JIT. This is a hack designed to work around the all-too-clever Glibc 79030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// strategy of making these functions work differently when inlined vs. when 79130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// not inlined, and hiding their real definitions in a separate archive file 79230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// that the dynamic linker can't see. For more info, search for 79330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274. 7941ab6084c9e785415da3a48083d53b25a38f0fb48Andrew Kaylor#if defined(__linux__) && defined(__GLIBC__) 79530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/* stat functions are redirecting to __xstat with a version number. On x86-64 79630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat' 79730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev * available as an exported symbol, so we have to add it explicitly. 79830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev */ 79930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevnamespace { 80030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevclass StatSymbols { 80130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevpublic: 80230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev StatSymbols() { 80330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat); 80430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat); 80530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat); 80630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64); 80730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64); 80830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64); 80930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64); 81030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64); 81130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64); 81230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit); 81330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod); 81430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 81530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev}; 81630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 81730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic StatSymbols initStatSymbols; 81830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#endif // __linux__ 81930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 82030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// jit_exit - Used to intercept the "exit" library call. 82130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic void jit_exit(int Status) { 82230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev runAtExitHandlers(); // Run atexit handlers... 82330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev exit(Status); 82430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 82530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 82630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// jit_atexit - Used to intercept the "atexit" library call. 82730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic int jit_atexit(void (*Fn)()) { 82830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev AtExitHandlers.push_back(Fn); // Take note of atexit handler... 82930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return 0; // Always successful 83030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 83130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 83230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevstatic int jit_noop() { 83330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return 0; 83430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 83530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 83630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev//===----------------------------------------------------------------------===// 83730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev// 83830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// getPointerToNamedFunction - This method returns the address of the specified 83930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// function by using the dynamic loader interface. As such it is only useful 84030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// for resolving library symbols, not code generated symbols. 84130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev/// 84230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshevvoid *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name, 8435fe019835c269ccbfe185276269bc53b3f9a7a86Eli Bendersky bool AbortOnFailure) { 84430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // Check to see if this is one of the functions we want to intercept. Note, 84530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // we cast to intptr_t here to silence a -pedantic warning that complains 84630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // about casting a function pointer to a normal pointer. 84730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Name == "exit") return (void*)(intptr_t)&jit_exit; 84830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Name == "atexit") return (void*)(intptr_t)&jit_atexit; 84930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 85030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // We should not invoke parent's ctors/dtors from generated main()! 85130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // On Mingw and Cygwin, the symbol __main is resolved to 85230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors 85330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // (and register wrong callee's dtors with atexit(3)). 85430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // We expect ExecutionEngine::runStaticConstructorsDestructors() 85530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // is called before ExecutionEngine::runFunctionAsMain() is called. 85630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Name == "__main") return (void*)(intptr_t)&jit_noop; 85730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 85830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev const char *NameStr = Name.c_str(); 85930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // If this is an asm specifier, skip the sentinal. 86030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (NameStr[0] == 1) ++NameStr; 86130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 86230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // If it's an external function, look it up in the process image... 86330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); 86430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Ptr) return Ptr; 86530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 86630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // If it wasn't found and if it starts with an underscore ('_') character, 86730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // try again without the underscore. 86830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (NameStr[0] == '_') { 86930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); 87030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Ptr) return Ptr; 87130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 87230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 87330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These 87430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // are references to hidden visibility symbols that dlsym cannot resolve. 87530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // If we have one of these, strip off $LDBLStub and try again. 87630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#if defined(__APPLE__) && defined(__ppc__) 87730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (Name.size() > 9 && Name[Name.size()-9] == '$' && 87830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) { 87930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // First try turning $LDBLStub into $LDBL128. If that fails, strip it off. 88030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev // This mirrors logic in libSystemStubs.a. 88130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev std::string Prefix = std::string(Name.begin(), Name.end()-9); 88230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false)) 88330b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return Ptr; 88430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (void *Ptr = getPointerToNamedFunction(Prefix, false)) 88530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev return Ptr; 88630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 88730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev#endif 88830b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 88930b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev if (AbortOnFailure) { 89030b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev report_fatal_error("Program used external function '"+Name+ 89130b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev "' which could not be resolved!"); 89230b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev } 893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 89430b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev} 89530b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 89630b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 89730b9e322e159df8eaabb5b194cec6e11ba99c261Danil Malyshev 8988907b4ba479bbfbe630a4c3abab32c7d49749a48Chris LattnerJITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { 8998907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner return new DefaultJITMemoryManager(); 9008907b4ba479bbfbe630a4c3abab32c7d49749a48Chris Lattner} 90110b4fc552f984dc978298d50c09c97c0764962fcReid Kleckner 90236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesconst size_t DefaultJITMemoryManager::DefaultCodeSlabSize; 90336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesconst size_t DefaultJITMemoryManager::DefaultSlabSize; 90436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesconst size_t DefaultJITMemoryManager::DefaultSizeThreshold; 905