15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Copyright (c) 2007, Google Inc. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// All rights reserved. 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Redistribution and use in source and binary forms, with or without 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// modification, are permitted provided that the following conditions are 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// met: 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Redistributions of source code must retain the above copyright 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// notice, this list of conditions and the following disclaimer. 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Redistributions in binary form must reproduce the above 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// copyright notice, this list of conditions and the following disclaimer 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// in the documentation and/or other materials provided with the 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// distribution. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Neither the name of Google Inc. nor the names of its 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// contributors may be used to endorse or promote products derived from 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// this software without specific prior written permission. 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// --- 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Author: Arun Sharma 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// A tcmalloc system allocator that uses a memory based filesystem such as 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// tmpfs or hugetlbfs 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Since these only exist on linux, we only register this allocator there. 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef __linux 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <config.h> 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <errno.h> // for errno, EINVAL 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <inttypes.h> // for PRId64 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <limits.h> // for PATH_MAX 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stddef.h> // for size_t, NULL 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_STDINT_H 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdint.h> // for int64_t, uintptr_t 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdio.h> // for snprintf 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdlib.h> // for mkstemp 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <string.h> // for strerror 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <sys/mman.h> // for mmap, MAP_FAILED, etc 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <sys/statfs.h> // for fstatfs, statfs 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <unistd.h> // for ftruncate, off_t, unlink 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <new> // for operator new 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <string> 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <gperftools/malloc_extension.h> 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/basictypes.h" 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/googleinit.h" 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/sysinfo.h" 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "internal_logging.h" 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO(sanjay): Move the code below into the tcmalloc namespace 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::kLog; 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::kCrash; 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::Log; 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using std::string; 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Path where hugetlbfs or tmpfs is mounted. The caller is " 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "responsible for ensuring that the path is unique and does " 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "not conflict with another process"); 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_int64(memfs_malloc_limit_mb, 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Limit total allocation size to the " 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "specified number of MiB. 0 == no limit."); 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_bool(memfs_malloc_abort_on_fail, 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false), 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "abort whenever memfs_malloc fails to satisfy an allocation " 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "for any reason."); 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_bool(memfs_malloc_ignore_mmap_fail, 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false), 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Ignore failures from mmap"); 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_bool(memfs_malloc_map_private, 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false), 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Use MAP_PRIVATE with mmap"); 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Hugetlbfs based allocator for tcmalloc 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class HugetlbSysAllocator: public SysAllocator { 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)public: 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) explicit HugetlbSysAllocator(SysAllocator* fallback) 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : failed_(true), // To disable allocator until Initialize() is called. 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) big_page_size_(0), 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) hugetlb_fd_(-1), 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) hugetlb_base_(0), 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fallback_(fallback) { 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* Alloc(size_t size, size_t *actual_size, size_t alignment); 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool Initialize(); 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool failed_; // Whether failed to allocate memory. 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)private: 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* AllocInternal(size_t size, size_t *actual_size, size_t alignment); 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int64 big_page_size_; 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int hugetlb_fd_; // file descriptor for hugetlb 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) off_t hugetlb_base_; 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SysAllocator* fallback_; // Default system allocator to fall back to. 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static char hugetlb_space[sizeof(HugetlbSysAllocator)]; 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// No locking needed here since we assume that tcmalloc calls 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// us with an internal lock held (see tcmalloc/system-alloc.cc). 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size, 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t alignment) { 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (failed_) { 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return fallback_->Alloc(size, actual_size, alignment); 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We don't respond to allocation requests smaller than big_page_size_ unless 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // the caller is ok to take more than they asked for. Used by MetaDataAlloc. 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (actual_size == NULL && size < big_page_size_) { 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return fallback_->Alloc(size, actual_size, alignment); 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Enforce huge page alignment. Be careful to deal with overflow. 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t new_alignment = alignment; 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (new_alignment < big_page_size_) new_alignment = big_page_size_; 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t aligned_size = ((size + new_alignment - 1) / 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_alignment) * new_alignment; 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (aligned_size < size) { 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return fallback_->Alloc(size, actual_size, alignment); 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = AllocInternal(aligned_size, actual_size, new_alignment); 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (result != NULL) { 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kLog, __FILE__, __LINE__, 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_); 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (FLAGS_memfs_malloc_abort_on_fail) { 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "memfs_malloc_abort_on_fail is set"); 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return fallback_->Alloc(size, actual_size, alignment); 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t alignment) { 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Ask for extra memory if alignment > pagesize 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t extra = 0; 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (alignment > big_page_size_) { 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) extra = alignment - big_page_size_; 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Test if this allocation would put us over the limit. 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (limit > 0 && hugetlb_base_ + size + extra > limit) { 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Disable the allocator when there's less than one page left. 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (limit - hugetlb_base_ < big_page_size_) { 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb"); 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) failed_ = true; 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else { 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kLog, __FILE__, __LINE__, 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "alloc too large (size, bytes left)", size, limit-hugetlb_base_); 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NULL; 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // hugetlbfs returns EINVAL for ftruncate. 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ret != 0 && errno != EINVAL) { 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kLog, __FILE__, __LINE__, 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "ftruncate failed", strerror(errno)); 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) failed_ = true; 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NULL; 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Note: size + extra does not overflow since: 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // size + alignment < (1<<NBITS). 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // and extra <= alignment 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // therefore size + extra < (1<<NBITS) 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *result; 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result = mmap(0, size + extra, PROT_WRITE|PROT_READ, 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED, 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) hugetlb_fd_, hugetlb_base_); 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (result == reinterpret_cast<void*>(MAP_FAILED)) { 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!FLAGS_memfs_malloc_ignore_mmap_fail) { 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kLog, __FILE__, __LINE__, 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "mmap failed (size, error)", size + extra, strerror(errno)); 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) failed_ = true; 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NULL; 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uintptr_t ptr = reinterpret_cast<uintptr_t>(result); 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Adjust the return memory so it is aligned 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t adjust = 0; 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if ((ptr & (alignment - 1)) != 0) { 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) adjust = alignment - (ptr & (alignment - 1)); 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ptr += adjust; 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) hugetlb_base_ += (size + extra); 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (actual_size) { 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *actual_size = size + extra - adjust; 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return reinterpret_cast<void*>(ptr); 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool HugetlbSysAllocator::Initialize() { 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char path[PATH_MAX]; 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int pathlen = FLAGS_memfs_malloc_path.size(); 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (pathlen + 8 > sizeof(path)) { 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long"); 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen); 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(path + pathlen, ".XXXXXX", 8); // Also copies terminating \0 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int hugetlb_fd = mkstemp(path); 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (hugetlb_fd == -1) { 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kLog, __FILE__, __LINE__, 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "warning: unable to create memfs_malloc_path", 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) path, strerror(errno)); 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Cleanup memory on process exit 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (unlink(path) == -1) { 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "fatal: error unlinking memfs_malloc_path", path, strerror(errno)); 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Use fstatfs to figure out the default page size for memfs 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) struct statfs sfs; 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (fstatfs(hugetlb_fd, &sfs) == -1) { 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "fatal: error fstatfs of memfs_malloc_path", strerror(errno)); 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int64 page_size = sfs.f_bsize; 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) hugetlb_fd_ = hugetlb_fd; 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) big_page_size_ = page_size; 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) failed_ = false; 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_MODULE_INITIALIZER(memfs_malloc, { 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (FLAGS_memfs_malloc_path.length()) { 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator(); 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc); 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (hp->Initialize()) { 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocExtension::instance()->SetSystemAllocator(hp); 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}); 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif /* ifdef __linux */ 268