1// Copyright (c) 2007, Google Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8//     * Redistributions of source code must retain the above copyright
9// notice, this list of conditions and the following disclaimer.
10//     * Redistributions in binary form must reproduce the above
11// copyright notice, this list of conditions and the following disclaimer
12// in the documentation and/or other materials provided with the
13// distribution.
14//     * Neither the name of Google Inc. nor the names of its
15// contributors may be used to endorse or promote products derived from
16// this software without specific prior written permission.
17//
18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30// ---
31// Author: Arun Sharma
32//
33// A tcmalloc system allocator that uses a memory based filesystem such as
34// tmpfs or hugetlbfs
35//
36// Since these only exist on linux, we only register this allocator there.
37
38#ifdef __linux
39
40#include <config.h>
41#include <errno.h>                      // for errno, EINVAL
42#include <inttypes.h>                   // for PRId64
43#include <limits.h>                     // for PATH_MAX
44#include <stddef.h>                     // for size_t, NULL
45#ifdef HAVE_STDINT_H
46#include <stdint.h>                     // for int64_t, uintptr_t
47#endif
48#include <stdio.h>                      // for snprintf
49#include <stdlib.h>                     // for mkstemp
50#include <string.h>                     // for strerror
51#include <sys/mman.h>                   // for mmap, MAP_FAILED, etc
52#include <sys/statfs.h>                 // for fstatfs, statfs
53#include <unistd.h>                     // for ftruncate, off_t, unlink
54#include <new>                          // for operator new
55#include <string>
56
57#include <gperftools/malloc_extension.h>
58#include "base/basictypes.h"
59#include "base/googleinit.h"
60#include "base/sysinfo.h"
61#include "internal_logging.h"
62
63// TODO(sanjay): Move the code below into the tcmalloc namespace
64using tcmalloc::kLog;
65using tcmalloc::kCrash;
66using tcmalloc::Log;
67using std::string;
68
69DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""),
70              "Path where hugetlbfs or tmpfs is mounted. The caller is "
71              "responsible for ensuring that the path is unique and does "
72              "not conflict with another process");
73DEFINE_int64(memfs_malloc_limit_mb,
74             EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0),
75             "Limit total allocation size to the "
76             "specified number of MiB.  0 == no limit.");
77DEFINE_bool(memfs_malloc_abort_on_fail,
78            EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false),
79            "abort() whenever memfs_malloc fails to satisfy an allocation "
80            "for any reason.");
81DEFINE_bool(memfs_malloc_ignore_mmap_fail,
82            EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false),
83            "Ignore failures from mmap");
84DEFINE_bool(memfs_malloc_map_private,
85            EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false),
86	    "Use MAP_PRIVATE with mmap");
87
88// Hugetlbfs based allocator for tcmalloc
89class HugetlbSysAllocator: public SysAllocator {
90public:
91  explicit HugetlbSysAllocator(SysAllocator* fallback)
92    : failed_(true),  // To disable allocator until Initialize() is called.
93      big_page_size_(0),
94      hugetlb_fd_(-1),
95      hugetlb_base_(0),
96      fallback_(fallback) {
97  }
98
99  void* Alloc(size_t size, size_t *actual_size, size_t alignment);
100  bool Initialize();
101
102  bool failed_;          // Whether failed to allocate memory.
103
104private:
105  void* AllocInternal(size_t size, size_t *actual_size, size_t alignment);
106
107  int64 big_page_size_;
108  int hugetlb_fd_;       // file descriptor for hugetlb
109  off_t hugetlb_base_;
110
111  SysAllocator* fallback_;  // Default system allocator to fall back to.
112};
113static char hugetlb_space[sizeof(HugetlbSysAllocator)];
114
115// No locking needed here since we assume that tcmalloc calls
116// us with an internal lock held (see tcmalloc/system-alloc.cc).
117void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size,
118                                 size_t alignment) {
119  if (failed_) {
120    return fallback_->Alloc(size, actual_size, alignment);
121  }
122
123  // We don't respond to allocation requests smaller than big_page_size_ unless
124  // the caller is ok to take more than they asked for. Used by MetaDataAlloc.
125  if (actual_size == NULL && size < big_page_size_) {
126    return fallback_->Alloc(size, actual_size, alignment);
127  }
128
129  // Enforce huge page alignment.  Be careful to deal with overflow.
130  size_t new_alignment = alignment;
131  if (new_alignment < big_page_size_) new_alignment = big_page_size_;
132  size_t aligned_size = ((size + new_alignment - 1) /
133                         new_alignment) * new_alignment;
134  if (aligned_size < size) {
135    return fallback_->Alloc(size, actual_size, alignment);
136  }
137
138  void* result = AllocInternal(aligned_size, actual_size, new_alignment);
139  if (result != NULL) {
140    return result;
141  }
142  Log(kLog, __FILE__, __LINE__,
143      "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_);
144  if (FLAGS_memfs_malloc_abort_on_fail) {
145    Log(kCrash, __FILE__, __LINE__,
146        "memfs_malloc_abort_on_fail is set");
147  }
148  return fallback_->Alloc(size, actual_size, alignment);
149}
150
151void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size,
152                                         size_t alignment) {
153  // Ask for extra memory if alignment > pagesize
154  size_t extra = 0;
155  if (alignment > big_page_size_) {
156    extra = alignment - big_page_size_;
157  }
158
159  // Test if this allocation would put us over the limit.
160  off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024;
161  if (limit > 0 && hugetlb_base_ + size + extra > limit) {
162    // Disable the allocator when there's less than one page left.
163    if (limit - hugetlb_base_ < big_page_size_) {
164      Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb");
165      failed_ = true;
166    }
167    else {
168      Log(kLog, __FILE__, __LINE__,
169          "alloc too large (size, bytes left)", size, limit-hugetlb_base_);
170    }
171    return NULL;
172  }
173
174  // This is not needed for hugetlbfs, but needed for tmpfs.  Annoyingly
175  // hugetlbfs returns EINVAL for ftruncate.
176  int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra);
177  if (ret != 0 && errno != EINVAL) {
178    Log(kLog, __FILE__, __LINE__,
179        "ftruncate failed", strerror(errno));
180    failed_ = true;
181    return NULL;
182  }
183
184  // Note: size + extra does not overflow since:
185  //            size + alignment < (1<<NBITS).
186  // and        extra <= alignment
187  // therefore  size + extra < (1<<NBITS)
188  void *result;
189  result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
190                FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED,
191                hugetlb_fd_, hugetlb_base_);
192  if (result == reinterpret_cast<void*>(MAP_FAILED)) {
193    if (!FLAGS_memfs_malloc_ignore_mmap_fail) {
194      Log(kLog, __FILE__, __LINE__,
195          "mmap failed (size, error)", size + extra, strerror(errno));
196      failed_ = true;
197    }
198    return NULL;
199  }
200  uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
201
202  // Adjust the return memory so it is aligned
203  size_t adjust = 0;
204  if ((ptr & (alignment - 1)) != 0) {
205    adjust = alignment - (ptr & (alignment - 1));
206  }
207  ptr += adjust;
208  hugetlb_base_ += (size + extra);
209
210  if (actual_size) {
211    *actual_size = size + extra - adjust;
212  }
213
214  return reinterpret_cast<void*>(ptr);
215}
216
217bool HugetlbSysAllocator::Initialize() {
218  char path[PATH_MAX];
219  const int pathlen = FLAGS_memfs_malloc_path.size();
220  if (pathlen + 8 > sizeof(path)) {
221    Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long");
222    return false;
223  }
224  memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen);
225  memcpy(path + pathlen, ".XXXXXX", 8);  // Also copies terminating \0
226
227  int hugetlb_fd = mkstemp(path);
228  if (hugetlb_fd == -1) {
229    Log(kLog, __FILE__, __LINE__,
230        "warning: unable to create memfs_malloc_path",
231        path, strerror(errno));
232    return false;
233  }
234
235  // Cleanup memory on process exit
236  if (unlink(path) == -1) {
237    Log(kCrash, __FILE__, __LINE__,
238        "fatal: error unlinking memfs_malloc_path", path, strerror(errno));
239    return false;
240  }
241
242  // Use fstatfs to figure out the default page size for memfs
243  struct statfs sfs;
244  if (fstatfs(hugetlb_fd, &sfs) == -1) {
245    Log(kCrash, __FILE__, __LINE__,
246        "fatal: error fstatfs of memfs_malloc_path", strerror(errno));
247    return false;
248  }
249  int64 page_size = sfs.f_bsize;
250
251  hugetlb_fd_ = hugetlb_fd;
252  big_page_size_ = page_size;
253  failed_ = false;
254  return true;
255}
256
257REGISTER_MODULE_INITIALIZER(memfs_malloc, {
258  if (FLAGS_memfs_malloc_path.length()) {
259    SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator();
260    HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc);
261    if (hp->Initialize()) {
262      MallocExtension::instance()->SetSystemAllocator(hp);
263    }
264  }
265});
266
267#endif   /* ifdef __linux */
268