1// Copyright (c) 2005, Google Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8//     * Redistributions of source code must retain the above copyright
9// notice, this list of conditions and the following disclaimer.
10//     * Redistributions in binary form must reproduce the above
11// copyright notice, this list of conditions and the following disclaimer
12// in the documentation and/or other materials provided with the
13// distribution.
14//     * Neither the name of Google Inc. nor the names of its
15// contributors may be used to endorse or promote products derived from
16// this software without specific prior written permission.
17//
18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30// ---
31// Author: Sanjay Ghemawat <opensource@google.com>
32
33// We define mmap() and mmap64(), which somewhat reimplements libc's mmap
34// syscall stubs.  Unfortunately libc only exports the stubs via weak symbols
35// (which we're overriding with our mmap64() and mmap() wrappers) so we can't
36// just call through to them.
37
38#ifndef __linux
39# error Should only be including malloc_hook_mmap_linux.h on linux systems.
40#endif
41
42#include <unistd.h>
43#include <syscall.h>
44#include <sys/mman.h>
45#include <errno.h>
46#include "base/linux_syscall_support.h"
47
48// The x86-32 case and the x86-64 case differ:
49// 32b has a mmap2() syscall, 64b does not.
50// 64b and 32b have different calling conventions for mmap().
51
52// I test for 64-bit first so I don't have to do things like
53// '#if (defined(__mips__) && !defined(__MIPS64__))' as a mips32 check.
54#if defined(__x86_64__) || defined(__PPC64__) || (defined(_MIPS_SIM) && _MIPS_SIM == _ABI64)
55
56static inline void* do_mmap64(void *start, size_t length,
57                              int prot, int flags,
58                              int fd, __off64_t offset) __THROW {
59  return sys_mmap(start, length, prot, flags, fd, offset);
60}
61
62#define MALLOC_HOOK_HAVE_DO_MMAP64 1
63
64#elif defined(__i386__) || defined(__PPC__) || defined(__mips__) || \
65      defined(__arm__)
66
67static inline void* do_mmap64(void *start, size_t length,
68                              int prot, int flags,
69                              int fd, __off64_t offset) __THROW {
70  void *result;
71
72  // Try mmap2() unless it's not supported
73  static bool have_mmap2 = true;
74  if (have_mmap2) {
75    static int pagesize = 0;
76    if (!pagesize) pagesize = getpagesize();
77
78    // Check that the offset is page aligned
79    if (offset & (pagesize - 1)) {
80      result = MAP_FAILED;
81      errno = EINVAL;
82      goto out;
83    }
84
85    result = (void *)syscall(SYS_mmap2,
86                             start, length, prot, flags, fd,
87                             (off_t) (offset / pagesize));
88    if (result != MAP_FAILED || errno != ENOSYS)  goto out;
89
90    // We don't have mmap2() after all - don't bother trying it in future
91    have_mmap2 = false;
92  }
93
94  if (((off_t)offset) != offset) {
95    // If we're trying to map a 64-bit offset, fail now since we don't
96    // have 64-bit mmap() support.
97    result = MAP_FAILED;
98    errno = EINVAL;
99    goto out;
100  }
101
102#ifdef __NR_mmap
103  {
104    // Fall back to old 32-bit offset mmap() call
105    // Old syscall interface cannot handle six args, so pass in an array
106    int32 args[6] = { (int32) start, (int32) length, prot, flags, fd,
107                      (off_t) offset };
108    result = (void *)syscall(SYS_mmap, args);
109  }
110#else
111  // Some Linux ports like ARM EABI Linux has no mmap, just mmap2.
112  result = MAP_FAILED;
113#endif
114
115 out:
116  return result;
117}
118
119#define MALLOC_HOOK_HAVE_DO_MMAP64 1
120
121#endif  // #if defined(__x86_64__)
122
123
124#ifdef MALLOC_HOOK_HAVE_DO_MMAP64
125
126// We use do_mmap64 abstraction to put MallocHook::InvokeMmapHook
127// calls right into mmap and mmap64, so that the stack frames in the caller's
128// stack are at the same offsets for all the calls of memory allocating
129// functions.
130
131// Put all callers of MallocHook::Invoke* in this module into
132// malloc_hook section,
133// so that MallocHook::GetCallerStackTrace can function accurately:
134
135// Make sure mmap doesn't get #define'd away by <sys/mman.h>
136# undef mmap
137
138extern "C" {
139  void* mmap64(void *start, size_t length, int prot, int flags,
140               int fd, __off64_t offset  ) __THROW
141    ATTRIBUTE_SECTION(malloc_hook);
142  void* mmap(void *start, size_t length,int prot, int flags,
143             int fd, off_t offset) __THROW
144    ATTRIBUTE_SECTION(malloc_hook);
145  int munmap(void* start, size_t length) __THROW
146    ATTRIBUTE_SECTION(malloc_hook);
147  void* mremap(void* old_addr, size_t old_size, size_t new_size,
148               int flags, ...) __THROW
149    ATTRIBUTE_SECTION(malloc_hook);
150  void* sbrk(ptrdiff_t increment) __THROW
151    ATTRIBUTE_SECTION(malloc_hook);
152}
153
154extern "C" void* mmap64(void *start, size_t length, int prot, int flags,
155                        int fd, __off64_t offset) __THROW {
156  MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset);
157  void *result;
158  if (!MallocHook::InvokeMmapReplacement(
159          start, length, prot, flags, fd, offset, &result)) {
160    result = do_mmap64(start, length, prot, flags, fd, offset);
161  }
162  MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset);
163  return result;
164}
165
166# if !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH)
167
168extern "C" void* mmap(void *start, size_t length, int prot, int flags,
169                      int fd, off_t offset) __THROW {
170  MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset);
171  void *result;
172  if (!MallocHook::InvokeMmapReplacement(
173          start, length, prot, flags, fd, offset, &result)) {
174    result = do_mmap64(start, length, prot, flags, fd,
175                       static_cast<size_t>(offset)); // avoid sign extension
176  }
177  MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset);
178  return result;
179}
180
181# endif  // !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH)
182
183extern "C" int munmap(void* start, size_t length) __THROW {
184  MallocHook::InvokeMunmapHook(start, length);
185  int result;
186  if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) {
187    result = sys_munmap(start, length);
188  }
189  return result;
190}
191
192extern "C" void* mremap(void* old_addr, size_t old_size, size_t new_size,
193                        int flags, ...) __THROW {
194  va_list ap;
195  va_start(ap, flags);
196  void *new_address = va_arg(ap, void *);
197  va_end(ap);
198  void* result = sys_mremap(old_addr, old_size, new_size, flags, new_address);
199  MallocHook::InvokeMremapHook(result, old_addr, old_size, new_size, flags,
200                               new_address);
201  return result;
202}
203
204// libc's version:
205extern "C" void* __sbrk(ptrdiff_t increment);
206
207extern "C" void* sbrk(ptrdiff_t increment) __THROW {
208  MallocHook::InvokePreSbrkHook(increment);
209  void *result = __sbrk(increment);
210  MallocHook::InvokeSbrkHook(result, increment);
211  return result;
212}
213
214/*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot,
215                                         int flags, int fd, off_t offset) {
216  void* result;
217  if (!MallocHook::InvokeMmapReplacement(
218          start, length, prot, flags, fd, offset, &result)) {
219    result = do_mmap64(start, length, prot, flags, fd, offset);
220  }
221  return result;
222}
223
224/*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) {
225  int result;
226  if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) {
227    result = syscall(SYS_munmap, start, length);
228  }
229  return result;
230}
231
232#undef MALLOC_HOOK_HAVE_DO_MMAP64
233
234#endif  // #ifdef MALLOC_HOOK_HAVE_DO_MMAP64
235