1/*
2 * Copyright (C) 2009 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *  * Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 *  * Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in
12 *    the documentation and/or other materials provided with the
13 *    distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29// Contains a thin layer that calls whatever real native allocator
30// has been defined. For the libc shared library, this allows the
31// implementation of a debug malloc that can intercept all of the allocation
32// calls and add special debugging code to attempt to catch allocation
33// errors. All of the debugging code is implemented in a separate shared
34// library that is only loaded when the property "libc.debug.malloc.options"
35// is set to a non-zero value. There are two functions exported to
36// allow ddms, or other external users to get information from the debug
37// allocation.
38//   get_malloc_leak_info: Returns information about all of the known native
39//                         allocations that are currently in use.
40//   free_malloc_leak_info: Frees the data allocated by the call to
41//                          get_malloc_leak_info.
42
43#include <pthread.h>
44
45#include <private/bionic_config.h>
46#include <private/bionic_globals.h>
47#include <private/bionic_malloc_dispatch.h>
48
49#include "jemalloc.h"
50#define Malloc(function)  je_ ## function
51
52static constexpr MallocDispatch __libc_malloc_default_dispatch
53  __attribute__((unused)) = {
54    Malloc(calloc),
55    Malloc(free),
56    Malloc(mallinfo),
57    Malloc(malloc),
58    Malloc(malloc_usable_size),
59    Malloc(memalign),
60    Malloc(posix_memalign),
61#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
62    Malloc(pvalloc),
63#endif
64    Malloc(realloc),
65#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
66    Malloc(valloc),
67#endif
68    Malloc(iterate),
69    Malloc(malloc_disable),
70    Malloc(malloc_enable),
71    Malloc(mallopt),
72    Malloc(aligned_alloc),
73  };
74
75// Malloc hooks.
76void* (*volatile __malloc_hook)(size_t, const void*);
77void* (*volatile __realloc_hook)(void*, size_t, const void*);
78void (*volatile __free_hook)(void*, const void*);
79void* (*volatile __memalign_hook)(size_t, size_t, const void*);
80
81// In a VM process, this is set to 1 after fork()ing out of zygote.
82int gMallocLeakZygoteChild = 0;
83
84// =============================================================================
85// Allocation functions
86// =============================================================================
87extern "C" void* calloc(size_t n_elements, size_t elem_size) {
88  auto _calloc = __libc_globals->malloc_dispatch.calloc;
89  if (__predict_false(_calloc != nullptr)) {
90    return _calloc(n_elements, elem_size);
91  }
92  return Malloc(calloc)(n_elements, elem_size);
93}
94
95extern "C" void free(void* mem) {
96  auto _free = __libc_globals->malloc_dispatch.free;
97  if (__predict_false(_free != nullptr)) {
98    _free(mem);
99  } else {
100    Malloc(free)(mem);
101  }
102}
103
104extern "C" struct mallinfo mallinfo() {
105  auto _mallinfo = __libc_globals->malloc_dispatch.mallinfo;
106  if (__predict_false(_mallinfo != nullptr)) {
107    return _mallinfo();
108  }
109  return Malloc(mallinfo)();
110}
111
112extern "C" int mallopt(int param, int value) {
113  auto _mallopt = __libc_globals->malloc_dispatch.mallopt;
114  if (__predict_false(_mallopt != nullptr)) {
115    return _mallopt(param, value);
116  }
117  return Malloc(mallopt)(param, value);
118}
119
120extern "C" void* malloc(size_t bytes) {
121  auto _malloc = __libc_globals->malloc_dispatch.malloc;
122  if (__predict_false(_malloc != nullptr)) {
123    return _malloc(bytes);
124  }
125  return Malloc(malloc)(bytes);
126}
127
128extern "C" size_t malloc_usable_size(const void* mem) {
129  auto _malloc_usable_size = __libc_globals->malloc_dispatch.malloc_usable_size;
130  if (__predict_false(_malloc_usable_size != nullptr)) {
131    return _malloc_usable_size(mem);
132  }
133  return Malloc(malloc_usable_size)(mem);
134}
135
136extern "C" void* memalign(size_t alignment, size_t bytes) {
137  auto _memalign = __libc_globals->malloc_dispatch.memalign;
138  if (__predict_false(_memalign != nullptr)) {
139    return _memalign(alignment, bytes);
140  }
141  return Malloc(memalign)(alignment, bytes);
142}
143
144extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) {
145  auto _posix_memalign = __libc_globals->malloc_dispatch.posix_memalign;
146  if (__predict_false(_posix_memalign != nullptr)) {
147    return _posix_memalign(memptr, alignment, size);
148  }
149  return Malloc(posix_memalign)(memptr, alignment, size);
150}
151
152extern "C" void* aligned_alloc(size_t alignment, size_t size) {
153  auto _aligned_alloc = __libc_globals->malloc_dispatch.aligned_alloc;
154  if (__predict_false(_aligned_alloc != nullptr)) {
155    return _aligned_alloc(alignment, size);
156  }
157  return Malloc(aligned_alloc)(alignment, size);
158}
159
160extern "C" void* realloc(void* old_mem, size_t bytes) {
161  auto _realloc = __libc_globals->malloc_dispatch.realloc;
162  if (__predict_false(_realloc != nullptr)) {
163    return _realloc(old_mem, bytes);
164  }
165  return Malloc(realloc)(old_mem, bytes);
166}
167
168#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
169extern "C" void* pvalloc(size_t bytes) {
170  auto _pvalloc = __libc_globals->malloc_dispatch.pvalloc;
171  if (__predict_false(_pvalloc != nullptr)) {
172    return _pvalloc(bytes);
173  }
174  return Malloc(pvalloc)(bytes);
175}
176
177extern "C" void* valloc(size_t bytes) {
178  auto _valloc = __libc_globals->malloc_dispatch.valloc;
179  if (__predict_false(_valloc != nullptr)) {
180    return _valloc(bytes);
181  }
182  return Malloc(valloc)(bytes);
183}
184#endif
185
186// We implement malloc debugging only in libc.so, so the code below
187// must be excluded if we compile this file for static libc.a
188#if !defined(LIBC_STATIC)
189
190#include <dlfcn.h>
191#include <stdio.h>
192#include <stdlib.h>
193
194#include <async_safe/log.h>
195#include <sys/system_properties.h>
196
197extern "C" int __cxa_atexit(void (*func)(void *), void *arg, void *dso);
198
199static const char* HOOKS_SHARED_LIB = "libc_malloc_hooks.so";
200static const char* HOOKS_PROPERTY_ENABLE = "libc.debug.hooks.enable";
201static const char* HOOKS_ENV_ENABLE = "LIBC_HOOKS_ENABLE";
202
203static const char* DEBUG_SHARED_LIB = "libc_malloc_debug.so";
204static const char* DEBUG_PROPERTY_OPTIONS = "libc.debug.malloc.options";
205static const char* DEBUG_PROPERTY_PROGRAM = "libc.debug.malloc.program";
206static const char* DEBUG_ENV_OPTIONS = "LIBC_DEBUG_MALLOC_OPTIONS";
207
208enum FunctionEnum : uint8_t {
209  FUNC_INITIALIZE,
210  FUNC_FINALIZE,
211  FUNC_GET_MALLOC_LEAK_INFO,
212  FUNC_FREE_MALLOC_LEAK_INFO,
213  FUNC_MALLOC_BACKTRACE,
214  FUNC_LAST,
215};
216static void* g_functions[FUNC_LAST];
217
218typedef void (*finalize_func_t)();
219typedef bool (*init_func_t)(const MallocDispatch*, int*, const char*);
220typedef void (*get_malloc_leak_info_func_t)(uint8_t**, size_t*, size_t*, size_t*, size_t*);
221typedef void (*free_malloc_leak_info_func_t)(uint8_t*);
222typedef ssize_t (*malloc_backtrace_func_t)(void*, uintptr_t*, size_t);
223
224// =============================================================================
225// Log functions
226// =============================================================================
227#define error_log(format, ...)  \
228    async_safe_format_log(ANDROID_LOG_ERROR, "libc", (format), ##__VA_ARGS__ )
229#define info_log(format, ...)  \
230    async_safe_format_log(ANDROID_LOG_INFO, "libc", (format), ##__VA_ARGS__ )
231// =============================================================================
232
233// =============================================================================
234// Exported for use by ddms.
235// =============================================================================
236
237// Retrieve native heap information.
238//
239// "*info" is set to a buffer we allocate
240// "*overall_size" is set to the size of the "info" buffer
241// "*info_size" is set to the size of a single entry
242// "*total_memory" is set to the sum of all allocations we're tracking; does
243//   not include heap overhead
244// "*backtrace_size" is set to the maximum number of entries in the back trace
245extern "C" void get_malloc_leak_info(uint8_t** info, size_t* overall_size,
246    size_t* info_size, size_t* total_memory, size_t* backtrace_size) {
247  void* func = g_functions[FUNC_GET_MALLOC_LEAK_INFO];
248  if (func == nullptr) {
249    return;
250  }
251  reinterpret_cast<get_malloc_leak_info_func_t>(func)(info, overall_size, info_size, total_memory,
252                                                      backtrace_size);
253}
254
255extern "C" void free_malloc_leak_info(uint8_t* info) {
256  void* func = g_functions[FUNC_FREE_MALLOC_LEAK_INFO];
257  if (func == nullptr) {
258    return;
259  }
260  reinterpret_cast<free_malloc_leak_info_func_t>(func)(info);
261}
262
263// =============================================================================
264
265template<typename FunctionType>
266static bool InitMallocFunction(void* malloc_impl_handler, FunctionType* func, const char* prefix, const char* suffix) {
267  char symbol[128];
268  snprintf(symbol, sizeof(symbol), "%s_%s", prefix, suffix);
269  *func = reinterpret_cast<FunctionType>(dlsym(malloc_impl_handler, symbol));
270  if (*func == nullptr) {
271    error_log("%s: dlsym(\"%s\") failed", getprogname(), symbol);
272    return false;
273  }
274  return true;
275}
276
277static bool InitMallocFunctions(void* impl_handler, MallocDispatch* table, const char* prefix) {
278  if (!InitMallocFunction<MallocCalloc>(impl_handler, &table->calloc, prefix, "calloc")) {
279    return false;
280  }
281  if (!InitMallocFunction<MallocFree>(impl_handler, &table->free, prefix, "free")) {
282    return false;
283  }
284  if (!InitMallocFunction<MallocMallinfo>(impl_handler, &table->mallinfo, prefix, "mallinfo")) {
285    return false;
286  }
287  if (!InitMallocFunction<MallocMallopt>(impl_handler, &table->mallopt, prefix, "mallopt")) {
288    return false;
289  }
290  if (!InitMallocFunction<MallocMalloc>(impl_handler, &table->malloc, prefix, "malloc")) {
291    return false;
292  }
293  if (!InitMallocFunction<MallocMallocUsableSize>(impl_handler, &table->malloc_usable_size, prefix,
294                                                  "malloc_usable_size")) {
295    return false;
296  }
297  if (!InitMallocFunction<MallocMemalign>(impl_handler, &table->memalign, prefix, "memalign")) {
298    return false;
299  }
300  if (!InitMallocFunction<MallocPosixMemalign>(impl_handler, &table->posix_memalign, prefix,
301                                               "posix_memalign")) {
302    return false;
303  }
304  if (!InitMallocFunction<MallocAlignedAlloc>(impl_handler, &table->aligned_alloc,
305                                              prefix, "aligned_alloc")) {
306    return false;
307  }
308  if (!InitMallocFunction<MallocRealloc>(impl_handler, &table->realloc, prefix, "realloc")) {
309    return false;
310  }
311  if (!InitMallocFunction<MallocIterate>(impl_handler, &table->iterate, prefix, "iterate")) {
312    return false;
313  }
314  if (!InitMallocFunction<MallocMallocDisable>(impl_handler, &table->malloc_disable, prefix,
315                                               "malloc_disable")) {
316    return false;
317  }
318  if (!InitMallocFunction<MallocMallocEnable>(impl_handler, &table->malloc_enable, prefix,
319                                              "malloc_enable")) {
320    return false;
321  }
322#if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
323  if (!InitMallocFunction<MallocPvalloc>(impl_handler, &table->pvalloc, prefix, "pvalloc")) {
324    return false;
325  }
326  if (!InitMallocFunction<MallocValloc>(impl_handler, &table->valloc, prefix, "valloc")) {
327    return false;
328  }
329#endif
330
331  return true;
332}
333
334static void malloc_fini_impl(void*) {
335  // Our BSD stdio implementation doesn't close the standard streams,
336  // it only flushes them. Other unclosed FILE*s will show up as
337  // malloc leaks, but to avoid the standard streams showing up in
338  // leak reports, close them here.
339  fclose(stdin);
340  fclose(stdout);
341  fclose(stderr);
342
343  reinterpret_cast<finalize_func_t>(g_functions[FUNC_FINALIZE])();
344}
345
346static bool CheckLoadMallocHooks(char** options) {
347  char* env = getenv(HOOKS_ENV_ENABLE);
348  if ((env == nullptr || env[0] == '\0' || env[0] == '0') &&
349    (__system_property_get(HOOKS_PROPERTY_ENABLE, *options) == 0 || *options[0] == '\0' || *options[0] == '0')) {
350    return false;
351  }
352  *options = nullptr;
353  return true;
354}
355
356static bool CheckLoadMallocDebug(char** options) {
357  // If DEBUG_MALLOC_ENV_OPTIONS is set then it overrides the system properties.
358  char* env = getenv(DEBUG_ENV_OPTIONS);
359  if (env == nullptr || env[0] == '\0') {
360    if (__system_property_get(DEBUG_PROPERTY_OPTIONS, *options) == 0 || *options[0] == '\0') {
361      return false;
362    }
363
364    // Check to see if only a specific program should have debug malloc enabled.
365    char program[PROP_VALUE_MAX];
366    if (__system_property_get(DEBUG_PROPERTY_PROGRAM, program) != 0 &&
367        strstr(getprogname(), program) == nullptr) {
368      return false;
369    }
370  } else {
371    *options = env;
372  }
373  return true;
374}
375
376static void ClearGlobalFunctions() {
377  for (size_t i = 0; i < FUNC_LAST; i++) {
378    g_functions[i] = nullptr;
379  }
380}
381
382static void* LoadSharedLibrary(const char* shared_lib, const char* prefix, MallocDispatch* dispatch_table) {
383  void* impl_handle = dlopen(shared_lib, RTLD_NOW | RTLD_LOCAL);
384  if (impl_handle == nullptr) {
385    error_log("%s: Unable to open shared library %s: %s", getprogname(), shared_lib, dlerror());
386    return nullptr;
387  }
388
389  static constexpr const char* names[] = {
390    "initialize",
391    "finalize",
392    "get_malloc_leak_info",
393    "free_malloc_leak_info",
394    "malloc_backtrace",
395  };
396  for (size_t i = 0; i < FUNC_LAST; i++) {
397    char symbol[128];
398    snprintf(symbol, sizeof(symbol), "%s_%s", prefix, names[i]);
399    g_functions[i] = dlsym(impl_handle, symbol);
400    if (g_functions[i] == nullptr) {
401      error_log("%s: %s routine not found in %s", getprogname(), symbol, shared_lib);
402      dlclose(impl_handle);
403      ClearGlobalFunctions();
404      return nullptr;
405    }
406  }
407
408  if (!InitMallocFunctions(impl_handle, dispatch_table, prefix)) {
409    dlclose(impl_handle);
410    ClearGlobalFunctions();
411    return nullptr;
412  }
413
414  return impl_handle;
415}
416
417// Initializes memory allocation framework once per process.
418static void malloc_init_impl(libc_globals* globals) {
419  const char* prefix;
420  const char* shared_lib;
421  char prop[PROP_VALUE_MAX];
422  char* options = prop;
423  // Prefer malloc debug since it existed first and is a more complete
424  // malloc interceptor than the hooks.
425  if (CheckLoadMallocDebug(&options)) {
426    prefix = "debug";
427    shared_lib = DEBUG_SHARED_LIB;
428  } else if (CheckLoadMallocHooks(&options)) {
429    prefix = "hooks";
430    shared_lib = HOOKS_SHARED_LIB;
431  } else {
432    return;
433  }
434
435  MallocDispatch dispatch_table;
436  void* impl_handle = LoadSharedLibrary(shared_lib, prefix, &dispatch_table);
437  if (impl_handle == nullptr) {
438    return;
439  }
440
441  init_func_t init_func = reinterpret_cast<init_func_t>(g_functions[FUNC_INITIALIZE]);
442  if (!init_func(&__libc_malloc_default_dispatch, &gMallocLeakZygoteChild, options)) {
443    dlclose(impl_handle);
444    ClearGlobalFunctions();
445    return;
446  }
447
448  globals->malloc_dispatch = dispatch_table;
449
450  info_log("%s: malloc %s enabled", getprogname(), prefix);
451
452  // Use atexit to trigger the cleanup function. This avoids a problem
453  // where another atexit function is used to cleanup allocated memory,
454  // but the finalize function was already called. This particular error
455  // seems to be triggered by a zygote spawned process calling exit.
456  int ret_value = __cxa_atexit(malloc_fini_impl, nullptr, nullptr);
457  if (ret_value != 0) {
458    error_log("failed to set atexit cleanup function: %d", ret_value);
459  }
460}
461
462// Initializes memory allocation framework.
463// This routine is called from __libc_init routines in libc_init_dynamic.cpp.
464__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals) {
465  malloc_init_impl(globals);
466}
467#endif  // !LIBC_STATIC
468
469// =============================================================================
470// Exported for use by libmemunreachable.
471// =============================================================================
472
473// Calls callback for every allocation in the anonymous heap mapping
474// [base, base+size).  Must be called between malloc_disable and malloc_enable.
475extern "C" int malloc_iterate(uintptr_t base, size_t size,
476    void (*callback)(uintptr_t base, size_t size, void* arg), void* arg) {
477  auto _iterate = __libc_globals->malloc_dispatch.iterate;
478  if (__predict_false(_iterate != nullptr)) {
479    return _iterate(base, size, callback, arg);
480  }
481  return Malloc(iterate)(base, size, callback, arg);
482}
483
484// Disable calls to malloc so malloc_iterate gets a consistent view of
485// allocated memory.
486extern "C" void malloc_disable() {
487  auto _malloc_disable = __libc_globals->malloc_dispatch.malloc_disable;
488  if (__predict_false(_malloc_disable != nullptr)) {
489    return _malloc_disable();
490  }
491  return Malloc(malloc_disable)();
492}
493
494// Re-enable calls to malloc after a previous call to malloc_disable.
495extern "C" void malloc_enable() {
496  auto _malloc_enable = __libc_globals->malloc_dispatch.malloc_enable;
497  if (__predict_false(_malloc_enable != nullptr)) {
498    return _malloc_enable();
499  }
500  return Malloc(malloc_enable)();
501}
502
503#ifndef LIBC_STATIC
504extern "C" ssize_t malloc_backtrace(void* pointer, uintptr_t* frames, size_t frame_count) {
505  void* func = g_functions[FUNC_MALLOC_BACKTRACE];
506  if (func == nullptr) {
507    return 0;
508  }
509  return reinterpret_cast<malloc_backtrace_func_t>(func)(pointer, frames, frame_count);
510}
511#else
512extern "C" ssize_t malloc_backtrace(void*, uintptr_t*, size_t) {
513  return 0;
514}
515#endif
516