smalloc.c revision 7982aa7da64b68f38c7ec685a428334e3af4e340
1d24c33a479fcd68debad128da057814495f65e20Jens Axboe/* 2d24c33a479fcd68debad128da057814495f65e20Jens Axboe * simple memory allocator, backed by mmap() so that it hands out memory 3d24c33a479fcd68debad128da057814495f65e20Jens Axboe * that can be shared across processes and threads 4d24c33a479fcd68debad128da057814495f65e20Jens Axboe */ 5d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <sys/mman.h> 6d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <stdio.h> 7d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <stdlib.h> 8d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <assert.h> 9d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <string.h> 10d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <unistd.h> 11e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran#include <inttypes.h> 12d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <sys/types.h> 13d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <limits.h> 143a8600b4ae9027d02aca7eb1990e5cda4e9f423aGreg Edwards#include <fcntl.h> 15d24c33a479fcd68debad128da057814495f65e20Jens Axboe 166548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe#include "mutex.h" 17b3268b928ec7596a1d294a46a2771c3da789e7d3Jens Axboe#include "arch/arch.h" 183a8600b4ae9027d02aca7eb1990e5cda4e9f423aGreg Edwards#include "os/os.h" 19d24c33a479fcd68debad128da057814495f65e20Jens Axboe 2055f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#define SMALLOC_REDZONE /* define to detect memory corruption */ 21d24c33a479fcd68debad128da057814495f65e20Jens Axboe 22ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#define SMALLOC_BPB 32 /* block size, bytes-per-bit in bitmap */ 23ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#define SMALLOC_BPI (sizeof(unsigned int) * 8) 24ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#define SMALLOC_BPL (SMALLOC_BPB * SMALLOC_BPI) 25ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 265e012980ff5f8396a3e3ebc432e1dd32cebedaa1Jens Axboe#define INITIAL_SIZE 8192*1024 /* new pool size */ 276885768609eafda8648bf50b05115d784cb04fb9Jens Axboe#define MAX_POOLS 128 /* maximum number of pools to setup */ 28d24c33a479fcd68debad128da057814495f65e20Jens Axboe 2955f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#define SMALLOC_PRE_RED 0xdeadbeefU 3055f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#define SMALLOC_POST_RED 0x5aa55aa5U 3155f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 322b386d2569c9078ca9790c4e6d318ec3835b8739Jens Axboeunsigned int smalloc_pool_size = INITIAL_SIZE; 33122426daab0fbd2fce244ac018d737a8f302f974Jens Axboeconst int int_mask = sizeof(int) - 1; 342b386d2569c9078ca9790c4e6d318ec3835b8739Jens Axboe 35d24c33a479fcd68debad128da057814495f65e20Jens Axboestruct pool { 366548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe struct fio_mutex *lock; /* protects this pool */ 37d24c33a479fcd68debad128da057814495f65e20Jens Axboe void *map; /* map of blocks */ 38ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int *bitmap; /* blocks free/busy map */ 39ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int free_blocks; /* free blocks */ 40ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int nr_blocks; /* total blocks */ 41ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int next_non_full; 42ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int mmap_size; 43ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe}; 44ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 45ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestruct block_hdr { 46ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int size; 47ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#ifdef SMALLOC_REDZONE 48ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int prered; 49ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#endif 50d24c33a479fcd68debad128da057814495f65e20Jens Axboe}; 51d24c33a479fcd68debad128da057814495f65e20Jens Axboe 52d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic struct pool mp[MAX_POOLS]; 53d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic unsigned int nr_pools; 54d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic unsigned int last_pool; 556548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboestatic struct fio_mutex *lock; 56d24c33a479fcd68debad128da057814495f65e20Jens Axboe 57d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic inline void pool_lock(struct pool *pool) 58d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 592e3e31e36b3565dcc5e36ba3842e7b207799601fJens Axboe fio_mutex_down(pool->lock); 60d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 61d24c33a479fcd68debad128da057814495f65e20Jens Axboe 62d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic inline void pool_unlock(struct pool *pool) 63d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 642e3e31e36b3565dcc5e36ba3842e7b207799601fJens Axboe fio_mutex_up(pool->lock); 65d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 66d24c33a479fcd68debad128da057814495f65e20Jens Axboe 6765864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboestatic inline void global_read_lock(void) 68d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 692e3e31e36b3565dcc5e36ba3842e7b207799601fJens Axboe fio_mutex_down_read(lock); 70d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 71d24c33a479fcd68debad128da057814495f65e20Jens Axboe 7265864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboestatic inline void global_read_unlock(void) 73d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 742e3e31e36b3565dcc5e36ba3842e7b207799601fJens Axboe fio_mutex_up_read(lock); 7565864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe} 7665864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe 7765864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboestatic inline void global_write_lock(void) 7865864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe{ 792e3e31e36b3565dcc5e36ba3842e7b207799601fJens Axboe fio_mutex_down_write(lock); 8065864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe} 8165864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe 8265864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboestatic inline void global_write_unlock(void) 8365864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe{ 842e3e31e36b3565dcc5e36ba3842e7b207799601fJens Axboe fio_mutex_up_write(lock); 85d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 86d24c33a479fcd68debad128da057814495f65e20Jens Axboe 87d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic inline int ptr_valid(struct pool *pool, void *ptr) 88d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 89dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe unsigned int pool_size = pool->nr_blocks * SMALLOC_BPL; 90ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 91ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return (ptr >= pool->map) && (ptr < pool->map + pool_size); 92d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 93d24c33a479fcd68debad128da057814495f65e20Jens Axboe 94808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboestatic inline unsigned int size_to_blocks(unsigned int size) 95808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboe{ 96808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboe return (size + SMALLOC_BPB - 1) / SMALLOC_BPB; 97808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboe} 98808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboe 99dcb69098630845b53ebb8034014d44b409e16f9eJens Axboestatic int blocks_iter(struct pool *pool, unsigned int pool_idx, 100dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe unsigned int idx, unsigned int nr_blocks, 101ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe int (*func)(unsigned int *map, unsigned int mask)) 102d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 103dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe 104ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe while (nr_blocks) { 105ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int this_blocks, mask; 106dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe unsigned int *map; 107dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe 108dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe if (pool_idx >= pool->nr_blocks) 109dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe return 0; 110dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe 111dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe map = &pool->bitmap[pool_idx]; 112ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 113ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe this_blocks = nr_blocks; 114ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (this_blocks + idx > SMALLOC_BPI) { 115ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe this_blocks = SMALLOC_BPI - idx; 116ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx = SMALLOC_BPI - this_blocks; 117ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 118ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 119ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (this_blocks == SMALLOC_BPI) 120ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe mask = -1U; 121ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe else 122ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe mask = ((1U << this_blocks) - 1) << idx; 123ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 124ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (!func(map, mask)) 125ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return 0; 126ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 127ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe nr_blocks -= this_blocks; 128ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx = 0; 129dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe pool_idx++; 130ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 131ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 132ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return 1; 133d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 134d24c33a479fcd68debad128da057814495f65e20Jens Axboe 135ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic int mask_cmp(unsigned int *map, unsigned int mask) 136d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 137ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return !(*map & mask); 138d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 139d24c33a479fcd68debad128da057814495f65e20Jens Axboe 140ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic int mask_clear(unsigned int *map, unsigned int mask) 141d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 142dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe assert((*map & mask) == mask); 143ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe *map &= ~mask; 144ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return 1; 145d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 146d24c33a479fcd68debad128da057814495f65e20Jens Axboe 147ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic int mask_set(unsigned int *map, unsigned int mask) 148d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 149dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe assert(!(*map & mask)); 150ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe *map |= mask; 151ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return 1; 152d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 153d24c33a479fcd68debad128da057814495f65e20Jens Axboe 154dcb69098630845b53ebb8034014d44b409e16f9eJens Axboestatic int blocks_free(struct pool *pool, unsigned int pool_idx, 155dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe unsigned int idx, unsigned int nr_blocks) 156d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 157dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp); 158d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 159d24c33a479fcd68debad128da057814495f65e20Jens Axboe 160dcb69098630845b53ebb8034014d44b409e16f9eJens Axboestatic void set_blocks(struct pool *pool, unsigned int pool_idx, 161dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe unsigned int idx, unsigned int nr_blocks) 162d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 163dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set); 164d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 165d24c33a479fcd68debad128da057814495f65e20Jens Axboe 166dcb69098630845b53ebb8034014d44b409e16f9eJens Axboestatic void clear_blocks(struct pool *pool, unsigned int pool_idx, 167dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe unsigned int idx, unsigned int nr_blocks) 168d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 169dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear); 170d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 171d24c33a479fcd68debad128da057814495f65e20Jens Axboe 172ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic int find_next_zero(int word, int start) 173ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe{ 174ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe assert(word != -1U); 175271067a6e278dafd62649257fe99ce536acfe7bbJiri Horky word >>= start; 176271067a6e278dafd62649257fe99ce536acfe7bbJiri Horky return ffz(word) + start; 177d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 178d24c33a479fcd68debad128da057814495f65e20Jens Axboe 179adf57099338307b3a4a129475a12aad31e35ade3Jens Axboestatic int add_pool(struct pool *pool, unsigned int alloc_size) 180d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 1818d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe int bitmap_blocks; 182b8a6582e89999f88c574b905b89743762d8080dfJens Axboe void *ptr; 183ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 18455f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#ifdef SMALLOC_REDZONE 185ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size += sizeof(unsigned int); 18655f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#endif 187ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size += sizeof(struct block_hdr); 188ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (alloc_size < INITIAL_SIZE) 189ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size = INITIAL_SIZE; 190ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 191ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe /* round up to nearest full number of blocks */ 192ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size = (alloc_size + SMALLOC_BPL - 1) & ~(SMALLOC_BPL - 1); 193ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe bitmap_blocks = alloc_size / SMALLOC_BPL; 194ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size += bitmap_blocks * sizeof(unsigned int); 195ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->mmap_size = alloc_size; 1960b9d69ecb14045cf3b2622ae922756b9889b25e6Jens Axboe 197ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->nr_blocks = bitmap_blocks; 198ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->free_blocks = bitmap_blocks * SMALLOC_BPB; 199adf57099338307b3a4a129475a12aad31e35ade3Jens Axboe 2008d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, 2018d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe MAP_SHARED | OS_MAP_ANON, -1, 0); 202d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (ptr == MAP_FAILED) 2038d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe goto out_fail; 204d24c33a479fcd68debad128da057814495f65e20Jens Axboe 205ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe memset(ptr, 0, alloc_size); 206ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->map = ptr; 207ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL); 208d24c33a479fcd68debad128da057814495f65e20Jens Axboe 209521da527743088a9bd2ab882f8b64799d49d5848Jens Axboe pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED); 210d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (!pool->lock) 2118d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe goto out_fail; 212d24c33a479fcd68debad128da057814495f65e20Jens Axboe 213d24c33a479fcd68debad128da057814495f65e20Jens Axboe nr_pools++; 214d24c33a479fcd68debad128da057814495f65e20Jens Axboe return 0; 2158d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboeout_fail: 216ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe fprintf(stderr, "smalloc: failed adding pool\n"); 217d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (pool->map) 218ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe munmap(pool->map, pool->mmap_size); 219d24c33a479fcd68debad128da057814495f65e20Jens Axboe return 1; 220d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 221d24c33a479fcd68debad128da057814495f65e20Jens Axboe 222d24c33a479fcd68debad128da057814495f65e20Jens Axboevoid sinit(void) 223d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 2244d4e80f2b4260f2c8b37a8612ce655502a799f7aJens Axboe int ret; 225d24c33a479fcd68debad128da057814495f65e20Jens Axboe 2266548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe lock = fio_mutex_rw_init(); 227adf57099338307b3a4a129475a12aad31e35ade3Jens Axboe ret = add_pool(&mp[0], INITIAL_SIZE); 228d24c33a479fcd68debad128da057814495f65e20Jens Axboe assert(!ret); 229d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 230d24c33a479fcd68debad128da057814495f65e20Jens Axboe 231d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic void cleanup_pool(struct pool *pool) 232d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 233443bb114d963a99082eef916025268a5a107092bJens Axboe /* 234443bb114d963a99082eef916025268a5a107092bJens Axboe * This will also remove the temporary file we used as a backing 235443bb114d963a99082eef916025268a5a107092bJens Axboe * store, it was already unlinked 236443bb114d963a99082eef916025268a5a107092bJens Axboe */ 237ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe munmap(pool->map, pool->mmap_size); 2386548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe 2396548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe if (pool->lock) 2406548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe fio_mutex_remove(pool->lock); 241d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 242d24c33a479fcd68debad128da057814495f65e20Jens Axboe 243d24c33a479fcd68debad128da057814495f65e20Jens Axboevoid scleanup(void) 244d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 245d24c33a479fcd68debad128da057814495f65e20Jens Axboe unsigned int i; 246d24c33a479fcd68debad128da057814495f65e20Jens Axboe 247d24c33a479fcd68debad128da057814495f65e20Jens Axboe for (i = 0; i < nr_pools; i++) 248d24c33a479fcd68debad128da057814495f65e20Jens Axboe cleanup_pool(&mp[i]); 249d24c33a479fcd68debad128da057814495f65e20Jens Axboe 2506548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe if (lock) 2516548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe fio_mutex_remove(lock); 252d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 253d24c33a479fcd68debad128da057814495f65e20Jens Axboe 25489da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe#ifdef SMALLOC_REDZONE 255cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboestatic void *postred_ptr(struct block_hdr *hdr) 256cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe{ 257e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran uintptr_t ptr; 258cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe 259e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int); 260cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe ptr = (ptr + int_mask) & ~int_mask; 261cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe 262cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe return (void *) ptr; 263cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe} 264cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe 265ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic void fill_redzone(struct block_hdr *hdr) 26655f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe{ 267cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe unsigned int *postred = postred_ptr(hdr); 26855f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 269ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe hdr->prered = SMALLOC_PRE_RED; 270ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe *postred = SMALLOC_POST_RED; 271ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe} 27255f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 273ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic void sfree_check_redzone(struct block_hdr *hdr) 274ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe{ 275cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe unsigned int *postred = postred_ptr(hdr); 276ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 277ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (hdr->prered != SMALLOC_PRE_RED) { 27855f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe fprintf(stderr, "smalloc pre redzone destroyed!\n"); 27955f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe fprintf(stderr, " ptr=%p, prered=%x, expected %x\n", 280ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe hdr, hdr->prered, SMALLOC_PRE_RED); 28155f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe assert(0); 28255f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe } 28355f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe if (*postred != SMALLOC_POST_RED) { 28455f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe fprintf(stderr, "smalloc post redzone destroyed!\n"); 28555f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe fprintf(stderr, " ptr=%p, postred=%x, expected %x\n", 286ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe hdr, *postred, SMALLOC_POST_RED); 28755f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe assert(0); 28855f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe } 28989da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe} 29089da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe#else 29189da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboestatic void fill_redzone(struct block_hdr *hdr) 29289da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe{ 29355f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe} 29455f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 29589da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboestatic void sfree_check_redzone(struct block_hdr *hdr) 29689da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe{ 29789da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe} 29889da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe#endif 29989da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe 300d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic void sfree_pool(struct pool *pool, void *ptr) 301d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 302ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe struct block_hdr *hdr; 303179446e0122d04ddbeb3d353ff68350490b0841cJens Axboe unsigned int i, idx; 304ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned long offset; 305d24c33a479fcd68debad128da057814495f65e20Jens Axboe 306d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (!ptr) 307d24c33a479fcd68debad128da057814495f65e20Jens Axboe return; 308d24c33a479fcd68debad128da057814495f65e20Jens Axboe 309ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe ptr -= sizeof(*hdr); 310ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe hdr = ptr; 31155f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 312d24c33a479fcd68debad128da057814495f65e20Jens Axboe assert(ptr_valid(pool, ptr)); 313d24c33a479fcd68debad128da057814495f65e20Jens Axboe 314ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe sfree_check_redzone(hdr); 315d24c33a479fcd68debad128da057814495f65e20Jens Axboe 316ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe offset = ptr - pool->map; 317ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe i = offset / SMALLOC_BPL; 318ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx = (offset % SMALLOC_BPL) / SMALLOC_BPB; 319d24c33a479fcd68debad128da057814495f65e20Jens Axboe 320ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool_lock(pool); 321dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe clear_blocks(pool, i, idx, size_to_blocks(hdr->size)); 322ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (i < pool->next_non_full) 323ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->next_non_full = i; 324179446e0122d04ddbeb3d353ff68350490b0841cJens Axboe pool->free_blocks += size_to_blocks(hdr->size); 325d24c33a479fcd68debad128da057814495f65e20Jens Axboe pool_unlock(pool); 326d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 327d24c33a479fcd68debad128da057814495f65e20Jens Axboe 328d24c33a479fcd68debad128da057814495f65e20Jens Axboevoid sfree(void *ptr) 329d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 330d24c33a479fcd68debad128da057814495f65e20Jens Axboe struct pool *pool = NULL; 331d24c33a479fcd68debad128da057814495f65e20Jens Axboe unsigned int i; 332d24c33a479fcd68debad128da057814495f65e20Jens Axboe 3338e5732e558509fc0f4ccdeb1e4d01ad038aead06Jens Axboe if (!ptr) 3348e5732e558509fc0f4ccdeb1e4d01ad038aead06Jens Axboe return; 3358e5732e558509fc0f4ccdeb1e4d01ad038aead06Jens Axboe 33665864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe global_read_lock(); 337d24c33a479fcd68debad128da057814495f65e20Jens Axboe 338d24c33a479fcd68debad128da057814495f65e20Jens Axboe for (i = 0; i < nr_pools; i++) { 339d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (ptr_valid(&mp[i], ptr)) { 340d24c33a479fcd68debad128da057814495f65e20Jens Axboe pool = &mp[i]; 341d24c33a479fcd68debad128da057814495f65e20Jens Axboe break; 342d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 343d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 344d24c33a479fcd68debad128da057814495f65e20Jens Axboe 34565864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe global_read_unlock(); 346d24c33a479fcd68debad128da057814495f65e20Jens Axboe 347d24c33a479fcd68debad128da057814495f65e20Jens Axboe assert(pool); 348d24c33a479fcd68debad128da057814495f65e20Jens Axboe sfree_pool(pool, ptr); 349d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 350d24c33a479fcd68debad128da057814495f65e20Jens Axboe 35155f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboestatic void *__smalloc_pool(struct pool *pool, unsigned int size) 352d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 353ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int nr_blocks; 354ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int i; 355ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int offset; 356ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int last_idx; 357ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe void *ret = NULL; 358d24c33a479fcd68debad128da057814495f65e20Jens Axboe 359d24c33a479fcd68debad128da057814495f65e20Jens Axboe pool_lock(pool); 360179446e0122d04ddbeb3d353ff68350490b0841cJens Axboe 361179446e0122d04ddbeb3d353ff68350490b0841cJens Axboe nr_blocks = size_to_blocks(size); 362ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (nr_blocks > pool->free_blocks) 3638e5732e558509fc0f4ccdeb1e4d01ad038aead06Jens Axboe goto fail; 3645ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe 365ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe i = pool->next_non_full; 366ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe last_idx = 0; 367ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe offset = -1U; 368ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe while (i < pool->nr_blocks) { 369ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int idx; 370d24c33a479fcd68debad128da057814495f65e20Jens Axboe 371ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (pool->bitmap[i] == -1U) { 372ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe i++; 373ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->next_non_full = i; 374ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe last_idx = 0; 375ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe continue; 376ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 377d24c33a479fcd68debad128da057814495f65e20Jens Axboe 378ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx = find_next_zero(pool->bitmap[i], last_idx); 379dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe if (!blocks_free(pool, i, idx, nr_blocks)) { 380ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx += nr_blocks; 381ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (idx < SMALLOC_BPI) 382ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe last_idx = idx; 383ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe else { 384ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe last_idx = 0; 385ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe while (idx >= SMALLOC_BPI) { 386ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe i++; 387ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx -= SMALLOC_BPI; 388ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 389ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 390ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe continue; 391d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 392dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe set_blocks(pool, i, idx, nr_blocks); 393ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe offset = i * SMALLOC_BPL + idx * SMALLOC_BPB; 394ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe break; 395ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 396ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 397ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (i < pool->nr_blocks) { 398ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->free_blocks -= nr_blocks; 399ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe ret = pool->map + offset; 400d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 401ec996e9c8f3baacababec55b3be746e035099c14Jens Axboefail: 402d24c33a479fcd68debad128da057814495f65e20Jens Axboe pool_unlock(pool); 403ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return ret; 404d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 405d24c33a479fcd68debad128da057814495f65e20Jens Axboe 40655f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboestatic void *smalloc_pool(struct pool *pool, unsigned int size) 40755f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe{ 40889da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe unsigned int alloc_size = size + sizeof(struct block_hdr); 40955f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe void *ptr; 41055f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 411cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe /* 412122426daab0fbd2fce244ac018d737a8f302f974Jens Axboe * Round to int alignment, so that the postred pointer will 413122426daab0fbd2fce244ac018d737a8f302f974Jens Axboe * be naturally aligned as well. 414cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe */ 415ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#ifdef SMALLOC_REDZONE 416122426daab0fbd2fce244ac018d737a8f302f974Jens Axboe alloc_size += sizeof(unsigned int); 417122426daab0fbd2fce244ac018d737a8f302f974Jens Axboe alloc_size = (alloc_size + int_mask) & ~int_mask; 418ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#endif 419ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 420ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe ptr = __smalloc_pool(pool, alloc_size); 42189da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe if (ptr) { 42289da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe struct block_hdr *hdr = ptr; 42355f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 42489da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe hdr->size = alloc_size; 42589da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe fill_redzone(hdr); 42655f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 42789da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe ptr += sizeof(*hdr); 42889da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe memset(ptr, 0, size); 42989da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe } 430ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 43155f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe return ptr; 43255f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe} 43355f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 4347982aa7da64b68f38c7ec685a428334e3af4e340Jens Axboevoid *smalloc(size_t size) 435d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 436d24c33a479fcd68debad128da057814495f65e20Jens Axboe unsigned int i; 437d24c33a479fcd68debad128da057814495f65e20Jens Axboe 4387982aa7da64b68f38c7ec685a428334e3af4e340Jens Axboe if (size != (unsigned int) size) 4397982aa7da64b68f38c7ec685a428334e3af4e340Jens Axboe return NULL; 4407982aa7da64b68f38c7ec685a428334e3af4e340Jens Axboe 441d1271dc19c335c39298e16b825f84fdd8650132eShaozhi Shawn Ye global_write_lock(); 442d24c33a479fcd68debad128da057814495f65e20Jens Axboe i = last_pool; 443d24c33a479fcd68debad128da057814495f65e20Jens Axboe 444d24c33a479fcd68debad128da057814495f65e20Jens Axboe do { 445d24c33a479fcd68debad128da057814495f65e20Jens Axboe for (; i < nr_pools; i++) { 446d24c33a479fcd68debad128da057814495f65e20Jens Axboe void *ptr = smalloc_pool(&mp[i], size); 447d24c33a479fcd68debad128da057814495f65e20Jens Axboe 448d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (ptr) { 449d24c33a479fcd68debad128da057814495f65e20Jens Axboe last_pool = i; 450d1271dc19c335c39298e16b825f84fdd8650132eShaozhi Shawn Ye global_write_unlock(); 451d24c33a479fcd68debad128da057814495f65e20Jens Axboe return ptr; 452d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 453d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 454d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (last_pool) { 455d24c33a479fcd68debad128da057814495f65e20Jens Axboe last_pool = 0; 456d24c33a479fcd68debad128da057814495f65e20Jens Axboe continue; 457d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 458d24c33a479fcd68debad128da057814495f65e20Jens Axboe 459ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (nr_pools + 1 > MAX_POOLS) 460d24c33a479fcd68debad128da057814495f65e20Jens Axboe break; 461d24c33a479fcd68debad128da057814495f65e20Jens Axboe else { 462d24c33a479fcd68debad128da057814495f65e20Jens Axboe i = nr_pools; 463adf57099338307b3a4a129475a12aad31e35ade3Jens Axboe if (add_pool(&mp[nr_pools], size)) 46465864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe goto out; 465d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 466d24c33a479fcd68debad128da057814495f65e20Jens Axboe } while (1); 467d24c33a479fcd68debad128da057814495f65e20Jens Axboe 46865864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboeout: 469d1271dc19c335c39298e16b825f84fdd8650132eShaozhi Shawn Ye global_write_unlock(); 470d24c33a479fcd68debad128da057814495f65e20Jens Axboe return NULL; 471d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 472d24c33a479fcd68debad128da057814495f65e20Jens Axboe 473d24c33a479fcd68debad128da057814495f65e20Jens Axboechar *smalloc_strdup(const char *str) 474d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 475d24c33a479fcd68debad128da057814495f65e20Jens Axboe char *ptr; 476d24c33a479fcd68debad128da057814495f65e20Jens Axboe 477d24c33a479fcd68debad128da057814495f65e20Jens Axboe ptr = smalloc(strlen(str) + 1); 478d24c33a479fcd68debad128da057814495f65e20Jens Axboe strcpy(ptr, str); 479d24c33a479fcd68debad128da057814495f65e20Jens Axboe return ptr; 480d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 481