1d24c33a479fcd68debad128da057814495f65e20Jens Axboe/* 2d24c33a479fcd68debad128da057814495f65e20Jens Axboe * simple memory allocator, backed by mmap() so that it hands out memory 3d24c33a479fcd68debad128da057814495f65e20Jens Axboe * that can be shared across processes and threads 4d24c33a479fcd68debad128da057814495f65e20Jens Axboe */ 5d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <sys/mman.h> 6d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <stdio.h> 7d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <stdlib.h> 8d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <assert.h> 9d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <string.h> 10d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <unistd.h> 11e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran#include <inttypes.h> 12d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <sys/types.h> 13d24c33a479fcd68debad128da057814495f65e20Jens Axboe#include <limits.h> 143a8600b4ae9027d02aca7eb1990e5cda4e9f423aGreg Edwards#include <fcntl.h> 15d24c33a479fcd68debad128da057814495f65e20Jens Axboe 166548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe#include "mutex.h" 17b3268b928ec7596a1d294a46a2771c3da789e7d3Jens Axboe#include "arch/arch.h" 183a8600b4ae9027d02aca7eb1990e5cda4e9f423aGreg Edwards#include "os/os.h" 1910aa136bddbaa7c845ab4eacb4a9a4a88d6657a3Jens Axboe#include "smalloc.h" 20d24c33a479fcd68debad128da057814495f65e20Jens Axboe 2155f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#define SMALLOC_REDZONE /* define to detect memory corruption */ 22d24c33a479fcd68debad128da057814495f65e20Jens Axboe 23ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#define SMALLOC_BPB 32 /* block size, bytes-per-bit in bitmap */ 24ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#define SMALLOC_BPI (sizeof(unsigned int) * 8) 25ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#define SMALLOC_BPL (SMALLOC_BPB * SMALLOC_BPI) 26ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 275e012980ff5f8396a3e3ebc432e1dd32cebedaa1Jens Axboe#define INITIAL_SIZE 8192*1024 /* new pool size */ 286885768609eafda8648bf50b05115d784cb04fb9Jens Axboe#define MAX_POOLS 128 /* maximum number of pools to setup */ 29d24c33a479fcd68debad128da057814495f65e20Jens Axboe 3055f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#define SMALLOC_PRE_RED 0xdeadbeefU 3155f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#define SMALLOC_POST_RED 0x5aa55aa5U 3255f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 332b386d2569c9078ca9790c4e6d318ec3835b8739Jens Axboeunsigned int smalloc_pool_size = INITIAL_SIZE; 3410aa136bddbaa7c845ab4eacb4a9a4a88d6657a3Jens Axboestatic const int int_mask = sizeof(int) - 1; 352b386d2569c9078ca9790c4e6d318ec3835b8739Jens Axboe 36d24c33a479fcd68debad128da057814495f65e20Jens Axboestruct pool { 376548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe struct fio_mutex *lock; /* protects this pool */ 38d24c33a479fcd68debad128da057814495f65e20Jens Axboe void *map; /* map of blocks */ 39ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int *bitmap; /* blocks free/busy map */ 40a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe size_t free_blocks; /* free blocks */ 41a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe size_t nr_blocks; /* total blocks */ 42a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe size_t next_non_full; 43a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe size_t mmap_size; 44ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe}; 45ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 46ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestruct block_hdr { 47a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe size_t size; 48ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#ifdef SMALLOC_REDZONE 49ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int prered; 50ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#endif 51d24c33a479fcd68debad128da057814495f65e20Jens Axboe}; 52d24c33a479fcd68debad128da057814495f65e20Jens Axboe 53d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic struct pool mp[MAX_POOLS]; 54d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic unsigned int nr_pools; 55d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic unsigned int last_pool; 56d7df1d133b0c3daad4ae4c731e0dae7b0181fd62Jens Axboestatic struct fio_rwlock *lock; 57d24c33a479fcd68debad128da057814495f65e20Jens Axboe 58d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic inline void pool_lock(struct pool *pool) 59d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 602e3e31e36b3565dcc5e36ba3842e7b207799601fJens Axboe fio_mutex_down(pool->lock); 61d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 62d24c33a479fcd68debad128da057814495f65e20Jens Axboe 63d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic inline void pool_unlock(struct pool *pool) 64d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 652e3e31e36b3565dcc5e36ba3842e7b207799601fJens Axboe fio_mutex_up(pool->lock); 66d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 67d24c33a479fcd68debad128da057814495f65e20Jens Axboe 6865864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboestatic inline void global_read_lock(void) 69d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 70d7df1d133b0c3daad4ae4c731e0dae7b0181fd62Jens Axboe fio_rwlock_read(lock); 71d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 72d24c33a479fcd68debad128da057814495f65e20Jens Axboe 7365864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboestatic inline void global_read_unlock(void) 74d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 75d7df1d133b0c3daad4ae4c731e0dae7b0181fd62Jens Axboe fio_rwlock_unlock(lock); 7665864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe} 7765864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe 7865864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboestatic inline void global_write_lock(void) 7965864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe{ 80d7df1d133b0c3daad4ae4c731e0dae7b0181fd62Jens Axboe fio_rwlock_write(lock); 8165864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe} 8265864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe 8365864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboestatic inline void global_write_unlock(void) 8465864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe{ 85d7df1d133b0c3daad4ae4c731e0dae7b0181fd62Jens Axboe fio_rwlock_unlock(lock); 86d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 87d24c33a479fcd68debad128da057814495f65e20Jens Axboe 88d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic inline int ptr_valid(struct pool *pool, void *ptr) 89d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 90dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe unsigned int pool_size = pool->nr_blocks * SMALLOC_BPL; 91ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 92ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return (ptr >= pool->map) && (ptr < pool->map + pool_size); 93d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 94d24c33a479fcd68debad128da057814495f65e20Jens Axboe 95a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboestatic inline size_t size_to_blocks(size_t size) 96808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboe{ 97808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboe return (size + SMALLOC_BPB - 1) / SMALLOC_BPB; 98808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboe} 99808e9ea8dd3dc3d631a7982fa0c4b01547cb811aJens Axboe 100dcb69098630845b53ebb8034014d44b409e16f9eJens Axboestatic int blocks_iter(struct pool *pool, unsigned int pool_idx, 101a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe unsigned int idx, size_t nr_blocks, 102ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe int (*func)(unsigned int *map, unsigned int mask)) 103d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 104dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe 105ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe while (nr_blocks) { 106ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int this_blocks, mask; 107dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe unsigned int *map; 108dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe 109dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe if (pool_idx >= pool->nr_blocks) 110dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe return 0; 111dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe 112dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe map = &pool->bitmap[pool_idx]; 113ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 114ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe this_blocks = nr_blocks; 115ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (this_blocks + idx > SMALLOC_BPI) { 116ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe this_blocks = SMALLOC_BPI - idx; 117ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx = SMALLOC_BPI - this_blocks; 118ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 119ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 120ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (this_blocks == SMALLOC_BPI) 121ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe mask = -1U; 122ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe else 123ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe mask = ((1U << this_blocks) - 1) << idx; 124ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 125ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (!func(map, mask)) 126ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return 0; 127ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 128ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe nr_blocks -= this_blocks; 129ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx = 0; 130dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe pool_idx++; 131ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 132ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 133ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return 1; 134d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 135d24c33a479fcd68debad128da057814495f65e20Jens Axboe 136ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic int mask_cmp(unsigned int *map, unsigned int mask) 137d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 138ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return !(*map & mask); 139d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 140d24c33a479fcd68debad128da057814495f65e20Jens Axboe 141ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic int mask_clear(unsigned int *map, unsigned int mask) 142d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 143dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe assert((*map & mask) == mask); 144ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe *map &= ~mask; 145ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return 1; 146d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 147d24c33a479fcd68debad128da057814495f65e20Jens Axboe 148ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic int mask_set(unsigned int *map, unsigned int mask) 149d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 150dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe assert(!(*map & mask)); 151ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe *map |= mask; 152ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return 1; 153d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 154d24c33a479fcd68debad128da057814495f65e20Jens Axboe 155dcb69098630845b53ebb8034014d44b409e16f9eJens Axboestatic int blocks_free(struct pool *pool, unsigned int pool_idx, 156a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe unsigned int idx, size_t nr_blocks) 157d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 158dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp); 159d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 160d24c33a479fcd68debad128da057814495f65e20Jens Axboe 161dcb69098630845b53ebb8034014d44b409e16f9eJens Axboestatic void set_blocks(struct pool *pool, unsigned int pool_idx, 162a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe unsigned int idx, size_t nr_blocks) 163d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 164dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set); 165d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 166d24c33a479fcd68debad128da057814495f65e20Jens Axboe 167dcb69098630845b53ebb8034014d44b409e16f9eJens Axboestatic void clear_blocks(struct pool *pool, unsigned int pool_idx, 168a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe unsigned int idx, size_t nr_blocks) 169d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 170dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear); 171d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 172d24c33a479fcd68debad128da057814495f65e20Jens Axboe 173ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic int find_next_zero(int word, int start) 174ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe{ 175ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe assert(word != -1U); 176271067a6e278dafd62649257fe99ce536acfe7bbJiri Horky word >>= start; 177271067a6e278dafd62649257fe99ce536acfe7bbJiri Horky return ffz(word) + start; 178d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 179d24c33a479fcd68debad128da057814495f65e20Jens Axboe 180adf57099338307b3a4a129475a12aad31e35ade3Jens Axboestatic int add_pool(struct pool *pool, unsigned int alloc_size) 181d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 1828d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe int bitmap_blocks; 183b8a6582e89999f88c574b905b89743762d8080dfJens Axboe void *ptr; 184ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 18555f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#ifdef SMALLOC_REDZONE 186ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size += sizeof(unsigned int); 18755f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe#endif 188ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size += sizeof(struct block_hdr); 189ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (alloc_size < INITIAL_SIZE) 190ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size = INITIAL_SIZE; 191ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 192ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe /* round up to nearest full number of blocks */ 193ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size = (alloc_size + SMALLOC_BPL - 1) & ~(SMALLOC_BPL - 1); 194ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe bitmap_blocks = alloc_size / SMALLOC_BPL; 195ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe alloc_size += bitmap_blocks * sizeof(unsigned int); 196ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->mmap_size = alloc_size; 1970b9d69ecb14045cf3b2622ae922756b9889b25e6Jens Axboe 198ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->nr_blocks = bitmap_blocks; 199ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->free_blocks = bitmap_blocks * SMALLOC_BPB; 200adf57099338307b3a4a129475a12aad31e35ade3Jens Axboe 2018d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, 2028d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe MAP_SHARED | OS_MAP_ANON, -1, 0); 203d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (ptr == MAP_FAILED) 2048d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe goto out_fail; 205d24c33a479fcd68debad128da057814495f65e20Jens Axboe 206ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe memset(ptr, 0, alloc_size); 207ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->map = ptr; 208ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL); 209d24c33a479fcd68debad128da057814495f65e20Jens Axboe 210521da527743088a9bd2ab882f8b64799d49d5848Jens Axboe pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED); 211d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (!pool->lock) 2128d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboe goto out_fail; 213d24c33a479fcd68debad128da057814495f65e20Jens Axboe 214d24c33a479fcd68debad128da057814495f65e20Jens Axboe nr_pools++; 215d24c33a479fcd68debad128da057814495f65e20Jens Axboe return 0; 2168d5844e9df308cc06d86a83d6bf28a29db46b6a9Jens Axboeout_fail: 217ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe fprintf(stderr, "smalloc: failed adding pool\n"); 218d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (pool->map) 219ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe munmap(pool->map, pool->mmap_size); 220d24c33a479fcd68debad128da057814495f65e20Jens Axboe return 1; 221d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 222d24c33a479fcd68debad128da057814495f65e20Jens Axboe 223d24c33a479fcd68debad128da057814495f65e20Jens Axboevoid sinit(void) 224d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 2254d4e80f2b4260f2c8b37a8612ce655502a799f7aJens Axboe int ret; 226d24c33a479fcd68debad128da057814495f65e20Jens Axboe 227d7df1d133b0c3daad4ae4c731e0dae7b0181fd62Jens Axboe lock = fio_rwlock_init(); 228adf57099338307b3a4a129475a12aad31e35ade3Jens Axboe ret = add_pool(&mp[0], INITIAL_SIZE); 229d24c33a479fcd68debad128da057814495f65e20Jens Axboe assert(!ret); 230d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 231d24c33a479fcd68debad128da057814495f65e20Jens Axboe 232d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic void cleanup_pool(struct pool *pool) 233d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 234443bb114d963a99082eef916025268a5a107092bJens Axboe /* 235443bb114d963a99082eef916025268a5a107092bJens Axboe * This will also remove the temporary file we used as a backing 236443bb114d963a99082eef916025268a5a107092bJens Axboe * store, it was already unlinked 237443bb114d963a99082eef916025268a5a107092bJens Axboe */ 238ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe munmap(pool->map, pool->mmap_size); 2396548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe 2406548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe if (pool->lock) 2416548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe fio_mutex_remove(pool->lock); 242d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 243d24c33a479fcd68debad128da057814495f65e20Jens Axboe 244d24c33a479fcd68debad128da057814495f65e20Jens Axboevoid scleanup(void) 245d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 246d24c33a479fcd68debad128da057814495f65e20Jens Axboe unsigned int i; 247d24c33a479fcd68debad128da057814495f65e20Jens Axboe 248d24c33a479fcd68debad128da057814495f65e20Jens Axboe for (i = 0; i < nr_pools; i++) 249d24c33a479fcd68debad128da057814495f65e20Jens Axboe cleanup_pool(&mp[i]); 250d24c33a479fcd68debad128da057814495f65e20Jens Axboe 2516548f47f241013c7f6e1bb4b8a341f7d7a78764fJens Axboe if (lock) 252d7df1d133b0c3daad4ae4c731e0dae7b0181fd62Jens Axboe fio_rwlock_remove(lock); 253d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 254d24c33a479fcd68debad128da057814495f65e20Jens Axboe 25589da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe#ifdef SMALLOC_REDZONE 256cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboestatic void *postred_ptr(struct block_hdr *hdr) 257cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe{ 258e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran uintptr_t ptr; 259cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe 260e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int); 261cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe ptr = (ptr + int_mask) & ~int_mask; 262cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe 263cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe return (void *) ptr; 264cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe} 265cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe 266ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic void fill_redzone(struct block_hdr *hdr) 26755f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe{ 268cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe unsigned int *postred = postred_ptr(hdr); 26955f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 270ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe hdr->prered = SMALLOC_PRE_RED; 271ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe *postred = SMALLOC_POST_RED; 272ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe} 27355f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 274ec996e9c8f3baacababec55b3be746e035099c14Jens Axboestatic void sfree_check_redzone(struct block_hdr *hdr) 275ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe{ 276cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe unsigned int *postred = postred_ptr(hdr); 277ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 278ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (hdr->prered != SMALLOC_PRE_RED) { 27955f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe fprintf(stderr, "smalloc pre redzone destroyed!\n"); 28055f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe fprintf(stderr, " ptr=%p, prered=%x, expected %x\n", 281ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe hdr, hdr->prered, SMALLOC_PRE_RED); 28255f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe assert(0); 28355f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe } 28455f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe if (*postred != SMALLOC_POST_RED) { 28555f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe fprintf(stderr, "smalloc post redzone destroyed!\n"); 28655f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe fprintf(stderr, " ptr=%p, postred=%x, expected %x\n", 287ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe hdr, *postred, SMALLOC_POST_RED); 28855f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe assert(0); 28955f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe } 29089da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe} 29189da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe#else 29289da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboestatic void fill_redzone(struct block_hdr *hdr) 29389da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe{ 29455f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe} 29555f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 29689da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboestatic void sfree_check_redzone(struct block_hdr *hdr) 29789da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe{ 29889da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe} 29989da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe#endif 30089da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe 301d24c33a479fcd68debad128da057814495f65e20Jens Axboestatic void sfree_pool(struct pool *pool, void *ptr) 302d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 303ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe struct block_hdr *hdr; 304179446e0122d04ddbeb3d353ff68350490b0841cJens Axboe unsigned int i, idx; 305ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned long offset; 306d24c33a479fcd68debad128da057814495f65e20Jens Axboe 307d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (!ptr) 308d24c33a479fcd68debad128da057814495f65e20Jens Axboe return; 309d24c33a479fcd68debad128da057814495f65e20Jens Axboe 310ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe ptr -= sizeof(*hdr); 311ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe hdr = ptr; 31255f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 313d24c33a479fcd68debad128da057814495f65e20Jens Axboe assert(ptr_valid(pool, ptr)); 314d24c33a479fcd68debad128da057814495f65e20Jens Axboe 315ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe sfree_check_redzone(hdr); 316d24c33a479fcd68debad128da057814495f65e20Jens Axboe 317ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe offset = ptr - pool->map; 318ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe i = offset / SMALLOC_BPL; 319ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx = (offset % SMALLOC_BPL) / SMALLOC_BPB; 320d24c33a479fcd68debad128da057814495f65e20Jens Axboe 321ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool_lock(pool); 322dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe clear_blocks(pool, i, idx, size_to_blocks(hdr->size)); 323ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (i < pool->next_non_full) 324ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->next_non_full = i; 325179446e0122d04ddbeb3d353ff68350490b0841cJens Axboe pool->free_blocks += size_to_blocks(hdr->size); 326d24c33a479fcd68debad128da057814495f65e20Jens Axboe pool_unlock(pool); 327d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 328d24c33a479fcd68debad128da057814495f65e20Jens Axboe 329d24c33a479fcd68debad128da057814495f65e20Jens Axboevoid sfree(void *ptr) 330d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 331d24c33a479fcd68debad128da057814495f65e20Jens Axboe struct pool *pool = NULL; 332d24c33a479fcd68debad128da057814495f65e20Jens Axboe unsigned int i; 333d24c33a479fcd68debad128da057814495f65e20Jens Axboe 3348e5732e558509fc0f4ccdeb1e4d01ad038aead06Jens Axboe if (!ptr) 3358e5732e558509fc0f4ccdeb1e4d01ad038aead06Jens Axboe return; 3368e5732e558509fc0f4ccdeb1e4d01ad038aead06Jens Axboe 33765864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe global_read_lock(); 338d24c33a479fcd68debad128da057814495f65e20Jens Axboe 339d24c33a479fcd68debad128da057814495f65e20Jens Axboe for (i = 0; i < nr_pools; i++) { 340d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (ptr_valid(&mp[i], ptr)) { 341d24c33a479fcd68debad128da057814495f65e20Jens Axboe pool = &mp[i]; 342d24c33a479fcd68debad128da057814495f65e20Jens Axboe break; 343d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 344d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 345d24c33a479fcd68debad128da057814495f65e20Jens Axboe 34665864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe global_read_unlock(); 347d24c33a479fcd68debad128da057814495f65e20Jens Axboe 348d24c33a479fcd68debad128da057814495f65e20Jens Axboe assert(pool); 349d24c33a479fcd68debad128da057814495f65e20Jens Axboe sfree_pool(pool, ptr); 350d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 351d24c33a479fcd68debad128da057814495f65e20Jens Axboe 352a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboestatic void *__smalloc_pool(struct pool *pool, size_t size) 353d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 354a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe size_t nr_blocks; 355ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int i; 356ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int offset; 357ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int last_idx; 358ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe void *ret = NULL; 359d24c33a479fcd68debad128da057814495f65e20Jens Axboe 360d24c33a479fcd68debad128da057814495f65e20Jens Axboe pool_lock(pool); 361179446e0122d04ddbeb3d353ff68350490b0841cJens Axboe 362179446e0122d04ddbeb3d353ff68350490b0841cJens Axboe nr_blocks = size_to_blocks(size); 363ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (nr_blocks > pool->free_blocks) 3648e5732e558509fc0f4ccdeb1e4d01ad038aead06Jens Axboe goto fail; 3655ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe 366ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe i = pool->next_non_full; 367ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe last_idx = 0; 368ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe offset = -1U; 369ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe while (i < pool->nr_blocks) { 370ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe unsigned int idx; 371d24c33a479fcd68debad128da057814495f65e20Jens Axboe 372ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (pool->bitmap[i] == -1U) { 373ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe i++; 374ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->next_non_full = i; 375ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe last_idx = 0; 376ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe continue; 377ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 378d24c33a479fcd68debad128da057814495f65e20Jens Axboe 379ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx = find_next_zero(pool->bitmap[i], last_idx); 380dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe if (!blocks_free(pool, i, idx, nr_blocks)) { 381ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx += nr_blocks; 382ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (idx < SMALLOC_BPI) 383ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe last_idx = idx; 384ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe else { 385ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe last_idx = 0; 386ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe while (idx >= SMALLOC_BPI) { 387ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe i++; 388ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe idx -= SMALLOC_BPI; 389ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 390ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 391ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe continue; 392d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 393dcb69098630845b53ebb8034014d44b409e16f9eJens Axboe set_blocks(pool, i, idx, nr_blocks); 394ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe offset = i * SMALLOC_BPL + idx * SMALLOC_BPB; 395ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe break; 396ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe } 397ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 398ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (i < pool->nr_blocks) { 399ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe pool->free_blocks -= nr_blocks; 400ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe ret = pool->map + offset; 401d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 402ec996e9c8f3baacababec55b3be746e035099c14Jens Axboefail: 403d24c33a479fcd68debad128da057814495f65e20Jens Axboe pool_unlock(pool); 404ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe return ret; 405d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 406d24c33a479fcd68debad128da057814495f65e20Jens Axboe 407a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboestatic void *smalloc_pool(struct pool *pool, size_t size) 40855f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe{ 409a3ebe7e079847413458b9d80bd7a4acc924d108bJens Axboe size_t alloc_size = size + sizeof(struct block_hdr); 41055f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe void *ptr; 41155f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 412cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe /* 413122426daab0fbd2fce244ac018d737a8f302f974Jens Axboe * Round to int alignment, so that the postred pointer will 414122426daab0fbd2fce244ac018d737a8f302f974Jens Axboe * be naturally aligned as well. 415cf98708da292cb9e77bb6aff4eda7652103f4f77Jens Axboe */ 416ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#ifdef SMALLOC_REDZONE 417122426daab0fbd2fce244ac018d737a8f302f974Jens Axboe alloc_size += sizeof(unsigned int); 418122426daab0fbd2fce244ac018d737a8f302f974Jens Axboe alloc_size = (alloc_size + int_mask) & ~int_mask; 419ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe#endif 420ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 421ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe ptr = __smalloc_pool(pool, alloc_size); 42289da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe if (ptr) { 42389da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe struct block_hdr *hdr = ptr; 42455f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 42589da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe hdr->size = alloc_size; 42689da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe fill_redzone(hdr); 42755f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 42889da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe ptr += sizeof(*hdr); 42989da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe memset(ptr, 0, size); 43089da54e865169abcf0a99c6e7ee12ec2c8ddd8b9Jens Axboe } 431ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe 43255f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe return ptr; 43355f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe} 43455f6491de3d5f734a1cbe3e7deed87a382d6c863Jens Axboe 4357982aa7da64b68f38c7ec685a428334e3af4e340Jens Axboevoid *smalloc(size_t size) 436d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 437d24c33a479fcd68debad128da057814495f65e20Jens Axboe unsigned int i; 438d24c33a479fcd68debad128da057814495f65e20Jens Axboe 4397982aa7da64b68f38c7ec685a428334e3af4e340Jens Axboe if (size != (unsigned int) size) 4407982aa7da64b68f38c7ec685a428334e3af4e340Jens Axboe return NULL; 4417982aa7da64b68f38c7ec685a428334e3af4e340Jens Axboe 442d1271dc19c335c39298e16b825f84fdd8650132eShaozhi Shawn Ye global_write_lock(); 443d24c33a479fcd68debad128da057814495f65e20Jens Axboe i = last_pool; 444d24c33a479fcd68debad128da057814495f65e20Jens Axboe 445d24c33a479fcd68debad128da057814495f65e20Jens Axboe do { 446d24c33a479fcd68debad128da057814495f65e20Jens Axboe for (; i < nr_pools; i++) { 447d24c33a479fcd68debad128da057814495f65e20Jens Axboe void *ptr = smalloc_pool(&mp[i], size); 448d24c33a479fcd68debad128da057814495f65e20Jens Axboe 449d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (ptr) { 450d24c33a479fcd68debad128da057814495f65e20Jens Axboe last_pool = i; 451d1271dc19c335c39298e16b825f84fdd8650132eShaozhi Shawn Ye global_write_unlock(); 452d24c33a479fcd68debad128da057814495f65e20Jens Axboe return ptr; 453d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 454d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 455d24c33a479fcd68debad128da057814495f65e20Jens Axboe if (last_pool) { 456d24c33a479fcd68debad128da057814495f65e20Jens Axboe last_pool = 0; 457d24c33a479fcd68debad128da057814495f65e20Jens Axboe continue; 458d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 459d24c33a479fcd68debad128da057814495f65e20Jens Axboe 460ec996e9c8f3baacababec55b3be746e035099c14Jens Axboe if (nr_pools + 1 > MAX_POOLS) 461d24c33a479fcd68debad128da057814495f65e20Jens Axboe break; 462d24c33a479fcd68debad128da057814495f65e20Jens Axboe else { 463d24c33a479fcd68debad128da057814495f65e20Jens Axboe i = nr_pools; 464adf57099338307b3a4a129475a12aad31e35ade3Jens Axboe if (add_pool(&mp[nr_pools], size)) 46565864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboe goto out; 466d24c33a479fcd68debad128da057814495f65e20Jens Axboe } 467d24c33a479fcd68debad128da057814495f65e20Jens Axboe } while (1); 468d24c33a479fcd68debad128da057814495f65e20Jens Axboe 46965864cf7c951a79c02b1d1f358f0eb5d25a9aff5Jens Axboeout: 470d1271dc19c335c39298e16b825f84fdd8650132eShaozhi Shawn Ye global_write_unlock(); 471d24c33a479fcd68debad128da057814495f65e20Jens Axboe return NULL; 472d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 473d24c33a479fcd68debad128da057814495f65e20Jens Axboe 474d24c33a479fcd68debad128da057814495f65e20Jens Axboechar *smalloc_strdup(const char *str) 475d24c33a479fcd68debad128da057814495f65e20Jens Axboe{ 476d24c33a479fcd68debad128da057814495f65e20Jens Axboe char *ptr; 477d24c33a479fcd68debad128da057814495f65e20Jens Axboe 478d24c33a479fcd68debad128da057814495f65e20Jens Axboe ptr = smalloc(strlen(str) + 1); 479d24c33a479fcd68debad128da057814495f65e20Jens Axboe strcpy(ptr, str); 480d24c33a479fcd68debad128da057814495f65e20Jens Axboe return ptr; 481d24c33a479fcd68debad128da057814495f65e20Jens Axboe} 482