mempool.c revision 0565d317768cc66b13e37184f29d9f270c2886dc
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * linux/mm/mempool.c 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * memory buffer pool support. Such pools are mostly used 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * for guaranteed, deadlock-free memory allocations during 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * extreme VM load. 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * started by Ingo Molnar, Copyright (C) 2001 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 13b95f1b31b75588306e32b2afd32166cad48f670bPaul Gortmaker#include <linux/export.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mempool.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/blkdev.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/writeback.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void add_element(mempool_t *pool, void *element) 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(pool->curr_nr >= pool->min_nr); 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->elements[pool->curr_nr++] = element; 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void *remove_element(mempool_t *pool) 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(pool->curr_nr <= 0); 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pool->elements[--pool->curr_nr]; 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 300565d317768cc66b13e37184f29d9f270c2886dcTejun Heo/** 310565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * mempool_destroy - deallocate a memory pool 320565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * @pool: pointer to the memory pool which was allocated via 330565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * mempool_create(). 340565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * 350565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * Free all reserved elements in @pool and @pool itself. This function 360565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * only sleeps if the free_fn() function sleeps. 370565d317768cc66b13e37184f29d9f270c2886dcTejun Heo */ 380565d317768cc66b13e37184f29d9f270c2886dcTejun Heovoid mempool_destroy(mempool_t *pool) 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr) { 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element = remove_element(pool); 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool->elements); 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool); 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 470565d317768cc66b13e37184f29d9f270c2886dcTejun HeoEXPORT_SYMBOL(mempool_destroy); 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create - create a memory pool 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @min_nr: the minimum number of elements guaranteed to be 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allocated for this pool. 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @alloc_fn: user-defined element-allocation function. 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @free_fn: user-defined element-freeing function. 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool_data: optional private data available to the user-defined functions. 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * this function creates and allocates a guaranteed size, preallocated 5872fd4a35a824331d7a0f4168d7576502d95d34b3Robert P. J. Day * memory pool. The pool can be used from the mempool_alloc() and mempool_free() 591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * functions. This function might sleep. Both the alloc_fn() and the free_fn() 6072fd4a35a824331d7a0f4168d7576502d95d34b3Robert P. J. Day * functions might sleep - as long as the mempool_alloc() function is not called 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * from IRQ contexts. 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 631946089a109251655c5438d92c539bd2930e71eaChristoph Lametermempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mempool_free_t *free_fn, void *pool_data) 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 661946089a109251655c5438d92c539bd2930e71eaChristoph Lameter return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1); 671946089a109251655c5438d92c539bd2930e71eaChristoph Lameter} 681946089a109251655c5438d92c539bd2930e71eaChristoph LameterEXPORT_SYMBOL(mempool_create); 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 701946089a109251655c5438d92c539bd2930e71eaChristoph Lametermempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, 711946089a109251655c5438d92c539bd2930e71eaChristoph Lameter mempool_free_t *free_fn, void *pool_data, int node_id) 721946089a109251655c5438d92c539bd2930e71eaChristoph Lameter{ 731946089a109251655c5438d92c539bd2930e71eaChristoph Lameter mempool_t *pool; 7494f6030ca792c57422f04a73e7a872d8325946d3Christoph Lameter pool = kmalloc_node(sizeof(*pool), GFP_KERNEL | __GFP_ZERO, node_id); 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pool) 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 771946089a109251655c5438d92c539bd2930e71eaChristoph Lameter pool->elements = kmalloc_node(min_nr * sizeof(void *), 781946089a109251655c5438d92c539bd2930e71eaChristoph Lameter GFP_KERNEL, node_id); 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pool->elements) { 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool); 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_init(&pool->lock); 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = min_nr; 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->pool_data = pool_data; 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds init_waitqueue_head(&pool->wait); 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->alloc = alloc_fn; 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free = free_fn; 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * First pre-allocate the guaranteed number of buffers. 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr < pool->min_nr) { 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = pool->alloc(GFP_KERNEL, pool->pool_data); 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(!element)) { 980565d317768cc66b13e37184f29d9f270c2886dcTejun Heo mempool_destroy(pool); 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pool; 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1051946089a109251655c5438d92c539bd2930e71eaChristoph LameterEXPORT_SYMBOL(mempool_create_node); 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_resize - resize an existing memory pool 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @new_min_nr: the new minimum number of elements guaranteed to be 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allocated for this pool. 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @gfp_mask: the usual allocation bitmask. 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This function shrinks/grows the pool. In the case of growing, 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * it cannot be guaranteed that the pool will be grown to the new 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * size immediately, but new mempool_free() calls will refill it. 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note, the caller must guarantee that no mempool_destroy is called 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * while this function is running. mempool_alloc() & mempool_free() 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * might be called (eg. from IRQ contexts) while this function executes. 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 123dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Viroint mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void **new_elements; 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(new_min_nr <= 0); 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (new_min_nr <= pool->min_nr) { 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (new_min_nr < pool->curr_nr) { 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = remove_element(pool); 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = new_min_nr; 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_unlock; 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Grow the pool */ 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask); 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!new_elements) 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(new_min_nr <= pool->min_nr)) { 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Raced, other resize will do our work */ 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(new_elements); 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memcpy(new_elements, pool->elements, 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->curr_nr * sizeof(*new_elements)); 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool->elements); 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->elements = new_elements; 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = new_min_nr; 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr < pool->min_nr) { 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = pool->alloc(gfp_mask, pool->pool_data); 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!element) 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pool->curr_nr < pool->min_nr) { 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); /* Raced */ 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_unlock: 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout: 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_resize); 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_alloc - allocate an element from a specific memory pool 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @gfp_mask: the usual allocation bitmask. 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18972fd4a35a824331d7a0f4168d7576502d95d34b3Robert P. J. Day * this function only sleeps if the alloc_fn() function sleeps or 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * returns NULL. Note that due to preallocation, this function 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * *never* fails when called from process contexts. (it might 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * fail if called from an IRQ context.) 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 194dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Virovoid * mempool_alloc(mempool_t *pool, gfp_t gfp_mask) 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 19801890a4c120f68366441bf5e193d1b9dd543d4d0Benjamin LaHaise wait_queue_t wait; 1996daa0e28627abf362138244a620a821a9027d816Al Viro gfp_t gfp_temp; 20020a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 20120a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin might_sleep_if(gfp_mask & __GFP_WAIT); 202b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin 203b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */ 204b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */ 205b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NOWARN; /* failures are OK */ 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 20720a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO); 20820a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 2091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrepeat_alloc: 21020a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 21120a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin element = pool->alloc(gfp_temp, pool->pool_data); 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (likely(element != NULL)) 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return element; 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (likely(pool->curr_nr)) { 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = remove_element(pool); 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 2195b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* paired with rmb in mempool_free(), read comment there */ 2205b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo smp_wmb(); 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return element; 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* We must not sleep in the GFP_ATOMIC case */ 2255b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo if (!(gfp_mask & __GFP_WAIT)) { 2265b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo spin_unlock_irqrestore(&pool->lock, flags); 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2285b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo } 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2305b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* Let's wait for someone else to return an element to @pool */ 23120a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin gfp_temp = gfp_mask; 23201890a4c120f68366441bf5e193d1b9dd543d4d0Benjamin LaHaise init_wait(&wait); 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2355b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo spin_unlock_irqrestore(&pool->lock, flags); 2365b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo 2375b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* 2385b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * FIXME: this should be io_schedule(). The timeout is there as a 2395b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * workaround for some DM problems in 2.6.18. 2405b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo */ 2415b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo io_schedule_timeout(5*HZ); 2425b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo 2435b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo finish_wait(&pool->wait, &wait); 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto repeat_alloc; 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_alloc); 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_free - return an element to the pool. 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @element: pool element pointer. 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * this function only sleeps if the free_fn() function sleeps. 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mempool_free(void *element, mempool_t *pool) 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 260c80e7a826c10cf5bce8487fbaede48bd0bd48d08Rusty Russell if (unlikely(element == NULL)) 261c80e7a826c10cf5bce8487fbaede48bd0bd48d08Rusty Russell return; 262c80e7a826c10cf5bce8487fbaede48bd0bd48d08Rusty Russell 2635b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* 2645b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * Paired with the wmb in mempool_alloc(). The preceding read is 2655b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * for @element and the following @pool->curr_nr. This ensures 2665b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * that the visible value of @pool->curr_nr is from after the 2675b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * allocation of @element. This is necessary for fringe cases 2685b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * where @element was passed to this task without going through 2695b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * barriers. 2705b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * 2715b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * For example, assume @p is %NULL at the beginning and one task 2725b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * performs "p = mempool_alloc(...);" while another task is doing 2735b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * "while (!p) cpu_relax(); mempool_free(p, ...);". This function 2745b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * may end up using curr_nr value which is from before allocation 2755b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * of @p without the following rmb. 2765b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo */ 2775b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo smp_rmb(); 2785b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo 2795b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* 2805b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * For correctness, we need a test which is guaranteed to trigger 2815b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * if curr_nr + #allocated == min_nr. Testing curr_nr < min_nr 2825b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * without locking achieves that and refilling as soon as possible 2835b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * is desirable. 2845b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * 2855b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * Because curr_nr visible here is always a value after the 2865b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * allocation of @element, any task which decremented curr_nr below 2875b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets 2885b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * incremented to min_nr afterwards. If curr_nr gets incremented 2895b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * to min_nr after the allocation of @element, the elements 2905b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * allocated after that are subject to the same guarantee. 2915b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * 2925b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * Waiters happen iff curr_nr is 0 and the above guarantee also 2935b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * ensures that there will be frees which return elements to the 2945b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * pool waking up the waiters. 2955b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo */ 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pool->curr_nr < pool->min_nr) { 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pool->curr_nr < pool->min_nr) { 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 3001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up(&pool->wait); 3021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_free); 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * A commonly used alloc and free fn. 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 313dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Virovoid *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) 3141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 315fcc234f888ba2365c44ba0507eb8a18eebf1f594Pekka Enberg struct kmem_cache *mem = pool_data; 3161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return kmem_cache_alloc(mem, gfp_mask); 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_alloc_slab); 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mempool_free_slab(void *element, void *pool_data) 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 322fcc234f888ba2365c44ba0507eb8a18eebf1f594Pekka Enberg struct kmem_cache *mem = pool_data; 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kmem_cache_free(mem, element); 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_free_slab); 3266e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson 3276e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson/* 32853184082b070dfb077218828fdf839826102ed96Matthew Dobson * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory 329183ff22bb6bd8188c904ebfb479656ae52230b72Simon Arlott * specified by pool_data 33053184082b070dfb077218828fdf839826102ed96Matthew Dobson */ 33153184082b070dfb077218828fdf839826102ed96Matthew Dobsonvoid *mempool_kmalloc(gfp_t gfp_mask, void *pool_data) 33253184082b070dfb077218828fdf839826102ed96Matthew Dobson{ 3335e2f89b5d5d87a7c3ba19fc85ba0c29adb65f639Figo.zhang size_t size = (size_t)pool_data; 33453184082b070dfb077218828fdf839826102ed96Matthew Dobson return kmalloc(size, gfp_mask); 33553184082b070dfb077218828fdf839826102ed96Matthew Dobson} 33653184082b070dfb077218828fdf839826102ed96Matthew DobsonEXPORT_SYMBOL(mempool_kmalloc); 33753184082b070dfb077218828fdf839826102ed96Matthew Dobson 33853184082b070dfb077218828fdf839826102ed96Matthew Dobsonvoid mempool_kfree(void *element, void *pool_data) 33953184082b070dfb077218828fdf839826102ed96Matthew Dobson{ 34053184082b070dfb077218828fdf839826102ed96Matthew Dobson kfree(element); 34153184082b070dfb077218828fdf839826102ed96Matthew Dobson} 34253184082b070dfb077218828fdf839826102ed96Matthew DobsonEXPORT_SYMBOL(mempool_kfree); 34353184082b070dfb077218828fdf839826102ed96Matthew Dobson 34453184082b070dfb077218828fdf839826102ed96Matthew Dobson/* 3456e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson * A simple mempool-backed page allocator that allocates pages 3466e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson * of the order specified by pool_data. 3476e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson */ 3486e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobsonvoid *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data) 3496e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson{ 3506e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson int order = (int)(long)pool_data; 3516e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson return alloc_pages(gfp_mask, order); 3526e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson} 3536e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew DobsonEXPORT_SYMBOL(mempool_alloc_pages); 3546e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson 3556e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobsonvoid mempool_free_pages(void *element, void *pool_data) 3566e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson{ 3576e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson int order = (int)(long)pool_data; 3586e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson __free_pages(element, order); 3596e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson} 3606e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew DobsonEXPORT_SYMBOL(mempool_free_pages); 361