mempool.c revision 6daa0e28627abf362138244a620a821a9027d816
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * linux/mm/mempool.c 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * memory buffer pool support. Such pools are mostly used 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * for guaranteed, deadlock-free memory allocations during 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * extreme VM load. 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * started by Ingo Molnar, Copyright (C) 2001 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mempool.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/blkdev.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/writeback.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void add_element(mempool_t *pool, void *element) 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(pool->curr_nr >= pool->min_nr); 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->elements[pool->curr_nr++] = element; 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void *remove_element(mempool_t *pool) 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(pool->curr_nr <= 0); 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pool->elements[--pool->curr_nr]; 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void free_pool(mempool_t *pool) 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr) { 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element = remove_element(pool); 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool->elements); 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool); 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create - create a memory pool 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @min_nr: the minimum number of elements guaranteed to be 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allocated for this pool. 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @alloc_fn: user-defined element-allocation function. 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @free_fn: user-defined element-freeing function. 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool_data: optional private data available to the user-defined functions. 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * this function creates and allocates a guaranteed size, preallocated 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * memory pool. The pool can be used from the mempool_alloc and mempool_free 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * functions. This function might sleep. Both the alloc_fn() and the free_fn() 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * functions might sleep - as long as the mempool_alloc function is not called 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * from IRQ contexts. 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 541946089a109251655c5438d92c539bd2930e71eaChristoph Lametermempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mempool_free_t *free_fn, void *pool_data) 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 571946089a109251655c5438d92c539bd2930e71eaChristoph Lameter return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1); 581946089a109251655c5438d92c539bd2930e71eaChristoph Lameter} 591946089a109251655c5438d92c539bd2930e71eaChristoph LameterEXPORT_SYMBOL(mempool_create); 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 611946089a109251655c5438d92c539bd2930e71eaChristoph Lametermempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, 621946089a109251655c5438d92c539bd2930e71eaChristoph Lameter mempool_free_t *free_fn, void *pool_data, int node_id) 631946089a109251655c5438d92c539bd2930e71eaChristoph Lameter{ 641946089a109251655c5438d92c539bd2930e71eaChristoph Lameter mempool_t *pool; 651946089a109251655c5438d92c539bd2930e71eaChristoph Lameter pool = kmalloc_node(sizeof(*pool), GFP_KERNEL, node_id); 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pool) 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memset(pool, 0, sizeof(*pool)); 691946089a109251655c5438d92c539bd2930e71eaChristoph Lameter pool->elements = kmalloc_node(min_nr * sizeof(void *), 701946089a109251655c5438d92c539bd2930e71eaChristoph Lameter GFP_KERNEL, node_id); 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pool->elements) { 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool); 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_init(&pool->lock); 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = min_nr; 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->pool_data = pool_data; 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds init_waitqueue_head(&pool->wait); 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->alloc = alloc_fn; 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free = free_fn; 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * First pre-allocate the guaranteed number of buffers. 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr < pool->min_nr) { 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = pool->alloc(GFP_KERNEL, pool->pool_data); 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(!element)) { 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds free_pool(pool); 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pool; 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 971946089a109251655c5438d92c539bd2930e71eaChristoph LameterEXPORT_SYMBOL(mempool_create_node); 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_resize - resize an existing memory pool 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @new_min_nr: the new minimum number of elements guaranteed to be 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allocated for this pool. 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @gfp_mask: the usual allocation bitmask. 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This function shrinks/grows the pool. In the case of growing, 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * it cannot be guaranteed that the pool will be grown to the new 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * size immediately, but new mempool_free() calls will refill it. 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note, the caller must guarantee that no mempool_destroy is called 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * while this function is running. mempool_alloc() & mempool_free() 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * might be called (eg. from IRQ contexts) while this function executes. 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 115dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Viroint mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void **new_elements; 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(new_min_nr <= 0); 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (new_min_nr <= pool->min_nr) { 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (new_min_nr < pool->curr_nr) { 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = remove_element(pool); 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = new_min_nr; 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_unlock; 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Grow the pool */ 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask); 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!new_elements) 1391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(new_min_nr <= pool->min_nr)) { 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Raced, other resize will do our work */ 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(new_elements); 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memcpy(new_elements, pool->elements, 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->curr_nr * sizeof(*new_elements)); 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool->elements); 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->elements = new_elements; 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = new_min_nr; 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr < pool->min_nr) { 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = pool->alloc(gfp_mask, pool->pool_data); 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!element) 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pool->curr_nr < pool->min_nr) { 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); /* Raced */ 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_unlock: 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout: 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_resize); 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_destroy - deallocate a memory pool 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * this function only sleeps if the free_fn() function sleeps. The caller 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * has to guarantee that all elements have been returned to the pool (ie: 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * freed) prior to calling mempool_destroy(). 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mempool_destroy(mempool_t *pool) 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pool->curr_nr != pool->min_nr) 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG(); /* There were outstanding elements */ 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds free_pool(pool); 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_destroy); 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_alloc - allocate an element from a specific memory pool 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @gfp_mask: the usual allocation bitmask. 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * this function only sleeps if the alloc_fn function sleeps or 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * returns NULL. Note that due to preallocation, this function 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * *never* fails when called from process contexts. (it might 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * fail if called from an IRQ context.) 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 203dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Virovoid * mempool_alloc(mempool_t *pool, gfp_t gfp_mask) 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 20701890a4c120f68366441bf5e193d1b9dd543d4d0Benjamin LaHaise wait_queue_t wait; 2086daa0e28627abf362138244a620a821a9027d816Al Viro gfp_t gfp_temp; 20920a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 21020a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin might_sleep_if(gfp_mask & __GFP_WAIT); 211b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin 212b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */ 213b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */ 214b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NOWARN; /* failures are OK */ 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 21620a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO); 21720a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrepeat_alloc: 21920a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 22020a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin element = pool->alloc(gfp_temp, pool->pool_data); 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (likely(element != NULL)) 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return element; 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (likely(pool->curr_nr)) { 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = remove_element(pool); 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return element; 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* We must not sleep in the GFP_ATOMIC case */ 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(gfp_mask & __GFP_WAIT)) 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 23620a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin /* Now start performing page reclaim */ 23720a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin gfp_temp = gfp_mask; 23801890a4c120f68366441bf5e193d1b9dd543d4d0Benjamin LaHaise init_wait(&wait); 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 240d59dd4620fb8d6422555a9e2b82a707718e68327Andrew Morton smp_mb(); 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pool->curr_nr) 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_schedule(); 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds finish_wait(&pool->wait, &wait); 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto repeat_alloc; 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_alloc); 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_free - return an element to the pool. 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @element: pool element pointer. 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * this function only sleeps if the free_fn() function sleeps. 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mempool_free(void *element, mempool_t *pool) 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 261d59dd4620fb8d6422555a9e2b82a707718e68327Andrew Morton smp_mb(); 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pool->curr_nr < pool->min_nr) { 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pool->curr_nr < pool->min_nr) { 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up(&pool->wait); 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_free); 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * A commonly used alloc and free fn. 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 279dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Virovoid *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kmem_cache_t *mem = (kmem_cache_t *) pool_data; 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return kmem_cache_alloc(mem, gfp_mask); 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_alloc_slab); 2851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mempool_free_slab(void *element, void *pool_data) 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kmem_cache_t *mem = (kmem_cache_t *) pool_data; 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kmem_cache_free(mem, element); 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_free_slab); 292