11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * linux/mm/mempool.c 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * memory buffer pool support. Such pools are mostly used 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * for guaranteed, deadlock-free memory allocations during 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * extreme VM load. 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * started by Ingo Molnar, Copyright (C) 2001 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 13174119628188b085c66fe7d86fbfb4cccb1bd864Catalin Marinas#include <linux/kmemleak.h> 14b95f1b31b75588306e32b2afd32166cad48f670bPaul Gortmaker#include <linux/export.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mempool.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/blkdev.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/writeback.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void add_element(mempool_t *pool, void *element) 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(pool->curr_nr >= pool->min_nr); 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->elements[pool->curr_nr++] = element; 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void *remove_element(mempool_t *pool) 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(pool->curr_nr <= 0); 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pool->elements[--pool->curr_nr]; 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 310565d317768cc66b13e37184f29d9f270c2886dcTejun Heo/** 320565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * mempool_destroy - deallocate a memory pool 330565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * @pool: pointer to the memory pool which was allocated via 340565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * mempool_create(). 350565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * 360565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * Free all reserved elements in @pool and @pool itself. This function 370565d317768cc66b13e37184f29d9f270c2886dcTejun Heo * only sleeps if the free_fn() function sleeps. 380565d317768cc66b13e37184f29d9f270c2886dcTejun Heo */ 390565d317768cc66b13e37184f29d9f270c2886dcTejun Heovoid mempool_destroy(mempool_t *pool) 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr) { 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element = remove_element(pool); 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool->elements); 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool); 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 480565d317768cc66b13e37184f29d9f270c2886dcTejun HeoEXPORT_SYMBOL(mempool_destroy); 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create - create a memory pool 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @min_nr: the minimum number of elements guaranteed to be 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allocated for this pool. 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @alloc_fn: user-defined element-allocation function. 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @free_fn: user-defined element-freeing function. 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool_data: optional private data available to the user-defined functions. 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * this function creates and allocates a guaranteed size, preallocated 5972fd4a35a824331d7a0f4168d7576502d95d34b3Robert P. J. Day * memory pool. The pool can be used from the mempool_alloc() and mempool_free() 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * functions. This function might sleep. Both the alloc_fn() and the free_fn() 6172fd4a35a824331d7a0f4168d7576502d95d34b3Robert P. J. Day * functions might sleep - as long as the mempool_alloc() function is not called 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * from IRQ contexts. 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 641946089a109251655c5438d92c539bd2930e71eaChristoph Lametermempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mempool_free_t *free_fn, void *pool_data) 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 67a91a5ac6858fbf7477131e1210cb3e897b668e6fTejun Heo return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data, 68a91a5ac6858fbf7477131e1210cb3e897b668e6fTejun Heo GFP_KERNEL, NUMA_NO_NODE); 691946089a109251655c5438d92c539bd2930e71eaChristoph Lameter} 701946089a109251655c5438d92c539bd2930e71eaChristoph LameterEXPORT_SYMBOL(mempool_create); 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 721946089a109251655c5438d92c539bd2930e71eaChristoph Lametermempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, 73a91a5ac6858fbf7477131e1210cb3e897b668e6fTejun Heo mempool_free_t *free_fn, void *pool_data, 74a91a5ac6858fbf7477131e1210cb3e897b668e6fTejun Heo gfp_t gfp_mask, int node_id) 751946089a109251655c5438d92c539bd2930e71eaChristoph Lameter{ 761946089a109251655c5438d92c539bd2930e71eaChristoph Lameter mempool_t *pool; 777b5219db00d0afaf3d2b0e8c443ffa892455ba75Joe Perches pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id); 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pool) 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 801946089a109251655c5438d92c539bd2930e71eaChristoph Lameter pool->elements = kmalloc_node(min_nr * sizeof(void *), 81a91a5ac6858fbf7477131e1210cb3e897b668e6fTejun Heo gfp_mask, node_id); 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pool->elements) { 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool); 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_init(&pool->lock); 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = min_nr; 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->pool_data = pool_data; 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds init_waitqueue_head(&pool->wait); 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->alloc = alloc_fn; 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free = free_fn; 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * First pre-allocate the guaranteed number of buffers. 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr < pool->min_nr) { 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 99a91a5ac6858fbf7477131e1210cb3e897b668e6fTejun Heo element = pool->alloc(gfp_mask, pool->pool_data); 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(!element)) { 1010565d317768cc66b13e37184f29d9f270c2886dcTejun Heo mempool_destroy(pool); 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pool; 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1081946089a109251655c5438d92c539bd2930e71eaChristoph LameterEXPORT_SYMBOL(mempool_create_node); 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_resize - resize an existing memory pool 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @new_min_nr: the new minimum number of elements guaranteed to be 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allocated for this pool. 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @gfp_mask: the usual allocation bitmask. 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This function shrinks/grows the pool. In the case of growing, 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * it cannot be guaranteed that the pool will be grown to the new 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * size immediately, but new mempool_free() calls will refill it. 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note, the caller must guarantee that no mempool_destroy is called 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * while this function is running. mempool_alloc() & mempool_free() 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * might be called (eg. from IRQ contexts) while this function executes. 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 126dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Viroint mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void **new_elements; 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG_ON(new_min_nr <= 0); 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (new_min_nr <= pool->min_nr) { 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (new_min_nr < pool->curr_nr) { 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = remove_element(pool); 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = new_min_nr; 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_unlock; 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Grow the pool */ 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask); 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!new_elements) 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(new_min_nr <= pool->min_nr)) { 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Raced, other resize will do our work */ 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(new_elements); 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memcpy(new_elements, pool->elements, 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->curr_nr * sizeof(*new_elements)); 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(pool->elements); 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->elements = new_elements; 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->min_nr = new_min_nr; 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (pool->curr_nr < pool->min_nr) { 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = pool->alloc(gfp_mask, pool->pool_data); 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!element) 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pool->curr_nr < pool->min_nr) { 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); /* Raced */ 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_unlock: 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout: 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_resize); 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_alloc - allocate an element from a specific memory pool 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @gfp_mask: the usual allocation bitmask. 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 19272fd4a35a824331d7a0f4168d7576502d95d34b3Robert P. J. Day * this function only sleeps if the alloc_fn() function sleeps or 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * returns NULL. Note that due to preallocation, this function 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * *never* fails when called from process contexts. (it might 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * fail if called from an IRQ context.) 1968bf8fcb07653fbaea74f96bba1e4ed0f851675abSebastian Ott * Note: using __GFP_ZERO is not supported. 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 198dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Virovoid * mempool_alloc(mempool_t *pool, gfp_t gfp_mask) 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *element; 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 20201890a4c120f68366441bf5e193d1b9dd543d4d0Benjamin LaHaise wait_queue_t wait; 2036daa0e28627abf362138244a620a821a9027d816Al Viro gfp_t gfp_temp; 20420a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 2058bf8fcb07653fbaea74f96bba1e4ed0f851675abSebastian Ott VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO); 20620a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin might_sleep_if(gfp_mask & __GFP_WAIT); 207b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin 208b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */ 209b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */ 210b84a35be0285229b0a8a5e2e04d79360c5b75562Nick Piggin gfp_mask |= __GFP_NOWARN; /* failures are OK */ 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 21220a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO); 21320a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrepeat_alloc: 21520a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin 21620a77776c24800d1e40a73f520cfcb32239568a9Nick Piggin element = pool->alloc(gfp_temp, pool->pool_data); 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (likely(element != NULL)) 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return element; 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (likely(pool->curr_nr)) { 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds element = remove_element(pool); 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 2245b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* paired with rmb in mempool_free(), read comment there */ 2255b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo smp_wmb(); 226174119628188b085c66fe7d86fbfb4cccb1bd864Catalin Marinas /* 227174119628188b085c66fe7d86fbfb4cccb1bd864Catalin Marinas * Update the allocation stack trace as this is more useful 228174119628188b085c66fe7d86fbfb4cccb1bd864Catalin Marinas * for debugging. 229174119628188b085c66fe7d86fbfb4cccb1bd864Catalin Marinas */ 230174119628188b085c66fe7d86fbfb4cccb1bd864Catalin Marinas kmemleak_update_trace(element); 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return element; 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2341ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo /* 2351ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo * We use gfp mask w/o __GFP_WAIT or IO for the first round. If 2361ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo * alloc failed with that and @pool was empty, retry immediately. 2371ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo */ 2381ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo if (gfp_temp != gfp_mask) { 2391ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo spin_unlock_irqrestore(&pool->lock, flags); 2401ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo gfp_temp = gfp_mask; 2411ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo goto repeat_alloc; 2421ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo } 2431ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo 2441ebb7044c9142c67d1d2b04d84010b4810a43fd8Tejun Heo /* We must not sleep if !__GFP_WAIT */ 2455b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo if (!(gfp_mask & __GFP_WAIT)) { 2465b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo spin_unlock_irqrestore(&pool->lock, flags); 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2485b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo } 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2505b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* Let's wait for someone else to return an element to @pool */ 25101890a4c120f68366441bf5e193d1b9dd543d4d0Benjamin LaHaise init_wait(&wait); 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2545b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo spin_unlock_irqrestore(&pool->lock, flags); 2555b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo 2565b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* 2575b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * FIXME: this should be io_schedule(). The timeout is there as a 2585b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * workaround for some DM problems in 2.6.18. 2595b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo */ 2605b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo io_schedule_timeout(5*HZ); 2615b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo 2625b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo finish_wait(&pool->wait, &wait); 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto repeat_alloc; 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_alloc); 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_free - return an element to the pool. 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @element: pool element pointer. 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @pool: pointer to the memory pool which was allocated via 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mempool_create(). 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * this function only sleeps if the free_fn() function sleeps. 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mempool_free(void *element, mempool_t *pool) 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 279c80e7a826c10cf5bce8487fbaede48bd0bd48d08Rusty Russell if (unlikely(element == NULL)) 280c80e7a826c10cf5bce8487fbaede48bd0bd48d08Rusty Russell return; 281c80e7a826c10cf5bce8487fbaede48bd0bd48d08Rusty Russell 2825b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* 2835b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * Paired with the wmb in mempool_alloc(). The preceding read is 2845b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * for @element and the following @pool->curr_nr. This ensures 2855b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * that the visible value of @pool->curr_nr is from after the 2865b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * allocation of @element. This is necessary for fringe cases 2875b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * where @element was passed to this task without going through 2885b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * barriers. 2895b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * 2905b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * For example, assume @p is %NULL at the beginning and one task 2915b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * performs "p = mempool_alloc(...);" while another task is doing 2925b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * "while (!p) cpu_relax(); mempool_free(p, ...);". This function 2935b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * may end up using curr_nr value which is from before allocation 2945b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * of @p without the following rmb. 2955b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo */ 2965b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo smp_rmb(); 2975b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo 2985b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo /* 2995b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * For correctness, we need a test which is guaranteed to trigger 3005b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * if curr_nr + #allocated == min_nr. Testing curr_nr < min_nr 3015b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * without locking achieves that and refilling as soon as possible 3025b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * is desirable. 3035b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * 3045b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * Because curr_nr visible here is always a value after the 3055b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * allocation of @element, any task which decremented curr_nr below 3065b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets 3075b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * incremented to min_nr afterwards. If curr_nr gets incremented 3085b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * to min_nr after the allocation of @element, the elements 3095b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * allocated after that are subject to the same guarantee. 3105b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * 3115b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * Waiters happen iff curr_nr is 0 and the above guarantee also 3125b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * ensures that there will be frees which return elements to the 3135b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo * pool waking up the waiters. 3145b990546e33477c34ee6fbc20fad6584386b46c3Tejun Heo */ 315eb9a3c62a0b6064c7f7e5b961ce00c646d21cb78Mikulas Patocka if (unlikely(pool->curr_nr < pool->min_nr)) { 3161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&pool->lock, flags); 317eb9a3c62a0b6064c7f7e5b961ce00c646d21cb78Mikulas Patocka if (likely(pool->curr_nr < pool->min_nr)) { 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_element(pool, element); 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up(&pool->wait); 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&pool->lock, flags); 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pool->free(element, pool->pool_data); 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_free); 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * A commonly used alloc and free fn. 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 332dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Virovoid *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 334fcc234f888ba2365c44ba0507eb8a18eebf1f594Pekka Enberg struct kmem_cache *mem = pool_data; 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return kmem_cache_alloc(mem, gfp_mask); 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_alloc_slab); 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid mempool_free_slab(void *element, void *pool_data) 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 341fcc234f888ba2365c44ba0507eb8a18eebf1f594Pekka Enberg struct kmem_cache *mem = pool_data; 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kmem_cache_free(mem, element); 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mempool_free_slab); 3456e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson 3466e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson/* 34753184082b070dfb077218828fdf839826102ed96Matthew Dobson * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory 348183ff22bb6bd8188c904ebfb479656ae52230b72Simon Arlott * specified by pool_data 34953184082b070dfb077218828fdf839826102ed96Matthew Dobson */ 35053184082b070dfb077218828fdf839826102ed96Matthew Dobsonvoid *mempool_kmalloc(gfp_t gfp_mask, void *pool_data) 35153184082b070dfb077218828fdf839826102ed96Matthew Dobson{ 3525e2f89b5d5d87a7c3ba19fc85ba0c29adb65f639Figo.zhang size_t size = (size_t)pool_data; 35353184082b070dfb077218828fdf839826102ed96Matthew Dobson return kmalloc(size, gfp_mask); 35453184082b070dfb077218828fdf839826102ed96Matthew Dobson} 35553184082b070dfb077218828fdf839826102ed96Matthew DobsonEXPORT_SYMBOL(mempool_kmalloc); 35653184082b070dfb077218828fdf839826102ed96Matthew Dobson 35753184082b070dfb077218828fdf839826102ed96Matthew Dobsonvoid mempool_kfree(void *element, void *pool_data) 35853184082b070dfb077218828fdf839826102ed96Matthew Dobson{ 35953184082b070dfb077218828fdf839826102ed96Matthew Dobson kfree(element); 36053184082b070dfb077218828fdf839826102ed96Matthew Dobson} 36153184082b070dfb077218828fdf839826102ed96Matthew DobsonEXPORT_SYMBOL(mempool_kfree); 36253184082b070dfb077218828fdf839826102ed96Matthew Dobson 36353184082b070dfb077218828fdf839826102ed96Matthew Dobson/* 3646e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson * A simple mempool-backed page allocator that allocates pages 3656e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson * of the order specified by pool_data. 3666e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson */ 3676e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobsonvoid *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data) 3686e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson{ 3696e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson int order = (int)(long)pool_data; 3706e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson return alloc_pages(gfp_mask, order); 3716e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson} 3726e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew DobsonEXPORT_SYMBOL(mempool_alloc_pages); 3736e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson 3746e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobsonvoid mempool_free_pages(void *element, void *pool_data) 3756e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson{ 3766e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson int order = (int)(long)pool_data; 3776e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson __free_pages(element, order); 3786e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew Dobson} 3796e0678f394c7bd21bfa5d252b071a09e10e7a749Matthew DobsonEXPORT_SYMBOL(mempool_free_pages); 380