11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 27b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * Copyright (c) 2000-2005 Silicon Graphics, Inc. 37b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * All Rights Reserved. 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 57b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * This program is free software; you can redistribute it and/or 67b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * modify it under the terms of the GNU General Public License as 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * published by the Free Software Foundation. 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 97b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * This program is distributed in the hope that it would be useful, 107b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * but WITHOUT ANY WARRANTY; without even the implied warranty of 117b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 127b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * GNU General Public License for more details. 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 147b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * You should have received a copy of the GNU General Public License 157b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * along with this program; if not, write the Free Software Foundation, 167b71876980d87c8f237b94d8529ee7fcc05ec2d9Nathan Scott * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/highmem.h> 205a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/swap.h> 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/blkdev.h> 233fcfab16c5b86eaa3db3a9a31adba550c5b67141Andrew Morton#include <linux/backing-dev.h> 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "kmem.h" 254f10700a2e4bb2ff3d3a80f08412e21109e6d4b5Dave Chinner#include "xfs_message.h" 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 27bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig/* 28bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig * Greedy allocation. May fail and may return vmalloced memory. 29bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig */ 30bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwigvoid * 31bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwigkmem_zalloc_greedy(size_t *size, size_t minsize, size_t maxsize) 32bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig{ 33bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig void *ptr; 34bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig size_t kmsize = maxsize; 35bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig 36fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner while (!(ptr = vzalloc(kmsize))) { 37bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig if ((kmsize >>= 1) <= minsize) 38bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig kmsize = minsize; 39bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig } 40bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig if (ptr) 41bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig *size = kmsize; 42bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig return ptr; 43bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig} 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid * 4677ba78776e90e8de541f13b326e284c74286252fAl Virokmem_alloc(size_t size, xfs_km_flags_t flags) 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4827496a8c67bef4d789d8e3c8317ca35813a507aeAl Viro int retries = 0; 4927496a8c67bef4d789d8e3c8317ca35813a507aeAl Viro gfp_t lflags = kmem_flags_convert(flags); 5027496a8c67bef4d789d8e3c8317ca35813a507aeAl Viro void *ptr; 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 53bdfb04301fa5fdd95f219539a9a5b9663b1e5fc2Christoph Hellwig ptr = kmalloc(size, lflags); 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ptr || (flags & (KM_MAYFAIL|KM_NOSLEEP))) 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ptr; 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(++retries % 100)) 574f10700a2e4bb2ff3d3a80f08412e21109e6d4b5Dave Chinner xfs_err(NULL, 584f10700a2e4bb2ff3d3a80f08412e21109e6d4b5Dave Chinner "possible memory allocation deadlock in %s (mode:0x%x)", 5934a622b2e1c8e11c8990184634f101c1aad42fecHarvey Harrison __func__, lflags); 608aa7e847d834ed937a9ad37a0f2ad5b8584c1ab0Jens Axboe congestion_wait(BLK_RW_ASYNC, HZ/50); 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } while (1); 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid * 65fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinnerkmem_zalloc_large(size_t size, xfs_km_flags_t flags) 66fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner{ 67ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner unsigned noio_flag = 0; 68fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner void *ptr; 69ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner gfp_t lflags; 70fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner 71fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner ptr = kmem_zalloc(size, flags | KM_MAYFAIL); 72fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner if (ptr) 73fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner return ptr; 74ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner 75ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner /* 76ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner * __vmalloc() will allocate data pages and auxillary structures (e.g. 77ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner * pagetables) with GFP_KERNEL, yet we may be under GFP_NOFS context 78ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner * here. Hence we need to tell memory reclaim that we are in such a 79ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner * context via PF_MEMALLOC_NOIO to prevent memory reclaim re-entering 80ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner * the filesystem here and potentially deadlocking. 81ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner */ 82ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner if ((current->flags & PF_FSTRANS) || (flags & KM_NOFS)) 83ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner noio_flag = memalloc_noio_save(); 84ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner 85ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner lflags = kmem_flags_convert(flags); 86ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner ptr = __vmalloc(size, lflags | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL); 87ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner 88ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner if ((current->flags & PF_FSTRANS) || (flags & KM_NOFS)) 89ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner memalloc_noio_restore(noio_flag); 90ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner 91ae687e58b3f09b1b3c0faf2cac8c27fbbefb5a48Dave Chinner return ptr; 92fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner} 93fdd3cceef46f2c18c618669cfae5c0f47d6982f9Dave Chinner 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid 95d3689d7687dbbc46c5004557d53349f6952fbc93Barry Naujokkmem_free(const void *ptr) 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 979e2779fa281cfda13ac060753d674bbcaa23367eChristoph Lameter if (!is_vmalloc_addr(ptr)) { 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(ptr); 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vfree(ptr); 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid * 105d3689d7687dbbc46c5004557d53349f6952fbc93Barry Naujokkmem_realloc(const void *ptr, size_t newsize, size_t oldsize, 10677ba78776e90e8de541f13b326e284c74286252fAl Viro xfs_km_flags_t flags) 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *new; 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds new = kmem_alloc(newsize, flags); 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ptr) { 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (new) 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memcpy(new, ptr, 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ((oldsize < newsize) ? oldsize : newsize)); 115f0e2d93c29dc39ffd24cac180a19d48f700c0706Denys Vlasenko kmem_free(ptr); 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return new; 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid * 12177ba78776e90e8de541f13b326e284c74286252fAl Virokmem_zone_alloc(kmem_zone_t *zone, xfs_km_flags_t flags) 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 12327496a8c67bef4d789d8e3c8317ca35813a507aeAl Viro int retries = 0; 12427496a8c67bef4d789d8e3c8317ca35813a507aeAl Viro gfp_t lflags = kmem_flags_convert(flags); 12527496a8c67bef4d789d8e3c8317ca35813a507aeAl Viro void *ptr; 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ptr = kmem_cache_alloc(zone, lflags); 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ptr || (flags & (KM_MAYFAIL|KM_NOSLEEP))) 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ptr; 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(++retries % 100)) 1324f10700a2e4bb2ff3d3a80f08412e21109e6d4b5Dave Chinner xfs_err(NULL, 1334f10700a2e4bb2ff3d3a80f08412e21109e6d4b5Dave Chinner "possible memory allocation deadlock in %s (mode:0x%x)", 13434a622b2e1c8e11c8990184634f101c1aad42fecHarvey Harrison __func__, lflags); 1358aa7e847d834ed937a9ad37a0f2ad5b8584c1ab0Jens Axboe congestion_wait(BLK_RW_ASYNC, HZ/50); 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } while (1); 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 138