slab.c revision 8c78f3075dab4be279e283f901f00e33ce44890a
1/* 2 * linux/mm/slab.c 3 * Written by Mark Hemment, 1996/97. 4 * (markhe@nextd.demon.co.uk) 5 * 6 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli 7 * 8 * Major cleanup, different bufctl logic, per-cpu arrays 9 * (c) 2000 Manfred Spraul 10 * 11 * Cleanup, make the head arrays unconditional, preparation for NUMA 12 * (c) 2002 Manfred Spraul 13 * 14 * An implementation of the Slab Allocator as described in outline in; 15 * UNIX Internals: The New Frontiers by Uresh Vahalia 16 * Pub: Prentice Hall ISBN 0-13-101908-2 17 * or with a little more detail in; 18 * The Slab Allocator: An Object-Caching Kernel Memory Allocator 19 * Jeff Bonwick (Sun Microsystems). 20 * Presented at: USENIX Summer 1994 Technical Conference 21 * 22 * The memory is organized in caches, one cache for each object type. 23 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct) 24 * Each cache consists out of many slabs (they are small (usually one 25 * page long) and always contiguous), and each slab contains multiple 26 * initialized objects. 27 * 28 * This means, that your constructor is used only for newly allocated 29 * slabs and you must pass objects with the same intializations to 30 * kmem_cache_free. 31 * 32 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM, 33 * normal). If you need a special memory type, then must create a new 34 * cache for that memory type. 35 * 36 * In order to reduce fragmentation, the slabs are sorted in 3 groups: 37 * full slabs with 0 free objects 38 * partial slabs 39 * empty slabs with no allocated objects 40 * 41 * If partial slabs exist, then new allocations come from these slabs, 42 * otherwise from empty slabs or new slabs are allocated. 43 * 44 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache 45 * during kmem_cache_destroy(). The caller must prevent concurrent allocs. 46 * 47 * Each cache has a short per-cpu head array, most allocs 48 * and frees go into that array, and if that array overflows, then 1/2 49 * of the entries in the array are given back into the global cache. 50 * The head array is strictly LIFO and should improve the cache hit rates. 51 * On SMP, it additionally reduces the spinlock operations. 52 * 53 * The c_cpuarray may not be read with enabled local interrupts - 54 * it's changed with a smp_call_function(). 55 * 56 * SMP synchronization: 57 * constructors and destructors are called without any locking. 58 * Several members in struct kmem_cache and struct slab never change, they 59 * are accessed without any locking. 60 * The per-cpu arrays are never accessed from the wrong cpu, no locking, 61 * and local interrupts are disabled so slab code is preempt-safe. 62 * The non-constant members are protected with a per-cache irq spinlock. 63 * 64 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch 65 * in 2000 - many ideas in the current implementation are derived from 66 * his patch. 67 * 68 * Further notes from the original documentation: 69 * 70 * 11 April '97. Started multi-threading - markhe 71 * The global cache-chain is protected by the mutex 'cache_chain_mutex'. 72 * The sem is only needed when accessing/extending the cache-chain, which 73 * can never happen inside an interrupt (kmem_cache_create(), 74 * kmem_cache_shrink() and kmem_cache_reap()). 75 * 76 * At present, each engine can be growing a cache. This should be blocked. 77 * 78 * 15 March 2005. NUMA slab allocator. 79 * Shai Fultheim <shai@scalex86.org>. 80 * Shobhit Dayal <shobhit@calsoftinc.com> 81 * Alok N Kataria <alokk@calsoftinc.com> 82 * Christoph Lameter <christoph@lameter.com> 83 * 84 * Modified the slab allocator to be node aware on NUMA systems. 85 * Each node has its own list of partial, free and full slabs. 86 * All object allocations for a node occur from node specific slab lists. 87 */ 88 89#include <linux/config.h> 90#include <linux/slab.h> 91#include <linux/mm.h> 92#include <linux/poison.h> 93#include <linux/swap.h> 94#include <linux/cache.h> 95#include <linux/interrupt.h> 96#include <linux/init.h> 97#include <linux/compiler.h> 98#include <linux/cpuset.h> 99#include <linux/seq_file.h> 100#include <linux/notifier.h> 101#include <linux/kallsyms.h> 102#include <linux/cpu.h> 103#include <linux/sysctl.h> 104#include <linux/module.h> 105#include <linux/rcupdate.h> 106#include <linux/string.h> 107#include <linux/nodemask.h> 108#include <linux/mempolicy.h> 109#include <linux/mutex.h> 110#include <linux/rtmutex.h> 111 112#include <asm/uaccess.h> 113#include <asm/cacheflush.h> 114#include <asm/tlbflush.h> 115#include <asm/page.h> 116 117/* 118 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_DEBUG_INITIAL, 119 * SLAB_RED_ZONE & SLAB_POISON. 120 * 0 for faster, smaller code (especially in the critical paths). 121 * 122 * STATS - 1 to collect stats for /proc/slabinfo. 123 * 0 for faster, smaller code (especially in the critical paths). 124 * 125 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible) 126 */ 127 128#ifdef CONFIG_DEBUG_SLAB 129#define DEBUG 1 130#define STATS 1 131#define FORCED_DEBUG 1 132#else 133#define DEBUG 0 134#define STATS 0 135#define FORCED_DEBUG 0 136#endif 137 138/* Shouldn't this be in a header file somewhere? */ 139#define BYTES_PER_WORD sizeof(void *) 140 141#ifndef cache_line_size 142#define cache_line_size() L1_CACHE_BYTES 143#endif 144 145#ifndef ARCH_KMALLOC_MINALIGN 146/* 147 * Enforce a minimum alignment for the kmalloc caches. 148 * Usually, the kmalloc caches are cache_line_size() aligned, except when 149 * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned. 150 * Some archs want to perform DMA into kmalloc caches and need a guaranteed 151 * alignment larger than BYTES_PER_WORD. ARCH_KMALLOC_MINALIGN allows that. 152 * Note that this flag disables some debug features. 153 */ 154#define ARCH_KMALLOC_MINALIGN 0 155#endif 156 157#ifndef ARCH_SLAB_MINALIGN 158/* 159 * Enforce a minimum alignment for all caches. 160 * Intended for archs that get misalignment faults even for BYTES_PER_WORD 161 * aligned buffers. Includes ARCH_KMALLOC_MINALIGN. 162 * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables 163 * some debug features. 164 */ 165#define ARCH_SLAB_MINALIGN 0 166#endif 167 168#ifndef ARCH_KMALLOC_FLAGS 169#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN 170#endif 171 172/* Legal flag mask for kmem_cache_create(). */ 173#if DEBUG 174# define CREATE_MASK (SLAB_DEBUG_INITIAL | SLAB_RED_ZONE | \ 175 SLAB_POISON | SLAB_HWCACHE_ALIGN | \ 176 SLAB_CACHE_DMA | \ 177 SLAB_MUST_HWCACHE_ALIGN | SLAB_STORE_USER | \ 178 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \ 179 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD) 180#else 181# define CREATE_MASK (SLAB_HWCACHE_ALIGN | \ 182 SLAB_CACHE_DMA | SLAB_MUST_HWCACHE_ALIGN | \ 183 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \ 184 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD) 185#endif 186 187/* 188 * kmem_bufctl_t: 189 * 190 * Bufctl's are used for linking objs within a slab 191 * linked offsets. 192 * 193 * This implementation relies on "struct page" for locating the cache & 194 * slab an object belongs to. 195 * This allows the bufctl structure to be small (one int), but limits 196 * the number of objects a slab (not a cache) can contain when off-slab 197 * bufctls are used. The limit is the size of the largest general cache 198 * that does not use off-slab slabs. 199 * For 32bit archs with 4 kB pages, is this 56. 200 * This is not serious, as it is only for large objects, when it is unwise 201 * to have too many per slab. 202 * Note: This limit can be raised by introducing a general cache whose size 203 * is less than 512 (PAGE_SIZE<<3), but greater than 256. 204 */ 205 206typedef unsigned int kmem_bufctl_t; 207#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0) 208#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1) 209#define BUFCTL_ACTIVE (((kmem_bufctl_t)(~0U))-2) 210#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-3) 211 212/* 213 * struct slab 214 * 215 * Manages the objs in a slab. Placed either at the beginning of mem allocated 216 * for a slab, or allocated from an general cache. 217 * Slabs are chained into three list: fully used, partial, fully free slabs. 218 */ 219struct slab { 220 struct list_head list; 221 unsigned long colouroff; 222 void *s_mem; /* including colour offset */ 223 unsigned int inuse; /* num of objs active in slab */ 224 kmem_bufctl_t free; 225 unsigned short nodeid; 226}; 227 228/* 229 * struct slab_rcu 230 * 231 * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to 232 * arrange for kmem_freepages to be called via RCU. This is useful if 233 * we need to approach a kernel structure obliquely, from its address 234 * obtained without the usual locking. We can lock the structure to 235 * stabilize it and check it's still at the given address, only if we 236 * can be sure that the memory has not been meanwhile reused for some 237 * other kind of object (which our subsystem's lock might corrupt). 238 * 239 * rcu_read_lock before reading the address, then rcu_read_unlock after 240 * taking the spinlock within the structure expected at that address. 241 * 242 * We assume struct slab_rcu can overlay struct slab when destroying. 243 */ 244struct slab_rcu { 245 struct rcu_head head; 246 struct kmem_cache *cachep; 247 void *addr; 248}; 249 250/* 251 * struct array_cache 252 * 253 * Purpose: 254 * - LIFO ordering, to hand out cache-warm objects from _alloc 255 * - reduce the number of linked list operations 256 * - reduce spinlock operations 257 * 258 * The limit is stored in the per-cpu structure to reduce the data cache 259 * footprint. 260 * 261 */ 262struct array_cache { 263 unsigned int avail; 264 unsigned int limit; 265 unsigned int batchcount; 266 unsigned int touched; 267 spinlock_t lock; 268 void *entry[0]; /* 269 * Must have this definition in here for the proper 270 * alignment of array_cache. Also simplifies accessing 271 * the entries. 272 * [0] is for gcc 2.95. It should really be []. 273 */ 274}; 275 276/* 277 * bootstrap: The caches do not work without cpuarrays anymore, but the 278 * cpuarrays are allocated from the generic caches... 279 */ 280#define BOOT_CPUCACHE_ENTRIES 1 281struct arraycache_init { 282 struct array_cache cache; 283 void *entries[BOOT_CPUCACHE_ENTRIES]; 284}; 285 286/* 287 * The slab lists for all objects. 288 */ 289struct kmem_list3 { 290 struct list_head slabs_partial; /* partial list first, better asm code */ 291 struct list_head slabs_full; 292 struct list_head slabs_free; 293 unsigned long free_objects; 294 unsigned int free_limit; 295 unsigned int colour_next; /* Per-node cache coloring */ 296 spinlock_t list_lock; 297 struct array_cache *shared; /* shared per node */ 298 struct array_cache **alien; /* on other nodes */ 299 unsigned long next_reap; /* updated without locking */ 300 int free_touched; /* updated without locking */ 301}; 302 303/* 304 * Need this for bootstrapping a per node allocator. 305 */ 306#define NUM_INIT_LISTS (2 * MAX_NUMNODES + 1) 307struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS]; 308#define CACHE_CACHE 0 309#define SIZE_AC 1 310#define SIZE_L3 (1 + MAX_NUMNODES) 311 312static int drain_freelist(struct kmem_cache *cache, 313 struct kmem_list3 *l3, int tofree); 314static void free_block(struct kmem_cache *cachep, void **objpp, int len, 315 int node); 316static void enable_cpucache(struct kmem_cache *cachep); 317static void cache_reap(void *unused); 318 319/* 320 * This function must be completely optimized away if a constant is passed to 321 * it. Mostly the same as what is in linux/slab.h except it returns an index. 322 */ 323static __always_inline int index_of(const size_t size) 324{ 325 extern void __bad_size(void); 326 327 if (__builtin_constant_p(size)) { 328 int i = 0; 329 330#define CACHE(x) \ 331 if (size <=x) \ 332 return i; \ 333 else \ 334 i++; 335#include "linux/kmalloc_sizes.h" 336#undef CACHE 337 __bad_size(); 338 } else 339 __bad_size(); 340 return 0; 341} 342 343static int slab_early_init = 1; 344 345#define INDEX_AC index_of(sizeof(struct arraycache_init)) 346#define INDEX_L3 index_of(sizeof(struct kmem_list3)) 347 348static void kmem_list3_init(struct kmem_list3 *parent) 349{ 350 INIT_LIST_HEAD(&parent->slabs_full); 351 INIT_LIST_HEAD(&parent->slabs_partial); 352 INIT_LIST_HEAD(&parent->slabs_free); 353 parent->shared = NULL; 354 parent->alien = NULL; 355 parent->colour_next = 0; 356 spin_lock_init(&parent->list_lock); 357 parent->free_objects = 0; 358 parent->free_touched = 0; 359} 360 361#define MAKE_LIST(cachep, listp, slab, nodeid) \ 362 do { \ 363 INIT_LIST_HEAD(listp); \ 364 list_splice(&(cachep->nodelists[nodeid]->slab), listp); \ 365 } while (0) 366 367#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ 368 do { \ 369 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \ 370 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \ 371 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \ 372 } while (0) 373 374/* 375 * struct kmem_cache 376 * 377 * manages a cache. 378 */ 379 380struct kmem_cache { 381/* 1) per-cpu data, touched during every alloc/free */ 382 struct array_cache *array[NR_CPUS]; 383/* 2) Cache tunables. Protected by cache_chain_mutex */ 384 unsigned int batchcount; 385 unsigned int limit; 386 unsigned int shared; 387 388 unsigned int buffer_size; 389/* 3) touched by every alloc & free from the backend */ 390 struct kmem_list3 *nodelists[MAX_NUMNODES]; 391 392 unsigned int flags; /* constant flags */ 393 unsigned int num; /* # of objs per slab */ 394 395/* 4) cache_grow/shrink */ 396 /* order of pgs per slab (2^n) */ 397 unsigned int gfporder; 398 399 /* force GFP flags, e.g. GFP_DMA */ 400 gfp_t gfpflags; 401 402 size_t colour; /* cache colouring range */ 403 unsigned int colour_off; /* colour offset */ 404 struct kmem_cache *slabp_cache; 405 unsigned int slab_size; 406 unsigned int dflags; /* dynamic flags */ 407 408 /* constructor func */ 409 void (*ctor) (void *, struct kmem_cache *, unsigned long); 410 411 /* de-constructor func */ 412 void (*dtor) (void *, struct kmem_cache *, unsigned long); 413 414/* 5) cache creation/removal */ 415 const char *name; 416 struct list_head next; 417 418/* 6) statistics */ 419#if STATS 420 unsigned long num_active; 421 unsigned long num_allocations; 422 unsigned long high_mark; 423 unsigned long grown; 424 unsigned long reaped; 425 unsigned long errors; 426 unsigned long max_freeable; 427 unsigned long node_allocs; 428 unsigned long node_frees; 429 unsigned long node_overflow; 430 atomic_t allochit; 431 atomic_t allocmiss; 432 atomic_t freehit; 433 atomic_t freemiss; 434#endif 435#if DEBUG 436 /* 437 * If debugging is enabled, then the allocator can add additional 438 * fields and/or padding to every object. buffer_size contains the total 439 * object size including these internal fields, the following two 440 * variables contain the offset to the user object and its size. 441 */ 442 int obj_offset; 443 int obj_size; 444#endif 445}; 446 447#define CFLGS_OFF_SLAB (0x80000000UL) 448#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB) 449 450#define BATCHREFILL_LIMIT 16 451/* 452 * Optimization question: fewer reaps means less probability for unnessary 453 * cpucache drain/refill cycles. 454 * 455 * OTOH the cpuarrays can contain lots of objects, 456 * which could lock up otherwise freeable slabs. 457 */ 458#define REAPTIMEOUT_CPUC (2*HZ) 459#define REAPTIMEOUT_LIST3 (4*HZ) 460 461#if STATS 462#define STATS_INC_ACTIVE(x) ((x)->num_active++) 463#define STATS_DEC_ACTIVE(x) ((x)->num_active--) 464#define STATS_INC_ALLOCED(x) ((x)->num_allocations++) 465#define STATS_INC_GROWN(x) ((x)->grown++) 466#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y)) 467#define STATS_SET_HIGH(x) \ 468 do { \ 469 if ((x)->num_active > (x)->high_mark) \ 470 (x)->high_mark = (x)->num_active; \ 471 } while (0) 472#define STATS_INC_ERR(x) ((x)->errors++) 473#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++) 474#define STATS_INC_NODEFREES(x) ((x)->node_frees++) 475#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++) 476#define STATS_SET_FREEABLE(x, i) \ 477 do { \ 478 if ((x)->max_freeable < i) \ 479 (x)->max_freeable = i; \ 480 } while (0) 481#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit) 482#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss) 483#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit) 484#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss) 485#else 486#define STATS_INC_ACTIVE(x) do { } while (0) 487#define STATS_DEC_ACTIVE(x) do { } while (0) 488#define STATS_INC_ALLOCED(x) do { } while (0) 489#define STATS_INC_GROWN(x) do { } while (0) 490#define STATS_ADD_REAPED(x,y) do { } while (0) 491#define STATS_SET_HIGH(x) do { } while (0) 492#define STATS_INC_ERR(x) do { } while (0) 493#define STATS_INC_NODEALLOCS(x) do { } while (0) 494#define STATS_INC_NODEFREES(x) do { } while (0) 495#define STATS_INC_ACOVERFLOW(x) do { } while (0) 496#define STATS_SET_FREEABLE(x, i) do { } while (0) 497#define STATS_INC_ALLOCHIT(x) do { } while (0) 498#define STATS_INC_ALLOCMISS(x) do { } while (0) 499#define STATS_INC_FREEHIT(x) do { } while (0) 500#define STATS_INC_FREEMISS(x) do { } while (0) 501#endif 502 503#if DEBUG 504 505/* 506 * memory layout of objects: 507 * 0 : objp 508 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that 509 * the end of an object is aligned with the end of the real 510 * allocation. Catches writes behind the end of the allocation. 511 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1: 512 * redzone word. 513 * cachep->obj_offset: The real object. 514 * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long] 515 * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address 516 * [BYTES_PER_WORD long] 517 */ 518static int obj_offset(struct kmem_cache *cachep) 519{ 520 return cachep->obj_offset; 521} 522 523static int obj_size(struct kmem_cache *cachep) 524{ 525 return cachep->obj_size; 526} 527 528static unsigned long *dbg_redzone1(struct kmem_cache *cachep, void *objp) 529{ 530 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); 531 return (unsigned long*) (objp+obj_offset(cachep)-BYTES_PER_WORD); 532} 533 534static unsigned long *dbg_redzone2(struct kmem_cache *cachep, void *objp) 535{ 536 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); 537 if (cachep->flags & SLAB_STORE_USER) 538 return (unsigned long *)(objp + cachep->buffer_size - 539 2 * BYTES_PER_WORD); 540 return (unsigned long *)(objp + cachep->buffer_size - BYTES_PER_WORD); 541} 542 543static void **dbg_userword(struct kmem_cache *cachep, void *objp) 544{ 545 BUG_ON(!(cachep->flags & SLAB_STORE_USER)); 546 return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD); 547} 548 549#else 550 551#define obj_offset(x) 0 552#define obj_size(cachep) (cachep->buffer_size) 553#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long *)NULL;}) 554#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long *)NULL;}) 555#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) 556 557#endif 558 559/* 560 * Maximum size of an obj (in 2^order pages) and absolute limit for the gfp 561 * order. 562 */ 563#if defined(CONFIG_LARGE_ALLOCS) 564#define MAX_OBJ_ORDER 13 /* up to 32Mb */ 565#define MAX_GFP_ORDER 13 /* up to 32Mb */ 566#elif defined(CONFIG_MMU) 567#define MAX_OBJ_ORDER 5 /* 32 pages */ 568#define MAX_GFP_ORDER 5 /* 32 pages */ 569#else 570#define MAX_OBJ_ORDER 8 /* up to 1Mb */ 571#define MAX_GFP_ORDER 8 /* up to 1Mb */ 572#endif 573 574/* 575 * Do not go above this order unless 0 objects fit into the slab. 576 */ 577#define BREAK_GFP_ORDER_HI 1 578#define BREAK_GFP_ORDER_LO 0 579static int slab_break_gfp_order = BREAK_GFP_ORDER_LO; 580 581/* 582 * Functions for storing/retrieving the cachep and or slab from the page 583 * allocator. These are used to find the slab an obj belongs to. With kfree(), 584 * these are used to find the cache which an obj belongs to. 585 */ 586static inline void page_set_cache(struct page *page, struct kmem_cache *cache) 587{ 588 page->lru.next = (struct list_head *)cache; 589} 590 591static inline struct kmem_cache *page_get_cache(struct page *page) 592{ 593 if (unlikely(PageCompound(page))) 594 page = (struct page *)page_private(page); 595 BUG_ON(!PageSlab(page)); 596 return (struct kmem_cache *)page->lru.next; 597} 598 599static inline void page_set_slab(struct page *page, struct slab *slab) 600{ 601 page->lru.prev = (struct list_head *)slab; 602} 603 604static inline struct slab *page_get_slab(struct page *page) 605{ 606 if (unlikely(PageCompound(page))) 607 page = (struct page *)page_private(page); 608 BUG_ON(!PageSlab(page)); 609 return (struct slab *)page->lru.prev; 610} 611 612static inline struct kmem_cache *virt_to_cache(const void *obj) 613{ 614 struct page *page = virt_to_page(obj); 615 return page_get_cache(page); 616} 617 618static inline struct slab *virt_to_slab(const void *obj) 619{ 620 struct page *page = virt_to_page(obj); 621 return page_get_slab(page); 622} 623 624static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab, 625 unsigned int idx) 626{ 627 return slab->s_mem + cache->buffer_size * idx; 628} 629 630static inline unsigned int obj_to_index(struct kmem_cache *cache, 631 struct slab *slab, void *obj) 632{ 633 return (unsigned)(obj - slab->s_mem) / cache->buffer_size; 634} 635 636/* 637 * These are the default caches for kmalloc. Custom caches can have other sizes. 638 */ 639struct cache_sizes malloc_sizes[] = { 640#define CACHE(x) { .cs_size = (x) }, 641#include <linux/kmalloc_sizes.h> 642 CACHE(ULONG_MAX) 643#undef CACHE 644}; 645EXPORT_SYMBOL(malloc_sizes); 646 647/* Must match cache_sizes above. Out of line to keep cache footprint low. */ 648struct cache_names { 649 char *name; 650 char *name_dma; 651}; 652 653static struct cache_names __initdata cache_names[] = { 654#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, 655#include <linux/kmalloc_sizes.h> 656 {NULL,} 657#undef CACHE 658}; 659 660static struct arraycache_init initarray_cache __initdata = 661 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 662static struct arraycache_init initarray_generic = 663 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 664 665/* internal cache of cache description objs */ 666static struct kmem_cache cache_cache = { 667 .batchcount = 1, 668 .limit = BOOT_CPUCACHE_ENTRIES, 669 .shared = 1, 670 .buffer_size = sizeof(struct kmem_cache), 671 .name = "kmem_cache", 672#if DEBUG 673 .obj_size = sizeof(struct kmem_cache), 674#endif 675}; 676 677#ifdef CONFIG_LOCKDEP 678 679/* 680 * Slab sometimes uses the kmalloc slabs to store the slab headers 681 * for other slabs "off slab". 682 * The locking for this is tricky in that it nests within the locks 683 * of all other slabs in a few places; to deal with this special 684 * locking we put on-slab caches into a separate lock-class. 685 */ 686static struct lock_class_key on_slab_key; 687 688static inline void init_lock_keys(struct cache_sizes *s) 689{ 690 int q; 691 692 for (q = 0; q < MAX_NUMNODES; q++) { 693 if (!s->cs_cachep->nodelists[q] || OFF_SLAB(s->cs_cachep)) 694 continue; 695 lockdep_set_class(&s->cs_cachep->nodelists[q]->list_lock, 696 &on_slab_key); 697 } 698} 699 700#else 701static inline void init_lock_keys(struct cache_sizes *s) 702{ 703} 704#endif 705 706 707 708/* Guard access to the cache-chain. */ 709static DEFINE_MUTEX(cache_chain_mutex); 710static struct list_head cache_chain; 711 712/* 713 * vm_enough_memory() looks at this to determine how many slab-allocated pages 714 * are possibly freeable under pressure 715 * 716 * SLAB_RECLAIM_ACCOUNT turns this on per-slab 717 */ 718atomic_t slab_reclaim_pages; 719 720/* 721 * chicken and egg problem: delay the per-cpu array allocation 722 * until the general caches are up. 723 */ 724static enum { 725 NONE, 726 PARTIAL_AC, 727 PARTIAL_L3, 728 FULL 729} g_cpucache_up; 730 731/* 732 * used by boot code to determine if it can use slab based allocator 733 */ 734int slab_is_available(void) 735{ 736 return g_cpucache_up == FULL; 737} 738 739static DEFINE_PER_CPU(struct work_struct, reap_work); 740 741static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) 742{ 743 return cachep->array[smp_processor_id()]; 744} 745 746static inline struct kmem_cache *__find_general_cachep(size_t size, 747 gfp_t gfpflags) 748{ 749 struct cache_sizes *csizep = malloc_sizes; 750 751#if DEBUG 752 /* This happens if someone tries to call 753 * kmem_cache_create(), or __kmalloc(), before 754 * the generic caches are initialized. 755 */ 756 BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); 757#endif 758 while (size > csizep->cs_size) 759 csizep++; 760 761 /* 762 * Really subtle: The last entry with cs->cs_size==ULONG_MAX 763 * has cs_{dma,}cachep==NULL. Thus no special case 764 * for large kmalloc calls required. 765 */ 766 if (unlikely(gfpflags & GFP_DMA)) 767 return csizep->cs_dmacachep; 768 return csizep->cs_cachep; 769} 770 771struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags) 772{ 773 return __find_general_cachep(size, gfpflags); 774} 775EXPORT_SYMBOL(kmem_find_general_cachep); 776 777static size_t slab_mgmt_size(size_t nr_objs, size_t align) 778{ 779 return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align); 780} 781 782/* 783 * Calculate the number of objects and left-over bytes for a given buffer size. 784 */ 785static void cache_estimate(unsigned long gfporder, size_t buffer_size, 786 size_t align, int flags, size_t *left_over, 787 unsigned int *num) 788{ 789 int nr_objs; 790 size_t mgmt_size; 791 size_t slab_size = PAGE_SIZE << gfporder; 792 793 /* 794 * The slab management structure can be either off the slab or 795 * on it. For the latter case, the memory allocated for a 796 * slab is used for: 797 * 798 * - The struct slab 799 * - One kmem_bufctl_t for each object 800 * - Padding to respect alignment of @align 801 * - @buffer_size bytes for each object 802 * 803 * If the slab management structure is off the slab, then the 804 * alignment will already be calculated into the size. Because 805 * the slabs are all pages aligned, the objects will be at the 806 * correct alignment when allocated. 807 */ 808 if (flags & CFLGS_OFF_SLAB) { 809 mgmt_size = 0; 810 nr_objs = slab_size / buffer_size; 811 812 if (nr_objs > SLAB_LIMIT) 813 nr_objs = SLAB_LIMIT; 814 } else { 815 /* 816 * Ignore padding for the initial guess. The padding 817 * is at most @align-1 bytes, and @buffer_size is at 818 * least @align. In the worst case, this result will 819 * be one greater than the number of objects that fit 820 * into the memory allocation when taking the padding 821 * into account. 822 */ 823 nr_objs = (slab_size - sizeof(struct slab)) / 824 (buffer_size + sizeof(kmem_bufctl_t)); 825 826 /* 827 * This calculated number will be either the right 828 * amount, or one greater than what we want. 829 */ 830 if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size 831 > slab_size) 832 nr_objs--; 833 834 if (nr_objs > SLAB_LIMIT) 835 nr_objs = SLAB_LIMIT; 836 837 mgmt_size = slab_mgmt_size(nr_objs, align); 838 } 839 *num = nr_objs; 840 *left_over = slab_size - nr_objs*buffer_size - mgmt_size; 841} 842 843#define slab_error(cachep, msg) __slab_error(__FUNCTION__, cachep, msg) 844 845static void __slab_error(const char *function, struct kmem_cache *cachep, 846 char *msg) 847{ 848 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n", 849 function, cachep->name, msg); 850 dump_stack(); 851} 852 853#ifdef CONFIG_NUMA 854/* 855 * Special reaping functions for NUMA systems called from cache_reap(). 856 * These take care of doing round robin flushing of alien caches (containing 857 * objects freed on different nodes from which they were allocated) and the 858 * flushing of remote pcps by calling drain_node_pages. 859 */ 860static DEFINE_PER_CPU(unsigned long, reap_node); 861 862static void init_reap_node(int cpu) 863{ 864 int node; 865 866 node = next_node(cpu_to_node(cpu), node_online_map); 867 if (node == MAX_NUMNODES) 868 node = first_node(node_online_map); 869 870 __get_cpu_var(reap_node) = node; 871} 872 873static void next_reap_node(void) 874{ 875 int node = __get_cpu_var(reap_node); 876 877 /* 878 * Also drain per cpu pages on remote zones 879 */ 880 if (node != numa_node_id()) 881 drain_node_pages(node); 882 883 node = next_node(node, node_online_map); 884 if (unlikely(node >= MAX_NUMNODES)) 885 node = first_node(node_online_map); 886 __get_cpu_var(reap_node) = node; 887} 888 889#else 890#define init_reap_node(cpu) do { } while (0) 891#define next_reap_node(void) do { } while (0) 892#endif 893 894/* 895 * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz 896 * via the workqueue/eventd. 897 * Add the CPU number into the expiration time to minimize the possibility of 898 * the CPUs getting into lockstep and contending for the global cache chain 899 * lock. 900 */ 901static void __devinit start_cpu_timer(int cpu) 902{ 903 struct work_struct *reap_work = &per_cpu(reap_work, cpu); 904 905 /* 906 * When this gets called from do_initcalls via cpucache_init(), 907 * init_workqueues() has already run, so keventd will be setup 908 * at that time. 909 */ 910 if (keventd_up() && reap_work->func == NULL) { 911 init_reap_node(cpu); 912 INIT_WORK(reap_work, cache_reap, NULL); 913 schedule_delayed_work_on(cpu, reap_work, HZ + 3 * cpu); 914 } 915} 916 917static struct array_cache *alloc_arraycache(int node, int entries, 918 int batchcount) 919{ 920 int memsize = sizeof(void *) * entries + sizeof(struct array_cache); 921 struct array_cache *nc = NULL; 922 923 nc = kmalloc_node(memsize, GFP_KERNEL, node); 924 if (nc) { 925 nc->avail = 0; 926 nc->limit = entries; 927 nc->batchcount = batchcount; 928 nc->touched = 0; 929 spin_lock_init(&nc->lock); 930 } 931 return nc; 932} 933 934/* 935 * Transfer objects in one arraycache to another. 936 * Locking must be handled by the caller. 937 * 938 * Return the number of entries transferred. 939 */ 940static int transfer_objects(struct array_cache *to, 941 struct array_cache *from, unsigned int max) 942{ 943 /* Figure out how many entries to transfer */ 944 int nr = min(min(from->avail, max), to->limit - to->avail); 945 946 if (!nr) 947 return 0; 948 949 memcpy(to->entry + to->avail, from->entry + from->avail -nr, 950 sizeof(void *) *nr); 951 952 from->avail -= nr; 953 to->avail += nr; 954 to->touched = 1; 955 return nr; 956} 957 958#ifdef CONFIG_NUMA 959static void *__cache_alloc_node(struct kmem_cache *, gfp_t, int); 960static void *alternate_node_alloc(struct kmem_cache *, gfp_t); 961 962static struct array_cache **alloc_alien_cache(int node, int limit) 963{ 964 struct array_cache **ac_ptr; 965 int memsize = sizeof(void *) * MAX_NUMNODES; 966 int i; 967 968 if (limit > 1) 969 limit = 12; 970 ac_ptr = kmalloc_node(memsize, GFP_KERNEL, node); 971 if (ac_ptr) { 972 for_each_node(i) { 973 if (i == node || !node_online(i)) { 974 ac_ptr[i] = NULL; 975 continue; 976 } 977 ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d); 978 if (!ac_ptr[i]) { 979 for (i--; i <= 0; i--) 980 kfree(ac_ptr[i]); 981 kfree(ac_ptr); 982 return NULL; 983 } 984 } 985 } 986 return ac_ptr; 987} 988 989static void free_alien_cache(struct array_cache **ac_ptr) 990{ 991 int i; 992 993 if (!ac_ptr) 994 return; 995 for_each_node(i) 996 kfree(ac_ptr[i]); 997 kfree(ac_ptr); 998} 999 1000static void __drain_alien_cache(struct kmem_cache *cachep, 1001 struct array_cache *ac, int node) 1002{ 1003 struct kmem_list3 *rl3 = cachep->nodelists[node]; 1004 1005 if (ac->avail) { 1006 spin_lock(&rl3->list_lock); 1007 /* 1008 * Stuff objects into the remote nodes shared array first. 1009 * That way we could avoid the overhead of putting the objects 1010 * into the free lists and getting them back later. 1011 */ 1012 if (rl3->shared) 1013 transfer_objects(rl3->shared, ac, ac->limit); 1014 1015 free_block(cachep, ac->entry, ac->avail, node); 1016 ac->avail = 0; 1017 spin_unlock(&rl3->list_lock); 1018 } 1019} 1020 1021/* 1022 * Called from cache_reap() to regularly drain alien caches round robin. 1023 */ 1024static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3) 1025{ 1026 int node = __get_cpu_var(reap_node); 1027 1028 if (l3->alien) { 1029 struct array_cache *ac = l3->alien[node]; 1030 1031 if (ac && ac->avail && spin_trylock_irq(&ac->lock)) { 1032 __drain_alien_cache(cachep, ac, node); 1033 spin_unlock_irq(&ac->lock); 1034 } 1035 } 1036} 1037 1038static void drain_alien_cache(struct kmem_cache *cachep, 1039 struct array_cache **alien) 1040{ 1041 int i = 0; 1042 struct array_cache *ac; 1043 unsigned long flags; 1044 1045 for_each_online_node(i) { 1046 ac = alien[i]; 1047 if (ac) { 1048 spin_lock_irqsave(&ac->lock, flags); 1049 __drain_alien_cache(cachep, ac, i); 1050 spin_unlock_irqrestore(&ac->lock, flags); 1051 } 1052 } 1053} 1054 1055static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) 1056{ 1057 struct slab *slabp = virt_to_slab(objp); 1058 int nodeid = slabp->nodeid; 1059 struct kmem_list3 *l3; 1060 struct array_cache *alien = NULL; 1061 1062 /* 1063 * Make sure we are not freeing a object from another node to the array 1064 * cache on this cpu. 1065 */ 1066 if (likely(slabp->nodeid == numa_node_id())) 1067 return 0; 1068 1069 l3 = cachep->nodelists[numa_node_id()]; 1070 STATS_INC_NODEFREES(cachep); 1071 if (l3->alien && l3->alien[nodeid]) { 1072 alien = l3->alien[nodeid]; 1073 spin_lock(&alien->lock); 1074 if (unlikely(alien->avail == alien->limit)) { 1075 STATS_INC_ACOVERFLOW(cachep); 1076 __drain_alien_cache(cachep, alien, nodeid); 1077 } 1078 alien->entry[alien->avail++] = objp; 1079 spin_unlock(&alien->lock); 1080 } else { 1081 spin_lock(&(cachep->nodelists[nodeid])->list_lock); 1082 free_block(cachep, &objp, 1, nodeid); 1083 spin_unlock(&(cachep->nodelists[nodeid])->list_lock); 1084 } 1085 return 1; 1086} 1087 1088#else 1089 1090#define drain_alien_cache(cachep, alien) do { } while (0) 1091#define reap_alien(cachep, l3) do { } while (0) 1092 1093static inline struct array_cache **alloc_alien_cache(int node, int limit) 1094{ 1095 return (struct array_cache **) 0x01020304ul; 1096} 1097 1098static inline void free_alien_cache(struct array_cache **ac_ptr) 1099{ 1100} 1101 1102static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) 1103{ 1104 return 0; 1105} 1106 1107#endif 1108 1109static int __cpuinit cpuup_callback(struct notifier_block *nfb, 1110 unsigned long action, void *hcpu) 1111{ 1112 long cpu = (long)hcpu; 1113 struct kmem_cache *cachep; 1114 struct kmem_list3 *l3 = NULL; 1115 int node = cpu_to_node(cpu); 1116 int memsize = sizeof(struct kmem_list3); 1117 1118 switch (action) { 1119 case CPU_UP_PREPARE: 1120 mutex_lock(&cache_chain_mutex); 1121 /* 1122 * We need to do this right in the beginning since 1123 * alloc_arraycache's are going to use this list. 1124 * kmalloc_node allows us to add the slab to the right 1125 * kmem_list3 and not this cpu's kmem_list3 1126 */ 1127 1128 list_for_each_entry(cachep, &cache_chain, next) { 1129 /* 1130 * Set up the size64 kmemlist for cpu before we can 1131 * begin anything. Make sure some other cpu on this 1132 * node has not already allocated this 1133 */ 1134 if (!cachep->nodelists[node]) { 1135 l3 = kmalloc_node(memsize, GFP_KERNEL, node); 1136 if (!l3) 1137 goto bad; 1138 kmem_list3_init(l3); 1139 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + 1140 ((unsigned long)cachep) % REAPTIMEOUT_LIST3; 1141 1142 /* 1143 * The l3s don't come and go as CPUs come and 1144 * go. cache_chain_mutex is sufficient 1145 * protection here. 1146 */ 1147 cachep->nodelists[node] = l3; 1148 } 1149 1150 spin_lock_irq(&cachep->nodelists[node]->list_lock); 1151 cachep->nodelists[node]->free_limit = 1152 (1 + nr_cpus_node(node)) * 1153 cachep->batchcount + cachep->num; 1154 spin_unlock_irq(&cachep->nodelists[node]->list_lock); 1155 } 1156 1157 /* 1158 * Now we can go ahead with allocating the shared arrays and 1159 * array caches 1160 */ 1161 list_for_each_entry(cachep, &cache_chain, next) { 1162 struct array_cache *nc; 1163 struct array_cache *shared; 1164 struct array_cache **alien; 1165 1166 nc = alloc_arraycache(node, cachep->limit, 1167 cachep->batchcount); 1168 if (!nc) 1169 goto bad; 1170 shared = alloc_arraycache(node, 1171 cachep->shared * cachep->batchcount, 1172 0xbaadf00d); 1173 if (!shared) 1174 goto bad; 1175 1176 alien = alloc_alien_cache(node, cachep->limit); 1177 if (!alien) 1178 goto bad; 1179 cachep->array[cpu] = nc; 1180 l3 = cachep->nodelists[node]; 1181 BUG_ON(!l3); 1182 1183 spin_lock_irq(&l3->list_lock); 1184 if (!l3->shared) { 1185 /* 1186 * We are serialised from CPU_DEAD or 1187 * CPU_UP_CANCELLED by the cpucontrol lock 1188 */ 1189 l3->shared = shared; 1190 shared = NULL; 1191 } 1192#ifdef CONFIG_NUMA 1193 if (!l3->alien) { 1194 l3->alien = alien; 1195 alien = NULL; 1196 } 1197#endif 1198 spin_unlock_irq(&l3->list_lock); 1199 kfree(shared); 1200 free_alien_cache(alien); 1201 } 1202 mutex_unlock(&cache_chain_mutex); 1203 break; 1204 case CPU_ONLINE: 1205 start_cpu_timer(cpu); 1206 break; 1207#ifdef CONFIG_HOTPLUG_CPU 1208 case CPU_DEAD: 1209 /* 1210 * Even if all the cpus of a node are down, we don't free the 1211 * kmem_list3 of any cache. This to avoid a race between 1212 * cpu_down, and a kmalloc allocation from another cpu for 1213 * memory from the node of the cpu going down. The list3 1214 * structure is usually allocated from kmem_cache_create() and 1215 * gets destroyed at kmem_cache_destroy(). 1216 */ 1217 /* fall thru */ 1218 case CPU_UP_CANCELED: 1219 mutex_lock(&cache_chain_mutex); 1220 list_for_each_entry(cachep, &cache_chain, next) { 1221 struct array_cache *nc; 1222 struct array_cache *shared; 1223 struct array_cache **alien; 1224 cpumask_t mask; 1225 1226 mask = node_to_cpumask(node); 1227 /* cpu is dead; no one can alloc from it. */ 1228 nc = cachep->array[cpu]; 1229 cachep->array[cpu] = NULL; 1230 l3 = cachep->nodelists[node]; 1231 1232 if (!l3) 1233 goto free_array_cache; 1234 1235 spin_lock_irq(&l3->list_lock); 1236 1237 /* Free limit for this kmem_list3 */ 1238 l3->free_limit -= cachep->batchcount; 1239 if (nc) 1240 free_block(cachep, nc->entry, nc->avail, node); 1241 1242 if (!cpus_empty(mask)) { 1243 spin_unlock_irq(&l3->list_lock); 1244 goto free_array_cache; 1245 } 1246 1247 shared = l3->shared; 1248 if (shared) { 1249 free_block(cachep, l3->shared->entry, 1250 l3->shared->avail, node); 1251 l3->shared = NULL; 1252 } 1253 1254 alien = l3->alien; 1255 l3->alien = NULL; 1256 1257 spin_unlock_irq(&l3->list_lock); 1258 1259 kfree(shared); 1260 if (alien) { 1261 drain_alien_cache(cachep, alien); 1262 free_alien_cache(alien); 1263 } 1264free_array_cache: 1265 kfree(nc); 1266 } 1267 /* 1268 * In the previous loop, all the objects were freed to 1269 * the respective cache's slabs, now we can go ahead and 1270 * shrink each nodelist to its limit. 1271 */ 1272 list_for_each_entry(cachep, &cache_chain, next) { 1273 l3 = cachep->nodelists[node]; 1274 if (!l3) 1275 continue; 1276 drain_freelist(cachep, l3, l3->free_objects); 1277 } 1278 mutex_unlock(&cache_chain_mutex); 1279 break; 1280#endif 1281 } 1282 return NOTIFY_OK; 1283bad: 1284 mutex_unlock(&cache_chain_mutex); 1285 return NOTIFY_BAD; 1286} 1287 1288static struct notifier_block __cpuinitdata cpucache_notifier = { 1289 &cpuup_callback, NULL, 0 1290}; 1291 1292/* 1293 * swap the static kmem_list3 with kmalloced memory 1294 */ 1295static void init_list(struct kmem_cache *cachep, struct kmem_list3 *list, 1296 int nodeid) 1297{ 1298 struct kmem_list3 *ptr; 1299 1300 BUG_ON(cachep->nodelists[nodeid] != list); 1301 ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, nodeid); 1302 BUG_ON(!ptr); 1303 1304 local_irq_disable(); 1305 memcpy(ptr, list, sizeof(struct kmem_list3)); 1306 /* 1307 * Do not assume that spinlocks can be initialized via memcpy: 1308 */ 1309 spin_lock_init(&ptr->list_lock); 1310 1311 MAKE_ALL_LISTS(cachep, ptr, nodeid); 1312 cachep->nodelists[nodeid] = ptr; 1313 local_irq_enable(); 1314} 1315 1316/* 1317 * Initialisation. Called after the page allocator have been initialised and 1318 * before smp_init(). 1319 */ 1320void __init kmem_cache_init(void) 1321{ 1322 size_t left_over; 1323 struct cache_sizes *sizes; 1324 struct cache_names *names; 1325 int i; 1326 int order; 1327 1328 for (i = 0; i < NUM_INIT_LISTS; i++) { 1329 kmem_list3_init(&initkmem_list3[i]); 1330 if (i < MAX_NUMNODES) 1331 cache_cache.nodelists[i] = NULL; 1332 } 1333 1334 /* 1335 * Fragmentation resistance on low memory - only use bigger 1336 * page orders on machines with more than 32MB of memory. 1337 */ 1338 if (num_physpages > (32 << 20) >> PAGE_SHIFT) 1339 slab_break_gfp_order = BREAK_GFP_ORDER_HI; 1340 1341 /* Bootstrap is tricky, because several objects are allocated 1342 * from caches that do not exist yet: 1343 * 1) initialize the cache_cache cache: it contains the struct 1344 * kmem_cache structures of all caches, except cache_cache itself: 1345 * cache_cache is statically allocated. 1346 * Initially an __init data area is used for the head array and the 1347 * kmem_list3 structures, it's replaced with a kmalloc allocated 1348 * array at the end of the bootstrap. 1349 * 2) Create the first kmalloc cache. 1350 * The struct kmem_cache for the new cache is allocated normally. 1351 * An __init data area is used for the head array. 1352 * 3) Create the remaining kmalloc caches, with minimally sized 1353 * head arrays. 1354 * 4) Replace the __init data head arrays for cache_cache and the first 1355 * kmalloc cache with kmalloc allocated arrays. 1356 * 5) Replace the __init data for kmem_list3 for cache_cache and 1357 * the other cache's with kmalloc allocated memory. 1358 * 6) Resize the head arrays of the kmalloc caches to their final sizes. 1359 */ 1360 1361 /* 1) create the cache_cache */ 1362 INIT_LIST_HEAD(&cache_chain); 1363 list_add(&cache_cache.next, &cache_chain); 1364 cache_cache.colour_off = cache_line_size(); 1365 cache_cache.array[smp_processor_id()] = &initarray_cache.cache; 1366 cache_cache.nodelists[numa_node_id()] = &initkmem_list3[CACHE_CACHE]; 1367 1368 cache_cache.buffer_size = ALIGN(cache_cache.buffer_size, 1369 cache_line_size()); 1370 1371 for (order = 0; order < MAX_ORDER; order++) { 1372 cache_estimate(order, cache_cache.buffer_size, 1373 cache_line_size(), 0, &left_over, &cache_cache.num); 1374 if (cache_cache.num) 1375 break; 1376 } 1377 BUG_ON(!cache_cache.num); 1378 cache_cache.gfporder = order; 1379 cache_cache.colour = left_over / cache_cache.colour_off; 1380 cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) + 1381 sizeof(struct slab), cache_line_size()); 1382 1383 /* 2+3) create the kmalloc caches */ 1384 sizes = malloc_sizes; 1385 names = cache_names; 1386 1387 /* 1388 * Initialize the caches that provide memory for the array cache and the 1389 * kmem_list3 structures first. Without this, further allocations will 1390 * bug. 1391 */ 1392 1393 sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, 1394 sizes[INDEX_AC].cs_size, 1395 ARCH_KMALLOC_MINALIGN, 1396 ARCH_KMALLOC_FLAGS|SLAB_PANIC, 1397 NULL, NULL); 1398 1399 if (INDEX_AC != INDEX_L3) { 1400 sizes[INDEX_L3].cs_cachep = 1401 kmem_cache_create(names[INDEX_L3].name, 1402 sizes[INDEX_L3].cs_size, 1403 ARCH_KMALLOC_MINALIGN, 1404 ARCH_KMALLOC_FLAGS|SLAB_PANIC, 1405 NULL, NULL); 1406 } 1407 1408 slab_early_init = 0; 1409 1410 while (sizes->cs_size != ULONG_MAX) { 1411 /* 1412 * For performance, all the general caches are L1 aligned. 1413 * This should be particularly beneficial on SMP boxes, as it 1414 * eliminates "false sharing". 1415 * Note for systems short on memory removing the alignment will 1416 * allow tighter packing of the smaller caches. 1417 */ 1418 if (!sizes->cs_cachep) { 1419 sizes->cs_cachep = kmem_cache_create(names->name, 1420 sizes->cs_size, 1421 ARCH_KMALLOC_MINALIGN, 1422 ARCH_KMALLOC_FLAGS|SLAB_PANIC, 1423 NULL, NULL); 1424 } 1425 init_lock_keys(sizes); 1426 1427 sizes->cs_dmacachep = kmem_cache_create(names->name_dma, 1428 sizes->cs_size, 1429 ARCH_KMALLOC_MINALIGN, 1430 ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| 1431 SLAB_PANIC, 1432 NULL, NULL); 1433 sizes++; 1434 names++; 1435 } 1436 /* 4) Replace the bootstrap head arrays */ 1437 { 1438 struct array_cache *ptr; 1439 1440 ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); 1441 1442 local_irq_disable(); 1443 BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache); 1444 memcpy(ptr, cpu_cache_get(&cache_cache), 1445 sizeof(struct arraycache_init)); 1446 /* 1447 * Do not assume that spinlocks can be initialized via memcpy: 1448 */ 1449 spin_lock_init(&ptr->lock); 1450 1451 cache_cache.array[smp_processor_id()] = ptr; 1452 local_irq_enable(); 1453 1454 ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); 1455 1456 local_irq_disable(); 1457 BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep) 1458 != &initarray_generic.cache); 1459 memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep), 1460 sizeof(struct arraycache_init)); 1461 /* 1462 * Do not assume that spinlocks can be initialized via memcpy: 1463 */ 1464 spin_lock_init(&ptr->lock); 1465 1466 malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = 1467 ptr; 1468 local_irq_enable(); 1469 } 1470 /* 5) Replace the bootstrap kmem_list3's */ 1471 { 1472 int node; 1473 /* Replace the static kmem_list3 structures for the boot cpu */ 1474 init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], 1475 numa_node_id()); 1476 1477 for_each_online_node(node) { 1478 init_list(malloc_sizes[INDEX_AC].cs_cachep, 1479 &initkmem_list3[SIZE_AC + node], node); 1480 1481 if (INDEX_AC != INDEX_L3) { 1482 init_list(malloc_sizes[INDEX_L3].cs_cachep, 1483 &initkmem_list3[SIZE_L3 + node], 1484 node); 1485 } 1486 } 1487 } 1488 1489 /* 6) resize the head arrays to their final sizes */ 1490 { 1491 struct kmem_cache *cachep; 1492 mutex_lock(&cache_chain_mutex); 1493 list_for_each_entry(cachep, &cache_chain, next) 1494 enable_cpucache(cachep); 1495 mutex_unlock(&cache_chain_mutex); 1496 } 1497 1498 /* Done! */ 1499 g_cpucache_up = FULL; 1500 1501 /* 1502 * Register a cpu startup notifier callback that initializes 1503 * cpu_cache_get for all new cpus 1504 */ 1505 register_cpu_notifier(&cpucache_notifier); 1506 1507 /* 1508 * The reap timers are started later, with a module init call: That part 1509 * of the kernel is not yet operational. 1510 */ 1511} 1512 1513static int __init cpucache_init(void) 1514{ 1515 int cpu; 1516 1517 /* 1518 * Register the timers that return unneeded pages to the page allocator 1519 */ 1520 for_each_online_cpu(cpu) 1521 start_cpu_timer(cpu); 1522 return 0; 1523} 1524__initcall(cpucache_init); 1525 1526/* 1527 * Interface to system's page allocator. No need to hold the cache-lock. 1528 * 1529 * If we requested dmaable memory, we will get it. Even if we 1530 * did not request dmaable memory, we might get it, but that 1531 * would be relatively rare and ignorable. 1532 */ 1533static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) 1534{ 1535 struct page *page; 1536 int nr_pages; 1537 int i; 1538 1539#ifndef CONFIG_MMU 1540 /* 1541 * Nommu uses slab's for process anonymous memory allocations, and thus 1542 * requires __GFP_COMP to properly refcount higher order allocations 1543 */ 1544 flags |= __GFP_COMP; 1545#endif 1546 flags |= cachep->gfpflags; 1547 1548 page = alloc_pages_node(nodeid, flags, cachep->gfporder); 1549 if (!page) 1550 return NULL; 1551 1552 nr_pages = (1 << cachep->gfporder); 1553 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1554 atomic_add(nr_pages, &slab_reclaim_pages); 1555 add_zone_page_state(page_zone(page), NR_SLAB, nr_pages); 1556 for (i = 0; i < nr_pages; i++) 1557 __SetPageSlab(page + i); 1558 return page_address(page); 1559} 1560 1561/* 1562 * Interface to system's page release. 1563 */ 1564static void kmem_freepages(struct kmem_cache *cachep, void *addr) 1565{ 1566 unsigned long i = (1 << cachep->gfporder); 1567 struct page *page = virt_to_page(addr); 1568 const unsigned long nr_freed = i; 1569 1570 sub_zone_page_state(page_zone(page), NR_SLAB, nr_freed); 1571 while (i--) { 1572 BUG_ON(!PageSlab(page)); 1573 __ClearPageSlab(page); 1574 page++; 1575 } 1576 if (current->reclaim_state) 1577 current->reclaim_state->reclaimed_slab += nr_freed; 1578 free_pages((unsigned long)addr, cachep->gfporder); 1579 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1580 atomic_sub(1 << cachep->gfporder, &slab_reclaim_pages); 1581} 1582 1583static void kmem_rcu_free(struct rcu_head *head) 1584{ 1585 struct slab_rcu *slab_rcu = (struct slab_rcu *)head; 1586 struct kmem_cache *cachep = slab_rcu->cachep; 1587 1588 kmem_freepages(cachep, slab_rcu->addr); 1589 if (OFF_SLAB(cachep)) 1590 kmem_cache_free(cachep->slabp_cache, slab_rcu); 1591} 1592 1593#if DEBUG 1594 1595#ifdef CONFIG_DEBUG_PAGEALLOC 1596static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr, 1597 unsigned long caller) 1598{ 1599 int size = obj_size(cachep); 1600 1601 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)]; 1602 1603 if (size < 5 * sizeof(unsigned long)) 1604 return; 1605 1606 *addr++ = 0x12345678; 1607 *addr++ = caller; 1608 *addr++ = smp_processor_id(); 1609 size -= 3 * sizeof(unsigned long); 1610 { 1611 unsigned long *sptr = &caller; 1612 unsigned long svalue; 1613 1614 while (!kstack_end(sptr)) { 1615 svalue = *sptr++; 1616 if (kernel_text_address(svalue)) { 1617 *addr++ = svalue; 1618 size -= sizeof(unsigned long); 1619 if (size <= sizeof(unsigned long)) 1620 break; 1621 } 1622 } 1623 1624 } 1625 *addr++ = 0x87654321; 1626} 1627#endif 1628 1629static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val) 1630{ 1631 int size = obj_size(cachep); 1632 addr = &((char *)addr)[obj_offset(cachep)]; 1633 1634 memset(addr, val, size); 1635 *(unsigned char *)(addr + size - 1) = POISON_END; 1636} 1637 1638static void dump_line(char *data, int offset, int limit) 1639{ 1640 int i; 1641 printk(KERN_ERR "%03x:", offset); 1642 for (i = 0; i < limit; i++) 1643 printk(" %02x", (unsigned char)data[offset + i]); 1644 printk("\n"); 1645} 1646#endif 1647 1648#if DEBUG 1649 1650static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) 1651{ 1652 int i, size; 1653 char *realobj; 1654 1655 if (cachep->flags & SLAB_RED_ZONE) { 1656 printk(KERN_ERR "Redzone: 0x%lx/0x%lx.\n", 1657 *dbg_redzone1(cachep, objp), 1658 *dbg_redzone2(cachep, objp)); 1659 } 1660 1661 if (cachep->flags & SLAB_STORE_USER) { 1662 printk(KERN_ERR "Last user: [<%p>]", 1663 *dbg_userword(cachep, objp)); 1664 print_symbol("(%s)", 1665 (unsigned long)*dbg_userword(cachep, objp)); 1666 printk("\n"); 1667 } 1668 realobj = (char *)objp + obj_offset(cachep); 1669 size = obj_size(cachep); 1670 for (i = 0; i < size && lines; i += 16, lines--) { 1671 int limit; 1672 limit = 16; 1673 if (i + limit > size) 1674 limit = size - i; 1675 dump_line(realobj, i, limit); 1676 } 1677} 1678 1679static void check_poison_obj(struct kmem_cache *cachep, void *objp) 1680{ 1681 char *realobj; 1682 int size, i; 1683 int lines = 0; 1684 1685 realobj = (char *)objp + obj_offset(cachep); 1686 size = obj_size(cachep); 1687 1688 for (i = 0; i < size; i++) { 1689 char exp = POISON_FREE; 1690 if (i == size - 1) 1691 exp = POISON_END; 1692 if (realobj[i] != exp) { 1693 int limit; 1694 /* Mismatch ! */ 1695 /* Print header */ 1696 if (lines == 0) { 1697 printk(KERN_ERR 1698 "Slab corruption: start=%p, len=%d\n", 1699 realobj, size); 1700 print_objinfo(cachep, objp, 0); 1701 } 1702 /* Hexdump the affected line */ 1703 i = (i / 16) * 16; 1704 limit = 16; 1705 if (i + limit > size) 1706 limit = size - i; 1707 dump_line(realobj, i, limit); 1708 i += 16; 1709 lines++; 1710 /* Limit to 5 lines */ 1711 if (lines > 5) 1712 break; 1713 } 1714 } 1715 if (lines != 0) { 1716 /* Print some data about the neighboring objects, if they 1717 * exist: 1718 */ 1719 struct slab *slabp = virt_to_slab(objp); 1720 unsigned int objnr; 1721 1722 objnr = obj_to_index(cachep, slabp, objp); 1723 if (objnr) { 1724 objp = index_to_obj(cachep, slabp, objnr - 1); 1725 realobj = (char *)objp + obj_offset(cachep); 1726 printk(KERN_ERR "Prev obj: start=%p, len=%d\n", 1727 realobj, size); 1728 print_objinfo(cachep, objp, 2); 1729 } 1730 if (objnr + 1 < cachep->num) { 1731 objp = index_to_obj(cachep, slabp, objnr + 1); 1732 realobj = (char *)objp + obj_offset(cachep); 1733 printk(KERN_ERR "Next obj: start=%p, len=%d\n", 1734 realobj, size); 1735 print_objinfo(cachep, objp, 2); 1736 } 1737 } 1738} 1739#endif 1740 1741#if DEBUG 1742/** 1743 * slab_destroy_objs - destroy a slab and its objects 1744 * @cachep: cache pointer being destroyed 1745 * @slabp: slab pointer being destroyed 1746 * 1747 * Call the registered destructor for each object in a slab that is being 1748 * destroyed. 1749 */ 1750static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp) 1751{ 1752 int i; 1753 for (i = 0; i < cachep->num; i++) { 1754 void *objp = index_to_obj(cachep, slabp, i); 1755 1756 if (cachep->flags & SLAB_POISON) { 1757#ifdef CONFIG_DEBUG_PAGEALLOC 1758 if (cachep->buffer_size % PAGE_SIZE == 0 && 1759 OFF_SLAB(cachep)) 1760 kernel_map_pages(virt_to_page(objp), 1761 cachep->buffer_size / PAGE_SIZE, 1); 1762 else 1763 check_poison_obj(cachep, objp); 1764#else 1765 check_poison_obj(cachep, objp); 1766#endif 1767 } 1768 if (cachep->flags & SLAB_RED_ZONE) { 1769 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) 1770 slab_error(cachep, "start of a freed object " 1771 "was overwritten"); 1772 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) 1773 slab_error(cachep, "end of a freed object " 1774 "was overwritten"); 1775 } 1776 if (cachep->dtor && !(cachep->flags & SLAB_POISON)) 1777 (cachep->dtor) (objp + obj_offset(cachep), cachep, 0); 1778 } 1779} 1780#else 1781static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp) 1782{ 1783 if (cachep->dtor) { 1784 int i; 1785 for (i = 0; i < cachep->num; i++) { 1786 void *objp = index_to_obj(cachep, slabp, i); 1787 (cachep->dtor) (objp, cachep, 0); 1788 } 1789 } 1790} 1791#endif 1792 1793/** 1794 * slab_destroy - destroy and release all objects in a slab 1795 * @cachep: cache pointer being destroyed 1796 * @slabp: slab pointer being destroyed 1797 * 1798 * Destroy all the objs in a slab, and release the mem back to the system. 1799 * Before calling the slab must have been unlinked from the cache. The 1800 * cache-lock is not held/needed. 1801 */ 1802static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp) 1803{ 1804 void *addr = slabp->s_mem - slabp->colouroff; 1805 1806 slab_destroy_objs(cachep, slabp); 1807 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) { 1808 struct slab_rcu *slab_rcu; 1809 1810 slab_rcu = (struct slab_rcu *)slabp; 1811 slab_rcu->cachep = cachep; 1812 slab_rcu->addr = addr; 1813 call_rcu(&slab_rcu->head, kmem_rcu_free); 1814 } else { 1815 kmem_freepages(cachep, addr); 1816 if (OFF_SLAB(cachep)) 1817 kmem_cache_free(cachep->slabp_cache, slabp); 1818 } 1819} 1820 1821/* 1822 * For setting up all the kmem_list3s for cache whose buffer_size is same as 1823 * size of kmem_list3. 1824 */ 1825static void set_up_list3s(struct kmem_cache *cachep, int index) 1826{ 1827 int node; 1828 1829 for_each_online_node(node) { 1830 cachep->nodelists[node] = &initkmem_list3[index + node]; 1831 cachep->nodelists[node]->next_reap = jiffies + 1832 REAPTIMEOUT_LIST3 + 1833 ((unsigned long)cachep) % REAPTIMEOUT_LIST3; 1834 } 1835} 1836 1837/** 1838 * calculate_slab_order - calculate size (page order) of slabs 1839 * @cachep: pointer to the cache that is being created 1840 * @size: size of objects to be created in this cache. 1841 * @align: required alignment for the objects. 1842 * @flags: slab allocation flags 1843 * 1844 * Also calculates the number of objects per slab. 1845 * 1846 * This could be made much more intelligent. For now, try to avoid using 1847 * high order pages for slabs. When the gfp() functions are more friendly 1848 * towards high-order requests, this should be changed. 1849 */ 1850static size_t calculate_slab_order(struct kmem_cache *cachep, 1851 size_t size, size_t align, unsigned long flags) 1852{ 1853 unsigned long offslab_limit; 1854 size_t left_over = 0; 1855 int gfporder; 1856 1857 for (gfporder = 0; gfporder <= MAX_GFP_ORDER; gfporder++) { 1858 unsigned int num; 1859 size_t remainder; 1860 1861 cache_estimate(gfporder, size, align, flags, &remainder, &num); 1862 if (!num) 1863 continue; 1864 1865 if (flags & CFLGS_OFF_SLAB) { 1866 /* 1867 * Max number of objs-per-slab for caches which 1868 * use off-slab slabs. Needed to avoid a possible 1869 * looping condition in cache_grow(). 1870 */ 1871 offslab_limit = size - sizeof(struct slab); 1872 offslab_limit /= sizeof(kmem_bufctl_t); 1873 1874 if (num > offslab_limit) 1875 break; 1876 } 1877 1878 /* Found something acceptable - save it away */ 1879 cachep->num = num; 1880 cachep->gfporder = gfporder; 1881 left_over = remainder; 1882 1883 /* 1884 * A VFS-reclaimable slab tends to have most allocations 1885 * as GFP_NOFS and we really don't want to have to be allocating 1886 * higher-order pages when we are unable to shrink dcache. 1887 */ 1888 if (flags & SLAB_RECLAIM_ACCOUNT) 1889 break; 1890 1891 /* 1892 * Large number of objects is good, but very large slabs are 1893 * currently bad for the gfp()s. 1894 */ 1895 if (gfporder >= slab_break_gfp_order) 1896 break; 1897 1898 /* 1899 * Acceptable internal fragmentation? 1900 */ 1901 if (left_over * 8 <= (PAGE_SIZE << gfporder)) 1902 break; 1903 } 1904 return left_over; 1905} 1906 1907static void setup_cpu_cache(struct kmem_cache *cachep) 1908{ 1909 if (g_cpucache_up == FULL) { 1910 enable_cpucache(cachep); 1911 return; 1912 } 1913 if (g_cpucache_up == NONE) { 1914 /* 1915 * Note: the first kmem_cache_create must create the cache 1916 * that's used by kmalloc(24), otherwise the creation of 1917 * further caches will BUG(). 1918 */ 1919 cachep->array[smp_processor_id()] = &initarray_generic.cache; 1920 1921 /* 1922 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is 1923 * the first cache, then we need to set up all its list3s, 1924 * otherwise the creation of further caches will BUG(). 1925 */ 1926 set_up_list3s(cachep, SIZE_AC); 1927 if (INDEX_AC == INDEX_L3) 1928 g_cpucache_up = PARTIAL_L3; 1929 else 1930 g_cpucache_up = PARTIAL_AC; 1931 } else { 1932 cachep->array[smp_processor_id()] = 1933 kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); 1934 1935 if (g_cpucache_up == PARTIAL_AC) { 1936 set_up_list3s(cachep, SIZE_L3); 1937 g_cpucache_up = PARTIAL_L3; 1938 } else { 1939 int node; 1940 for_each_online_node(node) { 1941 cachep->nodelists[node] = 1942 kmalloc_node(sizeof(struct kmem_list3), 1943 GFP_KERNEL, node); 1944 BUG_ON(!cachep->nodelists[node]); 1945 kmem_list3_init(cachep->nodelists[node]); 1946 } 1947 } 1948 } 1949 cachep->nodelists[numa_node_id()]->next_reap = 1950 jiffies + REAPTIMEOUT_LIST3 + 1951 ((unsigned long)cachep) % REAPTIMEOUT_LIST3; 1952 1953 cpu_cache_get(cachep)->avail = 0; 1954 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; 1955 cpu_cache_get(cachep)->batchcount = 1; 1956 cpu_cache_get(cachep)->touched = 0; 1957 cachep->batchcount = 1; 1958 cachep->limit = BOOT_CPUCACHE_ENTRIES; 1959} 1960 1961/** 1962 * kmem_cache_create - Create a cache. 1963 * @name: A string which is used in /proc/slabinfo to identify this cache. 1964 * @size: The size of objects to be created in this cache. 1965 * @align: The required alignment for the objects. 1966 * @flags: SLAB flags 1967 * @ctor: A constructor for the objects. 1968 * @dtor: A destructor for the objects. 1969 * 1970 * Returns a ptr to the cache on success, NULL on failure. 1971 * Cannot be called within a int, but can be interrupted. 1972 * The @ctor is run when new pages are allocated by the cache 1973 * and the @dtor is run before the pages are handed back. 1974 * 1975 * @name must be valid until the cache is destroyed. This implies that 1976 * the module calling this has to destroy the cache before getting unloaded. 1977 * 1978 * The flags are 1979 * 1980 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 1981 * to catch references to uninitialised memory. 1982 * 1983 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check 1984 * for buffer overruns. 1985 * 1986 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 1987 * cacheline. This can be beneficial if you're counting cycles as closely 1988 * as davem. 1989 */ 1990struct kmem_cache * 1991kmem_cache_create (const char *name, size_t size, size_t align, 1992 unsigned long flags, 1993 void (*ctor)(void*, struct kmem_cache *, unsigned long), 1994 void (*dtor)(void*, struct kmem_cache *, unsigned long)) 1995{ 1996 size_t left_over, slab_size, ralign; 1997 struct kmem_cache *cachep = NULL, *pc; 1998 1999 /* 2000 * Sanity checks... these are all serious usage bugs. 2001 */ 2002 if (!name || in_interrupt() || (size < BYTES_PER_WORD) || 2003 (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE) || (dtor && !ctor)) { 2004 printk(KERN_ERR "%s: Early error in slab %s\n", __FUNCTION__, 2005 name); 2006 BUG(); 2007 } 2008 2009 /* 2010 * Prevent CPUs from coming and going. 2011 * lock_cpu_hotplug() nests outside cache_chain_mutex 2012 */ 2013 lock_cpu_hotplug(); 2014 2015 mutex_lock(&cache_chain_mutex); 2016 2017 list_for_each_entry(pc, &cache_chain, next) { 2018 mm_segment_t old_fs = get_fs(); 2019 char tmp; 2020 int res; 2021 2022 /* 2023 * This happens when the module gets unloaded and doesn't 2024 * destroy its slab cache and no-one else reuses the vmalloc 2025 * area of the module. Print a warning. 2026 */ 2027 set_fs(KERNEL_DS); 2028 res = __get_user(tmp, pc->name); 2029 set_fs(old_fs); 2030 if (res) { 2031 printk("SLAB: cache with size %d has lost its name\n", 2032 pc->buffer_size); 2033 continue; 2034 } 2035 2036 if (!strcmp(pc->name, name)) { 2037 printk("kmem_cache_create: duplicate cache %s\n", name); 2038 dump_stack(); 2039 goto oops; 2040 } 2041 } 2042 2043#if DEBUG 2044 WARN_ON(strchr(name, ' ')); /* It confuses parsers */ 2045 if ((flags & SLAB_DEBUG_INITIAL) && !ctor) { 2046 /* No constructor, but inital state check requested */ 2047 printk(KERN_ERR "%s: No con, but init state check " 2048 "requested - %s\n", __FUNCTION__, name); 2049 flags &= ~SLAB_DEBUG_INITIAL; 2050 } 2051#if FORCED_DEBUG 2052 /* 2053 * Enable redzoning and last user accounting, except for caches with 2054 * large objects, if the increased size would increase the object size 2055 * above the next power of two: caches with object sizes just above a 2056 * power of two have a significant amount of internal fragmentation. 2057 */ 2058 if (size < 4096 || fls(size - 1) == fls(size-1 + 3 * BYTES_PER_WORD)) 2059 flags |= SLAB_RED_ZONE | SLAB_STORE_USER; 2060 if (!(flags & SLAB_DESTROY_BY_RCU)) 2061 flags |= SLAB_POISON; 2062#endif 2063 if (flags & SLAB_DESTROY_BY_RCU) 2064 BUG_ON(flags & SLAB_POISON); 2065#endif 2066 if (flags & SLAB_DESTROY_BY_RCU) 2067 BUG_ON(dtor); 2068 2069 /* 2070 * Always checks flags, a caller might be expecting debug support which 2071 * isn't available. 2072 */ 2073 BUG_ON(flags & ~CREATE_MASK); 2074 2075 /* 2076 * Check that size is in terms of words. This is needed to avoid 2077 * unaligned accesses for some archs when redzoning is used, and makes 2078 * sure any on-slab bufctl's are also correctly aligned. 2079 */ 2080 if (size & (BYTES_PER_WORD - 1)) { 2081 size += (BYTES_PER_WORD - 1); 2082 size &= ~(BYTES_PER_WORD - 1); 2083 } 2084 2085 /* calculate the final buffer alignment: */ 2086 2087 /* 1) arch recommendation: can be overridden for debug */ 2088 if (flags & SLAB_HWCACHE_ALIGN) { 2089 /* 2090 * Default alignment: as specified by the arch code. Except if 2091 * an object is really small, then squeeze multiple objects into 2092 * one cacheline. 2093 */ 2094 ralign = cache_line_size(); 2095 while (size <= ralign / 2) 2096 ralign /= 2; 2097 } else { 2098 ralign = BYTES_PER_WORD; 2099 } 2100 /* 2) arch mandated alignment: disables debug if necessary */ 2101 if (ralign < ARCH_SLAB_MINALIGN) { 2102 ralign = ARCH_SLAB_MINALIGN; 2103 if (ralign > BYTES_PER_WORD) 2104 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); 2105 } 2106 /* 3) caller mandated alignment: disables debug if necessary */ 2107 if (ralign < align) { 2108 ralign = align; 2109 if (ralign > BYTES_PER_WORD) 2110 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); 2111 } 2112 /* 2113 * 4) Store it. Note that the debug code below can reduce 2114 * the alignment to BYTES_PER_WORD. 2115 */ 2116 align = ralign; 2117 2118 /* Get cache's description obj. */ 2119 cachep = kmem_cache_zalloc(&cache_cache, SLAB_KERNEL); 2120 if (!cachep) 2121 goto oops; 2122 2123#if DEBUG 2124 cachep->obj_size = size; 2125 2126 if (flags & SLAB_RED_ZONE) { 2127 /* redzoning only works with word aligned caches */ 2128 align = BYTES_PER_WORD; 2129 2130 /* add space for red zone words */ 2131 cachep->obj_offset += BYTES_PER_WORD; 2132 size += 2 * BYTES_PER_WORD; 2133 } 2134 if (flags & SLAB_STORE_USER) { 2135 /* user store requires word alignment and 2136 * one word storage behind the end of the real 2137 * object. 2138 */ 2139 align = BYTES_PER_WORD; 2140 size += BYTES_PER_WORD; 2141 } 2142#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) 2143 if (size >= malloc_sizes[INDEX_L3 + 1].cs_size 2144 && cachep->obj_size > cache_line_size() && size < PAGE_SIZE) { 2145 cachep->obj_offset += PAGE_SIZE - size; 2146 size = PAGE_SIZE; 2147 } 2148#endif 2149#endif 2150 2151 /* 2152 * Determine if the slab management is 'on' or 'off' slab. 2153 * (bootstrapping cannot cope with offslab caches so don't do 2154 * it too early on.) 2155 */ 2156 if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init) 2157 /* 2158 * Size is large, assume best to place the slab management obj 2159 * off-slab (should allow better packing of objs). 2160 */ 2161 flags |= CFLGS_OFF_SLAB; 2162 2163 size = ALIGN(size, align); 2164 2165 left_over = calculate_slab_order(cachep, size, align, flags); 2166 2167 if (!cachep->num) { 2168 printk("kmem_cache_create: couldn't create cache %s.\n", name); 2169 kmem_cache_free(&cache_cache, cachep); 2170 cachep = NULL; 2171 goto oops; 2172 } 2173 slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t) 2174 + sizeof(struct slab), align); 2175 2176 /* 2177 * If the slab has been placed off-slab, and we have enough space then 2178 * move it on-slab. This is at the expense of any extra colouring. 2179 */ 2180 if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) { 2181 flags &= ~CFLGS_OFF_SLAB; 2182 left_over -= slab_size; 2183 } 2184 2185 if (flags & CFLGS_OFF_SLAB) { 2186 /* really off slab. No need for manual alignment */ 2187 slab_size = 2188 cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab); 2189 } 2190 2191 cachep->colour_off = cache_line_size(); 2192 /* Offset must be a multiple of the alignment. */ 2193 if (cachep->colour_off < align) 2194 cachep->colour_off = align; 2195 cachep->colour = left_over / cachep->colour_off; 2196 cachep->slab_size = slab_size; 2197 cachep->flags = flags; 2198 cachep->gfpflags = 0; 2199 if (flags & SLAB_CACHE_DMA) 2200 cachep->gfpflags |= GFP_DMA; 2201 cachep->buffer_size = size; 2202 2203 if (flags & CFLGS_OFF_SLAB) 2204 cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u); 2205 cachep->ctor = ctor; 2206 cachep->dtor = dtor; 2207 cachep->name = name; 2208 2209 2210 setup_cpu_cache(cachep); 2211 2212 /* cache setup completed, link it into the list */ 2213 list_add(&cachep->next, &cache_chain); 2214oops: 2215 if (!cachep && (flags & SLAB_PANIC)) 2216 panic("kmem_cache_create(): failed to create slab `%s'\n", 2217 name); 2218 mutex_unlock(&cache_chain_mutex); 2219 unlock_cpu_hotplug(); 2220 return cachep; 2221} 2222EXPORT_SYMBOL(kmem_cache_create); 2223 2224#if DEBUG 2225static void check_irq_off(void) 2226{ 2227 BUG_ON(!irqs_disabled()); 2228} 2229 2230static void check_irq_on(void) 2231{ 2232 BUG_ON(irqs_disabled()); 2233} 2234 2235static void check_spinlock_acquired(struct kmem_cache *cachep) 2236{ 2237#ifdef CONFIG_SMP 2238 check_irq_off(); 2239 assert_spin_locked(&cachep->nodelists[numa_node_id()]->list_lock); 2240#endif 2241} 2242 2243static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) 2244{ 2245#ifdef CONFIG_SMP 2246 check_irq_off(); 2247 assert_spin_locked(&cachep->nodelists[node]->list_lock); 2248#endif 2249} 2250 2251#else 2252#define check_irq_off() do { } while(0) 2253#define check_irq_on() do { } while(0) 2254#define check_spinlock_acquired(x) do { } while(0) 2255#define check_spinlock_acquired_node(x, y) do { } while(0) 2256#endif 2257 2258static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3, 2259 struct array_cache *ac, 2260 int force, int node); 2261 2262static void do_drain(void *arg) 2263{ 2264 struct kmem_cache *cachep = arg; 2265 struct array_cache *ac; 2266 int node = numa_node_id(); 2267 2268 check_irq_off(); 2269 ac = cpu_cache_get(cachep); 2270 spin_lock(&cachep->nodelists[node]->list_lock); 2271 free_block(cachep, ac->entry, ac->avail, node); 2272 spin_unlock(&cachep->nodelists[node]->list_lock); 2273 ac->avail = 0; 2274} 2275 2276static void drain_cpu_caches(struct kmem_cache *cachep) 2277{ 2278 struct kmem_list3 *l3; 2279 int node; 2280 2281 on_each_cpu(do_drain, cachep, 1, 1); 2282 check_irq_on(); 2283 for_each_online_node(node) { 2284 l3 = cachep->nodelists[node]; 2285 if (l3 && l3->alien) 2286 drain_alien_cache(cachep, l3->alien); 2287 } 2288 2289 for_each_online_node(node) { 2290 l3 = cachep->nodelists[node]; 2291 if (l3) 2292 drain_array(cachep, l3, l3->shared, 1, node); 2293 } 2294} 2295 2296/* 2297 * Remove slabs from the list of free slabs. 2298 * Specify the number of slabs to drain in tofree. 2299 * 2300 * Returns the actual number of slabs released. 2301 */ 2302static int drain_freelist(struct kmem_cache *cache, 2303 struct kmem_list3 *l3, int tofree) 2304{ 2305 struct list_head *p; 2306 int nr_freed; 2307 struct slab *slabp; 2308 2309 nr_freed = 0; 2310 while (nr_freed < tofree && !list_empty(&l3->slabs_free)) { 2311 2312 spin_lock_irq(&l3->list_lock); 2313 p = l3->slabs_free.prev; 2314 if (p == &l3->slabs_free) { 2315 spin_unlock_irq(&l3->list_lock); 2316 goto out; 2317 } 2318 2319 slabp = list_entry(p, struct slab, list); 2320#if DEBUG 2321 BUG_ON(slabp->inuse); 2322#endif 2323 list_del(&slabp->list); 2324 /* 2325 * Safe to drop the lock. The slab is no longer linked 2326 * to the cache. 2327 */ 2328 l3->free_objects -= cache->num; 2329 spin_unlock_irq(&l3->list_lock); 2330 slab_destroy(cache, slabp); 2331 nr_freed++; 2332 } 2333out: 2334 return nr_freed; 2335} 2336 2337static int __cache_shrink(struct kmem_cache *cachep) 2338{ 2339 int ret = 0, i = 0; 2340 struct kmem_list3 *l3; 2341 2342 drain_cpu_caches(cachep); 2343 2344 check_irq_on(); 2345 for_each_online_node(i) { 2346 l3 = cachep->nodelists[i]; 2347 if (!l3) 2348 continue; 2349 2350 drain_freelist(cachep, l3, l3->free_objects); 2351 2352 ret += !list_empty(&l3->slabs_full) || 2353 !list_empty(&l3->slabs_partial); 2354 } 2355 return (ret ? 1 : 0); 2356} 2357 2358/** 2359 * kmem_cache_shrink - Shrink a cache. 2360 * @cachep: The cache to shrink. 2361 * 2362 * Releases as many slabs as possible for a cache. 2363 * To help debugging, a zero exit status indicates all slabs were released. 2364 */ 2365int kmem_cache_shrink(struct kmem_cache *cachep) 2366{ 2367 BUG_ON(!cachep || in_interrupt()); 2368 2369 return __cache_shrink(cachep); 2370} 2371EXPORT_SYMBOL(kmem_cache_shrink); 2372 2373/** 2374 * kmem_cache_destroy - delete a cache 2375 * @cachep: the cache to destroy 2376 * 2377 * Remove a struct kmem_cache object from the slab cache. 2378 * Returns 0 on success. 2379 * 2380 * It is expected this function will be called by a module when it is 2381 * unloaded. This will remove the cache completely, and avoid a duplicate 2382 * cache being allocated each time a module is loaded and unloaded, if the 2383 * module doesn't have persistent in-kernel storage across loads and unloads. 2384 * 2385 * The cache must be empty before calling this function. 2386 * 2387 * The caller must guarantee that noone will allocate memory from the cache 2388 * during the kmem_cache_destroy(). 2389 */ 2390int kmem_cache_destroy(struct kmem_cache *cachep) 2391{ 2392 int i; 2393 struct kmem_list3 *l3; 2394 2395 BUG_ON(!cachep || in_interrupt()); 2396 2397 /* Don't let CPUs to come and go */ 2398 lock_cpu_hotplug(); 2399 2400 /* Find the cache in the chain of caches. */ 2401 mutex_lock(&cache_chain_mutex); 2402 /* 2403 * the chain is never empty, cache_cache is never destroyed 2404 */ 2405 list_del(&cachep->next); 2406 mutex_unlock(&cache_chain_mutex); 2407 2408 if (__cache_shrink(cachep)) { 2409 slab_error(cachep, "Can't free all objects"); 2410 mutex_lock(&cache_chain_mutex); 2411 list_add(&cachep->next, &cache_chain); 2412 mutex_unlock(&cache_chain_mutex); 2413 unlock_cpu_hotplug(); 2414 return 1; 2415 } 2416 2417 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) 2418 synchronize_rcu(); 2419 2420 for_each_online_cpu(i) 2421 kfree(cachep->array[i]); 2422 2423 /* NUMA: free the list3 structures */ 2424 for_each_online_node(i) { 2425 l3 = cachep->nodelists[i]; 2426 if (l3) { 2427 kfree(l3->shared); 2428 free_alien_cache(l3->alien); 2429 kfree(l3); 2430 } 2431 } 2432 kmem_cache_free(&cache_cache, cachep); 2433 unlock_cpu_hotplug(); 2434 return 0; 2435} 2436EXPORT_SYMBOL(kmem_cache_destroy); 2437 2438/* Get the memory for a slab management obj. */ 2439static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp, 2440 int colour_off, gfp_t local_flags, 2441 int nodeid) 2442{ 2443 struct slab *slabp; 2444 2445 if (OFF_SLAB(cachep)) { 2446 /* Slab management obj is off-slab. */ 2447 slabp = kmem_cache_alloc_node(cachep->slabp_cache, 2448 local_flags, nodeid); 2449 if (!slabp) 2450 return NULL; 2451 } else { 2452 slabp = objp + colour_off; 2453 colour_off += cachep->slab_size; 2454 } 2455 slabp->inuse = 0; 2456 slabp->colouroff = colour_off; 2457 slabp->s_mem = objp + colour_off; 2458 slabp->nodeid = nodeid; 2459 return slabp; 2460} 2461 2462static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp) 2463{ 2464 return (kmem_bufctl_t *) (slabp + 1); 2465} 2466 2467static void cache_init_objs(struct kmem_cache *cachep, 2468 struct slab *slabp, unsigned long ctor_flags) 2469{ 2470 int i; 2471 2472 for (i = 0; i < cachep->num; i++) { 2473 void *objp = index_to_obj(cachep, slabp, i); 2474#if DEBUG 2475 /* need to poison the objs? */ 2476 if (cachep->flags & SLAB_POISON) 2477 poison_obj(cachep, objp, POISON_FREE); 2478 if (cachep->flags & SLAB_STORE_USER) 2479 *dbg_userword(cachep, objp) = NULL; 2480 2481 if (cachep->flags & SLAB_RED_ZONE) { 2482 *dbg_redzone1(cachep, objp) = RED_INACTIVE; 2483 *dbg_redzone2(cachep, objp) = RED_INACTIVE; 2484 } 2485 /* 2486 * Constructors are not allowed to allocate memory from the same 2487 * cache which they are a constructor for. Otherwise, deadlock. 2488 * They must also be threaded. 2489 */ 2490 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) 2491 cachep->ctor(objp + obj_offset(cachep), cachep, 2492 ctor_flags); 2493 2494 if (cachep->flags & SLAB_RED_ZONE) { 2495 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) 2496 slab_error(cachep, "constructor overwrote the" 2497 " end of an object"); 2498 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) 2499 slab_error(cachep, "constructor overwrote the" 2500 " start of an object"); 2501 } 2502 if ((cachep->buffer_size % PAGE_SIZE) == 0 && 2503 OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) 2504 kernel_map_pages(virt_to_page(objp), 2505 cachep->buffer_size / PAGE_SIZE, 0); 2506#else 2507 if (cachep->ctor) 2508 cachep->ctor(objp, cachep, ctor_flags); 2509#endif 2510 slab_bufctl(slabp)[i] = i + 1; 2511 } 2512 slab_bufctl(slabp)[i - 1] = BUFCTL_END; 2513 slabp->free = 0; 2514} 2515 2516static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags) 2517{ 2518 if (flags & SLAB_DMA) 2519 BUG_ON(!(cachep->gfpflags & GFP_DMA)); 2520 else 2521 BUG_ON(cachep->gfpflags & GFP_DMA); 2522} 2523 2524static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp, 2525 int nodeid) 2526{ 2527 void *objp = index_to_obj(cachep, slabp, slabp->free); 2528 kmem_bufctl_t next; 2529 2530 slabp->inuse++; 2531 next = slab_bufctl(slabp)[slabp->free]; 2532#if DEBUG 2533 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; 2534 WARN_ON(slabp->nodeid != nodeid); 2535#endif 2536 slabp->free = next; 2537 2538 return objp; 2539} 2540 2541static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp, 2542 void *objp, int nodeid) 2543{ 2544 unsigned int objnr = obj_to_index(cachep, slabp, objp); 2545 2546#if DEBUG 2547 /* Verify that the slab belongs to the intended node */ 2548 WARN_ON(slabp->nodeid != nodeid); 2549 2550 if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) { 2551 printk(KERN_ERR "slab: double free detected in cache " 2552 "'%s', objp %p\n", cachep->name, objp); 2553 BUG(); 2554 } 2555#endif 2556 slab_bufctl(slabp)[objnr] = slabp->free; 2557 slabp->free = objnr; 2558 slabp->inuse--; 2559} 2560 2561/* 2562 * Map pages beginning at addr to the given cache and slab. This is required 2563 * for the slab allocator to be able to lookup the cache and slab of a 2564 * virtual address for kfree, ksize, kmem_ptr_validate, and slab debugging. 2565 */ 2566static void slab_map_pages(struct kmem_cache *cache, struct slab *slab, 2567 void *addr) 2568{ 2569 int nr_pages; 2570 struct page *page; 2571 2572 page = virt_to_page(addr); 2573 2574 nr_pages = 1; 2575 if (likely(!PageCompound(page))) 2576 nr_pages <<= cache->gfporder; 2577 2578 do { 2579 page_set_cache(page, cache); 2580 page_set_slab(page, slab); 2581 page++; 2582 } while (--nr_pages); 2583} 2584 2585/* 2586 * Grow (by 1) the number of slabs within a cache. This is called by 2587 * kmem_cache_alloc() when there are no active objs left in a cache. 2588 */ 2589static int cache_grow(struct kmem_cache *cachep, gfp_t flags, int nodeid) 2590{ 2591 struct slab *slabp; 2592 void *objp; 2593 size_t offset; 2594 gfp_t local_flags; 2595 unsigned long ctor_flags; 2596 struct kmem_list3 *l3; 2597 2598 /* 2599 * Be lazy and only check for valid flags here, keeping it out of the 2600 * critical path in kmem_cache_alloc(). 2601 */ 2602 BUG_ON(flags & ~(SLAB_DMA | SLAB_LEVEL_MASK | SLAB_NO_GROW)); 2603 if (flags & SLAB_NO_GROW) 2604 return 0; 2605 2606 ctor_flags = SLAB_CTOR_CONSTRUCTOR; 2607 local_flags = (flags & SLAB_LEVEL_MASK); 2608 if (!(local_flags & __GFP_WAIT)) 2609 /* 2610 * Not allowed to sleep. Need to tell a constructor about 2611 * this - it might need to know... 2612 */ 2613 ctor_flags |= SLAB_CTOR_ATOMIC; 2614 2615 /* Take the l3 list lock to change the colour_next on this node */ 2616 check_irq_off(); 2617 l3 = cachep->nodelists[nodeid]; 2618 spin_lock(&l3->list_lock); 2619 2620 /* Get colour for the slab, and cal the next value. */ 2621 offset = l3->colour_next; 2622 l3->colour_next++; 2623 if (l3->colour_next >= cachep->colour) 2624 l3->colour_next = 0; 2625 spin_unlock(&l3->list_lock); 2626 2627 offset *= cachep->colour_off; 2628 2629 if (local_flags & __GFP_WAIT) 2630 local_irq_enable(); 2631 2632 /* 2633 * The test for missing atomic flag is performed here, rather than 2634 * the more obvious place, simply to reduce the critical path length 2635 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they 2636 * will eventually be caught here (where it matters). 2637 */ 2638 kmem_flagcheck(cachep, flags); 2639 2640 /* 2641 * Get mem for the objs. Attempt to allocate a physical page from 2642 * 'nodeid'. 2643 */ 2644 objp = kmem_getpages(cachep, flags, nodeid); 2645 if (!objp) 2646 goto failed; 2647 2648 /* Get slab management. */ 2649 slabp = alloc_slabmgmt(cachep, objp, offset, local_flags, nodeid); 2650 if (!slabp) 2651 goto opps1; 2652 2653 slabp->nodeid = nodeid; 2654 slab_map_pages(cachep, slabp, objp); 2655 2656 cache_init_objs(cachep, slabp, ctor_flags); 2657 2658 if (local_flags & __GFP_WAIT) 2659 local_irq_disable(); 2660 check_irq_off(); 2661 spin_lock(&l3->list_lock); 2662 2663 /* Make slab active. */ 2664 list_add_tail(&slabp->list, &(l3->slabs_free)); 2665 STATS_INC_GROWN(cachep); 2666 l3->free_objects += cachep->num; 2667 spin_unlock(&l3->list_lock); 2668 return 1; 2669opps1: 2670 kmem_freepages(cachep, objp); 2671failed: 2672 if (local_flags & __GFP_WAIT) 2673 local_irq_disable(); 2674 return 0; 2675} 2676 2677#if DEBUG 2678 2679/* 2680 * Perform extra freeing checks: 2681 * - detect bad pointers. 2682 * - POISON/RED_ZONE checking 2683 * - destructor calls, for caches with POISON+dtor 2684 */ 2685static void kfree_debugcheck(const void *objp) 2686{ 2687 struct page *page; 2688 2689 if (!virt_addr_valid(objp)) { 2690 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n", 2691 (unsigned long)objp); 2692 BUG(); 2693 } 2694 page = virt_to_page(objp); 2695 if (!PageSlab(page)) { 2696 printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n", 2697 (unsigned long)objp); 2698 BUG(); 2699 } 2700} 2701 2702static inline void verify_redzone_free(struct kmem_cache *cache, void *obj) 2703{ 2704 unsigned long redzone1, redzone2; 2705 2706 redzone1 = *dbg_redzone1(cache, obj); 2707 redzone2 = *dbg_redzone2(cache, obj); 2708 2709 /* 2710 * Redzone is ok. 2711 */ 2712 if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE) 2713 return; 2714 2715 if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE) 2716 slab_error(cache, "double free detected"); 2717 else 2718 slab_error(cache, "memory outside object was overwritten"); 2719 2720 printk(KERN_ERR "%p: redzone 1:0x%lx, redzone 2:0x%lx.\n", 2721 obj, redzone1, redzone2); 2722} 2723 2724static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, 2725 void *caller) 2726{ 2727 struct page *page; 2728 unsigned int objnr; 2729 struct slab *slabp; 2730 2731 objp -= obj_offset(cachep); 2732 kfree_debugcheck(objp); 2733 page = virt_to_page(objp); 2734 2735 slabp = page_get_slab(page); 2736 2737 if (cachep->flags & SLAB_RED_ZONE) { 2738 verify_redzone_free(cachep, objp); 2739 *dbg_redzone1(cachep, objp) = RED_INACTIVE; 2740 *dbg_redzone2(cachep, objp) = RED_INACTIVE; 2741 } 2742 if (cachep->flags & SLAB_STORE_USER) 2743 *dbg_userword(cachep, objp) = caller; 2744 2745 objnr = obj_to_index(cachep, slabp, objp); 2746 2747 BUG_ON(objnr >= cachep->num); 2748 BUG_ON(objp != index_to_obj(cachep, slabp, objnr)); 2749 2750 if (cachep->flags & SLAB_DEBUG_INITIAL) { 2751 /* 2752 * Need to call the slab's constructor so the caller can 2753 * perform a verify of its state (debugging). Called without 2754 * the cache-lock held. 2755 */ 2756 cachep->ctor(objp + obj_offset(cachep), 2757 cachep, SLAB_CTOR_CONSTRUCTOR | SLAB_CTOR_VERIFY); 2758 } 2759 if (cachep->flags & SLAB_POISON && cachep->dtor) { 2760 /* we want to cache poison the object, 2761 * call the destruction callback 2762 */ 2763 cachep->dtor(objp + obj_offset(cachep), cachep, 0); 2764 } 2765#ifdef CONFIG_DEBUG_SLAB_LEAK 2766 slab_bufctl(slabp)[objnr] = BUFCTL_FREE; 2767#endif 2768 if (cachep->flags & SLAB_POISON) { 2769#ifdef CONFIG_DEBUG_PAGEALLOC 2770 if ((cachep->buffer_size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { 2771 store_stackinfo(cachep, objp, (unsigned long)caller); 2772 kernel_map_pages(virt_to_page(objp), 2773 cachep->buffer_size / PAGE_SIZE, 0); 2774 } else { 2775 poison_obj(cachep, objp, POISON_FREE); 2776 } 2777#else 2778 poison_obj(cachep, objp, POISON_FREE); 2779#endif 2780 } 2781 return objp; 2782} 2783 2784static void check_slabp(struct kmem_cache *cachep, struct slab *slabp) 2785{ 2786 kmem_bufctl_t i; 2787 int entries = 0; 2788 2789 /* Check slab's freelist to see if this obj is there. */ 2790 for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) { 2791 entries++; 2792 if (entries > cachep->num || i >= cachep->num) 2793 goto bad; 2794 } 2795 if (entries != cachep->num - slabp->inuse) { 2796bad: 2797 printk(KERN_ERR "slab: Internal list corruption detected in " 2798 "cache '%s'(%d), slabp %p(%d). Hexdump:\n", 2799 cachep->name, cachep->num, slabp, slabp->inuse); 2800 for (i = 0; 2801 i < sizeof(*slabp) + cachep->num * sizeof(kmem_bufctl_t); 2802 i++) { 2803 if (i % 16 == 0) 2804 printk("\n%03x:", i); 2805 printk(" %02x", ((unsigned char *)slabp)[i]); 2806 } 2807 printk("\n"); 2808 BUG(); 2809 } 2810} 2811#else 2812#define kfree_debugcheck(x) do { } while(0) 2813#define cache_free_debugcheck(x,objp,z) (objp) 2814#define check_slabp(x,y) do { } while(0) 2815#endif 2816 2817static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags) 2818{ 2819 int batchcount; 2820 struct kmem_list3 *l3; 2821 struct array_cache *ac; 2822 2823 check_irq_off(); 2824 ac = cpu_cache_get(cachep); 2825retry: 2826 batchcount = ac->batchcount; 2827 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { 2828 /* 2829 * If there was little recent activity on this cache, then 2830 * perform only a partial refill. Otherwise we could generate 2831 * refill bouncing. 2832 */ 2833 batchcount = BATCHREFILL_LIMIT; 2834 } 2835 l3 = cachep->nodelists[numa_node_id()]; 2836 2837 BUG_ON(ac->avail > 0 || !l3); 2838 spin_lock(&l3->list_lock); 2839 2840 /* See if we can refill from the shared array */ 2841 if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) 2842 goto alloc_done; 2843 2844 while (batchcount > 0) { 2845 struct list_head *entry; 2846 struct slab *slabp; 2847 /* Get slab alloc is to come from. */ 2848 entry = l3->slabs_partial.next; 2849 if (entry == &l3->slabs_partial) { 2850 l3->free_touched = 1; 2851 entry = l3->slabs_free.next; 2852 if (entry == &l3->slabs_free) 2853 goto must_grow; 2854 } 2855 2856 slabp = list_entry(entry, struct slab, list); 2857 check_slabp(cachep, slabp); 2858 check_spinlock_acquired(cachep); 2859 while (slabp->inuse < cachep->num && batchcount--) { 2860 STATS_INC_ALLOCED(cachep); 2861 STATS_INC_ACTIVE(cachep); 2862 STATS_SET_HIGH(cachep); 2863 2864 ac->entry[ac->avail++] = slab_get_obj(cachep, slabp, 2865 numa_node_id()); 2866 } 2867 check_slabp(cachep, slabp); 2868 2869 /* move slabp to correct slabp list: */ 2870 list_del(&slabp->list); 2871 if (slabp->free == BUFCTL_END) 2872 list_add(&slabp->list, &l3->slabs_full); 2873 else 2874 list_add(&slabp->list, &l3->slabs_partial); 2875 } 2876 2877must_grow: 2878 l3->free_objects -= ac->avail; 2879alloc_done: 2880 spin_unlock(&l3->list_lock); 2881 2882 if (unlikely(!ac->avail)) { 2883 int x; 2884 x = cache_grow(cachep, flags, numa_node_id()); 2885 2886 /* cache_grow can reenable interrupts, then ac could change. */ 2887 ac = cpu_cache_get(cachep); 2888 if (!x && ac->avail == 0) /* no objects in sight? abort */ 2889 return NULL; 2890 2891 if (!ac->avail) /* objects refilled by interrupt? */ 2892 goto retry; 2893 } 2894 ac->touched = 1; 2895 return ac->entry[--ac->avail]; 2896} 2897 2898static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep, 2899 gfp_t flags) 2900{ 2901 might_sleep_if(flags & __GFP_WAIT); 2902#if DEBUG 2903 kmem_flagcheck(cachep, flags); 2904#endif 2905} 2906 2907#if DEBUG 2908static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, 2909 gfp_t flags, void *objp, void *caller) 2910{ 2911 if (!objp) 2912 return objp; 2913 if (cachep->flags & SLAB_POISON) { 2914#ifdef CONFIG_DEBUG_PAGEALLOC 2915 if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) 2916 kernel_map_pages(virt_to_page(objp), 2917 cachep->buffer_size / PAGE_SIZE, 1); 2918 else 2919 check_poison_obj(cachep, objp); 2920#else 2921 check_poison_obj(cachep, objp); 2922#endif 2923 poison_obj(cachep, objp, POISON_INUSE); 2924 } 2925 if (cachep->flags & SLAB_STORE_USER) 2926 *dbg_userword(cachep, objp) = caller; 2927 2928 if (cachep->flags & SLAB_RED_ZONE) { 2929 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || 2930 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { 2931 slab_error(cachep, "double free, or memory outside" 2932 " object was overwritten"); 2933 printk(KERN_ERR 2934 "%p: redzone 1:0x%lx, redzone 2:0x%lx\n", 2935 objp, *dbg_redzone1(cachep, objp), 2936 *dbg_redzone2(cachep, objp)); 2937 } 2938 *dbg_redzone1(cachep, objp) = RED_ACTIVE; 2939 *dbg_redzone2(cachep, objp) = RED_ACTIVE; 2940 } 2941#ifdef CONFIG_DEBUG_SLAB_LEAK 2942 { 2943 struct slab *slabp; 2944 unsigned objnr; 2945 2946 slabp = page_get_slab(virt_to_page(objp)); 2947 objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size; 2948 slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE; 2949 } 2950#endif 2951 objp += obj_offset(cachep); 2952 if (cachep->ctor && cachep->flags & SLAB_POISON) { 2953 unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR; 2954 2955 if (!(flags & __GFP_WAIT)) 2956 ctor_flags |= SLAB_CTOR_ATOMIC; 2957 2958 cachep->ctor(objp, cachep, ctor_flags); 2959 } 2960 return objp; 2961} 2962#else 2963#define cache_alloc_debugcheck_after(a,b,objp,d) (objp) 2964#endif 2965 2966static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) 2967{ 2968 void *objp; 2969 struct array_cache *ac; 2970 2971#ifdef CONFIG_NUMA 2972 if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY))) { 2973 objp = alternate_node_alloc(cachep, flags); 2974 if (objp != NULL) 2975 return objp; 2976 } 2977#endif 2978 2979 check_irq_off(); 2980 ac = cpu_cache_get(cachep); 2981 if (likely(ac->avail)) { 2982 STATS_INC_ALLOCHIT(cachep); 2983 ac->touched = 1; 2984 objp = ac->entry[--ac->avail]; 2985 } else { 2986 STATS_INC_ALLOCMISS(cachep); 2987 objp = cache_alloc_refill(cachep, flags); 2988 } 2989 return objp; 2990} 2991 2992static __always_inline void *__cache_alloc(struct kmem_cache *cachep, 2993 gfp_t flags, void *caller) 2994{ 2995 unsigned long save_flags; 2996 void *objp; 2997 2998 cache_alloc_debugcheck_before(cachep, flags); 2999 3000 local_irq_save(save_flags); 3001 objp = ____cache_alloc(cachep, flags); 3002 local_irq_restore(save_flags); 3003 objp = cache_alloc_debugcheck_after(cachep, flags, objp, 3004 caller); 3005 prefetchw(objp); 3006 return objp; 3007} 3008 3009#ifdef CONFIG_NUMA 3010/* 3011 * Try allocating on another node if PF_SPREAD_SLAB|PF_MEMPOLICY. 3012 * 3013 * If we are in_interrupt, then process context, including cpusets and 3014 * mempolicy, may not apply and should not be used for allocation policy. 3015 */ 3016static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) 3017{ 3018 int nid_alloc, nid_here; 3019 3020 if (in_interrupt()) 3021 return NULL; 3022 nid_alloc = nid_here = numa_node_id(); 3023 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) 3024 nid_alloc = cpuset_mem_spread_node(); 3025 else if (current->mempolicy) 3026 nid_alloc = slab_node(current->mempolicy); 3027 if (nid_alloc != nid_here) 3028 return __cache_alloc_node(cachep, flags, nid_alloc); 3029 return NULL; 3030} 3031 3032/* 3033 * A interface to enable slab creation on nodeid 3034 */ 3035static void *__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, 3036 int nodeid) 3037{ 3038 struct list_head *entry; 3039 struct slab *slabp; 3040 struct kmem_list3 *l3; 3041 void *obj; 3042 int x; 3043 3044 l3 = cachep->nodelists[nodeid]; 3045 BUG_ON(!l3); 3046 3047retry: 3048 check_irq_off(); 3049 spin_lock(&l3->list_lock); 3050 entry = l3->slabs_partial.next; 3051 if (entry == &l3->slabs_partial) { 3052 l3->free_touched = 1; 3053 entry = l3->slabs_free.next; 3054 if (entry == &l3->slabs_free) 3055 goto must_grow; 3056 } 3057 3058 slabp = list_entry(entry, struct slab, list); 3059 check_spinlock_acquired_node(cachep, nodeid); 3060 check_slabp(cachep, slabp); 3061 3062 STATS_INC_NODEALLOCS(cachep); 3063 STATS_INC_ACTIVE(cachep); 3064 STATS_SET_HIGH(cachep); 3065 3066 BUG_ON(slabp->inuse == cachep->num); 3067 3068 obj = slab_get_obj(cachep, slabp, nodeid); 3069 check_slabp(cachep, slabp); 3070 l3->free_objects--; 3071 /* move slabp to correct slabp list: */ 3072 list_del(&slabp->list); 3073 3074 if (slabp->free == BUFCTL_END) 3075 list_add(&slabp->list, &l3->slabs_full); 3076 else 3077 list_add(&slabp->list, &l3->slabs_partial); 3078 3079 spin_unlock(&l3->list_lock); 3080 goto done; 3081 3082must_grow: 3083 spin_unlock(&l3->list_lock); 3084 x = cache_grow(cachep, flags, nodeid); 3085 3086 if (!x) 3087 return NULL; 3088 3089 goto retry; 3090done: 3091 return obj; 3092} 3093#endif 3094 3095/* 3096 * Caller needs to acquire correct kmem_list's list_lock 3097 */ 3098static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, 3099 int node) 3100{ 3101 int i; 3102 struct kmem_list3 *l3; 3103 3104 for (i = 0; i < nr_objects; i++) { 3105 void *objp = objpp[i]; 3106 struct slab *slabp; 3107 3108 slabp = virt_to_slab(objp); 3109 l3 = cachep->nodelists[node]; 3110 list_del(&slabp->list); 3111 check_spinlock_acquired_node(cachep, node); 3112 check_slabp(cachep, slabp); 3113 slab_put_obj(cachep, slabp, objp, node); 3114 STATS_DEC_ACTIVE(cachep); 3115 l3->free_objects++; 3116 check_slabp(cachep, slabp); 3117 3118 /* fixup slab chains */ 3119 if (slabp->inuse == 0) { 3120 if (l3->free_objects > l3->free_limit) { 3121 l3->free_objects -= cachep->num; 3122 slab_destroy(cachep, slabp); 3123 } else { 3124 list_add(&slabp->list, &l3->slabs_free); 3125 } 3126 } else { 3127 /* Unconditionally move a slab to the end of the 3128 * partial list on free - maximum time for the 3129 * other objects to be freed, too. 3130 */ 3131 list_add_tail(&slabp->list, &l3->slabs_partial); 3132 } 3133 } 3134} 3135 3136static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) 3137{ 3138 int batchcount; 3139 struct kmem_list3 *l3; 3140 int node = numa_node_id(); 3141 3142 batchcount = ac->batchcount; 3143#if DEBUG 3144 BUG_ON(!batchcount || batchcount > ac->avail); 3145#endif 3146 check_irq_off(); 3147 l3 = cachep->nodelists[node]; 3148 spin_lock(&l3->list_lock); 3149 if (l3->shared) { 3150 struct array_cache *shared_array = l3->shared; 3151 int max = shared_array->limit - shared_array->avail; 3152 if (max) { 3153 if (batchcount > max) 3154 batchcount = max; 3155 memcpy(&(shared_array->entry[shared_array->avail]), 3156 ac->entry, sizeof(void *) * batchcount); 3157 shared_array->avail += batchcount; 3158 goto free_done; 3159 } 3160 } 3161 3162 free_block(cachep, ac->entry, batchcount, node); 3163free_done: 3164#if STATS 3165 { 3166 int i = 0; 3167 struct list_head *p; 3168 3169 p = l3->slabs_free.next; 3170 while (p != &(l3->slabs_free)) { 3171 struct slab *slabp; 3172 3173 slabp = list_entry(p, struct slab, list); 3174 BUG_ON(slabp->inuse); 3175 3176 i++; 3177 p = p->next; 3178 } 3179 STATS_SET_FREEABLE(cachep, i); 3180 } 3181#endif 3182 spin_unlock(&l3->list_lock); 3183 ac->avail -= batchcount; 3184 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail); 3185} 3186 3187/* 3188 * Release an obj back to its cache. If the obj has a constructed state, it must 3189 * be in this state _before_ it is released. Called with disabled ints. 3190 */ 3191static inline void __cache_free(struct kmem_cache *cachep, void *objp) 3192{ 3193 struct array_cache *ac = cpu_cache_get(cachep); 3194 3195 check_irq_off(); 3196 objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0)); 3197 3198 if (cache_free_alien(cachep, objp)) 3199 return; 3200 3201 if (likely(ac->avail < ac->limit)) { 3202 STATS_INC_FREEHIT(cachep); 3203 ac->entry[ac->avail++] = objp; 3204 return; 3205 } else { 3206 STATS_INC_FREEMISS(cachep); 3207 cache_flusharray(cachep, ac); 3208 ac->entry[ac->avail++] = objp; 3209 } 3210} 3211 3212/** 3213 * kmem_cache_alloc - Allocate an object 3214 * @cachep: The cache to allocate from. 3215 * @flags: See kmalloc(). 3216 * 3217 * Allocate an object from this cache. The flags are only relevant 3218 * if the cache has no available objects. 3219 */ 3220void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3221{ 3222 return __cache_alloc(cachep, flags, __builtin_return_address(0)); 3223} 3224EXPORT_SYMBOL(kmem_cache_alloc); 3225 3226/** 3227 * kmem_cache_alloc - Allocate an object. The memory is set to zero. 3228 * @cache: The cache to allocate from. 3229 * @flags: See kmalloc(). 3230 * 3231 * Allocate an object from this cache and set the allocated memory to zero. 3232 * The flags are only relevant if the cache has no available objects. 3233 */ 3234void *kmem_cache_zalloc(struct kmem_cache *cache, gfp_t flags) 3235{ 3236 void *ret = __cache_alloc(cache, flags, __builtin_return_address(0)); 3237 if (ret) 3238 memset(ret, 0, obj_size(cache)); 3239 return ret; 3240} 3241EXPORT_SYMBOL(kmem_cache_zalloc); 3242 3243/** 3244 * kmem_ptr_validate - check if an untrusted pointer might 3245 * be a slab entry. 3246 * @cachep: the cache we're checking against 3247 * @ptr: pointer to validate 3248 * 3249 * This verifies that the untrusted pointer looks sane: 3250 * it is _not_ a guarantee that the pointer is actually 3251 * part of the slab cache in question, but it at least 3252 * validates that the pointer can be dereferenced and 3253 * looks half-way sane. 3254 * 3255 * Currently only used for dentry validation. 3256 */ 3257int fastcall kmem_ptr_validate(struct kmem_cache *cachep, void *ptr) 3258{ 3259 unsigned long addr = (unsigned long)ptr; 3260 unsigned long min_addr = PAGE_OFFSET; 3261 unsigned long align_mask = BYTES_PER_WORD - 1; 3262 unsigned long size = cachep->buffer_size; 3263 struct page *page; 3264 3265 if (unlikely(addr < min_addr)) 3266 goto out; 3267 if (unlikely(addr > (unsigned long)high_memory - size)) 3268 goto out; 3269 if (unlikely(addr & align_mask)) 3270 goto out; 3271 if (unlikely(!kern_addr_valid(addr))) 3272 goto out; 3273 if (unlikely(!kern_addr_valid(addr + size - 1))) 3274 goto out; 3275 page = virt_to_page(ptr); 3276 if (unlikely(!PageSlab(page))) 3277 goto out; 3278 if (unlikely(page_get_cache(page) != cachep)) 3279 goto out; 3280 return 1; 3281out: 3282 return 0; 3283} 3284 3285#ifdef CONFIG_NUMA 3286/** 3287 * kmem_cache_alloc_node - Allocate an object on the specified node 3288 * @cachep: The cache to allocate from. 3289 * @flags: See kmalloc(). 3290 * @nodeid: node number of the target node. 3291 * 3292 * Identical to kmem_cache_alloc, except that this function is slow 3293 * and can sleep. And it will allocate memory on the given node, which 3294 * can improve the performance for cpu bound structures. 3295 * New and improved: it will now make sure that the object gets 3296 * put on the correct node list so that there is no false sharing. 3297 */ 3298void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) 3299{ 3300 unsigned long save_flags; 3301 void *ptr; 3302 3303 cache_alloc_debugcheck_before(cachep, flags); 3304 local_irq_save(save_flags); 3305 3306 if (nodeid == -1 || nodeid == numa_node_id() || 3307 !cachep->nodelists[nodeid]) 3308 ptr = ____cache_alloc(cachep, flags); 3309 else 3310 ptr = __cache_alloc_node(cachep, flags, nodeid); 3311 local_irq_restore(save_flags); 3312 3313 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, 3314 __builtin_return_address(0)); 3315 3316 return ptr; 3317} 3318EXPORT_SYMBOL(kmem_cache_alloc_node); 3319 3320void *kmalloc_node(size_t size, gfp_t flags, int node) 3321{ 3322 struct kmem_cache *cachep; 3323 3324 cachep = kmem_find_general_cachep(size, flags); 3325 if (unlikely(cachep == NULL)) 3326 return NULL; 3327 return kmem_cache_alloc_node(cachep, flags, node); 3328} 3329EXPORT_SYMBOL(kmalloc_node); 3330#endif 3331 3332/** 3333 * __do_kmalloc - allocate memory 3334 * @size: how many bytes of memory are required. 3335 * @flags: the type of memory to allocate (see kmalloc). 3336 * @caller: function caller for debug tracking of the caller 3337 */ 3338static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, 3339 void *caller) 3340{ 3341 struct kmem_cache *cachep; 3342 3343 /* If you want to save a few bytes .text space: replace 3344 * __ with kmem_. 3345 * Then kmalloc uses the uninlined functions instead of the inline 3346 * functions. 3347 */ 3348 cachep = __find_general_cachep(size, flags); 3349 if (unlikely(cachep == NULL)) 3350 return NULL; 3351 return __cache_alloc(cachep, flags, caller); 3352} 3353 3354 3355void *__kmalloc(size_t size, gfp_t flags) 3356{ 3357#ifndef CONFIG_DEBUG_SLAB 3358 return __do_kmalloc(size, flags, NULL); 3359#else 3360 return __do_kmalloc(size, flags, __builtin_return_address(0)); 3361#endif 3362} 3363EXPORT_SYMBOL(__kmalloc); 3364 3365#ifdef CONFIG_DEBUG_SLAB 3366void *__kmalloc_track_caller(size_t size, gfp_t flags, void *caller) 3367{ 3368 return __do_kmalloc(size, flags, caller); 3369} 3370EXPORT_SYMBOL(__kmalloc_track_caller); 3371#endif 3372 3373#ifdef CONFIG_SMP 3374/** 3375 * __alloc_percpu - allocate one copy of the object for every present 3376 * cpu in the system, zeroing them. 3377 * Objects should be dereferenced using the per_cpu_ptr macro only. 3378 * 3379 * @size: how many bytes of memory are required. 3380 */ 3381void *__alloc_percpu(size_t size) 3382{ 3383 int i; 3384 struct percpu_data *pdata = kmalloc(sizeof(*pdata), GFP_KERNEL); 3385 3386 if (!pdata) 3387 return NULL; 3388 3389 /* 3390 * Cannot use for_each_online_cpu since a cpu may come online 3391 * and we have no way of figuring out how to fix the array 3392 * that we have allocated then.... 3393 */ 3394 for_each_possible_cpu(i) { 3395 int node = cpu_to_node(i); 3396 3397 if (node_online(node)) 3398 pdata->ptrs[i] = kmalloc_node(size, GFP_KERNEL, node); 3399 else 3400 pdata->ptrs[i] = kmalloc(size, GFP_KERNEL); 3401 3402 if (!pdata->ptrs[i]) 3403 goto unwind_oom; 3404 memset(pdata->ptrs[i], 0, size); 3405 } 3406 3407 /* Catch derefs w/o wrappers */ 3408 return (void *)(~(unsigned long)pdata); 3409 3410unwind_oom: 3411 while (--i >= 0) { 3412 if (!cpu_possible(i)) 3413 continue; 3414 kfree(pdata->ptrs[i]); 3415 } 3416 kfree(pdata); 3417 return NULL; 3418} 3419EXPORT_SYMBOL(__alloc_percpu); 3420#endif 3421 3422/** 3423 * kmem_cache_free - Deallocate an object 3424 * @cachep: The cache the allocation was from. 3425 * @objp: The previously allocated object. 3426 * 3427 * Free an object which was previously allocated from this 3428 * cache. 3429 */ 3430void kmem_cache_free(struct kmem_cache *cachep, void *objp) 3431{ 3432 unsigned long flags; 3433 3434 BUG_ON(virt_to_cache(objp) != cachep); 3435 3436 local_irq_save(flags); 3437 __cache_free(cachep, objp); 3438 local_irq_restore(flags); 3439} 3440EXPORT_SYMBOL(kmem_cache_free); 3441 3442/** 3443 * kfree - free previously allocated memory 3444 * @objp: pointer returned by kmalloc. 3445 * 3446 * If @objp is NULL, no operation is performed. 3447 * 3448 * Don't free memory not originally allocated by kmalloc() 3449 * or you will run into trouble. 3450 */ 3451void kfree(const void *objp) 3452{ 3453 struct kmem_cache *c; 3454 unsigned long flags; 3455 3456 if (unlikely(!objp)) 3457 return; 3458 local_irq_save(flags); 3459 kfree_debugcheck(objp); 3460 c = virt_to_cache(objp); 3461 debug_check_no_locks_freed(objp, obj_size(c)); 3462 __cache_free(c, (void *)objp); 3463 local_irq_restore(flags); 3464} 3465EXPORT_SYMBOL(kfree); 3466 3467#ifdef CONFIG_SMP 3468/** 3469 * free_percpu - free previously allocated percpu memory 3470 * @objp: pointer returned by alloc_percpu. 3471 * 3472 * Don't free memory not originally allocated by alloc_percpu() 3473 * The complemented objp is to check for that. 3474 */ 3475void free_percpu(const void *objp) 3476{ 3477 int i; 3478 struct percpu_data *p = (struct percpu_data *)(~(unsigned long)objp); 3479 3480 /* 3481 * We allocate for all cpus so we cannot use for online cpu here. 3482 */ 3483 for_each_possible_cpu(i) 3484 kfree(p->ptrs[i]); 3485 kfree(p); 3486} 3487EXPORT_SYMBOL(free_percpu); 3488#endif 3489 3490unsigned int kmem_cache_size(struct kmem_cache *cachep) 3491{ 3492 return obj_size(cachep); 3493} 3494EXPORT_SYMBOL(kmem_cache_size); 3495 3496const char *kmem_cache_name(struct kmem_cache *cachep) 3497{ 3498 return cachep->name; 3499} 3500EXPORT_SYMBOL_GPL(kmem_cache_name); 3501 3502/* 3503 * This initializes kmem_list3 or resizes varioius caches for all nodes. 3504 */ 3505static int alloc_kmemlist(struct kmem_cache *cachep) 3506{ 3507 int node; 3508 struct kmem_list3 *l3; 3509 struct array_cache *new_shared; 3510 struct array_cache **new_alien; 3511 3512 for_each_online_node(node) { 3513 3514 new_alien = alloc_alien_cache(node, cachep->limit); 3515 if (!new_alien) 3516 goto fail; 3517 3518 new_shared = alloc_arraycache(node, 3519 cachep->shared*cachep->batchcount, 3520 0xbaadf00d); 3521 if (!new_shared) { 3522 free_alien_cache(new_alien); 3523 goto fail; 3524 } 3525 3526 l3 = cachep->nodelists[node]; 3527 if (l3) { 3528 struct array_cache *shared = l3->shared; 3529 3530 spin_lock_irq(&l3->list_lock); 3531 3532 if (shared) 3533 free_block(cachep, shared->entry, 3534 shared->avail, node); 3535 3536 l3->shared = new_shared; 3537 if (!l3->alien) { 3538 l3->alien = new_alien; 3539 new_alien = NULL; 3540 } 3541 l3->free_limit = (1 + nr_cpus_node(node)) * 3542 cachep->batchcount + cachep->num; 3543 spin_unlock_irq(&l3->list_lock); 3544 kfree(shared); 3545 free_alien_cache(new_alien); 3546 continue; 3547 } 3548 l3 = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, node); 3549 if (!l3) { 3550 free_alien_cache(new_alien); 3551 kfree(new_shared); 3552 goto fail; 3553 } 3554 3555 kmem_list3_init(l3); 3556 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + 3557 ((unsigned long)cachep) % REAPTIMEOUT_LIST3; 3558 l3->shared = new_shared; 3559 l3->alien = new_alien; 3560 l3->free_limit = (1 + nr_cpus_node(node)) * 3561 cachep->batchcount + cachep->num; 3562 cachep->nodelists[node] = l3; 3563 } 3564 return 0; 3565 3566fail: 3567 if (!cachep->next.next) { 3568 /* Cache is not active yet. Roll back what we did */ 3569 node--; 3570 while (node >= 0) { 3571 if (cachep->nodelists[node]) { 3572 l3 = cachep->nodelists[node]; 3573 3574 kfree(l3->shared); 3575 free_alien_cache(l3->alien); 3576 kfree(l3); 3577 cachep->nodelists[node] = NULL; 3578 } 3579 node--; 3580 } 3581 } 3582 return -ENOMEM; 3583} 3584 3585struct ccupdate_struct { 3586 struct kmem_cache *cachep; 3587 struct array_cache *new[NR_CPUS]; 3588}; 3589 3590static void do_ccupdate_local(void *info) 3591{ 3592 struct ccupdate_struct *new = info; 3593 struct array_cache *old; 3594 3595 check_irq_off(); 3596 old = cpu_cache_get(new->cachep); 3597 3598 new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()]; 3599 new->new[smp_processor_id()] = old; 3600} 3601 3602/* Always called with the cache_chain_mutex held */ 3603static int do_tune_cpucache(struct kmem_cache *cachep, int limit, 3604 int batchcount, int shared) 3605{ 3606 struct ccupdate_struct new; 3607 int i, err; 3608 3609 memset(&new.new, 0, sizeof(new.new)); 3610 for_each_online_cpu(i) { 3611 new.new[i] = alloc_arraycache(cpu_to_node(i), limit, 3612 batchcount); 3613 if (!new.new[i]) { 3614 for (i--; i >= 0; i--) 3615 kfree(new.new[i]); 3616 return -ENOMEM; 3617 } 3618 } 3619 new.cachep = cachep; 3620 3621 on_each_cpu(do_ccupdate_local, (void *)&new, 1, 1); 3622 3623 check_irq_on(); 3624 cachep->batchcount = batchcount; 3625 cachep->limit = limit; 3626 cachep->shared = shared; 3627 3628 for_each_online_cpu(i) { 3629 struct array_cache *ccold = new.new[i]; 3630 if (!ccold) 3631 continue; 3632 spin_lock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock); 3633 free_block(cachep, ccold->entry, ccold->avail, cpu_to_node(i)); 3634 spin_unlock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock); 3635 kfree(ccold); 3636 } 3637 3638 err = alloc_kmemlist(cachep); 3639 if (err) { 3640 printk(KERN_ERR "alloc_kmemlist failed for %s, error %d.\n", 3641 cachep->name, -err); 3642 BUG(); 3643 } 3644 return 0; 3645} 3646 3647/* Called with cache_chain_mutex held always */ 3648static void enable_cpucache(struct kmem_cache *cachep) 3649{ 3650 int err; 3651 int limit, shared; 3652 3653 /* 3654 * The head array serves three purposes: 3655 * - create a LIFO ordering, i.e. return objects that are cache-warm 3656 * - reduce the number of spinlock operations. 3657 * - reduce the number of linked list operations on the slab and 3658 * bufctl chains: array operations are cheaper. 3659 * The numbers are guessed, we should auto-tune as described by 3660 * Bonwick. 3661 */ 3662 if (cachep->buffer_size > 131072) 3663 limit = 1; 3664 else if (cachep->buffer_size > PAGE_SIZE) 3665 limit = 8; 3666 else if (cachep->buffer_size > 1024) 3667 limit = 24; 3668 else if (cachep->buffer_size > 256) 3669 limit = 54; 3670 else 3671 limit = 120; 3672 3673 /* 3674 * CPU bound tasks (e.g. network routing) can exhibit cpu bound 3675 * allocation behaviour: Most allocs on one cpu, most free operations 3676 * on another cpu. For these cases, an efficient object passing between 3677 * cpus is necessary. This is provided by a shared array. The array 3678 * replaces Bonwick's magazine layer. 3679 * On uniprocessor, it's functionally equivalent (but less efficient) 3680 * to a larger limit. Thus disabled by default. 3681 */ 3682 shared = 0; 3683#ifdef CONFIG_SMP 3684 if (cachep->buffer_size <= PAGE_SIZE) 3685 shared = 8; 3686#endif 3687 3688#if DEBUG 3689 /* 3690 * With debugging enabled, large batchcount lead to excessively long 3691 * periods with disabled local interrupts. Limit the batchcount 3692 */ 3693 if (limit > 32) 3694 limit = 32; 3695#endif 3696 err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared); 3697 if (err) 3698 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n", 3699 cachep->name, -err); 3700} 3701 3702/* 3703 * Drain an array if it contains any elements taking the l3 lock only if 3704 * necessary. Note that the l3 listlock also protects the array_cache 3705 * if drain_array() is used on the shared array. 3706 */ 3707void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3, 3708 struct array_cache *ac, int force, int node) 3709{ 3710 int tofree; 3711 3712 if (!ac || !ac->avail) 3713 return; 3714 if (ac->touched && !force) { 3715 ac->touched = 0; 3716 } else { 3717 spin_lock_irq(&l3->list_lock); 3718 if (ac->avail) { 3719 tofree = force ? ac->avail : (ac->limit + 4) / 5; 3720 if (tofree > ac->avail) 3721 tofree = (ac->avail + 1) / 2; 3722 free_block(cachep, ac->entry, tofree, node); 3723 ac->avail -= tofree; 3724 memmove(ac->entry, &(ac->entry[tofree]), 3725 sizeof(void *) * ac->avail); 3726 } 3727 spin_unlock_irq(&l3->list_lock); 3728 } 3729} 3730 3731/** 3732 * cache_reap - Reclaim memory from caches. 3733 * @unused: unused parameter 3734 * 3735 * Called from workqueue/eventd every few seconds. 3736 * Purpose: 3737 * - clear the per-cpu caches for this CPU. 3738 * - return freeable pages to the main free memory pool. 3739 * 3740 * If we cannot acquire the cache chain mutex then just give up - we'll try 3741 * again on the next iteration. 3742 */ 3743static void cache_reap(void *unused) 3744{ 3745 struct kmem_cache *searchp; 3746 struct kmem_list3 *l3; 3747 int node = numa_node_id(); 3748 3749 if (!mutex_trylock(&cache_chain_mutex)) { 3750 /* Give up. Setup the next iteration. */ 3751 schedule_delayed_work(&__get_cpu_var(reap_work), 3752 REAPTIMEOUT_CPUC); 3753 return; 3754 } 3755 3756 list_for_each_entry(searchp, &cache_chain, next) { 3757 check_irq_on(); 3758 3759 /* 3760 * We only take the l3 lock if absolutely necessary and we 3761 * have established with reasonable certainty that 3762 * we can do some work if the lock was obtained. 3763 */ 3764 l3 = searchp->nodelists[node]; 3765 3766 reap_alien(searchp, l3); 3767 3768 drain_array(searchp, l3, cpu_cache_get(searchp), 0, node); 3769 3770 /* 3771 * These are racy checks but it does not matter 3772 * if we skip one check or scan twice. 3773 */ 3774 if (time_after(l3->next_reap, jiffies)) 3775 goto next; 3776 3777 l3->next_reap = jiffies + REAPTIMEOUT_LIST3; 3778 3779 drain_array(searchp, l3, l3->shared, 0, node); 3780 3781 if (l3->free_touched) 3782 l3->free_touched = 0; 3783 else { 3784 int freed; 3785 3786 freed = drain_freelist(searchp, l3, (l3->free_limit + 3787 5 * searchp->num - 1) / (5 * searchp->num)); 3788 STATS_ADD_REAPED(searchp, freed); 3789 } 3790next: 3791 cond_resched(); 3792 } 3793 check_irq_on(); 3794 mutex_unlock(&cache_chain_mutex); 3795 next_reap_node(); 3796 refresh_cpu_vm_stats(smp_processor_id()); 3797 /* Set up the next iteration */ 3798 schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC); 3799} 3800 3801#ifdef CONFIG_PROC_FS 3802 3803static void print_slabinfo_header(struct seq_file *m) 3804{ 3805 /* 3806 * Output format version, so at least we can change it 3807 * without _too_ many complaints. 3808 */ 3809#if STATS 3810 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n"); 3811#else 3812 seq_puts(m, "slabinfo - version: 2.1\n"); 3813#endif 3814 seq_puts(m, "# name <active_objs> <num_objs> <objsize> " 3815 "<objperslab> <pagesperslab>"); 3816 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); 3817 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); 3818#if STATS 3819 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> " 3820 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>"); 3821 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>"); 3822#endif 3823 seq_putc(m, '\n'); 3824} 3825 3826static void *s_start(struct seq_file *m, loff_t *pos) 3827{ 3828 loff_t n = *pos; 3829 struct list_head *p; 3830 3831 mutex_lock(&cache_chain_mutex); 3832 if (!n) 3833 print_slabinfo_header(m); 3834 p = cache_chain.next; 3835 while (n--) { 3836 p = p->next; 3837 if (p == &cache_chain) 3838 return NULL; 3839 } 3840 return list_entry(p, struct kmem_cache, next); 3841} 3842 3843static void *s_next(struct seq_file *m, void *p, loff_t *pos) 3844{ 3845 struct kmem_cache *cachep = p; 3846 ++*pos; 3847 return cachep->next.next == &cache_chain ? 3848 NULL : list_entry(cachep->next.next, struct kmem_cache, next); 3849} 3850 3851static void s_stop(struct seq_file *m, void *p) 3852{ 3853 mutex_unlock(&cache_chain_mutex); 3854} 3855 3856static int s_show(struct seq_file *m, void *p) 3857{ 3858 struct kmem_cache *cachep = p; 3859 struct slab *slabp; 3860 unsigned long active_objs; 3861 unsigned long num_objs; 3862 unsigned long active_slabs = 0; 3863 unsigned long num_slabs, free_objects = 0, shared_avail = 0; 3864 const char *name; 3865 char *error = NULL; 3866 int node; 3867 struct kmem_list3 *l3; 3868 3869 active_objs = 0; 3870 num_slabs = 0; 3871 for_each_online_node(node) { 3872 l3 = cachep->nodelists[node]; 3873 if (!l3) 3874 continue; 3875 3876 check_irq_on(); 3877 spin_lock_irq(&l3->list_lock); 3878 3879 list_for_each_entry(slabp, &l3->slabs_full, list) { 3880 if (slabp->inuse != cachep->num && !error) 3881 error = "slabs_full accounting error"; 3882 active_objs += cachep->num; 3883 active_slabs++; 3884 } 3885 list_for_each_entry(slabp, &l3->slabs_partial, list) { 3886 if (slabp->inuse == cachep->num && !error) 3887 error = "slabs_partial inuse accounting error"; 3888 if (!slabp->inuse && !error) 3889 error = "slabs_partial/inuse accounting error"; 3890 active_objs += slabp->inuse; 3891 active_slabs++; 3892 } 3893 list_for_each_entry(slabp, &l3->slabs_free, list) { 3894 if (slabp->inuse && !error) 3895 error = "slabs_free/inuse accounting error"; 3896 num_slabs++; 3897 } 3898 free_objects += l3->free_objects; 3899 if (l3->shared) 3900 shared_avail += l3->shared->avail; 3901 3902 spin_unlock_irq(&l3->list_lock); 3903 } 3904 num_slabs += active_slabs; 3905 num_objs = num_slabs * cachep->num; 3906 if (num_objs - active_objs != free_objects && !error) 3907 error = "free_objects accounting error"; 3908 3909 name = cachep->name; 3910 if (error) 3911 printk(KERN_ERR "slab: cache %s error: %s\n", name, error); 3912 3913 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 3914 name, active_objs, num_objs, cachep->buffer_size, 3915 cachep->num, (1 << cachep->gfporder)); 3916 seq_printf(m, " : tunables %4u %4u %4u", 3917 cachep->limit, cachep->batchcount, cachep->shared); 3918 seq_printf(m, " : slabdata %6lu %6lu %6lu", 3919 active_slabs, num_slabs, shared_avail); 3920#if STATS 3921 { /* list3 stats */ 3922 unsigned long high = cachep->high_mark; 3923 unsigned long allocs = cachep->num_allocations; 3924 unsigned long grown = cachep->grown; 3925 unsigned long reaped = cachep->reaped; 3926 unsigned long errors = cachep->errors; 3927 unsigned long max_freeable = cachep->max_freeable; 3928 unsigned long node_allocs = cachep->node_allocs; 3929 unsigned long node_frees = cachep->node_frees; 3930 unsigned long overflows = cachep->node_overflow; 3931 3932 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu \ 3933 %4lu %4lu %4lu %4lu %4lu", allocs, high, grown, 3934 reaped, errors, max_freeable, node_allocs, 3935 node_frees, overflows); 3936 } 3937 /* cpu stats */ 3938 { 3939 unsigned long allochit = atomic_read(&cachep->allochit); 3940 unsigned long allocmiss = atomic_read(&cachep->allocmiss); 3941 unsigned long freehit = atomic_read(&cachep->freehit); 3942 unsigned long freemiss = atomic_read(&cachep->freemiss); 3943 3944 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu", 3945 allochit, allocmiss, freehit, freemiss); 3946 } 3947#endif 3948 seq_putc(m, '\n'); 3949 return 0; 3950} 3951 3952/* 3953 * slabinfo_op - iterator that generates /proc/slabinfo 3954 * 3955 * Output layout: 3956 * cache-name 3957 * num-active-objs 3958 * total-objs 3959 * object size 3960 * num-active-slabs 3961 * total-slabs 3962 * num-pages-per-slab 3963 * + further values on SMP and with statistics enabled 3964 */ 3965 3966struct seq_operations slabinfo_op = { 3967 .start = s_start, 3968 .next = s_next, 3969 .stop = s_stop, 3970 .show = s_show, 3971}; 3972 3973#define MAX_SLABINFO_WRITE 128 3974/** 3975 * slabinfo_write - Tuning for the slab allocator 3976 * @file: unused 3977 * @buffer: user buffer 3978 * @count: data length 3979 * @ppos: unused 3980 */ 3981ssize_t slabinfo_write(struct file *file, const char __user * buffer, 3982 size_t count, loff_t *ppos) 3983{ 3984 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp; 3985 int limit, batchcount, shared, res; 3986 struct kmem_cache *cachep; 3987 3988 if (count > MAX_SLABINFO_WRITE) 3989 return -EINVAL; 3990 if (copy_from_user(&kbuf, buffer, count)) 3991 return -EFAULT; 3992 kbuf[MAX_SLABINFO_WRITE] = '\0'; 3993 3994 tmp = strchr(kbuf, ' '); 3995 if (!tmp) 3996 return -EINVAL; 3997 *tmp = '\0'; 3998 tmp++; 3999 if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3) 4000 return -EINVAL; 4001 4002 /* Find the cache in the chain of caches. */ 4003 mutex_lock(&cache_chain_mutex); 4004 res = -EINVAL; 4005 list_for_each_entry(cachep, &cache_chain, next) { 4006 if (!strcmp(cachep->name, kbuf)) { 4007 if (limit < 1 || batchcount < 1 || 4008 batchcount > limit || shared < 0) { 4009 res = 0; 4010 } else { 4011 res = do_tune_cpucache(cachep, limit, 4012 batchcount, shared); 4013 } 4014 break; 4015 } 4016 } 4017 mutex_unlock(&cache_chain_mutex); 4018 if (res >= 0) 4019 res = count; 4020 return res; 4021} 4022 4023#ifdef CONFIG_DEBUG_SLAB_LEAK 4024 4025static void *leaks_start(struct seq_file *m, loff_t *pos) 4026{ 4027 loff_t n = *pos; 4028 struct list_head *p; 4029 4030 mutex_lock(&cache_chain_mutex); 4031 p = cache_chain.next; 4032 while (n--) { 4033 p = p->next; 4034 if (p == &cache_chain) 4035 return NULL; 4036 } 4037 return list_entry(p, struct kmem_cache, next); 4038} 4039 4040static inline int add_caller(unsigned long *n, unsigned long v) 4041{ 4042 unsigned long *p; 4043 int l; 4044 if (!v) 4045 return 1; 4046 l = n[1]; 4047 p = n + 2; 4048 while (l) { 4049 int i = l/2; 4050 unsigned long *q = p + 2 * i; 4051 if (*q == v) { 4052 q[1]++; 4053 return 1; 4054 } 4055 if (*q > v) { 4056 l = i; 4057 } else { 4058 p = q + 2; 4059 l -= i + 1; 4060 } 4061 } 4062 if (++n[1] == n[0]) 4063 return 0; 4064 memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n)); 4065 p[0] = v; 4066 p[1] = 1; 4067 return 1; 4068} 4069 4070static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s) 4071{ 4072 void *p; 4073 int i; 4074 if (n[0] == n[1]) 4075 return; 4076 for (i = 0, p = s->s_mem; i < c->num; i++, p += c->buffer_size) { 4077 if (slab_bufctl(s)[i] != BUFCTL_ACTIVE) 4078 continue; 4079 if (!add_caller(n, (unsigned long)*dbg_userword(c, p))) 4080 return; 4081 } 4082} 4083 4084static void show_symbol(struct seq_file *m, unsigned long address) 4085{ 4086#ifdef CONFIG_KALLSYMS 4087 char *modname; 4088 const char *name; 4089 unsigned long offset, size; 4090 char namebuf[KSYM_NAME_LEN+1]; 4091 4092 name = kallsyms_lookup(address, &size, &offset, &modname, namebuf); 4093 4094 if (name) { 4095 seq_printf(m, "%s+%#lx/%#lx", name, offset, size); 4096 if (modname) 4097 seq_printf(m, " [%s]", modname); 4098 return; 4099 } 4100#endif 4101 seq_printf(m, "%p", (void *)address); 4102} 4103 4104static int leaks_show(struct seq_file *m, void *p) 4105{ 4106 struct kmem_cache *cachep = p; 4107 struct slab *slabp; 4108 struct kmem_list3 *l3; 4109 const char *name; 4110 unsigned long *n = m->private; 4111 int node; 4112 int i; 4113 4114 if (!(cachep->flags & SLAB_STORE_USER)) 4115 return 0; 4116 if (!(cachep->flags & SLAB_RED_ZONE)) 4117 return 0; 4118 4119 /* OK, we can do it */ 4120 4121 n[1] = 0; 4122 4123 for_each_online_node(node) { 4124 l3 = cachep->nodelists[node]; 4125 if (!l3) 4126 continue; 4127 4128 check_irq_on(); 4129 spin_lock_irq(&l3->list_lock); 4130 4131 list_for_each_entry(slabp, &l3->slabs_full, list) 4132 handle_slab(n, cachep, slabp); 4133 list_for_each_entry(slabp, &l3->slabs_partial, list) 4134 handle_slab(n, cachep, slabp); 4135 spin_unlock_irq(&l3->list_lock); 4136 } 4137 name = cachep->name; 4138 if (n[0] == n[1]) { 4139 /* Increase the buffer size */ 4140 mutex_unlock(&cache_chain_mutex); 4141 m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL); 4142 if (!m->private) { 4143 /* Too bad, we are really out */ 4144 m->private = n; 4145 mutex_lock(&cache_chain_mutex); 4146 return -ENOMEM; 4147 } 4148 *(unsigned long *)m->private = n[0] * 2; 4149 kfree(n); 4150 mutex_lock(&cache_chain_mutex); 4151 /* Now make sure this entry will be retried */ 4152 m->count = m->size; 4153 return 0; 4154 } 4155 for (i = 0; i < n[1]; i++) { 4156 seq_printf(m, "%s: %lu ", name, n[2*i+3]); 4157 show_symbol(m, n[2*i+2]); 4158 seq_putc(m, '\n'); 4159 } 4160 return 0; 4161} 4162 4163struct seq_operations slabstats_op = { 4164 .start = leaks_start, 4165 .next = s_next, 4166 .stop = s_stop, 4167 .show = leaks_show, 4168}; 4169#endif 4170#endif 4171 4172/** 4173 * ksize - get the actual amount of memory allocated for a given object 4174 * @objp: Pointer to the object 4175 * 4176 * kmalloc may internally round up allocations and return more memory 4177 * than requested. ksize() can be used to determine the actual amount of 4178 * memory allocated. The caller may use this additional memory, even though 4179 * a smaller amount of memory was initially specified with the kmalloc call. 4180 * The caller must guarantee that objp points to a valid object previously 4181 * allocated with either kmalloc() or kmem_cache_alloc(). The object 4182 * must not be freed during the duration of the call. 4183 */ 4184unsigned int ksize(const void *objp) 4185{ 4186 if (unlikely(objp == NULL)) 4187 return 0; 4188 4189 return obj_size(virt_to_cache(objp)); 4190} 4191