vmalloc.c revision 734269521e320ad14ed39ae9b64d482b9028dcd2
1/* 2 * linux/mm/vmalloc.c 3 * 4 * Copyright (C) 1993 Linus Torvalds 5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 8 * Numa awareness, Christoph Lameter, SGI, June 2005 9 */ 10 11#include <linux/vmalloc.h> 12#include <linux/mm.h> 13#include <linux/module.h> 14#include <linux/highmem.h> 15#include <linux/slab.h> 16#include <linux/spinlock.h> 17#include <linux/interrupt.h> 18#include <linux/proc_fs.h> 19#include <linux/seq_file.h> 20#include <linux/debugobjects.h> 21#include <linux/kallsyms.h> 22#include <linux/list.h> 23#include <linux/rbtree.h> 24#include <linux/radix-tree.h> 25#include <linux/rcupdate.h> 26#include <linux/bootmem.h> 27 28#include <asm/atomic.h> 29#include <asm/uaccess.h> 30#include <asm/tlbflush.h> 31 32 33/*** Page table manipulation functions ***/ 34 35static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end) 36{ 37 pte_t *pte; 38 39 pte = pte_offset_kernel(pmd, addr); 40 do { 41 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); 42 WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 43 } while (pte++, addr += PAGE_SIZE, addr != end); 44} 45 46static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end) 47{ 48 pmd_t *pmd; 49 unsigned long next; 50 51 pmd = pmd_offset(pud, addr); 52 do { 53 next = pmd_addr_end(addr, end); 54 if (pmd_none_or_clear_bad(pmd)) 55 continue; 56 vunmap_pte_range(pmd, addr, next); 57 } while (pmd++, addr = next, addr != end); 58} 59 60static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end) 61{ 62 pud_t *pud; 63 unsigned long next; 64 65 pud = pud_offset(pgd, addr); 66 do { 67 next = pud_addr_end(addr, end); 68 if (pud_none_or_clear_bad(pud)) 69 continue; 70 vunmap_pmd_range(pud, addr, next); 71 } while (pud++, addr = next, addr != end); 72} 73 74static void vunmap_page_range(unsigned long addr, unsigned long end) 75{ 76 pgd_t *pgd; 77 unsigned long next; 78 79 BUG_ON(addr >= end); 80 pgd = pgd_offset_k(addr); 81 do { 82 next = pgd_addr_end(addr, end); 83 if (pgd_none_or_clear_bad(pgd)) 84 continue; 85 vunmap_pud_range(pgd, addr, next); 86 } while (pgd++, addr = next, addr != end); 87} 88 89static int vmap_pte_range(pmd_t *pmd, unsigned long addr, 90 unsigned long end, pgprot_t prot, struct page **pages, int *nr) 91{ 92 pte_t *pte; 93 94 /* 95 * nr is a running index into the array which helps higher level 96 * callers keep track of where we're up to. 97 */ 98 99 pte = pte_alloc_kernel(pmd, addr); 100 if (!pte) 101 return -ENOMEM; 102 do { 103 struct page *page = pages[*nr]; 104 105 if (WARN_ON(!pte_none(*pte))) 106 return -EBUSY; 107 if (WARN_ON(!page)) 108 return -ENOMEM; 109 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 110 (*nr)++; 111 } while (pte++, addr += PAGE_SIZE, addr != end); 112 return 0; 113} 114 115static int vmap_pmd_range(pud_t *pud, unsigned long addr, 116 unsigned long end, pgprot_t prot, struct page **pages, int *nr) 117{ 118 pmd_t *pmd; 119 unsigned long next; 120 121 pmd = pmd_alloc(&init_mm, pud, addr); 122 if (!pmd) 123 return -ENOMEM; 124 do { 125 next = pmd_addr_end(addr, end); 126 if (vmap_pte_range(pmd, addr, next, prot, pages, nr)) 127 return -ENOMEM; 128 } while (pmd++, addr = next, addr != end); 129 return 0; 130} 131 132static int vmap_pud_range(pgd_t *pgd, unsigned long addr, 133 unsigned long end, pgprot_t prot, struct page **pages, int *nr) 134{ 135 pud_t *pud; 136 unsigned long next; 137 138 pud = pud_alloc(&init_mm, pgd, addr); 139 if (!pud) 140 return -ENOMEM; 141 do { 142 next = pud_addr_end(addr, end); 143 if (vmap_pmd_range(pud, addr, next, prot, pages, nr)) 144 return -ENOMEM; 145 } while (pud++, addr = next, addr != end); 146 return 0; 147} 148 149/* 150 * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and 151 * will have pfns corresponding to the "pages" array. 152 * 153 * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N] 154 */ 155static int vmap_page_range(unsigned long start, unsigned long end, 156 pgprot_t prot, struct page **pages) 157{ 158 pgd_t *pgd; 159 unsigned long next; 160 unsigned long addr = start; 161 int err = 0; 162 int nr = 0; 163 164 BUG_ON(addr >= end); 165 pgd = pgd_offset_k(addr); 166 do { 167 next = pgd_addr_end(addr, end); 168 err = vmap_pud_range(pgd, addr, next, prot, pages, &nr); 169 if (err) 170 break; 171 } while (pgd++, addr = next, addr != end); 172 flush_cache_vmap(start, end); 173 174 if (unlikely(err)) 175 return err; 176 return nr; 177} 178 179static inline int is_vmalloc_or_module_addr(const void *x) 180{ 181 /* 182 * ARM, x86-64 and sparc64 put modules in a special place, 183 * and fall back on vmalloc() if that fails. Others 184 * just put it in the vmalloc space. 185 */ 186#if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 187 unsigned long addr = (unsigned long)x; 188 if (addr >= MODULES_VADDR && addr < MODULES_END) 189 return 1; 190#endif 191 return is_vmalloc_addr(x); 192} 193 194/* 195 * Walk a vmap address to the struct page it maps. 196 */ 197struct page *vmalloc_to_page(const void *vmalloc_addr) 198{ 199 unsigned long addr = (unsigned long) vmalloc_addr; 200 struct page *page = NULL; 201 pgd_t *pgd = pgd_offset_k(addr); 202 203 /* 204 * XXX we might need to change this if we add VIRTUAL_BUG_ON for 205 * architectures that do not vmalloc module space 206 */ 207 VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 208 209 if (!pgd_none(*pgd)) { 210 pud_t *pud = pud_offset(pgd, addr); 211 if (!pud_none(*pud)) { 212 pmd_t *pmd = pmd_offset(pud, addr); 213 if (!pmd_none(*pmd)) { 214 pte_t *ptep, pte; 215 216 ptep = pte_offset_map(pmd, addr); 217 pte = *ptep; 218 if (pte_present(pte)) 219 page = pte_page(pte); 220 pte_unmap(ptep); 221 } 222 } 223 } 224 return page; 225} 226EXPORT_SYMBOL(vmalloc_to_page); 227 228/* 229 * Map a vmalloc()-space virtual address to the physical page frame number. 230 */ 231unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 232{ 233 return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 234} 235EXPORT_SYMBOL(vmalloc_to_pfn); 236 237 238/*** Global kva allocator ***/ 239 240#define VM_LAZY_FREE 0x01 241#define VM_LAZY_FREEING 0x02 242#define VM_VM_AREA 0x04 243 244struct vmap_area { 245 unsigned long va_start; 246 unsigned long va_end; 247 unsigned long flags; 248 struct rb_node rb_node; /* address sorted rbtree */ 249 struct list_head list; /* address sorted list */ 250 struct list_head purge_list; /* "lazy purge" list */ 251 void *private; 252 struct rcu_head rcu_head; 253}; 254 255static DEFINE_SPINLOCK(vmap_area_lock); 256static struct rb_root vmap_area_root = RB_ROOT; 257static LIST_HEAD(vmap_area_list); 258 259static struct vmap_area *__find_vmap_area(unsigned long addr) 260{ 261 struct rb_node *n = vmap_area_root.rb_node; 262 263 while (n) { 264 struct vmap_area *va; 265 266 va = rb_entry(n, struct vmap_area, rb_node); 267 if (addr < va->va_start) 268 n = n->rb_left; 269 else if (addr > va->va_start) 270 n = n->rb_right; 271 else 272 return va; 273 } 274 275 return NULL; 276} 277 278static void __insert_vmap_area(struct vmap_area *va) 279{ 280 struct rb_node **p = &vmap_area_root.rb_node; 281 struct rb_node *parent = NULL; 282 struct rb_node *tmp; 283 284 while (*p) { 285 struct vmap_area *tmp; 286 287 parent = *p; 288 tmp = rb_entry(parent, struct vmap_area, rb_node); 289 if (va->va_start < tmp->va_end) 290 p = &(*p)->rb_left; 291 else if (va->va_end > tmp->va_start) 292 p = &(*p)->rb_right; 293 else 294 BUG(); 295 } 296 297 rb_link_node(&va->rb_node, parent, p); 298 rb_insert_color(&va->rb_node, &vmap_area_root); 299 300 /* address-sort this list so it is usable like the vmlist */ 301 tmp = rb_prev(&va->rb_node); 302 if (tmp) { 303 struct vmap_area *prev; 304 prev = rb_entry(tmp, struct vmap_area, rb_node); 305 list_add_rcu(&va->list, &prev->list); 306 } else 307 list_add_rcu(&va->list, &vmap_area_list); 308} 309 310static void purge_vmap_area_lazy(void); 311 312/* 313 * Allocate a region of KVA of the specified size and alignment, within the 314 * vstart and vend. 315 */ 316static struct vmap_area *alloc_vmap_area(unsigned long size, 317 unsigned long align, 318 unsigned long vstart, unsigned long vend, 319 int node, gfp_t gfp_mask) 320{ 321 struct vmap_area *va; 322 struct rb_node *n; 323 unsigned long addr; 324 int purged = 0; 325 326 BUG_ON(size & ~PAGE_MASK); 327 328 va = kmalloc_node(sizeof(struct vmap_area), 329 gfp_mask & GFP_RECLAIM_MASK, node); 330 if (unlikely(!va)) 331 return ERR_PTR(-ENOMEM); 332 333retry: 334 addr = ALIGN(vstart, align); 335 336 spin_lock(&vmap_area_lock); 337 /* XXX: could have a last_hole cache */ 338 n = vmap_area_root.rb_node; 339 if (n) { 340 struct vmap_area *first = NULL; 341 342 do { 343 struct vmap_area *tmp; 344 tmp = rb_entry(n, struct vmap_area, rb_node); 345 if (tmp->va_end >= addr) { 346 if (!first && tmp->va_start < addr + size) 347 first = tmp; 348 n = n->rb_left; 349 } else { 350 first = tmp; 351 n = n->rb_right; 352 } 353 } while (n); 354 355 if (!first) 356 goto found; 357 358 if (first->va_end < addr) { 359 n = rb_next(&first->rb_node); 360 if (n) 361 first = rb_entry(n, struct vmap_area, rb_node); 362 else 363 goto found; 364 } 365 366 while (addr + size > first->va_start && addr + size <= vend) { 367 addr = ALIGN(first->va_end + PAGE_SIZE, align); 368 369 n = rb_next(&first->rb_node); 370 if (n) 371 first = rb_entry(n, struct vmap_area, rb_node); 372 else 373 goto found; 374 } 375 } 376found: 377 if (addr + size > vend) { 378 spin_unlock(&vmap_area_lock); 379 if (!purged) { 380 purge_vmap_area_lazy(); 381 purged = 1; 382 goto retry; 383 } 384 if (printk_ratelimit()) 385 printk(KERN_WARNING 386 "vmap allocation for size %lu failed: " 387 "use vmalloc=<size> to increase size.\n", size); 388 return ERR_PTR(-EBUSY); 389 } 390 391 BUG_ON(addr & (align-1)); 392 393 va->va_start = addr; 394 va->va_end = addr + size; 395 va->flags = 0; 396 __insert_vmap_area(va); 397 spin_unlock(&vmap_area_lock); 398 399 return va; 400} 401 402static void rcu_free_va(struct rcu_head *head) 403{ 404 struct vmap_area *va = container_of(head, struct vmap_area, rcu_head); 405 406 kfree(va); 407} 408 409static void __free_vmap_area(struct vmap_area *va) 410{ 411 BUG_ON(RB_EMPTY_NODE(&va->rb_node)); 412 rb_erase(&va->rb_node, &vmap_area_root); 413 RB_CLEAR_NODE(&va->rb_node); 414 list_del_rcu(&va->list); 415 416 call_rcu(&va->rcu_head, rcu_free_va); 417} 418 419/* 420 * Free a region of KVA allocated by alloc_vmap_area 421 */ 422static void free_vmap_area(struct vmap_area *va) 423{ 424 spin_lock(&vmap_area_lock); 425 __free_vmap_area(va); 426 spin_unlock(&vmap_area_lock); 427} 428 429/* 430 * Clear the pagetable entries of a given vmap_area 431 */ 432static void unmap_vmap_area(struct vmap_area *va) 433{ 434 vunmap_page_range(va->va_start, va->va_end); 435} 436 437static void vmap_debug_free_range(unsigned long start, unsigned long end) 438{ 439 /* 440 * Unmap page tables and force a TLB flush immediately if 441 * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free 442 * bugs similarly to those in linear kernel virtual address 443 * space after a page has been freed. 444 * 445 * All the lazy freeing logic is still retained, in order to 446 * minimise intrusiveness of this debugging feature. 447 * 448 * This is going to be *slow* (linear kernel virtual address 449 * debugging doesn't do a broadcast TLB flush so it is a lot 450 * faster). 451 */ 452#ifdef CONFIG_DEBUG_PAGEALLOC 453 vunmap_page_range(start, end); 454 flush_tlb_kernel_range(start, end); 455#endif 456} 457 458/* 459 * lazy_max_pages is the maximum amount of virtual address space we gather up 460 * before attempting to purge with a TLB flush. 461 * 462 * There is a tradeoff here: a larger number will cover more kernel page tables 463 * and take slightly longer to purge, but it will linearly reduce the number of 464 * global TLB flushes that must be performed. It would seem natural to scale 465 * this number up linearly with the number of CPUs (because vmapping activity 466 * could also scale linearly with the number of CPUs), however it is likely 467 * that in practice, workloads might be constrained in other ways that mean 468 * vmap activity will not scale linearly with CPUs. Also, I want to be 469 * conservative and not introduce a big latency on huge systems, so go with 470 * a less aggressive log scale. It will still be an improvement over the old 471 * code, and it will be simple to change the scale factor if we find that it 472 * becomes a problem on bigger systems. 473 */ 474static unsigned long lazy_max_pages(void) 475{ 476 unsigned int log; 477 478 log = fls(num_online_cpus()); 479 480 return log * (32UL * 1024 * 1024 / PAGE_SIZE); 481} 482 483static atomic_t vmap_lazy_nr = ATOMIC_INIT(0); 484 485/* 486 * Purges all lazily-freed vmap areas. 487 * 488 * If sync is 0 then don't purge if there is already a purge in progress. 489 * If force_flush is 1, then flush kernel TLBs between *start and *end even 490 * if we found no lazy vmap areas to unmap (callers can use this to optimise 491 * their own TLB flushing). 492 * Returns with *start = min(*start, lowest purged address) 493 * *end = max(*end, highest purged address) 494 */ 495static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end, 496 int sync, int force_flush) 497{ 498 static DEFINE_SPINLOCK(purge_lock); 499 LIST_HEAD(valist); 500 struct vmap_area *va; 501 int nr = 0; 502 503 /* 504 * If sync is 0 but force_flush is 1, we'll go sync anyway but callers 505 * should not expect such behaviour. This just simplifies locking for 506 * the case that isn't actually used at the moment anyway. 507 */ 508 if (!sync && !force_flush) { 509 if (!spin_trylock(&purge_lock)) 510 return; 511 } else 512 spin_lock(&purge_lock); 513 514 rcu_read_lock(); 515 list_for_each_entry_rcu(va, &vmap_area_list, list) { 516 if (va->flags & VM_LAZY_FREE) { 517 if (va->va_start < *start) 518 *start = va->va_start; 519 if (va->va_end > *end) 520 *end = va->va_end; 521 nr += (va->va_end - va->va_start) >> PAGE_SHIFT; 522 unmap_vmap_area(va); 523 list_add_tail(&va->purge_list, &valist); 524 va->flags |= VM_LAZY_FREEING; 525 va->flags &= ~VM_LAZY_FREE; 526 } 527 } 528 rcu_read_unlock(); 529 530 if (nr) { 531 BUG_ON(nr > atomic_read(&vmap_lazy_nr)); 532 atomic_sub(nr, &vmap_lazy_nr); 533 } 534 535 if (nr || force_flush) 536 flush_tlb_kernel_range(*start, *end); 537 538 if (nr) { 539 spin_lock(&vmap_area_lock); 540 list_for_each_entry(va, &valist, purge_list) 541 __free_vmap_area(va); 542 spin_unlock(&vmap_area_lock); 543 } 544 spin_unlock(&purge_lock); 545} 546 547/* 548 * Kick off a purge of the outstanding lazy areas. Don't bother if somebody 549 * is already purging. 550 */ 551static void try_purge_vmap_area_lazy(void) 552{ 553 unsigned long start = ULONG_MAX, end = 0; 554 555 __purge_vmap_area_lazy(&start, &end, 0, 0); 556} 557 558/* 559 * Kick off a purge of the outstanding lazy areas. 560 */ 561static void purge_vmap_area_lazy(void) 562{ 563 unsigned long start = ULONG_MAX, end = 0; 564 565 __purge_vmap_area_lazy(&start, &end, 1, 0); 566} 567 568/* 569 * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been 570 * called for the correct range previously. 571 */ 572static void free_unmap_vmap_area_noflush(struct vmap_area *va) 573{ 574 va->flags |= VM_LAZY_FREE; 575 atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr); 576 if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages())) 577 try_purge_vmap_area_lazy(); 578} 579 580/* 581 * Free and unmap a vmap area 582 */ 583static void free_unmap_vmap_area(struct vmap_area *va) 584{ 585 flush_cache_vunmap(va->va_start, va->va_end); 586 free_unmap_vmap_area_noflush(va); 587} 588 589static struct vmap_area *find_vmap_area(unsigned long addr) 590{ 591 struct vmap_area *va; 592 593 spin_lock(&vmap_area_lock); 594 va = __find_vmap_area(addr); 595 spin_unlock(&vmap_area_lock); 596 597 return va; 598} 599 600static void free_unmap_vmap_area_addr(unsigned long addr) 601{ 602 struct vmap_area *va; 603 604 va = find_vmap_area(addr); 605 BUG_ON(!va); 606 free_unmap_vmap_area(va); 607} 608 609 610/*** Per cpu kva allocator ***/ 611 612/* 613 * vmap space is limited especially on 32 bit architectures. Ensure there is 614 * room for at least 16 percpu vmap blocks per CPU. 615 */ 616/* 617 * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 618 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 619 * instead (we just need a rough idea) 620 */ 621#if BITS_PER_LONG == 32 622#define VMALLOC_SPACE (128UL*1024*1024) 623#else 624#define VMALLOC_SPACE (128UL*1024*1024*1024) 625#endif 626 627#define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 628#define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 629#define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 630#define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 631#define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 632#define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 633#define VMAP_BBMAP_BITS VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 634 VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 635 VMALLOC_PAGES / NR_CPUS / 16)) 636 637#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 638 639static bool vmap_initialized __read_mostly = false; 640 641struct vmap_block_queue { 642 spinlock_t lock; 643 struct list_head free; 644 struct list_head dirty; 645 unsigned int nr_dirty; 646}; 647 648struct vmap_block { 649 spinlock_t lock; 650 struct vmap_area *va; 651 struct vmap_block_queue *vbq; 652 unsigned long free, dirty; 653 DECLARE_BITMAP(alloc_map, VMAP_BBMAP_BITS); 654 DECLARE_BITMAP(dirty_map, VMAP_BBMAP_BITS); 655 union { 656 struct { 657 struct list_head free_list; 658 struct list_head dirty_list; 659 }; 660 struct rcu_head rcu_head; 661 }; 662}; 663 664/* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 665static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 666 667/* 668 * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block 669 * in the free path. Could get rid of this if we change the API to return a 670 * "cookie" from alloc, to be passed to free. But no big deal yet. 671 */ 672static DEFINE_SPINLOCK(vmap_block_tree_lock); 673static RADIX_TREE(vmap_block_tree, GFP_ATOMIC); 674 675/* 676 * We should probably have a fallback mechanism to allocate virtual memory 677 * out of partially filled vmap blocks. However vmap block sizing should be 678 * fairly reasonable according to the vmalloc size, so it shouldn't be a 679 * big problem. 680 */ 681 682static unsigned long addr_to_vb_idx(unsigned long addr) 683{ 684 addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 685 addr /= VMAP_BLOCK_SIZE; 686 return addr; 687} 688 689static struct vmap_block *new_vmap_block(gfp_t gfp_mask) 690{ 691 struct vmap_block_queue *vbq; 692 struct vmap_block *vb; 693 struct vmap_area *va; 694 unsigned long vb_idx; 695 int node, err; 696 697 node = numa_node_id(); 698 699 vb = kmalloc_node(sizeof(struct vmap_block), 700 gfp_mask & GFP_RECLAIM_MASK, node); 701 if (unlikely(!vb)) 702 return ERR_PTR(-ENOMEM); 703 704 va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 705 VMALLOC_START, VMALLOC_END, 706 node, gfp_mask); 707 if (unlikely(IS_ERR(va))) { 708 kfree(vb); 709 return ERR_PTR(PTR_ERR(va)); 710 } 711 712 err = radix_tree_preload(gfp_mask); 713 if (unlikely(err)) { 714 kfree(vb); 715 free_vmap_area(va); 716 return ERR_PTR(err); 717 } 718 719 spin_lock_init(&vb->lock); 720 vb->va = va; 721 vb->free = VMAP_BBMAP_BITS; 722 vb->dirty = 0; 723 bitmap_zero(vb->alloc_map, VMAP_BBMAP_BITS); 724 bitmap_zero(vb->dirty_map, VMAP_BBMAP_BITS); 725 INIT_LIST_HEAD(&vb->free_list); 726 INIT_LIST_HEAD(&vb->dirty_list); 727 728 vb_idx = addr_to_vb_idx(va->va_start); 729 spin_lock(&vmap_block_tree_lock); 730 err = radix_tree_insert(&vmap_block_tree, vb_idx, vb); 731 spin_unlock(&vmap_block_tree_lock); 732 BUG_ON(err); 733 radix_tree_preload_end(); 734 735 vbq = &get_cpu_var(vmap_block_queue); 736 vb->vbq = vbq; 737 spin_lock(&vbq->lock); 738 list_add(&vb->free_list, &vbq->free); 739 spin_unlock(&vbq->lock); 740 put_cpu_var(vmap_cpu_blocks); 741 742 return vb; 743} 744 745static void rcu_free_vb(struct rcu_head *head) 746{ 747 struct vmap_block *vb = container_of(head, struct vmap_block, rcu_head); 748 749 kfree(vb); 750} 751 752static void free_vmap_block(struct vmap_block *vb) 753{ 754 struct vmap_block *tmp; 755 unsigned long vb_idx; 756 757 spin_lock(&vb->vbq->lock); 758 if (!list_empty(&vb->free_list)) 759 list_del(&vb->free_list); 760 if (!list_empty(&vb->dirty_list)) 761 list_del(&vb->dirty_list); 762 spin_unlock(&vb->vbq->lock); 763 764 vb_idx = addr_to_vb_idx(vb->va->va_start); 765 spin_lock(&vmap_block_tree_lock); 766 tmp = radix_tree_delete(&vmap_block_tree, vb_idx); 767 spin_unlock(&vmap_block_tree_lock); 768 BUG_ON(tmp != vb); 769 770 free_unmap_vmap_area_noflush(vb->va); 771 call_rcu(&vb->rcu_head, rcu_free_vb); 772} 773 774static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 775{ 776 struct vmap_block_queue *vbq; 777 struct vmap_block *vb; 778 unsigned long addr = 0; 779 unsigned int order; 780 781 BUG_ON(size & ~PAGE_MASK); 782 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 783 order = get_order(size); 784 785again: 786 rcu_read_lock(); 787 vbq = &get_cpu_var(vmap_block_queue); 788 list_for_each_entry_rcu(vb, &vbq->free, free_list) { 789 int i; 790 791 spin_lock(&vb->lock); 792 i = bitmap_find_free_region(vb->alloc_map, 793 VMAP_BBMAP_BITS, order); 794 795 if (i >= 0) { 796 addr = vb->va->va_start + (i << PAGE_SHIFT); 797 BUG_ON(addr_to_vb_idx(addr) != 798 addr_to_vb_idx(vb->va->va_start)); 799 vb->free -= 1UL << order; 800 if (vb->free == 0) { 801 spin_lock(&vbq->lock); 802 list_del_init(&vb->free_list); 803 spin_unlock(&vbq->lock); 804 } 805 spin_unlock(&vb->lock); 806 break; 807 } 808 spin_unlock(&vb->lock); 809 } 810 put_cpu_var(vmap_cpu_blocks); 811 rcu_read_unlock(); 812 813 if (!addr) { 814 vb = new_vmap_block(gfp_mask); 815 if (IS_ERR(vb)) 816 return vb; 817 goto again; 818 } 819 820 return (void *)addr; 821} 822 823static void vb_free(const void *addr, unsigned long size) 824{ 825 unsigned long offset; 826 unsigned long vb_idx; 827 unsigned int order; 828 struct vmap_block *vb; 829 830 BUG_ON(size & ~PAGE_MASK); 831 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 832 833 flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size); 834 835 order = get_order(size); 836 837 offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1); 838 839 vb_idx = addr_to_vb_idx((unsigned long)addr); 840 rcu_read_lock(); 841 vb = radix_tree_lookup(&vmap_block_tree, vb_idx); 842 rcu_read_unlock(); 843 BUG_ON(!vb); 844 845 spin_lock(&vb->lock); 846 bitmap_allocate_region(vb->dirty_map, offset >> PAGE_SHIFT, order); 847 if (!vb->dirty) { 848 spin_lock(&vb->vbq->lock); 849 list_add(&vb->dirty_list, &vb->vbq->dirty); 850 spin_unlock(&vb->vbq->lock); 851 } 852 vb->dirty += 1UL << order; 853 if (vb->dirty == VMAP_BBMAP_BITS) { 854 BUG_ON(vb->free || !list_empty(&vb->free_list)); 855 spin_unlock(&vb->lock); 856 free_vmap_block(vb); 857 } else 858 spin_unlock(&vb->lock); 859} 860 861/** 862 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 863 * 864 * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 865 * to amortize TLB flushing overheads. What this means is that any page you 866 * have now, may, in a former life, have been mapped into kernel virtual 867 * address by the vmap layer and so there might be some CPUs with TLB entries 868 * still referencing that page (additional to the regular 1:1 kernel mapping). 869 * 870 * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 871 * be sure that none of the pages we have control over will have any aliases 872 * from the vmap layer. 873 */ 874void vm_unmap_aliases(void) 875{ 876 unsigned long start = ULONG_MAX, end = 0; 877 int cpu; 878 int flush = 0; 879 880 if (unlikely(!vmap_initialized)) 881 return; 882 883 for_each_possible_cpu(cpu) { 884 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 885 struct vmap_block *vb; 886 887 rcu_read_lock(); 888 list_for_each_entry_rcu(vb, &vbq->free, free_list) { 889 int i; 890 891 spin_lock(&vb->lock); 892 i = find_first_bit(vb->dirty_map, VMAP_BBMAP_BITS); 893 while (i < VMAP_BBMAP_BITS) { 894 unsigned long s, e; 895 int j; 896 j = find_next_zero_bit(vb->dirty_map, 897 VMAP_BBMAP_BITS, i); 898 899 s = vb->va->va_start + (i << PAGE_SHIFT); 900 e = vb->va->va_start + (j << PAGE_SHIFT); 901 vunmap_page_range(s, e); 902 flush = 1; 903 904 if (s < start) 905 start = s; 906 if (e > end) 907 end = e; 908 909 i = j; 910 i = find_next_bit(vb->dirty_map, 911 VMAP_BBMAP_BITS, i); 912 } 913 spin_unlock(&vb->lock); 914 } 915 rcu_read_unlock(); 916 } 917 918 __purge_vmap_area_lazy(&start, &end, 1, flush); 919} 920EXPORT_SYMBOL_GPL(vm_unmap_aliases); 921 922/** 923 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 924 * @mem: the pointer returned by vm_map_ram 925 * @count: the count passed to that vm_map_ram call (cannot unmap partial) 926 */ 927void vm_unmap_ram(const void *mem, unsigned int count) 928{ 929 unsigned long size = count << PAGE_SHIFT; 930 unsigned long addr = (unsigned long)mem; 931 932 BUG_ON(!addr); 933 BUG_ON(addr < VMALLOC_START); 934 BUG_ON(addr > VMALLOC_END); 935 BUG_ON(addr & (PAGE_SIZE-1)); 936 937 debug_check_no_locks_freed(mem, size); 938 vmap_debug_free_range(addr, addr+size); 939 940 if (likely(count <= VMAP_MAX_ALLOC)) 941 vb_free(mem, size); 942 else 943 free_unmap_vmap_area_addr(addr); 944} 945EXPORT_SYMBOL(vm_unmap_ram); 946 947/** 948 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 949 * @pages: an array of pointers to the pages to be mapped 950 * @count: number of pages 951 * @node: prefer to allocate data structures on this node 952 * @prot: memory protection to use. PAGE_KERNEL for regular RAM 953 * 954 * Returns: a pointer to the address that has been mapped, or %NULL on failure 955 */ 956void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot) 957{ 958 unsigned long size = count << PAGE_SHIFT; 959 unsigned long addr; 960 void *mem; 961 962 if (likely(count <= VMAP_MAX_ALLOC)) { 963 mem = vb_alloc(size, GFP_KERNEL); 964 if (IS_ERR(mem)) 965 return NULL; 966 addr = (unsigned long)mem; 967 } else { 968 struct vmap_area *va; 969 va = alloc_vmap_area(size, PAGE_SIZE, 970 VMALLOC_START, VMALLOC_END, node, GFP_KERNEL); 971 if (IS_ERR(va)) 972 return NULL; 973 974 addr = va->va_start; 975 mem = (void *)addr; 976 } 977 if (vmap_page_range(addr, addr + size, prot, pages) < 0) { 978 vm_unmap_ram(mem, count); 979 return NULL; 980 } 981 return mem; 982} 983EXPORT_SYMBOL(vm_map_ram); 984 985void __init vmalloc_init(void) 986{ 987 struct vmap_area *va; 988 struct vm_struct *tmp; 989 int i; 990 991 for_each_possible_cpu(i) { 992 struct vmap_block_queue *vbq; 993 994 vbq = &per_cpu(vmap_block_queue, i); 995 spin_lock_init(&vbq->lock); 996 INIT_LIST_HEAD(&vbq->free); 997 INIT_LIST_HEAD(&vbq->dirty); 998 vbq->nr_dirty = 0; 999 } 1000 1001 /* Import existing vmlist entries. */ 1002 for (tmp = vmlist; tmp; tmp = tmp->next) { 1003 va = alloc_bootmem(sizeof(struct vmap_area)); 1004 va->flags = tmp->flags | VM_VM_AREA; 1005 va->va_start = (unsigned long)tmp->addr; 1006 va->va_end = va->va_start + tmp->size; 1007 __insert_vmap_area(va); 1008 } 1009 vmap_initialized = true; 1010} 1011 1012void unmap_kernel_range(unsigned long addr, unsigned long size) 1013{ 1014 unsigned long end = addr + size; 1015 1016 flush_cache_vunmap(addr, end); 1017 vunmap_page_range(addr, end); 1018 flush_tlb_kernel_range(addr, end); 1019} 1020 1021int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages) 1022{ 1023 unsigned long addr = (unsigned long)area->addr; 1024 unsigned long end = addr + area->size - PAGE_SIZE; 1025 int err; 1026 1027 err = vmap_page_range(addr, end, prot, *pages); 1028 if (err > 0) { 1029 *pages += err; 1030 err = 0; 1031 } 1032 1033 return err; 1034} 1035EXPORT_SYMBOL_GPL(map_vm_area); 1036 1037/*** Old vmalloc interfaces ***/ 1038DEFINE_RWLOCK(vmlist_lock); 1039struct vm_struct *vmlist; 1040 1041static struct vm_struct *__get_vm_area_node(unsigned long size, 1042 unsigned long flags, unsigned long start, unsigned long end, 1043 int node, gfp_t gfp_mask, void *caller) 1044{ 1045 static struct vmap_area *va; 1046 struct vm_struct *area; 1047 struct vm_struct *tmp, **p; 1048 unsigned long align = 1; 1049 1050 BUG_ON(in_interrupt()); 1051 if (flags & VM_IOREMAP) { 1052 int bit = fls(size); 1053 1054 if (bit > IOREMAP_MAX_ORDER) 1055 bit = IOREMAP_MAX_ORDER; 1056 else if (bit < PAGE_SHIFT) 1057 bit = PAGE_SHIFT; 1058 1059 align = 1ul << bit; 1060 } 1061 1062 size = PAGE_ALIGN(size); 1063 if (unlikely(!size)) 1064 return NULL; 1065 1066 area = kmalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 1067 if (unlikely(!area)) 1068 return NULL; 1069 1070 /* 1071 * We always allocate a guard page. 1072 */ 1073 size += PAGE_SIZE; 1074 1075 va = alloc_vmap_area(size, align, start, end, node, gfp_mask); 1076 if (IS_ERR(va)) { 1077 kfree(area); 1078 return NULL; 1079 } 1080 1081 area->flags = flags; 1082 area->addr = (void *)va->va_start; 1083 area->size = size; 1084 area->pages = NULL; 1085 area->nr_pages = 0; 1086 area->phys_addr = 0; 1087 area->caller = caller; 1088 va->private = area; 1089 va->flags |= VM_VM_AREA; 1090 1091 write_lock(&vmlist_lock); 1092 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 1093 if (tmp->addr >= area->addr) 1094 break; 1095 } 1096 area->next = *p; 1097 *p = area; 1098 write_unlock(&vmlist_lock); 1099 1100 return area; 1101} 1102 1103struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, 1104 unsigned long start, unsigned long end) 1105{ 1106 return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL, 1107 __builtin_return_address(0)); 1108} 1109EXPORT_SYMBOL_GPL(__get_vm_area); 1110 1111/** 1112 * get_vm_area - reserve a contiguous kernel virtual area 1113 * @size: size of the area 1114 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 1115 * 1116 * Search an area of @size in the kernel virtual mapping area, 1117 * and reserved it for out purposes. Returns the area descriptor 1118 * on success or %NULL on failure. 1119 */ 1120struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 1121{ 1122 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, 1123 -1, GFP_KERNEL, __builtin_return_address(0)); 1124} 1125 1126struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 1127 void *caller) 1128{ 1129 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, 1130 -1, GFP_KERNEL, caller); 1131} 1132 1133struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, 1134 int node, gfp_t gfp_mask) 1135{ 1136 return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node, 1137 gfp_mask, __builtin_return_address(0)); 1138} 1139 1140static struct vm_struct *find_vm_area(const void *addr) 1141{ 1142 struct vmap_area *va; 1143 1144 va = find_vmap_area((unsigned long)addr); 1145 if (va && va->flags & VM_VM_AREA) 1146 return va->private; 1147 1148 return NULL; 1149} 1150 1151/** 1152 * remove_vm_area - find and remove a continuous kernel virtual area 1153 * @addr: base address 1154 * 1155 * Search for the kernel VM area starting at @addr, and remove it. 1156 * This function returns the found VM area, but using it is NOT safe 1157 * on SMP machines, except for its size or flags. 1158 */ 1159struct vm_struct *remove_vm_area(const void *addr) 1160{ 1161 struct vmap_area *va; 1162 1163 va = find_vmap_area((unsigned long)addr); 1164 if (va && va->flags & VM_VM_AREA) { 1165 struct vm_struct *vm = va->private; 1166 struct vm_struct *tmp, **p; 1167 1168 vmap_debug_free_range(va->va_start, va->va_end); 1169 free_unmap_vmap_area(va); 1170 vm->size -= PAGE_SIZE; 1171 1172 write_lock(&vmlist_lock); 1173 for (p = &vmlist; (tmp = *p) != vm; p = &tmp->next) 1174 ; 1175 *p = tmp->next; 1176 write_unlock(&vmlist_lock); 1177 1178 return vm; 1179 } 1180 return NULL; 1181} 1182 1183static void __vunmap(const void *addr, int deallocate_pages) 1184{ 1185 struct vm_struct *area; 1186 1187 if (!addr) 1188 return; 1189 1190 if ((PAGE_SIZE-1) & (unsigned long)addr) { 1191 WARN(1, KERN_ERR "Trying to vfree() bad address (%p)\n", addr); 1192 return; 1193 } 1194 1195 area = remove_vm_area(addr); 1196 if (unlikely(!area)) { 1197 WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 1198 addr); 1199 return; 1200 } 1201 1202 debug_check_no_locks_freed(addr, area->size); 1203 debug_check_no_obj_freed(addr, area->size); 1204 1205 if (deallocate_pages) { 1206 int i; 1207 1208 for (i = 0; i < area->nr_pages; i++) { 1209 struct page *page = area->pages[i]; 1210 1211 BUG_ON(!page); 1212 __free_page(page); 1213 } 1214 1215 if (area->flags & VM_VPAGES) 1216 vfree(area->pages); 1217 else 1218 kfree(area->pages); 1219 } 1220 1221 kfree(area); 1222 return; 1223} 1224 1225/** 1226 * vfree - release memory allocated by vmalloc() 1227 * @addr: memory base address 1228 * 1229 * Free the virtually continuous memory area starting at @addr, as 1230 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is 1231 * NULL, no operation is performed. 1232 * 1233 * Must not be called in interrupt context. 1234 */ 1235void vfree(const void *addr) 1236{ 1237 BUG_ON(in_interrupt()); 1238 __vunmap(addr, 1); 1239} 1240EXPORT_SYMBOL(vfree); 1241 1242/** 1243 * vunmap - release virtual mapping obtained by vmap() 1244 * @addr: memory base address 1245 * 1246 * Free the virtually contiguous memory area starting at @addr, 1247 * which was created from the page array passed to vmap(). 1248 * 1249 * Must not be called in interrupt context. 1250 */ 1251void vunmap(const void *addr) 1252{ 1253 BUG_ON(in_interrupt()); 1254 __vunmap(addr, 0); 1255} 1256EXPORT_SYMBOL(vunmap); 1257 1258/** 1259 * vmap - map an array of pages into virtually contiguous space 1260 * @pages: array of page pointers 1261 * @count: number of pages to map 1262 * @flags: vm_area->flags 1263 * @prot: page protection for the mapping 1264 * 1265 * Maps @count pages from @pages into contiguous kernel virtual 1266 * space. 1267 */ 1268void *vmap(struct page **pages, unsigned int count, 1269 unsigned long flags, pgprot_t prot) 1270{ 1271 struct vm_struct *area; 1272 1273 if (count > num_physpages) 1274 return NULL; 1275 1276 area = get_vm_area_caller((count << PAGE_SHIFT), flags, 1277 __builtin_return_address(0)); 1278 if (!area) 1279 return NULL; 1280 1281 if (map_vm_area(area, prot, &pages)) { 1282 vunmap(area->addr); 1283 return NULL; 1284 } 1285 1286 return area->addr; 1287} 1288EXPORT_SYMBOL(vmap); 1289 1290static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, 1291 int node, void *caller); 1292static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 1293 pgprot_t prot, int node, void *caller) 1294{ 1295 struct page **pages; 1296 unsigned int nr_pages, array_size, i; 1297 1298 nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT; 1299 array_size = (nr_pages * sizeof(struct page *)); 1300 1301 area->nr_pages = nr_pages; 1302 /* Please note that the recursion is strictly bounded. */ 1303 if (array_size > PAGE_SIZE) { 1304 pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO, 1305 PAGE_KERNEL, node, caller); 1306 area->flags |= VM_VPAGES; 1307 } else { 1308 pages = kmalloc_node(array_size, 1309 (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO, 1310 node); 1311 } 1312 area->pages = pages; 1313 area->caller = caller; 1314 if (!area->pages) { 1315 remove_vm_area(area->addr); 1316 kfree(area); 1317 return NULL; 1318 } 1319 1320 for (i = 0; i < area->nr_pages; i++) { 1321 struct page *page; 1322 1323 if (node < 0) 1324 page = alloc_page(gfp_mask); 1325 else 1326 page = alloc_pages_node(node, gfp_mask, 0); 1327 1328 if (unlikely(!page)) { 1329 /* Successfully allocated i pages, free them in __vunmap() */ 1330 area->nr_pages = i; 1331 goto fail; 1332 } 1333 area->pages[i] = page; 1334 } 1335 1336 if (map_vm_area(area, prot, &pages)) 1337 goto fail; 1338 return area->addr; 1339 1340fail: 1341 vfree(area->addr); 1342 return NULL; 1343} 1344 1345void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot) 1346{ 1347 return __vmalloc_area_node(area, gfp_mask, prot, -1, 1348 __builtin_return_address(0)); 1349} 1350 1351/** 1352 * __vmalloc_node - allocate virtually contiguous memory 1353 * @size: allocation size 1354 * @gfp_mask: flags for the page level allocator 1355 * @prot: protection mask for the allocated pages 1356 * @node: node to use for allocation or -1 1357 * @caller: caller's return address 1358 * 1359 * Allocate enough pages to cover @size from the page level 1360 * allocator with @gfp_mask flags. Map them into contiguous 1361 * kernel virtual space, using a pagetable protection of @prot. 1362 */ 1363static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, 1364 int node, void *caller) 1365{ 1366 struct vm_struct *area; 1367 1368 size = PAGE_ALIGN(size); 1369 if (!size || (size >> PAGE_SHIFT) > num_physpages) 1370 return NULL; 1371 1372 area = __get_vm_area_node(size, VM_ALLOC, VMALLOC_START, VMALLOC_END, 1373 node, gfp_mask, caller); 1374 1375 if (!area) 1376 return NULL; 1377 1378 return __vmalloc_area_node(area, gfp_mask, prot, node, caller); 1379} 1380 1381void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) 1382{ 1383 return __vmalloc_node(size, gfp_mask, prot, -1, 1384 __builtin_return_address(0)); 1385} 1386EXPORT_SYMBOL(__vmalloc); 1387 1388/** 1389 * vmalloc - allocate virtually contiguous memory 1390 * @size: allocation size 1391 * Allocate enough pages to cover @size from the page level 1392 * allocator and map them into contiguous kernel virtual space. 1393 * 1394 * For tight control over page level allocator and protection flags 1395 * use __vmalloc() instead. 1396 */ 1397void *vmalloc(unsigned long size) 1398{ 1399 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, 1400 -1, __builtin_return_address(0)); 1401} 1402EXPORT_SYMBOL(vmalloc); 1403 1404/** 1405 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 1406 * @size: allocation size 1407 * 1408 * The resulting memory area is zeroed so it can be mapped to userspace 1409 * without leaking data. 1410 */ 1411void *vmalloc_user(unsigned long size) 1412{ 1413 struct vm_struct *area; 1414 void *ret; 1415 1416 ret = __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, 1417 PAGE_KERNEL, -1, __builtin_return_address(0)); 1418 if (ret) { 1419 area = find_vm_area(ret); 1420 area->flags |= VM_USERMAP; 1421 } 1422 return ret; 1423} 1424EXPORT_SYMBOL(vmalloc_user); 1425 1426/** 1427 * vmalloc_node - allocate memory on a specific node 1428 * @size: allocation size 1429 * @node: numa node 1430 * 1431 * Allocate enough pages to cover @size from the page level 1432 * allocator and map them into contiguous kernel virtual space. 1433 * 1434 * For tight control over page level allocator and protection flags 1435 * use __vmalloc() instead. 1436 */ 1437void *vmalloc_node(unsigned long size, int node) 1438{ 1439 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, 1440 node, __builtin_return_address(0)); 1441} 1442EXPORT_SYMBOL(vmalloc_node); 1443 1444#ifndef PAGE_KERNEL_EXEC 1445# define PAGE_KERNEL_EXEC PAGE_KERNEL 1446#endif 1447 1448/** 1449 * vmalloc_exec - allocate virtually contiguous, executable memory 1450 * @size: allocation size 1451 * 1452 * Kernel-internal function to allocate enough pages to cover @size 1453 * the page level allocator and map them into contiguous and 1454 * executable kernel virtual space. 1455 * 1456 * For tight control over page level allocator and protection flags 1457 * use __vmalloc() instead. 1458 */ 1459 1460void *vmalloc_exec(unsigned long size) 1461{ 1462 return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC, 1463 -1, __builtin_return_address(0)); 1464} 1465 1466#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 1467#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL 1468#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 1469#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL 1470#else 1471#define GFP_VMALLOC32 GFP_KERNEL 1472#endif 1473 1474/** 1475 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 1476 * @size: allocation size 1477 * 1478 * Allocate enough 32bit PA addressable pages to cover @size from the 1479 * page level allocator and map them into contiguous kernel virtual space. 1480 */ 1481void *vmalloc_32(unsigned long size) 1482{ 1483 return __vmalloc_node(size, GFP_VMALLOC32, PAGE_KERNEL, 1484 -1, __builtin_return_address(0)); 1485} 1486EXPORT_SYMBOL(vmalloc_32); 1487 1488/** 1489 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 1490 * @size: allocation size 1491 * 1492 * The resulting memory area is 32bit addressable and zeroed so it can be 1493 * mapped to userspace without leaking data. 1494 */ 1495void *vmalloc_32_user(unsigned long size) 1496{ 1497 struct vm_struct *area; 1498 void *ret; 1499 1500 ret = __vmalloc_node(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 1501 -1, __builtin_return_address(0)); 1502 if (ret) { 1503 area = find_vm_area(ret); 1504 area->flags |= VM_USERMAP; 1505 } 1506 return ret; 1507} 1508EXPORT_SYMBOL(vmalloc_32_user); 1509 1510long vread(char *buf, char *addr, unsigned long count) 1511{ 1512 struct vm_struct *tmp; 1513 char *vaddr, *buf_start = buf; 1514 unsigned long n; 1515 1516 /* Don't allow overflow */ 1517 if ((unsigned long) addr + count < count) 1518 count = -(unsigned long) addr; 1519 1520 read_lock(&vmlist_lock); 1521 for (tmp = vmlist; tmp; tmp = tmp->next) { 1522 vaddr = (char *) tmp->addr; 1523 if (addr >= vaddr + tmp->size - PAGE_SIZE) 1524 continue; 1525 while (addr < vaddr) { 1526 if (count == 0) 1527 goto finished; 1528 *buf = '\0'; 1529 buf++; 1530 addr++; 1531 count--; 1532 } 1533 n = vaddr + tmp->size - PAGE_SIZE - addr; 1534 do { 1535 if (count == 0) 1536 goto finished; 1537 *buf = *addr; 1538 buf++; 1539 addr++; 1540 count--; 1541 } while (--n > 0); 1542 } 1543finished: 1544 read_unlock(&vmlist_lock); 1545 return buf - buf_start; 1546} 1547 1548long vwrite(char *buf, char *addr, unsigned long count) 1549{ 1550 struct vm_struct *tmp; 1551 char *vaddr, *buf_start = buf; 1552 unsigned long n; 1553 1554 /* Don't allow overflow */ 1555 if ((unsigned long) addr + count < count) 1556 count = -(unsigned long) addr; 1557 1558 read_lock(&vmlist_lock); 1559 for (tmp = vmlist; tmp; tmp = tmp->next) { 1560 vaddr = (char *) tmp->addr; 1561 if (addr >= vaddr + tmp->size - PAGE_SIZE) 1562 continue; 1563 while (addr < vaddr) { 1564 if (count == 0) 1565 goto finished; 1566 buf++; 1567 addr++; 1568 count--; 1569 } 1570 n = vaddr + tmp->size - PAGE_SIZE - addr; 1571 do { 1572 if (count == 0) 1573 goto finished; 1574 *addr = *buf; 1575 buf++; 1576 addr++; 1577 count--; 1578 } while (--n > 0); 1579 } 1580finished: 1581 read_unlock(&vmlist_lock); 1582 return buf - buf_start; 1583} 1584 1585/** 1586 * remap_vmalloc_range - map vmalloc pages to userspace 1587 * @vma: vma to cover (map full range of vma) 1588 * @addr: vmalloc memory 1589 * @pgoff: number of pages into addr before first page to map 1590 * 1591 * Returns: 0 for success, -Exxx on failure 1592 * 1593 * This function checks that addr is a valid vmalloc'ed area, and 1594 * that it is big enough to cover the vma. Will return failure if 1595 * that criteria isn't met. 1596 * 1597 * Similar to remap_pfn_range() (see mm/memory.c) 1598 */ 1599int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 1600 unsigned long pgoff) 1601{ 1602 struct vm_struct *area; 1603 unsigned long uaddr = vma->vm_start; 1604 unsigned long usize = vma->vm_end - vma->vm_start; 1605 1606 if ((PAGE_SIZE-1) & (unsigned long)addr) 1607 return -EINVAL; 1608 1609 area = find_vm_area(addr); 1610 if (!area) 1611 return -EINVAL; 1612 1613 if (!(area->flags & VM_USERMAP)) 1614 return -EINVAL; 1615 1616 if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE) 1617 return -EINVAL; 1618 1619 addr += pgoff << PAGE_SHIFT; 1620 do { 1621 struct page *page = vmalloc_to_page(addr); 1622 int ret; 1623 1624 ret = vm_insert_page(vma, uaddr, page); 1625 if (ret) 1626 return ret; 1627 1628 uaddr += PAGE_SIZE; 1629 addr += PAGE_SIZE; 1630 usize -= PAGE_SIZE; 1631 } while (usize > 0); 1632 1633 /* Prevent "things" like memory migration? VM_flags need a cleanup... */ 1634 vma->vm_flags |= VM_RESERVED; 1635 1636 return 0; 1637} 1638EXPORT_SYMBOL(remap_vmalloc_range); 1639 1640/* 1641 * Implement a stub for vmalloc_sync_all() if the architecture chose not to 1642 * have one. 1643 */ 1644void __attribute__((weak)) vmalloc_sync_all(void) 1645{ 1646} 1647 1648 1649static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data) 1650{ 1651 /* apply_to_page_range() does all the hard work. */ 1652 return 0; 1653} 1654 1655/** 1656 * alloc_vm_area - allocate a range of kernel address space 1657 * @size: size of the area 1658 * 1659 * Returns: NULL on failure, vm_struct on success 1660 * 1661 * This function reserves a range of kernel address space, and 1662 * allocates pagetables to map that range. No actual mappings 1663 * are created. If the kernel address space is not shared 1664 * between processes, it syncs the pagetable across all 1665 * processes. 1666 */ 1667struct vm_struct *alloc_vm_area(size_t size) 1668{ 1669 struct vm_struct *area; 1670 1671 area = get_vm_area_caller(size, VM_IOREMAP, 1672 __builtin_return_address(0)); 1673 if (area == NULL) 1674 return NULL; 1675 1676 /* 1677 * This ensures that page tables are constructed for this region 1678 * of kernel virtual address space and mapped into init_mm. 1679 */ 1680 if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 1681 area->size, f, NULL)) { 1682 free_vm_area(area); 1683 return NULL; 1684 } 1685 1686 /* Make sure the pagetables are constructed in process kernel 1687 mappings */ 1688 vmalloc_sync_all(); 1689 1690 return area; 1691} 1692EXPORT_SYMBOL_GPL(alloc_vm_area); 1693 1694void free_vm_area(struct vm_struct *area) 1695{ 1696 struct vm_struct *ret; 1697 ret = remove_vm_area(area->addr); 1698 BUG_ON(ret != area); 1699 kfree(area); 1700} 1701EXPORT_SYMBOL_GPL(free_vm_area); 1702 1703 1704#ifdef CONFIG_PROC_FS 1705static void *s_start(struct seq_file *m, loff_t *pos) 1706{ 1707 loff_t n = *pos; 1708 struct vm_struct *v; 1709 1710 read_lock(&vmlist_lock); 1711 v = vmlist; 1712 while (n > 0 && v) { 1713 n--; 1714 v = v->next; 1715 } 1716 if (!n) 1717 return v; 1718 1719 return NULL; 1720 1721} 1722 1723static void *s_next(struct seq_file *m, void *p, loff_t *pos) 1724{ 1725 struct vm_struct *v = p; 1726 1727 ++*pos; 1728 return v->next; 1729} 1730 1731static void s_stop(struct seq_file *m, void *p) 1732{ 1733 read_unlock(&vmlist_lock); 1734} 1735 1736static void show_numa_info(struct seq_file *m, struct vm_struct *v) 1737{ 1738 if (NUMA_BUILD) { 1739 unsigned int nr, *counters = m->private; 1740 1741 if (!counters) 1742 return; 1743 1744 memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 1745 1746 for (nr = 0; nr < v->nr_pages; nr++) 1747 counters[page_to_nid(v->pages[nr])]++; 1748 1749 for_each_node_state(nr, N_HIGH_MEMORY) 1750 if (counters[nr]) 1751 seq_printf(m, " N%u=%u", nr, counters[nr]); 1752 } 1753} 1754 1755static int s_show(struct seq_file *m, void *p) 1756{ 1757 struct vm_struct *v = p; 1758 1759 seq_printf(m, "0x%p-0x%p %7ld", 1760 v->addr, v->addr + v->size, v->size); 1761 1762 if (v->caller) { 1763 char buff[KSYM_SYMBOL_LEN]; 1764 1765 seq_putc(m, ' '); 1766 sprint_symbol(buff, (unsigned long)v->caller); 1767 seq_puts(m, buff); 1768 } 1769 1770 if (v->nr_pages) 1771 seq_printf(m, " pages=%d", v->nr_pages); 1772 1773 if (v->phys_addr) 1774 seq_printf(m, " phys=%lx", v->phys_addr); 1775 1776 if (v->flags & VM_IOREMAP) 1777 seq_printf(m, " ioremap"); 1778 1779 if (v->flags & VM_ALLOC) 1780 seq_printf(m, " vmalloc"); 1781 1782 if (v->flags & VM_MAP) 1783 seq_printf(m, " vmap"); 1784 1785 if (v->flags & VM_USERMAP) 1786 seq_printf(m, " user"); 1787 1788 if (v->flags & VM_VPAGES) 1789 seq_printf(m, " vpages"); 1790 1791 show_numa_info(m, v); 1792 seq_putc(m, '\n'); 1793 return 0; 1794} 1795 1796static const struct seq_operations vmalloc_op = { 1797 .start = s_start, 1798 .next = s_next, 1799 .stop = s_stop, 1800 .show = s_show, 1801}; 1802 1803static int vmalloc_open(struct inode *inode, struct file *file) 1804{ 1805 unsigned int *ptr = NULL; 1806 int ret; 1807 1808 if (NUMA_BUILD) 1809 ptr = kmalloc(nr_node_ids * sizeof(unsigned int), GFP_KERNEL); 1810 ret = seq_open(file, &vmalloc_op); 1811 if (!ret) { 1812 struct seq_file *m = file->private_data; 1813 m->private = ptr; 1814 } else 1815 kfree(ptr); 1816 return ret; 1817} 1818 1819static const struct file_operations proc_vmalloc_operations = { 1820 .open = vmalloc_open, 1821 .read = seq_read, 1822 .llseek = seq_lseek, 1823 .release = seq_release_private, 1824}; 1825 1826static int __init proc_vmalloc_init(void) 1827{ 1828 proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations); 1829 return 0; 1830} 1831module_init(proc_vmalloc_init); 1832#endif 1833 1834