hugetlb.c revision f96efd585b8d847181f81bf16721f96ded18d9fe
1/* 2 * Generic hugetlb support. 3 * (C) William Irwin, April 2004 4 */ 5#include <linux/gfp.h> 6#include <linux/list.h> 7#include <linux/init.h> 8#include <linux/module.h> 9#include <linux/mm.h> 10#include <linux/sysctl.h> 11#include <linux/highmem.h> 12#include <linux/nodemask.h> 13#include <linux/pagemap.h> 14#include <linux/mempolicy.h> 15#include <linux/cpuset.h> 16#include <linux/mutex.h> 17 18#include <asm/page.h> 19#include <asm/pgtable.h> 20 21#include <linux/hugetlb.h> 22#include "internal.h" 23 24const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL; 25static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages; 26unsigned long max_huge_pages; 27static struct list_head hugepage_freelists[MAX_NUMNODES]; 28static unsigned int nr_huge_pages_node[MAX_NUMNODES]; 29static unsigned int free_huge_pages_node[MAX_NUMNODES]; 30/* 31 * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages 32 */ 33static DEFINE_SPINLOCK(hugetlb_lock); 34 35static void clear_huge_page(struct page *page, unsigned long addr) 36{ 37 int i; 38 39 might_sleep(); 40 for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) { 41 cond_resched(); 42 clear_user_highpage(page + i, addr); 43 } 44} 45 46static void copy_huge_page(struct page *dst, struct page *src, 47 unsigned long addr, struct vm_area_struct *vma) 48{ 49 int i; 50 51 might_sleep(); 52 for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) { 53 cond_resched(); 54 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma); 55 } 56} 57 58static void enqueue_huge_page(struct page *page) 59{ 60 int nid = page_to_nid(page); 61 list_add(&page->lru, &hugepage_freelists[nid]); 62 free_huge_pages++; 63 free_huge_pages_node[nid]++; 64} 65 66static struct page *dequeue_huge_page(struct vm_area_struct *vma, 67 unsigned long address) 68{ 69 int nid; 70 struct page *page = NULL; 71 struct zonelist *zonelist = huge_zonelist(vma, address); 72 struct zone **z; 73 74 for (z = zonelist->zones; *z; z++) { 75 nid = zone_to_nid(*z); 76 if (cpuset_zone_allowed_softwall(*z, GFP_HIGHUSER) && 77 !list_empty(&hugepage_freelists[nid])) 78 break; 79 } 80 81 if (*z) { 82 page = list_entry(hugepage_freelists[nid].next, 83 struct page, lru); 84 list_del(&page->lru); 85 free_huge_pages--; 86 free_huge_pages_node[nid]--; 87 } 88 return page; 89} 90 91static void free_huge_page(struct page *page) 92{ 93 BUG_ON(page_count(page)); 94 95 INIT_LIST_HEAD(&page->lru); 96 97 spin_lock(&hugetlb_lock); 98 enqueue_huge_page(page); 99 spin_unlock(&hugetlb_lock); 100} 101 102static int alloc_fresh_huge_page(void) 103{ 104 static int prev_nid; 105 struct page *page; 106 static DEFINE_SPINLOCK(nid_lock); 107 int nid; 108 109 spin_lock(&nid_lock); 110 nid = next_node(prev_nid, node_online_map); 111 if (nid == MAX_NUMNODES) 112 nid = first_node(node_online_map); 113 prev_nid = nid; 114 spin_unlock(&nid_lock); 115 116 page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN, 117 HUGETLB_PAGE_ORDER); 118 if (page) { 119 set_compound_page_dtor(page, free_huge_page); 120 spin_lock(&hugetlb_lock); 121 nr_huge_pages++; 122 nr_huge_pages_node[page_to_nid(page)]++; 123 spin_unlock(&hugetlb_lock); 124 put_page(page); /* free it into the hugepage allocator */ 125 return 1; 126 } 127 return 0; 128} 129 130static struct page *alloc_huge_page(struct vm_area_struct *vma, 131 unsigned long addr) 132{ 133 struct page *page; 134 135 spin_lock(&hugetlb_lock); 136 if (vma->vm_flags & VM_MAYSHARE) 137 resv_huge_pages--; 138 else if (free_huge_pages <= resv_huge_pages) 139 goto fail; 140 141 page = dequeue_huge_page(vma, addr); 142 if (!page) 143 goto fail; 144 145 spin_unlock(&hugetlb_lock); 146 set_page_refcounted(page); 147 return page; 148 149fail: 150 if (vma->vm_flags & VM_MAYSHARE) 151 resv_huge_pages++; 152 spin_unlock(&hugetlb_lock); 153 return NULL; 154} 155 156static int __init hugetlb_init(void) 157{ 158 unsigned long i; 159 160 if (HPAGE_SHIFT == 0) 161 return 0; 162 163 for (i = 0; i < MAX_NUMNODES; ++i) 164 INIT_LIST_HEAD(&hugepage_freelists[i]); 165 166 for (i = 0; i < max_huge_pages; ++i) { 167 if (!alloc_fresh_huge_page()) 168 break; 169 } 170 max_huge_pages = free_huge_pages = nr_huge_pages = i; 171 printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages); 172 return 0; 173} 174module_init(hugetlb_init); 175 176static int __init hugetlb_setup(char *s) 177{ 178 if (sscanf(s, "%lu", &max_huge_pages) <= 0) 179 max_huge_pages = 0; 180 return 1; 181} 182__setup("hugepages=", hugetlb_setup); 183 184static unsigned int cpuset_mems_nr(unsigned int *array) 185{ 186 int node; 187 unsigned int nr = 0; 188 189 for_each_node_mask(node, cpuset_current_mems_allowed) 190 nr += array[node]; 191 192 return nr; 193} 194 195#ifdef CONFIG_SYSCTL 196static void update_and_free_page(struct page *page) 197{ 198 int i; 199 nr_huge_pages--; 200 nr_huge_pages_node[page_to_nid(page)]--; 201 for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) { 202 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced | 203 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved | 204 1 << PG_private | 1<< PG_writeback); 205 } 206 page[1].lru.next = NULL; 207 set_page_refcounted(page); 208 __free_pages(page, HUGETLB_PAGE_ORDER); 209} 210 211#ifdef CONFIG_HIGHMEM 212static void try_to_free_low(unsigned long count) 213{ 214 int i; 215 216 for (i = 0; i < MAX_NUMNODES; ++i) { 217 struct page *page, *next; 218 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) { 219 if (PageHighMem(page)) 220 continue; 221 list_del(&page->lru); 222 update_and_free_page(page); 223 free_huge_pages--; 224 free_huge_pages_node[page_to_nid(page)]--; 225 if (count >= nr_huge_pages) 226 return; 227 } 228 } 229} 230#else 231static inline void try_to_free_low(unsigned long count) 232{ 233} 234#endif 235 236static unsigned long set_max_huge_pages(unsigned long count) 237{ 238 while (count > nr_huge_pages) { 239 if (!alloc_fresh_huge_page()) 240 return nr_huge_pages; 241 } 242 if (count >= nr_huge_pages) 243 return nr_huge_pages; 244 245 spin_lock(&hugetlb_lock); 246 count = max(count, resv_huge_pages); 247 try_to_free_low(count); 248 while (count < nr_huge_pages) { 249 struct page *page = dequeue_huge_page(NULL, 0); 250 if (!page) 251 break; 252 update_and_free_page(page); 253 } 254 spin_unlock(&hugetlb_lock); 255 return nr_huge_pages; 256} 257 258int hugetlb_sysctl_handler(struct ctl_table *table, int write, 259 struct file *file, void __user *buffer, 260 size_t *length, loff_t *ppos) 261{ 262 proc_doulongvec_minmax(table, write, file, buffer, length, ppos); 263 max_huge_pages = set_max_huge_pages(max_huge_pages); 264 return 0; 265} 266#endif /* CONFIG_SYSCTL */ 267 268int hugetlb_report_meminfo(char *buf) 269{ 270 return sprintf(buf, 271 "HugePages_Total: %5lu\n" 272 "HugePages_Free: %5lu\n" 273 "HugePages_Rsvd: %5lu\n" 274 "Hugepagesize: %5lu kB\n", 275 nr_huge_pages, 276 free_huge_pages, 277 resv_huge_pages, 278 HPAGE_SIZE/1024); 279} 280 281int hugetlb_report_node_meminfo(int nid, char *buf) 282{ 283 return sprintf(buf, 284 "Node %d HugePages_Total: %5u\n" 285 "Node %d HugePages_Free: %5u\n", 286 nid, nr_huge_pages_node[nid], 287 nid, free_huge_pages_node[nid]); 288} 289 290/* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 291unsigned long hugetlb_total_pages(void) 292{ 293 return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE); 294} 295 296/* 297 * We cannot handle pagefaults against hugetlb pages at all. They cause 298 * handle_mm_fault() to try to instantiate regular-sized pages in the 299 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 300 * this far. 301 */ 302static struct page *hugetlb_nopage(struct vm_area_struct *vma, 303 unsigned long address, int *unused) 304{ 305 BUG(); 306 return NULL; 307} 308 309struct vm_operations_struct hugetlb_vm_ops = { 310 .nopage = hugetlb_nopage, 311}; 312 313static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 314 int writable) 315{ 316 pte_t entry; 317 318 if (writable) { 319 entry = 320 pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))); 321 } else { 322 entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot)); 323 } 324 entry = pte_mkyoung(entry); 325 entry = pte_mkhuge(entry); 326 327 return entry; 328} 329 330static void set_huge_ptep_writable(struct vm_area_struct *vma, 331 unsigned long address, pte_t *ptep) 332{ 333 pte_t entry; 334 335 entry = pte_mkwrite(pte_mkdirty(*ptep)); 336 if (ptep_set_access_flags(vma, address, ptep, entry, 1)) { 337 update_mmu_cache(vma, address, entry); 338 lazy_mmu_prot_update(entry); 339 } 340} 341 342 343int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 344 struct vm_area_struct *vma) 345{ 346 pte_t *src_pte, *dst_pte, entry; 347 struct page *ptepage; 348 unsigned long addr; 349 int cow; 350 351 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 352 353 for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) { 354 src_pte = huge_pte_offset(src, addr); 355 if (!src_pte) 356 continue; 357 dst_pte = huge_pte_alloc(dst, addr); 358 if (!dst_pte) 359 goto nomem; 360 spin_lock(&dst->page_table_lock); 361 spin_lock(&src->page_table_lock); 362 if (!pte_none(*src_pte)) { 363 if (cow) 364 ptep_set_wrprotect(src, addr, src_pte); 365 entry = *src_pte; 366 ptepage = pte_page(entry); 367 get_page(ptepage); 368 set_huge_pte_at(dst, addr, dst_pte, entry); 369 } 370 spin_unlock(&src->page_table_lock); 371 spin_unlock(&dst->page_table_lock); 372 } 373 return 0; 374 375nomem: 376 return -ENOMEM; 377} 378 379void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 380 unsigned long end) 381{ 382 struct mm_struct *mm = vma->vm_mm; 383 unsigned long address; 384 pte_t *ptep; 385 pte_t pte; 386 struct page *page; 387 struct page *tmp; 388 /* 389 * A page gathering list, protected by per file i_mmap_lock. The 390 * lock is used to avoid list corruption from multiple unmapping 391 * of the same page since we are using page->lru. 392 */ 393 LIST_HEAD(page_list); 394 395 WARN_ON(!is_vm_hugetlb_page(vma)); 396 BUG_ON(start & ~HPAGE_MASK); 397 BUG_ON(end & ~HPAGE_MASK); 398 399 spin_lock(&mm->page_table_lock); 400 for (address = start; address < end; address += HPAGE_SIZE) { 401 ptep = huge_pte_offset(mm, address); 402 if (!ptep) 403 continue; 404 405 if (huge_pmd_unshare(mm, &address, ptep)) 406 continue; 407 408 pte = huge_ptep_get_and_clear(mm, address, ptep); 409 if (pte_none(pte)) 410 continue; 411 412 page = pte_page(pte); 413 if (pte_dirty(pte)) 414 set_page_dirty(page); 415 list_add(&page->lru, &page_list); 416 } 417 spin_unlock(&mm->page_table_lock); 418 flush_tlb_range(vma, start, end); 419 list_for_each_entry_safe(page, tmp, &page_list, lru) { 420 list_del(&page->lru); 421 put_page(page); 422 } 423} 424 425void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 426 unsigned long end) 427{ 428 /* 429 * It is undesirable to test vma->vm_file as it should be non-null 430 * for valid hugetlb area. However, vm_file will be NULL in the error 431 * cleanup path of do_mmap_pgoff. When hugetlbfs ->mmap method fails, 432 * do_mmap_pgoff() nullifies vma->vm_file before calling this function 433 * to clean up. Since no pte has actually been setup, it is safe to 434 * do nothing in this case. 435 */ 436 if (vma->vm_file) { 437 spin_lock(&vma->vm_file->f_mapping->i_mmap_lock); 438 __unmap_hugepage_range(vma, start, end); 439 spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock); 440 } 441} 442 443static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 444 unsigned long address, pte_t *ptep, pte_t pte) 445{ 446 struct page *old_page, *new_page; 447 int avoidcopy; 448 449 old_page = pte_page(pte); 450 451 /* If no-one else is actually using this page, avoid the copy 452 * and just make the page writable */ 453 avoidcopy = (page_count(old_page) == 1); 454 if (avoidcopy) { 455 set_huge_ptep_writable(vma, address, ptep); 456 return VM_FAULT_MINOR; 457 } 458 459 page_cache_get(old_page); 460 new_page = alloc_huge_page(vma, address); 461 462 if (!new_page) { 463 page_cache_release(old_page); 464 return VM_FAULT_OOM; 465 } 466 467 spin_unlock(&mm->page_table_lock); 468 copy_huge_page(new_page, old_page, address, vma); 469 spin_lock(&mm->page_table_lock); 470 471 ptep = huge_pte_offset(mm, address & HPAGE_MASK); 472 if (likely(pte_same(*ptep, pte))) { 473 /* Break COW */ 474 set_huge_pte_at(mm, address, ptep, 475 make_huge_pte(vma, new_page, 1)); 476 /* Make the old page be freed below */ 477 new_page = old_page; 478 } 479 page_cache_release(new_page); 480 page_cache_release(old_page); 481 return VM_FAULT_MINOR; 482} 483 484int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, 485 unsigned long address, pte_t *ptep, int write_access) 486{ 487 int ret = VM_FAULT_SIGBUS; 488 unsigned long idx; 489 unsigned long size; 490 struct page *page; 491 struct address_space *mapping; 492 pte_t new_pte; 493 494 mapping = vma->vm_file->f_mapping; 495 idx = ((address - vma->vm_start) >> HPAGE_SHIFT) 496 + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT)); 497 498 /* 499 * Use page lock to guard against racing truncation 500 * before we get page_table_lock. 501 */ 502retry: 503 page = find_lock_page(mapping, idx); 504 if (!page) { 505 size = i_size_read(mapping->host) >> HPAGE_SHIFT; 506 if (idx >= size) 507 goto out; 508 if (hugetlb_get_quota(mapping)) 509 goto out; 510 page = alloc_huge_page(vma, address); 511 if (!page) { 512 hugetlb_put_quota(mapping); 513 ret = VM_FAULT_OOM; 514 goto out; 515 } 516 clear_huge_page(page, address); 517 518 if (vma->vm_flags & VM_SHARED) { 519 int err; 520 521 err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 522 if (err) { 523 put_page(page); 524 hugetlb_put_quota(mapping); 525 if (err == -EEXIST) 526 goto retry; 527 goto out; 528 } 529 } else 530 lock_page(page); 531 } 532 533 spin_lock(&mm->page_table_lock); 534 size = i_size_read(mapping->host) >> HPAGE_SHIFT; 535 if (idx >= size) 536 goto backout; 537 538 ret = VM_FAULT_MINOR; 539 if (!pte_none(*ptep)) 540 goto backout; 541 542 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 543 && (vma->vm_flags & VM_SHARED))); 544 set_huge_pte_at(mm, address, ptep, new_pte); 545 546 if (write_access && !(vma->vm_flags & VM_SHARED)) { 547 /* Optimization, do the COW without a second fault */ 548 ret = hugetlb_cow(mm, vma, address, ptep, new_pte); 549 } 550 551 spin_unlock(&mm->page_table_lock); 552 unlock_page(page); 553out: 554 return ret; 555 556backout: 557 spin_unlock(&mm->page_table_lock); 558 hugetlb_put_quota(mapping); 559 unlock_page(page); 560 put_page(page); 561 goto out; 562} 563 564int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 565 unsigned long address, int write_access) 566{ 567 pte_t *ptep; 568 pte_t entry; 569 int ret; 570 static DEFINE_MUTEX(hugetlb_instantiation_mutex); 571 572 ptep = huge_pte_alloc(mm, address); 573 if (!ptep) 574 return VM_FAULT_OOM; 575 576 /* 577 * Serialize hugepage allocation and instantiation, so that we don't 578 * get spurious allocation failures if two CPUs race to instantiate 579 * the same page in the page cache. 580 */ 581 mutex_lock(&hugetlb_instantiation_mutex); 582 entry = *ptep; 583 if (pte_none(entry)) { 584 ret = hugetlb_no_page(mm, vma, address, ptep, write_access); 585 mutex_unlock(&hugetlb_instantiation_mutex); 586 return ret; 587 } 588 589 ret = VM_FAULT_MINOR; 590 591 spin_lock(&mm->page_table_lock); 592 /* Check for a racing update before calling hugetlb_cow */ 593 if (likely(pte_same(entry, *ptep))) 594 if (write_access && !pte_write(entry)) 595 ret = hugetlb_cow(mm, vma, address, ptep, entry); 596 spin_unlock(&mm->page_table_lock); 597 mutex_unlock(&hugetlb_instantiation_mutex); 598 599 return ret; 600} 601 602int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 603 struct page **pages, struct vm_area_struct **vmas, 604 unsigned long *position, int *length, int i) 605{ 606 unsigned long pfn_offset; 607 unsigned long vaddr = *position; 608 int remainder = *length; 609 610 spin_lock(&mm->page_table_lock); 611 while (vaddr < vma->vm_end && remainder) { 612 pte_t *pte; 613 struct page *page; 614 615 /* 616 * Some archs (sparc64, sh*) have multiple pte_ts to 617 * each hugepage. We have to make * sure we get the 618 * first, for the page indexing below to work. 619 */ 620 pte = huge_pte_offset(mm, vaddr & HPAGE_MASK); 621 622 if (!pte || pte_none(*pte)) { 623 int ret; 624 625 spin_unlock(&mm->page_table_lock); 626 ret = hugetlb_fault(mm, vma, vaddr, 0); 627 spin_lock(&mm->page_table_lock); 628 if (ret == VM_FAULT_MINOR) 629 continue; 630 631 remainder = 0; 632 if (!i) 633 i = -EFAULT; 634 break; 635 } 636 637 pfn_offset = (vaddr & ~HPAGE_MASK) >> PAGE_SHIFT; 638 page = pte_page(*pte); 639same_page: 640 if (pages) { 641 get_page(page); 642 pages[i] = page + pfn_offset; 643 } 644 645 if (vmas) 646 vmas[i] = vma; 647 648 vaddr += PAGE_SIZE; 649 ++pfn_offset; 650 --remainder; 651 ++i; 652 if (vaddr < vma->vm_end && remainder && 653 pfn_offset < HPAGE_SIZE/PAGE_SIZE) { 654 /* 655 * We use pfn_offset to avoid touching the pageframes 656 * of this compound page. 657 */ 658 goto same_page; 659 } 660 } 661 spin_unlock(&mm->page_table_lock); 662 *length = remainder; 663 *position = vaddr; 664 665 return i; 666} 667 668void hugetlb_change_protection(struct vm_area_struct *vma, 669 unsigned long address, unsigned long end, pgprot_t newprot) 670{ 671 struct mm_struct *mm = vma->vm_mm; 672 unsigned long start = address; 673 pte_t *ptep; 674 pte_t pte; 675 676 BUG_ON(address >= end); 677 flush_cache_range(vma, address, end); 678 679 spin_lock(&vma->vm_file->f_mapping->i_mmap_lock); 680 spin_lock(&mm->page_table_lock); 681 for (; address < end; address += HPAGE_SIZE) { 682 ptep = huge_pte_offset(mm, address); 683 if (!ptep) 684 continue; 685 if (huge_pmd_unshare(mm, &address, ptep)) 686 continue; 687 if (!pte_none(*ptep)) { 688 pte = huge_ptep_get_and_clear(mm, address, ptep); 689 pte = pte_mkhuge(pte_modify(pte, newprot)); 690 set_huge_pte_at(mm, address, ptep, pte); 691 lazy_mmu_prot_update(pte); 692 } 693 } 694 spin_unlock(&mm->page_table_lock); 695 spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock); 696 697 flush_tlb_range(vma, start, end); 698} 699 700struct file_region { 701 struct list_head link; 702 long from; 703 long to; 704}; 705 706static long region_add(struct list_head *head, long f, long t) 707{ 708 struct file_region *rg, *nrg, *trg; 709 710 /* Locate the region we are either in or before. */ 711 list_for_each_entry(rg, head, link) 712 if (f <= rg->to) 713 break; 714 715 /* Round our left edge to the current segment if it encloses us. */ 716 if (f > rg->from) 717 f = rg->from; 718 719 /* Check for and consume any regions we now overlap with. */ 720 nrg = rg; 721 list_for_each_entry_safe(rg, trg, rg->link.prev, link) { 722 if (&rg->link == head) 723 break; 724 if (rg->from > t) 725 break; 726 727 /* If this area reaches higher then extend our area to 728 * include it completely. If this is not the first area 729 * which we intend to reuse, free it. */ 730 if (rg->to > t) 731 t = rg->to; 732 if (rg != nrg) { 733 list_del(&rg->link); 734 kfree(rg); 735 } 736 } 737 nrg->from = f; 738 nrg->to = t; 739 return 0; 740} 741 742static long region_chg(struct list_head *head, long f, long t) 743{ 744 struct file_region *rg, *nrg; 745 long chg = 0; 746 747 /* Locate the region we are before or in. */ 748 list_for_each_entry(rg, head, link) 749 if (f <= rg->to) 750 break; 751 752 /* If we are below the current region then a new region is required. 753 * Subtle, allocate a new region at the position but make it zero 754 * size such that we can guarentee to record the reservation. */ 755 if (&rg->link == head || t < rg->from) { 756 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 757 if (nrg == 0) 758 return -ENOMEM; 759 nrg->from = f; 760 nrg->to = f; 761 INIT_LIST_HEAD(&nrg->link); 762 list_add(&nrg->link, rg->link.prev); 763 764 return t - f; 765 } 766 767 /* Round our left edge to the current segment if it encloses us. */ 768 if (f > rg->from) 769 f = rg->from; 770 chg = t - f; 771 772 /* Check for and consume any regions we now overlap with. */ 773 list_for_each_entry(rg, rg->link.prev, link) { 774 if (&rg->link == head) 775 break; 776 if (rg->from > t) 777 return chg; 778 779 /* We overlap with this area, if it extends futher than 780 * us then we must extend ourselves. Account for its 781 * existing reservation. */ 782 if (rg->to > t) { 783 chg += rg->to - t; 784 t = rg->to; 785 } 786 chg -= rg->to - rg->from; 787 } 788 return chg; 789} 790 791static long region_truncate(struct list_head *head, long end) 792{ 793 struct file_region *rg, *trg; 794 long chg = 0; 795 796 /* Locate the region we are either in or before. */ 797 list_for_each_entry(rg, head, link) 798 if (end <= rg->to) 799 break; 800 if (&rg->link == head) 801 return 0; 802 803 /* If we are in the middle of a region then adjust it. */ 804 if (end > rg->from) { 805 chg = rg->to - end; 806 rg->to = end; 807 rg = list_entry(rg->link.next, typeof(*rg), link); 808 } 809 810 /* Drop any remaining regions. */ 811 list_for_each_entry_safe(rg, trg, rg->link.prev, link) { 812 if (&rg->link == head) 813 break; 814 chg += rg->to - rg->from; 815 list_del(&rg->link); 816 kfree(rg); 817 } 818 return chg; 819} 820 821static int hugetlb_acct_memory(long delta) 822{ 823 int ret = -ENOMEM; 824 825 spin_lock(&hugetlb_lock); 826 if ((delta + resv_huge_pages) <= free_huge_pages) { 827 resv_huge_pages += delta; 828 ret = 0; 829 } 830 spin_unlock(&hugetlb_lock); 831 return ret; 832} 833 834int hugetlb_reserve_pages(struct inode *inode, long from, long to) 835{ 836 long ret, chg; 837 838 chg = region_chg(&inode->i_mapping->private_list, from, to); 839 if (chg < 0) 840 return chg; 841 /* 842 * When cpuset is configured, it breaks the strict hugetlb page 843 * reservation as the accounting is done on a global variable. Such 844 * reservation is completely rubbish in the presence of cpuset because 845 * the reservation is not checked against page availability for the 846 * current cpuset. Application can still potentially OOM'ed by kernel 847 * with lack of free htlb page in cpuset that the task is in. 848 * Attempt to enforce strict accounting with cpuset is almost 849 * impossible (or too ugly) because cpuset is too fluid that 850 * task or memory node can be dynamically moved between cpusets. 851 * 852 * The change of semantics for shared hugetlb mapping with cpuset is 853 * undesirable. However, in order to preserve some of the semantics, 854 * we fall back to check against current free page availability as 855 * a best attempt and hopefully to minimize the impact of changing 856 * semantics that cpuset has. 857 */ 858 if (chg > cpuset_mems_nr(free_huge_pages_node)) 859 return -ENOMEM; 860 861 ret = hugetlb_acct_memory(chg); 862 if (ret < 0) 863 return ret; 864 region_add(&inode->i_mapping->private_list, from, to); 865 return 0; 866} 867 868void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed) 869{ 870 long chg = region_truncate(&inode->i_mapping->private_list, offset); 871 hugetlb_acct_memory(freed - chg); 872} 873