hugetlb.c revision ed4d4902ebdd7ca8b5a51daaf6bebf4b172895cc
1/* 2 * Generic hugetlb support. 3 * (C) Nadia Yvette Chambers, April 2004 4 */ 5#include <linux/list.h> 6#include <linux/init.h> 7#include <linux/module.h> 8#include <linux/mm.h> 9#include <linux/seq_file.h> 10#include <linux/sysctl.h> 11#include <linux/highmem.h> 12#include <linux/mmu_notifier.h> 13#include <linux/nodemask.h> 14#include <linux/pagemap.h> 15#include <linux/mempolicy.h> 16#include <linux/compiler.h> 17#include <linux/cpuset.h> 18#include <linux/mutex.h> 19#include <linux/bootmem.h> 20#include <linux/sysfs.h> 21#include <linux/slab.h> 22#include <linux/rmap.h> 23#include <linux/swap.h> 24#include <linux/swapops.h> 25#include <linux/page-isolation.h> 26#include <linux/jhash.h> 27 28#include <asm/page.h> 29#include <asm/pgtable.h> 30#include <asm/tlb.h> 31 32#include <linux/io.h> 33#include <linux/hugetlb.h> 34#include <linux/hugetlb_cgroup.h> 35#include <linux/node.h> 36#include "internal.h" 37 38unsigned long hugepages_treat_as_movable; 39 40int hugetlb_max_hstate __read_mostly; 41unsigned int default_hstate_idx; 42struct hstate hstates[HUGE_MAX_HSTATE]; 43 44__initdata LIST_HEAD(huge_boot_pages); 45 46/* for command line parsing */ 47static struct hstate * __initdata parsed_hstate; 48static unsigned long __initdata default_hstate_max_huge_pages; 49static unsigned long __initdata default_hstate_size; 50 51/* 52 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 53 * free_huge_pages, and surplus_huge_pages. 54 */ 55DEFINE_SPINLOCK(hugetlb_lock); 56 57/* 58 * Serializes faults on the same logical page. This is used to 59 * prevent spurious OOMs when the hugepage pool is fully utilized. 60 */ 61static int num_fault_mutexes; 62static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp; 63 64static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) 65{ 66 bool free = (spool->count == 0) && (spool->used_hpages == 0); 67 68 spin_unlock(&spool->lock); 69 70 /* If no pages are used, and no other handles to the subpool 71 * remain, free the subpool the subpool remain */ 72 if (free) 73 kfree(spool); 74} 75 76struct hugepage_subpool *hugepage_new_subpool(long nr_blocks) 77{ 78 struct hugepage_subpool *spool; 79 80 spool = kmalloc(sizeof(*spool), GFP_KERNEL); 81 if (!spool) 82 return NULL; 83 84 spin_lock_init(&spool->lock); 85 spool->count = 1; 86 spool->max_hpages = nr_blocks; 87 spool->used_hpages = 0; 88 89 return spool; 90} 91 92void hugepage_put_subpool(struct hugepage_subpool *spool) 93{ 94 spin_lock(&spool->lock); 95 BUG_ON(!spool->count); 96 spool->count--; 97 unlock_or_release_subpool(spool); 98} 99 100static int hugepage_subpool_get_pages(struct hugepage_subpool *spool, 101 long delta) 102{ 103 int ret = 0; 104 105 if (!spool) 106 return 0; 107 108 spin_lock(&spool->lock); 109 if ((spool->used_hpages + delta) <= spool->max_hpages) { 110 spool->used_hpages += delta; 111 } else { 112 ret = -ENOMEM; 113 } 114 spin_unlock(&spool->lock); 115 116 return ret; 117} 118 119static void hugepage_subpool_put_pages(struct hugepage_subpool *spool, 120 long delta) 121{ 122 if (!spool) 123 return; 124 125 spin_lock(&spool->lock); 126 spool->used_hpages -= delta; 127 /* If hugetlbfs_put_super couldn't free spool due to 128 * an outstanding quota reference, free it now. */ 129 unlock_or_release_subpool(spool); 130} 131 132static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 133{ 134 return HUGETLBFS_SB(inode->i_sb)->spool; 135} 136 137static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 138{ 139 return subpool_inode(file_inode(vma->vm_file)); 140} 141 142/* 143 * Region tracking -- allows tracking of reservations and instantiated pages 144 * across the pages in a mapping. 145 * 146 * The region data structures are embedded into a resv_map and 147 * protected by a resv_map's lock 148 */ 149struct file_region { 150 struct list_head link; 151 long from; 152 long to; 153}; 154 155static long region_add(struct resv_map *resv, long f, long t) 156{ 157 struct list_head *head = &resv->regions; 158 struct file_region *rg, *nrg, *trg; 159 160 spin_lock(&resv->lock); 161 /* Locate the region we are either in or before. */ 162 list_for_each_entry(rg, head, link) 163 if (f <= rg->to) 164 break; 165 166 /* Round our left edge to the current segment if it encloses us. */ 167 if (f > rg->from) 168 f = rg->from; 169 170 /* Check for and consume any regions we now overlap with. */ 171 nrg = rg; 172 list_for_each_entry_safe(rg, trg, rg->link.prev, link) { 173 if (&rg->link == head) 174 break; 175 if (rg->from > t) 176 break; 177 178 /* If this area reaches higher then extend our area to 179 * include it completely. If this is not the first area 180 * which we intend to reuse, free it. */ 181 if (rg->to > t) 182 t = rg->to; 183 if (rg != nrg) { 184 list_del(&rg->link); 185 kfree(rg); 186 } 187 } 188 nrg->from = f; 189 nrg->to = t; 190 spin_unlock(&resv->lock); 191 return 0; 192} 193 194static long region_chg(struct resv_map *resv, long f, long t) 195{ 196 struct list_head *head = &resv->regions; 197 struct file_region *rg, *nrg = NULL; 198 long chg = 0; 199 200retry: 201 spin_lock(&resv->lock); 202 /* Locate the region we are before or in. */ 203 list_for_each_entry(rg, head, link) 204 if (f <= rg->to) 205 break; 206 207 /* If we are below the current region then a new region is required. 208 * Subtle, allocate a new region at the position but make it zero 209 * size such that we can guarantee to record the reservation. */ 210 if (&rg->link == head || t < rg->from) { 211 if (!nrg) { 212 spin_unlock(&resv->lock); 213 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 214 if (!nrg) 215 return -ENOMEM; 216 217 nrg->from = f; 218 nrg->to = f; 219 INIT_LIST_HEAD(&nrg->link); 220 goto retry; 221 } 222 223 list_add(&nrg->link, rg->link.prev); 224 chg = t - f; 225 goto out_nrg; 226 } 227 228 /* Round our left edge to the current segment if it encloses us. */ 229 if (f > rg->from) 230 f = rg->from; 231 chg = t - f; 232 233 /* Check for and consume any regions we now overlap with. */ 234 list_for_each_entry(rg, rg->link.prev, link) { 235 if (&rg->link == head) 236 break; 237 if (rg->from > t) 238 goto out; 239 240 /* We overlap with this area, if it extends further than 241 * us then we must extend ourselves. Account for its 242 * existing reservation. */ 243 if (rg->to > t) { 244 chg += rg->to - t; 245 t = rg->to; 246 } 247 chg -= rg->to - rg->from; 248 } 249 250out: 251 spin_unlock(&resv->lock); 252 /* We already know we raced and no longer need the new region */ 253 kfree(nrg); 254 return chg; 255out_nrg: 256 spin_unlock(&resv->lock); 257 return chg; 258} 259 260static long region_truncate(struct resv_map *resv, long end) 261{ 262 struct list_head *head = &resv->regions; 263 struct file_region *rg, *trg; 264 long chg = 0; 265 266 spin_lock(&resv->lock); 267 /* Locate the region we are either in or before. */ 268 list_for_each_entry(rg, head, link) 269 if (end <= rg->to) 270 break; 271 if (&rg->link == head) 272 goto out; 273 274 /* If we are in the middle of a region then adjust it. */ 275 if (end > rg->from) { 276 chg = rg->to - end; 277 rg->to = end; 278 rg = list_entry(rg->link.next, typeof(*rg), link); 279 } 280 281 /* Drop any remaining regions. */ 282 list_for_each_entry_safe(rg, trg, rg->link.prev, link) { 283 if (&rg->link == head) 284 break; 285 chg += rg->to - rg->from; 286 list_del(&rg->link); 287 kfree(rg); 288 } 289 290out: 291 spin_unlock(&resv->lock); 292 return chg; 293} 294 295static long region_count(struct resv_map *resv, long f, long t) 296{ 297 struct list_head *head = &resv->regions; 298 struct file_region *rg; 299 long chg = 0; 300 301 spin_lock(&resv->lock); 302 /* Locate each segment we overlap with, and count that overlap. */ 303 list_for_each_entry(rg, head, link) { 304 long seg_from; 305 long seg_to; 306 307 if (rg->to <= f) 308 continue; 309 if (rg->from >= t) 310 break; 311 312 seg_from = max(rg->from, f); 313 seg_to = min(rg->to, t); 314 315 chg += seg_to - seg_from; 316 } 317 spin_unlock(&resv->lock); 318 319 return chg; 320} 321 322/* 323 * Convert the address within this vma to the page offset within 324 * the mapping, in pagecache page units; huge pages here. 325 */ 326static pgoff_t vma_hugecache_offset(struct hstate *h, 327 struct vm_area_struct *vma, unsigned long address) 328{ 329 return ((address - vma->vm_start) >> huge_page_shift(h)) + 330 (vma->vm_pgoff >> huge_page_order(h)); 331} 332 333pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 334 unsigned long address) 335{ 336 return vma_hugecache_offset(hstate_vma(vma), vma, address); 337} 338 339/* 340 * Return the size of the pages allocated when backing a VMA. In the majority 341 * cases this will be same size as used by the page table entries. 342 */ 343unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 344{ 345 struct hstate *hstate; 346 347 if (!is_vm_hugetlb_page(vma)) 348 return PAGE_SIZE; 349 350 hstate = hstate_vma(vma); 351 352 return 1UL << huge_page_shift(hstate); 353} 354EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 355 356/* 357 * Return the page size being used by the MMU to back a VMA. In the majority 358 * of cases, the page size used by the kernel matches the MMU size. On 359 * architectures where it differs, an architecture-specific version of this 360 * function is required. 361 */ 362#ifndef vma_mmu_pagesize 363unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 364{ 365 return vma_kernel_pagesize(vma); 366} 367#endif 368 369/* 370 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 371 * bits of the reservation map pointer, which are always clear due to 372 * alignment. 373 */ 374#define HPAGE_RESV_OWNER (1UL << 0) 375#define HPAGE_RESV_UNMAPPED (1UL << 1) 376#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 377 378/* 379 * These helpers are used to track how many pages are reserved for 380 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 381 * is guaranteed to have their future faults succeed. 382 * 383 * With the exception of reset_vma_resv_huge_pages() which is called at fork(), 384 * the reserve counters are updated with the hugetlb_lock held. It is safe 385 * to reset the VMA at fork() time as it is not in use yet and there is no 386 * chance of the global counters getting corrupted as a result of the values. 387 * 388 * The private mapping reservation is represented in a subtly different 389 * manner to a shared mapping. A shared mapping has a region map associated 390 * with the underlying file, this region map represents the backing file 391 * pages which have ever had a reservation assigned which this persists even 392 * after the page is instantiated. A private mapping has a region map 393 * associated with the original mmap which is attached to all VMAs which 394 * reference it, this region map represents those offsets which have consumed 395 * reservation ie. where pages have been instantiated. 396 */ 397static unsigned long get_vma_private_data(struct vm_area_struct *vma) 398{ 399 return (unsigned long)vma->vm_private_data; 400} 401 402static void set_vma_private_data(struct vm_area_struct *vma, 403 unsigned long value) 404{ 405 vma->vm_private_data = (void *)value; 406} 407 408struct resv_map *resv_map_alloc(void) 409{ 410 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 411 if (!resv_map) 412 return NULL; 413 414 kref_init(&resv_map->refs); 415 spin_lock_init(&resv_map->lock); 416 INIT_LIST_HEAD(&resv_map->regions); 417 418 return resv_map; 419} 420 421void resv_map_release(struct kref *ref) 422{ 423 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 424 425 /* Clear out any active regions before we release the map. */ 426 region_truncate(resv_map, 0); 427 kfree(resv_map); 428} 429 430static inline struct resv_map *inode_resv_map(struct inode *inode) 431{ 432 return inode->i_mapping->private_data; 433} 434 435static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 436{ 437 VM_BUG_ON(!is_vm_hugetlb_page(vma)); 438 if (vma->vm_flags & VM_MAYSHARE) { 439 struct address_space *mapping = vma->vm_file->f_mapping; 440 struct inode *inode = mapping->host; 441 442 return inode_resv_map(inode); 443 444 } else { 445 return (struct resv_map *)(get_vma_private_data(vma) & 446 ~HPAGE_RESV_MASK); 447 } 448} 449 450static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 451{ 452 VM_BUG_ON(!is_vm_hugetlb_page(vma)); 453 VM_BUG_ON(vma->vm_flags & VM_MAYSHARE); 454 455 set_vma_private_data(vma, (get_vma_private_data(vma) & 456 HPAGE_RESV_MASK) | (unsigned long)map); 457} 458 459static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 460{ 461 VM_BUG_ON(!is_vm_hugetlb_page(vma)); 462 VM_BUG_ON(vma->vm_flags & VM_MAYSHARE); 463 464 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 465} 466 467static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 468{ 469 VM_BUG_ON(!is_vm_hugetlb_page(vma)); 470 471 return (get_vma_private_data(vma) & flag) != 0; 472} 473 474/* Reset counters to 0 and clear all HPAGE_RESV_* flags */ 475void reset_vma_resv_huge_pages(struct vm_area_struct *vma) 476{ 477 VM_BUG_ON(!is_vm_hugetlb_page(vma)); 478 if (!(vma->vm_flags & VM_MAYSHARE)) 479 vma->vm_private_data = (void *)0; 480} 481 482/* Returns true if the VMA has associated reserve pages */ 483static int vma_has_reserves(struct vm_area_struct *vma, long chg) 484{ 485 if (vma->vm_flags & VM_NORESERVE) { 486 /* 487 * This address is already reserved by other process(chg == 0), 488 * so, we should decrement reserved count. Without decrementing, 489 * reserve count remains after releasing inode, because this 490 * allocated page will go into page cache and is regarded as 491 * coming from reserved pool in releasing step. Currently, we 492 * don't have any other solution to deal with this situation 493 * properly, so add work-around here. 494 */ 495 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 496 return 1; 497 else 498 return 0; 499 } 500 501 /* Shared mappings always use reserves */ 502 if (vma->vm_flags & VM_MAYSHARE) 503 return 1; 504 505 /* 506 * Only the process that called mmap() has reserves for 507 * private mappings. 508 */ 509 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 510 return 1; 511 512 return 0; 513} 514 515static void enqueue_huge_page(struct hstate *h, struct page *page) 516{ 517 int nid = page_to_nid(page); 518 list_move(&page->lru, &h->hugepage_freelists[nid]); 519 h->free_huge_pages++; 520 h->free_huge_pages_node[nid]++; 521} 522 523static struct page *dequeue_huge_page_node(struct hstate *h, int nid) 524{ 525 struct page *page; 526 527 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) 528 if (!is_migrate_isolate_page(page)) 529 break; 530 /* 531 * if 'non-isolated free hugepage' not found on the list, 532 * the allocation fails. 533 */ 534 if (&h->hugepage_freelists[nid] == &page->lru) 535 return NULL; 536 list_move(&page->lru, &h->hugepage_activelist); 537 set_page_refcounted(page); 538 h->free_huge_pages--; 539 h->free_huge_pages_node[nid]--; 540 return page; 541} 542 543/* Movability of hugepages depends on migration support. */ 544static inline gfp_t htlb_alloc_mask(struct hstate *h) 545{ 546 if (hugepages_treat_as_movable || hugepage_migration_supported(h)) 547 return GFP_HIGHUSER_MOVABLE; 548 else 549 return GFP_HIGHUSER; 550} 551 552static struct page *dequeue_huge_page_vma(struct hstate *h, 553 struct vm_area_struct *vma, 554 unsigned long address, int avoid_reserve, 555 long chg) 556{ 557 struct page *page = NULL; 558 struct mempolicy *mpol; 559 nodemask_t *nodemask; 560 struct zonelist *zonelist; 561 struct zone *zone; 562 struct zoneref *z; 563 unsigned int cpuset_mems_cookie; 564 565 /* 566 * A child process with MAP_PRIVATE mappings created by their parent 567 * have no page reserves. This check ensures that reservations are 568 * not "stolen". The child may still get SIGKILLed 569 */ 570 if (!vma_has_reserves(vma, chg) && 571 h->free_huge_pages - h->resv_huge_pages == 0) 572 goto err; 573 574 /* If reserves cannot be used, ensure enough pages are in the pool */ 575 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) 576 goto err; 577 578retry_cpuset: 579 cpuset_mems_cookie = read_mems_allowed_begin(); 580 zonelist = huge_zonelist(vma, address, 581 htlb_alloc_mask(h), &mpol, &nodemask); 582 583 for_each_zone_zonelist_nodemask(zone, z, zonelist, 584 MAX_NR_ZONES - 1, nodemask) { 585 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) { 586 page = dequeue_huge_page_node(h, zone_to_nid(zone)); 587 if (page) { 588 if (avoid_reserve) 589 break; 590 if (!vma_has_reserves(vma, chg)) 591 break; 592 593 SetPagePrivate(page); 594 h->resv_huge_pages--; 595 break; 596 } 597 } 598 } 599 600 mpol_cond_put(mpol); 601 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) 602 goto retry_cpuset; 603 return page; 604 605err: 606 return NULL; 607} 608 609/* 610 * common helper functions for hstate_next_node_to_{alloc|free}. 611 * We may have allocated or freed a huge page based on a different 612 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 613 * be outside of *nodes_allowed. Ensure that we use an allowed 614 * node for alloc or free. 615 */ 616static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 617{ 618 nid = next_node(nid, *nodes_allowed); 619 if (nid == MAX_NUMNODES) 620 nid = first_node(*nodes_allowed); 621 VM_BUG_ON(nid >= MAX_NUMNODES); 622 623 return nid; 624} 625 626static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 627{ 628 if (!node_isset(nid, *nodes_allowed)) 629 nid = next_node_allowed(nid, nodes_allowed); 630 return nid; 631} 632 633/* 634 * returns the previously saved node ["this node"] from which to 635 * allocate a persistent huge page for the pool and advance the 636 * next node from which to allocate, handling wrap at end of node 637 * mask. 638 */ 639static int hstate_next_node_to_alloc(struct hstate *h, 640 nodemask_t *nodes_allowed) 641{ 642 int nid; 643 644 VM_BUG_ON(!nodes_allowed); 645 646 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 647 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 648 649 return nid; 650} 651 652/* 653 * helper for free_pool_huge_page() - return the previously saved 654 * node ["this node"] from which to free a huge page. Advance the 655 * next node id whether or not we find a free huge page to free so 656 * that the next attempt to free addresses the next node. 657 */ 658static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 659{ 660 int nid; 661 662 VM_BUG_ON(!nodes_allowed); 663 664 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 665 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 666 667 return nid; 668} 669 670#define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 671 for (nr_nodes = nodes_weight(*mask); \ 672 nr_nodes > 0 && \ 673 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 674 nr_nodes--) 675 676#define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 677 for (nr_nodes = nodes_weight(*mask); \ 678 nr_nodes > 0 && \ 679 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 680 nr_nodes--) 681 682#if defined(CONFIG_CMA) && defined(CONFIG_X86_64) 683static void destroy_compound_gigantic_page(struct page *page, 684 unsigned long order) 685{ 686 int i; 687 int nr_pages = 1 << order; 688 struct page *p = page + 1; 689 690 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 691 __ClearPageTail(p); 692 set_page_refcounted(p); 693 p->first_page = NULL; 694 } 695 696 set_compound_order(page, 0); 697 __ClearPageHead(page); 698} 699 700static void free_gigantic_page(struct page *page, unsigned order) 701{ 702 free_contig_range(page_to_pfn(page), 1 << order); 703} 704 705static int __alloc_gigantic_page(unsigned long start_pfn, 706 unsigned long nr_pages) 707{ 708 unsigned long end_pfn = start_pfn + nr_pages; 709 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 710} 711 712static bool pfn_range_valid_gigantic(unsigned long start_pfn, 713 unsigned long nr_pages) 714{ 715 unsigned long i, end_pfn = start_pfn + nr_pages; 716 struct page *page; 717 718 for (i = start_pfn; i < end_pfn; i++) { 719 if (!pfn_valid(i)) 720 return false; 721 722 page = pfn_to_page(i); 723 724 if (PageReserved(page)) 725 return false; 726 727 if (page_count(page) > 0) 728 return false; 729 730 if (PageHuge(page)) 731 return false; 732 } 733 734 return true; 735} 736 737static bool zone_spans_last_pfn(const struct zone *zone, 738 unsigned long start_pfn, unsigned long nr_pages) 739{ 740 unsigned long last_pfn = start_pfn + nr_pages - 1; 741 return zone_spans_pfn(zone, last_pfn); 742} 743 744static struct page *alloc_gigantic_page(int nid, unsigned order) 745{ 746 unsigned long nr_pages = 1 << order; 747 unsigned long ret, pfn, flags; 748 struct zone *z; 749 750 z = NODE_DATA(nid)->node_zones; 751 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) { 752 spin_lock_irqsave(&z->lock, flags); 753 754 pfn = ALIGN(z->zone_start_pfn, nr_pages); 755 while (zone_spans_last_pfn(z, pfn, nr_pages)) { 756 if (pfn_range_valid_gigantic(pfn, nr_pages)) { 757 /* 758 * We release the zone lock here because 759 * alloc_contig_range() will also lock the zone 760 * at some point. If there's an allocation 761 * spinning on this lock, it may win the race 762 * and cause alloc_contig_range() to fail... 763 */ 764 spin_unlock_irqrestore(&z->lock, flags); 765 ret = __alloc_gigantic_page(pfn, nr_pages); 766 if (!ret) 767 return pfn_to_page(pfn); 768 spin_lock_irqsave(&z->lock, flags); 769 } 770 pfn += nr_pages; 771 } 772 773 spin_unlock_irqrestore(&z->lock, flags); 774 } 775 776 return NULL; 777} 778 779static void prep_new_huge_page(struct hstate *h, struct page *page, int nid); 780static void prep_compound_gigantic_page(struct page *page, unsigned long order); 781 782static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid) 783{ 784 struct page *page; 785 786 page = alloc_gigantic_page(nid, huge_page_order(h)); 787 if (page) { 788 prep_compound_gigantic_page(page, huge_page_order(h)); 789 prep_new_huge_page(h, page, nid); 790 } 791 792 return page; 793} 794 795static int alloc_fresh_gigantic_page(struct hstate *h, 796 nodemask_t *nodes_allowed) 797{ 798 struct page *page = NULL; 799 int nr_nodes, node; 800 801 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 802 page = alloc_fresh_gigantic_page_node(h, node); 803 if (page) 804 return 1; 805 } 806 807 return 0; 808} 809 810static inline bool gigantic_page_supported(void) { return true; } 811#else 812static inline bool gigantic_page_supported(void) { return false; } 813static inline void free_gigantic_page(struct page *page, unsigned order) { } 814static inline void destroy_compound_gigantic_page(struct page *page, 815 unsigned long order) { } 816static inline int alloc_fresh_gigantic_page(struct hstate *h, 817 nodemask_t *nodes_allowed) { return 0; } 818#endif 819 820static void update_and_free_page(struct hstate *h, struct page *page) 821{ 822 int i; 823 824 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 825 return; 826 827 h->nr_huge_pages--; 828 h->nr_huge_pages_node[page_to_nid(page)]--; 829 for (i = 0; i < pages_per_huge_page(h); i++) { 830 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 831 1 << PG_referenced | 1 << PG_dirty | 832 1 << PG_active | 1 << PG_private | 833 1 << PG_writeback); 834 } 835 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); 836 set_compound_page_dtor(page, NULL); 837 set_page_refcounted(page); 838 if (hstate_is_gigantic(h)) { 839 destroy_compound_gigantic_page(page, huge_page_order(h)); 840 free_gigantic_page(page, huge_page_order(h)); 841 } else { 842 arch_release_hugepage(page); 843 __free_pages(page, huge_page_order(h)); 844 } 845} 846 847struct hstate *size_to_hstate(unsigned long size) 848{ 849 struct hstate *h; 850 851 for_each_hstate(h) { 852 if (huge_page_size(h) == size) 853 return h; 854 } 855 return NULL; 856} 857 858void free_huge_page(struct page *page) 859{ 860 /* 861 * Can't pass hstate in here because it is called from the 862 * compound page destructor. 863 */ 864 struct hstate *h = page_hstate(page); 865 int nid = page_to_nid(page); 866 struct hugepage_subpool *spool = 867 (struct hugepage_subpool *)page_private(page); 868 bool restore_reserve; 869 870 set_page_private(page, 0); 871 page->mapping = NULL; 872 BUG_ON(page_count(page)); 873 BUG_ON(page_mapcount(page)); 874 restore_reserve = PagePrivate(page); 875 ClearPagePrivate(page); 876 877 spin_lock(&hugetlb_lock); 878 hugetlb_cgroup_uncharge_page(hstate_index(h), 879 pages_per_huge_page(h), page); 880 if (restore_reserve) 881 h->resv_huge_pages++; 882 883 if (h->surplus_huge_pages_node[nid]) { 884 /* remove the page from active list */ 885 list_del(&page->lru); 886 update_and_free_page(h, page); 887 h->surplus_huge_pages--; 888 h->surplus_huge_pages_node[nid]--; 889 } else { 890 arch_clear_hugepage_flags(page); 891 enqueue_huge_page(h, page); 892 } 893 spin_unlock(&hugetlb_lock); 894 hugepage_subpool_put_pages(spool, 1); 895} 896 897static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) 898{ 899 INIT_LIST_HEAD(&page->lru); 900 set_compound_page_dtor(page, free_huge_page); 901 spin_lock(&hugetlb_lock); 902 set_hugetlb_cgroup(page, NULL); 903 h->nr_huge_pages++; 904 h->nr_huge_pages_node[nid]++; 905 spin_unlock(&hugetlb_lock); 906 put_page(page); /* free it into the hugepage allocator */ 907} 908 909static void prep_compound_gigantic_page(struct page *page, unsigned long order) 910{ 911 int i; 912 int nr_pages = 1 << order; 913 struct page *p = page + 1; 914 915 /* we rely on prep_new_huge_page to set the destructor */ 916 set_compound_order(page, order); 917 __SetPageHead(page); 918 __ClearPageReserved(page); 919 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 920 __SetPageTail(p); 921 /* 922 * For gigantic hugepages allocated through bootmem at 923 * boot, it's safer to be consistent with the not-gigantic 924 * hugepages and clear the PG_reserved bit from all tail pages 925 * too. Otherwse drivers using get_user_pages() to access tail 926 * pages may get the reference counting wrong if they see 927 * PG_reserved set on a tail page (despite the head page not 928 * having PG_reserved set). Enforcing this consistency between 929 * head and tail pages allows drivers to optimize away a check 930 * on the head page when they need know if put_page() is needed 931 * after get_user_pages(). 932 */ 933 __ClearPageReserved(p); 934 set_page_count(p, 0); 935 p->first_page = page; 936 } 937} 938 939/* 940 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 941 * transparent huge pages. See the PageTransHuge() documentation for more 942 * details. 943 */ 944int PageHuge(struct page *page) 945{ 946 if (!PageCompound(page)) 947 return 0; 948 949 page = compound_head(page); 950 return get_compound_page_dtor(page) == free_huge_page; 951} 952EXPORT_SYMBOL_GPL(PageHuge); 953 954/* 955 * PageHeadHuge() only returns true for hugetlbfs head page, but not for 956 * normal or transparent huge pages. 957 */ 958int PageHeadHuge(struct page *page_head) 959{ 960 if (!PageHead(page_head)) 961 return 0; 962 963 return get_compound_page_dtor(page_head) == free_huge_page; 964} 965 966pgoff_t __basepage_index(struct page *page) 967{ 968 struct page *page_head = compound_head(page); 969 pgoff_t index = page_index(page_head); 970 unsigned long compound_idx; 971 972 if (!PageHuge(page_head)) 973 return page_index(page); 974 975 if (compound_order(page_head) >= MAX_ORDER) 976 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 977 else 978 compound_idx = page - page_head; 979 980 return (index << compound_order(page_head)) + compound_idx; 981} 982 983static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) 984{ 985 struct page *page; 986 987 page = alloc_pages_exact_node(nid, 988 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE| 989 __GFP_REPEAT|__GFP_NOWARN, 990 huge_page_order(h)); 991 if (page) { 992 if (arch_prepare_hugepage(page)) { 993 __free_pages(page, huge_page_order(h)); 994 return NULL; 995 } 996 prep_new_huge_page(h, page, nid); 997 } 998 999 return page; 1000} 1001 1002static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed) 1003{ 1004 struct page *page; 1005 int nr_nodes, node; 1006 int ret = 0; 1007 1008 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1009 page = alloc_fresh_huge_page_node(h, node); 1010 if (page) { 1011 ret = 1; 1012 break; 1013 } 1014 } 1015 1016 if (ret) 1017 count_vm_event(HTLB_BUDDY_PGALLOC); 1018 else 1019 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1020 1021 return ret; 1022} 1023 1024/* 1025 * Free huge page from pool from next node to free. 1026 * Attempt to keep persistent huge pages more or less 1027 * balanced over allowed nodes. 1028 * Called with hugetlb_lock locked. 1029 */ 1030static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1031 bool acct_surplus) 1032{ 1033 int nr_nodes, node; 1034 int ret = 0; 1035 1036 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1037 /* 1038 * If we're returning unused surplus pages, only examine 1039 * nodes with surplus pages. 1040 */ 1041 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 1042 !list_empty(&h->hugepage_freelists[node])) { 1043 struct page *page = 1044 list_entry(h->hugepage_freelists[node].next, 1045 struct page, lru); 1046 list_del(&page->lru); 1047 h->free_huge_pages--; 1048 h->free_huge_pages_node[node]--; 1049 if (acct_surplus) { 1050 h->surplus_huge_pages--; 1051 h->surplus_huge_pages_node[node]--; 1052 } 1053 update_and_free_page(h, page); 1054 ret = 1; 1055 break; 1056 } 1057 } 1058 1059 return ret; 1060} 1061 1062/* 1063 * Dissolve a given free hugepage into free buddy pages. This function does 1064 * nothing for in-use (including surplus) hugepages. 1065 */ 1066static void dissolve_free_huge_page(struct page *page) 1067{ 1068 spin_lock(&hugetlb_lock); 1069 if (PageHuge(page) && !page_count(page)) { 1070 struct hstate *h = page_hstate(page); 1071 int nid = page_to_nid(page); 1072 list_del(&page->lru); 1073 h->free_huge_pages--; 1074 h->free_huge_pages_node[nid]--; 1075 update_and_free_page(h, page); 1076 } 1077 spin_unlock(&hugetlb_lock); 1078} 1079 1080/* 1081 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 1082 * make specified memory blocks removable from the system. 1083 * Note that start_pfn should aligned with (minimum) hugepage size. 1084 */ 1085void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 1086{ 1087 unsigned int order = 8 * sizeof(void *); 1088 unsigned long pfn; 1089 struct hstate *h; 1090 1091 /* Set scan step to minimum hugepage size */ 1092 for_each_hstate(h) 1093 if (order > huge_page_order(h)) 1094 order = huge_page_order(h); 1095 VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order)); 1096 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) 1097 dissolve_free_huge_page(pfn_to_page(pfn)); 1098} 1099 1100static struct page *alloc_buddy_huge_page(struct hstate *h, int nid) 1101{ 1102 struct page *page; 1103 unsigned int r_nid; 1104 1105 if (hstate_is_gigantic(h)) 1106 return NULL; 1107 1108 /* 1109 * Assume we will successfully allocate the surplus page to 1110 * prevent racing processes from causing the surplus to exceed 1111 * overcommit 1112 * 1113 * This however introduces a different race, where a process B 1114 * tries to grow the static hugepage pool while alloc_pages() is 1115 * called by process A. B will only examine the per-node 1116 * counters in determining if surplus huge pages can be 1117 * converted to normal huge pages in adjust_pool_surplus(). A 1118 * won't be able to increment the per-node counter, until the 1119 * lock is dropped by B, but B doesn't drop hugetlb_lock until 1120 * no more huge pages can be converted from surplus to normal 1121 * state (and doesn't try to convert again). Thus, we have a 1122 * case where a surplus huge page exists, the pool is grown, and 1123 * the surplus huge page still exists after, even though it 1124 * should just have been converted to a normal huge page. This 1125 * does not leak memory, though, as the hugepage will be freed 1126 * once it is out of use. It also does not allow the counters to 1127 * go out of whack in adjust_pool_surplus() as we don't modify 1128 * the node values until we've gotten the hugepage and only the 1129 * per-node value is checked there. 1130 */ 1131 spin_lock(&hugetlb_lock); 1132 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 1133 spin_unlock(&hugetlb_lock); 1134 return NULL; 1135 } else { 1136 h->nr_huge_pages++; 1137 h->surplus_huge_pages++; 1138 } 1139 spin_unlock(&hugetlb_lock); 1140 1141 if (nid == NUMA_NO_NODE) 1142 page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP| 1143 __GFP_REPEAT|__GFP_NOWARN, 1144 huge_page_order(h)); 1145 else 1146 page = alloc_pages_exact_node(nid, 1147 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE| 1148 __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h)); 1149 1150 if (page && arch_prepare_hugepage(page)) { 1151 __free_pages(page, huge_page_order(h)); 1152 page = NULL; 1153 } 1154 1155 spin_lock(&hugetlb_lock); 1156 if (page) { 1157 INIT_LIST_HEAD(&page->lru); 1158 r_nid = page_to_nid(page); 1159 set_compound_page_dtor(page, free_huge_page); 1160 set_hugetlb_cgroup(page, NULL); 1161 /* 1162 * We incremented the global counters already 1163 */ 1164 h->nr_huge_pages_node[r_nid]++; 1165 h->surplus_huge_pages_node[r_nid]++; 1166 __count_vm_event(HTLB_BUDDY_PGALLOC); 1167 } else { 1168 h->nr_huge_pages--; 1169 h->surplus_huge_pages--; 1170 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1171 } 1172 spin_unlock(&hugetlb_lock); 1173 1174 return page; 1175} 1176 1177/* 1178 * This allocation function is useful in the context where vma is irrelevant. 1179 * E.g. soft-offlining uses this function because it only cares physical 1180 * address of error page. 1181 */ 1182struct page *alloc_huge_page_node(struct hstate *h, int nid) 1183{ 1184 struct page *page = NULL; 1185 1186 spin_lock(&hugetlb_lock); 1187 if (h->free_huge_pages - h->resv_huge_pages > 0) 1188 page = dequeue_huge_page_node(h, nid); 1189 spin_unlock(&hugetlb_lock); 1190 1191 if (!page) 1192 page = alloc_buddy_huge_page(h, nid); 1193 1194 return page; 1195} 1196 1197/* 1198 * Increase the hugetlb pool such that it can accommodate a reservation 1199 * of size 'delta'. 1200 */ 1201static int gather_surplus_pages(struct hstate *h, int delta) 1202{ 1203 struct list_head surplus_list; 1204 struct page *page, *tmp; 1205 int ret, i; 1206 int needed, allocated; 1207 bool alloc_ok = true; 1208 1209 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 1210 if (needed <= 0) { 1211 h->resv_huge_pages += delta; 1212 return 0; 1213 } 1214 1215 allocated = 0; 1216 INIT_LIST_HEAD(&surplus_list); 1217 1218 ret = -ENOMEM; 1219retry: 1220 spin_unlock(&hugetlb_lock); 1221 for (i = 0; i < needed; i++) { 1222 page = alloc_buddy_huge_page(h, NUMA_NO_NODE); 1223 if (!page) { 1224 alloc_ok = false; 1225 break; 1226 } 1227 list_add(&page->lru, &surplus_list); 1228 } 1229 allocated += i; 1230 1231 /* 1232 * After retaking hugetlb_lock, we need to recalculate 'needed' 1233 * because either resv_huge_pages or free_huge_pages may have changed. 1234 */ 1235 spin_lock(&hugetlb_lock); 1236 needed = (h->resv_huge_pages + delta) - 1237 (h->free_huge_pages + allocated); 1238 if (needed > 0) { 1239 if (alloc_ok) 1240 goto retry; 1241 /* 1242 * We were not able to allocate enough pages to 1243 * satisfy the entire reservation so we free what 1244 * we've allocated so far. 1245 */ 1246 goto free; 1247 } 1248 /* 1249 * The surplus_list now contains _at_least_ the number of extra pages 1250 * needed to accommodate the reservation. Add the appropriate number 1251 * of pages to the hugetlb pool and free the extras back to the buddy 1252 * allocator. Commit the entire reservation here to prevent another 1253 * process from stealing the pages as they are added to the pool but 1254 * before they are reserved. 1255 */ 1256 needed += allocated; 1257 h->resv_huge_pages += delta; 1258 ret = 0; 1259 1260 /* Free the needed pages to the hugetlb pool */ 1261 list_for_each_entry_safe(page, tmp, &surplus_list, lru) { 1262 if ((--needed) < 0) 1263 break; 1264 /* 1265 * This page is now managed by the hugetlb allocator and has 1266 * no users -- drop the buddy allocator's reference. 1267 */ 1268 put_page_testzero(page); 1269 VM_BUG_ON_PAGE(page_count(page), page); 1270 enqueue_huge_page(h, page); 1271 } 1272free: 1273 spin_unlock(&hugetlb_lock); 1274 1275 /* Free unnecessary surplus pages to the buddy allocator */ 1276 list_for_each_entry_safe(page, tmp, &surplus_list, lru) 1277 put_page(page); 1278 spin_lock(&hugetlb_lock); 1279 1280 return ret; 1281} 1282 1283/* 1284 * When releasing a hugetlb pool reservation, any surplus pages that were 1285 * allocated to satisfy the reservation must be explicitly freed if they were 1286 * never used. 1287 * Called with hugetlb_lock held. 1288 */ 1289static void return_unused_surplus_pages(struct hstate *h, 1290 unsigned long unused_resv_pages) 1291{ 1292 unsigned long nr_pages; 1293 1294 /* Uncommit the reservation */ 1295 h->resv_huge_pages -= unused_resv_pages; 1296 1297 /* Cannot return gigantic pages currently */ 1298 if (hstate_is_gigantic(h)) 1299 return; 1300 1301 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 1302 1303 /* 1304 * We want to release as many surplus pages as possible, spread 1305 * evenly across all nodes with memory. Iterate across these nodes 1306 * until we can no longer free unreserved surplus pages. This occurs 1307 * when the nodes with surplus pages have no free pages. 1308 * free_pool_huge_page() will balance the the freed pages across the 1309 * on-line nodes with memory and will handle the hstate accounting. 1310 */ 1311 while (nr_pages--) { 1312 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1)) 1313 break; 1314 cond_resched_lock(&hugetlb_lock); 1315 } 1316} 1317 1318/* 1319 * Determine if the huge page at addr within the vma has an associated 1320 * reservation. Where it does not we will need to logically increase 1321 * reservation and actually increase subpool usage before an allocation 1322 * can occur. Where any new reservation would be required the 1323 * reservation change is prepared, but not committed. Once the page 1324 * has been allocated from the subpool and instantiated the change should 1325 * be committed via vma_commit_reservation. No action is required on 1326 * failure. 1327 */ 1328static long vma_needs_reservation(struct hstate *h, 1329 struct vm_area_struct *vma, unsigned long addr) 1330{ 1331 struct resv_map *resv; 1332 pgoff_t idx; 1333 long chg; 1334 1335 resv = vma_resv_map(vma); 1336 if (!resv) 1337 return 1; 1338 1339 idx = vma_hugecache_offset(h, vma, addr); 1340 chg = region_chg(resv, idx, idx + 1); 1341 1342 if (vma->vm_flags & VM_MAYSHARE) 1343 return chg; 1344 else 1345 return chg < 0 ? chg : 0; 1346} 1347static void vma_commit_reservation(struct hstate *h, 1348 struct vm_area_struct *vma, unsigned long addr) 1349{ 1350 struct resv_map *resv; 1351 pgoff_t idx; 1352 1353 resv = vma_resv_map(vma); 1354 if (!resv) 1355 return; 1356 1357 idx = vma_hugecache_offset(h, vma, addr); 1358 region_add(resv, idx, idx + 1); 1359} 1360 1361static struct page *alloc_huge_page(struct vm_area_struct *vma, 1362 unsigned long addr, int avoid_reserve) 1363{ 1364 struct hugepage_subpool *spool = subpool_vma(vma); 1365 struct hstate *h = hstate_vma(vma); 1366 struct page *page; 1367 long chg; 1368 int ret, idx; 1369 struct hugetlb_cgroup *h_cg; 1370 1371 idx = hstate_index(h); 1372 /* 1373 * Processes that did not create the mapping will have no 1374 * reserves and will not have accounted against subpool 1375 * limit. Check that the subpool limit can be made before 1376 * satisfying the allocation MAP_NORESERVE mappings may also 1377 * need pages and subpool limit allocated allocated if no reserve 1378 * mapping overlaps. 1379 */ 1380 chg = vma_needs_reservation(h, vma, addr); 1381 if (chg < 0) 1382 return ERR_PTR(-ENOMEM); 1383 if (chg || avoid_reserve) 1384 if (hugepage_subpool_get_pages(spool, 1)) 1385 return ERR_PTR(-ENOSPC); 1386 1387 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 1388 if (ret) 1389 goto out_subpool_put; 1390 1391 spin_lock(&hugetlb_lock); 1392 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg); 1393 if (!page) { 1394 spin_unlock(&hugetlb_lock); 1395 page = alloc_buddy_huge_page(h, NUMA_NO_NODE); 1396 if (!page) 1397 goto out_uncharge_cgroup; 1398 1399 spin_lock(&hugetlb_lock); 1400 list_move(&page->lru, &h->hugepage_activelist); 1401 /* Fall through */ 1402 } 1403 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); 1404 spin_unlock(&hugetlb_lock); 1405 1406 set_page_private(page, (unsigned long)spool); 1407 1408 vma_commit_reservation(h, vma, addr); 1409 return page; 1410 1411out_uncharge_cgroup: 1412 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 1413out_subpool_put: 1414 if (chg || avoid_reserve) 1415 hugepage_subpool_put_pages(spool, 1); 1416 return ERR_PTR(-ENOSPC); 1417} 1418 1419/* 1420 * alloc_huge_page()'s wrapper which simply returns the page if allocation 1421 * succeeds, otherwise NULL. This function is called from new_vma_page(), 1422 * where no ERR_VALUE is expected to be returned. 1423 */ 1424struct page *alloc_huge_page_noerr(struct vm_area_struct *vma, 1425 unsigned long addr, int avoid_reserve) 1426{ 1427 struct page *page = alloc_huge_page(vma, addr, avoid_reserve); 1428 if (IS_ERR(page)) 1429 page = NULL; 1430 return page; 1431} 1432 1433int __weak alloc_bootmem_huge_page(struct hstate *h) 1434{ 1435 struct huge_bootmem_page *m; 1436 int nr_nodes, node; 1437 1438 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 1439 void *addr; 1440 1441 addr = memblock_virt_alloc_try_nid_nopanic( 1442 huge_page_size(h), huge_page_size(h), 1443 0, BOOTMEM_ALLOC_ACCESSIBLE, node); 1444 if (addr) { 1445 /* 1446 * Use the beginning of the huge page to store the 1447 * huge_bootmem_page struct (until gather_bootmem 1448 * puts them into the mem_map). 1449 */ 1450 m = addr; 1451 goto found; 1452 } 1453 } 1454 return 0; 1455 1456found: 1457 BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1)); 1458 /* Put them into a private list first because mem_map is not up yet */ 1459 list_add(&m->list, &huge_boot_pages); 1460 m->hstate = h; 1461 return 1; 1462} 1463 1464static void __init prep_compound_huge_page(struct page *page, int order) 1465{ 1466 if (unlikely(order > (MAX_ORDER - 1))) 1467 prep_compound_gigantic_page(page, order); 1468 else 1469 prep_compound_page(page, order); 1470} 1471 1472/* Put bootmem huge pages into the standard lists after mem_map is up */ 1473static void __init gather_bootmem_prealloc(void) 1474{ 1475 struct huge_bootmem_page *m; 1476 1477 list_for_each_entry(m, &huge_boot_pages, list) { 1478 struct hstate *h = m->hstate; 1479 struct page *page; 1480 1481#ifdef CONFIG_HIGHMEM 1482 page = pfn_to_page(m->phys >> PAGE_SHIFT); 1483 memblock_free_late(__pa(m), 1484 sizeof(struct huge_bootmem_page)); 1485#else 1486 page = virt_to_page(m); 1487#endif 1488 WARN_ON(page_count(page) != 1); 1489 prep_compound_huge_page(page, h->order); 1490 WARN_ON(PageReserved(page)); 1491 prep_new_huge_page(h, page, page_to_nid(page)); 1492 /* 1493 * If we had gigantic hugepages allocated at boot time, we need 1494 * to restore the 'stolen' pages to totalram_pages in order to 1495 * fix confusing memory reports from free(1) and another 1496 * side-effects, like CommitLimit going negative. 1497 */ 1498 if (hstate_is_gigantic(h)) 1499 adjust_managed_page_count(page, 1 << h->order); 1500 } 1501} 1502 1503static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 1504{ 1505 unsigned long i; 1506 1507 for (i = 0; i < h->max_huge_pages; ++i) { 1508 if (hstate_is_gigantic(h)) { 1509 if (!alloc_bootmem_huge_page(h)) 1510 break; 1511 } else if (!alloc_fresh_huge_page(h, 1512 &node_states[N_MEMORY])) 1513 break; 1514 } 1515 h->max_huge_pages = i; 1516} 1517 1518static void __init hugetlb_init_hstates(void) 1519{ 1520 struct hstate *h; 1521 1522 for_each_hstate(h) { 1523 /* oversize hugepages were init'ed in early boot */ 1524 if (!hstate_is_gigantic(h)) 1525 hugetlb_hstate_alloc_pages(h); 1526 } 1527} 1528 1529static char * __init memfmt(char *buf, unsigned long n) 1530{ 1531 if (n >= (1UL << 30)) 1532 sprintf(buf, "%lu GB", n >> 30); 1533 else if (n >= (1UL << 20)) 1534 sprintf(buf, "%lu MB", n >> 20); 1535 else 1536 sprintf(buf, "%lu KB", n >> 10); 1537 return buf; 1538} 1539 1540static void __init report_hugepages(void) 1541{ 1542 struct hstate *h; 1543 1544 for_each_hstate(h) { 1545 char buf[32]; 1546 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", 1547 memfmt(buf, huge_page_size(h)), 1548 h->free_huge_pages); 1549 } 1550} 1551 1552#ifdef CONFIG_HIGHMEM 1553static void try_to_free_low(struct hstate *h, unsigned long count, 1554 nodemask_t *nodes_allowed) 1555{ 1556 int i; 1557 1558 if (hstate_is_gigantic(h)) 1559 return; 1560 1561 for_each_node_mask(i, *nodes_allowed) { 1562 struct page *page, *next; 1563 struct list_head *freel = &h->hugepage_freelists[i]; 1564 list_for_each_entry_safe(page, next, freel, lru) { 1565 if (count >= h->nr_huge_pages) 1566 return; 1567 if (PageHighMem(page)) 1568 continue; 1569 list_del(&page->lru); 1570 update_and_free_page(h, page); 1571 h->free_huge_pages--; 1572 h->free_huge_pages_node[page_to_nid(page)]--; 1573 } 1574 } 1575} 1576#else 1577static inline void try_to_free_low(struct hstate *h, unsigned long count, 1578 nodemask_t *nodes_allowed) 1579{ 1580} 1581#endif 1582 1583/* 1584 * Increment or decrement surplus_huge_pages. Keep node-specific counters 1585 * balanced by operating on them in a round-robin fashion. 1586 * Returns 1 if an adjustment was made. 1587 */ 1588static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 1589 int delta) 1590{ 1591 int nr_nodes, node; 1592 1593 VM_BUG_ON(delta != -1 && delta != 1); 1594 1595 if (delta < 0) { 1596 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1597 if (h->surplus_huge_pages_node[node]) 1598 goto found; 1599 } 1600 } else { 1601 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1602 if (h->surplus_huge_pages_node[node] < 1603 h->nr_huge_pages_node[node]) 1604 goto found; 1605 } 1606 } 1607 return 0; 1608 1609found: 1610 h->surplus_huge_pages += delta; 1611 h->surplus_huge_pages_node[node] += delta; 1612 return 1; 1613} 1614 1615#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 1616static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, 1617 nodemask_t *nodes_allowed) 1618{ 1619 unsigned long min_count, ret; 1620 1621 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 1622 return h->max_huge_pages; 1623 1624 /* 1625 * Increase the pool size 1626 * First take pages out of surplus state. Then make up the 1627 * remaining difference by allocating fresh huge pages. 1628 * 1629 * We might race with alloc_buddy_huge_page() here and be unable 1630 * to convert a surplus huge page to a normal huge page. That is 1631 * not critical, though, it just means the overall size of the 1632 * pool might be one hugepage larger than it needs to be, but 1633 * within all the constraints specified by the sysctls. 1634 */ 1635 spin_lock(&hugetlb_lock); 1636 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 1637 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 1638 break; 1639 } 1640 1641 while (count > persistent_huge_pages(h)) { 1642 /* 1643 * If this allocation races such that we no longer need the 1644 * page, free_huge_page will handle it by freeing the page 1645 * and reducing the surplus. 1646 */ 1647 spin_unlock(&hugetlb_lock); 1648 if (hstate_is_gigantic(h)) 1649 ret = alloc_fresh_gigantic_page(h, nodes_allowed); 1650 else 1651 ret = alloc_fresh_huge_page(h, nodes_allowed); 1652 spin_lock(&hugetlb_lock); 1653 if (!ret) 1654 goto out; 1655 1656 /* Bail for signals. Probably ctrl-c from user */ 1657 if (signal_pending(current)) 1658 goto out; 1659 } 1660 1661 /* 1662 * Decrease the pool size 1663 * First return free pages to the buddy allocator (being careful 1664 * to keep enough around to satisfy reservations). Then place 1665 * pages into surplus state as needed so the pool will shrink 1666 * to the desired size as pages become free. 1667 * 1668 * By placing pages into the surplus state independent of the 1669 * overcommit value, we are allowing the surplus pool size to 1670 * exceed overcommit. There are few sane options here. Since 1671 * alloc_buddy_huge_page() is checking the global counter, 1672 * though, we'll note that we're not allowed to exceed surplus 1673 * and won't grow the pool anywhere else. Not until one of the 1674 * sysctls are changed, or the surplus pages go out of use. 1675 */ 1676 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 1677 min_count = max(count, min_count); 1678 try_to_free_low(h, min_count, nodes_allowed); 1679 while (min_count < persistent_huge_pages(h)) { 1680 if (!free_pool_huge_page(h, nodes_allowed, 0)) 1681 break; 1682 cond_resched_lock(&hugetlb_lock); 1683 } 1684 while (count < persistent_huge_pages(h)) { 1685 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 1686 break; 1687 } 1688out: 1689 ret = persistent_huge_pages(h); 1690 spin_unlock(&hugetlb_lock); 1691 return ret; 1692} 1693 1694#define HSTATE_ATTR_RO(_name) \ 1695 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 1696 1697#define HSTATE_ATTR(_name) \ 1698 static struct kobj_attribute _name##_attr = \ 1699 __ATTR(_name, 0644, _name##_show, _name##_store) 1700 1701static struct kobject *hugepages_kobj; 1702static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 1703 1704static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 1705 1706static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 1707{ 1708 int i; 1709 1710 for (i = 0; i < HUGE_MAX_HSTATE; i++) 1711 if (hstate_kobjs[i] == kobj) { 1712 if (nidp) 1713 *nidp = NUMA_NO_NODE; 1714 return &hstates[i]; 1715 } 1716 1717 return kobj_to_node_hstate(kobj, nidp); 1718} 1719 1720static ssize_t nr_hugepages_show_common(struct kobject *kobj, 1721 struct kobj_attribute *attr, char *buf) 1722{ 1723 struct hstate *h; 1724 unsigned long nr_huge_pages; 1725 int nid; 1726 1727 h = kobj_to_hstate(kobj, &nid); 1728 if (nid == NUMA_NO_NODE) 1729 nr_huge_pages = h->nr_huge_pages; 1730 else 1731 nr_huge_pages = h->nr_huge_pages_node[nid]; 1732 1733 return sprintf(buf, "%lu\n", nr_huge_pages); 1734} 1735 1736static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 1737 struct hstate *h, int nid, 1738 unsigned long count, size_t len) 1739{ 1740 int err; 1741 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY); 1742 1743 if (hstate_is_gigantic(h) && !gigantic_page_supported()) { 1744 err = -EINVAL; 1745 goto out; 1746 } 1747 1748 if (nid == NUMA_NO_NODE) { 1749 /* 1750 * global hstate attribute 1751 */ 1752 if (!(obey_mempolicy && 1753 init_nodemask_of_mempolicy(nodes_allowed))) { 1754 NODEMASK_FREE(nodes_allowed); 1755 nodes_allowed = &node_states[N_MEMORY]; 1756 } 1757 } else if (nodes_allowed) { 1758 /* 1759 * per node hstate attribute: adjust count to global, 1760 * but restrict alloc/free to the specified node. 1761 */ 1762 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 1763 init_nodemask_of_node(nodes_allowed, nid); 1764 } else 1765 nodes_allowed = &node_states[N_MEMORY]; 1766 1767 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed); 1768 1769 if (nodes_allowed != &node_states[N_MEMORY]) 1770 NODEMASK_FREE(nodes_allowed); 1771 1772 return len; 1773out: 1774 NODEMASK_FREE(nodes_allowed); 1775 return err; 1776} 1777 1778static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 1779 struct kobject *kobj, const char *buf, 1780 size_t len) 1781{ 1782 struct hstate *h; 1783 unsigned long count; 1784 int nid; 1785 int err; 1786 1787 err = kstrtoul(buf, 10, &count); 1788 if (err) 1789 return err; 1790 1791 h = kobj_to_hstate(kobj, &nid); 1792 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 1793} 1794 1795static ssize_t nr_hugepages_show(struct kobject *kobj, 1796 struct kobj_attribute *attr, char *buf) 1797{ 1798 return nr_hugepages_show_common(kobj, attr, buf); 1799} 1800 1801static ssize_t nr_hugepages_store(struct kobject *kobj, 1802 struct kobj_attribute *attr, const char *buf, size_t len) 1803{ 1804 return nr_hugepages_store_common(false, kobj, buf, len); 1805} 1806HSTATE_ATTR(nr_hugepages); 1807 1808#ifdef CONFIG_NUMA 1809 1810/* 1811 * hstate attribute for optionally mempolicy-based constraint on persistent 1812 * huge page alloc/free. 1813 */ 1814static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 1815 struct kobj_attribute *attr, char *buf) 1816{ 1817 return nr_hugepages_show_common(kobj, attr, buf); 1818} 1819 1820static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 1821 struct kobj_attribute *attr, const char *buf, size_t len) 1822{ 1823 return nr_hugepages_store_common(true, kobj, buf, len); 1824} 1825HSTATE_ATTR(nr_hugepages_mempolicy); 1826#endif 1827 1828 1829static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 1830 struct kobj_attribute *attr, char *buf) 1831{ 1832 struct hstate *h = kobj_to_hstate(kobj, NULL); 1833 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); 1834} 1835 1836static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 1837 struct kobj_attribute *attr, const char *buf, size_t count) 1838{ 1839 int err; 1840 unsigned long input; 1841 struct hstate *h = kobj_to_hstate(kobj, NULL); 1842 1843 if (hstate_is_gigantic(h)) 1844 return -EINVAL; 1845 1846 err = kstrtoul(buf, 10, &input); 1847 if (err) 1848 return err; 1849 1850 spin_lock(&hugetlb_lock); 1851 h->nr_overcommit_huge_pages = input; 1852 spin_unlock(&hugetlb_lock); 1853 1854 return count; 1855} 1856HSTATE_ATTR(nr_overcommit_hugepages); 1857 1858static ssize_t free_hugepages_show(struct kobject *kobj, 1859 struct kobj_attribute *attr, char *buf) 1860{ 1861 struct hstate *h; 1862 unsigned long free_huge_pages; 1863 int nid; 1864 1865 h = kobj_to_hstate(kobj, &nid); 1866 if (nid == NUMA_NO_NODE) 1867 free_huge_pages = h->free_huge_pages; 1868 else 1869 free_huge_pages = h->free_huge_pages_node[nid]; 1870 1871 return sprintf(buf, "%lu\n", free_huge_pages); 1872} 1873HSTATE_ATTR_RO(free_hugepages); 1874 1875static ssize_t resv_hugepages_show(struct kobject *kobj, 1876 struct kobj_attribute *attr, char *buf) 1877{ 1878 struct hstate *h = kobj_to_hstate(kobj, NULL); 1879 return sprintf(buf, "%lu\n", h->resv_huge_pages); 1880} 1881HSTATE_ATTR_RO(resv_hugepages); 1882 1883static ssize_t surplus_hugepages_show(struct kobject *kobj, 1884 struct kobj_attribute *attr, char *buf) 1885{ 1886 struct hstate *h; 1887 unsigned long surplus_huge_pages; 1888 int nid; 1889 1890 h = kobj_to_hstate(kobj, &nid); 1891 if (nid == NUMA_NO_NODE) 1892 surplus_huge_pages = h->surplus_huge_pages; 1893 else 1894 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 1895 1896 return sprintf(buf, "%lu\n", surplus_huge_pages); 1897} 1898HSTATE_ATTR_RO(surplus_hugepages); 1899 1900static struct attribute *hstate_attrs[] = { 1901 &nr_hugepages_attr.attr, 1902 &nr_overcommit_hugepages_attr.attr, 1903 &free_hugepages_attr.attr, 1904 &resv_hugepages_attr.attr, 1905 &surplus_hugepages_attr.attr, 1906#ifdef CONFIG_NUMA 1907 &nr_hugepages_mempolicy_attr.attr, 1908#endif 1909 NULL, 1910}; 1911 1912static struct attribute_group hstate_attr_group = { 1913 .attrs = hstate_attrs, 1914}; 1915 1916static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 1917 struct kobject **hstate_kobjs, 1918 struct attribute_group *hstate_attr_group) 1919{ 1920 int retval; 1921 int hi = hstate_index(h); 1922 1923 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 1924 if (!hstate_kobjs[hi]) 1925 return -ENOMEM; 1926 1927 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 1928 if (retval) 1929 kobject_put(hstate_kobjs[hi]); 1930 1931 return retval; 1932} 1933 1934static void __init hugetlb_sysfs_init(void) 1935{ 1936 struct hstate *h; 1937 int err; 1938 1939 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 1940 if (!hugepages_kobj) 1941 return; 1942 1943 for_each_hstate(h) { 1944 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 1945 hstate_kobjs, &hstate_attr_group); 1946 if (err) 1947 pr_err("Hugetlb: Unable to add hstate %s", h->name); 1948 } 1949} 1950 1951#ifdef CONFIG_NUMA 1952 1953/* 1954 * node_hstate/s - associate per node hstate attributes, via their kobjects, 1955 * with node devices in node_devices[] using a parallel array. The array 1956 * index of a node device or _hstate == node id. 1957 * This is here to avoid any static dependency of the node device driver, in 1958 * the base kernel, on the hugetlb module. 1959 */ 1960struct node_hstate { 1961 struct kobject *hugepages_kobj; 1962 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 1963}; 1964struct node_hstate node_hstates[MAX_NUMNODES]; 1965 1966/* 1967 * A subset of global hstate attributes for node devices 1968 */ 1969static struct attribute *per_node_hstate_attrs[] = { 1970 &nr_hugepages_attr.attr, 1971 &free_hugepages_attr.attr, 1972 &surplus_hugepages_attr.attr, 1973 NULL, 1974}; 1975 1976static struct attribute_group per_node_hstate_attr_group = { 1977 .attrs = per_node_hstate_attrs, 1978}; 1979 1980/* 1981 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 1982 * Returns node id via non-NULL nidp. 1983 */ 1984static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 1985{ 1986 int nid; 1987 1988 for (nid = 0; nid < nr_node_ids; nid++) { 1989 struct node_hstate *nhs = &node_hstates[nid]; 1990 int i; 1991 for (i = 0; i < HUGE_MAX_HSTATE; i++) 1992 if (nhs->hstate_kobjs[i] == kobj) { 1993 if (nidp) 1994 *nidp = nid; 1995 return &hstates[i]; 1996 } 1997 } 1998 1999 BUG(); 2000 return NULL; 2001} 2002 2003/* 2004 * Unregister hstate attributes from a single node device. 2005 * No-op if no hstate attributes attached. 2006 */ 2007static void hugetlb_unregister_node(struct node *node) 2008{ 2009 struct hstate *h; 2010 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2011 2012 if (!nhs->hugepages_kobj) 2013 return; /* no hstate attributes */ 2014 2015 for_each_hstate(h) { 2016 int idx = hstate_index(h); 2017 if (nhs->hstate_kobjs[idx]) { 2018 kobject_put(nhs->hstate_kobjs[idx]); 2019 nhs->hstate_kobjs[idx] = NULL; 2020 } 2021 } 2022 2023 kobject_put(nhs->hugepages_kobj); 2024 nhs->hugepages_kobj = NULL; 2025} 2026 2027/* 2028 * hugetlb module exit: unregister hstate attributes from node devices 2029 * that have them. 2030 */ 2031static void hugetlb_unregister_all_nodes(void) 2032{ 2033 int nid; 2034 2035 /* 2036 * disable node device registrations. 2037 */ 2038 register_hugetlbfs_with_node(NULL, NULL); 2039 2040 /* 2041 * remove hstate attributes from any nodes that have them. 2042 */ 2043 for (nid = 0; nid < nr_node_ids; nid++) 2044 hugetlb_unregister_node(node_devices[nid]); 2045} 2046 2047/* 2048 * Register hstate attributes for a single node device. 2049 * No-op if attributes already registered. 2050 */ 2051static void hugetlb_register_node(struct node *node) 2052{ 2053 struct hstate *h; 2054 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2055 int err; 2056 2057 if (nhs->hugepages_kobj) 2058 return; /* already allocated */ 2059 2060 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 2061 &node->dev.kobj); 2062 if (!nhs->hugepages_kobj) 2063 return; 2064 2065 for_each_hstate(h) { 2066 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 2067 nhs->hstate_kobjs, 2068 &per_node_hstate_attr_group); 2069 if (err) { 2070 pr_err("Hugetlb: Unable to add hstate %s for node %d\n", 2071 h->name, node->dev.id); 2072 hugetlb_unregister_node(node); 2073 break; 2074 } 2075 } 2076} 2077 2078/* 2079 * hugetlb init time: register hstate attributes for all registered node 2080 * devices of nodes that have memory. All on-line nodes should have 2081 * registered their associated device by this time. 2082 */ 2083static void hugetlb_register_all_nodes(void) 2084{ 2085 int nid; 2086 2087 for_each_node_state(nid, N_MEMORY) { 2088 struct node *node = node_devices[nid]; 2089 if (node->dev.id == nid) 2090 hugetlb_register_node(node); 2091 } 2092 2093 /* 2094 * Let the node device driver know we're here so it can 2095 * [un]register hstate attributes on node hotplug. 2096 */ 2097 register_hugetlbfs_with_node(hugetlb_register_node, 2098 hugetlb_unregister_node); 2099} 2100#else /* !CONFIG_NUMA */ 2101 2102static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2103{ 2104 BUG(); 2105 if (nidp) 2106 *nidp = -1; 2107 return NULL; 2108} 2109 2110static void hugetlb_unregister_all_nodes(void) { } 2111 2112static void hugetlb_register_all_nodes(void) { } 2113 2114#endif 2115 2116static void __exit hugetlb_exit(void) 2117{ 2118 struct hstate *h; 2119 2120 hugetlb_unregister_all_nodes(); 2121 2122 for_each_hstate(h) { 2123 kobject_put(hstate_kobjs[hstate_index(h)]); 2124 } 2125 2126 kobject_put(hugepages_kobj); 2127 kfree(htlb_fault_mutex_table); 2128} 2129module_exit(hugetlb_exit); 2130 2131static int __init hugetlb_init(void) 2132{ 2133 int i; 2134 2135 if (!hugepages_supported()) 2136 return 0; 2137 2138 if (!size_to_hstate(default_hstate_size)) { 2139 default_hstate_size = HPAGE_SIZE; 2140 if (!size_to_hstate(default_hstate_size)) 2141 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 2142 } 2143 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size)); 2144 if (default_hstate_max_huge_pages) 2145 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 2146 2147 hugetlb_init_hstates(); 2148 gather_bootmem_prealloc(); 2149 report_hugepages(); 2150 2151 hugetlb_sysfs_init(); 2152 hugetlb_register_all_nodes(); 2153 hugetlb_cgroup_file_init(); 2154 2155#ifdef CONFIG_SMP 2156 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 2157#else 2158 num_fault_mutexes = 1; 2159#endif 2160 htlb_fault_mutex_table = 2161 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL); 2162 BUG_ON(!htlb_fault_mutex_table); 2163 2164 for (i = 0; i < num_fault_mutexes; i++) 2165 mutex_init(&htlb_fault_mutex_table[i]); 2166 return 0; 2167} 2168module_init(hugetlb_init); 2169 2170/* Should be called on processing a hugepagesz=... option */ 2171void __init hugetlb_add_hstate(unsigned order) 2172{ 2173 struct hstate *h; 2174 unsigned long i; 2175 2176 if (size_to_hstate(PAGE_SIZE << order)) { 2177 pr_warning("hugepagesz= specified twice, ignoring\n"); 2178 return; 2179 } 2180 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 2181 BUG_ON(order == 0); 2182 h = &hstates[hugetlb_max_hstate++]; 2183 h->order = order; 2184 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); 2185 h->nr_huge_pages = 0; 2186 h->free_huge_pages = 0; 2187 for (i = 0; i < MAX_NUMNODES; ++i) 2188 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 2189 INIT_LIST_HEAD(&h->hugepage_activelist); 2190 h->next_nid_to_alloc = first_node(node_states[N_MEMORY]); 2191 h->next_nid_to_free = first_node(node_states[N_MEMORY]); 2192 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 2193 huge_page_size(h)/1024); 2194 2195 parsed_hstate = h; 2196} 2197 2198static int __init hugetlb_nrpages_setup(char *s) 2199{ 2200 unsigned long *mhp; 2201 static unsigned long *last_mhp; 2202 2203 /* 2204 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet, 2205 * so this hugepages= parameter goes to the "default hstate". 2206 */ 2207 if (!hugetlb_max_hstate) 2208 mhp = &default_hstate_max_huge_pages; 2209 else 2210 mhp = &parsed_hstate->max_huge_pages; 2211 2212 if (mhp == last_mhp) { 2213 pr_warning("hugepages= specified twice without " 2214 "interleaving hugepagesz=, ignoring\n"); 2215 return 1; 2216 } 2217 2218 if (sscanf(s, "%lu", mhp) <= 0) 2219 *mhp = 0; 2220 2221 /* 2222 * Global state is always initialized later in hugetlb_init. 2223 * But we need to allocate >= MAX_ORDER hstates here early to still 2224 * use the bootmem allocator. 2225 */ 2226 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) 2227 hugetlb_hstate_alloc_pages(parsed_hstate); 2228 2229 last_mhp = mhp; 2230 2231 return 1; 2232} 2233__setup("hugepages=", hugetlb_nrpages_setup); 2234 2235static int __init hugetlb_default_setup(char *s) 2236{ 2237 default_hstate_size = memparse(s, &s); 2238 return 1; 2239} 2240__setup("default_hugepagesz=", hugetlb_default_setup); 2241 2242static unsigned int cpuset_mems_nr(unsigned int *array) 2243{ 2244 int node; 2245 unsigned int nr = 0; 2246 2247 for_each_node_mask(node, cpuset_current_mems_allowed) 2248 nr += array[node]; 2249 2250 return nr; 2251} 2252 2253#ifdef CONFIG_SYSCTL 2254static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 2255 struct ctl_table *table, int write, 2256 void __user *buffer, size_t *length, loff_t *ppos) 2257{ 2258 struct hstate *h = &default_hstate; 2259 unsigned long tmp = h->max_huge_pages; 2260 int ret; 2261 2262 if (!hugepages_supported()) 2263 return -ENOTSUPP; 2264 2265 table->data = &tmp; 2266 table->maxlen = sizeof(unsigned long); 2267 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2268 if (ret) 2269 goto out; 2270 2271 if (write) 2272 ret = __nr_hugepages_store_common(obey_mempolicy, h, 2273 NUMA_NO_NODE, tmp, *length); 2274out: 2275 return ret; 2276} 2277 2278int hugetlb_sysctl_handler(struct ctl_table *table, int write, 2279 void __user *buffer, size_t *length, loff_t *ppos) 2280{ 2281 2282 return hugetlb_sysctl_handler_common(false, table, write, 2283 buffer, length, ppos); 2284} 2285 2286#ifdef CONFIG_NUMA 2287int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 2288 void __user *buffer, size_t *length, loff_t *ppos) 2289{ 2290 return hugetlb_sysctl_handler_common(true, table, write, 2291 buffer, length, ppos); 2292} 2293#endif /* CONFIG_NUMA */ 2294 2295int hugetlb_overcommit_handler(struct ctl_table *table, int write, 2296 void __user *buffer, 2297 size_t *length, loff_t *ppos) 2298{ 2299 struct hstate *h = &default_hstate; 2300 unsigned long tmp; 2301 int ret; 2302 2303 if (!hugepages_supported()) 2304 return -ENOTSUPP; 2305 2306 tmp = h->nr_overcommit_huge_pages; 2307 2308 if (write && hstate_is_gigantic(h)) 2309 return -EINVAL; 2310 2311 table->data = &tmp; 2312 table->maxlen = sizeof(unsigned long); 2313 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2314 if (ret) 2315 goto out; 2316 2317 if (write) { 2318 spin_lock(&hugetlb_lock); 2319 h->nr_overcommit_huge_pages = tmp; 2320 spin_unlock(&hugetlb_lock); 2321 } 2322out: 2323 return ret; 2324} 2325 2326#endif /* CONFIG_SYSCTL */ 2327 2328void hugetlb_report_meminfo(struct seq_file *m) 2329{ 2330 struct hstate *h = &default_hstate; 2331 if (!hugepages_supported()) 2332 return; 2333 seq_printf(m, 2334 "HugePages_Total: %5lu\n" 2335 "HugePages_Free: %5lu\n" 2336 "HugePages_Rsvd: %5lu\n" 2337 "HugePages_Surp: %5lu\n" 2338 "Hugepagesize: %8lu kB\n", 2339 h->nr_huge_pages, 2340 h->free_huge_pages, 2341 h->resv_huge_pages, 2342 h->surplus_huge_pages, 2343 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 2344} 2345 2346int hugetlb_report_node_meminfo(int nid, char *buf) 2347{ 2348 struct hstate *h = &default_hstate; 2349 if (!hugepages_supported()) 2350 return 0; 2351 return sprintf(buf, 2352 "Node %d HugePages_Total: %5u\n" 2353 "Node %d HugePages_Free: %5u\n" 2354 "Node %d HugePages_Surp: %5u\n", 2355 nid, h->nr_huge_pages_node[nid], 2356 nid, h->free_huge_pages_node[nid], 2357 nid, h->surplus_huge_pages_node[nid]); 2358} 2359 2360void hugetlb_show_meminfo(void) 2361{ 2362 struct hstate *h; 2363 int nid; 2364 2365 if (!hugepages_supported()) 2366 return; 2367 2368 for_each_node_state(nid, N_MEMORY) 2369 for_each_hstate(h) 2370 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 2371 nid, 2372 h->nr_huge_pages_node[nid], 2373 h->free_huge_pages_node[nid], 2374 h->surplus_huge_pages_node[nid], 2375 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 2376} 2377 2378/* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 2379unsigned long hugetlb_total_pages(void) 2380{ 2381 struct hstate *h; 2382 unsigned long nr_total_pages = 0; 2383 2384 for_each_hstate(h) 2385 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 2386 return nr_total_pages; 2387} 2388 2389static int hugetlb_acct_memory(struct hstate *h, long delta) 2390{ 2391 int ret = -ENOMEM; 2392 2393 spin_lock(&hugetlb_lock); 2394 /* 2395 * When cpuset is configured, it breaks the strict hugetlb page 2396 * reservation as the accounting is done on a global variable. Such 2397 * reservation is completely rubbish in the presence of cpuset because 2398 * the reservation is not checked against page availability for the 2399 * current cpuset. Application can still potentially OOM'ed by kernel 2400 * with lack of free htlb page in cpuset that the task is in. 2401 * Attempt to enforce strict accounting with cpuset is almost 2402 * impossible (or too ugly) because cpuset is too fluid that 2403 * task or memory node can be dynamically moved between cpusets. 2404 * 2405 * The change of semantics for shared hugetlb mapping with cpuset is 2406 * undesirable. However, in order to preserve some of the semantics, 2407 * we fall back to check against current free page availability as 2408 * a best attempt and hopefully to minimize the impact of changing 2409 * semantics that cpuset has. 2410 */ 2411 if (delta > 0) { 2412 if (gather_surplus_pages(h, delta) < 0) 2413 goto out; 2414 2415 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) { 2416 return_unused_surplus_pages(h, delta); 2417 goto out; 2418 } 2419 } 2420 2421 ret = 0; 2422 if (delta < 0) 2423 return_unused_surplus_pages(h, (unsigned long) -delta); 2424 2425out: 2426 spin_unlock(&hugetlb_lock); 2427 return ret; 2428} 2429 2430static void hugetlb_vm_op_open(struct vm_area_struct *vma) 2431{ 2432 struct resv_map *resv = vma_resv_map(vma); 2433 2434 /* 2435 * This new VMA should share its siblings reservation map if present. 2436 * The VMA will only ever have a valid reservation map pointer where 2437 * it is being copied for another still existing VMA. As that VMA 2438 * has a reference to the reservation map it cannot disappear until 2439 * after this open call completes. It is therefore safe to take a 2440 * new reference here without additional locking. 2441 */ 2442 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 2443 kref_get(&resv->refs); 2444} 2445 2446static void hugetlb_vm_op_close(struct vm_area_struct *vma) 2447{ 2448 struct hstate *h = hstate_vma(vma); 2449 struct resv_map *resv = vma_resv_map(vma); 2450 struct hugepage_subpool *spool = subpool_vma(vma); 2451 unsigned long reserve, start, end; 2452 2453 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 2454 return; 2455 2456 start = vma_hugecache_offset(h, vma, vma->vm_start); 2457 end = vma_hugecache_offset(h, vma, vma->vm_end); 2458 2459 reserve = (end - start) - region_count(resv, start, end); 2460 2461 kref_put(&resv->refs, resv_map_release); 2462 2463 if (reserve) { 2464 hugetlb_acct_memory(h, -reserve); 2465 hugepage_subpool_put_pages(spool, reserve); 2466 } 2467} 2468 2469/* 2470 * We cannot handle pagefaults against hugetlb pages at all. They cause 2471 * handle_mm_fault() to try to instantiate regular-sized pages in the 2472 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 2473 * this far. 2474 */ 2475static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2476{ 2477 BUG(); 2478 return 0; 2479} 2480 2481const struct vm_operations_struct hugetlb_vm_ops = { 2482 .fault = hugetlb_vm_op_fault, 2483 .open = hugetlb_vm_op_open, 2484 .close = hugetlb_vm_op_close, 2485}; 2486 2487static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 2488 int writable) 2489{ 2490 pte_t entry; 2491 2492 if (writable) { 2493 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 2494 vma->vm_page_prot))); 2495 } else { 2496 entry = huge_pte_wrprotect(mk_huge_pte(page, 2497 vma->vm_page_prot)); 2498 } 2499 entry = pte_mkyoung(entry); 2500 entry = pte_mkhuge(entry); 2501 entry = arch_make_huge_pte(entry, vma, page, writable); 2502 2503 return entry; 2504} 2505 2506static void set_huge_ptep_writable(struct vm_area_struct *vma, 2507 unsigned long address, pte_t *ptep) 2508{ 2509 pte_t entry; 2510 2511 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 2512 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 2513 update_mmu_cache(vma, address, ptep); 2514} 2515 2516static int is_hugetlb_entry_migration(pte_t pte) 2517{ 2518 swp_entry_t swp; 2519 2520 if (huge_pte_none(pte) || pte_present(pte)) 2521 return 0; 2522 swp = pte_to_swp_entry(pte); 2523 if (non_swap_entry(swp) && is_migration_entry(swp)) 2524 return 1; 2525 else 2526 return 0; 2527} 2528 2529static int is_hugetlb_entry_hwpoisoned(pte_t pte) 2530{ 2531 swp_entry_t swp; 2532 2533 if (huge_pte_none(pte) || pte_present(pte)) 2534 return 0; 2535 swp = pte_to_swp_entry(pte); 2536 if (non_swap_entry(swp) && is_hwpoison_entry(swp)) 2537 return 1; 2538 else 2539 return 0; 2540} 2541 2542int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 2543 struct vm_area_struct *vma) 2544{ 2545 pte_t *src_pte, *dst_pte, entry; 2546 struct page *ptepage; 2547 unsigned long addr; 2548 int cow; 2549 struct hstate *h = hstate_vma(vma); 2550 unsigned long sz = huge_page_size(h); 2551 unsigned long mmun_start; /* For mmu_notifiers */ 2552 unsigned long mmun_end; /* For mmu_notifiers */ 2553 int ret = 0; 2554 2555 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 2556 2557 mmun_start = vma->vm_start; 2558 mmun_end = vma->vm_end; 2559 if (cow) 2560 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end); 2561 2562 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { 2563 spinlock_t *src_ptl, *dst_ptl; 2564 src_pte = huge_pte_offset(src, addr); 2565 if (!src_pte) 2566 continue; 2567 dst_pte = huge_pte_alloc(dst, addr, sz); 2568 if (!dst_pte) { 2569 ret = -ENOMEM; 2570 break; 2571 } 2572 2573 /* If the pagetables are shared don't copy or take references */ 2574 if (dst_pte == src_pte) 2575 continue; 2576 2577 dst_ptl = huge_pte_lock(h, dst, dst_pte); 2578 src_ptl = huge_pte_lockptr(h, src, src_pte); 2579 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 2580 entry = huge_ptep_get(src_pte); 2581 if (huge_pte_none(entry)) { /* skip none entry */ 2582 ; 2583 } else if (unlikely(is_hugetlb_entry_migration(entry) || 2584 is_hugetlb_entry_hwpoisoned(entry))) { 2585 swp_entry_t swp_entry = pte_to_swp_entry(entry); 2586 2587 if (is_write_migration_entry(swp_entry) && cow) { 2588 /* 2589 * COW mappings require pages in both 2590 * parent and child to be set to read. 2591 */ 2592 make_migration_entry_read(&swp_entry); 2593 entry = swp_entry_to_pte(swp_entry); 2594 set_huge_pte_at(src, addr, src_pte, entry); 2595 } 2596 set_huge_pte_at(dst, addr, dst_pte, entry); 2597 } else { 2598 if (cow) 2599 huge_ptep_set_wrprotect(src, addr, src_pte); 2600 entry = huge_ptep_get(src_pte); 2601 ptepage = pte_page(entry); 2602 get_page(ptepage); 2603 page_dup_rmap(ptepage); 2604 set_huge_pte_at(dst, addr, dst_pte, entry); 2605 } 2606 spin_unlock(src_ptl); 2607 spin_unlock(dst_ptl); 2608 } 2609 2610 if (cow) 2611 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end); 2612 2613 return ret; 2614} 2615 2616void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 2617 unsigned long start, unsigned long end, 2618 struct page *ref_page) 2619{ 2620 int force_flush = 0; 2621 struct mm_struct *mm = vma->vm_mm; 2622 unsigned long address; 2623 pte_t *ptep; 2624 pte_t pte; 2625 spinlock_t *ptl; 2626 struct page *page; 2627 struct hstate *h = hstate_vma(vma); 2628 unsigned long sz = huge_page_size(h); 2629 const unsigned long mmun_start = start; /* For mmu_notifiers */ 2630 const unsigned long mmun_end = end; /* For mmu_notifiers */ 2631 2632 WARN_ON(!is_vm_hugetlb_page(vma)); 2633 BUG_ON(start & ~huge_page_mask(h)); 2634 BUG_ON(end & ~huge_page_mask(h)); 2635 2636 tlb_start_vma(tlb, vma); 2637 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2638again: 2639 for (address = start; address < end; address += sz) { 2640 ptep = huge_pte_offset(mm, address); 2641 if (!ptep) 2642 continue; 2643 2644 ptl = huge_pte_lock(h, mm, ptep); 2645 if (huge_pmd_unshare(mm, &address, ptep)) 2646 goto unlock; 2647 2648 pte = huge_ptep_get(ptep); 2649 if (huge_pte_none(pte)) 2650 goto unlock; 2651 2652 /* 2653 * HWPoisoned hugepage is already unmapped and dropped reference 2654 */ 2655 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 2656 huge_pte_clear(mm, address, ptep); 2657 goto unlock; 2658 } 2659 2660 page = pte_page(pte); 2661 /* 2662 * If a reference page is supplied, it is because a specific 2663 * page is being unmapped, not a range. Ensure the page we 2664 * are about to unmap is the actual page of interest. 2665 */ 2666 if (ref_page) { 2667 if (page != ref_page) 2668 goto unlock; 2669 2670 /* 2671 * Mark the VMA as having unmapped its page so that 2672 * future faults in this VMA will fail rather than 2673 * looking like data was lost 2674 */ 2675 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 2676 } 2677 2678 pte = huge_ptep_get_and_clear(mm, address, ptep); 2679 tlb_remove_tlb_entry(tlb, ptep, address); 2680 if (huge_pte_dirty(pte)) 2681 set_page_dirty(page); 2682 2683 page_remove_rmap(page); 2684 force_flush = !__tlb_remove_page(tlb, page); 2685 if (force_flush) { 2686 spin_unlock(ptl); 2687 break; 2688 } 2689 /* Bail out after unmapping reference page if supplied */ 2690 if (ref_page) { 2691 spin_unlock(ptl); 2692 break; 2693 } 2694unlock: 2695 spin_unlock(ptl); 2696 } 2697 /* 2698 * mmu_gather ran out of room to batch pages, we break out of 2699 * the PTE lock to avoid doing the potential expensive TLB invalidate 2700 * and page-free while holding it. 2701 */ 2702 if (force_flush) { 2703 force_flush = 0; 2704 tlb_flush_mmu(tlb); 2705 if (address < end && !ref_page) 2706 goto again; 2707 } 2708 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2709 tlb_end_vma(tlb, vma); 2710} 2711 2712void __unmap_hugepage_range_final(struct mmu_gather *tlb, 2713 struct vm_area_struct *vma, unsigned long start, 2714 unsigned long end, struct page *ref_page) 2715{ 2716 __unmap_hugepage_range(tlb, vma, start, end, ref_page); 2717 2718 /* 2719 * Clear this flag so that x86's huge_pmd_share page_table_shareable 2720 * test will fail on a vma being torn down, and not grab a page table 2721 * on its way out. We're lucky that the flag has such an appropriate 2722 * name, and can in fact be safely cleared here. We could clear it 2723 * before the __unmap_hugepage_range above, but all that's necessary 2724 * is to clear it before releasing the i_mmap_mutex. This works 2725 * because in the context this is called, the VMA is about to be 2726 * destroyed and the i_mmap_mutex is held. 2727 */ 2728 vma->vm_flags &= ~VM_MAYSHARE; 2729} 2730 2731void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 2732 unsigned long end, struct page *ref_page) 2733{ 2734 struct mm_struct *mm; 2735 struct mmu_gather tlb; 2736 2737 mm = vma->vm_mm; 2738 2739 tlb_gather_mmu(&tlb, mm, start, end); 2740 __unmap_hugepage_range(&tlb, vma, start, end, ref_page); 2741 tlb_finish_mmu(&tlb, start, end); 2742} 2743 2744/* 2745 * This is called when the original mapper is failing to COW a MAP_PRIVATE 2746 * mappping it owns the reserve page for. The intention is to unmap the page 2747 * from other VMAs and let the children be SIGKILLed if they are faulting the 2748 * same region. 2749 */ 2750static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 2751 struct page *page, unsigned long address) 2752{ 2753 struct hstate *h = hstate_vma(vma); 2754 struct vm_area_struct *iter_vma; 2755 struct address_space *mapping; 2756 pgoff_t pgoff; 2757 2758 /* 2759 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 2760 * from page cache lookup which is in HPAGE_SIZE units. 2761 */ 2762 address = address & huge_page_mask(h); 2763 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 2764 vma->vm_pgoff; 2765 mapping = file_inode(vma->vm_file)->i_mapping; 2766 2767 /* 2768 * Take the mapping lock for the duration of the table walk. As 2769 * this mapping should be shared between all the VMAs, 2770 * __unmap_hugepage_range() is called as the lock is already held 2771 */ 2772 mutex_lock(&mapping->i_mmap_mutex); 2773 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 2774 /* Do not unmap the current VMA */ 2775 if (iter_vma == vma) 2776 continue; 2777 2778 /* 2779 * Unmap the page from other VMAs without their own reserves. 2780 * They get marked to be SIGKILLed if they fault in these 2781 * areas. This is because a future no-page fault on this VMA 2782 * could insert a zeroed page instead of the data existing 2783 * from the time of fork. This would look like data corruption 2784 */ 2785 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 2786 unmap_hugepage_range(iter_vma, address, 2787 address + huge_page_size(h), page); 2788 } 2789 mutex_unlock(&mapping->i_mmap_mutex); 2790} 2791 2792/* 2793 * Hugetlb_cow() should be called with page lock of the original hugepage held. 2794 * Called with hugetlb_instantiation_mutex held and pte_page locked so we 2795 * cannot race with other handlers or page migration. 2796 * Keep the pte_same checks anyway to make transition from the mutex easier. 2797 */ 2798static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 2799 unsigned long address, pte_t *ptep, pte_t pte, 2800 struct page *pagecache_page, spinlock_t *ptl) 2801{ 2802 struct hstate *h = hstate_vma(vma); 2803 struct page *old_page, *new_page; 2804 int ret = 0, outside_reserve = 0; 2805 unsigned long mmun_start; /* For mmu_notifiers */ 2806 unsigned long mmun_end; /* For mmu_notifiers */ 2807 2808 old_page = pte_page(pte); 2809 2810retry_avoidcopy: 2811 /* If no-one else is actually using this page, avoid the copy 2812 * and just make the page writable */ 2813 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) { 2814 page_move_anon_rmap(old_page, vma, address); 2815 set_huge_ptep_writable(vma, address, ptep); 2816 return 0; 2817 } 2818 2819 /* 2820 * If the process that created a MAP_PRIVATE mapping is about to 2821 * perform a COW due to a shared page count, attempt to satisfy 2822 * the allocation without using the existing reserves. The pagecache 2823 * page is used to determine if the reserve at this address was 2824 * consumed or not. If reserves were used, a partial faulted mapping 2825 * at the time of fork() could consume its reserves on COW instead 2826 * of the full address range. 2827 */ 2828 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 2829 old_page != pagecache_page) 2830 outside_reserve = 1; 2831 2832 page_cache_get(old_page); 2833 2834 /* 2835 * Drop page table lock as buddy allocator may be called. It will 2836 * be acquired again before returning to the caller, as expected. 2837 */ 2838 spin_unlock(ptl); 2839 new_page = alloc_huge_page(vma, address, outside_reserve); 2840 2841 if (IS_ERR(new_page)) { 2842 /* 2843 * If a process owning a MAP_PRIVATE mapping fails to COW, 2844 * it is due to references held by a child and an insufficient 2845 * huge page pool. To guarantee the original mappers 2846 * reliability, unmap the page from child processes. The child 2847 * may get SIGKILLed if it later faults. 2848 */ 2849 if (outside_reserve) { 2850 page_cache_release(old_page); 2851 BUG_ON(huge_pte_none(pte)); 2852 unmap_ref_private(mm, vma, old_page, address); 2853 BUG_ON(huge_pte_none(pte)); 2854 spin_lock(ptl); 2855 ptep = huge_pte_offset(mm, address & huge_page_mask(h)); 2856 if (likely(ptep && 2857 pte_same(huge_ptep_get(ptep), pte))) 2858 goto retry_avoidcopy; 2859 /* 2860 * race occurs while re-acquiring page table 2861 * lock, and our job is done. 2862 */ 2863 return 0; 2864 } 2865 2866 ret = (PTR_ERR(new_page) == -ENOMEM) ? 2867 VM_FAULT_OOM : VM_FAULT_SIGBUS; 2868 goto out_release_old; 2869 } 2870 2871 /* 2872 * When the original hugepage is shared one, it does not have 2873 * anon_vma prepared. 2874 */ 2875 if (unlikely(anon_vma_prepare(vma))) { 2876 ret = VM_FAULT_OOM; 2877 goto out_release_all; 2878 } 2879 2880 copy_user_huge_page(new_page, old_page, address, vma, 2881 pages_per_huge_page(h)); 2882 __SetPageUptodate(new_page); 2883 2884 mmun_start = address & huge_page_mask(h); 2885 mmun_end = mmun_start + huge_page_size(h); 2886 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2887 2888 /* 2889 * Retake the page table lock to check for racing updates 2890 * before the page tables are altered 2891 */ 2892 spin_lock(ptl); 2893 ptep = huge_pte_offset(mm, address & huge_page_mask(h)); 2894 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 2895 ClearPagePrivate(new_page); 2896 2897 /* Break COW */ 2898 huge_ptep_clear_flush(vma, address, ptep); 2899 set_huge_pte_at(mm, address, ptep, 2900 make_huge_pte(vma, new_page, 1)); 2901 page_remove_rmap(old_page); 2902 hugepage_add_new_anon_rmap(new_page, vma, address); 2903 /* Make the old page be freed below */ 2904 new_page = old_page; 2905 } 2906 spin_unlock(ptl); 2907 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2908out_release_all: 2909 page_cache_release(new_page); 2910out_release_old: 2911 page_cache_release(old_page); 2912 2913 spin_lock(ptl); /* Caller expects lock to be held */ 2914 return ret; 2915} 2916 2917/* Return the pagecache page at a given address within a VMA */ 2918static struct page *hugetlbfs_pagecache_page(struct hstate *h, 2919 struct vm_area_struct *vma, unsigned long address) 2920{ 2921 struct address_space *mapping; 2922 pgoff_t idx; 2923 2924 mapping = vma->vm_file->f_mapping; 2925 idx = vma_hugecache_offset(h, vma, address); 2926 2927 return find_lock_page(mapping, idx); 2928} 2929 2930/* 2931 * Return whether there is a pagecache page to back given address within VMA. 2932 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. 2933 */ 2934static bool hugetlbfs_pagecache_present(struct hstate *h, 2935 struct vm_area_struct *vma, unsigned long address) 2936{ 2937 struct address_space *mapping; 2938 pgoff_t idx; 2939 struct page *page; 2940 2941 mapping = vma->vm_file->f_mapping; 2942 idx = vma_hugecache_offset(h, vma, address); 2943 2944 page = find_get_page(mapping, idx); 2945 if (page) 2946 put_page(page); 2947 return page != NULL; 2948} 2949 2950static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, 2951 struct address_space *mapping, pgoff_t idx, 2952 unsigned long address, pte_t *ptep, unsigned int flags) 2953{ 2954 struct hstate *h = hstate_vma(vma); 2955 int ret = VM_FAULT_SIGBUS; 2956 int anon_rmap = 0; 2957 unsigned long size; 2958 struct page *page; 2959 pte_t new_pte; 2960 spinlock_t *ptl; 2961 2962 /* 2963 * Currently, we are forced to kill the process in the event the 2964 * original mapper has unmapped pages from the child due to a failed 2965 * COW. Warn that such a situation has occurred as it may not be obvious 2966 */ 2967 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 2968 pr_warning("PID %d killed due to inadequate hugepage pool\n", 2969 current->pid); 2970 return ret; 2971 } 2972 2973 /* 2974 * Use page lock to guard against racing truncation 2975 * before we get page_table_lock. 2976 */ 2977retry: 2978 page = find_lock_page(mapping, idx); 2979 if (!page) { 2980 size = i_size_read(mapping->host) >> huge_page_shift(h); 2981 if (idx >= size) 2982 goto out; 2983 page = alloc_huge_page(vma, address, 0); 2984 if (IS_ERR(page)) { 2985 ret = PTR_ERR(page); 2986 if (ret == -ENOMEM) 2987 ret = VM_FAULT_OOM; 2988 else 2989 ret = VM_FAULT_SIGBUS; 2990 goto out; 2991 } 2992 clear_huge_page(page, address, pages_per_huge_page(h)); 2993 __SetPageUptodate(page); 2994 2995 if (vma->vm_flags & VM_MAYSHARE) { 2996 int err; 2997 struct inode *inode = mapping->host; 2998 2999 err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 3000 if (err) { 3001 put_page(page); 3002 if (err == -EEXIST) 3003 goto retry; 3004 goto out; 3005 } 3006 ClearPagePrivate(page); 3007 3008 spin_lock(&inode->i_lock); 3009 inode->i_blocks += blocks_per_huge_page(h); 3010 spin_unlock(&inode->i_lock); 3011 } else { 3012 lock_page(page); 3013 if (unlikely(anon_vma_prepare(vma))) { 3014 ret = VM_FAULT_OOM; 3015 goto backout_unlocked; 3016 } 3017 anon_rmap = 1; 3018 } 3019 } else { 3020 /* 3021 * If memory error occurs between mmap() and fault, some process 3022 * don't have hwpoisoned swap entry for errored virtual address. 3023 * So we need to block hugepage fault by PG_hwpoison bit check. 3024 */ 3025 if (unlikely(PageHWPoison(page))) { 3026 ret = VM_FAULT_HWPOISON | 3027 VM_FAULT_SET_HINDEX(hstate_index(h)); 3028 goto backout_unlocked; 3029 } 3030 } 3031 3032 /* 3033 * If we are going to COW a private mapping later, we examine the 3034 * pending reservations for this page now. This will ensure that 3035 * any allocations necessary to record that reservation occur outside 3036 * the spinlock. 3037 */ 3038 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) 3039 if (vma_needs_reservation(h, vma, address) < 0) { 3040 ret = VM_FAULT_OOM; 3041 goto backout_unlocked; 3042 } 3043 3044 ptl = huge_pte_lockptr(h, mm, ptep); 3045 spin_lock(ptl); 3046 size = i_size_read(mapping->host) >> huge_page_shift(h); 3047 if (idx >= size) 3048 goto backout; 3049 3050 ret = 0; 3051 if (!huge_pte_none(huge_ptep_get(ptep))) 3052 goto backout; 3053 3054 if (anon_rmap) { 3055 ClearPagePrivate(page); 3056 hugepage_add_new_anon_rmap(page, vma, address); 3057 } else 3058 page_dup_rmap(page); 3059 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 3060 && (vma->vm_flags & VM_SHARED))); 3061 set_huge_pte_at(mm, address, ptep, new_pte); 3062 3063 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3064 /* Optimization, do the COW without a second fault */ 3065 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl); 3066 } 3067 3068 spin_unlock(ptl); 3069 unlock_page(page); 3070out: 3071 return ret; 3072 3073backout: 3074 spin_unlock(ptl); 3075backout_unlocked: 3076 unlock_page(page); 3077 put_page(page); 3078 goto out; 3079} 3080 3081#ifdef CONFIG_SMP 3082static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3083 struct vm_area_struct *vma, 3084 struct address_space *mapping, 3085 pgoff_t idx, unsigned long address) 3086{ 3087 unsigned long key[2]; 3088 u32 hash; 3089 3090 if (vma->vm_flags & VM_SHARED) { 3091 key[0] = (unsigned long) mapping; 3092 key[1] = idx; 3093 } else { 3094 key[0] = (unsigned long) mm; 3095 key[1] = address >> huge_page_shift(h); 3096 } 3097 3098 hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0); 3099 3100 return hash & (num_fault_mutexes - 1); 3101} 3102#else 3103/* 3104 * For uniprocesor systems we always use a single mutex, so just 3105 * return 0 and avoid the hashing overhead. 3106 */ 3107static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3108 struct vm_area_struct *vma, 3109 struct address_space *mapping, 3110 pgoff_t idx, unsigned long address) 3111{ 3112 return 0; 3113} 3114#endif 3115 3116int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3117 unsigned long address, unsigned int flags) 3118{ 3119 pte_t *ptep, entry; 3120 spinlock_t *ptl; 3121 int ret; 3122 u32 hash; 3123 pgoff_t idx; 3124 struct page *page = NULL; 3125 struct page *pagecache_page = NULL; 3126 struct hstate *h = hstate_vma(vma); 3127 struct address_space *mapping; 3128 3129 address &= huge_page_mask(h); 3130 3131 ptep = huge_pte_offset(mm, address); 3132 if (ptep) { 3133 entry = huge_ptep_get(ptep); 3134 if (unlikely(is_hugetlb_entry_migration(entry))) { 3135 migration_entry_wait_huge(vma, mm, ptep); 3136 return 0; 3137 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 3138 return VM_FAULT_HWPOISON_LARGE | 3139 VM_FAULT_SET_HINDEX(hstate_index(h)); 3140 } 3141 3142 ptep = huge_pte_alloc(mm, address, huge_page_size(h)); 3143 if (!ptep) 3144 return VM_FAULT_OOM; 3145 3146 mapping = vma->vm_file->f_mapping; 3147 idx = vma_hugecache_offset(h, vma, address); 3148 3149 /* 3150 * Serialize hugepage allocation and instantiation, so that we don't 3151 * get spurious allocation failures if two CPUs race to instantiate 3152 * the same page in the page cache. 3153 */ 3154 hash = fault_mutex_hash(h, mm, vma, mapping, idx, address); 3155 mutex_lock(&htlb_fault_mutex_table[hash]); 3156 3157 entry = huge_ptep_get(ptep); 3158 if (huge_pte_none(entry)) { 3159 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); 3160 goto out_mutex; 3161 } 3162 3163 ret = 0; 3164 3165 /* 3166 * If we are going to COW the mapping later, we examine the pending 3167 * reservations for this page now. This will ensure that any 3168 * allocations necessary to record that reservation occur outside the 3169 * spinlock. For private mappings, we also lookup the pagecache 3170 * page now as it is used to determine if a reservation has been 3171 * consumed. 3172 */ 3173 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 3174 if (vma_needs_reservation(h, vma, address) < 0) { 3175 ret = VM_FAULT_OOM; 3176 goto out_mutex; 3177 } 3178 3179 if (!(vma->vm_flags & VM_MAYSHARE)) 3180 pagecache_page = hugetlbfs_pagecache_page(h, 3181 vma, address); 3182 } 3183 3184 /* 3185 * hugetlb_cow() requires page locks of pte_page(entry) and 3186 * pagecache_page, so here we need take the former one 3187 * when page != pagecache_page or !pagecache_page. 3188 * Note that locking order is always pagecache_page -> page, 3189 * so no worry about deadlock. 3190 */ 3191 page = pte_page(entry); 3192 get_page(page); 3193 if (page != pagecache_page) 3194 lock_page(page); 3195 3196 ptl = huge_pte_lockptr(h, mm, ptep); 3197 spin_lock(ptl); 3198 /* Check for a racing update before calling hugetlb_cow */ 3199 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 3200 goto out_ptl; 3201 3202 3203 if (flags & FAULT_FLAG_WRITE) { 3204 if (!huge_pte_write(entry)) { 3205 ret = hugetlb_cow(mm, vma, address, ptep, entry, 3206 pagecache_page, ptl); 3207 goto out_ptl; 3208 } 3209 entry = huge_pte_mkdirty(entry); 3210 } 3211 entry = pte_mkyoung(entry); 3212 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 3213 flags & FAULT_FLAG_WRITE)) 3214 update_mmu_cache(vma, address, ptep); 3215 3216out_ptl: 3217 spin_unlock(ptl); 3218 3219 if (pagecache_page) { 3220 unlock_page(pagecache_page); 3221 put_page(pagecache_page); 3222 } 3223 if (page != pagecache_page) 3224 unlock_page(page); 3225 put_page(page); 3226 3227out_mutex: 3228 mutex_unlock(&htlb_fault_mutex_table[hash]); 3229 return ret; 3230} 3231 3232long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 3233 struct page **pages, struct vm_area_struct **vmas, 3234 unsigned long *position, unsigned long *nr_pages, 3235 long i, unsigned int flags) 3236{ 3237 unsigned long pfn_offset; 3238 unsigned long vaddr = *position; 3239 unsigned long remainder = *nr_pages; 3240 struct hstate *h = hstate_vma(vma); 3241 3242 while (vaddr < vma->vm_end && remainder) { 3243 pte_t *pte; 3244 spinlock_t *ptl = NULL; 3245 int absent; 3246 struct page *page; 3247 3248 /* 3249 * Some archs (sparc64, sh*) have multiple pte_ts to 3250 * each hugepage. We have to make sure we get the 3251 * first, for the page indexing below to work. 3252 * 3253 * Note that page table lock is not held when pte is null. 3254 */ 3255 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h)); 3256 if (pte) 3257 ptl = huge_pte_lock(h, mm, pte); 3258 absent = !pte || huge_pte_none(huge_ptep_get(pte)); 3259 3260 /* 3261 * When coredumping, it suits get_dump_page if we just return 3262 * an error where there's an empty slot with no huge pagecache 3263 * to back it. This way, we avoid allocating a hugepage, and 3264 * the sparse dumpfile avoids allocating disk blocks, but its 3265 * huge holes still show up with zeroes where they need to be. 3266 */ 3267 if (absent && (flags & FOLL_DUMP) && 3268 !hugetlbfs_pagecache_present(h, vma, vaddr)) { 3269 if (pte) 3270 spin_unlock(ptl); 3271 remainder = 0; 3272 break; 3273 } 3274 3275 /* 3276 * We need call hugetlb_fault for both hugepages under migration 3277 * (in which case hugetlb_fault waits for the migration,) and 3278 * hwpoisoned hugepages (in which case we need to prevent the 3279 * caller from accessing to them.) In order to do this, we use 3280 * here is_swap_pte instead of is_hugetlb_entry_migration and 3281 * is_hugetlb_entry_hwpoisoned. This is because it simply covers 3282 * both cases, and because we can't follow correct pages 3283 * directly from any kind of swap entries. 3284 */ 3285 if (absent || is_swap_pte(huge_ptep_get(pte)) || 3286 ((flags & FOLL_WRITE) && 3287 !huge_pte_write(huge_ptep_get(pte)))) { 3288 int ret; 3289 3290 if (pte) 3291 spin_unlock(ptl); 3292 ret = hugetlb_fault(mm, vma, vaddr, 3293 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0); 3294 if (!(ret & VM_FAULT_ERROR)) 3295 continue; 3296 3297 remainder = 0; 3298 break; 3299 } 3300 3301 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; 3302 page = pte_page(huge_ptep_get(pte)); 3303same_page: 3304 if (pages) { 3305 pages[i] = mem_map_offset(page, pfn_offset); 3306 get_page_foll(pages[i]); 3307 } 3308 3309 if (vmas) 3310 vmas[i] = vma; 3311 3312 vaddr += PAGE_SIZE; 3313 ++pfn_offset; 3314 --remainder; 3315 ++i; 3316 if (vaddr < vma->vm_end && remainder && 3317 pfn_offset < pages_per_huge_page(h)) { 3318 /* 3319 * We use pfn_offset to avoid touching the pageframes 3320 * of this compound page. 3321 */ 3322 goto same_page; 3323 } 3324 spin_unlock(ptl); 3325 } 3326 *nr_pages = remainder; 3327 *position = vaddr; 3328 3329 return i ? i : -EFAULT; 3330} 3331 3332unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 3333 unsigned long address, unsigned long end, pgprot_t newprot) 3334{ 3335 struct mm_struct *mm = vma->vm_mm; 3336 unsigned long start = address; 3337 pte_t *ptep; 3338 pte_t pte; 3339 struct hstate *h = hstate_vma(vma); 3340 unsigned long pages = 0; 3341 3342 BUG_ON(address >= end); 3343 flush_cache_range(vma, address, end); 3344 3345 mmu_notifier_invalidate_range_start(mm, start, end); 3346 mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex); 3347 for (; address < end; address += huge_page_size(h)) { 3348 spinlock_t *ptl; 3349 ptep = huge_pte_offset(mm, address); 3350 if (!ptep) 3351 continue; 3352 ptl = huge_pte_lock(h, mm, ptep); 3353 if (huge_pmd_unshare(mm, &address, ptep)) { 3354 pages++; 3355 spin_unlock(ptl); 3356 continue; 3357 } 3358 if (!huge_pte_none(huge_ptep_get(ptep))) { 3359 pte = huge_ptep_get_and_clear(mm, address, ptep); 3360 pte = pte_mkhuge(huge_pte_modify(pte, newprot)); 3361 pte = arch_make_huge_pte(pte, vma, NULL, 0); 3362 set_huge_pte_at(mm, address, ptep, pte); 3363 pages++; 3364 } 3365 spin_unlock(ptl); 3366 } 3367 /* 3368 * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare 3369 * may have cleared our pud entry and done put_page on the page table: 3370 * once we release i_mmap_mutex, another task can do the final put_page 3371 * and that page table be reused and filled with junk. 3372 */ 3373 flush_tlb_range(vma, start, end); 3374 mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex); 3375 mmu_notifier_invalidate_range_end(mm, start, end); 3376 3377 return pages << h->order; 3378} 3379 3380int hugetlb_reserve_pages(struct inode *inode, 3381 long from, long to, 3382 struct vm_area_struct *vma, 3383 vm_flags_t vm_flags) 3384{ 3385 long ret, chg; 3386 struct hstate *h = hstate_inode(inode); 3387 struct hugepage_subpool *spool = subpool_inode(inode); 3388 struct resv_map *resv_map; 3389 3390 /* 3391 * Only apply hugepage reservation if asked. At fault time, an 3392 * attempt will be made for VM_NORESERVE to allocate a page 3393 * without using reserves 3394 */ 3395 if (vm_flags & VM_NORESERVE) 3396 return 0; 3397 3398 /* 3399 * Shared mappings base their reservation on the number of pages that 3400 * are already allocated on behalf of the file. Private mappings need 3401 * to reserve the full area even if read-only as mprotect() may be 3402 * called to make the mapping read-write. Assume !vma is a shm mapping 3403 */ 3404 if (!vma || vma->vm_flags & VM_MAYSHARE) { 3405 resv_map = inode_resv_map(inode); 3406 3407 chg = region_chg(resv_map, from, to); 3408 3409 } else { 3410 resv_map = resv_map_alloc(); 3411 if (!resv_map) 3412 return -ENOMEM; 3413 3414 chg = to - from; 3415 3416 set_vma_resv_map(vma, resv_map); 3417 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 3418 } 3419 3420 if (chg < 0) { 3421 ret = chg; 3422 goto out_err; 3423 } 3424 3425 /* There must be enough pages in the subpool for the mapping */ 3426 if (hugepage_subpool_get_pages(spool, chg)) { 3427 ret = -ENOSPC; 3428 goto out_err; 3429 } 3430 3431 /* 3432 * Check enough hugepages are available for the reservation. 3433 * Hand the pages back to the subpool if there are not 3434 */ 3435 ret = hugetlb_acct_memory(h, chg); 3436 if (ret < 0) { 3437 hugepage_subpool_put_pages(spool, chg); 3438 goto out_err; 3439 } 3440 3441 /* 3442 * Account for the reservations made. Shared mappings record regions 3443 * that have reservations as they are shared by multiple VMAs. 3444 * When the last VMA disappears, the region map says how much 3445 * the reservation was and the page cache tells how much of 3446 * the reservation was consumed. Private mappings are per-VMA and 3447 * only the consumed reservations are tracked. When the VMA 3448 * disappears, the original reservation is the VMA size and the 3449 * consumed reservations are stored in the map. Hence, nothing 3450 * else has to be done for private mappings here 3451 */ 3452 if (!vma || vma->vm_flags & VM_MAYSHARE) 3453 region_add(resv_map, from, to); 3454 return 0; 3455out_err: 3456 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3457 kref_put(&resv_map->refs, resv_map_release); 3458 return ret; 3459} 3460 3461void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed) 3462{ 3463 struct hstate *h = hstate_inode(inode); 3464 struct resv_map *resv_map = inode_resv_map(inode); 3465 long chg = 0; 3466 struct hugepage_subpool *spool = subpool_inode(inode); 3467 3468 if (resv_map) 3469 chg = region_truncate(resv_map, offset); 3470 spin_lock(&inode->i_lock); 3471 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 3472 spin_unlock(&inode->i_lock); 3473 3474 hugepage_subpool_put_pages(spool, (chg - freed)); 3475 hugetlb_acct_memory(h, -(chg - freed)); 3476} 3477 3478#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 3479static unsigned long page_table_shareable(struct vm_area_struct *svma, 3480 struct vm_area_struct *vma, 3481 unsigned long addr, pgoff_t idx) 3482{ 3483 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 3484 svma->vm_start; 3485 unsigned long sbase = saddr & PUD_MASK; 3486 unsigned long s_end = sbase + PUD_SIZE; 3487 3488 /* Allow segments to share if only one is marked locked */ 3489 unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED; 3490 unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED; 3491 3492 /* 3493 * match the virtual addresses, permission and the alignment of the 3494 * page table page. 3495 */ 3496 if (pmd_index(addr) != pmd_index(saddr) || 3497 vm_flags != svm_flags || 3498 sbase < svma->vm_start || svma->vm_end < s_end) 3499 return 0; 3500 3501 return saddr; 3502} 3503 3504static int vma_shareable(struct vm_area_struct *vma, unsigned long addr) 3505{ 3506 unsigned long base = addr & PUD_MASK; 3507 unsigned long end = base + PUD_SIZE; 3508 3509 /* 3510 * check on proper vm_flags and page table alignment 3511 */ 3512 if (vma->vm_flags & VM_MAYSHARE && 3513 vma->vm_start <= base && end <= vma->vm_end) 3514 return 1; 3515 return 0; 3516} 3517 3518/* 3519 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 3520 * and returns the corresponding pte. While this is not necessary for the 3521 * !shared pmd case because we can allocate the pmd later as well, it makes the 3522 * code much cleaner. pmd allocation is essential for the shared case because 3523 * pud has to be populated inside the same i_mmap_mutex section - otherwise 3524 * racing tasks could either miss the sharing (see huge_pte_offset) or select a 3525 * bad pmd for sharing. 3526 */ 3527pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 3528{ 3529 struct vm_area_struct *vma = find_vma(mm, addr); 3530 struct address_space *mapping = vma->vm_file->f_mapping; 3531 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 3532 vma->vm_pgoff; 3533 struct vm_area_struct *svma; 3534 unsigned long saddr; 3535 pte_t *spte = NULL; 3536 pte_t *pte; 3537 spinlock_t *ptl; 3538 3539 if (!vma_shareable(vma, addr)) 3540 return (pte_t *)pmd_alloc(mm, pud, addr); 3541 3542 mutex_lock(&mapping->i_mmap_mutex); 3543 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 3544 if (svma == vma) 3545 continue; 3546 3547 saddr = page_table_shareable(svma, vma, addr, idx); 3548 if (saddr) { 3549 spte = huge_pte_offset(svma->vm_mm, saddr); 3550 if (spte) { 3551 get_page(virt_to_page(spte)); 3552 break; 3553 } 3554 } 3555 } 3556 3557 if (!spte) 3558 goto out; 3559 3560 ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte); 3561 spin_lock(ptl); 3562 if (pud_none(*pud)) 3563 pud_populate(mm, pud, 3564 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 3565 else 3566 put_page(virt_to_page(spte)); 3567 spin_unlock(ptl); 3568out: 3569 pte = (pte_t *)pmd_alloc(mm, pud, addr); 3570 mutex_unlock(&mapping->i_mmap_mutex); 3571 return pte; 3572} 3573 3574/* 3575 * unmap huge page backed by shared pte. 3576 * 3577 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 3578 * indicated by page_count > 1, unmap is achieved by clearing pud and 3579 * decrementing the ref count. If count == 1, the pte page is not shared. 3580 * 3581 * called with page table lock held. 3582 * 3583 * returns: 1 successfully unmapped a shared pte page 3584 * 0 the underlying pte page is not shared, or it is the last user 3585 */ 3586int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 3587{ 3588 pgd_t *pgd = pgd_offset(mm, *addr); 3589 pud_t *pud = pud_offset(pgd, *addr); 3590 3591 BUG_ON(page_count(virt_to_page(ptep)) == 0); 3592 if (page_count(virt_to_page(ptep)) == 1) 3593 return 0; 3594 3595 pud_clear(pud); 3596 put_page(virt_to_page(ptep)); 3597 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; 3598 return 1; 3599} 3600#define want_pmd_share() (1) 3601#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 3602pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 3603{ 3604 return NULL; 3605} 3606#define want_pmd_share() (0) 3607#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 3608 3609#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 3610pte_t *huge_pte_alloc(struct mm_struct *mm, 3611 unsigned long addr, unsigned long sz) 3612{ 3613 pgd_t *pgd; 3614 pud_t *pud; 3615 pte_t *pte = NULL; 3616 3617 pgd = pgd_offset(mm, addr); 3618 pud = pud_alloc(mm, pgd, addr); 3619 if (pud) { 3620 if (sz == PUD_SIZE) { 3621 pte = (pte_t *)pud; 3622 } else { 3623 BUG_ON(sz != PMD_SIZE); 3624 if (want_pmd_share() && pud_none(*pud)) 3625 pte = huge_pmd_share(mm, addr, pud); 3626 else 3627 pte = (pte_t *)pmd_alloc(mm, pud, addr); 3628 } 3629 } 3630 BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); 3631 3632 return pte; 3633} 3634 3635pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 3636{ 3637 pgd_t *pgd; 3638 pud_t *pud; 3639 pmd_t *pmd = NULL; 3640 3641 pgd = pgd_offset(mm, addr); 3642 if (pgd_present(*pgd)) { 3643 pud = pud_offset(pgd, addr); 3644 if (pud_present(*pud)) { 3645 if (pud_huge(*pud)) 3646 return (pte_t *)pud; 3647 pmd = pmd_offset(pud, addr); 3648 } 3649 } 3650 return (pte_t *) pmd; 3651} 3652 3653struct page * 3654follow_huge_pmd(struct mm_struct *mm, unsigned long address, 3655 pmd_t *pmd, int write) 3656{ 3657 struct page *page; 3658 3659 page = pte_page(*(pte_t *)pmd); 3660 if (page) 3661 page += ((address & ~PMD_MASK) >> PAGE_SHIFT); 3662 return page; 3663} 3664 3665struct page * 3666follow_huge_pud(struct mm_struct *mm, unsigned long address, 3667 pud_t *pud, int write) 3668{ 3669 struct page *page; 3670 3671 page = pte_page(*(pte_t *)pud); 3672 if (page) 3673 page += ((address & ~PUD_MASK) >> PAGE_SHIFT); 3674 return page; 3675} 3676 3677#else /* !CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 3678 3679/* Can be overriden by architectures */ 3680struct page * __weak 3681follow_huge_pud(struct mm_struct *mm, unsigned long address, 3682 pud_t *pud, int write) 3683{ 3684 BUG(); 3685 return NULL; 3686} 3687 3688#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 3689 3690#ifdef CONFIG_MEMORY_FAILURE 3691 3692/* Should be called in hugetlb_lock */ 3693static int is_hugepage_on_freelist(struct page *hpage) 3694{ 3695 struct page *page; 3696 struct page *tmp; 3697 struct hstate *h = page_hstate(hpage); 3698 int nid = page_to_nid(hpage); 3699 3700 list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru) 3701 if (page == hpage) 3702 return 1; 3703 return 0; 3704} 3705 3706/* 3707 * This function is called from memory failure code. 3708 * Assume the caller holds page lock of the head page. 3709 */ 3710int dequeue_hwpoisoned_huge_page(struct page *hpage) 3711{ 3712 struct hstate *h = page_hstate(hpage); 3713 int nid = page_to_nid(hpage); 3714 int ret = -EBUSY; 3715 3716 spin_lock(&hugetlb_lock); 3717 if (is_hugepage_on_freelist(hpage)) { 3718 /* 3719 * Hwpoisoned hugepage isn't linked to activelist or freelist, 3720 * but dangling hpage->lru can trigger list-debug warnings 3721 * (this happens when we call unpoison_memory() on it), 3722 * so let it point to itself with list_del_init(). 3723 */ 3724 list_del_init(&hpage->lru); 3725 set_page_refcounted(hpage); 3726 h->free_huge_pages--; 3727 h->free_huge_pages_node[nid]--; 3728 ret = 0; 3729 } 3730 spin_unlock(&hugetlb_lock); 3731 return ret; 3732} 3733#endif 3734 3735bool isolate_huge_page(struct page *page, struct list_head *list) 3736{ 3737 VM_BUG_ON_PAGE(!PageHead(page), page); 3738 if (!get_page_unless_zero(page)) 3739 return false; 3740 spin_lock(&hugetlb_lock); 3741 list_move_tail(&page->lru, list); 3742 spin_unlock(&hugetlb_lock); 3743 return true; 3744} 3745 3746void putback_active_hugepage(struct page *page) 3747{ 3748 VM_BUG_ON_PAGE(!PageHead(page), page); 3749 spin_lock(&hugetlb_lock); 3750 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); 3751 spin_unlock(&hugetlb_lock); 3752 put_page(page); 3753} 3754 3755bool is_hugepage_active(struct page *page) 3756{ 3757 VM_BUG_ON_PAGE(!PageHuge(page), page); 3758 /* 3759 * This function can be called for a tail page because the caller, 3760 * scan_movable_pages, scans through a given pfn-range which typically 3761 * covers one memory block. In systems using gigantic hugepage (1GB 3762 * for x86_64,) a hugepage is larger than a memory block, and we don't 3763 * support migrating such large hugepages for now, so return false 3764 * when called for tail pages. 3765 */ 3766 if (PageTail(page)) 3767 return false; 3768 /* 3769 * Refcount of a hwpoisoned hugepages is 1, but they are not active, 3770 * so we should return false for them. 3771 */ 3772 if (unlikely(PageHWPoison(page))) 3773 return false; 3774 return page_count(page) > 0; 3775} 3776