page_alloc.c revision 54114994f4de7e8076fc250e44501e55e19b75b5
1/* 2 * linux/mm/page_alloc.c 3 * 4 * Manages the free list, the system allocates free pages here. 5 * Note that kmalloc() lives in slab.c 6 * 7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 8 * Swap reorganised 29.12.95, Stephen Tweedie 9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 15 */ 16 17#include <linux/stddef.h> 18#include <linux/mm.h> 19#include <linux/swap.h> 20#include <linux/interrupt.h> 21#include <linux/pagemap.h> 22#include <linux/bootmem.h> 23#include <linux/compiler.h> 24#include <linux/kernel.h> 25#include <linux/module.h> 26#include <linux/suspend.h> 27#include <linux/pagevec.h> 28#include <linux/blkdev.h> 29#include <linux/slab.h> 30#include <linux/notifier.h> 31#include <linux/topology.h> 32#include <linux/sysctl.h> 33#include <linux/cpu.h> 34#include <linux/cpuset.h> 35#include <linux/memory_hotplug.h> 36#include <linux/nodemask.h> 37#include <linux/vmalloc.h> 38#include <linux/mempolicy.h> 39#include <linux/stop_machine.h> 40#include <linux/sort.h> 41#include <linux/pfn.h> 42#include <linux/backing-dev.h> 43#include <linux/fault-inject.h> 44 45#include <asm/tlbflush.h> 46#include <asm/div64.h> 47#include "internal.h" 48 49/* 50 * MCD - HACK: Find somewhere to initialize this EARLY, or make this 51 * initializer cleaner 52 */ 53nodemask_t node_online_map __read_mostly = { { [0] = 1UL } }; 54EXPORT_SYMBOL(node_online_map); 55nodemask_t node_possible_map __read_mostly = NODE_MASK_ALL; 56EXPORT_SYMBOL(node_possible_map); 57unsigned long totalram_pages __read_mostly; 58unsigned long totalreserve_pages __read_mostly; 59long nr_swap_pages; 60int percpu_pagelist_fraction; 61 62static void __free_pages_ok(struct page *page, unsigned int order); 63 64/* 65 * results with 256, 32 in the lowmem_reserve sysctl: 66 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 67 * 1G machine -> (16M dma, 784M normal, 224M high) 68 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 69 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 70 * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA 71 * 72 * TBD: should special case ZONE_DMA32 machines here - in those we normally 73 * don't need any ZONE_NORMAL reservation 74 */ 75int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { 76#ifdef CONFIG_ZONE_DMA 77 256, 78#endif 79#ifdef CONFIG_ZONE_DMA32 80 256, 81#endif 82#ifdef CONFIG_HIGHMEM 83 32 84#endif 85}; 86 87EXPORT_SYMBOL(totalram_pages); 88 89static char * const zone_names[MAX_NR_ZONES] = { 90#ifdef CONFIG_ZONE_DMA 91 "DMA", 92#endif 93#ifdef CONFIG_ZONE_DMA32 94 "DMA32", 95#endif 96 "Normal", 97#ifdef CONFIG_HIGHMEM 98 "HighMem" 99#endif 100}; 101 102int min_free_kbytes = 1024; 103 104unsigned long __meminitdata nr_kernel_pages; 105unsigned long __meminitdata nr_all_pages; 106static unsigned long __meminitdata dma_reserve; 107 108#ifdef CONFIG_ARCH_POPULATES_NODE_MAP 109 /* 110 * MAX_ACTIVE_REGIONS determines the maxmimum number of distinct 111 * ranges of memory (RAM) that may be registered with add_active_range(). 112 * Ranges passed to add_active_range() will be merged if possible 113 * so the number of times add_active_range() can be called is 114 * related to the number of nodes and the number of holes 115 */ 116 #ifdef CONFIG_MAX_ACTIVE_REGIONS 117 /* Allow an architecture to set MAX_ACTIVE_REGIONS to save memory */ 118 #define MAX_ACTIVE_REGIONS CONFIG_MAX_ACTIVE_REGIONS 119 #else 120 #if MAX_NUMNODES >= 32 121 /* If there can be many nodes, allow up to 50 holes per node */ 122 #define MAX_ACTIVE_REGIONS (MAX_NUMNODES*50) 123 #else 124 /* By default, allow up to 256 distinct regions */ 125 #define MAX_ACTIVE_REGIONS 256 126 #endif 127 #endif 128 129 static struct node_active_region __meminitdata early_node_map[MAX_ACTIVE_REGIONS]; 130 static int __meminitdata nr_nodemap_entries; 131 static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; 132 static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; 133#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE 134 static unsigned long __meminitdata node_boundary_start_pfn[MAX_NUMNODES]; 135 static unsigned long __meminitdata node_boundary_end_pfn[MAX_NUMNODES]; 136#endif /* CONFIG_MEMORY_HOTPLUG_RESERVE */ 137#endif /* CONFIG_ARCH_POPULATES_NODE_MAP */ 138 139#if MAX_NUMNODES > 1 140int nr_node_ids __read_mostly = MAX_NUMNODES; 141EXPORT_SYMBOL(nr_node_ids); 142#endif 143 144#ifdef CONFIG_DEBUG_VM 145static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 146{ 147 int ret = 0; 148 unsigned seq; 149 unsigned long pfn = page_to_pfn(page); 150 151 do { 152 seq = zone_span_seqbegin(zone); 153 if (pfn >= zone->zone_start_pfn + zone->spanned_pages) 154 ret = 1; 155 else if (pfn < zone->zone_start_pfn) 156 ret = 1; 157 } while (zone_span_seqretry(zone, seq)); 158 159 return ret; 160} 161 162static int page_is_consistent(struct zone *zone, struct page *page) 163{ 164 if (!pfn_valid_within(page_to_pfn(page))) 165 return 0; 166 if (zone != page_zone(page)) 167 return 0; 168 169 return 1; 170} 171/* 172 * Temporary debugging check for pages not lying within a given zone. 173 */ 174static int bad_range(struct zone *zone, struct page *page) 175{ 176 if (page_outside_zone_boundaries(zone, page)) 177 return 1; 178 if (!page_is_consistent(zone, page)) 179 return 1; 180 181 return 0; 182} 183#else 184static inline int bad_range(struct zone *zone, struct page *page) 185{ 186 return 0; 187} 188#endif 189 190static void bad_page(struct page *page) 191{ 192 printk(KERN_EMERG "Bad page state in process '%s'\n" 193 KERN_EMERG "page:%p flags:0x%0*lx mapping:%p mapcount:%d count:%d\n" 194 KERN_EMERG "Trying to fix it up, but a reboot is needed\n" 195 KERN_EMERG "Backtrace:\n", 196 current->comm, page, (int)(2*sizeof(unsigned long)), 197 (unsigned long)page->flags, page->mapping, 198 page_mapcount(page), page_count(page)); 199 dump_stack(); 200 page->flags &= ~(1 << PG_lru | 201 1 << PG_private | 202 1 << PG_locked | 203 1 << PG_active | 204 1 << PG_dirty | 205 1 << PG_reclaim | 206 1 << PG_slab | 207 1 << PG_swapcache | 208 1 << PG_writeback | 209 1 << PG_buddy ); 210 set_page_count(page, 0); 211 reset_page_mapcount(page); 212 page->mapping = NULL; 213 add_taint(TAINT_BAD_PAGE); 214} 215 216/* 217 * Higher-order pages are called "compound pages". They are structured thusly: 218 * 219 * The first PAGE_SIZE page is called the "head page". 220 * 221 * The remaining PAGE_SIZE pages are called "tail pages". 222 * 223 * All pages have PG_compound set. All pages have their ->private pointing at 224 * the head page (even the head page has this). 225 * 226 * The first tail page's ->lru.next holds the address of the compound page's 227 * put_page() function. Its ->lru.prev holds the order of allocation. 228 * This usage means that zero-order pages may not be compound. 229 */ 230 231static void free_compound_page(struct page *page) 232{ 233 __free_pages_ok(page, compound_order(page)); 234} 235 236static void prep_compound_page(struct page *page, unsigned long order) 237{ 238 int i; 239 int nr_pages = 1 << order; 240 241 set_compound_page_dtor(page, free_compound_page); 242 set_compound_order(page, order); 243 __SetPageHead(page); 244 for (i = 1; i < nr_pages; i++) { 245 struct page *p = page + i; 246 247 __SetPageTail(p); 248 p->first_page = page; 249 } 250} 251 252static void destroy_compound_page(struct page *page, unsigned long order) 253{ 254 int i; 255 int nr_pages = 1 << order; 256 257 if (unlikely(compound_order(page) != order)) 258 bad_page(page); 259 260 if (unlikely(!PageHead(page))) 261 bad_page(page); 262 __ClearPageHead(page); 263 for (i = 1; i < nr_pages; i++) { 264 struct page *p = page + i; 265 266 if (unlikely(!PageTail(p) | 267 (p->first_page != page))) 268 bad_page(page); 269 __ClearPageTail(p); 270 } 271} 272 273static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags) 274{ 275 int i; 276 277 VM_BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM); 278 /* 279 * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO 280 * and __GFP_HIGHMEM from hard or soft interrupt context. 281 */ 282 VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt()); 283 for (i = 0; i < (1 << order); i++) 284 clear_highpage(page + i); 285} 286 287/* 288 * function for dealing with page's order in buddy system. 289 * zone->lock is already acquired when we use these. 290 * So, we don't need atomic page->flags operations here. 291 */ 292static inline unsigned long page_order(struct page *page) 293{ 294 return page_private(page); 295} 296 297static inline void set_page_order(struct page *page, int order) 298{ 299 set_page_private(page, order); 300 __SetPageBuddy(page); 301} 302 303static inline void rmv_page_order(struct page *page) 304{ 305 __ClearPageBuddy(page); 306 set_page_private(page, 0); 307} 308 309/* 310 * Locate the struct page for both the matching buddy in our 311 * pair (buddy1) and the combined O(n+1) page they form (page). 312 * 313 * 1) Any buddy B1 will have an order O twin B2 which satisfies 314 * the following equation: 315 * B2 = B1 ^ (1 << O) 316 * For example, if the starting buddy (buddy2) is #8 its order 317 * 1 buddy is #10: 318 * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 319 * 320 * 2) Any buddy B will have an order O+1 parent P which 321 * satisfies the following equation: 322 * P = B & ~(1 << O) 323 * 324 * Assumption: *_mem_map is contiguous at least up to MAX_ORDER 325 */ 326static inline struct page * 327__page_find_buddy(struct page *page, unsigned long page_idx, unsigned int order) 328{ 329 unsigned long buddy_idx = page_idx ^ (1 << order); 330 331 return page + (buddy_idx - page_idx); 332} 333 334static inline unsigned long 335__find_combined_index(unsigned long page_idx, unsigned int order) 336{ 337 return (page_idx & ~(1 << order)); 338} 339 340/* 341 * This function checks whether a page is free && is the buddy 342 * we can do coalesce a page and its buddy if 343 * (a) the buddy is not in a hole && 344 * (b) the buddy is in the buddy system && 345 * (c) a page and its buddy have the same order && 346 * (d) a page and its buddy are in the same zone. 347 * 348 * For recording whether a page is in the buddy system, we use PG_buddy. 349 * Setting, clearing, and testing PG_buddy is serialized by zone->lock. 350 * 351 * For recording page's order, we use page_private(page). 352 */ 353static inline int page_is_buddy(struct page *page, struct page *buddy, 354 int order) 355{ 356 if (!pfn_valid_within(page_to_pfn(buddy))) 357 return 0; 358 359 if (page_zone_id(page) != page_zone_id(buddy)) 360 return 0; 361 362 if (PageBuddy(buddy) && page_order(buddy) == order) { 363 BUG_ON(page_count(buddy) != 0); 364 return 1; 365 } 366 return 0; 367} 368 369/* 370 * Freeing function for a buddy system allocator. 371 * 372 * The concept of a buddy system is to maintain direct-mapped table 373 * (containing bit values) for memory blocks of various "orders". 374 * The bottom level table contains the map for the smallest allocatable 375 * units of memory (here, pages), and each level above it describes 376 * pairs of units from the levels below, hence, "buddies". 377 * At a high level, all that happens here is marking the table entry 378 * at the bottom level available, and propagating the changes upward 379 * as necessary, plus some accounting needed to play nicely with other 380 * parts of the VM system. 381 * At each level, we keep a list of pages, which are heads of continuous 382 * free pages of length of (1 << order) and marked with PG_buddy. Page's 383 * order is recorded in page_private(page) field. 384 * So when we are allocating or freeing one, we can derive the state of the 385 * other. That is, if we allocate a small block, and both were 386 * free, the remainder of the region must be split into blocks. 387 * If a block is freed, and its buddy is also free, then this 388 * triggers coalescing into a block of larger size. 389 * 390 * -- wli 391 */ 392 393static inline void __free_one_page(struct page *page, 394 struct zone *zone, unsigned int order) 395{ 396 unsigned long page_idx; 397 int order_size = 1 << order; 398 399 if (unlikely(PageCompound(page))) 400 destroy_compound_page(page, order); 401 402 page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1); 403 404 VM_BUG_ON(page_idx & (order_size - 1)); 405 VM_BUG_ON(bad_range(zone, page)); 406 407 __mod_zone_page_state(zone, NR_FREE_PAGES, order_size); 408 while (order < MAX_ORDER-1) { 409 unsigned long combined_idx; 410 struct free_area *area; 411 struct page *buddy; 412 413 buddy = __page_find_buddy(page, page_idx, order); 414 if (!page_is_buddy(page, buddy, order)) 415 break; /* Move the buddy up one level. */ 416 417 list_del(&buddy->lru); 418 area = zone->free_area + order; 419 area->nr_free--; 420 rmv_page_order(buddy); 421 combined_idx = __find_combined_index(page_idx, order); 422 page = page + (combined_idx - page_idx); 423 page_idx = combined_idx; 424 order++; 425 } 426 set_page_order(page, order); 427 list_add(&page->lru, &zone->free_area[order].free_list); 428 zone->free_area[order].nr_free++; 429} 430 431static inline int free_pages_check(struct page *page) 432{ 433 if (unlikely(page_mapcount(page) | 434 (page->mapping != NULL) | 435 (page_count(page) != 0) | 436 (page->flags & ( 437 1 << PG_lru | 438 1 << PG_private | 439 1 << PG_locked | 440 1 << PG_active | 441 1 << PG_slab | 442 1 << PG_swapcache | 443 1 << PG_writeback | 444 1 << PG_reserved | 445 1 << PG_buddy )))) 446 bad_page(page); 447 /* 448 * PageReclaim == PageTail. It is only an error 449 * for PageReclaim to be set if PageCompound is clear. 450 */ 451 if (unlikely(!PageCompound(page) && PageReclaim(page))) 452 bad_page(page); 453 if (PageDirty(page)) 454 __ClearPageDirty(page); 455 /* 456 * For now, we report if PG_reserved was found set, but do not 457 * clear it, and do not free the page. But we shall soon need 458 * to do more, for when the ZERO_PAGE count wraps negative. 459 */ 460 return PageReserved(page); 461} 462 463/* 464 * Frees a list of pages. 465 * Assumes all pages on list are in same zone, and of same order. 466 * count is the number of pages to free. 467 * 468 * If the zone was previously in an "all pages pinned" state then look to 469 * see if this freeing clears that state. 470 * 471 * And clear the zone's pages_scanned counter, to hold off the "all pages are 472 * pinned" detection logic. 473 */ 474static void free_pages_bulk(struct zone *zone, int count, 475 struct list_head *list, int order) 476{ 477 spin_lock(&zone->lock); 478 zone->all_unreclaimable = 0; 479 zone->pages_scanned = 0; 480 while (count--) { 481 struct page *page; 482 483 VM_BUG_ON(list_empty(list)); 484 page = list_entry(list->prev, struct page, lru); 485 /* have to delete it as __free_one_page list manipulates */ 486 list_del(&page->lru); 487 __free_one_page(page, zone, order); 488 } 489 spin_unlock(&zone->lock); 490} 491 492static void free_one_page(struct zone *zone, struct page *page, int order) 493{ 494 spin_lock(&zone->lock); 495 zone->all_unreclaimable = 0; 496 zone->pages_scanned = 0; 497 __free_one_page(page, zone, order); 498 spin_unlock(&zone->lock); 499} 500 501static void __free_pages_ok(struct page *page, unsigned int order) 502{ 503 unsigned long flags; 504 int i; 505 int reserved = 0; 506 507 for (i = 0 ; i < (1 << order) ; ++i) 508 reserved += free_pages_check(page + i); 509 if (reserved) 510 return; 511 512 if (!PageHighMem(page)) 513 debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order); 514 arch_free_page(page, order); 515 kernel_map_pages(page, 1 << order, 0); 516 517 local_irq_save(flags); 518 __count_vm_events(PGFREE, 1 << order); 519 free_one_page(page_zone(page), page, order); 520 local_irq_restore(flags); 521} 522 523/* 524 * permit the bootmem allocator to evade page validation on high-order frees 525 */ 526void fastcall __init __free_pages_bootmem(struct page *page, unsigned int order) 527{ 528 if (order == 0) { 529 __ClearPageReserved(page); 530 set_page_count(page, 0); 531 set_page_refcounted(page); 532 __free_page(page); 533 } else { 534 int loop; 535 536 prefetchw(page); 537 for (loop = 0; loop < BITS_PER_LONG; loop++) { 538 struct page *p = &page[loop]; 539 540 if (loop + 1 < BITS_PER_LONG) 541 prefetchw(p + 1); 542 __ClearPageReserved(p); 543 set_page_count(p, 0); 544 } 545 546 set_page_refcounted(page); 547 __free_pages(page, order); 548 } 549} 550 551 552/* 553 * The order of subdivision here is critical for the IO subsystem. 554 * Please do not alter this order without good reasons and regression 555 * testing. Specifically, as large blocks of memory are subdivided, 556 * the order in which smaller blocks are delivered depends on the order 557 * they're subdivided in this function. This is the primary factor 558 * influencing the order in which pages are delivered to the IO 559 * subsystem according to empirical testing, and this is also justified 560 * by considering the behavior of a buddy system containing a single 561 * large block of memory acted on by a series of small allocations. 562 * This behavior is a critical factor in sglist merging's success. 563 * 564 * -- wli 565 */ 566static inline void expand(struct zone *zone, struct page *page, 567 int low, int high, struct free_area *area) 568{ 569 unsigned long size = 1 << high; 570 571 while (high > low) { 572 area--; 573 high--; 574 size >>= 1; 575 VM_BUG_ON(bad_range(zone, &page[size])); 576 list_add(&page[size].lru, &area->free_list); 577 area->nr_free++; 578 set_page_order(&page[size], high); 579 } 580} 581 582/* 583 * This page is about to be returned from the page allocator 584 */ 585static int prep_new_page(struct page *page, int order, gfp_t gfp_flags) 586{ 587 if (unlikely(page_mapcount(page) | 588 (page->mapping != NULL) | 589 (page_count(page) != 0) | 590 (page->flags & ( 591 1 << PG_lru | 592 1 << PG_private | 593 1 << PG_locked | 594 1 << PG_active | 595 1 << PG_dirty | 596 1 << PG_reclaim | 597 1 << PG_slab | 598 1 << PG_swapcache | 599 1 << PG_writeback | 600 1 << PG_reserved | 601 1 << PG_buddy )))) 602 bad_page(page); 603 604 /* 605 * For now, we report if PG_reserved was found set, but do not 606 * clear it, and do not allocate the page: as a safety net. 607 */ 608 if (PageReserved(page)) 609 return 1; 610 611 page->flags &= ~(1 << PG_uptodate | 1 << PG_error | 612 1 << PG_referenced | 1 << PG_arch_1 | 613 1 << PG_owner_priv_1 | 1 << PG_mappedtodisk); 614 set_page_private(page, 0); 615 set_page_refcounted(page); 616 617 arch_alloc_page(page, order); 618 kernel_map_pages(page, 1 << order, 1); 619 620 if (gfp_flags & __GFP_ZERO) 621 prep_zero_page(page, order, gfp_flags); 622 623 if (order && (gfp_flags & __GFP_COMP)) 624 prep_compound_page(page, order); 625 626 return 0; 627} 628 629/* 630 * Do the hard work of removing an element from the buddy allocator. 631 * Call me with the zone->lock already held. 632 */ 633static struct page *__rmqueue(struct zone *zone, unsigned int order) 634{ 635 struct free_area * area; 636 unsigned int current_order; 637 struct page *page; 638 639 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 640 area = zone->free_area + current_order; 641 if (list_empty(&area->free_list)) 642 continue; 643 644 page = list_entry(area->free_list.next, struct page, lru); 645 list_del(&page->lru); 646 rmv_page_order(page); 647 area->nr_free--; 648 __mod_zone_page_state(zone, NR_FREE_PAGES, - (1UL << order)); 649 expand(zone, page, order, current_order, area); 650 return page; 651 } 652 653 return NULL; 654} 655 656/* 657 * Obtain a specified number of elements from the buddy allocator, all under 658 * a single hold of the lock, for efficiency. Add them to the supplied list. 659 * Returns the number of new pages which were placed at *list. 660 */ 661static int rmqueue_bulk(struct zone *zone, unsigned int order, 662 unsigned long count, struct list_head *list) 663{ 664 int i; 665 666 spin_lock(&zone->lock); 667 for (i = 0; i < count; ++i) { 668 struct page *page = __rmqueue(zone, order); 669 if (unlikely(page == NULL)) 670 break; 671 list_add_tail(&page->lru, list); 672 } 673 spin_unlock(&zone->lock); 674 return i; 675} 676 677#ifdef CONFIG_NUMA 678/* 679 * Called from the vmstat counter updater to drain pagesets of this 680 * currently executing processor on remote nodes after they have 681 * expired. 682 * 683 * Note that this function must be called with the thread pinned to 684 * a single processor. 685 */ 686void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 687{ 688 unsigned long flags; 689 int to_drain; 690 691 local_irq_save(flags); 692 if (pcp->count >= pcp->batch) 693 to_drain = pcp->batch; 694 else 695 to_drain = pcp->count; 696 free_pages_bulk(zone, to_drain, &pcp->list, 0); 697 pcp->count -= to_drain; 698 local_irq_restore(flags); 699} 700#endif 701 702static void __drain_pages(unsigned int cpu) 703{ 704 unsigned long flags; 705 struct zone *zone; 706 int i; 707 708 for_each_zone(zone) { 709 struct per_cpu_pageset *pset; 710 711 if (!populated_zone(zone)) 712 continue; 713 714 pset = zone_pcp(zone, cpu); 715 for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) { 716 struct per_cpu_pages *pcp; 717 718 pcp = &pset->pcp[i]; 719 local_irq_save(flags); 720 free_pages_bulk(zone, pcp->count, &pcp->list, 0); 721 pcp->count = 0; 722 local_irq_restore(flags); 723 } 724 } 725} 726 727#ifdef CONFIG_PM 728 729void mark_free_pages(struct zone *zone) 730{ 731 unsigned long pfn, max_zone_pfn; 732 unsigned long flags; 733 int order; 734 struct list_head *curr; 735 736 if (!zone->spanned_pages) 737 return; 738 739 spin_lock_irqsave(&zone->lock, flags); 740 741 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 742 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 743 if (pfn_valid(pfn)) { 744 struct page *page = pfn_to_page(pfn); 745 746 if (!swsusp_page_is_forbidden(page)) 747 swsusp_unset_page_free(page); 748 } 749 750 for (order = MAX_ORDER - 1; order >= 0; --order) 751 list_for_each(curr, &zone->free_area[order].free_list) { 752 unsigned long i; 753 754 pfn = page_to_pfn(list_entry(curr, struct page, lru)); 755 for (i = 0; i < (1UL << order); i++) 756 swsusp_set_page_free(pfn_to_page(pfn + i)); 757 } 758 759 spin_unlock_irqrestore(&zone->lock, flags); 760} 761 762/* 763 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 764 */ 765void drain_local_pages(void) 766{ 767 unsigned long flags; 768 769 local_irq_save(flags); 770 __drain_pages(smp_processor_id()); 771 local_irq_restore(flags); 772} 773#endif /* CONFIG_PM */ 774 775/* 776 * Free a 0-order page 777 */ 778static void fastcall free_hot_cold_page(struct page *page, int cold) 779{ 780 struct zone *zone = page_zone(page); 781 struct per_cpu_pages *pcp; 782 unsigned long flags; 783 784 if (PageAnon(page)) 785 page->mapping = NULL; 786 if (free_pages_check(page)) 787 return; 788 789 if (!PageHighMem(page)) 790 debug_check_no_locks_freed(page_address(page), PAGE_SIZE); 791 arch_free_page(page, 0); 792 kernel_map_pages(page, 1, 0); 793 794 pcp = &zone_pcp(zone, get_cpu())->pcp[cold]; 795 local_irq_save(flags); 796 __count_vm_event(PGFREE); 797 list_add(&page->lru, &pcp->list); 798 pcp->count++; 799 if (pcp->count >= pcp->high) { 800 free_pages_bulk(zone, pcp->batch, &pcp->list, 0); 801 pcp->count -= pcp->batch; 802 } 803 local_irq_restore(flags); 804 put_cpu(); 805} 806 807void fastcall free_hot_page(struct page *page) 808{ 809 free_hot_cold_page(page, 0); 810} 811 812void fastcall free_cold_page(struct page *page) 813{ 814 free_hot_cold_page(page, 1); 815} 816 817/* 818 * split_page takes a non-compound higher-order page, and splits it into 819 * n (1<<order) sub-pages: page[0..n] 820 * Each sub-page must be freed individually. 821 * 822 * Note: this is probably too low level an operation for use in drivers. 823 * Please consult with lkml before using this in your driver. 824 */ 825void split_page(struct page *page, unsigned int order) 826{ 827 int i; 828 829 VM_BUG_ON(PageCompound(page)); 830 VM_BUG_ON(!page_count(page)); 831 for (i = 1; i < (1 << order); i++) 832 set_page_refcounted(page + i); 833} 834 835/* 836 * Really, prep_compound_page() should be called from __rmqueue_bulk(). But 837 * we cheat by calling it from here, in the order > 0 path. Saves a branch 838 * or two. 839 */ 840static struct page *buffered_rmqueue(struct zonelist *zonelist, 841 struct zone *zone, int order, gfp_t gfp_flags) 842{ 843 unsigned long flags; 844 struct page *page; 845 int cold = !!(gfp_flags & __GFP_COLD); 846 int cpu; 847 848again: 849 cpu = get_cpu(); 850 if (likely(order == 0)) { 851 struct per_cpu_pages *pcp; 852 853 pcp = &zone_pcp(zone, cpu)->pcp[cold]; 854 local_irq_save(flags); 855 if (!pcp->count) { 856 pcp->count = rmqueue_bulk(zone, 0, 857 pcp->batch, &pcp->list); 858 if (unlikely(!pcp->count)) 859 goto failed; 860 } 861 page = list_entry(pcp->list.next, struct page, lru); 862 list_del(&page->lru); 863 pcp->count--; 864 } else { 865 spin_lock_irqsave(&zone->lock, flags); 866 page = __rmqueue(zone, order); 867 spin_unlock(&zone->lock); 868 if (!page) 869 goto failed; 870 } 871 872 __count_zone_vm_events(PGALLOC, zone, 1 << order); 873 zone_statistics(zonelist, zone); 874 local_irq_restore(flags); 875 put_cpu(); 876 877 VM_BUG_ON(bad_range(zone, page)); 878 if (prep_new_page(page, order, gfp_flags)) 879 goto again; 880 return page; 881 882failed: 883 local_irq_restore(flags); 884 put_cpu(); 885 return NULL; 886} 887 888#define ALLOC_NO_WATERMARKS 0x01 /* don't check watermarks at all */ 889#define ALLOC_WMARK_MIN 0x02 /* use pages_min watermark */ 890#define ALLOC_WMARK_LOW 0x04 /* use pages_low watermark */ 891#define ALLOC_WMARK_HIGH 0x08 /* use pages_high watermark */ 892#define ALLOC_HARDER 0x10 /* try to alloc harder */ 893#define ALLOC_HIGH 0x20 /* __GFP_HIGH set */ 894#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ 895 896#ifdef CONFIG_FAIL_PAGE_ALLOC 897 898static struct fail_page_alloc_attr { 899 struct fault_attr attr; 900 901 u32 ignore_gfp_highmem; 902 u32 ignore_gfp_wait; 903 u32 min_order; 904 905#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 906 907 struct dentry *ignore_gfp_highmem_file; 908 struct dentry *ignore_gfp_wait_file; 909 struct dentry *min_order_file; 910 911#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 912 913} fail_page_alloc = { 914 .attr = FAULT_ATTR_INITIALIZER, 915 .ignore_gfp_wait = 1, 916 .ignore_gfp_highmem = 1, 917 .min_order = 1, 918}; 919 920static int __init setup_fail_page_alloc(char *str) 921{ 922 return setup_fault_attr(&fail_page_alloc.attr, str); 923} 924__setup("fail_page_alloc=", setup_fail_page_alloc); 925 926static int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 927{ 928 if (order < fail_page_alloc.min_order) 929 return 0; 930 if (gfp_mask & __GFP_NOFAIL) 931 return 0; 932 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 933 return 0; 934 if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT)) 935 return 0; 936 937 return should_fail(&fail_page_alloc.attr, 1 << order); 938} 939 940#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 941 942static int __init fail_page_alloc_debugfs(void) 943{ 944 mode_t mode = S_IFREG | S_IRUSR | S_IWUSR; 945 struct dentry *dir; 946 int err; 947 948 err = init_fault_attr_dentries(&fail_page_alloc.attr, 949 "fail_page_alloc"); 950 if (err) 951 return err; 952 dir = fail_page_alloc.attr.dentries.dir; 953 954 fail_page_alloc.ignore_gfp_wait_file = 955 debugfs_create_bool("ignore-gfp-wait", mode, dir, 956 &fail_page_alloc.ignore_gfp_wait); 957 958 fail_page_alloc.ignore_gfp_highmem_file = 959 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 960 &fail_page_alloc.ignore_gfp_highmem); 961 fail_page_alloc.min_order_file = 962 debugfs_create_u32("min-order", mode, dir, 963 &fail_page_alloc.min_order); 964 965 if (!fail_page_alloc.ignore_gfp_wait_file || 966 !fail_page_alloc.ignore_gfp_highmem_file || 967 !fail_page_alloc.min_order_file) { 968 err = -ENOMEM; 969 debugfs_remove(fail_page_alloc.ignore_gfp_wait_file); 970 debugfs_remove(fail_page_alloc.ignore_gfp_highmem_file); 971 debugfs_remove(fail_page_alloc.min_order_file); 972 cleanup_fault_attr_dentries(&fail_page_alloc.attr); 973 } 974 975 return err; 976} 977 978late_initcall(fail_page_alloc_debugfs); 979 980#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 981 982#else /* CONFIG_FAIL_PAGE_ALLOC */ 983 984static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 985{ 986 return 0; 987} 988 989#endif /* CONFIG_FAIL_PAGE_ALLOC */ 990 991/* 992 * Return 1 if free pages are above 'mark'. This takes into account the order 993 * of the allocation. 994 */ 995int zone_watermark_ok(struct zone *z, int order, unsigned long mark, 996 int classzone_idx, int alloc_flags) 997{ 998 /* free_pages my go negative - that's OK */ 999 long min = mark; 1000 long free_pages = zone_page_state(z, NR_FREE_PAGES) - (1 << order) + 1; 1001 int o; 1002 1003 if (alloc_flags & ALLOC_HIGH) 1004 min -= min / 2; 1005 if (alloc_flags & ALLOC_HARDER) 1006 min -= min / 4; 1007 1008 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 1009 return 0; 1010 for (o = 0; o < order; o++) { 1011 /* At the next order, this order's pages become unavailable */ 1012 free_pages -= z->free_area[o].nr_free << o; 1013 1014 /* Require fewer higher order pages to be free */ 1015 min >>= 1; 1016 1017 if (free_pages <= min) 1018 return 0; 1019 } 1020 return 1; 1021} 1022 1023#ifdef CONFIG_NUMA 1024/* 1025 * zlc_setup - Setup for "zonelist cache". Uses cached zone data to 1026 * skip over zones that are not allowed by the cpuset, or that have 1027 * been recently (in last second) found to be nearly full. See further 1028 * comments in mmzone.h. Reduces cache footprint of zonelist scans 1029 * that have to skip over alot of full or unallowed zones. 1030 * 1031 * If the zonelist cache is present in the passed in zonelist, then 1032 * returns a pointer to the allowed node mask (either the current 1033 * tasks mems_allowed, or node_online_map.) 1034 * 1035 * If the zonelist cache is not available for this zonelist, does 1036 * nothing and returns NULL. 1037 * 1038 * If the fullzones BITMAP in the zonelist cache is stale (more than 1039 * a second since last zap'd) then we zap it out (clear its bits.) 1040 * 1041 * We hold off even calling zlc_setup, until after we've checked the 1042 * first zone in the zonelist, on the theory that most allocations will 1043 * be satisfied from that first zone, so best to examine that zone as 1044 * quickly as we can. 1045 */ 1046static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) 1047{ 1048 struct zonelist_cache *zlc; /* cached zonelist speedup info */ 1049 nodemask_t *allowednodes; /* zonelist_cache approximation */ 1050 1051 zlc = zonelist->zlcache_ptr; 1052 if (!zlc) 1053 return NULL; 1054 1055 if (jiffies - zlc->last_full_zap > 1 * HZ) { 1056 bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); 1057 zlc->last_full_zap = jiffies; 1058 } 1059 1060 allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ? 1061 &cpuset_current_mems_allowed : 1062 &node_online_map; 1063 return allowednodes; 1064} 1065 1066/* 1067 * Given 'z' scanning a zonelist, run a couple of quick checks to see 1068 * if it is worth looking at further for free memory: 1069 * 1) Check that the zone isn't thought to be full (doesn't have its 1070 * bit set in the zonelist_cache fullzones BITMAP). 1071 * 2) Check that the zones node (obtained from the zonelist_cache 1072 * z_to_n[] mapping) is allowed in the passed in allowednodes mask. 1073 * Return true (non-zero) if zone is worth looking at further, or 1074 * else return false (zero) if it is not. 1075 * 1076 * This check -ignores- the distinction between various watermarks, 1077 * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is 1078 * found to be full for any variation of these watermarks, it will 1079 * be considered full for up to one second by all requests, unless 1080 * we are so low on memory on all allowed nodes that we are forced 1081 * into the second scan of the zonelist. 1082 * 1083 * In the second scan we ignore this zonelist cache and exactly 1084 * apply the watermarks to all zones, even it is slower to do so. 1085 * We are low on memory in the second scan, and should leave no stone 1086 * unturned looking for a free page. 1087 */ 1088static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zone **z, 1089 nodemask_t *allowednodes) 1090{ 1091 struct zonelist_cache *zlc; /* cached zonelist speedup info */ 1092 int i; /* index of *z in zonelist zones */ 1093 int n; /* node that zone *z is on */ 1094 1095 zlc = zonelist->zlcache_ptr; 1096 if (!zlc) 1097 return 1; 1098 1099 i = z - zonelist->zones; 1100 n = zlc->z_to_n[i]; 1101 1102 /* This zone is worth trying if it is allowed but not full */ 1103 return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones); 1104} 1105 1106/* 1107 * Given 'z' scanning a zonelist, set the corresponding bit in 1108 * zlc->fullzones, so that subsequent attempts to allocate a page 1109 * from that zone don't waste time re-examining it. 1110 */ 1111static void zlc_mark_zone_full(struct zonelist *zonelist, struct zone **z) 1112{ 1113 struct zonelist_cache *zlc; /* cached zonelist speedup info */ 1114 int i; /* index of *z in zonelist zones */ 1115 1116 zlc = zonelist->zlcache_ptr; 1117 if (!zlc) 1118 return; 1119 1120 i = z - zonelist->zones; 1121 1122 set_bit(i, zlc->fullzones); 1123} 1124 1125#else /* CONFIG_NUMA */ 1126 1127static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) 1128{ 1129 return NULL; 1130} 1131 1132static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zone **z, 1133 nodemask_t *allowednodes) 1134{ 1135 return 1; 1136} 1137 1138static void zlc_mark_zone_full(struct zonelist *zonelist, struct zone **z) 1139{ 1140} 1141#endif /* CONFIG_NUMA */ 1142 1143/* 1144 * get_page_from_freelist goes through the zonelist trying to allocate 1145 * a page. 1146 */ 1147static struct page * 1148get_page_from_freelist(gfp_t gfp_mask, unsigned int order, 1149 struct zonelist *zonelist, int alloc_flags) 1150{ 1151 struct zone **z; 1152 struct page *page = NULL; 1153 int classzone_idx = zone_idx(zonelist->zones[0]); 1154 struct zone *zone; 1155 nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */ 1156 int zlc_active = 0; /* set if using zonelist_cache */ 1157 int did_zlc_setup = 0; /* just call zlc_setup() one time */ 1158 1159zonelist_scan: 1160 /* 1161 * Scan zonelist, looking for a zone with enough free. 1162 * See also cpuset_zone_allowed() comment in kernel/cpuset.c. 1163 */ 1164 z = zonelist->zones; 1165 1166 do { 1167 if (NUMA_BUILD && zlc_active && 1168 !zlc_zone_worth_trying(zonelist, z, allowednodes)) 1169 continue; 1170 zone = *z; 1171 if (unlikely(NUMA_BUILD && (gfp_mask & __GFP_THISNODE) && 1172 zone->zone_pgdat != zonelist->zones[0]->zone_pgdat)) 1173 break; 1174 if ((alloc_flags & ALLOC_CPUSET) && 1175 !cpuset_zone_allowed_softwall(zone, gfp_mask)) 1176 goto try_next_zone; 1177 1178 if (!(alloc_flags & ALLOC_NO_WATERMARKS)) { 1179 unsigned long mark; 1180 if (alloc_flags & ALLOC_WMARK_MIN) 1181 mark = zone->pages_min; 1182 else if (alloc_flags & ALLOC_WMARK_LOW) 1183 mark = zone->pages_low; 1184 else 1185 mark = zone->pages_high; 1186 if (!zone_watermark_ok(zone, order, mark, 1187 classzone_idx, alloc_flags)) { 1188 if (!zone_reclaim_mode || 1189 !zone_reclaim(zone, gfp_mask, order)) 1190 goto this_zone_full; 1191 } 1192 } 1193 1194 page = buffered_rmqueue(zonelist, zone, order, gfp_mask); 1195 if (page) 1196 break; 1197this_zone_full: 1198 if (NUMA_BUILD) 1199 zlc_mark_zone_full(zonelist, z); 1200try_next_zone: 1201 if (NUMA_BUILD && !did_zlc_setup) { 1202 /* we do zlc_setup after the first zone is tried */ 1203 allowednodes = zlc_setup(zonelist, alloc_flags); 1204 zlc_active = 1; 1205 did_zlc_setup = 1; 1206 } 1207 } while (*(++z) != NULL); 1208 1209 if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) { 1210 /* Disable zlc cache for second zonelist scan */ 1211 zlc_active = 0; 1212 goto zonelist_scan; 1213 } 1214 return page; 1215} 1216 1217/* 1218 * This is the 'heart' of the zoned buddy allocator. 1219 */ 1220struct page * fastcall 1221__alloc_pages(gfp_t gfp_mask, unsigned int order, 1222 struct zonelist *zonelist) 1223{ 1224 const gfp_t wait = gfp_mask & __GFP_WAIT; 1225 struct zone **z; 1226 struct page *page; 1227 struct reclaim_state reclaim_state; 1228 struct task_struct *p = current; 1229 int do_retry; 1230 int alloc_flags; 1231 int did_some_progress; 1232 1233 might_sleep_if(wait); 1234 1235 if (should_fail_alloc_page(gfp_mask, order)) 1236 return NULL; 1237 1238restart: 1239 z = zonelist->zones; /* the list of zones suitable for gfp_mask */ 1240 1241 if (unlikely(*z == NULL)) { 1242 /* Should this ever happen?? */ 1243 return NULL; 1244 } 1245 1246 page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, order, 1247 zonelist, ALLOC_WMARK_LOW|ALLOC_CPUSET); 1248 if (page) 1249 goto got_pg; 1250 1251 /* 1252 * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and 1253 * __GFP_NOWARN set) should not cause reclaim since the subsystem 1254 * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim 1255 * using a larger set of nodes after it has established that the 1256 * allowed per node queues are empty and that nodes are 1257 * over allocated. 1258 */ 1259 if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE) 1260 goto nopage; 1261 1262 for (z = zonelist->zones; *z; z++) 1263 wakeup_kswapd(*z, order); 1264 1265 /* 1266 * OK, we're below the kswapd watermark and have kicked background 1267 * reclaim. Now things get more complex, so set up alloc_flags according 1268 * to how we want to proceed. 1269 * 1270 * The caller may dip into page reserves a bit more if the caller 1271 * cannot run direct reclaim, or if the caller has realtime scheduling 1272 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 1273 * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH). 1274 */ 1275 alloc_flags = ALLOC_WMARK_MIN; 1276 if ((unlikely(rt_task(p)) && !in_interrupt()) || !wait) 1277 alloc_flags |= ALLOC_HARDER; 1278 if (gfp_mask & __GFP_HIGH) 1279 alloc_flags |= ALLOC_HIGH; 1280 if (wait) 1281 alloc_flags |= ALLOC_CPUSET; 1282 1283 /* 1284 * Go through the zonelist again. Let __GFP_HIGH and allocations 1285 * coming from realtime tasks go deeper into reserves. 1286 * 1287 * This is the last chance, in general, before the goto nopage. 1288 * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc. 1289 * See also cpuset_zone_allowed() comment in kernel/cpuset.c. 1290 */ 1291 page = get_page_from_freelist(gfp_mask, order, zonelist, alloc_flags); 1292 if (page) 1293 goto got_pg; 1294 1295 /* This allocation should allow future memory freeing. */ 1296 1297rebalance: 1298 if (((p->flags & PF_MEMALLOC) || unlikely(test_thread_flag(TIF_MEMDIE))) 1299 && !in_interrupt()) { 1300 if (!(gfp_mask & __GFP_NOMEMALLOC)) { 1301nofail_alloc: 1302 /* go through the zonelist yet again, ignoring mins */ 1303 page = get_page_from_freelist(gfp_mask, order, 1304 zonelist, ALLOC_NO_WATERMARKS); 1305 if (page) 1306 goto got_pg; 1307 if (gfp_mask & __GFP_NOFAIL) { 1308 congestion_wait(WRITE, HZ/50); 1309 goto nofail_alloc; 1310 } 1311 } 1312 goto nopage; 1313 } 1314 1315 /* Atomic allocations - we can't balance anything */ 1316 if (!wait) 1317 goto nopage; 1318 1319 cond_resched(); 1320 1321 /* We now go into synchronous reclaim */ 1322 cpuset_memory_pressure_bump(); 1323 p->flags |= PF_MEMALLOC; 1324 reclaim_state.reclaimed_slab = 0; 1325 p->reclaim_state = &reclaim_state; 1326 1327 did_some_progress = try_to_free_pages(zonelist->zones, gfp_mask); 1328 1329 p->reclaim_state = NULL; 1330 p->flags &= ~PF_MEMALLOC; 1331 1332 cond_resched(); 1333 1334 if (likely(did_some_progress)) { 1335 page = get_page_from_freelist(gfp_mask, order, 1336 zonelist, alloc_flags); 1337 if (page) 1338 goto got_pg; 1339 } else if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) { 1340 /* 1341 * Go through the zonelist yet one more time, keep 1342 * very high watermark here, this is only to catch 1343 * a parallel oom killing, we must fail if we're still 1344 * under heavy pressure. 1345 */ 1346 page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, order, 1347 zonelist, ALLOC_WMARK_HIGH|ALLOC_CPUSET); 1348 if (page) 1349 goto got_pg; 1350 1351 out_of_memory(zonelist, gfp_mask, order); 1352 goto restart; 1353 } 1354 1355 /* 1356 * Don't let big-order allocations loop unless the caller explicitly 1357 * requests that. Wait for some write requests to complete then retry. 1358 * 1359 * In this implementation, __GFP_REPEAT means __GFP_NOFAIL for order 1360 * <= 3, but that may not be true in other implementations. 1361 */ 1362 do_retry = 0; 1363 if (!(gfp_mask & __GFP_NORETRY)) { 1364 if ((order <= 3) || (gfp_mask & __GFP_REPEAT)) 1365 do_retry = 1; 1366 if (gfp_mask & __GFP_NOFAIL) 1367 do_retry = 1; 1368 } 1369 if (do_retry) { 1370 congestion_wait(WRITE, HZ/50); 1371 goto rebalance; 1372 } 1373 1374nopage: 1375 if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) { 1376 printk(KERN_WARNING "%s: page allocation failure." 1377 " order:%d, mode:0x%x\n", 1378 p->comm, order, gfp_mask); 1379 dump_stack(); 1380 show_mem(); 1381 } 1382got_pg: 1383 return page; 1384} 1385 1386EXPORT_SYMBOL(__alloc_pages); 1387 1388/* 1389 * Common helper functions. 1390 */ 1391fastcall unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 1392{ 1393 struct page * page; 1394 page = alloc_pages(gfp_mask, order); 1395 if (!page) 1396 return 0; 1397 return (unsigned long) page_address(page); 1398} 1399 1400EXPORT_SYMBOL(__get_free_pages); 1401 1402fastcall unsigned long get_zeroed_page(gfp_t gfp_mask) 1403{ 1404 struct page * page; 1405 1406 /* 1407 * get_zeroed_page() returns a 32-bit address, which cannot represent 1408 * a highmem page 1409 */ 1410 VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); 1411 1412 page = alloc_pages(gfp_mask | __GFP_ZERO, 0); 1413 if (page) 1414 return (unsigned long) page_address(page); 1415 return 0; 1416} 1417 1418EXPORT_SYMBOL(get_zeroed_page); 1419 1420void __pagevec_free(struct pagevec *pvec) 1421{ 1422 int i = pagevec_count(pvec); 1423 1424 while (--i >= 0) 1425 free_hot_cold_page(pvec->pages[i], pvec->cold); 1426} 1427 1428fastcall void __free_pages(struct page *page, unsigned int order) 1429{ 1430 if (put_page_testzero(page)) { 1431 if (order == 0) 1432 free_hot_page(page); 1433 else 1434 __free_pages_ok(page, order); 1435 } 1436} 1437 1438EXPORT_SYMBOL(__free_pages); 1439 1440fastcall void free_pages(unsigned long addr, unsigned int order) 1441{ 1442 if (addr != 0) { 1443 VM_BUG_ON(!virt_addr_valid((void *)addr)); 1444 __free_pages(virt_to_page((void *)addr), order); 1445 } 1446} 1447 1448EXPORT_SYMBOL(free_pages); 1449 1450static unsigned int nr_free_zone_pages(int offset) 1451{ 1452 /* Just pick one node, since fallback list is circular */ 1453 pg_data_t *pgdat = NODE_DATA(numa_node_id()); 1454 unsigned int sum = 0; 1455 1456 struct zonelist *zonelist = pgdat->node_zonelists + offset; 1457 struct zone **zonep = zonelist->zones; 1458 struct zone *zone; 1459 1460 for (zone = *zonep++; zone; zone = *zonep++) { 1461 unsigned long size = zone->present_pages; 1462 unsigned long high = zone->pages_high; 1463 if (size > high) 1464 sum += size - high; 1465 } 1466 1467 return sum; 1468} 1469 1470/* 1471 * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL 1472 */ 1473unsigned int nr_free_buffer_pages(void) 1474{ 1475 return nr_free_zone_pages(gfp_zone(GFP_USER)); 1476} 1477 1478/* 1479 * Amount of free RAM allocatable within all zones 1480 */ 1481unsigned int nr_free_pagecache_pages(void) 1482{ 1483 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER)); 1484} 1485 1486static inline void show_node(struct zone *zone) 1487{ 1488 if (NUMA_BUILD) 1489 printk("Node %d ", zone_to_nid(zone)); 1490} 1491 1492void si_meminfo(struct sysinfo *val) 1493{ 1494 val->totalram = totalram_pages; 1495 val->sharedram = 0; 1496 val->freeram = global_page_state(NR_FREE_PAGES); 1497 val->bufferram = nr_blockdev_pages(); 1498 val->totalhigh = totalhigh_pages; 1499 val->freehigh = nr_free_highpages(); 1500 val->mem_unit = PAGE_SIZE; 1501} 1502 1503EXPORT_SYMBOL(si_meminfo); 1504 1505#ifdef CONFIG_NUMA 1506void si_meminfo_node(struct sysinfo *val, int nid) 1507{ 1508 pg_data_t *pgdat = NODE_DATA(nid); 1509 1510 val->totalram = pgdat->node_present_pages; 1511 val->freeram = node_page_state(nid, NR_FREE_PAGES); 1512#ifdef CONFIG_HIGHMEM 1513 val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages; 1514 val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM], 1515 NR_FREE_PAGES); 1516#else 1517 val->totalhigh = 0; 1518 val->freehigh = 0; 1519#endif 1520 val->mem_unit = PAGE_SIZE; 1521} 1522#endif 1523 1524#define K(x) ((x) << (PAGE_SHIFT-10)) 1525 1526/* 1527 * Show free area list (used inside shift_scroll-lock stuff) 1528 * We also calculate the percentage fragmentation. We do this by counting the 1529 * memory on each free list with the exception of the first item on the list. 1530 */ 1531void show_free_areas(void) 1532{ 1533 int cpu; 1534 struct zone *zone; 1535 1536 for_each_zone(zone) { 1537 if (!populated_zone(zone)) 1538 continue; 1539 1540 show_node(zone); 1541 printk("%s per-cpu:\n", zone->name); 1542 1543 for_each_online_cpu(cpu) { 1544 struct per_cpu_pageset *pageset; 1545 1546 pageset = zone_pcp(zone, cpu); 1547 1548 printk("CPU %4d: Hot: hi:%5d, btch:%4d usd:%4d " 1549 "Cold: hi:%5d, btch:%4d usd:%4d\n", 1550 cpu, pageset->pcp[0].high, 1551 pageset->pcp[0].batch, pageset->pcp[0].count, 1552 pageset->pcp[1].high, pageset->pcp[1].batch, 1553 pageset->pcp[1].count); 1554 } 1555 } 1556 1557 printk("Active:%lu inactive:%lu dirty:%lu writeback:%lu unstable:%lu\n" 1558 " free:%lu slab:%lu mapped:%lu pagetables:%lu bounce:%lu\n", 1559 global_page_state(NR_ACTIVE), 1560 global_page_state(NR_INACTIVE), 1561 global_page_state(NR_FILE_DIRTY), 1562 global_page_state(NR_WRITEBACK), 1563 global_page_state(NR_UNSTABLE_NFS), 1564 global_page_state(NR_FREE_PAGES), 1565 global_page_state(NR_SLAB_RECLAIMABLE) + 1566 global_page_state(NR_SLAB_UNRECLAIMABLE), 1567 global_page_state(NR_FILE_MAPPED), 1568 global_page_state(NR_PAGETABLE), 1569 global_page_state(NR_BOUNCE)); 1570 1571 for_each_zone(zone) { 1572 int i; 1573 1574 if (!populated_zone(zone)) 1575 continue; 1576 1577 show_node(zone); 1578 printk("%s" 1579 " free:%lukB" 1580 " min:%lukB" 1581 " low:%lukB" 1582 " high:%lukB" 1583 " active:%lukB" 1584 " inactive:%lukB" 1585 " present:%lukB" 1586 " pages_scanned:%lu" 1587 " all_unreclaimable? %s" 1588 "\n", 1589 zone->name, 1590 K(zone_page_state(zone, NR_FREE_PAGES)), 1591 K(zone->pages_min), 1592 K(zone->pages_low), 1593 K(zone->pages_high), 1594 K(zone_page_state(zone, NR_ACTIVE)), 1595 K(zone_page_state(zone, NR_INACTIVE)), 1596 K(zone->present_pages), 1597 zone->pages_scanned, 1598 (zone->all_unreclaimable ? "yes" : "no") 1599 ); 1600 printk("lowmem_reserve[]:"); 1601 for (i = 0; i < MAX_NR_ZONES; i++) 1602 printk(" %lu", zone->lowmem_reserve[i]); 1603 printk("\n"); 1604 } 1605 1606 for_each_zone(zone) { 1607 unsigned long nr[MAX_ORDER], flags, order, total = 0; 1608 1609 if (!populated_zone(zone)) 1610 continue; 1611 1612 show_node(zone); 1613 printk("%s: ", zone->name); 1614 1615 spin_lock_irqsave(&zone->lock, flags); 1616 for (order = 0; order < MAX_ORDER; order++) { 1617 nr[order] = zone->free_area[order].nr_free; 1618 total += nr[order] << order; 1619 } 1620 spin_unlock_irqrestore(&zone->lock, flags); 1621 for (order = 0; order < MAX_ORDER; order++) 1622 printk("%lu*%lukB ", nr[order], K(1UL) << order); 1623 printk("= %lukB\n", K(total)); 1624 } 1625 1626 show_swap_cache_info(); 1627} 1628 1629/* 1630 * Builds allocation fallback zone lists. 1631 * 1632 * Add all populated zones of a node to the zonelist. 1633 */ 1634static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, 1635 int nr_zones, enum zone_type zone_type) 1636{ 1637 struct zone *zone; 1638 1639 BUG_ON(zone_type >= MAX_NR_ZONES); 1640 zone_type++; 1641 1642 do { 1643 zone_type--; 1644 zone = pgdat->node_zones + zone_type; 1645 if (populated_zone(zone)) { 1646 zonelist->zones[nr_zones++] = zone; 1647 check_highest_zone(zone_type); 1648 } 1649 1650 } while (zone_type); 1651 return nr_zones; 1652} 1653 1654 1655/* 1656 * zonelist_order: 1657 * 0 = automatic detection of better ordering. 1658 * 1 = order by ([node] distance, -zonetype) 1659 * 2 = order by (-zonetype, [node] distance) 1660 * 1661 * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create 1662 * the same zonelist. So only NUMA can configure this param. 1663 */ 1664#define ZONELIST_ORDER_DEFAULT 0 1665#define ZONELIST_ORDER_NODE 1 1666#define ZONELIST_ORDER_ZONE 2 1667 1668/* zonelist order in the kernel. 1669 * set_zonelist_order() will set this to NODE or ZONE. 1670 */ 1671static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; 1672static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; 1673 1674 1675#ifdef CONFIG_NUMA 1676/* The value user specified ....changed by config */ 1677static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; 1678/* string for sysctl */ 1679#define NUMA_ZONELIST_ORDER_LEN 16 1680char numa_zonelist_order[16] = "default"; 1681 1682/* 1683 * interface for configure zonelist ordering. 1684 * command line option "numa_zonelist_order" 1685 * = "[dD]efault - default, automatic configuration. 1686 * = "[nN]ode - order by node locality, then by zone within node 1687 * = "[zZ]one - order by zone, then by locality within zone 1688 */ 1689 1690static int __parse_numa_zonelist_order(char *s) 1691{ 1692 if (*s == 'd' || *s == 'D') { 1693 user_zonelist_order = ZONELIST_ORDER_DEFAULT; 1694 } else if (*s == 'n' || *s == 'N') { 1695 user_zonelist_order = ZONELIST_ORDER_NODE; 1696 } else if (*s == 'z' || *s == 'Z') { 1697 user_zonelist_order = ZONELIST_ORDER_ZONE; 1698 } else { 1699 printk(KERN_WARNING 1700 "Ignoring invalid numa_zonelist_order value: " 1701 "%s\n", s); 1702 return -EINVAL; 1703 } 1704 return 0; 1705} 1706 1707static __init int setup_numa_zonelist_order(char *s) 1708{ 1709 if (s) 1710 return __parse_numa_zonelist_order(s); 1711 return 0; 1712} 1713early_param("numa_zonelist_order", setup_numa_zonelist_order); 1714 1715/* 1716 * sysctl handler for numa_zonelist_order 1717 */ 1718int numa_zonelist_order_handler(ctl_table *table, int write, 1719 struct file *file, void __user *buffer, size_t *length, 1720 loff_t *ppos) 1721{ 1722 char saved_string[NUMA_ZONELIST_ORDER_LEN]; 1723 int ret; 1724 1725 if (write) 1726 strncpy(saved_string, (char*)table->data, 1727 NUMA_ZONELIST_ORDER_LEN); 1728 ret = proc_dostring(table, write, file, buffer, length, ppos); 1729 if (ret) 1730 return ret; 1731 if (write) { 1732 int oldval = user_zonelist_order; 1733 if (__parse_numa_zonelist_order((char*)table->data)) { 1734 /* 1735 * bogus value. restore saved string 1736 */ 1737 strncpy((char*)table->data, saved_string, 1738 NUMA_ZONELIST_ORDER_LEN); 1739 user_zonelist_order = oldval; 1740 } else if (oldval != user_zonelist_order) 1741 build_all_zonelists(); 1742 } 1743 return 0; 1744} 1745 1746 1747#define MAX_NODE_LOAD (num_online_nodes()) 1748static int node_load[MAX_NUMNODES]; 1749 1750/** 1751 * find_next_best_node - find the next node that should appear in a given node's fallback list 1752 * @node: node whose fallback list we're appending 1753 * @used_node_mask: nodemask_t of already used nodes 1754 * 1755 * We use a number of factors to determine which is the next node that should 1756 * appear on a given node's fallback list. The node should not have appeared 1757 * already in @node's fallback list, and it should be the next closest node 1758 * according to the distance array (which contains arbitrary distance values 1759 * from each node to each node in the system), and should also prefer nodes 1760 * with no CPUs, since presumably they'll have very little allocation pressure 1761 * on them otherwise. 1762 * It returns -1 if no node is found. 1763 */ 1764static int find_next_best_node(int node, nodemask_t *used_node_mask) 1765{ 1766 int n, val; 1767 int min_val = INT_MAX; 1768 int best_node = -1; 1769 1770 /* Use the local node if we haven't already */ 1771 if (!node_isset(node, *used_node_mask)) { 1772 node_set(node, *used_node_mask); 1773 return node; 1774 } 1775 1776 for_each_online_node(n) { 1777 cpumask_t tmp; 1778 1779 /* Don't want a node to appear more than once */ 1780 if (node_isset(n, *used_node_mask)) 1781 continue; 1782 1783 /* Use the distance array to find the distance */ 1784 val = node_distance(node, n); 1785 1786 /* Penalize nodes under us ("prefer the next node") */ 1787 val += (n < node); 1788 1789 /* Give preference to headless and unused nodes */ 1790 tmp = node_to_cpumask(n); 1791 if (!cpus_empty(tmp)) 1792 val += PENALTY_FOR_NODE_WITH_CPUS; 1793 1794 /* Slight preference for less loaded node */ 1795 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 1796 val += node_load[n]; 1797 1798 if (val < min_val) { 1799 min_val = val; 1800 best_node = n; 1801 } 1802 } 1803 1804 if (best_node >= 0) 1805 node_set(best_node, *used_node_mask); 1806 1807 return best_node; 1808} 1809 1810 1811/* 1812 * Build zonelists ordered by node and zones within node. 1813 * This results in maximum locality--normal zone overflows into local 1814 * DMA zone, if any--but risks exhausting DMA zone. 1815 */ 1816static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) 1817{ 1818 enum zone_type i; 1819 int j; 1820 struct zonelist *zonelist; 1821 1822 for (i = 0; i < MAX_NR_ZONES; i++) { 1823 zonelist = pgdat->node_zonelists + i; 1824 for (j = 0; zonelist->zones[j] != NULL; j++) 1825 ; 1826 j = build_zonelists_node(NODE_DATA(node), zonelist, j, i); 1827 zonelist->zones[j] = NULL; 1828 } 1829} 1830 1831/* 1832 * Build zonelists ordered by zone and nodes within zones. 1833 * This results in conserving DMA zone[s] until all Normal memory is 1834 * exhausted, but results in overflowing to remote node while memory 1835 * may still exist in local DMA zone. 1836 */ 1837static int node_order[MAX_NUMNODES]; 1838 1839static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) 1840{ 1841 enum zone_type i; 1842 int pos, j, node; 1843 int zone_type; /* needs to be signed */ 1844 struct zone *z; 1845 struct zonelist *zonelist; 1846 1847 for (i = 0; i < MAX_NR_ZONES; i++) { 1848 zonelist = pgdat->node_zonelists + i; 1849 pos = 0; 1850 for (zone_type = i; zone_type >= 0; zone_type--) { 1851 for (j = 0; j < nr_nodes; j++) { 1852 node = node_order[j]; 1853 z = &NODE_DATA(node)->node_zones[zone_type]; 1854 if (populated_zone(z)) { 1855 zonelist->zones[pos++] = z; 1856 check_highest_zone(zone_type); 1857 } 1858 } 1859 } 1860 zonelist->zones[pos] = NULL; 1861 } 1862} 1863 1864static int default_zonelist_order(void) 1865{ 1866 int nid, zone_type; 1867 unsigned long low_kmem_size,total_size; 1868 struct zone *z; 1869 int average_size; 1870 /* 1871 * ZONE_DMA and ZONE_DMA32 can be very small area in the sytem. 1872 * If they are really small and used heavily, the system can fall 1873 * into OOM very easily. 1874 * This function detect ZONE_DMA/DMA32 size and confgigures zone order. 1875 */ 1876 /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */ 1877 low_kmem_size = 0; 1878 total_size = 0; 1879 for_each_online_node(nid) { 1880 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 1881 z = &NODE_DATA(nid)->node_zones[zone_type]; 1882 if (populated_zone(z)) { 1883 if (zone_type < ZONE_NORMAL) 1884 low_kmem_size += z->present_pages; 1885 total_size += z->present_pages; 1886 } 1887 } 1888 } 1889 if (!low_kmem_size || /* there are no DMA area. */ 1890 low_kmem_size > total_size/2) /* DMA/DMA32 is big. */ 1891 return ZONELIST_ORDER_NODE; 1892 /* 1893 * look into each node's config. 1894 * If there is a node whose DMA/DMA32 memory is very big area on 1895 * local memory, NODE_ORDER may be suitable. 1896 */ 1897 average_size = total_size / (num_online_nodes() + 1); 1898 for_each_online_node(nid) { 1899 low_kmem_size = 0; 1900 total_size = 0; 1901 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 1902 z = &NODE_DATA(nid)->node_zones[zone_type]; 1903 if (populated_zone(z)) { 1904 if (zone_type < ZONE_NORMAL) 1905 low_kmem_size += z->present_pages; 1906 total_size += z->present_pages; 1907 } 1908 } 1909 if (low_kmem_size && 1910 total_size > average_size && /* ignore small node */ 1911 low_kmem_size > total_size * 70/100) 1912 return ZONELIST_ORDER_NODE; 1913 } 1914 return ZONELIST_ORDER_ZONE; 1915} 1916 1917static void set_zonelist_order(void) 1918{ 1919 if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) 1920 current_zonelist_order = default_zonelist_order(); 1921 else 1922 current_zonelist_order = user_zonelist_order; 1923} 1924 1925static void build_zonelists(pg_data_t *pgdat) 1926{ 1927 int j, node, load; 1928 enum zone_type i; 1929 nodemask_t used_mask; 1930 int local_node, prev_node; 1931 struct zonelist *zonelist; 1932 int order = current_zonelist_order; 1933 1934 /* initialize zonelists */ 1935 for (i = 0; i < MAX_NR_ZONES; i++) { 1936 zonelist = pgdat->node_zonelists + i; 1937 zonelist->zones[0] = NULL; 1938 } 1939 1940 /* NUMA-aware ordering of nodes */ 1941 local_node = pgdat->node_id; 1942 load = num_online_nodes(); 1943 prev_node = local_node; 1944 nodes_clear(used_mask); 1945 1946 memset(node_load, 0, sizeof(node_load)); 1947 memset(node_order, 0, sizeof(node_order)); 1948 j = 0; 1949 1950 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 1951 int distance = node_distance(local_node, node); 1952 1953 /* 1954 * If another node is sufficiently far away then it is better 1955 * to reclaim pages in a zone before going off node. 1956 */ 1957 if (distance > RECLAIM_DISTANCE) 1958 zone_reclaim_mode = 1; 1959 1960 /* 1961 * We don't want to pressure a particular node. 1962 * So adding penalty to the first node in same 1963 * distance group to make it round-robin. 1964 */ 1965 if (distance != node_distance(local_node, prev_node)) 1966 node_load[node] = load; 1967 1968 prev_node = node; 1969 load--; 1970 if (order == ZONELIST_ORDER_NODE) 1971 build_zonelists_in_node_order(pgdat, node); 1972 else 1973 node_order[j++] = node; /* remember order */ 1974 } 1975 1976 if (order == ZONELIST_ORDER_ZONE) { 1977 /* calculate node order -- i.e., DMA last! */ 1978 build_zonelists_in_zone_order(pgdat, j); 1979 } 1980} 1981 1982/* Construct the zonelist performance cache - see further mmzone.h */ 1983static void build_zonelist_cache(pg_data_t *pgdat) 1984{ 1985 int i; 1986 1987 for (i = 0; i < MAX_NR_ZONES; i++) { 1988 struct zonelist *zonelist; 1989 struct zonelist_cache *zlc; 1990 struct zone **z; 1991 1992 zonelist = pgdat->node_zonelists + i; 1993 zonelist->zlcache_ptr = zlc = &zonelist->zlcache; 1994 bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); 1995 for (z = zonelist->zones; *z; z++) 1996 zlc->z_to_n[z - zonelist->zones] = zone_to_nid(*z); 1997 } 1998} 1999 2000 2001#else /* CONFIG_NUMA */ 2002 2003static void set_zonelist_order(void) 2004{ 2005 current_zonelist_order = ZONELIST_ORDER_ZONE; 2006} 2007 2008static void build_zonelists(pg_data_t *pgdat) 2009{ 2010 int node, local_node; 2011 enum zone_type i,j; 2012 2013 local_node = pgdat->node_id; 2014 for (i = 0; i < MAX_NR_ZONES; i++) { 2015 struct zonelist *zonelist; 2016 2017 zonelist = pgdat->node_zonelists + i; 2018 2019 j = build_zonelists_node(pgdat, zonelist, 0, i); 2020 /* 2021 * Now we build the zonelist so that it contains the zones 2022 * of all the other nodes. 2023 * We don't want to pressure a particular node, so when 2024 * building the zones for node N, we make sure that the 2025 * zones coming right after the local ones are those from 2026 * node N+1 (modulo N) 2027 */ 2028 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 2029 if (!node_online(node)) 2030 continue; 2031 j = build_zonelists_node(NODE_DATA(node), zonelist, j, i); 2032 } 2033 for (node = 0; node < local_node; node++) { 2034 if (!node_online(node)) 2035 continue; 2036 j = build_zonelists_node(NODE_DATA(node), zonelist, j, i); 2037 } 2038 2039 zonelist->zones[j] = NULL; 2040 } 2041} 2042 2043/* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */ 2044static void build_zonelist_cache(pg_data_t *pgdat) 2045{ 2046 int i; 2047 2048 for (i = 0; i < MAX_NR_ZONES; i++) 2049 pgdat->node_zonelists[i].zlcache_ptr = NULL; 2050} 2051 2052#endif /* CONFIG_NUMA */ 2053 2054/* return values int ....just for stop_machine_run() */ 2055static int __build_all_zonelists(void *dummy) 2056{ 2057 int nid; 2058 2059 for_each_online_node(nid) { 2060 build_zonelists(NODE_DATA(nid)); 2061 build_zonelist_cache(NODE_DATA(nid)); 2062 } 2063 return 0; 2064} 2065 2066void build_all_zonelists(void) 2067{ 2068 set_zonelist_order(); 2069 2070 if (system_state == SYSTEM_BOOTING) { 2071 __build_all_zonelists(NULL); 2072 cpuset_init_current_mems_allowed(); 2073 } else { 2074 /* we have to stop all cpus to guaranntee there is no user 2075 of zonelist */ 2076 stop_machine_run(__build_all_zonelists, NULL, NR_CPUS); 2077 /* cpuset refresh routine should be here */ 2078 } 2079 vm_total_pages = nr_free_pagecache_pages(); 2080 printk("Built %i zonelists in %s order. Total pages: %ld\n", 2081 num_online_nodes(), 2082 zonelist_order_name[current_zonelist_order], 2083 vm_total_pages); 2084#ifdef CONFIG_NUMA 2085 printk("Policy zone: %s\n", zone_names[policy_zone]); 2086#endif 2087} 2088 2089/* 2090 * Helper functions to size the waitqueue hash table. 2091 * Essentially these want to choose hash table sizes sufficiently 2092 * large so that collisions trying to wait on pages are rare. 2093 * But in fact, the number of active page waitqueues on typical 2094 * systems is ridiculously low, less than 200. So this is even 2095 * conservative, even though it seems large. 2096 * 2097 * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to 2098 * waitqueues, i.e. the size of the waitq table given the number of pages. 2099 */ 2100#define PAGES_PER_WAITQUEUE 256 2101 2102#ifndef CONFIG_MEMORY_HOTPLUG 2103static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) 2104{ 2105 unsigned long size = 1; 2106 2107 pages /= PAGES_PER_WAITQUEUE; 2108 2109 while (size < pages) 2110 size <<= 1; 2111 2112 /* 2113 * Once we have dozens or even hundreds of threads sleeping 2114 * on IO we've got bigger problems than wait queue collision. 2115 * Limit the size of the wait table to a reasonable size. 2116 */ 2117 size = min(size, 4096UL); 2118 2119 return max(size, 4UL); 2120} 2121#else 2122/* 2123 * A zone's size might be changed by hot-add, so it is not possible to determine 2124 * a suitable size for its wait_table. So we use the maximum size now. 2125 * 2126 * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: 2127 * 2128 * i386 (preemption config) : 4096 x 16 = 64Kbyte. 2129 * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. 2130 * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. 2131 * 2132 * The maximum entries are prepared when a zone's memory is (512K + 256) pages 2133 * or more by the traditional way. (See above). It equals: 2134 * 2135 * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. 2136 * ia64(16K page size) : = ( 8G + 4M)byte. 2137 * powerpc (64K page size) : = (32G +16M)byte. 2138 */ 2139static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) 2140{ 2141 return 4096UL; 2142} 2143#endif 2144 2145/* 2146 * This is an integer logarithm so that shifts can be used later 2147 * to extract the more random high bits from the multiplicative 2148 * hash function before the remainder is taken. 2149 */ 2150static inline unsigned long wait_table_bits(unsigned long size) 2151{ 2152 return ffz(~size); 2153} 2154 2155#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) 2156 2157/* 2158 * Initially all pages are reserved - free ones are freed 2159 * up by free_all_bootmem() once the early boot process is 2160 * done. Non-atomic initialization, single-pass. 2161 */ 2162void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 2163 unsigned long start_pfn, enum memmap_context context) 2164{ 2165 struct page *page; 2166 unsigned long end_pfn = start_pfn + size; 2167 unsigned long pfn; 2168 2169 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 2170 /* 2171 * There can be holes in boot-time mem_map[]s 2172 * handed to this function. They do not 2173 * exist on hotplugged memory. 2174 */ 2175 if (context == MEMMAP_EARLY) { 2176 if (!early_pfn_valid(pfn)) 2177 continue; 2178 if (!early_pfn_in_nid(pfn, nid)) 2179 continue; 2180 } 2181 page = pfn_to_page(pfn); 2182 set_page_links(page, zone, nid, pfn); 2183 init_page_count(page); 2184 reset_page_mapcount(page); 2185 SetPageReserved(page); 2186 INIT_LIST_HEAD(&page->lru); 2187#ifdef WANT_PAGE_VIRTUAL 2188 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 2189 if (!is_highmem_idx(zone)) 2190 set_page_address(page, __va(pfn << PAGE_SHIFT)); 2191#endif 2192 } 2193} 2194 2195static void __meminit zone_init_free_lists(struct pglist_data *pgdat, 2196 struct zone *zone, unsigned long size) 2197{ 2198 int order; 2199 for (order = 0; order < MAX_ORDER ; order++) { 2200 INIT_LIST_HEAD(&zone->free_area[order].free_list); 2201 zone->free_area[order].nr_free = 0; 2202 } 2203} 2204 2205#ifndef __HAVE_ARCH_MEMMAP_INIT 2206#define memmap_init(size, nid, zone, start_pfn) \ 2207 memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) 2208#endif 2209 2210static int __devinit zone_batchsize(struct zone *zone) 2211{ 2212 int batch; 2213 2214 /* 2215 * The per-cpu-pages pools are set to around 1000th of the 2216 * size of the zone. But no more than 1/2 of a meg. 2217 * 2218 * OK, so we don't know how big the cache is. So guess. 2219 */ 2220 batch = zone->present_pages / 1024; 2221 if (batch * PAGE_SIZE > 512 * 1024) 2222 batch = (512 * 1024) / PAGE_SIZE; 2223 batch /= 4; /* We effectively *= 4 below */ 2224 if (batch < 1) 2225 batch = 1; 2226 2227 /* 2228 * Clamp the batch to a 2^n - 1 value. Having a power 2229 * of 2 value was found to be more likely to have 2230 * suboptimal cache aliasing properties in some cases. 2231 * 2232 * For example if 2 tasks are alternately allocating 2233 * batches of pages, one task can end up with a lot 2234 * of pages of one half of the possible page colors 2235 * and the other with pages of the other colors. 2236 */ 2237 batch = (1 << (fls(batch + batch/2)-1)) - 1; 2238 2239 return batch; 2240} 2241 2242inline void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 2243{ 2244 struct per_cpu_pages *pcp; 2245 2246 memset(p, 0, sizeof(*p)); 2247 2248 pcp = &p->pcp[0]; /* hot */ 2249 pcp->count = 0; 2250 pcp->high = 6 * batch; 2251 pcp->batch = max(1UL, 1 * batch); 2252 INIT_LIST_HEAD(&pcp->list); 2253 2254 pcp = &p->pcp[1]; /* cold*/ 2255 pcp->count = 0; 2256 pcp->high = 2 * batch; 2257 pcp->batch = max(1UL, batch/2); 2258 INIT_LIST_HEAD(&pcp->list); 2259} 2260 2261/* 2262 * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist 2263 * to the value high for the pageset p. 2264 */ 2265 2266static void setup_pagelist_highmark(struct per_cpu_pageset *p, 2267 unsigned long high) 2268{ 2269 struct per_cpu_pages *pcp; 2270 2271 pcp = &p->pcp[0]; /* hot list */ 2272 pcp->high = high; 2273 pcp->batch = max(1UL, high/4); 2274 if ((high/4) > (PAGE_SHIFT * 8)) 2275 pcp->batch = PAGE_SHIFT * 8; 2276} 2277 2278 2279#ifdef CONFIG_NUMA 2280/* 2281 * Boot pageset table. One per cpu which is going to be used for all 2282 * zones and all nodes. The parameters will be set in such a way 2283 * that an item put on a list will immediately be handed over to 2284 * the buddy list. This is safe since pageset manipulation is done 2285 * with interrupts disabled. 2286 * 2287 * Some NUMA counter updates may also be caught by the boot pagesets. 2288 * 2289 * The boot_pagesets must be kept even after bootup is complete for 2290 * unused processors and/or zones. They do play a role for bootstrapping 2291 * hotplugged processors. 2292 * 2293 * zoneinfo_show() and maybe other functions do 2294 * not check if the processor is online before following the pageset pointer. 2295 * Other parts of the kernel may not check if the zone is available. 2296 */ 2297static struct per_cpu_pageset boot_pageset[NR_CPUS]; 2298 2299/* 2300 * Dynamically allocate memory for the 2301 * per cpu pageset array in struct zone. 2302 */ 2303static int __cpuinit process_zones(int cpu) 2304{ 2305 struct zone *zone, *dzone; 2306 2307 for_each_zone(zone) { 2308 2309 if (!populated_zone(zone)) 2310 continue; 2311 2312 zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset), 2313 GFP_KERNEL, cpu_to_node(cpu)); 2314 if (!zone_pcp(zone, cpu)) 2315 goto bad; 2316 2317 setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone)); 2318 2319 if (percpu_pagelist_fraction) 2320 setup_pagelist_highmark(zone_pcp(zone, cpu), 2321 (zone->present_pages / percpu_pagelist_fraction)); 2322 } 2323 2324 return 0; 2325bad: 2326 for_each_zone(dzone) { 2327 if (dzone == zone) 2328 break; 2329 kfree(zone_pcp(dzone, cpu)); 2330 zone_pcp(dzone, cpu) = NULL; 2331 } 2332 return -ENOMEM; 2333} 2334 2335static inline void free_zone_pagesets(int cpu) 2336{ 2337 struct zone *zone; 2338 2339 for_each_zone(zone) { 2340 struct per_cpu_pageset *pset = zone_pcp(zone, cpu); 2341 2342 /* Free per_cpu_pageset if it is slab allocated */ 2343 if (pset != &boot_pageset[cpu]) 2344 kfree(pset); 2345 zone_pcp(zone, cpu) = NULL; 2346 } 2347} 2348 2349static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb, 2350 unsigned long action, 2351 void *hcpu) 2352{ 2353 int cpu = (long)hcpu; 2354 int ret = NOTIFY_OK; 2355 2356 switch (action) { 2357 case CPU_UP_PREPARE: 2358 case CPU_UP_PREPARE_FROZEN: 2359 if (process_zones(cpu)) 2360 ret = NOTIFY_BAD; 2361 break; 2362 case CPU_UP_CANCELED: 2363 case CPU_UP_CANCELED_FROZEN: 2364 case CPU_DEAD: 2365 case CPU_DEAD_FROZEN: 2366 free_zone_pagesets(cpu); 2367 break; 2368 default: 2369 break; 2370 } 2371 return ret; 2372} 2373 2374static struct notifier_block __cpuinitdata pageset_notifier = 2375 { &pageset_cpuup_callback, NULL, 0 }; 2376 2377void __init setup_per_cpu_pageset(void) 2378{ 2379 int err; 2380 2381 /* Initialize per_cpu_pageset for cpu 0. 2382 * A cpuup callback will do this for every cpu 2383 * as it comes online 2384 */ 2385 err = process_zones(smp_processor_id()); 2386 BUG_ON(err); 2387 register_cpu_notifier(&pageset_notifier); 2388} 2389 2390#endif 2391 2392static noinline __init_refok 2393int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) 2394{ 2395 int i; 2396 struct pglist_data *pgdat = zone->zone_pgdat; 2397 size_t alloc_size; 2398 2399 /* 2400 * The per-page waitqueue mechanism uses hashed waitqueues 2401 * per zone. 2402 */ 2403 zone->wait_table_hash_nr_entries = 2404 wait_table_hash_nr_entries(zone_size_pages); 2405 zone->wait_table_bits = 2406 wait_table_bits(zone->wait_table_hash_nr_entries); 2407 alloc_size = zone->wait_table_hash_nr_entries 2408 * sizeof(wait_queue_head_t); 2409 2410 if (system_state == SYSTEM_BOOTING) { 2411 zone->wait_table = (wait_queue_head_t *) 2412 alloc_bootmem_node(pgdat, alloc_size); 2413 } else { 2414 /* 2415 * This case means that a zone whose size was 0 gets new memory 2416 * via memory hot-add. 2417 * But it may be the case that a new node was hot-added. In 2418 * this case vmalloc() will not be able to use this new node's 2419 * memory - this wait_table must be initialized to use this new 2420 * node itself as well. 2421 * To use this new node's memory, further consideration will be 2422 * necessary. 2423 */ 2424 zone->wait_table = (wait_queue_head_t *)vmalloc(alloc_size); 2425 } 2426 if (!zone->wait_table) 2427 return -ENOMEM; 2428 2429 for(i = 0; i < zone->wait_table_hash_nr_entries; ++i) 2430 init_waitqueue_head(zone->wait_table + i); 2431 2432 return 0; 2433} 2434 2435static __meminit void zone_pcp_init(struct zone *zone) 2436{ 2437 int cpu; 2438 unsigned long batch = zone_batchsize(zone); 2439 2440 for (cpu = 0; cpu < NR_CPUS; cpu++) { 2441#ifdef CONFIG_NUMA 2442 /* Early boot. Slab allocator not functional yet */ 2443 zone_pcp(zone, cpu) = &boot_pageset[cpu]; 2444 setup_pageset(&boot_pageset[cpu],0); 2445#else 2446 setup_pageset(zone_pcp(zone,cpu), batch); 2447#endif 2448 } 2449 if (zone->present_pages) 2450 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", 2451 zone->name, zone->present_pages, batch); 2452} 2453 2454__meminit int init_currently_empty_zone(struct zone *zone, 2455 unsigned long zone_start_pfn, 2456 unsigned long size, 2457 enum memmap_context context) 2458{ 2459 struct pglist_data *pgdat = zone->zone_pgdat; 2460 int ret; 2461 ret = zone_wait_table_init(zone, size); 2462 if (ret) 2463 return ret; 2464 pgdat->nr_zones = zone_idx(zone) + 1; 2465 2466 zone->zone_start_pfn = zone_start_pfn; 2467 2468 memmap_init(size, pgdat->node_id, zone_idx(zone), zone_start_pfn); 2469 2470 zone_init_free_lists(pgdat, zone, zone->spanned_pages); 2471 2472 return 0; 2473} 2474 2475#ifdef CONFIG_ARCH_POPULATES_NODE_MAP 2476/* 2477 * Basic iterator support. Return the first range of PFNs for a node 2478 * Note: nid == MAX_NUMNODES returns first region regardless of node 2479 */ 2480static int __meminit first_active_region_index_in_nid(int nid) 2481{ 2482 int i; 2483 2484 for (i = 0; i < nr_nodemap_entries; i++) 2485 if (nid == MAX_NUMNODES || early_node_map[i].nid == nid) 2486 return i; 2487 2488 return -1; 2489} 2490 2491/* 2492 * Basic iterator support. Return the next active range of PFNs for a node 2493 * Note: nid == MAX_NUMNODES returns next region regardles of node 2494 */ 2495static int __meminit next_active_region_index_in_nid(int index, int nid) 2496{ 2497 for (index = index + 1; index < nr_nodemap_entries; index++) 2498 if (nid == MAX_NUMNODES || early_node_map[index].nid == nid) 2499 return index; 2500 2501 return -1; 2502} 2503 2504#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 2505/* 2506 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 2507 * Architectures may implement their own version but if add_active_range() 2508 * was used and there are no special requirements, this is a convenient 2509 * alternative 2510 */ 2511int __meminit early_pfn_to_nid(unsigned long pfn) 2512{ 2513 int i; 2514 2515 for (i = 0; i < nr_nodemap_entries; i++) { 2516 unsigned long start_pfn = early_node_map[i].start_pfn; 2517 unsigned long end_pfn = early_node_map[i].end_pfn; 2518 2519 if (start_pfn <= pfn && pfn < end_pfn) 2520 return early_node_map[i].nid; 2521 } 2522 2523 return 0; 2524} 2525#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 2526 2527/* Basic iterator support to walk early_node_map[] */ 2528#define for_each_active_range_index_in_nid(i, nid) \ 2529 for (i = first_active_region_index_in_nid(nid); i != -1; \ 2530 i = next_active_region_index_in_nid(i, nid)) 2531 2532/** 2533 * free_bootmem_with_active_regions - Call free_bootmem_node for each active range 2534 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 2535 * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node 2536 * 2537 * If an architecture guarantees that all ranges registered with 2538 * add_active_ranges() contain no holes and may be freed, this 2539 * this function may be used instead of calling free_bootmem() manually. 2540 */ 2541void __init free_bootmem_with_active_regions(int nid, 2542 unsigned long max_low_pfn) 2543{ 2544 int i; 2545 2546 for_each_active_range_index_in_nid(i, nid) { 2547 unsigned long size_pages = 0; 2548 unsigned long end_pfn = early_node_map[i].end_pfn; 2549 2550 if (early_node_map[i].start_pfn >= max_low_pfn) 2551 continue; 2552 2553 if (end_pfn > max_low_pfn) 2554 end_pfn = max_low_pfn; 2555 2556 size_pages = end_pfn - early_node_map[i].start_pfn; 2557 free_bootmem_node(NODE_DATA(early_node_map[i].nid), 2558 PFN_PHYS(early_node_map[i].start_pfn), 2559 size_pages << PAGE_SHIFT); 2560 } 2561} 2562 2563/** 2564 * sparse_memory_present_with_active_regions - Call memory_present for each active range 2565 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 2566 * 2567 * If an architecture guarantees that all ranges registered with 2568 * add_active_ranges() contain no holes and may be freed, this 2569 * function may be used instead of calling memory_present() manually. 2570 */ 2571void __init sparse_memory_present_with_active_regions(int nid) 2572{ 2573 int i; 2574 2575 for_each_active_range_index_in_nid(i, nid) 2576 memory_present(early_node_map[i].nid, 2577 early_node_map[i].start_pfn, 2578 early_node_map[i].end_pfn); 2579} 2580 2581/** 2582 * push_node_boundaries - Push node boundaries to at least the requested boundary 2583 * @nid: The nid of the node to push the boundary for 2584 * @start_pfn: The start pfn of the node 2585 * @end_pfn: The end pfn of the node 2586 * 2587 * In reserve-based hot-add, mem_map is allocated that is unused until hotadd 2588 * time. Specifically, on x86_64, SRAT will report ranges that can potentially 2589 * be hotplugged even though no physical memory exists. This function allows 2590 * an arch to push out the node boundaries so mem_map is allocated that can 2591 * be used later. 2592 */ 2593#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE 2594void __init push_node_boundaries(unsigned int nid, 2595 unsigned long start_pfn, unsigned long end_pfn) 2596{ 2597 printk(KERN_DEBUG "Entering push_node_boundaries(%u, %lu, %lu)\n", 2598 nid, start_pfn, end_pfn); 2599 2600 /* Initialise the boundary for this node if necessary */ 2601 if (node_boundary_end_pfn[nid] == 0) 2602 node_boundary_start_pfn[nid] = -1UL; 2603 2604 /* Update the boundaries */ 2605 if (node_boundary_start_pfn[nid] > start_pfn) 2606 node_boundary_start_pfn[nid] = start_pfn; 2607 if (node_boundary_end_pfn[nid] < end_pfn) 2608 node_boundary_end_pfn[nid] = end_pfn; 2609} 2610 2611/* If necessary, push the node boundary out for reserve hotadd */ 2612static void __meminit account_node_boundary(unsigned int nid, 2613 unsigned long *start_pfn, unsigned long *end_pfn) 2614{ 2615 printk(KERN_DEBUG "Entering account_node_boundary(%u, %lu, %lu)\n", 2616 nid, *start_pfn, *end_pfn); 2617 2618 /* Return if boundary information has not been provided */ 2619 if (node_boundary_end_pfn[nid] == 0) 2620 return; 2621 2622 /* Check the boundaries and update if necessary */ 2623 if (node_boundary_start_pfn[nid] < *start_pfn) 2624 *start_pfn = node_boundary_start_pfn[nid]; 2625 if (node_boundary_end_pfn[nid] > *end_pfn) 2626 *end_pfn = node_boundary_end_pfn[nid]; 2627} 2628#else 2629void __init push_node_boundaries(unsigned int nid, 2630 unsigned long start_pfn, unsigned long end_pfn) {} 2631 2632static void __meminit account_node_boundary(unsigned int nid, 2633 unsigned long *start_pfn, unsigned long *end_pfn) {} 2634#endif 2635 2636 2637/** 2638 * get_pfn_range_for_nid - Return the start and end page frames for a node 2639 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 2640 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 2641 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 2642 * 2643 * It returns the start and end page frame of a node based on information 2644 * provided by an arch calling add_active_range(). If called for a node 2645 * with no available memory, a warning is printed and the start and end 2646 * PFNs will be 0. 2647 */ 2648void __meminit get_pfn_range_for_nid(unsigned int nid, 2649 unsigned long *start_pfn, unsigned long *end_pfn) 2650{ 2651 int i; 2652 *start_pfn = -1UL; 2653 *end_pfn = 0; 2654 2655 for_each_active_range_index_in_nid(i, nid) { 2656 *start_pfn = min(*start_pfn, early_node_map[i].start_pfn); 2657 *end_pfn = max(*end_pfn, early_node_map[i].end_pfn); 2658 } 2659 2660 if (*start_pfn == -1UL) { 2661 printk(KERN_WARNING "Node %u active with no memory\n", nid); 2662 *start_pfn = 0; 2663 } 2664 2665 /* Push the node boundaries out if requested */ 2666 account_node_boundary(nid, start_pfn, end_pfn); 2667} 2668 2669/* 2670 * Return the number of pages a zone spans in a node, including holes 2671 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 2672 */ 2673static unsigned long __meminit zone_spanned_pages_in_node(int nid, 2674 unsigned long zone_type, 2675 unsigned long *ignored) 2676{ 2677 unsigned long node_start_pfn, node_end_pfn; 2678 unsigned long zone_start_pfn, zone_end_pfn; 2679 2680 /* Get the start and end of the node and zone */ 2681 get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn); 2682 zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; 2683 zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; 2684 2685 /* Check that this node has pages within the zone's required range */ 2686 if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn) 2687 return 0; 2688 2689 /* Move the zone boundaries inside the node if necessary */ 2690 zone_end_pfn = min(zone_end_pfn, node_end_pfn); 2691 zone_start_pfn = max(zone_start_pfn, node_start_pfn); 2692 2693 /* Return the spanned pages */ 2694 return zone_end_pfn - zone_start_pfn; 2695} 2696 2697/* 2698 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 2699 * then all holes in the requested range will be accounted for. 2700 */ 2701unsigned long __meminit __absent_pages_in_range(int nid, 2702 unsigned long range_start_pfn, 2703 unsigned long range_end_pfn) 2704{ 2705 int i = 0; 2706 unsigned long prev_end_pfn = 0, hole_pages = 0; 2707 unsigned long start_pfn; 2708 2709 /* Find the end_pfn of the first active range of pfns in the node */ 2710 i = first_active_region_index_in_nid(nid); 2711 if (i == -1) 2712 return 0; 2713 2714 /* Account for ranges before physical memory on this node */ 2715 if (early_node_map[i].start_pfn > range_start_pfn) 2716 hole_pages = early_node_map[i].start_pfn - range_start_pfn; 2717 2718 prev_end_pfn = early_node_map[i].start_pfn; 2719 2720 /* Find all holes for the zone within the node */ 2721 for (; i != -1; i = next_active_region_index_in_nid(i, nid)) { 2722 2723 /* No need to continue if prev_end_pfn is outside the zone */ 2724 if (prev_end_pfn >= range_end_pfn) 2725 break; 2726 2727 /* Make sure the end of the zone is not within the hole */ 2728 start_pfn = min(early_node_map[i].start_pfn, range_end_pfn); 2729 prev_end_pfn = max(prev_end_pfn, range_start_pfn); 2730 2731 /* Update the hole size cound and move on */ 2732 if (start_pfn > range_start_pfn) { 2733 BUG_ON(prev_end_pfn > start_pfn); 2734 hole_pages += start_pfn - prev_end_pfn; 2735 } 2736 prev_end_pfn = early_node_map[i].end_pfn; 2737 } 2738 2739 /* Account for ranges past physical memory on this node */ 2740 if (range_end_pfn > prev_end_pfn) 2741 hole_pages += range_end_pfn - 2742 max(range_start_pfn, prev_end_pfn); 2743 2744 return hole_pages; 2745} 2746 2747/** 2748 * absent_pages_in_range - Return number of page frames in holes within a range 2749 * @start_pfn: The start PFN to start searching for holes 2750 * @end_pfn: The end PFN to stop searching for holes 2751 * 2752 * It returns the number of pages frames in memory holes within a range. 2753 */ 2754unsigned long __init absent_pages_in_range(unsigned long start_pfn, 2755 unsigned long end_pfn) 2756{ 2757 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 2758} 2759 2760/* Return the number of page frames in holes in a zone on a node */ 2761static unsigned long __meminit zone_absent_pages_in_node(int nid, 2762 unsigned long zone_type, 2763 unsigned long *ignored) 2764{ 2765 unsigned long node_start_pfn, node_end_pfn; 2766 unsigned long zone_start_pfn, zone_end_pfn; 2767 2768 get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn); 2769 zone_start_pfn = max(arch_zone_lowest_possible_pfn[zone_type], 2770 node_start_pfn); 2771 zone_end_pfn = min(arch_zone_highest_possible_pfn[zone_type], 2772 node_end_pfn); 2773 2774 return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 2775} 2776 2777#else 2778static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, 2779 unsigned long zone_type, 2780 unsigned long *zones_size) 2781{ 2782 return zones_size[zone_type]; 2783} 2784 2785static inline unsigned long __meminit zone_absent_pages_in_node(int nid, 2786 unsigned long zone_type, 2787 unsigned long *zholes_size) 2788{ 2789 if (!zholes_size) 2790 return 0; 2791 2792 return zholes_size[zone_type]; 2793} 2794 2795#endif 2796 2797static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, 2798 unsigned long *zones_size, unsigned long *zholes_size) 2799{ 2800 unsigned long realtotalpages, totalpages = 0; 2801 enum zone_type i; 2802 2803 for (i = 0; i < MAX_NR_ZONES; i++) 2804 totalpages += zone_spanned_pages_in_node(pgdat->node_id, i, 2805 zones_size); 2806 pgdat->node_spanned_pages = totalpages; 2807 2808 realtotalpages = totalpages; 2809 for (i = 0; i < MAX_NR_ZONES; i++) 2810 realtotalpages -= 2811 zone_absent_pages_in_node(pgdat->node_id, i, 2812 zholes_size); 2813 pgdat->node_present_pages = realtotalpages; 2814 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 2815 realtotalpages); 2816} 2817 2818/* 2819 * Set up the zone data structures: 2820 * - mark all pages reserved 2821 * - mark all memory queues empty 2822 * - clear the memory bitmaps 2823 */ 2824static void __meminit free_area_init_core(struct pglist_data *pgdat, 2825 unsigned long *zones_size, unsigned long *zholes_size) 2826{ 2827 enum zone_type j; 2828 int nid = pgdat->node_id; 2829 unsigned long zone_start_pfn = pgdat->node_start_pfn; 2830 int ret; 2831 2832 pgdat_resize_init(pgdat); 2833 pgdat->nr_zones = 0; 2834 init_waitqueue_head(&pgdat->kswapd_wait); 2835 pgdat->kswapd_max_order = 0; 2836 2837 for (j = 0; j < MAX_NR_ZONES; j++) { 2838 struct zone *zone = pgdat->node_zones + j; 2839 unsigned long size, realsize, memmap_pages; 2840 2841 size = zone_spanned_pages_in_node(nid, j, zones_size); 2842 realsize = size - zone_absent_pages_in_node(nid, j, 2843 zholes_size); 2844 2845 /* 2846 * Adjust realsize so that it accounts for how much memory 2847 * is used by this zone for memmap. This affects the watermark 2848 * and per-cpu initialisations 2849 */ 2850 memmap_pages = (size * sizeof(struct page)) >> PAGE_SHIFT; 2851 if (realsize >= memmap_pages) { 2852 realsize -= memmap_pages; 2853 printk(KERN_DEBUG 2854 " %s zone: %lu pages used for memmap\n", 2855 zone_names[j], memmap_pages); 2856 } else 2857 printk(KERN_WARNING 2858 " %s zone: %lu pages exceeds realsize %lu\n", 2859 zone_names[j], memmap_pages, realsize); 2860 2861 /* Account for reserved pages */ 2862 if (j == 0 && realsize > dma_reserve) { 2863 realsize -= dma_reserve; 2864 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 2865 zone_names[0], dma_reserve); 2866 } 2867 2868 if (!is_highmem_idx(j)) 2869 nr_kernel_pages += realsize; 2870 nr_all_pages += realsize; 2871 2872 zone->spanned_pages = size; 2873 zone->present_pages = realsize; 2874#ifdef CONFIG_NUMA 2875 zone->node = nid; 2876 zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio) 2877 / 100; 2878 zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100; 2879#endif 2880 zone->name = zone_names[j]; 2881 spin_lock_init(&zone->lock); 2882 spin_lock_init(&zone->lru_lock); 2883 zone_seqlock_init(zone); 2884 zone->zone_pgdat = pgdat; 2885 2886 zone->prev_priority = DEF_PRIORITY; 2887 2888 zone_pcp_init(zone); 2889 INIT_LIST_HEAD(&zone->active_list); 2890 INIT_LIST_HEAD(&zone->inactive_list); 2891 zone->nr_scan_active = 0; 2892 zone->nr_scan_inactive = 0; 2893 zap_zone_vm_stats(zone); 2894 atomic_set(&zone->reclaim_in_progress, 0); 2895 if (!size) 2896 continue; 2897 2898 ret = init_currently_empty_zone(zone, zone_start_pfn, 2899 size, MEMMAP_EARLY); 2900 BUG_ON(ret); 2901 zone_start_pfn += size; 2902 } 2903} 2904 2905static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat) 2906{ 2907 /* Skip empty nodes */ 2908 if (!pgdat->node_spanned_pages) 2909 return; 2910 2911#ifdef CONFIG_FLAT_NODE_MEM_MAP 2912 /* ia64 gets its own node_mem_map, before this, without bootmem */ 2913 if (!pgdat->node_mem_map) { 2914 unsigned long size, start, end; 2915 struct page *map; 2916 2917 /* 2918 * The zone's endpoints aren't required to be MAX_ORDER 2919 * aligned but the node_mem_map endpoints must be in order 2920 * for the buddy allocator to function correctly. 2921 */ 2922 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 2923 end = pgdat->node_start_pfn + pgdat->node_spanned_pages; 2924 end = ALIGN(end, MAX_ORDER_NR_PAGES); 2925 size = (end - start) * sizeof(struct page); 2926 map = alloc_remap(pgdat->node_id, size); 2927 if (!map) 2928 map = alloc_bootmem_node(pgdat, size); 2929 pgdat->node_mem_map = map + (pgdat->node_start_pfn - start); 2930 } 2931#ifndef CONFIG_NEED_MULTIPLE_NODES 2932 /* 2933 * With no DISCONTIG, the global mem_map is just set as node 0's 2934 */ 2935 if (pgdat == NODE_DATA(0)) { 2936 mem_map = NODE_DATA(0)->node_mem_map; 2937#ifdef CONFIG_ARCH_POPULATES_NODE_MAP 2938 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 2939 mem_map -= pgdat->node_start_pfn; 2940#endif /* CONFIG_ARCH_POPULATES_NODE_MAP */ 2941 } 2942#endif 2943#endif /* CONFIG_FLAT_NODE_MEM_MAP */ 2944} 2945 2946void __meminit free_area_init_node(int nid, struct pglist_data *pgdat, 2947 unsigned long *zones_size, unsigned long node_start_pfn, 2948 unsigned long *zholes_size) 2949{ 2950 pgdat->node_id = nid; 2951 pgdat->node_start_pfn = node_start_pfn; 2952 calculate_node_totalpages(pgdat, zones_size, zholes_size); 2953 2954 alloc_node_mem_map(pgdat); 2955 2956 free_area_init_core(pgdat, zones_size, zholes_size); 2957} 2958 2959#ifdef CONFIG_ARCH_POPULATES_NODE_MAP 2960 2961#if MAX_NUMNODES > 1 2962/* 2963 * Figure out the number of possible node ids. 2964 */ 2965static void __init setup_nr_node_ids(void) 2966{ 2967 unsigned int node; 2968 unsigned int highest = 0; 2969 2970 for_each_node_mask(node, node_possible_map) 2971 highest = node; 2972 nr_node_ids = highest + 1; 2973} 2974#else 2975static inline void setup_nr_node_ids(void) 2976{ 2977} 2978#endif 2979 2980/** 2981 * add_active_range - Register a range of PFNs backed by physical memory 2982 * @nid: The node ID the range resides on 2983 * @start_pfn: The start PFN of the available physical memory 2984 * @end_pfn: The end PFN of the available physical memory 2985 * 2986 * These ranges are stored in an early_node_map[] and later used by 2987 * free_area_init_nodes() to calculate zone sizes and holes. If the 2988 * range spans a memory hole, it is up to the architecture to ensure 2989 * the memory is not freed by the bootmem allocator. If possible 2990 * the range being registered will be merged with existing ranges. 2991 */ 2992void __init add_active_range(unsigned int nid, unsigned long start_pfn, 2993 unsigned long end_pfn) 2994{ 2995 int i; 2996 2997 printk(KERN_DEBUG "Entering add_active_range(%d, %lu, %lu) " 2998 "%d entries of %d used\n", 2999 nid, start_pfn, end_pfn, 3000 nr_nodemap_entries, MAX_ACTIVE_REGIONS); 3001 3002 /* Merge with existing active regions if possible */ 3003 for (i = 0; i < nr_nodemap_entries; i++) { 3004 if (early_node_map[i].nid != nid) 3005 continue; 3006 3007 /* Skip if an existing region covers this new one */ 3008 if (start_pfn >= early_node_map[i].start_pfn && 3009 end_pfn <= early_node_map[i].end_pfn) 3010 return; 3011 3012 /* Merge forward if suitable */ 3013 if (start_pfn <= early_node_map[i].end_pfn && 3014 end_pfn > early_node_map[i].end_pfn) { 3015 early_node_map[i].end_pfn = end_pfn; 3016 return; 3017 } 3018 3019 /* Merge backward if suitable */ 3020 if (start_pfn < early_node_map[i].end_pfn && 3021 end_pfn >= early_node_map[i].start_pfn) { 3022 early_node_map[i].start_pfn = start_pfn; 3023 return; 3024 } 3025 } 3026 3027 /* Check that early_node_map is large enough */ 3028 if (i >= MAX_ACTIVE_REGIONS) { 3029 printk(KERN_CRIT "More than %d memory regions, truncating\n", 3030 MAX_ACTIVE_REGIONS); 3031 return; 3032 } 3033 3034 early_node_map[i].nid = nid; 3035 early_node_map[i].start_pfn = start_pfn; 3036 early_node_map[i].end_pfn = end_pfn; 3037 nr_nodemap_entries = i + 1; 3038} 3039 3040/** 3041 * shrink_active_range - Shrink an existing registered range of PFNs 3042 * @nid: The node id the range is on that should be shrunk 3043 * @old_end_pfn: The old end PFN of the range 3044 * @new_end_pfn: The new PFN of the range 3045 * 3046 * i386 with NUMA use alloc_remap() to store a node_mem_map on a local node. 3047 * The map is kept at the end physical page range that has already been 3048 * registered with add_active_range(). This function allows an arch to shrink 3049 * an existing registered range. 3050 */ 3051void __init shrink_active_range(unsigned int nid, unsigned long old_end_pfn, 3052 unsigned long new_end_pfn) 3053{ 3054 int i; 3055 3056 /* Find the old active region end and shrink */ 3057 for_each_active_range_index_in_nid(i, nid) 3058 if (early_node_map[i].end_pfn == old_end_pfn) { 3059 early_node_map[i].end_pfn = new_end_pfn; 3060 break; 3061 } 3062} 3063 3064/** 3065 * remove_all_active_ranges - Remove all currently registered regions 3066 * 3067 * During discovery, it may be found that a table like SRAT is invalid 3068 * and an alternative discovery method must be used. This function removes 3069 * all currently registered regions. 3070 */ 3071void __init remove_all_active_ranges(void) 3072{ 3073 memset(early_node_map, 0, sizeof(early_node_map)); 3074 nr_nodemap_entries = 0; 3075#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE 3076 memset(node_boundary_start_pfn, 0, sizeof(node_boundary_start_pfn)); 3077 memset(node_boundary_end_pfn, 0, sizeof(node_boundary_end_pfn)); 3078#endif /* CONFIG_MEMORY_HOTPLUG_RESERVE */ 3079} 3080 3081/* Compare two active node_active_regions */ 3082static int __init cmp_node_active_region(const void *a, const void *b) 3083{ 3084 struct node_active_region *arange = (struct node_active_region *)a; 3085 struct node_active_region *brange = (struct node_active_region *)b; 3086 3087 /* Done this way to avoid overflows */ 3088 if (arange->start_pfn > brange->start_pfn) 3089 return 1; 3090 if (arange->start_pfn < brange->start_pfn) 3091 return -1; 3092 3093 return 0; 3094} 3095 3096/* sort the node_map by start_pfn */ 3097static void __init sort_node_map(void) 3098{ 3099 sort(early_node_map, (size_t)nr_nodemap_entries, 3100 sizeof(struct node_active_region), 3101 cmp_node_active_region, NULL); 3102} 3103 3104/* Find the lowest pfn for a node */ 3105unsigned long __init find_min_pfn_for_node(unsigned long nid) 3106{ 3107 int i; 3108 unsigned long min_pfn = ULONG_MAX; 3109 3110 /* Assuming a sorted map, the first range found has the starting pfn */ 3111 for_each_active_range_index_in_nid(i, nid) 3112 min_pfn = min(min_pfn, early_node_map[i].start_pfn); 3113 3114 if (min_pfn == ULONG_MAX) { 3115 printk(KERN_WARNING 3116 "Could not find start_pfn for node %lu\n", nid); 3117 return 0; 3118 } 3119 3120 return min_pfn; 3121} 3122 3123/** 3124 * find_min_pfn_with_active_regions - Find the minimum PFN registered 3125 * 3126 * It returns the minimum PFN based on information provided via 3127 * add_active_range(). 3128 */ 3129unsigned long __init find_min_pfn_with_active_regions(void) 3130{ 3131 return find_min_pfn_for_node(MAX_NUMNODES); 3132} 3133 3134/** 3135 * find_max_pfn_with_active_regions - Find the maximum PFN registered 3136 * 3137 * It returns the maximum PFN based on information provided via 3138 * add_active_range(). 3139 */ 3140unsigned long __init find_max_pfn_with_active_regions(void) 3141{ 3142 int i; 3143 unsigned long max_pfn = 0; 3144 3145 for (i = 0; i < nr_nodemap_entries; i++) 3146 max_pfn = max(max_pfn, early_node_map[i].end_pfn); 3147 3148 return max_pfn; 3149} 3150 3151/** 3152 * free_area_init_nodes - Initialise all pg_data_t and zone data 3153 * @max_zone_pfn: an array of max PFNs for each zone 3154 * 3155 * This will call free_area_init_node() for each active node in the system. 3156 * Using the page ranges provided by add_active_range(), the size of each 3157 * zone in each node and their holes is calculated. If the maximum PFN 3158 * between two adjacent zones match, it is assumed that the zone is empty. 3159 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 3160 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 3161 * starts where the previous one ended. For example, ZONE_DMA32 starts 3162 * at arch_max_dma_pfn. 3163 */ 3164void __init free_area_init_nodes(unsigned long *max_zone_pfn) 3165{ 3166 unsigned long nid; 3167 enum zone_type i; 3168 3169 /* Sort early_node_map as initialisation assumes it is sorted */ 3170 sort_node_map(); 3171 3172 /* Record where the zone boundaries are */ 3173 memset(arch_zone_lowest_possible_pfn, 0, 3174 sizeof(arch_zone_lowest_possible_pfn)); 3175 memset(arch_zone_highest_possible_pfn, 0, 3176 sizeof(arch_zone_highest_possible_pfn)); 3177 arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions(); 3178 arch_zone_highest_possible_pfn[0] = max_zone_pfn[0]; 3179 for (i = 1; i < MAX_NR_ZONES; i++) { 3180 arch_zone_lowest_possible_pfn[i] = 3181 arch_zone_highest_possible_pfn[i-1]; 3182 arch_zone_highest_possible_pfn[i] = 3183 max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]); 3184 } 3185 3186 /* Print out the zone ranges */ 3187 printk("Zone PFN ranges:\n"); 3188 for (i = 0; i < MAX_NR_ZONES; i++) 3189 printk(" %-8s %8lu -> %8lu\n", 3190 zone_names[i], 3191 arch_zone_lowest_possible_pfn[i], 3192 arch_zone_highest_possible_pfn[i]); 3193 3194 /* Print out the early_node_map[] */ 3195 printk("early_node_map[%d] active PFN ranges\n", nr_nodemap_entries); 3196 for (i = 0; i < nr_nodemap_entries; i++) 3197 printk(" %3d: %8lu -> %8lu\n", early_node_map[i].nid, 3198 early_node_map[i].start_pfn, 3199 early_node_map[i].end_pfn); 3200 3201 /* Initialise every node */ 3202 setup_nr_node_ids(); 3203 for_each_online_node(nid) { 3204 pg_data_t *pgdat = NODE_DATA(nid); 3205 free_area_init_node(nid, pgdat, NULL, 3206 find_min_pfn_for_node(nid), NULL); 3207 } 3208} 3209#endif /* CONFIG_ARCH_POPULATES_NODE_MAP */ 3210 3211/** 3212 * set_dma_reserve - set the specified number of pages reserved in the first zone 3213 * @new_dma_reserve: The number of pages to mark reserved 3214 * 3215 * The per-cpu batchsize and zone watermarks are determined by present_pages. 3216 * In the DMA zone, a significant percentage may be consumed by kernel image 3217 * and other unfreeable allocations which can skew the watermarks badly. This 3218 * function may optionally be used to account for unfreeable pages in the 3219 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 3220 * smaller per-cpu batchsize. 3221 */ 3222void __init set_dma_reserve(unsigned long new_dma_reserve) 3223{ 3224 dma_reserve = new_dma_reserve; 3225} 3226 3227#ifndef CONFIG_NEED_MULTIPLE_NODES 3228static bootmem_data_t contig_bootmem_data; 3229struct pglist_data contig_page_data = { .bdata = &contig_bootmem_data }; 3230 3231EXPORT_SYMBOL(contig_page_data); 3232#endif 3233 3234void __init free_area_init(unsigned long *zones_size) 3235{ 3236 free_area_init_node(0, NODE_DATA(0), zones_size, 3237 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 3238} 3239 3240static int page_alloc_cpu_notify(struct notifier_block *self, 3241 unsigned long action, void *hcpu) 3242{ 3243 int cpu = (unsigned long)hcpu; 3244 3245 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 3246 local_irq_disable(); 3247 __drain_pages(cpu); 3248 vm_events_fold_cpu(cpu); 3249 local_irq_enable(); 3250 refresh_cpu_vm_stats(cpu); 3251 } 3252 return NOTIFY_OK; 3253} 3254 3255void __init page_alloc_init(void) 3256{ 3257 hotcpu_notifier(page_alloc_cpu_notify, 0); 3258} 3259 3260/* 3261 * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio 3262 * or min_free_kbytes changes. 3263 */ 3264static void calculate_totalreserve_pages(void) 3265{ 3266 struct pglist_data *pgdat; 3267 unsigned long reserve_pages = 0; 3268 enum zone_type i, j; 3269 3270 for_each_online_pgdat(pgdat) { 3271 for (i = 0; i < MAX_NR_ZONES; i++) { 3272 struct zone *zone = pgdat->node_zones + i; 3273 unsigned long max = 0; 3274 3275 /* Find valid and maximum lowmem_reserve in the zone */ 3276 for (j = i; j < MAX_NR_ZONES; j++) { 3277 if (zone->lowmem_reserve[j] > max) 3278 max = zone->lowmem_reserve[j]; 3279 } 3280 3281 /* we treat pages_high as reserved pages. */ 3282 max += zone->pages_high; 3283 3284 if (max > zone->present_pages) 3285 max = zone->present_pages; 3286 reserve_pages += max; 3287 } 3288 } 3289 totalreserve_pages = reserve_pages; 3290} 3291 3292/* 3293 * setup_per_zone_lowmem_reserve - called whenever 3294 * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone 3295 * has a correct pages reserved value, so an adequate number of 3296 * pages are left in the zone after a successful __alloc_pages(). 3297 */ 3298static void setup_per_zone_lowmem_reserve(void) 3299{ 3300 struct pglist_data *pgdat; 3301 enum zone_type j, idx; 3302 3303 for_each_online_pgdat(pgdat) { 3304 for (j = 0; j < MAX_NR_ZONES; j++) { 3305 struct zone *zone = pgdat->node_zones + j; 3306 unsigned long present_pages = zone->present_pages; 3307 3308 zone->lowmem_reserve[j] = 0; 3309 3310 idx = j; 3311 while (idx) { 3312 struct zone *lower_zone; 3313 3314 idx--; 3315 3316 if (sysctl_lowmem_reserve_ratio[idx] < 1) 3317 sysctl_lowmem_reserve_ratio[idx] = 1; 3318 3319 lower_zone = pgdat->node_zones + idx; 3320 lower_zone->lowmem_reserve[j] = present_pages / 3321 sysctl_lowmem_reserve_ratio[idx]; 3322 present_pages += lower_zone->present_pages; 3323 } 3324 } 3325 } 3326 3327 /* update totalreserve_pages */ 3328 calculate_totalreserve_pages(); 3329} 3330 3331/** 3332 * setup_per_zone_pages_min - called when min_free_kbytes changes. 3333 * 3334 * Ensures that the pages_{min,low,high} values for each zone are set correctly 3335 * with respect to min_free_kbytes. 3336 */ 3337void setup_per_zone_pages_min(void) 3338{ 3339 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 3340 unsigned long lowmem_pages = 0; 3341 struct zone *zone; 3342 unsigned long flags; 3343 3344 /* Calculate total number of !ZONE_HIGHMEM pages */ 3345 for_each_zone(zone) { 3346 if (!is_highmem(zone)) 3347 lowmem_pages += zone->present_pages; 3348 } 3349 3350 for_each_zone(zone) { 3351 u64 tmp; 3352 3353 spin_lock_irqsave(&zone->lru_lock, flags); 3354 tmp = (u64)pages_min * zone->present_pages; 3355 do_div(tmp, lowmem_pages); 3356 if (is_highmem(zone)) { 3357 /* 3358 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 3359 * need highmem pages, so cap pages_min to a small 3360 * value here. 3361 * 3362 * The (pages_high-pages_low) and (pages_low-pages_min) 3363 * deltas controls asynch page reclaim, and so should 3364 * not be capped for highmem. 3365 */ 3366 int min_pages; 3367 3368 min_pages = zone->present_pages / 1024; 3369 if (min_pages < SWAP_CLUSTER_MAX) 3370 min_pages = SWAP_CLUSTER_MAX; 3371 if (min_pages > 128) 3372 min_pages = 128; 3373 zone->pages_min = min_pages; 3374 } else { 3375 /* 3376 * If it's a lowmem zone, reserve a number of pages 3377 * proportionate to the zone's size. 3378 */ 3379 zone->pages_min = tmp; 3380 } 3381 3382 zone->pages_low = zone->pages_min + (tmp >> 2); 3383 zone->pages_high = zone->pages_min + (tmp >> 1); 3384 spin_unlock_irqrestore(&zone->lru_lock, flags); 3385 } 3386 3387 /* update totalreserve_pages */ 3388 calculate_totalreserve_pages(); 3389} 3390 3391/* 3392 * Initialise min_free_kbytes. 3393 * 3394 * For small machines we want it small (128k min). For large machines 3395 * we want it large (64MB max). But it is not linear, because network 3396 * bandwidth does not increase linearly with machine size. We use 3397 * 3398 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 3399 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 3400 * 3401 * which yields 3402 * 3403 * 16MB: 512k 3404 * 32MB: 724k 3405 * 64MB: 1024k 3406 * 128MB: 1448k 3407 * 256MB: 2048k 3408 * 512MB: 2896k 3409 * 1024MB: 4096k 3410 * 2048MB: 5792k 3411 * 4096MB: 8192k 3412 * 8192MB: 11584k 3413 * 16384MB: 16384k 3414 */ 3415static int __init init_per_zone_pages_min(void) 3416{ 3417 unsigned long lowmem_kbytes; 3418 3419 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 3420 3421 min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 3422 if (min_free_kbytes < 128) 3423 min_free_kbytes = 128; 3424 if (min_free_kbytes > 65536) 3425 min_free_kbytes = 65536; 3426 setup_per_zone_pages_min(); 3427 setup_per_zone_lowmem_reserve(); 3428 return 0; 3429} 3430module_init(init_per_zone_pages_min) 3431 3432/* 3433 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 3434 * that we can call two helper functions whenever min_free_kbytes 3435 * changes. 3436 */ 3437int min_free_kbytes_sysctl_handler(ctl_table *table, int write, 3438 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 3439{ 3440 proc_dointvec(table, write, file, buffer, length, ppos); 3441 if (write) 3442 setup_per_zone_pages_min(); 3443 return 0; 3444} 3445 3446#ifdef CONFIG_NUMA 3447int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write, 3448 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 3449{ 3450 struct zone *zone; 3451 int rc; 3452 3453 rc = proc_dointvec_minmax(table, write, file, buffer, length, ppos); 3454 if (rc) 3455 return rc; 3456 3457 for_each_zone(zone) 3458 zone->min_unmapped_pages = (zone->present_pages * 3459 sysctl_min_unmapped_ratio) / 100; 3460 return 0; 3461} 3462 3463int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write, 3464 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 3465{ 3466 struct zone *zone; 3467 int rc; 3468 3469 rc = proc_dointvec_minmax(table, write, file, buffer, length, ppos); 3470 if (rc) 3471 return rc; 3472 3473 for_each_zone(zone) 3474 zone->min_slab_pages = (zone->present_pages * 3475 sysctl_min_slab_ratio) / 100; 3476 return 0; 3477} 3478#endif 3479 3480/* 3481 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 3482 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 3483 * whenever sysctl_lowmem_reserve_ratio changes. 3484 * 3485 * The reserve ratio obviously has absolutely no relation with the 3486 * pages_min watermarks. The lowmem reserve ratio can only make sense 3487 * if in function of the boot time zone sizes. 3488 */ 3489int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write, 3490 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 3491{ 3492 proc_dointvec_minmax(table, write, file, buffer, length, ppos); 3493 setup_per_zone_lowmem_reserve(); 3494 return 0; 3495} 3496 3497/* 3498 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 3499 * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist 3500 * can have before it gets flushed back to buddy allocator. 3501 */ 3502 3503int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write, 3504 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 3505{ 3506 struct zone *zone; 3507 unsigned int cpu; 3508 int ret; 3509 3510 ret = proc_dointvec_minmax(table, write, file, buffer, length, ppos); 3511 if (!write || (ret == -EINVAL)) 3512 return ret; 3513 for_each_zone(zone) { 3514 for_each_online_cpu(cpu) { 3515 unsigned long high; 3516 high = zone->present_pages / percpu_pagelist_fraction; 3517 setup_pagelist_highmark(zone_pcp(zone, cpu), high); 3518 } 3519 } 3520 return 0; 3521} 3522 3523int hashdist = HASHDIST_DEFAULT; 3524 3525#ifdef CONFIG_NUMA 3526static int __init set_hashdist(char *str) 3527{ 3528 if (!str) 3529 return 0; 3530 hashdist = simple_strtoul(str, &str, 0); 3531 return 1; 3532} 3533__setup("hashdist=", set_hashdist); 3534#endif 3535 3536/* 3537 * allocate a large system hash table from bootmem 3538 * - it is assumed that the hash table must contain an exact power-of-2 3539 * quantity of entries 3540 * - limit is the number of hash buckets, not the total allocation size 3541 */ 3542void *__init alloc_large_system_hash(const char *tablename, 3543 unsigned long bucketsize, 3544 unsigned long numentries, 3545 int scale, 3546 int flags, 3547 unsigned int *_hash_shift, 3548 unsigned int *_hash_mask, 3549 unsigned long limit) 3550{ 3551 unsigned long long max = limit; 3552 unsigned long log2qty, size; 3553 void *table = NULL; 3554 3555 /* allow the kernel cmdline to have a say */ 3556 if (!numentries) { 3557 /* round applicable memory size up to nearest megabyte */ 3558 numentries = nr_kernel_pages; 3559 numentries += (1UL << (20 - PAGE_SHIFT)) - 1; 3560 numentries >>= 20 - PAGE_SHIFT; 3561 numentries <<= 20 - PAGE_SHIFT; 3562 3563 /* limit to 1 bucket per 2^scale bytes of low memory */ 3564 if (scale > PAGE_SHIFT) 3565 numentries >>= (scale - PAGE_SHIFT); 3566 else 3567 numentries <<= (PAGE_SHIFT - scale); 3568 3569 /* Make sure we've got at least a 0-order allocation.. */ 3570 if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 3571 numentries = PAGE_SIZE / bucketsize; 3572 } 3573 numentries = roundup_pow_of_two(numentries); 3574 3575 /* limit allocation size to 1/16 total memory by default */ 3576 if (max == 0) { 3577 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 3578 do_div(max, bucketsize); 3579 } 3580 3581 if (numentries > max) 3582 numentries = max; 3583 3584 log2qty = ilog2(numentries); 3585 3586 do { 3587 size = bucketsize << log2qty; 3588 if (flags & HASH_EARLY) 3589 table = alloc_bootmem(size); 3590 else if (hashdist) 3591 table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); 3592 else { 3593 unsigned long order; 3594 for (order = 0; ((1UL << order) << PAGE_SHIFT) < size; order++) 3595 ; 3596 table = (void*) __get_free_pages(GFP_ATOMIC, order); 3597 /* 3598 * If bucketsize is not a power-of-two, we may free 3599 * some pages at the end of hash table. 3600 */ 3601 if (table) { 3602 unsigned long alloc_end = (unsigned long)table + 3603 (PAGE_SIZE << order); 3604 unsigned long used = (unsigned long)table + 3605 PAGE_ALIGN(size); 3606 split_page(virt_to_page(table), order); 3607 while (used < alloc_end) { 3608 free_page(used); 3609 used += PAGE_SIZE; 3610 } 3611 } 3612 } 3613 } while (!table && size > PAGE_SIZE && --log2qty); 3614 3615 if (!table) 3616 panic("Failed to allocate %s hash table\n", tablename); 3617 3618 printk(KERN_INFO "%s hash table entries: %d (order: %d, %lu bytes)\n", 3619 tablename, 3620 (1U << log2qty), 3621 ilog2(size) - PAGE_SHIFT, 3622 size); 3623 3624 if (_hash_shift) 3625 *_hash_shift = log2qty; 3626 if (_hash_mask) 3627 *_hash_mask = (1 << log2qty) - 1; 3628 3629 return table; 3630} 3631 3632#ifdef CONFIG_OUT_OF_LINE_PFN_TO_PAGE 3633struct page *pfn_to_page(unsigned long pfn) 3634{ 3635 return __pfn_to_page(pfn); 3636} 3637unsigned long page_to_pfn(struct page *page) 3638{ 3639 return __page_to_pfn(page); 3640} 3641EXPORT_SYMBOL(pfn_to_page); 3642EXPORT_SYMBOL(page_to_pfn); 3643#endif /* CONFIG_OUT_OF_LINE_PFN_TO_PAGE */ 3644 3645 3646