page_alloc.c revision 12b1c5f382194d3f656e78fb5c9c8f2bfbe8ed8a
1/* 2 * linux/mm/page_alloc.c 3 * 4 * Manages the free list, the system allocates free pages here. 5 * Note that kmalloc() lives in slab.c 6 * 7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 8 * Swap reorganised 29.12.95, Stephen Tweedie 9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 15 */ 16 17#include <linux/config.h> 18#include <linux/stddef.h> 19#include <linux/mm.h> 20#include <linux/swap.h> 21#include <linux/interrupt.h> 22#include <linux/pagemap.h> 23#include <linux/bootmem.h> 24#include <linux/compiler.h> 25#include <linux/module.h> 26#include <linux/suspend.h> 27#include <linux/pagevec.h> 28#include <linux/blkdev.h> 29#include <linux/slab.h> 30#include <linux/notifier.h> 31#include <linux/topology.h> 32#include <linux/sysctl.h> 33#include <linux/cpu.h> 34#include <linux/cpuset.h> 35#include <linux/nodemask.h> 36#include <linux/vmalloc.h> 37 38#include <asm/tlbflush.h> 39#include "internal.h" 40 41/* 42 * MCD - HACK: Find somewhere to initialize this EARLY, or make this 43 * initializer cleaner 44 */ 45nodemask_t node_online_map = { { [0] = 1UL } }; 46EXPORT_SYMBOL(node_online_map); 47nodemask_t node_possible_map = NODE_MASK_ALL; 48EXPORT_SYMBOL(node_possible_map); 49struct pglist_data *pgdat_list; 50unsigned long totalram_pages; 51unsigned long totalhigh_pages; 52long nr_swap_pages; 53 54/* 55 * results with 256, 32 in the lowmem_reserve sysctl: 56 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 57 * 1G machine -> (16M dma, 784M normal, 224M high) 58 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 59 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 60 * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA 61 */ 62int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { 256, 32 }; 63 64EXPORT_SYMBOL(totalram_pages); 65EXPORT_SYMBOL(nr_swap_pages); 66 67/* 68 * Used by page_zone() to look up the address of the struct zone whose 69 * id is encoded in the upper bits of page->flags 70 */ 71struct zone *zone_table[1 << ZONETABLE_SHIFT]; 72EXPORT_SYMBOL(zone_table); 73 74static char *zone_names[MAX_NR_ZONES] = { "DMA", "Normal", "HighMem" }; 75int min_free_kbytes = 1024; 76 77unsigned long __initdata nr_kernel_pages; 78unsigned long __initdata nr_all_pages; 79 80/* 81 * Temporary debugging check for pages not lying within a given zone. 82 */ 83static int bad_range(struct zone *zone, struct page *page) 84{ 85 if (page_to_pfn(page) >= zone->zone_start_pfn + zone->spanned_pages) 86 return 1; 87 if (page_to_pfn(page) < zone->zone_start_pfn) 88 return 1; 89#ifdef CONFIG_HOLES_IN_ZONE 90 if (!pfn_valid(page_to_pfn(page))) 91 return 1; 92#endif 93 if (zone != page_zone(page)) 94 return 1; 95 return 0; 96} 97 98static void bad_page(const char *function, struct page *page) 99{ 100 printk(KERN_EMERG "Bad page state at %s (in process '%s', page %p)\n", 101 function, current->comm, page); 102 printk(KERN_EMERG "flags:0x%0*lx mapping:%p mapcount:%d count:%d\n", 103 (int)(2*sizeof(page_flags_t)), (unsigned long)page->flags, 104 page->mapping, page_mapcount(page), page_count(page)); 105 printk(KERN_EMERG "Backtrace:\n"); 106 dump_stack(); 107 printk(KERN_EMERG "Trying to fix it up, but a reboot is needed\n"); 108 page->flags &= ~(1 << PG_lru | 109 1 << PG_private | 110 1 << PG_locked | 111 1 << PG_active | 112 1 << PG_dirty | 113 1 << PG_reclaim | 114 1 << PG_slab | 115 1 << PG_swapcache | 116 1 << PG_writeback); 117 set_page_count(page, 0); 118 reset_page_mapcount(page); 119 page->mapping = NULL; 120 tainted |= TAINT_BAD_PAGE; 121} 122 123#ifndef CONFIG_HUGETLB_PAGE 124#define prep_compound_page(page, order) do { } while (0) 125#define destroy_compound_page(page, order) do { } while (0) 126#else 127/* 128 * Higher-order pages are called "compound pages". They are structured thusly: 129 * 130 * The first PAGE_SIZE page is called the "head page". 131 * 132 * The remaining PAGE_SIZE pages are called "tail pages". 133 * 134 * All pages have PG_compound set. All pages have their ->private pointing at 135 * the head page (even the head page has this). 136 * 137 * The first tail page's ->mapping, if non-zero, holds the address of the 138 * compound page's put_page() function. 139 * 140 * The order of the allocation is stored in the first tail page's ->index 141 * This is only for debug at present. This usage means that zero-order pages 142 * may not be compound. 143 */ 144static void prep_compound_page(struct page *page, unsigned long order) 145{ 146 int i; 147 int nr_pages = 1 << order; 148 149 page[1].mapping = NULL; 150 page[1].index = order; 151 for (i = 0; i < nr_pages; i++) { 152 struct page *p = page + i; 153 154 SetPageCompound(p); 155 p->private = (unsigned long)page; 156 } 157} 158 159static void destroy_compound_page(struct page *page, unsigned long order) 160{ 161 int i; 162 int nr_pages = 1 << order; 163 164 if (!PageCompound(page)) 165 return; 166 167 if (page[1].index != order) 168 bad_page(__FUNCTION__, page); 169 170 for (i = 0; i < nr_pages; i++) { 171 struct page *p = page + i; 172 173 if (!PageCompound(p)) 174 bad_page(__FUNCTION__, page); 175 if (p->private != (unsigned long)page) 176 bad_page(__FUNCTION__, page); 177 ClearPageCompound(p); 178 } 179} 180#endif /* CONFIG_HUGETLB_PAGE */ 181 182/* 183 * function for dealing with page's order in buddy system. 184 * zone->lock is already acquired when we use these. 185 * So, we don't need atomic page->flags operations here. 186 */ 187static inline unsigned long page_order(struct page *page) { 188 return page->private; 189} 190 191static inline void set_page_order(struct page *page, int order) { 192 page->private = order; 193 __SetPagePrivate(page); 194} 195 196static inline void rmv_page_order(struct page *page) 197{ 198 __ClearPagePrivate(page); 199 page->private = 0; 200} 201 202/* 203 * Locate the struct page for both the matching buddy in our 204 * pair (buddy1) and the combined O(n+1) page they form (page). 205 * 206 * 1) Any buddy B1 will have an order O twin B2 which satisfies 207 * the following equation: 208 * B2 = B1 ^ (1 << O) 209 * For example, if the starting buddy (buddy2) is #8 its order 210 * 1 buddy is #10: 211 * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 212 * 213 * 2) Any buddy B will have an order O+1 parent P which 214 * satisfies the following equation: 215 * P = B & ~(1 << O) 216 * 217 * Assumption: *_mem_map is contigious at least up to MAX_ORDER 218 */ 219static inline struct page * 220__page_find_buddy(struct page *page, unsigned long page_idx, unsigned int order) 221{ 222 unsigned long buddy_idx = page_idx ^ (1 << order); 223 224 return page + (buddy_idx - page_idx); 225} 226 227static inline unsigned long 228__find_combined_index(unsigned long page_idx, unsigned int order) 229{ 230 return (page_idx & ~(1 << order)); 231} 232 233/* 234 * This function checks whether a page is free && is the buddy 235 * we can do coalesce a page and its buddy if 236 * (a) the buddy is free && 237 * (b) the buddy is on the buddy system && 238 * (c) a page and its buddy have the same order. 239 * for recording page's order, we use page->private and PG_private. 240 * 241 */ 242static inline int page_is_buddy(struct page *page, int order) 243{ 244 if (PagePrivate(page) && 245 (page_order(page) == order) && 246 !PageReserved(page) && 247 page_count(page) == 0) 248 return 1; 249 return 0; 250} 251 252/* 253 * Freeing function for a buddy system allocator. 254 * 255 * The concept of a buddy system is to maintain direct-mapped table 256 * (containing bit values) for memory blocks of various "orders". 257 * The bottom level table contains the map for the smallest allocatable 258 * units of memory (here, pages), and each level above it describes 259 * pairs of units from the levels below, hence, "buddies". 260 * At a high level, all that happens here is marking the table entry 261 * at the bottom level available, and propagating the changes upward 262 * as necessary, plus some accounting needed to play nicely with other 263 * parts of the VM system. 264 * At each level, we keep a list of pages, which are heads of continuous 265 * free pages of length of (1 << order) and marked with PG_Private.Page's 266 * order is recorded in page->private field. 267 * So when we are allocating or freeing one, we can derive the state of the 268 * other. That is, if we allocate a small block, and both were 269 * free, the remainder of the region must be split into blocks. 270 * If a block is freed, and its buddy is also free, then this 271 * triggers coalescing into a block of larger size. 272 * 273 * -- wli 274 */ 275 276static inline void __free_pages_bulk (struct page *page, 277 struct zone *zone, unsigned int order) 278{ 279 unsigned long page_idx; 280 int order_size = 1 << order; 281 282 if (unlikely(order)) 283 destroy_compound_page(page, order); 284 285 page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1); 286 287 BUG_ON(page_idx & (order_size - 1)); 288 BUG_ON(bad_range(zone, page)); 289 290 zone->free_pages += order_size; 291 while (order < MAX_ORDER-1) { 292 unsigned long combined_idx; 293 struct free_area *area; 294 struct page *buddy; 295 296 combined_idx = __find_combined_index(page_idx, order); 297 buddy = __page_find_buddy(page, page_idx, order); 298 299 if (bad_range(zone, buddy)) 300 break; 301 if (!page_is_buddy(buddy, order)) 302 break; /* Move the buddy up one level. */ 303 list_del(&buddy->lru); 304 area = zone->free_area + order; 305 area->nr_free--; 306 rmv_page_order(buddy); 307 page = page + (combined_idx - page_idx); 308 page_idx = combined_idx; 309 order++; 310 } 311 set_page_order(page, order); 312 list_add(&page->lru, &zone->free_area[order].free_list); 313 zone->free_area[order].nr_free++; 314} 315 316static inline void free_pages_check(const char *function, struct page *page) 317{ 318 if ( page_mapcount(page) || 319 page->mapping != NULL || 320 page_count(page) != 0 || 321 (page->flags & ( 322 1 << PG_lru | 323 1 << PG_private | 324 1 << PG_locked | 325 1 << PG_active | 326 1 << PG_reclaim | 327 1 << PG_slab | 328 1 << PG_swapcache | 329 1 << PG_writeback ))) 330 bad_page(function, page); 331 if (PageDirty(page)) 332 ClearPageDirty(page); 333} 334 335/* 336 * Frees a list of pages. 337 * Assumes all pages on list are in same zone, and of same order. 338 * count is the number of pages to free, or 0 for all on the list. 339 * 340 * If the zone was previously in an "all pages pinned" state then look to 341 * see if this freeing clears that state. 342 * 343 * And clear the zone's pages_scanned counter, to hold off the "all pages are 344 * pinned" detection logic. 345 */ 346static int 347free_pages_bulk(struct zone *zone, int count, 348 struct list_head *list, unsigned int order) 349{ 350 unsigned long flags; 351 struct page *page = NULL; 352 int ret = 0; 353 354 spin_lock_irqsave(&zone->lock, flags); 355 zone->all_unreclaimable = 0; 356 zone->pages_scanned = 0; 357 while (!list_empty(list) && count--) { 358 page = list_entry(list->prev, struct page, lru); 359 /* have to delete it as __free_pages_bulk list manipulates */ 360 list_del(&page->lru); 361 __free_pages_bulk(page, zone, order); 362 ret++; 363 } 364 spin_unlock_irqrestore(&zone->lock, flags); 365 return ret; 366} 367 368void __free_pages_ok(struct page *page, unsigned int order) 369{ 370 LIST_HEAD(list); 371 int i; 372 373 arch_free_page(page, order); 374 375 mod_page_state(pgfree, 1 << order); 376 377#ifndef CONFIG_MMU 378 if (order > 0) 379 for (i = 1 ; i < (1 << order) ; ++i) 380 __put_page(page + i); 381#endif 382 383 for (i = 0 ; i < (1 << order) ; ++i) 384 free_pages_check(__FUNCTION__, page + i); 385 list_add(&page->lru, &list); 386 kernel_map_pages(page, 1<<order, 0); 387 free_pages_bulk(page_zone(page), 1, &list, order); 388} 389 390 391/* 392 * The order of subdivision here is critical for the IO subsystem. 393 * Please do not alter this order without good reasons and regression 394 * testing. Specifically, as large blocks of memory are subdivided, 395 * the order in which smaller blocks are delivered depends on the order 396 * they're subdivided in this function. This is the primary factor 397 * influencing the order in which pages are delivered to the IO 398 * subsystem according to empirical testing, and this is also justified 399 * by considering the behavior of a buddy system containing a single 400 * large block of memory acted on by a series of small allocations. 401 * This behavior is a critical factor in sglist merging's success. 402 * 403 * -- wli 404 */ 405static inline struct page * 406expand(struct zone *zone, struct page *page, 407 int low, int high, struct free_area *area) 408{ 409 unsigned long size = 1 << high; 410 411 while (high > low) { 412 area--; 413 high--; 414 size >>= 1; 415 BUG_ON(bad_range(zone, &page[size])); 416 list_add(&page[size].lru, &area->free_list); 417 area->nr_free++; 418 set_page_order(&page[size], high); 419 } 420 return page; 421} 422 423void set_page_refs(struct page *page, int order) 424{ 425#ifdef CONFIG_MMU 426 set_page_count(page, 1); 427#else 428 int i; 429 430 /* 431 * We need to reference all the pages for this order, otherwise if 432 * anyone accesses one of the pages with (get/put) it will be freed. 433 * - eg: access_process_vm() 434 */ 435 for (i = 0; i < (1 << order); i++) 436 set_page_count(page + i, 1); 437#endif /* CONFIG_MMU */ 438} 439 440/* 441 * This page is about to be returned from the page allocator 442 */ 443static void prep_new_page(struct page *page, int order) 444{ 445 if ( page_mapcount(page) || 446 page->mapping != NULL || 447 page_count(page) != 0 || 448 (page->flags & ( 449 1 << PG_lru | 450 1 << PG_private | 451 1 << PG_locked | 452 1 << PG_active | 453 1 << PG_dirty | 454 1 << PG_reclaim | 455 1 << PG_slab | 456 1 << PG_swapcache | 457 1 << PG_writeback ))) 458 bad_page(__FUNCTION__, page); 459 460 page->flags &= ~(1 << PG_uptodate | 1 << PG_error | 461 1 << PG_referenced | 1 << PG_arch_1 | 462 1 << PG_checked | 1 << PG_mappedtodisk); 463 page->private = 0; 464 set_page_refs(page, order); 465 kernel_map_pages(page, 1 << order, 1); 466} 467 468/* 469 * Do the hard work of removing an element from the buddy allocator. 470 * Call me with the zone->lock already held. 471 */ 472static struct page *__rmqueue(struct zone *zone, unsigned int order) 473{ 474 struct free_area * area; 475 unsigned int current_order; 476 struct page *page; 477 478 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 479 area = zone->free_area + current_order; 480 if (list_empty(&area->free_list)) 481 continue; 482 483 page = list_entry(area->free_list.next, struct page, lru); 484 list_del(&page->lru); 485 rmv_page_order(page); 486 area->nr_free--; 487 zone->free_pages -= 1UL << order; 488 return expand(zone, page, order, current_order, area); 489 } 490 491 return NULL; 492} 493 494/* 495 * Obtain a specified number of elements from the buddy allocator, all under 496 * a single hold of the lock, for efficiency. Add them to the supplied list. 497 * Returns the number of new pages which were placed at *list. 498 */ 499static int rmqueue_bulk(struct zone *zone, unsigned int order, 500 unsigned long count, struct list_head *list) 501{ 502 unsigned long flags; 503 int i; 504 int allocated = 0; 505 struct page *page; 506 507 spin_lock_irqsave(&zone->lock, flags); 508 for (i = 0; i < count; ++i) { 509 page = __rmqueue(zone, order); 510 if (page == NULL) 511 break; 512 allocated++; 513 list_add_tail(&page->lru, list); 514 } 515 spin_unlock_irqrestore(&zone->lock, flags); 516 return allocated; 517} 518 519#ifdef CONFIG_NUMA 520/* Called from the slab reaper to drain remote pagesets */ 521void drain_remote_pages(void) 522{ 523 struct zone *zone; 524 int i; 525 unsigned long flags; 526 527 local_irq_save(flags); 528 for_each_zone(zone) { 529 struct per_cpu_pageset *pset; 530 531 /* Do not drain local pagesets */ 532 if (zone->zone_pgdat->node_id == numa_node_id()) 533 continue; 534 535 pset = zone->pageset[smp_processor_id()]; 536 for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) { 537 struct per_cpu_pages *pcp; 538 539 pcp = &pset->pcp[i]; 540 if (pcp->count) 541 pcp->count -= free_pages_bulk(zone, pcp->count, 542 &pcp->list, 0); 543 } 544 } 545 local_irq_restore(flags); 546} 547#endif 548 549#if defined(CONFIG_PM) || defined(CONFIG_HOTPLUG_CPU) 550static void __drain_pages(unsigned int cpu) 551{ 552 struct zone *zone; 553 int i; 554 555 for_each_zone(zone) { 556 struct per_cpu_pageset *pset; 557 558 pset = zone_pcp(zone, cpu); 559 for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) { 560 struct per_cpu_pages *pcp; 561 562 pcp = &pset->pcp[i]; 563 pcp->count -= free_pages_bulk(zone, pcp->count, 564 &pcp->list, 0); 565 } 566 } 567} 568#endif /* CONFIG_PM || CONFIG_HOTPLUG_CPU */ 569 570#ifdef CONFIG_PM 571 572void mark_free_pages(struct zone *zone) 573{ 574 unsigned long zone_pfn, flags; 575 int order; 576 struct list_head *curr; 577 578 if (!zone->spanned_pages) 579 return; 580 581 spin_lock_irqsave(&zone->lock, flags); 582 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) 583 ClearPageNosaveFree(pfn_to_page(zone_pfn + zone->zone_start_pfn)); 584 585 for (order = MAX_ORDER - 1; order >= 0; --order) 586 list_for_each(curr, &zone->free_area[order].free_list) { 587 unsigned long start_pfn, i; 588 589 start_pfn = page_to_pfn(list_entry(curr, struct page, lru)); 590 591 for (i=0; i < (1<<order); i++) 592 SetPageNosaveFree(pfn_to_page(start_pfn+i)); 593 } 594 spin_unlock_irqrestore(&zone->lock, flags); 595} 596 597/* 598 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 599 */ 600void drain_local_pages(void) 601{ 602 unsigned long flags; 603 604 local_irq_save(flags); 605 __drain_pages(smp_processor_id()); 606 local_irq_restore(flags); 607} 608#endif /* CONFIG_PM */ 609 610static void zone_statistics(struct zonelist *zonelist, struct zone *z) 611{ 612#ifdef CONFIG_NUMA 613 unsigned long flags; 614 int cpu; 615 pg_data_t *pg = z->zone_pgdat; 616 pg_data_t *orig = zonelist->zones[0]->zone_pgdat; 617 struct per_cpu_pageset *p; 618 619 local_irq_save(flags); 620 cpu = smp_processor_id(); 621 p = zone_pcp(z,cpu); 622 if (pg == orig) { 623 p->numa_hit++; 624 } else { 625 p->numa_miss++; 626 zone_pcp(zonelist->zones[0], cpu)->numa_foreign++; 627 } 628 if (pg == NODE_DATA(numa_node_id())) 629 p->local_node++; 630 else 631 p->other_node++; 632 local_irq_restore(flags); 633#endif 634} 635 636/* 637 * Free a 0-order page 638 */ 639static void FASTCALL(free_hot_cold_page(struct page *page, int cold)); 640static void fastcall free_hot_cold_page(struct page *page, int cold) 641{ 642 struct zone *zone = page_zone(page); 643 struct per_cpu_pages *pcp; 644 unsigned long flags; 645 646 arch_free_page(page, 0); 647 648 kernel_map_pages(page, 1, 0); 649 inc_page_state(pgfree); 650 if (PageAnon(page)) 651 page->mapping = NULL; 652 free_pages_check(__FUNCTION__, page); 653 pcp = &zone_pcp(zone, get_cpu())->pcp[cold]; 654 local_irq_save(flags); 655 list_add(&page->lru, &pcp->list); 656 pcp->count++; 657 if (pcp->count >= pcp->high) 658 pcp->count -= free_pages_bulk(zone, pcp->batch, &pcp->list, 0); 659 local_irq_restore(flags); 660 put_cpu(); 661} 662 663void fastcall free_hot_page(struct page *page) 664{ 665 free_hot_cold_page(page, 0); 666} 667 668void fastcall free_cold_page(struct page *page) 669{ 670 free_hot_cold_page(page, 1); 671} 672 673static inline void prep_zero_page(struct page *page, int order, unsigned int __nocast gfp_flags) 674{ 675 int i; 676 677 BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM); 678 for(i = 0; i < (1 << order); i++) 679 clear_highpage(page + i); 680} 681 682/* 683 * Really, prep_compound_page() should be called from __rmqueue_bulk(). But 684 * we cheat by calling it from here, in the order > 0 path. Saves a branch 685 * or two. 686 */ 687static struct page * 688buffered_rmqueue(struct zone *zone, int order, unsigned int __nocast gfp_flags) 689{ 690 unsigned long flags; 691 struct page *page = NULL; 692 int cold = !!(gfp_flags & __GFP_COLD); 693 694 if (order == 0) { 695 struct per_cpu_pages *pcp; 696 697 pcp = &zone_pcp(zone, get_cpu())->pcp[cold]; 698 local_irq_save(flags); 699 if (pcp->count <= pcp->low) 700 pcp->count += rmqueue_bulk(zone, 0, 701 pcp->batch, &pcp->list); 702 if (pcp->count) { 703 page = list_entry(pcp->list.next, struct page, lru); 704 list_del(&page->lru); 705 pcp->count--; 706 } 707 local_irq_restore(flags); 708 put_cpu(); 709 } 710 711 if (page == NULL) { 712 spin_lock_irqsave(&zone->lock, flags); 713 page = __rmqueue(zone, order); 714 spin_unlock_irqrestore(&zone->lock, flags); 715 } 716 717 if (page != NULL) { 718 BUG_ON(bad_range(zone, page)); 719 mod_page_state_zone(zone, pgalloc, 1 << order); 720 prep_new_page(page, order); 721 722 if (gfp_flags & __GFP_ZERO) 723 prep_zero_page(page, order, gfp_flags); 724 725 if (order && (gfp_flags & __GFP_COMP)) 726 prep_compound_page(page, order); 727 } 728 return page; 729} 730 731/* 732 * Return 1 if free pages are above 'mark'. This takes into account the order 733 * of the allocation. 734 */ 735int zone_watermark_ok(struct zone *z, int order, unsigned long mark, 736 int classzone_idx, int can_try_harder, int gfp_high) 737{ 738 /* free_pages my go negative - that's OK */ 739 long min = mark, free_pages = z->free_pages - (1 << order) + 1; 740 int o; 741 742 if (gfp_high) 743 min -= min / 2; 744 if (can_try_harder) 745 min -= min / 4; 746 747 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 748 return 0; 749 for (o = 0; o < order; o++) { 750 /* At the next order, this order's pages become unavailable */ 751 free_pages -= z->free_area[o].nr_free << o; 752 753 /* Require fewer higher order pages to be free */ 754 min >>= 1; 755 756 if (free_pages <= min) 757 return 0; 758 } 759 return 1; 760} 761 762static inline int 763should_reclaim_zone(struct zone *z, unsigned int gfp_mask) 764{ 765 if (!z->reclaim_pages) 766 return 0; 767 if (gfp_mask & __GFP_NORECLAIM) 768 return 0; 769 return 1; 770} 771 772/* 773 * This is the 'heart' of the zoned buddy allocator. 774 */ 775struct page * fastcall 776__alloc_pages(unsigned int __nocast gfp_mask, unsigned int order, 777 struct zonelist *zonelist) 778{ 779 const int wait = gfp_mask & __GFP_WAIT; 780 struct zone **zones, *z; 781 struct page *page; 782 struct reclaim_state reclaim_state; 783 struct task_struct *p = current; 784 int i; 785 int classzone_idx; 786 int do_retry; 787 int can_try_harder; 788 int did_some_progress; 789 790 might_sleep_if(wait); 791 792 /* 793 * The caller may dip into page reserves a bit more if the caller 794 * cannot run direct reclaim, or is the caller has realtime scheduling 795 * policy 796 */ 797 can_try_harder = (unlikely(rt_task(p)) && !in_interrupt()) || !wait; 798 799 zones = zonelist->zones; /* the list of zones suitable for gfp_mask */ 800 801 if (unlikely(zones[0] == NULL)) { 802 /* Should this ever happen?? */ 803 return NULL; 804 } 805 806 classzone_idx = zone_idx(zones[0]); 807 808restart: 809 /* Go through the zonelist once, looking for a zone with enough free */ 810 for (i = 0; (z = zones[i]) != NULL; i++) { 811 int do_reclaim = should_reclaim_zone(z, gfp_mask); 812 813 if (!cpuset_zone_allowed(z)) 814 continue; 815 816 /* 817 * If the zone is to attempt early page reclaim then this loop 818 * will try to reclaim pages and check the watermark a second 819 * time before giving up and falling back to the next zone. 820 */ 821zone_reclaim_retry: 822 if (!zone_watermark_ok(z, order, z->pages_low, 823 classzone_idx, 0, 0)) { 824 if (!do_reclaim) 825 continue; 826 else { 827 zone_reclaim(z, gfp_mask, order); 828 /* Only try reclaim once */ 829 do_reclaim = 0; 830 goto zone_reclaim_retry; 831 } 832 } 833 834 page = buffered_rmqueue(z, order, gfp_mask); 835 if (page) 836 goto got_pg; 837 } 838 839 for (i = 0; (z = zones[i]) != NULL; i++) 840 wakeup_kswapd(z, order); 841 842 /* 843 * Go through the zonelist again. Let __GFP_HIGH and allocations 844 * coming from realtime tasks to go deeper into reserves 845 * 846 * This is the last chance, in general, before the goto nopage. 847 * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc. 848 */ 849 for (i = 0; (z = zones[i]) != NULL; i++) { 850 if (!zone_watermark_ok(z, order, z->pages_min, 851 classzone_idx, can_try_harder, 852 gfp_mask & __GFP_HIGH)) 853 continue; 854 855 if (wait && !cpuset_zone_allowed(z)) 856 continue; 857 858 page = buffered_rmqueue(z, order, gfp_mask); 859 if (page) 860 goto got_pg; 861 } 862 863 /* This allocation should allow future memory freeing. */ 864 865 if (((p->flags & PF_MEMALLOC) || unlikely(test_thread_flag(TIF_MEMDIE))) 866 && !in_interrupt()) { 867 if (!(gfp_mask & __GFP_NOMEMALLOC)) { 868 /* go through the zonelist yet again, ignoring mins */ 869 for (i = 0; (z = zones[i]) != NULL; i++) { 870 if (!cpuset_zone_allowed(z)) 871 continue; 872 page = buffered_rmqueue(z, order, gfp_mask); 873 if (page) 874 goto got_pg; 875 } 876 } 877 goto nopage; 878 } 879 880 /* Atomic allocations - we can't balance anything */ 881 if (!wait) 882 goto nopage; 883 884rebalance: 885 cond_resched(); 886 887 /* We now go into synchronous reclaim */ 888 p->flags |= PF_MEMALLOC; 889 reclaim_state.reclaimed_slab = 0; 890 p->reclaim_state = &reclaim_state; 891 892 did_some_progress = try_to_free_pages(zones, gfp_mask); 893 894 p->reclaim_state = NULL; 895 p->flags &= ~PF_MEMALLOC; 896 897 cond_resched(); 898 899 if (likely(did_some_progress)) { 900 for (i = 0; (z = zones[i]) != NULL; i++) { 901 if (!zone_watermark_ok(z, order, z->pages_min, 902 classzone_idx, can_try_harder, 903 gfp_mask & __GFP_HIGH)) 904 continue; 905 906 if (!cpuset_zone_allowed(z)) 907 continue; 908 909 page = buffered_rmqueue(z, order, gfp_mask); 910 if (page) 911 goto got_pg; 912 } 913 } else if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) { 914 /* 915 * Go through the zonelist yet one more time, keep 916 * very high watermark here, this is only to catch 917 * a parallel oom killing, we must fail if we're still 918 * under heavy pressure. 919 */ 920 for (i = 0; (z = zones[i]) != NULL; i++) { 921 if (!zone_watermark_ok(z, order, z->pages_high, 922 classzone_idx, 0, 0)) 923 continue; 924 925 if (!cpuset_zone_allowed(z)) 926 continue; 927 928 page = buffered_rmqueue(z, order, gfp_mask); 929 if (page) 930 goto got_pg; 931 } 932 933 out_of_memory(gfp_mask, order); 934 goto restart; 935 } 936 937 /* 938 * Don't let big-order allocations loop unless the caller explicitly 939 * requests that. Wait for some write requests to complete then retry. 940 * 941 * In this implementation, __GFP_REPEAT means __GFP_NOFAIL for order 942 * <= 3, but that may not be true in other implementations. 943 */ 944 do_retry = 0; 945 if (!(gfp_mask & __GFP_NORETRY)) { 946 if ((order <= 3) || (gfp_mask & __GFP_REPEAT)) 947 do_retry = 1; 948 if (gfp_mask & __GFP_NOFAIL) 949 do_retry = 1; 950 } 951 if (do_retry) { 952 blk_congestion_wait(WRITE, HZ/50); 953 goto rebalance; 954 } 955 956nopage: 957 if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) { 958 printk(KERN_WARNING "%s: page allocation failure." 959 " order:%d, mode:0x%x\n", 960 p->comm, order, gfp_mask); 961 dump_stack(); 962 show_mem(); 963 } 964 return NULL; 965got_pg: 966 zone_statistics(zonelist, z); 967 return page; 968} 969 970EXPORT_SYMBOL(__alloc_pages); 971 972/* 973 * Common helper functions. 974 */ 975fastcall unsigned long __get_free_pages(unsigned int __nocast gfp_mask, unsigned int order) 976{ 977 struct page * page; 978 page = alloc_pages(gfp_mask, order); 979 if (!page) 980 return 0; 981 return (unsigned long) page_address(page); 982} 983 984EXPORT_SYMBOL(__get_free_pages); 985 986fastcall unsigned long get_zeroed_page(unsigned int __nocast gfp_mask) 987{ 988 struct page * page; 989 990 /* 991 * get_zeroed_page() returns a 32-bit address, which cannot represent 992 * a highmem page 993 */ 994 BUG_ON(gfp_mask & __GFP_HIGHMEM); 995 996 page = alloc_pages(gfp_mask | __GFP_ZERO, 0); 997 if (page) 998 return (unsigned long) page_address(page); 999 return 0; 1000} 1001 1002EXPORT_SYMBOL(get_zeroed_page); 1003 1004void __pagevec_free(struct pagevec *pvec) 1005{ 1006 int i = pagevec_count(pvec); 1007 1008 while (--i >= 0) 1009 free_hot_cold_page(pvec->pages[i], pvec->cold); 1010} 1011 1012fastcall void __free_pages(struct page *page, unsigned int order) 1013{ 1014 if (!PageReserved(page) && put_page_testzero(page)) { 1015 if (order == 0) 1016 free_hot_page(page); 1017 else 1018 __free_pages_ok(page, order); 1019 } 1020} 1021 1022EXPORT_SYMBOL(__free_pages); 1023 1024fastcall void free_pages(unsigned long addr, unsigned int order) 1025{ 1026 if (addr != 0) { 1027 BUG_ON(!virt_addr_valid((void *)addr)); 1028 __free_pages(virt_to_page((void *)addr), order); 1029 } 1030} 1031 1032EXPORT_SYMBOL(free_pages); 1033 1034/* 1035 * Total amount of free (allocatable) RAM: 1036 */ 1037unsigned int nr_free_pages(void) 1038{ 1039 unsigned int sum = 0; 1040 struct zone *zone; 1041 1042 for_each_zone(zone) 1043 sum += zone->free_pages; 1044 1045 return sum; 1046} 1047 1048EXPORT_SYMBOL(nr_free_pages); 1049 1050#ifdef CONFIG_NUMA 1051unsigned int nr_free_pages_pgdat(pg_data_t *pgdat) 1052{ 1053 unsigned int i, sum = 0; 1054 1055 for (i = 0; i < MAX_NR_ZONES; i++) 1056 sum += pgdat->node_zones[i].free_pages; 1057 1058 return sum; 1059} 1060#endif 1061 1062static unsigned int nr_free_zone_pages(int offset) 1063{ 1064 pg_data_t *pgdat; 1065 unsigned int sum = 0; 1066 1067 for_each_pgdat(pgdat) { 1068 struct zonelist *zonelist = pgdat->node_zonelists + offset; 1069 struct zone **zonep = zonelist->zones; 1070 struct zone *zone; 1071 1072 for (zone = *zonep++; zone; zone = *zonep++) { 1073 unsigned long size = zone->present_pages; 1074 unsigned long high = zone->pages_high; 1075 if (size > high) 1076 sum += size - high; 1077 } 1078 } 1079 1080 return sum; 1081} 1082 1083/* 1084 * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL 1085 */ 1086unsigned int nr_free_buffer_pages(void) 1087{ 1088 return nr_free_zone_pages(GFP_USER & GFP_ZONEMASK); 1089} 1090 1091/* 1092 * Amount of free RAM allocatable within all zones 1093 */ 1094unsigned int nr_free_pagecache_pages(void) 1095{ 1096 return nr_free_zone_pages(GFP_HIGHUSER & GFP_ZONEMASK); 1097} 1098 1099#ifdef CONFIG_HIGHMEM 1100unsigned int nr_free_highpages (void) 1101{ 1102 pg_data_t *pgdat; 1103 unsigned int pages = 0; 1104 1105 for_each_pgdat(pgdat) 1106 pages += pgdat->node_zones[ZONE_HIGHMEM].free_pages; 1107 1108 return pages; 1109} 1110#endif 1111 1112#ifdef CONFIG_NUMA 1113static void show_node(struct zone *zone) 1114{ 1115 printk("Node %d ", zone->zone_pgdat->node_id); 1116} 1117#else 1118#define show_node(zone) do { } while (0) 1119#endif 1120 1121/* 1122 * Accumulate the page_state information across all CPUs. 1123 * The result is unavoidably approximate - it can change 1124 * during and after execution of this function. 1125 */ 1126static DEFINE_PER_CPU(struct page_state, page_states) = {0}; 1127 1128atomic_t nr_pagecache = ATOMIC_INIT(0); 1129EXPORT_SYMBOL(nr_pagecache); 1130#ifdef CONFIG_SMP 1131DEFINE_PER_CPU(long, nr_pagecache_local) = 0; 1132#endif 1133 1134void __get_page_state(struct page_state *ret, int nr) 1135{ 1136 int cpu = 0; 1137 1138 memset(ret, 0, sizeof(*ret)); 1139 1140 cpu = first_cpu(cpu_online_map); 1141 while (cpu < NR_CPUS) { 1142 unsigned long *in, *out, off; 1143 1144 in = (unsigned long *)&per_cpu(page_states, cpu); 1145 1146 cpu = next_cpu(cpu, cpu_online_map); 1147 1148 if (cpu < NR_CPUS) 1149 prefetch(&per_cpu(page_states, cpu)); 1150 1151 out = (unsigned long *)ret; 1152 for (off = 0; off < nr; off++) 1153 *out++ += *in++; 1154 } 1155} 1156 1157void get_page_state(struct page_state *ret) 1158{ 1159 int nr; 1160 1161 nr = offsetof(struct page_state, GET_PAGE_STATE_LAST); 1162 nr /= sizeof(unsigned long); 1163 1164 __get_page_state(ret, nr + 1); 1165} 1166 1167void get_full_page_state(struct page_state *ret) 1168{ 1169 __get_page_state(ret, sizeof(*ret) / sizeof(unsigned long)); 1170} 1171 1172unsigned long __read_page_state(unsigned long offset) 1173{ 1174 unsigned long ret = 0; 1175 int cpu; 1176 1177 for_each_online_cpu(cpu) { 1178 unsigned long in; 1179 1180 in = (unsigned long)&per_cpu(page_states, cpu) + offset; 1181 ret += *((unsigned long *)in); 1182 } 1183 return ret; 1184} 1185 1186void __mod_page_state(unsigned long offset, unsigned long delta) 1187{ 1188 unsigned long flags; 1189 void* ptr; 1190 1191 local_irq_save(flags); 1192 ptr = &__get_cpu_var(page_states); 1193 *(unsigned long*)(ptr + offset) += delta; 1194 local_irq_restore(flags); 1195} 1196 1197EXPORT_SYMBOL(__mod_page_state); 1198 1199void __get_zone_counts(unsigned long *active, unsigned long *inactive, 1200 unsigned long *free, struct pglist_data *pgdat) 1201{ 1202 struct zone *zones = pgdat->node_zones; 1203 int i; 1204 1205 *active = 0; 1206 *inactive = 0; 1207 *free = 0; 1208 for (i = 0; i < MAX_NR_ZONES; i++) { 1209 *active += zones[i].nr_active; 1210 *inactive += zones[i].nr_inactive; 1211 *free += zones[i].free_pages; 1212 } 1213} 1214 1215void get_zone_counts(unsigned long *active, 1216 unsigned long *inactive, unsigned long *free) 1217{ 1218 struct pglist_data *pgdat; 1219 1220 *active = 0; 1221 *inactive = 0; 1222 *free = 0; 1223 for_each_pgdat(pgdat) { 1224 unsigned long l, m, n; 1225 __get_zone_counts(&l, &m, &n, pgdat); 1226 *active += l; 1227 *inactive += m; 1228 *free += n; 1229 } 1230} 1231 1232void si_meminfo(struct sysinfo *val) 1233{ 1234 val->totalram = totalram_pages; 1235 val->sharedram = 0; 1236 val->freeram = nr_free_pages(); 1237 val->bufferram = nr_blockdev_pages(); 1238#ifdef CONFIG_HIGHMEM 1239 val->totalhigh = totalhigh_pages; 1240 val->freehigh = nr_free_highpages(); 1241#else 1242 val->totalhigh = 0; 1243 val->freehigh = 0; 1244#endif 1245 val->mem_unit = PAGE_SIZE; 1246} 1247 1248EXPORT_SYMBOL(si_meminfo); 1249 1250#ifdef CONFIG_NUMA 1251void si_meminfo_node(struct sysinfo *val, int nid) 1252{ 1253 pg_data_t *pgdat = NODE_DATA(nid); 1254 1255 val->totalram = pgdat->node_present_pages; 1256 val->freeram = nr_free_pages_pgdat(pgdat); 1257 val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages; 1258 val->freehigh = pgdat->node_zones[ZONE_HIGHMEM].free_pages; 1259 val->mem_unit = PAGE_SIZE; 1260} 1261#endif 1262 1263#define K(x) ((x) << (PAGE_SHIFT-10)) 1264 1265/* 1266 * Show free area list (used inside shift_scroll-lock stuff) 1267 * We also calculate the percentage fragmentation. We do this by counting the 1268 * memory on each free list with the exception of the first item on the list. 1269 */ 1270void show_free_areas(void) 1271{ 1272 struct page_state ps; 1273 int cpu, temperature; 1274 unsigned long active; 1275 unsigned long inactive; 1276 unsigned long free; 1277 struct zone *zone; 1278 1279 for_each_zone(zone) { 1280 show_node(zone); 1281 printk("%s per-cpu:", zone->name); 1282 1283 if (!zone->present_pages) { 1284 printk(" empty\n"); 1285 continue; 1286 } else 1287 printk("\n"); 1288 1289 for (cpu = 0; cpu < NR_CPUS; ++cpu) { 1290 struct per_cpu_pageset *pageset; 1291 1292 if (!cpu_possible(cpu)) 1293 continue; 1294 1295 pageset = zone_pcp(zone, cpu); 1296 1297 for (temperature = 0; temperature < 2; temperature++) 1298 printk("cpu %d %s: low %d, high %d, batch %d used:%d\n", 1299 cpu, 1300 temperature ? "cold" : "hot", 1301 pageset->pcp[temperature].low, 1302 pageset->pcp[temperature].high, 1303 pageset->pcp[temperature].batch, 1304 pageset->pcp[temperature].count); 1305 } 1306 } 1307 1308 get_page_state(&ps); 1309 get_zone_counts(&active, &inactive, &free); 1310 1311 printk("Free pages: %11ukB (%ukB HighMem)\n", 1312 K(nr_free_pages()), 1313 K(nr_free_highpages())); 1314 1315 printk("Active:%lu inactive:%lu dirty:%lu writeback:%lu " 1316 "unstable:%lu free:%u slab:%lu mapped:%lu pagetables:%lu\n", 1317 active, 1318 inactive, 1319 ps.nr_dirty, 1320 ps.nr_writeback, 1321 ps.nr_unstable, 1322 nr_free_pages(), 1323 ps.nr_slab, 1324 ps.nr_mapped, 1325 ps.nr_page_table_pages); 1326 1327 for_each_zone(zone) { 1328 int i; 1329 1330 show_node(zone); 1331 printk("%s" 1332 " free:%lukB" 1333 " min:%lukB" 1334 " low:%lukB" 1335 " high:%lukB" 1336 " active:%lukB" 1337 " inactive:%lukB" 1338 " present:%lukB" 1339 " pages_scanned:%lu" 1340 " all_unreclaimable? %s" 1341 "\n", 1342 zone->name, 1343 K(zone->free_pages), 1344 K(zone->pages_min), 1345 K(zone->pages_low), 1346 K(zone->pages_high), 1347 K(zone->nr_active), 1348 K(zone->nr_inactive), 1349 K(zone->present_pages), 1350 zone->pages_scanned, 1351 (zone->all_unreclaimable ? "yes" : "no") 1352 ); 1353 printk("lowmem_reserve[]:"); 1354 for (i = 0; i < MAX_NR_ZONES; i++) 1355 printk(" %lu", zone->lowmem_reserve[i]); 1356 printk("\n"); 1357 } 1358 1359 for_each_zone(zone) { 1360 unsigned long nr, flags, order, total = 0; 1361 1362 show_node(zone); 1363 printk("%s: ", zone->name); 1364 if (!zone->present_pages) { 1365 printk("empty\n"); 1366 continue; 1367 } 1368 1369 spin_lock_irqsave(&zone->lock, flags); 1370 for (order = 0; order < MAX_ORDER; order++) { 1371 nr = zone->free_area[order].nr_free; 1372 total += nr << order; 1373 printk("%lu*%lukB ", nr, K(1UL) << order); 1374 } 1375 spin_unlock_irqrestore(&zone->lock, flags); 1376 printk("= %lukB\n", K(total)); 1377 } 1378 1379 show_swap_cache_info(); 1380} 1381 1382/* 1383 * Builds allocation fallback zone lists. 1384 */ 1385static int __init build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, int j, int k) 1386{ 1387 switch (k) { 1388 struct zone *zone; 1389 default: 1390 BUG(); 1391 case ZONE_HIGHMEM: 1392 zone = pgdat->node_zones + ZONE_HIGHMEM; 1393 if (zone->present_pages) { 1394#ifndef CONFIG_HIGHMEM 1395 BUG(); 1396#endif 1397 zonelist->zones[j++] = zone; 1398 } 1399 case ZONE_NORMAL: 1400 zone = pgdat->node_zones + ZONE_NORMAL; 1401 if (zone->present_pages) 1402 zonelist->zones[j++] = zone; 1403 case ZONE_DMA: 1404 zone = pgdat->node_zones + ZONE_DMA; 1405 if (zone->present_pages) 1406 zonelist->zones[j++] = zone; 1407 } 1408 1409 return j; 1410} 1411 1412#ifdef CONFIG_NUMA 1413#define MAX_NODE_LOAD (num_online_nodes()) 1414static int __initdata node_load[MAX_NUMNODES]; 1415/** 1416 * find_next_best_node - find the next node that should appear in a given node's fallback list 1417 * @node: node whose fallback list we're appending 1418 * @used_node_mask: nodemask_t of already used nodes 1419 * 1420 * We use a number of factors to determine which is the next node that should 1421 * appear on a given node's fallback list. The node should not have appeared 1422 * already in @node's fallback list, and it should be the next closest node 1423 * according to the distance array (which contains arbitrary distance values 1424 * from each node to each node in the system), and should also prefer nodes 1425 * with no CPUs, since presumably they'll have very little allocation pressure 1426 * on them otherwise. 1427 * It returns -1 if no node is found. 1428 */ 1429static int __init find_next_best_node(int node, nodemask_t *used_node_mask) 1430{ 1431 int i, n, val; 1432 int min_val = INT_MAX; 1433 int best_node = -1; 1434 1435 for_each_online_node(i) { 1436 cpumask_t tmp; 1437 1438 /* Start from local node */ 1439 n = (node+i) % num_online_nodes(); 1440 1441 /* Don't want a node to appear more than once */ 1442 if (node_isset(n, *used_node_mask)) 1443 continue; 1444 1445 /* Use the local node if we haven't already */ 1446 if (!node_isset(node, *used_node_mask)) { 1447 best_node = node; 1448 break; 1449 } 1450 1451 /* Use the distance array to find the distance */ 1452 val = node_distance(node, n); 1453 1454 /* Give preference to headless and unused nodes */ 1455 tmp = node_to_cpumask(n); 1456 if (!cpus_empty(tmp)) 1457 val += PENALTY_FOR_NODE_WITH_CPUS; 1458 1459 /* Slight preference for less loaded node */ 1460 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 1461 val += node_load[n]; 1462 1463 if (val < min_val) { 1464 min_val = val; 1465 best_node = n; 1466 } 1467 } 1468 1469 if (best_node >= 0) 1470 node_set(best_node, *used_node_mask); 1471 1472 return best_node; 1473} 1474 1475static void __init build_zonelists(pg_data_t *pgdat) 1476{ 1477 int i, j, k, node, local_node; 1478 int prev_node, load; 1479 struct zonelist *zonelist; 1480 nodemask_t used_mask; 1481 1482 /* initialize zonelists */ 1483 for (i = 0; i < GFP_ZONETYPES; i++) { 1484 zonelist = pgdat->node_zonelists + i; 1485 zonelist->zones[0] = NULL; 1486 } 1487 1488 /* NUMA-aware ordering of nodes */ 1489 local_node = pgdat->node_id; 1490 load = num_online_nodes(); 1491 prev_node = local_node; 1492 nodes_clear(used_mask); 1493 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 1494 /* 1495 * We don't want to pressure a particular node. 1496 * So adding penalty to the first node in same 1497 * distance group to make it round-robin. 1498 */ 1499 if (node_distance(local_node, node) != 1500 node_distance(local_node, prev_node)) 1501 node_load[node] += load; 1502 prev_node = node; 1503 load--; 1504 for (i = 0; i < GFP_ZONETYPES; i++) { 1505 zonelist = pgdat->node_zonelists + i; 1506 for (j = 0; zonelist->zones[j] != NULL; j++); 1507 1508 k = ZONE_NORMAL; 1509 if (i & __GFP_HIGHMEM) 1510 k = ZONE_HIGHMEM; 1511 if (i & __GFP_DMA) 1512 k = ZONE_DMA; 1513 1514 j = build_zonelists_node(NODE_DATA(node), zonelist, j, k); 1515 zonelist->zones[j] = NULL; 1516 } 1517 } 1518} 1519 1520#else /* CONFIG_NUMA */ 1521 1522static void __init build_zonelists(pg_data_t *pgdat) 1523{ 1524 int i, j, k, node, local_node; 1525 1526 local_node = pgdat->node_id; 1527 for (i = 0; i < GFP_ZONETYPES; i++) { 1528 struct zonelist *zonelist; 1529 1530 zonelist = pgdat->node_zonelists + i; 1531 1532 j = 0; 1533 k = ZONE_NORMAL; 1534 if (i & __GFP_HIGHMEM) 1535 k = ZONE_HIGHMEM; 1536 if (i & __GFP_DMA) 1537 k = ZONE_DMA; 1538 1539 j = build_zonelists_node(pgdat, zonelist, j, k); 1540 /* 1541 * Now we build the zonelist so that it contains the zones 1542 * of all the other nodes. 1543 * We don't want to pressure a particular node, so when 1544 * building the zones for node N, we make sure that the 1545 * zones coming right after the local ones are those from 1546 * node N+1 (modulo N) 1547 */ 1548 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 1549 if (!node_online(node)) 1550 continue; 1551 j = build_zonelists_node(NODE_DATA(node), zonelist, j, k); 1552 } 1553 for (node = 0; node < local_node; node++) { 1554 if (!node_online(node)) 1555 continue; 1556 j = build_zonelists_node(NODE_DATA(node), zonelist, j, k); 1557 } 1558 1559 zonelist->zones[j] = NULL; 1560 } 1561} 1562 1563#endif /* CONFIG_NUMA */ 1564 1565void __init build_all_zonelists(void) 1566{ 1567 int i; 1568 1569 for_each_online_node(i) 1570 build_zonelists(NODE_DATA(i)); 1571 printk("Built %i zonelists\n", num_online_nodes()); 1572 cpuset_init_current_mems_allowed(); 1573} 1574 1575/* 1576 * Helper functions to size the waitqueue hash table. 1577 * Essentially these want to choose hash table sizes sufficiently 1578 * large so that collisions trying to wait on pages are rare. 1579 * But in fact, the number of active page waitqueues on typical 1580 * systems is ridiculously low, less than 200. So this is even 1581 * conservative, even though it seems large. 1582 * 1583 * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to 1584 * waitqueues, i.e. the size of the waitq table given the number of pages. 1585 */ 1586#define PAGES_PER_WAITQUEUE 256 1587 1588static inline unsigned long wait_table_size(unsigned long pages) 1589{ 1590 unsigned long size = 1; 1591 1592 pages /= PAGES_PER_WAITQUEUE; 1593 1594 while (size < pages) 1595 size <<= 1; 1596 1597 /* 1598 * Once we have dozens or even hundreds of threads sleeping 1599 * on IO we've got bigger problems than wait queue collision. 1600 * Limit the size of the wait table to a reasonable size. 1601 */ 1602 size = min(size, 4096UL); 1603 1604 return max(size, 4UL); 1605} 1606 1607/* 1608 * This is an integer logarithm so that shifts can be used later 1609 * to extract the more random high bits from the multiplicative 1610 * hash function before the remainder is taken. 1611 */ 1612static inline unsigned long wait_table_bits(unsigned long size) 1613{ 1614 return ffz(~size); 1615} 1616 1617#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) 1618 1619static void __init calculate_zone_totalpages(struct pglist_data *pgdat, 1620 unsigned long *zones_size, unsigned long *zholes_size) 1621{ 1622 unsigned long realtotalpages, totalpages = 0; 1623 int i; 1624 1625 for (i = 0; i < MAX_NR_ZONES; i++) 1626 totalpages += zones_size[i]; 1627 pgdat->node_spanned_pages = totalpages; 1628 1629 realtotalpages = totalpages; 1630 if (zholes_size) 1631 for (i = 0; i < MAX_NR_ZONES; i++) 1632 realtotalpages -= zholes_size[i]; 1633 pgdat->node_present_pages = realtotalpages; 1634 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 1635} 1636 1637 1638/* 1639 * Initially all pages are reserved - free ones are freed 1640 * up by free_all_bootmem() once the early boot process is 1641 * done. Non-atomic initialization, single-pass. 1642 */ 1643void __init memmap_init_zone(unsigned long size, int nid, unsigned long zone, 1644 unsigned long start_pfn) 1645{ 1646 struct page *page; 1647 unsigned long end_pfn = start_pfn + size; 1648 unsigned long pfn; 1649 1650 for (pfn = start_pfn; pfn < end_pfn; pfn++, page++) { 1651 if (!early_pfn_valid(pfn)) 1652 continue; 1653 if (!early_pfn_in_nid(pfn, nid)) 1654 continue; 1655 page = pfn_to_page(pfn); 1656 set_page_links(page, zone, nid, pfn); 1657 set_page_count(page, 0); 1658 reset_page_mapcount(page); 1659 SetPageReserved(page); 1660 INIT_LIST_HEAD(&page->lru); 1661#ifdef WANT_PAGE_VIRTUAL 1662 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1663 if (!is_highmem_idx(zone)) 1664 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1665#endif 1666 } 1667} 1668 1669void zone_init_free_lists(struct pglist_data *pgdat, struct zone *zone, 1670 unsigned long size) 1671{ 1672 int order; 1673 for (order = 0; order < MAX_ORDER ; order++) { 1674 INIT_LIST_HEAD(&zone->free_area[order].free_list); 1675 zone->free_area[order].nr_free = 0; 1676 } 1677} 1678 1679#define ZONETABLE_INDEX(x, zone_nr) ((x << ZONES_SHIFT) | zone_nr) 1680void zonetable_add(struct zone *zone, int nid, int zid, unsigned long pfn, 1681 unsigned long size) 1682{ 1683 unsigned long snum = pfn_to_section_nr(pfn); 1684 unsigned long end = pfn_to_section_nr(pfn + size); 1685 1686 if (FLAGS_HAS_NODE) 1687 zone_table[ZONETABLE_INDEX(nid, zid)] = zone; 1688 else 1689 for (; snum <= end; snum++) 1690 zone_table[ZONETABLE_INDEX(snum, zid)] = zone; 1691} 1692 1693#ifndef __HAVE_ARCH_MEMMAP_INIT 1694#define memmap_init(size, nid, zone, start_pfn) \ 1695 memmap_init_zone((size), (nid), (zone), (start_pfn)) 1696#endif 1697 1698static int __devinit zone_batchsize(struct zone *zone) 1699{ 1700 int batch; 1701 1702 /* 1703 * The per-cpu-pages pools are set to around 1000th of the 1704 * size of the zone. But no more than 1/4 of a meg - there's 1705 * no point in going beyond the size of L2 cache. 1706 * 1707 * OK, so we don't know how big the cache is. So guess. 1708 */ 1709 batch = zone->present_pages / 1024; 1710 if (batch * PAGE_SIZE > 256 * 1024) 1711 batch = (256 * 1024) / PAGE_SIZE; 1712 batch /= 4; /* We effectively *= 4 below */ 1713 if (batch < 1) 1714 batch = 1; 1715 1716 /* 1717 * Clamp the batch to a 2^n - 1 value. Having a power 1718 * of 2 value was found to be more likely to have 1719 * suboptimal cache aliasing properties in some cases. 1720 * 1721 * For example if 2 tasks are alternately allocating 1722 * batches of pages, one task can end up with a lot 1723 * of pages of one half of the possible page colors 1724 * and the other with pages of the other colors. 1725 */ 1726 batch = (1 << fls(batch + batch/2)) - 1; 1727 return batch; 1728} 1729 1730inline void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 1731{ 1732 struct per_cpu_pages *pcp; 1733 1734 pcp = &p->pcp[0]; /* hot */ 1735 pcp->count = 0; 1736 pcp->low = 2 * batch; 1737 pcp->high = 6 * batch; 1738 pcp->batch = max(1UL, 1 * batch); 1739 INIT_LIST_HEAD(&pcp->list); 1740 1741 pcp = &p->pcp[1]; /* cold*/ 1742 pcp->count = 0; 1743 pcp->low = 0; 1744 pcp->high = 2 * batch; 1745 pcp->batch = max(1UL, 1 * batch); 1746 INIT_LIST_HEAD(&pcp->list); 1747} 1748 1749#ifdef CONFIG_NUMA 1750/* 1751 * Boot pageset table. One per cpu which is going to be used for all 1752 * zones and all nodes. The parameters will be set in such a way 1753 * that an item put on a list will immediately be handed over to 1754 * the buddy list. This is safe since pageset manipulation is done 1755 * with interrupts disabled. 1756 * 1757 * Some NUMA counter updates may also be caught by the boot pagesets. 1758 * 1759 * The boot_pagesets must be kept even after bootup is complete for 1760 * unused processors and/or zones. They do play a role for bootstrapping 1761 * hotplugged processors. 1762 * 1763 * zoneinfo_show() and maybe other functions do 1764 * not check if the processor is online before following the pageset pointer. 1765 * Other parts of the kernel may not check if the zone is available. 1766 */ 1767static struct per_cpu_pageset 1768 boot_pageset[NR_CPUS]; 1769 1770/* 1771 * Dynamically allocate memory for the 1772 * per cpu pageset array in struct zone. 1773 */ 1774static int __devinit process_zones(int cpu) 1775{ 1776 struct zone *zone, *dzone; 1777 1778 for_each_zone(zone) { 1779 1780 zone->pageset[cpu] = kmalloc_node(sizeof(struct per_cpu_pageset), 1781 GFP_KERNEL, cpu_to_node(cpu)); 1782 if (!zone->pageset[cpu]) 1783 goto bad; 1784 1785 setup_pageset(zone->pageset[cpu], zone_batchsize(zone)); 1786 } 1787 1788 return 0; 1789bad: 1790 for_each_zone(dzone) { 1791 if (dzone == zone) 1792 break; 1793 kfree(dzone->pageset[cpu]); 1794 dzone->pageset[cpu] = NULL; 1795 } 1796 return -ENOMEM; 1797} 1798 1799static inline void free_zone_pagesets(int cpu) 1800{ 1801#ifdef CONFIG_NUMA 1802 struct zone *zone; 1803 1804 for_each_zone(zone) { 1805 struct per_cpu_pageset *pset = zone_pcp(zone, cpu); 1806 1807 zone_pcp(zone, cpu) = NULL; 1808 kfree(pset); 1809 } 1810#endif 1811} 1812 1813static int __devinit pageset_cpuup_callback(struct notifier_block *nfb, 1814 unsigned long action, 1815 void *hcpu) 1816{ 1817 int cpu = (long)hcpu; 1818 int ret = NOTIFY_OK; 1819 1820 switch (action) { 1821 case CPU_UP_PREPARE: 1822 if (process_zones(cpu)) 1823 ret = NOTIFY_BAD; 1824 break; 1825#ifdef CONFIG_HOTPLUG_CPU 1826 case CPU_DEAD: 1827 free_zone_pagesets(cpu); 1828 break; 1829#endif 1830 default: 1831 break; 1832 } 1833 return ret; 1834} 1835 1836static struct notifier_block pageset_notifier = 1837 { &pageset_cpuup_callback, NULL, 0 }; 1838 1839void __init setup_per_cpu_pageset() 1840{ 1841 int err; 1842 1843 /* Initialize per_cpu_pageset for cpu 0. 1844 * A cpuup callback will do this for every cpu 1845 * as it comes online 1846 */ 1847 err = process_zones(smp_processor_id()); 1848 BUG_ON(err); 1849 register_cpu_notifier(&pageset_notifier); 1850} 1851 1852#endif 1853 1854/* 1855 * Set up the zone data structures: 1856 * - mark all pages reserved 1857 * - mark all memory queues empty 1858 * - clear the memory bitmaps 1859 */ 1860static void __init free_area_init_core(struct pglist_data *pgdat, 1861 unsigned long *zones_size, unsigned long *zholes_size) 1862{ 1863 unsigned long i, j; 1864 int cpu, nid = pgdat->node_id; 1865 unsigned long zone_start_pfn = pgdat->node_start_pfn; 1866 1867 pgdat->nr_zones = 0; 1868 init_waitqueue_head(&pgdat->kswapd_wait); 1869 pgdat->kswapd_max_order = 0; 1870 1871 for (j = 0; j < MAX_NR_ZONES; j++) { 1872 struct zone *zone = pgdat->node_zones + j; 1873 unsigned long size, realsize; 1874 unsigned long batch; 1875 1876 realsize = size = zones_size[j]; 1877 if (zholes_size) 1878 realsize -= zholes_size[j]; 1879 1880 if (j == ZONE_DMA || j == ZONE_NORMAL) 1881 nr_kernel_pages += realsize; 1882 nr_all_pages += realsize; 1883 1884 zone->spanned_pages = size; 1885 zone->present_pages = realsize; 1886 zone->name = zone_names[j]; 1887 spin_lock_init(&zone->lock); 1888 spin_lock_init(&zone->lru_lock); 1889 zone->zone_pgdat = pgdat; 1890 zone->free_pages = 0; 1891 1892 zone->temp_priority = zone->prev_priority = DEF_PRIORITY; 1893 1894 batch = zone_batchsize(zone); 1895 1896 for (cpu = 0; cpu < NR_CPUS; cpu++) { 1897#ifdef CONFIG_NUMA 1898 /* Early boot. Slab allocator not functional yet */ 1899 zone->pageset[cpu] = &boot_pageset[cpu]; 1900 setup_pageset(&boot_pageset[cpu],0); 1901#else 1902 setup_pageset(zone_pcp(zone,cpu), batch); 1903#endif 1904 } 1905 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", 1906 zone_names[j], realsize, batch); 1907 INIT_LIST_HEAD(&zone->active_list); 1908 INIT_LIST_HEAD(&zone->inactive_list); 1909 zone->nr_scan_active = 0; 1910 zone->nr_scan_inactive = 0; 1911 zone->nr_active = 0; 1912 zone->nr_inactive = 0; 1913 atomic_set(&zone->reclaim_in_progress, -1); 1914 if (!size) 1915 continue; 1916 1917 /* 1918 * The per-page waitqueue mechanism uses hashed waitqueues 1919 * per zone. 1920 */ 1921 zone->wait_table_size = wait_table_size(size); 1922 zone->wait_table_bits = 1923 wait_table_bits(zone->wait_table_size); 1924 zone->wait_table = (wait_queue_head_t *) 1925 alloc_bootmem_node(pgdat, zone->wait_table_size 1926 * sizeof(wait_queue_head_t)); 1927 1928 for(i = 0; i < zone->wait_table_size; ++i) 1929 init_waitqueue_head(zone->wait_table + i); 1930 1931 pgdat->nr_zones = j+1; 1932 1933 zone->zone_mem_map = pfn_to_page(zone_start_pfn); 1934 zone->zone_start_pfn = zone_start_pfn; 1935 1936 memmap_init(size, nid, j, zone_start_pfn); 1937 1938 zonetable_add(zone, nid, j, zone_start_pfn, size); 1939 1940 zone_start_pfn += size; 1941 1942 zone_init_free_lists(pgdat, zone, zone->spanned_pages); 1943 } 1944} 1945 1946static void __init alloc_node_mem_map(struct pglist_data *pgdat) 1947{ 1948 /* Skip empty nodes */ 1949 if (!pgdat->node_spanned_pages) 1950 return; 1951 1952#ifdef CONFIG_FLAT_NODE_MEM_MAP 1953 /* ia64 gets its own node_mem_map, before this, without bootmem */ 1954 if (!pgdat->node_mem_map) { 1955 unsigned long size; 1956 struct page *map; 1957 1958 size = (pgdat->node_spanned_pages + 1) * sizeof(struct page); 1959 map = alloc_remap(pgdat->node_id, size); 1960 if (!map) 1961 map = alloc_bootmem_node(pgdat, size); 1962 pgdat->node_mem_map = map; 1963 } 1964#ifdef CONFIG_FLATMEM 1965 /* 1966 * With no DISCONTIG, the global mem_map is just set as node 0's 1967 */ 1968 if (pgdat == NODE_DATA(0)) 1969 mem_map = NODE_DATA(0)->node_mem_map; 1970#endif 1971#endif /* CONFIG_FLAT_NODE_MEM_MAP */ 1972} 1973 1974void __init free_area_init_node(int nid, struct pglist_data *pgdat, 1975 unsigned long *zones_size, unsigned long node_start_pfn, 1976 unsigned long *zholes_size) 1977{ 1978 pgdat->node_id = nid; 1979 pgdat->node_start_pfn = node_start_pfn; 1980 calculate_zone_totalpages(pgdat, zones_size, zholes_size); 1981 1982 alloc_node_mem_map(pgdat); 1983 1984 free_area_init_core(pgdat, zones_size, zholes_size); 1985} 1986 1987#ifndef CONFIG_NEED_MULTIPLE_NODES 1988static bootmem_data_t contig_bootmem_data; 1989struct pglist_data contig_page_data = { .bdata = &contig_bootmem_data }; 1990 1991EXPORT_SYMBOL(contig_page_data); 1992#endif 1993 1994void __init free_area_init(unsigned long *zones_size) 1995{ 1996 free_area_init_node(0, NODE_DATA(0), zones_size, 1997 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 1998} 1999 2000#ifdef CONFIG_PROC_FS 2001 2002#include <linux/seq_file.h> 2003 2004static void *frag_start(struct seq_file *m, loff_t *pos) 2005{ 2006 pg_data_t *pgdat; 2007 loff_t node = *pos; 2008 2009 for (pgdat = pgdat_list; pgdat && node; pgdat = pgdat->pgdat_next) 2010 --node; 2011 2012 return pgdat; 2013} 2014 2015static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 2016{ 2017 pg_data_t *pgdat = (pg_data_t *)arg; 2018 2019 (*pos)++; 2020 return pgdat->pgdat_next; 2021} 2022 2023static void frag_stop(struct seq_file *m, void *arg) 2024{ 2025} 2026 2027/* 2028 * This walks the free areas for each zone. 2029 */ 2030static int frag_show(struct seq_file *m, void *arg) 2031{ 2032 pg_data_t *pgdat = (pg_data_t *)arg; 2033 struct zone *zone; 2034 struct zone *node_zones = pgdat->node_zones; 2035 unsigned long flags; 2036 int order; 2037 2038 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 2039 if (!zone->present_pages) 2040 continue; 2041 2042 spin_lock_irqsave(&zone->lock, flags); 2043 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 2044 for (order = 0; order < MAX_ORDER; ++order) 2045 seq_printf(m, "%6lu ", zone->free_area[order].nr_free); 2046 spin_unlock_irqrestore(&zone->lock, flags); 2047 seq_putc(m, '\n'); 2048 } 2049 return 0; 2050} 2051 2052struct seq_operations fragmentation_op = { 2053 .start = frag_start, 2054 .next = frag_next, 2055 .stop = frag_stop, 2056 .show = frag_show, 2057}; 2058 2059/* 2060 * Output information about zones in @pgdat. 2061 */ 2062static int zoneinfo_show(struct seq_file *m, void *arg) 2063{ 2064 pg_data_t *pgdat = arg; 2065 struct zone *zone; 2066 struct zone *node_zones = pgdat->node_zones; 2067 unsigned long flags; 2068 2069 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; zone++) { 2070 int i; 2071 2072 if (!zone->present_pages) 2073 continue; 2074 2075 spin_lock_irqsave(&zone->lock, flags); 2076 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 2077 seq_printf(m, 2078 "\n pages free %lu" 2079 "\n min %lu" 2080 "\n low %lu" 2081 "\n high %lu" 2082 "\n active %lu" 2083 "\n inactive %lu" 2084 "\n scanned %lu (a: %lu i: %lu)" 2085 "\n spanned %lu" 2086 "\n present %lu", 2087 zone->free_pages, 2088 zone->pages_min, 2089 zone->pages_low, 2090 zone->pages_high, 2091 zone->nr_active, 2092 zone->nr_inactive, 2093 zone->pages_scanned, 2094 zone->nr_scan_active, zone->nr_scan_inactive, 2095 zone->spanned_pages, 2096 zone->present_pages); 2097 seq_printf(m, 2098 "\n protection: (%lu", 2099 zone->lowmem_reserve[0]); 2100 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 2101 seq_printf(m, ", %lu", zone->lowmem_reserve[i]); 2102 seq_printf(m, 2103 ")" 2104 "\n pagesets"); 2105 for (i = 0; i < ARRAY_SIZE(zone->pageset); i++) { 2106 struct per_cpu_pageset *pageset; 2107 int j; 2108 2109 pageset = zone_pcp(zone, i); 2110 for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) { 2111 if (pageset->pcp[j].count) 2112 break; 2113 } 2114 if (j == ARRAY_SIZE(pageset->pcp)) 2115 continue; 2116 for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) { 2117 seq_printf(m, 2118 "\n cpu: %i pcp: %i" 2119 "\n count: %i" 2120 "\n low: %i" 2121 "\n high: %i" 2122 "\n batch: %i", 2123 i, j, 2124 pageset->pcp[j].count, 2125 pageset->pcp[j].low, 2126 pageset->pcp[j].high, 2127 pageset->pcp[j].batch); 2128 } 2129#ifdef CONFIG_NUMA 2130 seq_printf(m, 2131 "\n numa_hit: %lu" 2132 "\n numa_miss: %lu" 2133 "\n numa_foreign: %lu" 2134 "\n interleave_hit: %lu" 2135 "\n local_node: %lu" 2136 "\n other_node: %lu", 2137 pageset->numa_hit, 2138 pageset->numa_miss, 2139 pageset->numa_foreign, 2140 pageset->interleave_hit, 2141 pageset->local_node, 2142 pageset->other_node); 2143#endif 2144 } 2145 seq_printf(m, 2146 "\n all_unreclaimable: %u" 2147 "\n prev_priority: %i" 2148 "\n temp_priority: %i" 2149 "\n start_pfn: %lu", 2150 zone->all_unreclaimable, 2151 zone->prev_priority, 2152 zone->temp_priority, 2153 zone->zone_start_pfn); 2154 spin_unlock_irqrestore(&zone->lock, flags); 2155 seq_putc(m, '\n'); 2156 } 2157 return 0; 2158} 2159 2160struct seq_operations zoneinfo_op = { 2161 .start = frag_start, /* iterate over all zones. The same as in 2162 * fragmentation. */ 2163 .next = frag_next, 2164 .stop = frag_stop, 2165 .show = zoneinfo_show, 2166}; 2167 2168static char *vmstat_text[] = { 2169 "nr_dirty", 2170 "nr_writeback", 2171 "nr_unstable", 2172 "nr_page_table_pages", 2173 "nr_mapped", 2174 "nr_slab", 2175 2176 "pgpgin", 2177 "pgpgout", 2178 "pswpin", 2179 "pswpout", 2180 "pgalloc_high", 2181 2182 "pgalloc_normal", 2183 "pgalloc_dma", 2184 "pgfree", 2185 "pgactivate", 2186 "pgdeactivate", 2187 2188 "pgfault", 2189 "pgmajfault", 2190 "pgrefill_high", 2191 "pgrefill_normal", 2192 "pgrefill_dma", 2193 2194 "pgsteal_high", 2195 "pgsteal_normal", 2196 "pgsteal_dma", 2197 "pgscan_kswapd_high", 2198 "pgscan_kswapd_normal", 2199 2200 "pgscan_kswapd_dma", 2201 "pgscan_direct_high", 2202 "pgscan_direct_normal", 2203 "pgscan_direct_dma", 2204 "pginodesteal", 2205 2206 "slabs_scanned", 2207 "kswapd_steal", 2208 "kswapd_inodesteal", 2209 "pageoutrun", 2210 "allocstall", 2211 2212 "pgrotated", 2213 "nr_bounce", 2214}; 2215 2216static void *vmstat_start(struct seq_file *m, loff_t *pos) 2217{ 2218 struct page_state *ps; 2219 2220 if (*pos >= ARRAY_SIZE(vmstat_text)) 2221 return NULL; 2222 2223 ps = kmalloc(sizeof(*ps), GFP_KERNEL); 2224 m->private = ps; 2225 if (!ps) 2226 return ERR_PTR(-ENOMEM); 2227 get_full_page_state(ps); 2228 ps->pgpgin /= 2; /* sectors -> kbytes */ 2229 ps->pgpgout /= 2; 2230 return (unsigned long *)ps + *pos; 2231} 2232 2233static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 2234{ 2235 (*pos)++; 2236 if (*pos >= ARRAY_SIZE(vmstat_text)) 2237 return NULL; 2238 return (unsigned long *)m->private + *pos; 2239} 2240 2241static int vmstat_show(struct seq_file *m, void *arg) 2242{ 2243 unsigned long *l = arg; 2244 unsigned long off = l - (unsigned long *)m->private; 2245 2246 seq_printf(m, "%s %lu\n", vmstat_text[off], *l); 2247 return 0; 2248} 2249 2250static void vmstat_stop(struct seq_file *m, void *arg) 2251{ 2252 kfree(m->private); 2253 m->private = NULL; 2254} 2255 2256struct seq_operations vmstat_op = { 2257 .start = vmstat_start, 2258 .next = vmstat_next, 2259 .stop = vmstat_stop, 2260 .show = vmstat_show, 2261}; 2262 2263#endif /* CONFIG_PROC_FS */ 2264 2265#ifdef CONFIG_HOTPLUG_CPU 2266static int page_alloc_cpu_notify(struct notifier_block *self, 2267 unsigned long action, void *hcpu) 2268{ 2269 int cpu = (unsigned long)hcpu; 2270 long *count; 2271 unsigned long *src, *dest; 2272 2273 if (action == CPU_DEAD) { 2274 int i; 2275 2276 /* Drain local pagecache count. */ 2277 count = &per_cpu(nr_pagecache_local, cpu); 2278 atomic_add(*count, &nr_pagecache); 2279 *count = 0; 2280 local_irq_disable(); 2281 __drain_pages(cpu); 2282 2283 /* Add dead cpu's page_states to our own. */ 2284 dest = (unsigned long *)&__get_cpu_var(page_states); 2285 src = (unsigned long *)&per_cpu(page_states, cpu); 2286 2287 for (i = 0; i < sizeof(struct page_state)/sizeof(unsigned long); 2288 i++) { 2289 dest[i] += src[i]; 2290 src[i] = 0; 2291 } 2292 2293 local_irq_enable(); 2294 } 2295 return NOTIFY_OK; 2296} 2297#endif /* CONFIG_HOTPLUG_CPU */ 2298 2299void __init page_alloc_init(void) 2300{ 2301 hotcpu_notifier(page_alloc_cpu_notify, 0); 2302} 2303 2304/* 2305 * setup_per_zone_lowmem_reserve - called whenever 2306 * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone 2307 * has a correct pages reserved value, so an adequate number of 2308 * pages are left in the zone after a successful __alloc_pages(). 2309 */ 2310static void setup_per_zone_lowmem_reserve(void) 2311{ 2312 struct pglist_data *pgdat; 2313 int j, idx; 2314 2315 for_each_pgdat(pgdat) { 2316 for (j = 0; j < MAX_NR_ZONES; j++) { 2317 struct zone *zone = pgdat->node_zones + j; 2318 unsigned long present_pages = zone->present_pages; 2319 2320 zone->lowmem_reserve[j] = 0; 2321 2322 for (idx = j-1; idx >= 0; idx--) { 2323 struct zone *lower_zone; 2324 2325 if (sysctl_lowmem_reserve_ratio[idx] < 1) 2326 sysctl_lowmem_reserve_ratio[idx] = 1; 2327 2328 lower_zone = pgdat->node_zones + idx; 2329 lower_zone->lowmem_reserve[j] = present_pages / 2330 sysctl_lowmem_reserve_ratio[idx]; 2331 present_pages += lower_zone->present_pages; 2332 } 2333 } 2334 } 2335} 2336 2337/* 2338 * setup_per_zone_pages_min - called when min_free_kbytes changes. Ensures 2339 * that the pages_{min,low,high} values for each zone are set correctly 2340 * with respect to min_free_kbytes. 2341 */ 2342static void setup_per_zone_pages_min(void) 2343{ 2344 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 2345 unsigned long lowmem_pages = 0; 2346 struct zone *zone; 2347 unsigned long flags; 2348 2349 /* Calculate total number of !ZONE_HIGHMEM pages */ 2350 for_each_zone(zone) { 2351 if (!is_highmem(zone)) 2352 lowmem_pages += zone->present_pages; 2353 } 2354 2355 for_each_zone(zone) { 2356 spin_lock_irqsave(&zone->lru_lock, flags); 2357 if (is_highmem(zone)) { 2358 /* 2359 * Often, highmem doesn't need to reserve any pages. 2360 * But the pages_min/low/high values are also used for 2361 * batching up page reclaim activity so we need a 2362 * decent value here. 2363 */ 2364 int min_pages; 2365 2366 min_pages = zone->present_pages / 1024; 2367 if (min_pages < SWAP_CLUSTER_MAX) 2368 min_pages = SWAP_CLUSTER_MAX; 2369 if (min_pages > 128) 2370 min_pages = 128; 2371 zone->pages_min = min_pages; 2372 } else { 2373 /* if it's a lowmem zone, reserve a number of pages 2374 * proportionate to the zone's size. 2375 */ 2376 zone->pages_min = (pages_min * zone->present_pages) / 2377 lowmem_pages; 2378 } 2379 2380 /* 2381 * When interpreting these watermarks, just keep in mind that: 2382 * zone->pages_min == (zone->pages_min * 4) / 4; 2383 */ 2384 zone->pages_low = (zone->pages_min * 5) / 4; 2385 zone->pages_high = (zone->pages_min * 6) / 4; 2386 spin_unlock_irqrestore(&zone->lru_lock, flags); 2387 } 2388} 2389 2390/* 2391 * Initialise min_free_kbytes. 2392 * 2393 * For small machines we want it small (128k min). For large machines 2394 * we want it large (64MB max). But it is not linear, because network 2395 * bandwidth does not increase linearly with machine size. We use 2396 * 2397 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 2398 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 2399 * 2400 * which yields 2401 * 2402 * 16MB: 512k 2403 * 32MB: 724k 2404 * 64MB: 1024k 2405 * 128MB: 1448k 2406 * 256MB: 2048k 2407 * 512MB: 2896k 2408 * 1024MB: 4096k 2409 * 2048MB: 5792k 2410 * 4096MB: 8192k 2411 * 8192MB: 11584k 2412 * 16384MB: 16384k 2413 */ 2414static int __init init_per_zone_pages_min(void) 2415{ 2416 unsigned long lowmem_kbytes; 2417 2418 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 2419 2420 min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 2421 if (min_free_kbytes < 128) 2422 min_free_kbytes = 128; 2423 if (min_free_kbytes > 65536) 2424 min_free_kbytes = 65536; 2425 setup_per_zone_pages_min(); 2426 setup_per_zone_lowmem_reserve(); 2427 return 0; 2428} 2429module_init(init_per_zone_pages_min) 2430 2431/* 2432 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 2433 * that we can call two helper functions whenever min_free_kbytes 2434 * changes. 2435 */ 2436int min_free_kbytes_sysctl_handler(ctl_table *table, int write, 2437 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 2438{ 2439 proc_dointvec(table, write, file, buffer, length, ppos); 2440 setup_per_zone_pages_min(); 2441 return 0; 2442} 2443 2444/* 2445 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 2446 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 2447 * whenever sysctl_lowmem_reserve_ratio changes. 2448 * 2449 * The reserve ratio obviously has absolutely no relation with the 2450 * pages_min watermarks. The lowmem reserve ratio can only make sense 2451 * if in function of the boot time zone sizes. 2452 */ 2453int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write, 2454 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 2455{ 2456 proc_dointvec_minmax(table, write, file, buffer, length, ppos); 2457 setup_per_zone_lowmem_reserve(); 2458 return 0; 2459} 2460 2461__initdata int hashdist = HASHDIST_DEFAULT; 2462 2463#ifdef CONFIG_NUMA 2464static int __init set_hashdist(char *str) 2465{ 2466 if (!str) 2467 return 0; 2468 hashdist = simple_strtoul(str, &str, 0); 2469 return 1; 2470} 2471__setup("hashdist=", set_hashdist); 2472#endif 2473 2474/* 2475 * allocate a large system hash table from bootmem 2476 * - it is assumed that the hash table must contain an exact power-of-2 2477 * quantity of entries 2478 * - limit is the number of hash buckets, not the total allocation size 2479 */ 2480void *__init alloc_large_system_hash(const char *tablename, 2481 unsigned long bucketsize, 2482 unsigned long numentries, 2483 int scale, 2484 int flags, 2485 unsigned int *_hash_shift, 2486 unsigned int *_hash_mask, 2487 unsigned long limit) 2488{ 2489 unsigned long long max = limit; 2490 unsigned long log2qty, size; 2491 void *table = NULL; 2492 2493 /* allow the kernel cmdline to have a say */ 2494 if (!numentries) { 2495 /* round applicable memory size up to nearest megabyte */ 2496 numentries = (flags & HASH_HIGHMEM) ? nr_all_pages : nr_kernel_pages; 2497 numentries += (1UL << (20 - PAGE_SHIFT)) - 1; 2498 numentries >>= 20 - PAGE_SHIFT; 2499 numentries <<= 20 - PAGE_SHIFT; 2500 2501 /* limit to 1 bucket per 2^scale bytes of low memory */ 2502 if (scale > PAGE_SHIFT) 2503 numentries >>= (scale - PAGE_SHIFT); 2504 else 2505 numentries <<= (PAGE_SHIFT - scale); 2506 } 2507 /* rounded up to nearest power of 2 in size */ 2508 numentries = 1UL << (long_log2(numentries) + 1); 2509 2510 /* limit allocation size to 1/16 total memory by default */ 2511 if (max == 0) { 2512 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 2513 do_div(max, bucketsize); 2514 } 2515 2516 if (numentries > max) 2517 numentries = max; 2518 2519 log2qty = long_log2(numentries); 2520 2521 do { 2522 size = bucketsize << log2qty; 2523 if (flags & HASH_EARLY) 2524 table = alloc_bootmem(size); 2525 else if (hashdist) 2526 table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); 2527 else { 2528 unsigned long order; 2529 for (order = 0; ((1UL << order) << PAGE_SHIFT) < size; order++) 2530 ; 2531 table = (void*) __get_free_pages(GFP_ATOMIC, order); 2532 } 2533 } while (!table && size > PAGE_SIZE && --log2qty); 2534 2535 if (!table) 2536 panic("Failed to allocate %s hash table\n", tablename); 2537 2538 printk("%s hash table entries: %d (order: %d, %lu bytes)\n", 2539 tablename, 2540 (1U << log2qty), 2541 long_log2(size) - PAGE_SHIFT, 2542 size); 2543 2544 if (_hash_shift) 2545 *_hash_shift = log2qty; 2546 if (_hash_mask) 2547 *_hash_mask = (1 << log2qty) - 1; 2548 2549 return table; 2550} 2551