vmscan.c revision 49d2e9cc4544369635cd6f4ef6d5bb0f757079a7
1/* 2 * linux/mm/vmscan.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 * 6 * Swap reorganised 29.12.95, Stephen Tweedie. 7 * kswapd added: 7.1.96 sct 8 * Removed kswapd_ctl limits, and swap out as many pages as needed 9 * to bring the system back to freepages.high: 2.4.97, Rik van Riel. 10 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). 11 * Multiqueue VM started 5.8.00, Rik van Riel. 12 */ 13 14#include <linux/mm.h> 15#include <linux/module.h> 16#include <linux/slab.h> 17#include <linux/kernel_stat.h> 18#include <linux/swap.h> 19#include <linux/pagemap.h> 20#include <linux/init.h> 21#include <linux/highmem.h> 22#include <linux/file.h> 23#include <linux/writeback.h> 24#include <linux/blkdev.h> 25#include <linux/buffer_head.h> /* for try_to_release_page(), 26 buffer_heads_over_limit */ 27#include <linux/mm_inline.h> 28#include <linux/pagevec.h> 29#include <linux/backing-dev.h> 30#include <linux/rmap.h> 31#include <linux/topology.h> 32#include <linux/cpu.h> 33#include <linux/cpuset.h> 34#include <linux/notifier.h> 35#include <linux/rwsem.h> 36 37#include <asm/tlbflush.h> 38#include <asm/div64.h> 39 40#include <linux/swapops.h> 41 42/* possible outcome of pageout() */ 43typedef enum { 44 /* failed to write page out, page is locked */ 45 PAGE_KEEP, 46 /* move page to the active list, page is locked */ 47 PAGE_ACTIVATE, 48 /* page has been sent to the disk successfully, page is unlocked */ 49 PAGE_SUCCESS, 50 /* page is clean and locked */ 51 PAGE_CLEAN, 52} pageout_t; 53 54struct scan_control { 55 /* Ask refill_inactive_zone, or shrink_cache to scan this many pages */ 56 unsigned long nr_to_scan; 57 58 /* Incremented by the number of inactive pages that were scanned */ 59 unsigned long nr_scanned; 60 61 /* Incremented by the number of pages reclaimed */ 62 unsigned long nr_reclaimed; 63 64 unsigned long nr_mapped; /* From page_state */ 65 66 /* Ask shrink_caches, or shrink_zone to scan at this priority */ 67 unsigned int priority; 68 69 /* This context's GFP mask */ 70 gfp_t gfp_mask; 71 72 int may_writepage; 73 74 /* This context's SWAP_CLUSTER_MAX. If freeing memory for 75 * suspend, we effectively ignore SWAP_CLUSTER_MAX. 76 * In this context, it doesn't matter that we scan the 77 * whole list at once. */ 78 int swap_cluster_max; 79}; 80 81/* 82 * The list of shrinker callbacks used by to apply pressure to 83 * ageable caches. 84 */ 85struct shrinker { 86 shrinker_t shrinker; 87 struct list_head list; 88 int seeks; /* seeks to recreate an obj */ 89 long nr; /* objs pending delete */ 90}; 91 92#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) 93 94#ifdef ARCH_HAS_PREFETCH 95#define prefetch_prev_lru_page(_page, _base, _field) \ 96 do { \ 97 if ((_page)->lru.prev != _base) { \ 98 struct page *prev; \ 99 \ 100 prev = lru_to_page(&(_page->lru)); \ 101 prefetch(&prev->_field); \ 102 } \ 103 } while (0) 104#else 105#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0) 106#endif 107 108#ifdef ARCH_HAS_PREFETCHW 109#define prefetchw_prev_lru_page(_page, _base, _field) \ 110 do { \ 111 if ((_page)->lru.prev != _base) { \ 112 struct page *prev; \ 113 \ 114 prev = lru_to_page(&(_page->lru)); \ 115 prefetchw(&prev->_field); \ 116 } \ 117 } while (0) 118#else 119#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0) 120#endif 121 122/* 123 * From 0 .. 100. Higher means more swappy. 124 */ 125int vm_swappiness = 60; 126static long total_memory; 127 128static LIST_HEAD(shrinker_list); 129static DECLARE_RWSEM(shrinker_rwsem); 130 131/* 132 * Add a shrinker callback to be called from the vm 133 */ 134struct shrinker *set_shrinker(int seeks, shrinker_t theshrinker) 135{ 136 struct shrinker *shrinker; 137 138 shrinker = kmalloc(sizeof(*shrinker), GFP_KERNEL); 139 if (shrinker) { 140 shrinker->shrinker = theshrinker; 141 shrinker->seeks = seeks; 142 shrinker->nr = 0; 143 down_write(&shrinker_rwsem); 144 list_add_tail(&shrinker->list, &shrinker_list); 145 up_write(&shrinker_rwsem); 146 } 147 return shrinker; 148} 149EXPORT_SYMBOL(set_shrinker); 150 151/* 152 * Remove one 153 */ 154void remove_shrinker(struct shrinker *shrinker) 155{ 156 down_write(&shrinker_rwsem); 157 list_del(&shrinker->list); 158 up_write(&shrinker_rwsem); 159 kfree(shrinker); 160} 161EXPORT_SYMBOL(remove_shrinker); 162 163#define SHRINK_BATCH 128 164/* 165 * Call the shrink functions to age shrinkable caches 166 * 167 * Here we assume it costs one seek to replace a lru page and that it also 168 * takes a seek to recreate a cache object. With this in mind we age equal 169 * percentages of the lru and ageable caches. This should balance the seeks 170 * generated by these structures. 171 * 172 * If the vm encounted mapped pages on the LRU it increase the pressure on 173 * slab to avoid swapping. 174 * 175 * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits. 176 * 177 * `lru_pages' represents the number of on-LRU pages in all the zones which 178 * are eligible for the caller's allocation attempt. It is used for balancing 179 * slab reclaim versus page reclaim. 180 * 181 * Returns the number of slab objects which we shrunk. 182 */ 183int shrink_slab(unsigned long scanned, gfp_t gfp_mask, unsigned long lru_pages) 184{ 185 struct shrinker *shrinker; 186 int ret = 0; 187 188 if (scanned == 0) 189 scanned = SWAP_CLUSTER_MAX; 190 191 if (!down_read_trylock(&shrinker_rwsem)) 192 return 1; /* Assume we'll be able to shrink next time */ 193 194 list_for_each_entry(shrinker, &shrinker_list, list) { 195 unsigned long long delta; 196 unsigned long total_scan; 197 unsigned long max_pass = (*shrinker->shrinker)(0, gfp_mask); 198 199 delta = (4 * scanned) / shrinker->seeks; 200 delta *= max_pass; 201 do_div(delta, lru_pages + 1); 202 shrinker->nr += delta; 203 if (shrinker->nr < 0) { 204 printk(KERN_ERR "%s: nr=%ld\n", 205 __FUNCTION__, shrinker->nr); 206 shrinker->nr = max_pass; 207 } 208 209 /* 210 * Avoid risking looping forever due to too large nr value: 211 * never try to free more than twice the estimate number of 212 * freeable entries. 213 */ 214 if (shrinker->nr > max_pass * 2) 215 shrinker->nr = max_pass * 2; 216 217 total_scan = shrinker->nr; 218 shrinker->nr = 0; 219 220 while (total_scan >= SHRINK_BATCH) { 221 long this_scan = SHRINK_BATCH; 222 int shrink_ret; 223 int nr_before; 224 225 nr_before = (*shrinker->shrinker)(0, gfp_mask); 226 shrink_ret = (*shrinker->shrinker)(this_scan, gfp_mask); 227 if (shrink_ret == -1) 228 break; 229 if (shrink_ret < nr_before) 230 ret += nr_before - shrink_ret; 231 mod_page_state(slabs_scanned, this_scan); 232 total_scan -= this_scan; 233 234 cond_resched(); 235 } 236 237 shrinker->nr += total_scan; 238 } 239 up_read(&shrinker_rwsem); 240 return ret; 241} 242 243/* Called without lock on whether page is mapped, so answer is unstable */ 244static inline int page_mapping_inuse(struct page *page) 245{ 246 struct address_space *mapping; 247 248 /* Page is in somebody's page tables. */ 249 if (page_mapped(page)) 250 return 1; 251 252 /* Be more reluctant to reclaim swapcache than pagecache */ 253 if (PageSwapCache(page)) 254 return 1; 255 256 mapping = page_mapping(page); 257 if (!mapping) 258 return 0; 259 260 /* File is mmap'd by somebody? */ 261 return mapping_mapped(mapping); 262} 263 264static inline int is_page_cache_freeable(struct page *page) 265{ 266 return page_count(page) - !!PagePrivate(page) == 2; 267} 268 269static int may_write_to_queue(struct backing_dev_info *bdi) 270{ 271 if (current->flags & PF_SWAPWRITE) 272 return 1; 273 if (!bdi_write_congested(bdi)) 274 return 1; 275 if (bdi == current->backing_dev_info) 276 return 1; 277 return 0; 278} 279 280/* 281 * We detected a synchronous write error writing a page out. Probably 282 * -ENOSPC. We need to propagate that into the address_space for a subsequent 283 * fsync(), msync() or close(). 284 * 285 * The tricky part is that after writepage we cannot touch the mapping: nothing 286 * prevents it from being freed up. But we have a ref on the page and once 287 * that page is locked, the mapping is pinned. 288 * 289 * We're allowed to run sleeping lock_page() here because we know the caller has 290 * __GFP_FS. 291 */ 292static void handle_write_error(struct address_space *mapping, 293 struct page *page, int error) 294{ 295 lock_page(page); 296 if (page_mapping(page) == mapping) { 297 if (error == -ENOSPC) 298 set_bit(AS_ENOSPC, &mapping->flags); 299 else 300 set_bit(AS_EIO, &mapping->flags); 301 } 302 unlock_page(page); 303} 304 305/* 306 * pageout is called by shrink_list() for each dirty page. Calls ->writepage(). 307 */ 308static pageout_t pageout(struct page *page, struct address_space *mapping) 309{ 310 /* 311 * If the page is dirty, only perform writeback if that write 312 * will be non-blocking. To prevent this allocation from being 313 * stalled by pagecache activity. But note that there may be 314 * stalls if we need to run get_block(). We could test 315 * PagePrivate for that. 316 * 317 * If this process is currently in generic_file_write() against 318 * this page's queue, we can perform writeback even if that 319 * will block. 320 * 321 * If the page is swapcache, write it back even if that would 322 * block, for some throttling. This happens by accident, because 323 * swap_backing_dev_info is bust: it doesn't reflect the 324 * congestion state of the swapdevs. Easy to fix, if needed. 325 * See swapfile.c:page_queue_congested(). 326 */ 327 if (!is_page_cache_freeable(page)) 328 return PAGE_KEEP; 329 if (!mapping) { 330 /* 331 * Some data journaling orphaned pages can have 332 * page->mapping == NULL while being dirty with clean buffers. 333 */ 334 if (PagePrivate(page)) { 335 if (try_to_free_buffers(page)) { 336 ClearPageDirty(page); 337 printk("%s: orphaned page\n", __FUNCTION__); 338 return PAGE_CLEAN; 339 } 340 } 341 return PAGE_KEEP; 342 } 343 if (mapping->a_ops->writepage == NULL) 344 return PAGE_ACTIVATE; 345 if (!may_write_to_queue(mapping->backing_dev_info)) 346 return PAGE_KEEP; 347 348 if (clear_page_dirty_for_io(page)) { 349 int res; 350 struct writeback_control wbc = { 351 .sync_mode = WB_SYNC_NONE, 352 .nr_to_write = SWAP_CLUSTER_MAX, 353 .nonblocking = 1, 354 .for_reclaim = 1, 355 }; 356 357 SetPageReclaim(page); 358 res = mapping->a_ops->writepage(page, &wbc); 359 if (res < 0) 360 handle_write_error(mapping, page, res); 361 if (res == AOP_WRITEPAGE_ACTIVATE) { 362 ClearPageReclaim(page); 363 return PAGE_ACTIVATE; 364 } 365 if (!PageWriteback(page)) { 366 /* synchronous write or broken a_ops? */ 367 ClearPageReclaim(page); 368 } 369 370 return PAGE_SUCCESS; 371 } 372 373 return PAGE_CLEAN; 374} 375 376static int remove_mapping(struct address_space *mapping, struct page *page) 377{ 378 if (!mapping) 379 return 0; /* truncate got there first */ 380 381 write_lock_irq(&mapping->tree_lock); 382 383 /* 384 * The non-racy check for busy page. It is critical to check 385 * PageDirty _after_ making sure that the page is freeable and 386 * not in use by anybody. (pagecache + us == 2) 387 */ 388 if (unlikely(page_count(page) != 2)) 389 goto cannot_free; 390 smp_rmb(); 391 if (unlikely(PageDirty(page))) 392 goto cannot_free; 393 394 if (PageSwapCache(page)) { 395 swp_entry_t swap = { .val = page_private(page) }; 396 __delete_from_swap_cache(page); 397 write_unlock_irq(&mapping->tree_lock); 398 swap_free(swap); 399 __put_page(page); /* The pagecache ref */ 400 return 1; 401 } 402 403 __remove_from_page_cache(page); 404 write_unlock_irq(&mapping->tree_lock); 405 __put_page(page); 406 return 1; 407 408cannot_free: 409 write_unlock_irq(&mapping->tree_lock); 410 return 0; 411} 412 413/* 414 * shrink_list adds the number of reclaimed pages to sc->nr_reclaimed 415 */ 416static int shrink_list(struct list_head *page_list, struct scan_control *sc) 417{ 418 LIST_HEAD(ret_pages); 419 struct pagevec freed_pvec; 420 int pgactivate = 0; 421 int reclaimed = 0; 422 423 cond_resched(); 424 425 pagevec_init(&freed_pvec, 1); 426 while (!list_empty(page_list)) { 427 struct address_space *mapping; 428 struct page *page; 429 int may_enter_fs; 430 int referenced; 431 432 cond_resched(); 433 434 page = lru_to_page(page_list); 435 list_del(&page->lru); 436 437 if (TestSetPageLocked(page)) 438 goto keep; 439 440 BUG_ON(PageActive(page)); 441 442 sc->nr_scanned++; 443 /* Double the slab pressure for mapped and swapcache pages */ 444 if (page_mapped(page) || PageSwapCache(page)) 445 sc->nr_scanned++; 446 447 if (PageWriteback(page)) 448 goto keep_locked; 449 450 referenced = page_referenced(page, 1); 451 /* In active use or really unfreeable? Activate it. */ 452 if (referenced && page_mapping_inuse(page)) 453 goto activate_locked; 454 455#ifdef CONFIG_SWAP 456 /* 457 * Anonymous process memory has backing store? 458 * Try to allocate it some swap space here. 459 */ 460 if (PageAnon(page) && !PageSwapCache(page)) { 461 if (!add_to_swap(page)) 462 goto activate_locked; 463 } 464#endif /* CONFIG_SWAP */ 465 466 mapping = page_mapping(page); 467 may_enter_fs = (sc->gfp_mask & __GFP_FS) || 468 (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO)); 469 470 /* 471 * The page is mapped into the page tables of one or more 472 * processes. Try to unmap it here. 473 */ 474 if (page_mapped(page) && mapping) { 475 switch (try_to_unmap(page)) { 476 case SWAP_FAIL: 477 goto activate_locked; 478 case SWAP_AGAIN: 479 goto keep_locked; 480 case SWAP_SUCCESS: 481 ; /* try to free the page below */ 482 } 483 } 484 485 if (PageDirty(page)) { 486 if (referenced) 487 goto keep_locked; 488 if (!may_enter_fs) 489 goto keep_locked; 490 if (laptop_mode && !sc->may_writepage) 491 goto keep_locked; 492 493 /* Page is dirty, try to write it out here */ 494 switch(pageout(page, mapping)) { 495 case PAGE_KEEP: 496 goto keep_locked; 497 case PAGE_ACTIVATE: 498 goto activate_locked; 499 case PAGE_SUCCESS: 500 if (PageWriteback(page) || PageDirty(page)) 501 goto keep; 502 /* 503 * A synchronous write - probably a ramdisk. Go 504 * ahead and try to reclaim the page. 505 */ 506 if (TestSetPageLocked(page)) 507 goto keep; 508 if (PageDirty(page) || PageWriteback(page)) 509 goto keep_locked; 510 mapping = page_mapping(page); 511 case PAGE_CLEAN: 512 ; /* try to free the page below */ 513 } 514 } 515 516 /* 517 * If the page has buffers, try to free the buffer mappings 518 * associated with this page. If we succeed we try to free 519 * the page as well. 520 * 521 * We do this even if the page is PageDirty(). 522 * try_to_release_page() does not perform I/O, but it is 523 * possible for a page to have PageDirty set, but it is actually 524 * clean (all its buffers are clean). This happens if the 525 * buffers were written out directly, with submit_bh(). ext3 526 * will do this, as well as the blockdev mapping. 527 * try_to_release_page() will discover that cleanness and will 528 * drop the buffers and mark the page clean - it can be freed. 529 * 530 * Rarely, pages can have buffers and no ->mapping. These are 531 * the pages which were not successfully invalidated in 532 * truncate_complete_page(). We try to drop those buffers here 533 * and if that worked, and the page is no longer mapped into 534 * process address space (page_count == 1) it can be freed. 535 * Otherwise, leave the page on the LRU so it is swappable. 536 */ 537 if (PagePrivate(page)) { 538 if (!try_to_release_page(page, sc->gfp_mask)) 539 goto activate_locked; 540 if (!mapping && page_count(page) == 1) 541 goto free_it; 542 } 543 544 if (!remove_mapping(mapping, page)) 545 goto keep_locked; 546 547free_it: 548 unlock_page(page); 549 reclaimed++; 550 if (!pagevec_add(&freed_pvec, page)) 551 __pagevec_release_nonlru(&freed_pvec); 552 continue; 553 554activate_locked: 555 SetPageActive(page); 556 pgactivate++; 557keep_locked: 558 unlock_page(page); 559keep: 560 list_add(&page->lru, &ret_pages); 561 BUG_ON(PageLRU(page)); 562 } 563 list_splice(&ret_pages, page_list); 564 if (pagevec_count(&freed_pvec)) 565 __pagevec_release_nonlru(&freed_pvec); 566 mod_page_state(pgactivate, pgactivate); 567 sc->nr_reclaimed += reclaimed; 568 return reclaimed; 569} 570 571/* 572 * swapout a single page 573 * page is locked upon entry, unlocked on exit 574 * 575 * return codes: 576 * 0 = complete 577 * 1 = retry 578 */ 579static int swap_page(struct page *page) 580{ 581 struct address_space *mapping = page_mapping(page); 582 583 if (page_mapped(page) && mapping) 584 if (try_to_unmap(page) != SWAP_SUCCESS) 585 goto unlock_retry; 586 587 if (PageDirty(page)) { 588 /* Page is dirty, try to write it out here */ 589 switch(pageout(page, mapping)) { 590 case PAGE_KEEP: 591 case PAGE_ACTIVATE: 592 goto unlock_retry; 593 594 case PAGE_SUCCESS: 595 goto retry; 596 597 case PAGE_CLEAN: 598 ; /* try to free the page below */ 599 } 600 } 601 602 if (PagePrivate(page)) { 603 if (!try_to_release_page(page, GFP_KERNEL) || 604 (!mapping && page_count(page) == 1)) 605 goto unlock_retry; 606 } 607 608 if (remove_mapping(mapping, page)) { 609 /* Success */ 610 unlock_page(page); 611 return 0; 612 } 613 614unlock_retry: 615 unlock_page(page); 616 617retry: 618 return 1; 619} 620/* 621 * migrate_pages 622 * 623 * Two lists are passed to this function. The first list 624 * contains the pages isolated from the LRU to be migrated. 625 * The second list contains new pages that the pages isolated 626 * can be moved to. If the second list is NULL then all 627 * pages are swapped out. 628 * 629 * The function returns after 10 attempts or if no pages 630 * are movable anymore because t has become empty 631 * or no retryable pages exist anymore. 632 * 633 * SIMPLIFIED VERSION: This implementation of migrate_pages 634 * is only swapping out pages and never touches the second 635 * list. The direct migration patchset 636 * extends this function to avoid the use of swap. 637 */ 638int migrate_pages(struct list_head *l, struct list_head *t) 639{ 640 int retry; 641 LIST_HEAD(failed); 642 int nr_failed = 0; 643 int pass = 0; 644 struct page *page; 645 struct page *page2; 646 int swapwrite = current->flags & PF_SWAPWRITE; 647 648 if (!swapwrite) 649 current->flags |= PF_SWAPWRITE; 650 651redo: 652 retry = 0; 653 654 list_for_each_entry_safe(page, page2, l, lru) { 655 cond_resched(); 656 657 /* 658 * Skip locked pages during the first two passes to give the 659 * functions holding the lock time to release the page. Later we use 660 * lock_page to have a higher chance of acquiring the lock. 661 */ 662 if (pass > 2) 663 lock_page(page); 664 else 665 if (TestSetPageLocked(page)) 666 goto retry_later; 667 668 /* 669 * Only wait on writeback if we have already done a pass where 670 * we we may have triggered writeouts for lots of pages. 671 */ 672 if (pass > 0) 673 wait_on_page_writeback(page); 674 else 675 if (PageWriteback(page)) { 676 unlock_page(page); 677 goto retry_later; 678 } 679 680#ifdef CONFIG_SWAP 681 if (PageAnon(page) && !PageSwapCache(page)) { 682 if (!add_to_swap(page)) { 683 unlock_page(page); 684 list_move(&page->lru, &failed); 685 nr_failed++; 686 continue; 687 } 688 } 689#endif /* CONFIG_SWAP */ 690 691 /* 692 * Page is properly locked and writeback is complete. 693 * Try to migrate the page. 694 */ 695 if (swap_page(page)) { 696retry_later: 697 retry++; 698 } 699 } 700 if (retry && pass++ < 10) 701 goto redo; 702 703 if (!swapwrite) 704 current->flags &= ~PF_SWAPWRITE; 705 706 if (!list_empty(&failed)) 707 list_splice(&failed, l); 708 709 return nr_failed + retry; 710} 711 712/* 713 * zone->lru_lock is heavily contended. Some of the functions that 714 * shrink the lists perform better by taking out a batch of pages 715 * and working on them outside the LRU lock. 716 * 717 * For pagecache intensive workloads, this function is the hottest 718 * spot in the kernel (apart from copy_*_user functions). 719 * 720 * Appropriate locks must be held before calling this function. 721 * 722 * @nr_to_scan: The number of pages to look through on the list. 723 * @src: The LRU list to pull pages off. 724 * @dst: The temp list to put pages on to. 725 * @scanned: The number of pages that were scanned. 726 * 727 * returns how many pages were moved onto *@dst. 728 */ 729static int isolate_lru_pages(int nr_to_scan, struct list_head *src, 730 struct list_head *dst, int *scanned) 731{ 732 int nr_taken = 0; 733 struct page *page; 734 int scan = 0; 735 736 while (scan++ < nr_to_scan && !list_empty(src)) { 737 page = lru_to_page(src); 738 prefetchw_prev_lru_page(page, src, flags); 739 740 switch (__isolate_lru_page(page)) { 741 case 1: 742 /* Succeeded to isolate page */ 743 list_move(&page->lru, dst); 744 nr_taken++; 745 break; 746 case -ENOENT: 747 /* Not possible to isolate */ 748 list_move(&page->lru, src); 749 break; 750 default: 751 BUG(); 752 } 753 } 754 755 *scanned = scan; 756 return nr_taken; 757} 758 759static void lru_add_drain_per_cpu(void *dummy) 760{ 761 lru_add_drain(); 762} 763 764/* 765 * Isolate one page from the LRU lists and put it on the 766 * indicated list. Do necessary cache draining if the 767 * page is not on the LRU lists yet. 768 * 769 * Result: 770 * 0 = page not on LRU list 771 * 1 = page removed from LRU list and added to the specified list. 772 * -ENOENT = page is being freed elsewhere. 773 */ 774int isolate_lru_page(struct page *page) 775{ 776 int rc = 0; 777 struct zone *zone = page_zone(page); 778 779redo: 780 spin_lock_irq(&zone->lru_lock); 781 rc = __isolate_lru_page(page); 782 if (rc == 1) { 783 if (PageActive(page)) 784 del_page_from_active_list(zone, page); 785 else 786 del_page_from_inactive_list(zone, page); 787 } 788 spin_unlock_irq(&zone->lru_lock); 789 if (rc == 0) { 790 /* 791 * Maybe this page is still waiting for a cpu to drain it 792 * from one of the lru lists? 793 */ 794 rc = schedule_on_each_cpu(lru_add_drain_per_cpu, NULL); 795 if (rc == 0 && PageLRU(page)) 796 goto redo; 797 } 798 return rc; 799} 800 801/* 802 * shrink_cache() adds the number of pages reclaimed to sc->nr_reclaimed 803 */ 804static void shrink_cache(struct zone *zone, struct scan_control *sc) 805{ 806 LIST_HEAD(page_list); 807 struct pagevec pvec; 808 int max_scan = sc->nr_to_scan; 809 810 pagevec_init(&pvec, 1); 811 812 lru_add_drain(); 813 spin_lock_irq(&zone->lru_lock); 814 while (max_scan > 0) { 815 struct page *page; 816 int nr_taken; 817 int nr_scan; 818 int nr_freed; 819 820 nr_taken = isolate_lru_pages(sc->swap_cluster_max, 821 &zone->inactive_list, 822 &page_list, &nr_scan); 823 zone->nr_inactive -= nr_taken; 824 zone->pages_scanned += nr_scan; 825 spin_unlock_irq(&zone->lru_lock); 826 827 if (nr_taken == 0) 828 goto done; 829 830 max_scan -= nr_scan; 831 nr_freed = shrink_list(&page_list, sc); 832 833 local_irq_disable(); 834 if (current_is_kswapd()) { 835 __mod_page_state_zone(zone, pgscan_kswapd, nr_scan); 836 __mod_page_state(kswapd_steal, nr_freed); 837 } else 838 __mod_page_state_zone(zone, pgscan_direct, nr_scan); 839 __mod_page_state_zone(zone, pgsteal, nr_freed); 840 841 spin_lock(&zone->lru_lock); 842 /* 843 * Put back any unfreeable pages. 844 */ 845 while (!list_empty(&page_list)) { 846 page = lru_to_page(&page_list); 847 if (TestSetPageLRU(page)) 848 BUG(); 849 list_del(&page->lru); 850 if (PageActive(page)) 851 add_page_to_active_list(zone, page); 852 else 853 add_page_to_inactive_list(zone, page); 854 if (!pagevec_add(&pvec, page)) { 855 spin_unlock_irq(&zone->lru_lock); 856 __pagevec_release(&pvec); 857 spin_lock_irq(&zone->lru_lock); 858 } 859 } 860 } 861 spin_unlock_irq(&zone->lru_lock); 862done: 863 pagevec_release(&pvec); 864} 865 866static inline void move_to_lru(struct page *page) 867{ 868 list_del(&page->lru); 869 if (PageActive(page)) { 870 /* 871 * lru_cache_add_active checks that 872 * the PG_active bit is off. 873 */ 874 ClearPageActive(page); 875 lru_cache_add_active(page); 876 } else { 877 lru_cache_add(page); 878 } 879 put_page(page); 880} 881 882/* 883 * Add isolated pages on the list back to the LRU 884 * 885 * returns the number of pages put back. 886 */ 887int putback_lru_pages(struct list_head *l) 888{ 889 struct page *page; 890 struct page *page2; 891 int count = 0; 892 893 list_for_each_entry_safe(page, page2, l, lru) { 894 move_to_lru(page); 895 count++; 896 } 897 return count; 898} 899 900/* 901 * This moves pages from the active list to the inactive list. 902 * 903 * We move them the other way if the page is referenced by one or more 904 * processes, from rmap. 905 * 906 * If the pages are mostly unmapped, the processing is fast and it is 907 * appropriate to hold zone->lru_lock across the whole operation. But if 908 * the pages are mapped, the processing is slow (page_referenced()) so we 909 * should drop zone->lru_lock around each page. It's impossible to balance 910 * this, so instead we remove the pages from the LRU while processing them. 911 * It is safe to rely on PG_active against the non-LRU pages in here because 912 * nobody will play with that bit on a non-LRU page. 913 * 914 * The downside is that we have to touch page->_count against each page. 915 * But we had to alter page->flags anyway. 916 */ 917static void 918refill_inactive_zone(struct zone *zone, struct scan_control *sc) 919{ 920 int pgmoved; 921 int pgdeactivate = 0; 922 int pgscanned; 923 int nr_pages = sc->nr_to_scan; 924 LIST_HEAD(l_hold); /* The pages which were snipped off */ 925 LIST_HEAD(l_inactive); /* Pages to go onto the inactive_list */ 926 LIST_HEAD(l_active); /* Pages to go onto the active_list */ 927 struct page *page; 928 struct pagevec pvec; 929 int reclaim_mapped = 0; 930 long mapped_ratio; 931 long distress; 932 long swap_tendency; 933 934 lru_add_drain(); 935 spin_lock_irq(&zone->lru_lock); 936 pgmoved = isolate_lru_pages(nr_pages, &zone->active_list, 937 &l_hold, &pgscanned); 938 zone->pages_scanned += pgscanned; 939 zone->nr_active -= pgmoved; 940 spin_unlock_irq(&zone->lru_lock); 941 942 /* 943 * `distress' is a measure of how much trouble we're having reclaiming 944 * pages. 0 -> no problems. 100 -> great trouble. 945 */ 946 distress = 100 >> zone->prev_priority; 947 948 /* 949 * The point of this algorithm is to decide when to start reclaiming 950 * mapped memory instead of just pagecache. Work out how much memory 951 * is mapped. 952 */ 953 mapped_ratio = (sc->nr_mapped * 100) / total_memory; 954 955 /* 956 * Now decide how much we really want to unmap some pages. The mapped 957 * ratio is downgraded - just because there's a lot of mapped memory 958 * doesn't necessarily mean that page reclaim isn't succeeding. 959 * 960 * The distress ratio is important - we don't want to start going oom. 961 * 962 * A 100% value of vm_swappiness overrides this algorithm altogether. 963 */ 964 swap_tendency = mapped_ratio / 2 + distress + vm_swappiness; 965 966 /* 967 * Now use this metric to decide whether to start moving mapped memory 968 * onto the inactive list. 969 */ 970 if (swap_tendency >= 100) 971 reclaim_mapped = 1; 972 973 while (!list_empty(&l_hold)) { 974 cond_resched(); 975 page = lru_to_page(&l_hold); 976 list_del(&page->lru); 977 if (page_mapped(page)) { 978 if (!reclaim_mapped || 979 (total_swap_pages == 0 && PageAnon(page)) || 980 page_referenced(page, 0)) { 981 list_add(&page->lru, &l_active); 982 continue; 983 } 984 } 985 list_add(&page->lru, &l_inactive); 986 } 987 988 pagevec_init(&pvec, 1); 989 pgmoved = 0; 990 spin_lock_irq(&zone->lru_lock); 991 while (!list_empty(&l_inactive)) { 992 page = lru_to_page(&l_inactive); 993 prefetchw_prev_lru_page(page, &l_inactive, flags); 994 if (TestSetPageLRU(page)) 995 BUG(); 996 if (!TestClearPageActive(page)) 997 BUG(); 998 list_move(&page->lru, &zone->inactive_list); 999 pgmoved++; 1000 if (!pagevec_add(&pvec, page)) { 1001 zone->nr_inactive += pgmoved; 1002 spin_unlock_irq(&zone->lru_lock); 1003 pgdeactivate += pgmoved; 1004 pgmoved = 0; 1005 if (buffer_heads_over_limit) 1006 pagevec_strip(&pvec); 1007 __pagevec_release(&pvec); 1008 spin_lock_irq(&zone->lru_lock); 1009 } 1010 } 1011 zone->nr_inactive += pgmoved; 1012 pgdeactivate += pgmoved; 1013 if (buffer_heads_over_limit) { 1014 spin_unlock_irq(&zone->lru_lock); 1015 pagevec_strip(&pvec); 1016 spin_lock_irq(&zone->lru_lock); 1017 } 1018 1019 pgmoved = 0; 1020 while (!list_empty(&l_active)) { 1021 page = lru_to_page(&l_active); 1022 prefetchw_prev_lru_page(page, &l_active, flags); 1023 if (TestSetPageLRU(page)) 1024 BUG(); 1025 BUG_ON(!PageActive(page)); 1026 list_move(&page->lru, &zone->active_list); 1027 pgmoved++; 1028 if (!pagevec_add(&pvec, page)) { 1029 zone->nr_active += pgmoved; 1030 pgmoved = 0; 1031 spin_unlock_irq(&zone->lru_lock); 1032 __pagevec_release(&pvec); 1033 spin_lock_irq(&zone->lru_lock); 1034 } 1035 } 1036 zone->nr_active += pgmoved; 1037 spin_unlock(&zone->lru_lock); 1038 1039 __mod_page_state_zone(zone, pgrefill, pgscanned); 1040 __mod_page_state(pgdeactivate, pgdeactivate); 1041 local_irq_enable(); 1042 1043 pagevec_release(&pvec); 1044} 1045 1046/* 1047 * This is a basic per-zone page freer. Used by both kswapd and direct reclaim. 1048 */ 1049static void 1050shrink_zone(struct zone *zone, struct scan_control *sc) 1051{ 1052 unsigned long nr_active; 1053 unsigned long nr_inactive; 1054 1055 atomic_inc(&zone->reclaim_in_progress); 1056 1057 /* 1058 * Add one to `nr_to_scan' just to make sure that the kernel will 1059 * slowly sift through the active list. 1060 */ 1061 zone->nr_scan_active += (zone->nr_active >> sc->priority) + 1; 1062 nr_active = zone->nr_scan_active; 1063 if (nr_active >= sc->swap_cluster_max) 1064 zone->nr_scan_active = 0; 1065 else 1066 nr_active = 0; 1067 1068 zone->nr_scan_inactive += (zone->nr_inactive >> sc->priority) + 1; 1069 nr_inactive = zone->nr_scan_inactive; 1070 if (nr_inactive >= sc->swap_cluster_max) 1071 zone->nr_scan_inactive = 0; 1072 else 1073 nr_inactive = 0; 1074 1075 while (nr_active || nr_inactive) { 1076 if (nr_active) { 1077 sc->nr_to_scan = min(nr_active, 1078 (unsigned long)sc->swap_cluster_max); 1079 nr_active -= sc->nr_to_scan; 1080 refill_inactive_zone(zone, sc); 1081 } 1082 1083 if (nr_inactive) { 1084 sc->nr_to_scan = min(nr_inactive, 1085 (unsigned long)sc->swap_cluster_max); 1086 nr_inactive -= sc->nr_to_scan; 1087 shrink_cache(zone, sc); 1088 } 1089 } 1090 1091 throttle_vm_writeout(); 1092 1093 atomic_dec(&zone->reclaim_in_progress); 1094} 1095 1096/* 1097 * This is the direct reclaim path, for page-allocating processes. We only 1098 * try to reclaim pages from zones which will satisfy the caller's allocation 1099 * request. 1100 * 1101 * We reclaim from a zone even if that zone is over pages_high. Because: 1102 * a) The caller may be trying to free *extra* pages to satisfy a higher-order 1103 * allocation or 1104 * b) The zones may be over pages_high but they must go *over* pages_high to 1105 * satisfy the `incremental min' zone defense algorithm. 1106 * 1107 * Returns the number of reclaimed pages. 1108 * 1109 * If a zone is deemed to be full of pinned pages then just give it a light 1110 * scan then give up on it. 1111 */ 1112static void 1113shrink_caches(struct zone **zones, struct scan_control *sc) 1114{ 1115 int i; 1116 1117 for (i = 0; zones[i] != NULL; i++) { 1118 struct zone *zone = zones[i]; 1119 1120 if (!populated_zone(zone)) 1121 continue; 1122 1123 if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) 1124 continue; 1125 1126 zone->temp_priority = sc->priority; 1127 if (zone->prev_priority > sc->priority) 1128 zone->prev_priority = sc->priority; 1129 1130 if (zone->all_unreclaimable && sc->priority != DEF_PRIORITY) 1131 continue; /* Let kswapd poll it */ 1132 1133 shrink_zone(zone, sc); 1134 } 1135} 1136 1137/* 1138 * This is the main entry point to direct page reclaim. 1139 * 1140 * If a full scan of the inactive list fails to free enough memory then we 1141 * are "out of memory" and something needs to be killed. 1142 * 1143 * If the caller is !__GFP_FS then the probability of a failure is reasonably 1144 * high - the zone may be full of dirty or under-writeback pages, which this 1145 * caller can't do much about. We kick pdflush and take explicit naps in the 1146 * hope that some of these pages can be written. But if the allocating task 1147 * holds filesystem locks which prevent writeout this might not work, and the 1148 * allocation attempt will fail. 1149 */ 1150int try_to_free_pages(struct zone **zones, gfp_t gfp_mask) 1151{ 1152 int priority; 1153 int ret = 0; 1154 int total_scanned = 0, total_reclaimed = 0; 1155 struct reclaim_state *reclaim_state = current->reclaim_state; 1156 struct scan_control sc; 1157 unsigned long lru_pages = 0; 1158 int i; 1159 1160 sc.gfp_mask = gfp_mask; 1161 sc.may_writepage = 0; 1162 1163 inc_page_state(allocstall); 1164 1165 for (i = 0; zones[i] != NULL; i++) { 1166 struct zone *zone = zones[i]; 1167 1168 if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) 1169 continue; 1170 1171 zone->temp_priority = DEF_PRIORITY; 1172 lru_pages += zone->nr_active + zone->nr_inactive; 1173 } 1174 1175 for (priority = DEF_PRIORITY; priority >= 0; priority--) { 1176 sc.nr_mapped = read_page_state(nr_mapped); 1177 sc.nr_scanned = 0; 1178 sc.nr_reclaimed = 0; 1179 sc.priority = priority; 1180 sc.swap_cluster_max = SWAP_CLUSTER_MAX; 1181 if (!priority) 1182 disable_swap_token(); 1183 shrink_caches(zones, &sc); 1184 shrink_slab(sc.nr_scanned, gfp_mask, lru_pages); 1185 if (reclaim_state) { 1186 sc.nr_reclaimed += reclaim_state->reclaimed_slab; 1187 reclaim_state->reclaimed_slab = 0; 1188 } 1189 total_scanned += sc.nr_scanned; 1190 total_reclaimed += sc.nr_reclaimed; 1191 if (total_reclaimed >= sc.swap_cluster_max) { 1192 ret = 1; 1193 goto out; 1194 } 1195 1196 /* 1197 * Try to write back as many pages as we just scanned. This 1198 * tends to cause slow streaming writers to write data to the 1199 * disk smoothly, at the dirtying rate, which is nice. But 1200 * that's undesirable in laptop mode, where we *want* lumpy 1201 * writeout. So in laptop mode, write out the whole world. 1202 */ 1203 if (total_scanned > sc.swap_cluster_max + sc.swap_cluster_max/2) { 1204 wakeup_pdflush(laptop_mode ? 0 : total_scanned); 1205 sc.may_writepage = 1; 1206 } 1207 1208 /* Take a nap, wait for some writeback to complete */ 1209 if (sc.nr_scanned && priority < DEF_PRIORITY - 2) 1210 blk_congestion_wait(WRITE, HZ/10); 1211 } 1212out: 1213 for (i = 0; zones[i] != 0; i++) { 1214 struct zone *zone = zones[i]; 1215 1216 if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) 1217 continue; 1218 1219 zone->prev_priority = zone->temp_priority; 1220 } 1221 return ret; 1222} 1223 1224/* 1225 * For kswapd, balance_pgdat() will work across all this node's zones until 1226 * they are all at pages_high. 1227 * 1228 * If `nr_pages' is non-zero then it is the number of pages which are to be 1229 * reclaimed, regardless of the zone occupancies. This is a software suspend 1230 * special. 1231 * 1232 * Returns the number of pages which were actually freed. 1233 * 1234 * There is special handling here for zones which are full of pinned pages. 1235 * This can happen if the pages are all mlocked, or if they are all used by 1236 * device drivers (say, ZONE_DMA). Or if they are all in use by hugetlb. 1237 * What we do is to detect the case where all pages in the zone have been 1238 * scanned twice and there has been zero successful reclaim. Mark the zone as 1239 * dead and from now on, only perform a short scan. Basically we're polling 1240 * the zone for when the problem goes away. 1241 * 1242 * kswapd scans the zones in the highmem->normal->dma direction. It skips 1243 * zones which have free_pages > pages_high, but once a zone is found to have 1244 * free_pages <= pages_high, we scan that zone and the lower zones regardless 1245 * of the number of free pages in the lower zones. This interoperates with 1246 * the page allocator fallback scheme to ensure that aging of pages is balanced 1247 * across the zones. 1248 */ 1249static int balance_pgdat(pg_data_t *pgdat, int nr_pages, int order) 1250{ 1251 int to_free = nr_pages; 1252 int all_zones_ok; 1253 int priority; 1254 int i; 1255 int total_scanned, total_reclaimed; 1256 struct reclaim_state *reclaim_state = current->reclaim_state; 1257 struct scan_control sc; 1258 1259loop_again: 1260 total_scanned = 0; 1261 total_reclaimed = 0; 1262 sc.gfp_mask = GFP_KERNEL; 1263 sc.may_writepage = 0; 1264 sc.nr_mapped = read_page_state(nr_mapped); 1265 1266 inc_page_state(pageoutrun); 1267 1268 for (i = 0; i < pgdat->nr_zones; i++) { 1269 struct zone *zone = pgdat->node_zones + i; 1270 1271 zone->temp_priority = DEF_PRIORITY; 1272 } 1273 1274 for (priority = DEF_PRIORITY; priority >= 0; priority--) { 1275 int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */ 1276 unsigned long lru_pages = 0; 1277 1278 /* The swap token gets in the way of swapout... */ 1279 if (!priority) 1280 disable_swap_token(); 1281 1282 all_zones_ok = 1; 1283 1284 if (nr_pages == 0) { 1285 /* 1286 * Scan in the highmem->dma direction for the highest 1287 * zone which needs scanning 1288 */ 1289 for (i = pgdat->nr_zones - 1; i >= 0; i--) { 1290 struct zone *zone = pgdat->node_zones + i; 1291 1292 if (!populated_zone(zone)) 1293 continue; 1294 1295 if (zone->all_unreclaimable && 1296 priority != DEF_PRIORITY) 1297 continue; 1298 1299 if (!zone_watermark_ok(zone, order, 1300 zone->pages_high, 0, 0)) { 1301 end_zone = i; 1302 goto scan; 1303 } 1304 } 1305 goto out; 1306 } else { 1307 end_zone = pgdat->nr_zones - 1; 1308 } 1309scan: 1310 for (i = 0; i <= end_zone; i++) { 1311 struct zone *zone = pgdat->node_zones + i; 1312 1313 lru_pages += zone->nr_active + zone->nr_inactive; 1314 } 1315 1316 /* 1317 * Now scan the zone in the dma->highmem direction, stopping 1318 * at the last zone which needs scanning. 1319 * 1320 * We do this because the page allocator works in the opposite 1321 * direction. This prevents the page allocator from allocating 1322 * pages behind kswapd's direction of progress, which would 1323 * cause too much scanning of the lower zones. 1324 */ 1325 for (i = 0; i <= end_zone; i++) { 1326 struct zone *zone = pgdat->node_zones + i; 1327 int nr_slab; 1328 1329 if (!populated_zone(zone)) 1330 continue; 1331 1332 if (zone->all_unreclaimable && priority != DEF_PRIORITY) 1333 continue; 1334 1335 if (nr_pages == 0) { /* Not software suspend */ 1336 if (!zone_watermark_ok(zone, order, 1337 zone->pages_high, end_zone, 0)) 1338 all_zones_ok = 0; 1339 } 1340 zone->temp_priority = priority; 1341 if (zone->prev_priority > priority) 1342 zone->prev_priority = priority; 1343 sc.nr_scanned = 0; 1344 sc.nr_reclaimed = 0; 1345 sc.priority = priority; 1346 sc.swap_cluster_max = nr_pages? nr_pages : SWAP_CLUSTER_MAX; 1347 atomic_inc(&zone->reclaim_in_progress); 1348 shrink_zone(zone, &sc); 1349 atomic_dec(&zone->reclaim_in_progress); 1350 reclaim_state->reclaimed_slab = 0; 1351 nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL, 1352 lru_pages); 1353 sc.nr_reclaimed += reclaim_state->reclaimed_slab; 1354 total_reclaimed += sc.nr_reclaimed; 1355 total_scanned += sc.nr_scanned; 1356 if (zone->all_unreclaimable) 1357 continue; 1358 if (nr_slab == 0 && zone->pages_scanned >= 1359 (zone->nr_active + zone->nr_inactive) * 4) 1360 zone->all_unreclaimable = 1; 1361 /* 1362 * If we've done a decent amount of scanning and 1363 * the reclaim ratio is low, start doing writepage 1364 * even in laptop mode 1365 */ 1366 if (total_scanned > SWAP_CLUSTER_MAX * 2 && 1367 total_scanned > total_reclaimed+total_reclaimed/2) 1368 sc.may_writepage = 1; 1369 } 1370 if (nr_pages && to_free > total_reclaimed) 1371 continue; /* swsusp: need to do more work */ 1372 if (all_zones_ok) 1373 break; /* kswapd: all done */ 1374 /* 1375 * OK, kswapd is getting into trouble. Take a nap, then take 1376 * another pass across the zones. 1377 */ 1378 if (total_scanned && priority < DEF_PRIORITY - 2) 1379 blk_congestion_wait(WRITE, HZ/10); 1380 1381 /* 1382 * We do this so kswapd doesn't build up large priorities for 1383 * example when it is freeing in parallel with allocators. It 1384 * matches the direct reclaim path behaviour in terms of impact 1385 * on zone->*_priority. 1386 */ 1387 if ((total_reclaimed >= SWAP_CLUSTER_MAX) && (!nr_pages)) 1388 break; 1389 } 1390out: 1391 for (i = 0; i < pgdat->nr_zones; i++) { 1392 struct zone *zone = pgdat->node_zones + i; 1393 1394 zone->prev_priority = zone->temp_priority; 1395 } 1396 if (!all_zones_ok) { 1397 cond_resched(); 1398 goto loop_again; 1399 } 1400 1401 return total_reclaimed; 1402} 1403 1404/* 1405 * The background pageout daemon, started as a kernel thread 1406 * from the init process. 1407 * 1408 * This basically trickles out pages so that we have _some_ 1409 * free memory available even if there is no other activity 1410 * that frees anything up. This is needed for things like routing 1411 * etc, where we otherwise might have all activity going on in 1412 * asynchronous contexts that cannot page things out. 1413 * 1414 * If there are applications that are active memory-allocators 1415 * (most normal use), this basically shouldn't matter. 1416 */ 1417static int kswapd(void *p) 1418{ 1419 unsigned long order; 1420 pg_data_t *pgdat = (pg_data_t*)p; 1421 struct task_struct *tsk = current; 1422 DEFINE_WAIT(wait); 1423 struct reclaim_state reclaim_state = { 1424 .reclaimed_slab = 0, 1425 }; 1426 cpumask_t cpumask; 1427 1428 daemonize("kswapd%d", pgdat->node_id); 1429 cpumask = node_to_cpumask(pgdat->node_id); 1430 if (!cpus_empty(cpumask)) 1431 set_cpus_allowed(tsk, cpumask); 1432 current->reclaim_state = &reclaim_state; 1433 1434 /* 1435 * Tell the memory management that we're a "memory allocator", 1436 * and that if we need more memory we should get access to it 1437 * regardless (see "__alloc_pages()"). "kswapd" should 1438 * never get caught in the normal page freeing logic. 1439 * 1440 * (Kswapd normally doesn't need memory anyway, but sometimes 1441 * you need a small amount of memory in order to be able to 1442 * page out something else, and this flag essentially protects 1443 * us from recursively trying to free more memory as we're 1444 * trying to free the first piece of memory in the first place). 1445 */ 1446 tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD; 1447 1448 order = 0; 1449 for ( ; ; ) { 1450 unsigned long new_order; 1451 1452 try_to_freeze(); 1453 1454 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 1455 new_order = pgdat->kswapd_max_order; 1456 pgdat->kswapd_max_order = 0; 1457 if (order < new_order) { 1458 /* 1459 * Don't sleep if someone wants a larger 'order' 1460 * allocation 1461 */ 1462 order = new_order; 1463 } else { 1464 schedule(); 1465 order = pgdat->kswapd_max_order; 1466 } 1467 finish_wait(&pgdat->kswapd_wait, &wait); 1468 1469 balance_pgdat(pgdat, 0, order); 1470 } 1471 return 0; 1472} 1473 1474/* 1475 * A zone is low on free memory, so wake its kswapd task to service it. 1476 */ 1477void wakeup_kswapd(struct zone *zone, int order) 1478{ 1479 pg_data_t *pgdat; 1480 1481 if (!populated_zone(zone)) 1482 return; 1483 1484 pgdat = zone->zone_pgdat; 1485 if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0)) 1486 return; 1487 if (pgdat->kswapd_max_order < order) 1488 pgdat->kswapd_max_order = order; 1489 if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) 1490 return; 1491 if (!waitqueue_active(&pgdat->kswapd_wait)) 1492 return; 1493 wake_up_interruptible(&pgdat->kswapd_wait); 1494} 1495 1496#ifdef CONFIG_PM 1497/* 1498 * Try to free `nr_pages' of memory, system-wide. Returns the number of freed 1499 * pages. 1500 */ 1501int shrink_all_memory(int nr_pages) 1502{ 1503 pg_data_t *pgdat; 1504 int nr_to_free = nr_pages; 1505 int ret = 0; 1506 struct reclaim_state reclaim_state = { 1507 .reclaimed_slab = 0, 1508 }; 1509 1510 current->reclaim_state = &reclaim_state; 1511 for_each_pgdat(pgdat) { 1512 int freed; 1513 freed = balance_pgdat(pgdat, nr_to_free, 0); 1514 ret += freed; 1515 nr_to_free -= freed; 1516 if (nr_to_free <= 0) 1517 break; 1518 } 1519 current->reclaim_state = NULL; 1520 return ret; 1521} 1522#endif 1523 1524#ifdef CONFIG_HOTPLUG_CPU 1525/* It's optimal to keep kswapds on the same CPUs as their memory, but 1526 not required for correctness. So if the last cpu in a node goes 1527 away, we get changed to run anywhere: as the first one comes back, 1528 restore their cpu bindings. */ 1529static int __devinit cpu_callback(struct notifier_block *nfb, 1530 unsigned long action, 1531 void *hcpu) 1532{ 1533 pg_data_t *pgdat; 1534 cpumask_t mask; 1535 1536 if (action == CPU_ONLINE) { 1537 for_each_pgdat(pgdat) { 1538 mask = node_to_cpumask(pgdat->node_id); 1539 if (any_online_cpu(mask) != NR_CPUS) 1540 /* One of our CPUs online: restore mask */ 1541 set_cpus_allowed(pgdat->kswapd, mask); 1542 } 1543 } 1544 return NOTIFY_OK; 1545} 1546#endif /* CONFIG_HOTPLUG_CPU */ 1547 1548static int __init kswapd_init(void) 1549{ 1550 pg_data_t *pgdat; 1551 swap_setup(); 1552 for_each_pgdat(pgdat) 1553 pgdat->kswapd 1554 = find_task_by_pid(kernel_thread(kswapd, pgdat, CLONE_KERNEL)); 1555 total_memory = nr_free_pagecache_pages(); 1556 hotcpu_notifier(cpu_callback, 0); 1557 return 0; 1558} 1559 1560module_init(kswapd_init) 1561