vmscan.c revision bbfd28eee9fbd73e780b19beb3dc562befbb94fa
1/* 2 * linux/mm/vmscan.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 * 6 * Swap reorganised 29.12.95, Stephen Tweedie. 7 * kswapd added: 7.1.96 sct 8 * Removed kswapd_ctl limits, and swap out as many pages as needed 9 * to bring the system back to freepages.high: 2.4.97, Rik van Riel. 10 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). 11 * Multiqueue VM started 5.8.00, Rik van Riel. 12 */ 13 14#include <linux/mm.h> 15#include <linux/module.h> 16#include <linux/slab.h> 17#include <linux/kernel_stat.h> 18#include <linux/swap.h> 19#include <linux/pagemap.h> 20#include <linux/init.h> 21#include <linux/highmem.h> 22#include <linux/vmstat.h> 23#include <linux/file.h> 24#include <linux/writeback.h> 25#include <linux/blkdev.h> 26#include <linux/buffer_head.h> /* for try_to_release_page(), 27 buffer_heads_over_limit */ 28#include <linux/mm_inline.h> 29#include <linux/pagevec.h> 30#include <linux/backing-dev.h> 31#include <linux/rmap.h> 32#include <linux/topology.h> 33#include <linux/cpu.h> 34#include <linux/cpuset.h> 35#include <linux/notifier.h> 36#include <linux/rwsem.h> 37#include <linux/delay.h> 38#include <linux/kthread.h> 39#include <linux/freezer.h> 40#include <linux/memcontrol.h> 41#include <linux/delayacct.h> 42 43#include <asm/tlbflush.h> 44#include <asm/div64.h> 45 46#include <linux/swapops.h> 47 48#include "internal.h" 49 50struct scan_control { 51 /* Incremented by the number of inactive pages that were scanned */ 52 unsigned long nr_scanned; 53 54 /* This context's GFP mask */ 55 gfp_t gfp_mask; 56 57 int may_writepage; 58 59 /* Can pages be swapped as part of reclaim? */ 60 int may_swap; 61 62 /* This context's SWAP_CLUSTER_MAX. If freeing memory for 63 * suspend, we effectively ignore SWAP_CLUSTER_MAX. 64 * In this context, it doesn't matter that we scan the 65 * whole list at once. */ 66 int swap_cluster_max; 67 68 int swappiness; 69 70 int all_unreclaimable; 71 72 int order; 73 74 /* Which cgroup do we reclaim from */ 75 struct mem_cgroup *mem_cgroup; 76 77 /* Pluggable isolate pages callback */ 78 unsigned long (*isolate_pages)(unsigned long nr, struct list_head *dst, 79 unsigned long *scanned, int order, int mode, 80 struct zone *z, struct mem_cgroup *mem_cont, 81 int active, int file); 82}; 83 84#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) 85 86#ifdef ARCH_HAS_PREFETCH 87#define prefetch_prev_lru_page(_page, _base, _field) \ 88 do { \ 89 if ((_page)->lru.prev != _base) { \ 90 struct page *prev; \ 91 \ 92 prev = lru_to_page(&(_page->lru)); \ 93 prefetch(&prev->_field); \ 94 } \ 95 } while (0) 96#else 97#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0) 98#endif 99 100#ifdef ARCH_HAS_PREFETCHW 101#define prefetchw_prev_lru_page(_page, _base, _field) \ 102 do { \ 103 if ((_page)->lru.prev != _base) { \ 104 struct page *prev; \ 105 \ 106 prev = lru_to_page(&(_page->lru)); \ 107 prefetchw(&prev->_field); \ 108 } \ 109 } while (0) 110#else 111#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0) 112#endif 113 114/* 115 * From 0 .. 100. Higher means more swappy. 116 */ 117int vm_swappiness = 60; 118long vm_total_pages; /* The total number of pages which the VM controls */ 119 120static LIST_HEAD(shrinker_list); 121static DECLARE_RWSEM(shrinker_rwsem); 122 123#ifdef CONFIG_CGROUP_MEM_RES_CTLR 124#define scan_global_lru(sc) (!(sc)->mem_cgroup) 125#else 126#define scan_global_lru(sc) (1) 127#endif 128 129/* 130 * Add a shrinker callback to be called from the vm 131 */ 132void register_shrinker(struct shrinker *shrinker) 133{ 134 shrinker->nr = 0; 135 down_write(&shrinker_rwsem); 136 list_add_tail(&shrinker->list, &shrinker_list); 137 up_write(&shrinker_rwsem); 138} 139EXPORT_SYMBOL(register_shrinker); 140 141/* 142 * Remove one 143 */ 144void unregister_shrinker(struct shrinker *shrinker) 145{ 146 down_write(&shrinker_rwsem); 147 list_del(&shrinker->list); 148 up_write(&shrinker_rwsem); 149} 150EXPORT_SYMBOL(unregister_shrinker); 151 152#define SHRINK_BATCH 128 153/* 154 * Call the shrink functions to age shrinkable caches 155 * 156 * Here we assume it costs one seek to replace a lru page and that it also 157 * takes a seek to recreate a cache object. With this in mind we age equal 158 * percentages of the lru and ageable caches. This should balance the seeks 159 * generated by these structures. 160 * 161 * If the vm encountered mapped pages on the LRU it increase the pressure on 162 * slab to avoid swapping. 163 * 164 * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits. 165 * 166 * `lru_pages' represents the number of on-LRU pages in all the zones which 167 * are eligible for the caller's allocation attempt. It is used for balancing 168 * slab reclaim versus page reclaim. 169 * 170 * Returns the number of slab objects which we shrunk. 171 */ 172unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask, 173 unsigned long lru_pages) 174{ 175 struct shrinker *shrinker; 176 unsigned long ret = 0; 177 178 if (scanned == 0) 179 scanned = SWAP_CLUSTER_MAX; 180 181 if (!down_read_trylock(&shrinker_rwsem)) 182 return 1; /* Assume we'll be able to shrink next time */ 183 184 list_for_each_entry(shrinker, &shrinker_list, list) { 185 unsigned long long delta; 186 unsigned long total_scan; 187 unsigned long max_pass = (*shrinker->shrink)(0, gfp_mask); 188 189 delta = (4 * scanned) / shrinker->seeks; 190 delta *= max_pass; 191 do_div(delta, lru_pages + 1); 192 shrinker->nr += delta; 193 if (shrinker->nr < 0) { 194 printk(KERN_ERR "%s: nr=%ld\n", 195 __func__, shrinker->nr); 196 shrinker->nr = max_pass; 197 } 198 199 /* 200 * Avoid risking looping forever due to too large nr value: 201 * never try to free more than twice the estimate number of 202 * freeable entries. 203 */ 204 if (shrinker->nr > max_pass * 2) 205 shrinker->nr = max_pass * 2; 206 207 total_scan = shrinker->nr; 208 shrinker->nr = 0; 209 210 while (total_scan >= SHRINK_BATCH) { 211 long this_scan = SHRINK_BATCH; 212 int shrink_ret; 213 int nr_before; 214 215 nr_before = (*shrinker->shrink)(0, gfp_mask); 216 shrink_ret = (*shrinker->shrink)(this_scan, gfp_mask); 217 if (shrink_ret == -1) 218 break; 219 if (shrink_ret < nr_before) 220 ret += nr_before - shrink_ret; 221 count_vm_events(SLABS_SCANNED, this_scan); 222 total_scan -= this_scan; 223 224 cond_resched(); 225 } 226 227 shrinker->nr += total_scan; 228 } 229 up_read(&shrinker_rwsem); 230 return ret; 231} 232 233/* Called without lock on whether page is mapped, so answer is unstable */ 234static inline int page_mapping_inuse(struct page *page) 235{ 236 struct address_space *mapping; 237 238 /* Page is in somebody's page tables. */ 239 if (page_mapped(page)) 240 return 1; 241 242 /* Be more reluctant to reclaim swapcache than pagecache */ 243 if (PageSwapCache(page)) 244 return 1; 245 246 mapping = page_mapping(page); 247 if (!mapping) 248 return 0; 249 250 /* File is mmap'd by somebody? */ 251 return mapping_mapped(mapping); 252} 253 254static inline int is_page_cache_freeable(struct page *page) 255{ 256 return page_count(page) - !!PagePrivate(page) == 2; 257} 258 259static int may_write_to_queue(struct backing_dev_info *bdi) 260{ 261 if (current->flags & PF_SWAPWRITE) 262 return 1; 263 if (!bdi_write_congested(bdi)) 264 return 1; 265 if (bdi == current->backing_dev_info) 266 return 1; 267 return 0; 268} 269 270/* 271 * We detected a synchronous write error writing a page out. Probably 272 * -ENOSPC. We need to propagate that into the address_space for a subsequent 273 * fsync(), msync() or close(). 274 * 275 * The tricky part is that after writepage we cannot touch the mapping: nothing 276 * prevents it from being freed up. But we have a ref on the page and once 277 * that page is locked, the mapping is pinned. 278 * 279 * We're allowed to run sleeping lock_page() here because we know the caller has 280 * __GFP_FS. 281 */ 282static void handle_write_error(struct address_space *mapping, 283 struct page *page, int error) 284{ 285 lock_page(page); 286 if (page_mapping(page) == mapping) 287 mapping_set_error(mapping, error); 288 unlock_page(page); 289} 290 291/* Request for sync pageout. */ 292enum pageout_io { 293 PAGEOUT_IO_ASYNC, 294 PAGEOUT_IO_SYNC, 295}; 296 297/* possible outcome of pageout() */ 298typedef enum { 299 /* failed to write page out, page is locked */ 300 PAGE_KEEP, 301 /* move page to the active list, page is locked */ 302 PAGE_ACTIVATE, 303 /* page has been sent to the disk successfully, page is unlocked */ 304 PAGE_SUCCESS, 305 /* page is clean and locked */ 306 PAGE_CLEAN, 307} pageout_t; 308 309/* 310 * pageout is called by shrink_page_list() for each dirty page. 311 * Calls ->writepage(). 312 */ 313static pageout_t pageout(struct page *page, struct address_space *mapping, 314 enum pageout_io sync_writeback) 315{ 316 /* 317 * If the page is dirty, only perform writeback if that write 318 * will be non-blocking. To prevent this allocation from being 319 * stalled by pagecache activity. But note that there may be 320 * stalls if we need to run get_block(). We could test 321 * PagePrivate for that. 322 * 323 * If this process is currently in generic_file_write() against 324 * this page's queue, we can perform writeback even if that 325 * will block. 326 * 327 * If the page is swapcache, write it back even if that would 328 * block, for some throttling. This happens by accident, because 329 * swap_backing_dev_info is bust: it doesn't reflect the 330 * congestion state of the swapdevs. Easy to fix, if needed. 331 * See swapfile.c:page_queue_congested(). 332 */ 333 if (!is_page_cache_freeable(page)) 334 return PAGE_KEEP; 335 if (!mapping) { 336 /* 337 * Some data journaling orphaned pages can have 338 * page->mapping == NULL while being dirty with clean buffers. 339 */ 340 if (PagePrivate(page)) { 341 if (try_to_free_buffers(page)) { 342 ClearPageDirty(page); 343 printk("%s: orphaned page\n", __func__); 344 return PAGE_CLEAN; 345 } 346 } 347 return PAGE_KEEP; 348 } 349 if (mapping->a_ops->writepage == NULL) 350 return PAGE_ACTIVATE; 351 if (!may_write_to_queue(mapping->backing_dev_info)) 352 return PAGE_KEEP; 353 354 if (clear_page_dirty_for_io(page)) { 355 int res; 356 struct writeback_control wbc = { 357 .sync_mode = WB_SYNC_NONE, 358 .nr_to_write = SWAP_CLUSTER_MAX, 359 .range_start = 0, 360 .range_end = LLONG_MAX, 361 .nonblocking = 1, 362 .for_reclaim = 1, 363 }; 364 365 SetPageReclaim(page); 366 res = mapping->a_ops->writepage(page, &wbc); 367 if (res < 0) 368 handle_write_error(mapping, page, res); 369 if (res == AOP_WRITEPAGE_ACTIVATE) { 370 ClearPageReclaim(page); 371 return PAGE_ACTIVATE; 372 } 373 374 /* 375 * Wait on writeback if requested to. This happens when 376 * direct reclaiming a large contiguous area and the 377 * first attempt to free a range of pages fails. 378 */ 379 if (PageWriteback(page) && sync_writeback == PAGEOUT_IO_SYNC) 380 wait_on_page_writeback(page); 381 382 if (!PageWriteback(page)) { 383 /* synchronous write or broken a_ops? */ 384 ClearPageReclaim(page); 385 } 386 inc_zone_page_state(page, NR_VMSCAN_WRITE); 387 return PAGE_SUCCESS; 388 } 389 390 return PAGE_CLEAN; 391} 392 393/* 394 * Same as remove_mapping, but if the page is removed from the mapping, it 395 * gets returned with a refcount of 0. 396 */ 397static int __remove_mapping(struct address_space *mapping, struct page *page) 398{ 399 BUG_ON(!PageLocked(page)); 400 BUG_ON(mapping != page_mapping(page)); 401 402 spin_lock_irq(&mapping->tree_lock); 403 /* 404 * The non racy check for a busy page. 405 * 406 * Must be careful with the order of the tests. When someone has 407 * a ref to the page, it may be possible that they dirty it then 408 * drop the reference. So if PageDirty is tested before page_count 409 * here, then the following race may occur: 410 * 411 * get_user_pages(&page); 412 * [user mapping goes away] 413 * write_to(page); 414 * !PageDirty(page) [good] 415 * SetPageDirty(page); 416 * put_page(page); 417 * !page_count(page) [good, discard it] 418 * 419 * [oops, our write_to data is lost] 420 * 421 * Reversing the order of the tests ensures such a situation cannot 422 * escape unnoticed. The smp_rmb is needed to ensure the page->flags 423 * load is not satisfied before that of page->_count. 424 * 425 * Note that if SetPageDirty is always performed via set_page_dirty, 426 * and thus under tree_lock, then this ordering is not required. 427 */ 428 if (!page_freeze_refs(page, 2)) 429 goto cannot_free; 430 /* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */ 431 if (unlikely(PageDirty(page))) { 432 page_unfreeze_refs(page, 2); 433 goto cannot_free; 434 } 435 436 if (PageSwapCache(page)) { 437 swp_entry_t swap = { .val = page_private(page) }; 438 __delete_from_swap_cache(page); 439 spin_unlock_irq(&mapping->tree_lock); 440 swap_free(swap); 441 } else { 442 __remove_from_page_cache(page); 443 spin_unlock_irq(&mapping->tree_lock); 444 } 445 446 return 1; 447 448cannot_free: 449 spin_unlock_irq(&mapping->tree_lock); 450 return 0; 451} 452 453/* 454 * Attempt to detach a locked page from its ->mapping. If it is dirty or if 455 * someone else has a ref on the page, abort and return 0. If it was 456 * successfully detached, return 1. Assumes the caller has a single ref on 457 * this page. 458 */ 459int remove_mapping(struct address_space *mapping, struct page *page) 460{ 461 if (__remove_mapping(mapping, page)) { 462 /* 463 * Unfreezing the refcount with 1 rather than 2 effectively 464 * drops the pagecache ref for us without requiring another 465 * atomic operation. 466 */ 467 page_unfreeze_refs(page, 1); 468 return 1; 469 } 470 return 0; 471} 472 473/** 474 * putback_lru_page - put previously isolated page onto appropriate LRU list 475 * @page: page to be put back to appropriate lru list 476 * 477 * Add previously isolated @page to appropriate LRU list. 478 * Page may still be unevictable for other reasons. 479 * 480 * lru_lock must not be held, interrupts must be enabled. 481 */ 482#ifdef CONFIG_UNEVICTABLE_LRU 483void putback_lru_page(struct page *page) 484{ 485 int lru; 486 int active = !!TestClearPageActive(page); 487 int was_unevictable = PageUnevictable(page); 488 489 VM_BUG_ON(PageLRU(page)); 490 491redo: 492 ClearPageUnevictable(page); 493 494 if (page_evictable(page, NULL)) { 495 /* 496 * For evictable pages, we can use the cache. 497 * In event of a race, worst case is we end up with an 498 * unevictable page on [in]active list. 499 * We know how to handle that. 500 */ 501 lru = active + page_is_file_cache(page); 502 lru_cache_add_lru(page, lru); 503 } else { 504 /* 505 * Put unevictable pages directly on zone's unevictable 506 * list. 507 */ 508 lru = LRU_UNEVICTABLE; 509 add_page_to_unevictable_list(page); 510 } 511 mem_cgroup_move_lists(page, lru); 512 513 /* 514 * page's status can change while we move it among lru. If an evictable 515 * page is on unevictable list, it never be freed. To avoid that, 516 * check after we added it to the list, again. 517 */ 518 if (lru == LRU_UNEVICTABLE && page_evictable(page, NULL)) { 519 if (!isolate_lru_page(page)) { 520 put_page(page); 521 goto redo; 522 } 523 /* This means someone else dropped this page from LRU 524 * So, it will be freed or putback to LRU again. There is 525 * nothing to do here. 526 */ 527 } 528 529 if (was_unevictable && lru != LRU_UNEVICTABLE) 530 count_vm_event(UNEVICTABLE_PGRESCUED); 531 else if (!was_unevictable && lru == LRU_UNEVICTABLE) 532 count_vm_event(UNEVICTABLE_PGCULLED); 533 534 put_page(page); /* drop ref from isolate */ 535} 536 537#else /* CONFIG_UNEVICTABLE_LRU */ 538 539void putback_lru_page(struct page *page) 540{ 541 int lru; 542 VM_BUG_ON(PageLRU(page)); 543 544 lru = !!TestClearPageActive(page) + page_is_file_cache(page); 545 lru_cache_add_lru(page, lru); 546 mem_cgroup_move_lists(page, lru); 547 put_page(page); 548} 549#endif /* CONFIG_UNEVICTABLE_LRU */ 550 551 552/* 553 * shrink_page_list() returns the number of reclaimed pages 554 */ 555static unsigned long shrink_page_list(struct list_head *page_list, 556 struct scan_control *sc, 557 enum pageout_io sync_writeback) 558{ 559 LIST_HEAD(ret_pages); 560 struct pagevec freed_pvec; 561 int pgactivate = 0; 562 unsigned long nr_reclaimed = 0; 563 564 cond_resched(); 565 566 pagevec_init(&freed_pvec, 1); 567 while (!list_empty(page_list)) { 568 struct address_space *mapping; 569 struct page *page; 570 int may_enter_fs; 571 int referenced; 572 573 cond_resched(); 574 575 page = lru_to_page(page_list); 576 list_del(&page->lru); 577 578 if (!trylock_page(page)) 579 goto keep; 580 581 VM_BUG_ON(PageActive(page)); 582 583 sc->nr_scanned++; 584 585 if (unlikely(!page_evictable(page, NULL))) { 586 unlock_page(page); 587 putback_lru_page(page); 588 continue; 589 } 590 591 if (!sc->may_swap && page_mapped(page)) 592 goto keep_locked; 593 594 /* Double the slab pressure for mapped and swapcache pages */ 595 if (page_mapped(page) || PageSwapCache(page)) 596 sc->nr_scanned++; 597 598 may_enter_fs = (sc->gfp_mask & __GFP_FS) || 599 (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO)); 600 601 if (PageWriteback(page)) { 602 /* 603 * Synchronous reclaim is performed in two passes, 604 * first an asynchronous pass over the list to 605 * start parallel writeback, and a second synchronous 606 * pass to wait for the IO to complete. Wait here 607 * for any page for which writeback has already 608 * started. 609 */ 610 if (sync_writeback == PAGEOUT_IO_SYNC && may_enter_fs) 611 wait_on_page_writeback(page); 612 else 613 goto keep_locked; 614 } 615 616 referenced = page_referenced(page, 1, sc->mem_cgroup); 617 /* In active use or really unfreeable? Activate it. */ 618 if (sc->order <= PAGE_ALLOC_COSTLY_ORDER && 619 referenced && page_mapping_inuse(page)) 620 goto activate_locked; 621 622#ifdef CONFIG_SWAP 623 /* 624 * Anonymous process memory has backing store? 625 * Try to allocate it some swap space here. 626 */ 627 if (PageAnon(page) && !PageSwapCache(page)) 628 if (!add_to_swap(page, GFP_ATOMIC)) 629 goto activate_locked; 630#endif /* CONFIG_SWAP */ 631 632 mapping = page_mapping(page); 633 634 /* 635 * The page is mapped into the page tables of one or more 636 * processes. Try to unmap it here. 637 */ 638 if (page_mapped(page) && mapping) { 639 switch (try_to_unmap(page, 0)) { 640 case SWAP_FAIL: 641 goto activate_locked; 642 case SWAP_AGAIN: 643 goto keep_locked; 644 case SWAP_SUCCESS: 645 ; /* try to free the page below */ 646 } 647 } 648 649 if (PageDirty(page)) { 650 if (sc->order <= PAGE_ALLOC_COSTLY_ORDER && referenced) 651 goto keep_locked; 652 if (!may_enter_fs) 653 goto keep_locked; 654 if (!sc->may_writepage) 655 goto keep_locked; 656 657 /* Page is dirty, try to write it out here */ 658 switch (pageout(page, mapping, sync_writeback)) { 659 case PAGE_KEEP: 660 goto keep_locked; 661 case PAGE_ACTIVATE: 662 goto activate_locked; 663 case PAGE_SUCCESS: 664 if (PageWriteback(page) || PageDirty(page)) 665 goto keep; 666 /* 667 * A synchronous write - probably a ramdisk. Go 668 * ahead and try to reclaim the page. 669 */ 670 if (!trylock_page(page)) 671 goto keep; 672 if (PageDirty(page) || PageWriteback(page)) 673 goto keep_locked; 674 mapping = page_mapping(page); 675 case PAGE_CLEAN: 676 ; /* try to free the page below */ 677 } 678 } 679 680 /* 681 * If the page has buffers, try to free the buffer mappings 682 * associated with this page. If we succeed we try to free 683 * the page as well. 684 * 685 * We do this even if the page is PageDirty(). 686 * try_to_release_page() does not perform I/O, but it is 687 * possible for a page to have PageDirty set, but it is actually 688 * clean (all its buffers are clean). This happens if the 689 * buffers were written out directly, with submit_bh(). ext3 690 * will do this, as well as the blockdev mapping. 691 * try_to_release_page() will discover that cleanness and will 692 * drop the buffers and mark the page clean - it can be freed. 693 * 694 * Rarely, pages can have buffers and no ->mapping. These are 695 * the pages which were not successfully invalidated in 696 * truncate_complete_page(). We try to drop those buffers here 697 * and if that worked, and the page is no longer mapped into 698 * process address space (page_count == 1) it can be freed. 699 * Otherwise, leave the page on the LRU so it is swappable. 700 */ 701 if (PagePrivate(page)) { 702 if (!try_to_release_page(page, sc->gfp_mask)) 703 goto activate_locked; 704 if (!mapping && page_count(page) == 1) { 705 unlock_page(page); 706 if (put_page_testzero(page)) 707 goto free_it; 708 else { 709 /* 710 * rare race with speculative reference. 711 * the speculative reference will free 712 * this page shortly, so we may 713 * increment nr_reclaimed here (and 714 * leave it off the LRU). 715 */ 716 nr_reclaimed++; 717 continue; 718 } 719 } 720 } 721 722 if (!mapping || !__remove_mapping(mapping, page)) 723 goto keep_locked; 724 725 unlock_page(page); 726free_it: 727 nr_reclaimed++; 728 if (!pagevec_add(&freed_pvec, page)) { 729 __pagevec_free(&freed_pvec); 730 pagevec_reinit(&freed_pvec); 731 } 732 continue; 733 734activate_locked: 735 /* Not a candidate for swapping, so reclaim swap space. */ 736 if (PageSwapCache(page) && vm_swap_full()) 737 remove_exclusive_swap_page_ref(page); 738 VM_BUG_ON(PageActive(page)); 739 SetPageActive(page); 740 pgactivate++; 741keep_locked: 742 unlock_page(page); 743keep: 744 list_add(&page->lru, &ret_pages); 745 VM_BUG_ON(PageLRU(page)); 746 } 747 list_splice(&ret_pages, page_list); 748 if (pagevec_count(&freed_pvec)) 749 __pagevec_free(&freed_pvec); 750 count_vm_events(PGACTIVATE, pgactivate); 751 return nr_reclaimed; 752} 753 754/* LRU Isolation modes. */ 755#define ISOLATE_INACTIVE 0 /* Isolate inactive pages. */ 756#define ISOLATE_ACTIVE 1 /* Isolate active pages. */ 757#define ISOLATE_BOTH 2 /* Isolate both active and inactive pages. */ 758 759/* 760 * Attempt to remove the specified page from its LRU. Only take this page 761 * if it is of the appropriate PageActive status. Pages which are being 762 * freed elsewhere are also ignored. 763 * 764 * page: page to consider 765 * mode: one of the LRU isolation modes defined above 766 * 767 * returns 0 on success, -ve errno on failure. 768 */ 769int __isolate_lru_page(struct page *page, int mode, int file) 770{ 771 int ret = -EINVAL; 772 773 /* Only take pages on the LRU. */ 774 if (!PageLRU(page)) 775 return ret; 776 777 /* 778 * When checking the active state, we need to be sure we are 779 * dealing with comparible boolean values. Take the logical not 780 * of each. 781 */ 782 if (mode != ISOLATE_BOTH && (!PageActive(page) != !mode)) 783 return ret; 784 785 if (mode != ISOLATE_BOTH && (!page_is_file_cache(page) != !file)) 786 return ret; 787 788 /* 789 * When this function is being called for lumpy reclaim, we 790 * initially look into all LRU pages, active, inactive and 791 * unevictable; only give shrink_page_list evictable pages. 792 */ 793 if (PageUnevictable(page)) 794 return ret; 795 796 ret = -EBUSY; 797 if (likely(get_page_unless_zero(page))) { 798 /* 799 * Be careful not to clear PageLRU until after we're 800 * sure the page is not being freed elsewhere -- the 801 * page release code relies on it. 802 */ 803 ClearPageLRU(page); 804 ret = 0; 805 } 806 807 return ret; 808} 809 810/* 811 * zone->lru_lock is heavily contended. Some of the functions that 812 * shrink the lists perform better by taking out a batch of pages 813 * and working on them outside the LRU lock. 814 * 815 * For pagecache intensive workloads, this function is the hottest 816 * spot in the kernel (apart from copy_*_user functions). 817 * 818 * Appropriate locks must be held before calling this function. 819 * 820 * @nr_to_scan: The number of pages to look through on the list. 821 * @src: The LRU list to pull pages off. 822 * @dst: The temp list to put pages on to. 823 * @scanned: The number of pages that were scanned. 824 * @order: The caller's attempted allocation order 825 * @mode: One of the LRU isolation modes 826 * @file: True [1] if isolating file [!anon] pages 827 * 828 * returns how many pages were moved onto *@dst. 829 */ 830static unsigned long isolate_lru_pages(unsigned long nr_to_scan, 831 struct list_head *src, struct list_head *dst, 832 unsigned long *scanned, int order, int mode, int file) 833{ 834 unsigned long nr_taken = 0; 835 unsigned long scan; 836 837 for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) { 838 struct page *page; 839 unsigned long pfn; 840 unsigned long end_pfn; 841 unsigned long page_pfn; 842 int zone_id; 843 844 page = lru_to_page(src); 845 prefetchw_prev_lru_page(page, src, flags); 846 847 VM_BUG_ON(!PageLRU(page)); 848 849 switch (__isolate_lru_page(page, mode, file)) { 850 case 0: 851 list_move(&page->lru, dst); 852 nr_taken++; 853 break; 854 855 case -EBUSY: 856 /* else it is being freed elsewhere */ 857 list_move(&page->lru, src); 858 continue; 859 860 default: 861 BUG(); 862 } 863 864 if (!order) 865 continue; 866 867 /* 868 * Attempt to take all pages in the order aligned region 869 * surrounding the tag page. Only take those pages of 870 * the same active state as that tag page. We may safely 871 * round the target page pfn down to the requested order 872 * as the mem_map is guarenteed valid out to MAX_ORDER, 873 * where that page is in a different zone we will detect 874 * it from its zone id and abort this block scan. 875 */ 876 zone_id = page_zone_id(page); 877 page_pfn = page_to_pfn(page); 878 pfn = page_pfn & ~((1 << order) - 1); 879 end_pfn = pfn + (1 << order); 880 for (; pfn < end_pfn; pfn++) { 881 struct page *cursor_page; 882 883 /* The target page is in the block, ignore it. */ 884 if (unlikely(pfn == page_pfn)) 885 continue; 886 887 /* Avoid holes within the zone. */ 888 if (unlikely(!pfn_valid_within(pfn))) 889 break; 890 891 cursor_page = pfn_to_page(pfn); 892 893 /* Check that we have not crossed a zone boundary. */ 894 if (unlikely(page_zone_id(cursor_page) != zone_id)) 895 continue; 896 switch (__isolate_lru_page(cursor_page, mode, file)) { 897 case 0: 898 list_move(&cursor_page->lru, dst); 899 nr_taken++; 900 scan++; 901 break; 902 903 case -EBUSY: 904 /* else it is being freed elsewhere */ 905 list_move(&cursor_page->lru, src); 906 default: 907 break; /* ! on LRU or wrong list */ 908 } 909 } 910 } 911 912 *scanned = scan; 913 return nr_taken; 914} 915 916static unsigned long isolate_pages_global(unsigned long nr, 917 struct list_head *dst, 918 unsigned long *scanned, int order, 919 int mode, struct zone *z, 920 struct mem_cgroup *mem_cont, 921 int active, int file) 922{ 923 int lru = LRU_BASE; 924 if (active) 925 lru += LRU_ACTIVE; 926 if (file) 927 lru += LRU_FILE; 928 return isolate_lru_pages(nr, &z->lru[lru].list, dst, scanned, order, 929 mode, !!file); 930} 931 932/* 933 * clear_active_flags() is a helper for shrink_active_list(), clearing 934 * any active bits from the pages in the list. 935 */ 936static unsigned long clear_active_flags(struct list_head *page_list, 937 unsigned int *count) 938{ 939 int nr_active = 0; 940 int lru; 941 struct page *page; 942 943 list_for_each_entry(page, page_list, lru) { 944 lru = page_is_file_cache(page); 945 if (PageActive(page)) { 946 lru += LRU_ACTIVE; 947 ClearPageActive(page); 948 nr_active++; 949 } 950 count[lru]++; 951 } 952 953 return nr_active; 954} 955 956/** 957 * isolate_lru_page - tries to isolate a page from its LRU list 958 * @page: page to isolate from its LRU list 959 * 960 * Isolates a @page from an LRU list, clears PageLRU and adjusts the 961 * vmstat statistic corresponding to whatever LRU list the page was on. 962 * 963 * Returns 0 if the page was removed from an LRU list. 964 * Returns -EBUSY if the page was not on an LRU list. 965 * 966 * The returned page will have PageLRU() cleared. If it was found on 967 * the active list, it will have PageActive set. If it was found on 968 * the unevictable list, it will have the PageUnevictable bit set. That flag 969 * may need to be cleared by the caller before letting the page go. 970 * 971 * The vmstat statistic corresponding to the list on which the page was 972 * found will be decremented. 973 * 974 * Restrictions: 975 * (1) Must be called with an elevated refcount on the page. This is a 976 * fundamentnal difference from isolate_lru_pages (which is called 977 * without a stable reference). 978 * (2) the lru_lock must not be held. 979 * (3) interrupts must be enabled. 980 */ 981int isolate_lru_page(struct page *page) 982{ 983 int ret = -EBUSY; 984 985 if (PageLRU(page)) { 986 struct zone *zone = page_zone(page); 987 988 spin_lock_irq(&zone->lru_lock); 989 if (PageLRU(page) && get_page_unless_zero(page)) { 990 int lru = page_lru(page); 991 ret = 0; 992 ClearPageLRU(page); 993 994 del_page_from_lru_list(zone, page, lru); 995 } 996 spin_unlock_irq(&zone->lru_lock); 997 } 998 return ret; 999} 1000 1001/* 1002 * shrink_inactive_list() is a helper for shrink_zone(). It returns the number 1003 * of reclaimed pages 1004 */ 1005static unsigned long shrink_inactive_list(unsigned long max_scan, 1006 struct zone *zone, struct scan_control *sc, 1007 int priority, int file) 1008{ 1009 LIST_HEAD(page_list); 1010 struct pagevec pvec; 1011 unsigned long nr_scanned = 0; 1012 unsigned long nr_reclaimed = 0; 1013 1014 pagevec_init(&pvec, 1); 1015 1016 lru_add_drain(); 1017 spin_lock_irq(&zone->lru_lock); 1018 do { 1019 struct page *page; 1020 unsigned long nr_taken; 1021 unsigned long nr_scan; 1022 unsigned long nr_freed; 1023 unsigned long nr_active; 1024 unsigned int count[NR_LRU_LISTS] = { 0, }; 1025 int mode = ISOLATE_INACTIVE; 1026 1027 /* 1028 * If we need a large contiguous chunk of memory, or have 1029 * trouble getting a small set of contiguous pages, we 1030 * will reclaim both active and inactive pages. 1031 * 1032 * We use the same threshold as pageout congestion_wait below. 1033 */ 1034 if (sc->order > PAGE_ALLOC_COSTLY_ORDER) 1035 mode = ISOLATE_BOTH; 1036 else if (sc->order && priority < DEF_PRIORITY - 2) 1037 mode = ISOLATE_BOTH; 1038 1039 nr_taken = sc->isolate_pages(sc->swap_cluster_max, 1040 &page_list, &nr_scan, sc->order, mode, 1041 zone, sc->mem_cgroup, 0, file); 1042 nr_active = clear_active_flags(&page_list, count); 1043 __count_vm_events(PGDEACTIVATE, nr_active); 1044 1045 __mod_zone_page_state(zone, NR_ACTIVE_FILE, 1046 -count[LRU_ACTIVE_FILE]); 1047 __mod_zone_page_state(zone, NR_INACTIVE_FILE, 1048 -count[LRU_INACTIVE_FILE]); 1049 __mod_zone_page_state(zone, NR_ACTIVE_ANON, 1050 -count[LRU_ACTIVE_ANON]); 1051 __mod_zone_page_state(zone, NR_INACTIVE_ANON, 1052 -count[LRU_INACTIVE_ANON]); 1053 1054 if (scan_global_lru(sc)) { 1055 zone->pages_scanned += nr_scan; 1056 zone->recent_scanned[0] += count[LRU_INACTIVE_ANON]; 1057 zone->recent_scanned[0] += count[LRU_ACTIVE_ANON]; 1058 zone->recent_scanned[1] += count[LRU_INACTIVE_FILE]; 1059 zone->recent_scanned[1] += count[LRU_ACTIVE_FILE]; 1060 } 1061 spin_unlock_irq(&zone->lru_lock); 1062 1063 nr_scanned += nr_scan; 1064 nr_freed = shrink_page_list(&page_list, sc, PAGEOUT_IO_ASYNC); 1065 1066 /* 1067 * If we are direct reclaiming for contiguous pages and we do 1068 * not reclaim everything in the list, try again and wait 1069 * for IO to complete. This will stall high-order allocations 1070 * but that should be acceptable to the caller 1071 */ 1072 if (nr_freed < nr_taken && !current_is_kswapd() && 1073 sc->order > PAGE_ALLOC_COSTLY_ORDER) { 1074 congestion_wait(WRITE, HZ/10); 1075 1076 /* 1077 * The attempt at page out may have made some 1078 * of the pages active, mark them inactive again. 1079 */ 1080 nr_active = clear_active_flags(&page_list, count); 1081 count_vm_events(PGDEACTIVATE, nr_active); 1082 1083 nr_freed += shrink_page_list(&page_list, sc, 1084 PAGEOUT_IO_SYNC); 1085 } 1086 1087 nr_reclaimed += nr_freed; 1088 local_irq_disable(); 1089 if (current_is_kswapd()) { 1090 __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scan); 1091 __count_vm_events(KSWAPD_STEAL, nr_freed); 1092 } else if (scan_global_lru(sc)) 1093 __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scan); 1094 1095 __count_zone_vm_events(PGSTEAL, zone, nr_freed); 1096 1097 if (nr_taken == 0) 1098 goto done; 1099 1100 spin_lock(&zone->lru_lock); 1101 /* 1102 * Put back any unfreeable pages. 1103 */ 1104 while (!list_empty(&page_list)) { 1105 int lru; 1106 page = lru_to_page(&page_list); 1107 VM_BUG_ON(PageLRU(page)); 1108 list_del(&page->lru); 1109 if (unlikely(!page_evictable(page, NULL))) { 1110 spin_unlock_irq(&zone->lru_lock); 1111 putback_lru_page(page); 1112 spin_lock_irq(&zone->lru_lock); 1113 continue; 1114 } 1115 SetPageLRU(page); 1116 lru = page_lru(page); 1117 add_page_to_lru_list(zone, page, lru); 1118 mem_cgroup_move_lists(page, lru); 1119 if (PageActive(page) && scan_global_lru(sc)) { 1120 int file = !!page_is_file_cache(page); 1121 zone->recent_rotated[file]++; 1122 } 1123 if (!pagevec_add(&pvec, page)) { 1124 spin_unlock_irq(&zone->lru_lock); 1125 __pagevec_release(&pvec); 1126 spin_lock_irq(&zone->lru_lock); 1127 } 1128 } 1129 } while (nr_scanned < max_scan); 1130 spin_unlock(&zone->lru_lock); 1131done: 1132 local_irq_enable(); 1133 pagevec_release(&pvec); 1134 return nr_reclaimed; 1135} 1136 1137/* 1138 * We are about to scan this zone at a certain priority level. If that priority 1139 * level is smaller (ie: more urgent) than the previous priority, then note 1140 * that priority level within the zone. This is done so that when the next 1141 * process comes in to scan this zone, it will immediately start out at this 1142 * priority level rather than having to build up its own scanning priority. 1143 * Here, this priority affects only the reclaim-mapped threshold. 1144 */ 1145static inline void note_zone_scanning_priority(struct zone *zone, int priority) 1146{ 1147 if (priority < zone->prev_priority) 1148 zone->prev_priority = priority; 1149} 1150 1151static inline int zone_is_near_oom(struct zone *zone) 1152{ 1153 return zone->pages_scanned >= (zone_lru_pages(zone) * 3); 1154} 1155 1156/* 1157 * This moves pages from the active list to the inactive list. 1158 * 1159 * We move them the other way if the page is referenced by one or more 1160 * processes, from rmap. 1161 * 1162 * If the pages are mostly unmapped, the processing is fast and it is 1163 * appropriate to hold zone->lru_lock across the whole operation. But if 1164 * the pages are mapped, the processing is slow (page_referenced()) so we 1165 * should drop zone->lru_lock around each page. It's impossible to balance 1166 * this, so instead we remove the pages from the LRU while processing them. 1167 * It is safe to rely on PG_active against the non-LRU pages in here because 1168 * nobody will play with that bit on a non-LRU page. 1169 * 1170 * The downside is that we have to touch page->_count against each page. 1171 * But we had to alter page->flags anyway. 1172 */ 1173 1174 1175static void shrink_active_list(unsigned long nr_pages, struct zone *zone, 1176 struct scan_control *sc, int priority, int file) 1177{ 1178 unsigned long pgmoved; 1179 int pgdeactivate = 0; 1180 unsigned long pgscanned; 1181 LIST_HEAD(l_hold); /* The pages which were snipped off */ 1182 LIST_HEAD(l_inactive); 1183 struct page *page; 1184 struct pagevec pvec; 1185 enum lru_list lru; 1186 1187 lru_add_drain(); 1188 spin_lock_irq(&zone->lru_lock); 1189 pgmoved = sc->isolate_pages(nr_pages, &l_hold, &pgscanned, sc->order, 1190 ISOLATE_ACTIVE, zone, 1191 sc->mem_cgroup, 1, file); 1192 /* 1193 * zone->pages_scanned is used for detect zone's oom 1194 * mem_cgroup remembers nr_scan by itself. 1195 */ 1196 if (scan_global_lru(sc)) { 1197 zone->pages_scanned += pgscanned; 1198 zone->recent_scanned[!!file] += pgmoved; 1199 } 1200 1201 if (file) 1202 __mod_zone_page_state(zone, NR_ACTIVE_FILE, -pgmoved); 1203 else 1204 __mod_zone_page_state(zone, NR_ACTIVE_ANON, -pgmoved); 1205 spin_unlock_irq(&zone->lru_lock); 1206 1207 pgmoved = 0; 1208 while (!list_empty(&l_hold)) { 1209 cond_resched(); 1210 page = lru_to_page(&l_hold); 1211 list_del(&page->lru); 1212 1213 if (unlikely(!page_evictable(page, NULL))) { 1214 putback_lru_page(page); 1215 continue; 1216 } 1217 1218 /* page_referenced clears PageReferenced */ 1219 if (page_mapping_inuse(page) && 1220 page_referenced(page, 0, sc->mem_cgroup)) 1221 pgmoved++; 1222 1223 list_add(&page->lru, &l_inactive); 1224 } 1225 1226 /* 1227 * Count referenced pages from currently used mappings as 1228 * rotated, even though they are moved to the inactive list. 1229 * This helps balance scan pressure between file and anonymous 1230 * pages in get_scan_ratio. 1231 */ 1232 zone->recent_rotated[!!file] += pgmoved; 1233 1234 /* 1235 * Move the pages to the [file or anon] inactive list. 1236 */ 1237 pagevec_init(&pvec, 1); 1238 1239 pgmoved = 0; 1240 lru = LRU_BASE + file * LRU_FILE; 1241 spin_lock_irq(&zone->lru_lock); 1242 while (!list_empty(&l_inactive)) { 1243 page = lru_to_page(&l_inactive); 1244 prefetchw_prev_lru_page(page, &l_inactive, flags); 1245 VM_BUG_ON(PageLRU(page)); 1246 SetPageLRU(page); 1247 VM_BUG_ON(!PageActive(page)); 1248 ClearPageActive(page); 1249 1250 list_move(&page->lru, &zone->lru[lru].list); 1251 mem_cgroup_move_lists(page, lru); 1252 pgmoved++; 1253 if (!pagevec_add(&pvec, page)) { 1254 __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved); 1255 spin_unlock_irq(&zone->lru_lock); 1256 pgdeactivate += pgmoved; 1257 pgmoved = 0; 1258 if (buffer_heads_over_limit) 1259 pagevec_strip(&pvec); 1260 __pagevec_release(&pvec); 1261 spin_lock_irq(&zone->lru_lock); 1262 } 1263 } 1264 __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved); 1265 pgdeactivate += pgmoved; 1266 if (buffer_heads_over_limit) { 1267 spin_unlock_irq(&zone->lru_lock); 1268 pagevec_strip(&pvec); 1269 spin_lock_irq(&zone->lru_lock); 1270 } 1271 __count_zone_vm_events(PGREFILL, zone, pgscanned); 1272 __count_vm_events(PGDEACTIVATE, pgdeactivate); 1273 spin_unlock_irq(&zone->lru_lock); 1274 if (vm_swap_full()) 1275 pagevec_swap_free(&pvec); 1276 1277 pagevec_release(&pvec); 1278} 1279 1280static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, 1281 struct zone *zone, struct scan_control *sc, int priority) 1282{ 1283 int file = is_file_lru(lru); 1284 1285 if (lru == LRU_ACTIVE_FILE) { 1286 shrink_active_list(nr_to_scan, zone, sc, priority, file); 1287 return 0; 1288 } 1289 1290 if (lru == LRU_ACTIVE_ANON && 1291 (!scan_global_lru(sc) || inactive_anon_is_low(zone))) { 1292 shrink_active_list(nr_to_scan, zone, sc, priority, file); 1293 return 0; 1294 } 1295 return shrink_inactive_list(nr_to_scan, zone, sc, priority, file); 1296} 1297 1298/* 1299 * Determine how aggressively the anon and file LRU lists should be 1300 * scanned. The relative value of each set of LRU lists is determined 1301 * by looking at the fraction of the pages scanned we did rotate back 1302 * onto the active list instead of evict. 1303 * 1304 * percent[0] specifies how much pressure to put on ram/swap backed 1305 * memory, while percent[1] determines pressure on the file LRUs. 1306 */ 1307static void get_scan_ratio(struct zone *zone, struct scan_control *sc, 1308 unsigned long *percent) 1309{ 1310 unsigned long anon, file, free; 1311 unsigned long anon_prio, file_prio; 1312 unsigned long ap, fp; 1313 1314 anon = zone_page_state(zone, NR_ACTIVE_ANON) + 1315 zone_page_state(zone, NR_INACTIVE_ANON); 1316 file = zone_page_state(zone, NR_ACTIVE_FILE) + 1317 zone_page_state(zone, NR_INACTIVE_FILE); 1318 free = zone_page_state(zone, NR_FREE_PAGES); 1319 1320 /* If we have no swap space, do not bother scanning anon pages. */ 1321 if (nr_swap_pages <= 0) { 1322 percent[0] = 0; 1323 percent[1] = 100; 1324 return; 1325 } 1326 1327 /* If we have very few page cache pages, force-scan anon pages. */ 1328 if (unlikely(file + free <= zone->pages_high)) { 1329 percent[0] = 100; 1330 percent[1] = 0; 1331 return; 1332 } 1333 1334 /* 1335 * OK, so we have swap space and a fair amount of page cache 1336 * pages. We use the recently rotated / recently scanned 1337 * ratios to determine how valuable each cache is. 1338 * 1339 * Because workloads change over time (and to avoid overflow) 1340 * we keep these statistics as a floating average, which ends 1341 * up weighing recent references more than old ones. 1342 * 1343 * anon in [0], file in [1] 1344 */ 1345 if (unlikely(zone->recent_scanned[0] > anon / 4)) { 1346 spin_lock_irq(&zone->lru_lock); 1347 zone->recent_scanned[0] /= 2; 1348 zone->recent_rotated[0] /= 2; 1349 spin_unlock_irq(&zone->lru_lock); 1350 } 1351 1352 if (unlikely(zone->recent_scanned[1] > file / 4)) { 1353 spin_lock_irq(&zone->lru_lock); 1354 zone->recent_scanned[1] /= 2; 1355 zone->recent_rotated[1] /= 2; 1356 spin_unlock_irq(&zone->lru_lock); 1357 } 1358 1359 /* 1360 * With swappiness at 100, anonymous and file have the same priority. 1361 * This scanning priority is essentially the inverse of IO cost. 1362 */ 1363 anon_prio = sc->swappiness; 1364 file_prio = 200 - sc->swappiness; 1365 1366 /* 1367 * anon recent_rotated[0] 1368 * %anon = 100 * ----------- / ----------------- * IO cost 1369 * anon + file rotate_sum 1370 */ 1371 ap = (anon_prio + 1) * (zone->recent_scanned[0] + 1); 1372 ap /= zone->recent_rotated[0] + 1; 1373 1374 fp = (file_prio + 1) * (zone->recent_scanned[1] + 1); 1375 fp /= zone->recent_rotated[1] + 1; 1376 1377 /* Normalize to percentages */ 1378 percent[0] = 100 * ap / (ap + fp + 1); 1379 percent[1] = 100 - percent[0]; 1380} 1381 1382 1383/* 1384 * This is a basic per-zone page freer. Used by both kswapd and direct reclaim. 1385 */ 1386static unsigned long shrink_zone(int priority, struct zone *zone, 1387 struct scan_control *sc) 1388{ 1389 unsigned long nr[NR_LRU_LISTS]; 1390 unsigned long nr_to_scan; 1391 unsigned long nr_reclaimed = 0; 1392 unsigned long percent[2]; /* anon @ 0; file @ 1 */ 1393 enum lru_list l; 1394 1395 get_scan_ratio(zone, sc, percent); 1396 1397 for_each_evictable_lru(l) { 1398 if (scan_global_lru(sc)) { 1399 int file = is_file_lru(l); 1400 int scan; 1401 /* 1402 * Add one to nr_to_scan just to make sure that the 1403 * kernel will slowly sift through each list. 1404 */ 1405 scan = zone_page_state(zone, NR_LRU_BASE + l); 1406 if (priority) { 1407 scan >>= priority; 1408 scan = (scan * percent[file]) / 100; 1409 } 1410 zone->lru[l].nr_scan += scan + 1; 1411 nr[l] = zone->lru[l].nr_scan; 1412 if (nr[l] >= sc->swap_cluster_max) 1413 zone->lru[l].nr_scan = 0; 1414 else 1415 nr[l] = 0; 1416 } else { 1417 /* 1418 * This reclaim occurs not because zone memory shortage 1419 * but because memory controller hits its limit. 1420 * Don't modify zone reclaim related data. 1421 */ 1422 nr[l] = mem_cgroup_calc_reclaim(sc->mem_cgroup, zone, 1423 priority, l); 1424 } 1425 } 1426 1427 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || 1428 nr[LRU_INACTIVE_FILE]) { 1429 for_each_evictable_lru(l) { 1430 if (nr[l]) { 1431 nr_to_scan = min(nr[l], 1432 (unsigned long)sc->swap_cluster_max); 1433 nr[l] -= nr_to_scan; 1434 1435 nr_reclaimed += shrink_list(l, nr_to_scan, 1436 zone, sc, priority); 1437 } 1438 } 1439 } 1440 1441 /* 1442 * Even if we did not try to evict anon pages at all, we want to 1443 * rebalance the anon lru active/inactive ratio. 1444 */ 1445 if (!scan_global_lru(sc) || inactive_anon_is_low(zone)) 1446 shrink_active_list(SWAP_CLUSTER_MAX, zone, sc, priority, 0); 1447 else if (!scan_global_lru(sc)) 1448 shrink_active_list(SWAP_CLUSTER_MAX, zone, sc, priority, 0); 1449 1450 throttle_vm_writeout(sc->gfp_mask); 1451 return nr_reclaimed; 1452} 1453 1454/* 1455 * This is the direct reclaim path, for page-allocating processes. We only 1456 * try to reclaim pages from zones which will satisfy the caller's allocation 1457 * request. 1458 * 1459 * We reclaim from a zone even if that zone is over pages_high. Because: 1460 * a) The caller may be trying to free *extra* pages to satisfy a higher-order 1461 * allocation or 1462 * b) The zones may be over pages_high but they must go *over* pages_high to 1463 * satisfy the `incremental min' zone defense algorithm. 1464 * 1465 * Returns the number of reclaimed pages. 1466 * 1467 * If a zone is deemed to be full of pinned pages then just give it a light 1468 * scan then give up on it. 1469 */ 1470static unsigned long shrink_zones(int priority, struct zonelist *zonelist, 1471 struct scan_control *sc) 1472{ 1473 enum zone_type high_zoneidx = gfp_zone(sc->gfp_mask); 1474 unsigned long nr_reclaimed = 0; 1475 struct zoneref *z; 1476 struct zone *zone; 1477 1478 sc->all_unreclaimable = 1; 1479 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 1480 if (!populated_zone(zone)) 1481 continue; 1482 /* 1483 * Take care memory controller reclaiming has small influence 1484 * to global LRU. 1485 */ 1486 if (scan_global_lru(sc)) { 1487 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL)) 1488 continue; 1489 note_zone_scanning_priority(zone, priority); 1490 1491 if (zone_is_all_unreclaimable(zone) && 1492 priority != DEF_PRIORITY) 1493 continue; /* Let kswapd poll it */ 1494 sc->all_unreclaimable = 0; 1495 } else { 1496 /* 1497 * Ignore cpuset limitation here. We just want to reduce 1498 * # of used pages by us regardless of memory shortage. 1499 */ 1500 sc->all_unreclaimable = 0; 1501 mem_cgroup_note_reclaim_priority(sc->mem_cgroup, 1502 priority); 1503 } 1504 1505 nr_reclaimed += shrink_zone(priority, zone, sc); 1506 } 1507 1508 return nr_reclaimed; 1509} 1510 1511/* 1512 * This is the main entry point to direct page reclaim. 1513 * 1514 * If a full scan of the inactive list fails to free enough memory then we 1515 * are "out of memory" and something needs to be killed. 1516 * 1517 * If the caller is !__GFP_FS then the probability of a failure is reasonably 1518 * high - the zone may be full of dirty or under-writeback pages, which this 1519 * caller can't do much about. We kick pdflush and take explicit naps in the 1520 * hope that some of these pages can be written. But if the allocating task 1521 * holds filesystem locks which prevent writeout this might not work, and the 1522 * allocation attempt will fail. 1523 * 1524 * returns: 0, if no pages reclaimed 1525 * else, the number of pages reclaimed 1526 */ 1527static unsigned long do_try_to_free_pages(struct zonelist *zonelist, 1528 struct scan_control *sc) 1529{ 1530 int priority; 1531 unsigned long ret = 0; 1532 unsigned long total_scanned = 0; 1533 unsigned long nr_reclaimed = 0; 1534 struct reclaim_state *reclaim_state = current->reclaim_state; 1535 unsigned long lru_pages = 0; 1536 struct zoneref *z; 1537 struct zone *zone; 1538 enum zone_type high_zoneidx = gfp_zone(sc->gfp_mask); 1539 1540 delayacct_freepages_start(); 1541 1542 if (scan_global_lru(sc)) 1543 count_vm_event(ALLOCSTALL); 1544 /* 1545 * mem_cgroup will not do shrink_slab. 1546 */ 1547 if (scan_global_lru(sc)) { 1548 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 1549 1550 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL)) 1551 continue; 1552 1553 lru_pages += zone_lru_pages(zone); 1554 } 1555 } 1556 1557 for (priority = DEF_PRIORITY; priority >= 0; priority--) { 1558 sc->nr_scanned = 0; 1559 if (!priority) 1560 disable_swap_token(); 1561 nr_reclaimed += shrink_zones(priority, zonelist, sc); 1562 /* 1563 * Don't shrink slabs when reclaiming memory from 1564 * over limit cgroups 1565 */ 1566 if (scan_global_lru(sc)) { 1567 shrink_slab(sc->nr_scanned, sc->gfp_mask, lru_pages); 1568 if (reclaim_state) { 1569 nr_reclaimed += reclaim_state->reclaimed_slab; 1570 reclaim_state->reclaimed_slab = 0; 1571 } 1572 } 1573 total_scanned += sc->nr_scanned; 1574 if (nr_reclaimed >= sc->swap_cluster_max) { 1575 ret = nr_reclaimed; 1576 goto out; 1577 } 1578 1579 /* 1580 * Try to write back as many pages as we just scanned. This 1581 * tends to cause slow streaming writers to write data to the 1582 * disk smoothly, at the dirtying rate, which is nice. But 1583 * that's undesirable in laptop mode, where we *want* lumpy 1584 * writeout. So in laptop mode, write out the whole world. 1585 */ 1586 if (total_scanned > sc->swap_cluster_max + 1587 sc->swap_cluster_max / 2) { 1588 wakeup_pdflush(laptop_mode ? 0 : total_scanned); 1589 sc->may_writepage = 1; 1590 } 1591 1592 /* Take a nap, wait for some writeback to complete */ 1593 if (sc->nr_scanned && priority < DEF_PRIORITY - 2) 1594 congestion_wait(WRITE, HZ/10); 1595 } 1596 /* top priority shrink_zones still had more to do? don't OOM, then */ 1597 if (!sc->all_unreclaimable && scan_global_lru(sc)) 1598 ret = nr_reclaimed; 1599out: 1600 /* 1601 * Now that we've scanned all the zones at this priority level, note 1602 * that level within the zone so that the next thread which performs 1603 * scanning of this zone will immediately start out at this priority 1604 * level. This affects only the decision whether or not to bring 1605 * mapped pages onto the inactive list. 1606 */ 1607 if (priority < 0) 1608 priority = 0; 1609 1610 if (scan_global_lru(sc)) { 1611 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 1612 1613 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL)) 1614 continue; 1615 1616 zone->prev_priority = priority; 1617 } 1618 } else 1619 mem_cgroup_record_reclaim_priority(sc->mem_cgroup, priority); 1620 1621 delayacct_freepages_end(); 1622 1623 return ret; 1624} 1625 1626unsigned long try_to_free_pages(struct zonelist *zonelist, int order, 1627 gfp_t gfp_mask) 1628{ 1629 struct scan_control sc = { 1630 .gfp_mask = gfp_mask, 1631 .may_writepage = !laptop_mode, 1632 .swap_cluster_max = SWAP_CLUSTER_MAX, 1633 .may_swap = 1, 1634 .swappiness = vm_swappiness, 1635 .order = order, 1636 .mem_cgroup = NULL, 1637 .isolate_pages = isolate_pages_global, 1638 }; 1639 1640 return do_try_to_free_pages(zonelist, &sc); 1641} 1642 1643#ifdef CONFIG_CGROUP_MEM_RES_CTLR 1644 1645unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont, 1646 gfp_t gfp_mask) 1647{ 1648 struct scan_control sc = { 1649 .may_writepage = !laptop_mode, 1650 .may_swap = 1, 1651 .swap_cluster_max = SWAP_CLUSTER_MAX, 1652 .swappiness = vm_swappiness, 1653 .order = 0, 1654 .mem_cgroup = mem_cont, 1655 .isolate_pages = mem_cgroup_isolate_pages, 1656 }; 1657 struct zonelist *zonelist; 1658 1659 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | 1660 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); 1661 zonelist = NODE_DATA(numa_node_id())->node_zonelists; 1662 return do_try_to_free_pages(zonelist, &sc); 1663} 1664#endif 1665 1666/* 1667 * For kswapd, balance_pgdat() will work across all this node's zones until 1668 * they are all at pages_high. 1669 * 1670 * Returns the number of pages which were actually freed. 1671 * 1672 * There is special handling here for zones which are full of pinned pages. 1673 * This can happen if the pages are all mlocked, or if they are all used by 1674 * device drivers (say, ZONE_DMA). Or if they are all in use by hugetlb. 1675 * What we do is to detect the case where all pages in the zone have been 1676 * scanned twice and there has been zero successful reclaim. Mark the zone as 1677 * dead and from now on, only perform a short scan. Basically we're polling 1678 * the zone for when the problem goes away. 1679 * 1680 * kswapd scans the zones in the highmem->normal->dma direction. It skips 1681 * zones which have free_pages > pages_high, but once a zone is found to have 1682 * free_pages <= pages_high, we scan that zone and the lower zones regardless 1683 * of the number of free pages in the lower zones. This interoperates with 1684 * the page allocator fallback scheme to ensure that aging of pages is balanced 1685 * across the zones. 1686 */ 1687static unsigned long balance_pgdat(pg_data_t *pgdat, int order) 1688{ 1689 int all_zones_ok; 1690 int priority; 1691 int i; 1692 unsigned long total_scanned; 1693 unsigned long nr_reclaimed; 1694 struct reclaim_state *reclaim_state = current->reclaim_state; 1695 struct scan_control sc = { 1696 .gfp_mask = GFP_KERNEL, 1697 .may_swap = 1, 1698 .swap_cluster_max = SWAP_CLUSTER_MAX, 1699 .swappiness = vm_swappiness, 1700 .order = order, 1701 .mem_cgroup = NULL, 1702 .isolate_pages = isolate_pages_global, 1703 }; 1704 /* 1705 * temp_priority is used to remember the scanning priority at which 1706 * this zone was successfully refilled to free_pages == pages_high. 1707 */ 1708 int temp_priority[MAX_NR_ZONES]; 1709 1710loop_again: 1711 total_scanned = 0; 1712 nr_reclaimed = 0; 1713 sc.may_writepage = !laptop_mode; 1714 count_vm_event(PAGEOUTRUN); 1715 1716 for (i = 0; i < pgdat->nr_zones; i++) 1717 temp_priority[i] = DEF_PRIORITY; 1718 1719 for (priority = DEF_PRIORITY; priority >= 0; priority--) { 1720 int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */ 1721 unsigned long lru_pages = 0; 1722 1723 /* The swap token gets in the way of swapout... */ 1724 if (!priority) 1725 disable_swap_token(); 1726 1727 all_zones_ok = 1; 1728 1729 /* 1730 * Scan in the highmem->dma direction for the highest 1731 * zone which needs scanning 1732 */ 1733 for (i = pgdat->nr_zones - 1; i >= 0; i--) { 1734 struct zone *zone = pgdat->node_zones + i; 1735 1736 if (!populated_zone(zone)) 1737 continue; 1738 1739 if (zone_is_all_unreclaimable(zone) && 1740 priority != DEF_PRIORITY) 1741 continue; 1742 1743 /* 1744 * Do some background aging of the anon list, to give 1745 * pages a chance to be referenced before reclaiming. 1746 */ 1747 if (inactive_anon_is_low(zone)) 1748 shrink_active_list(SWAP_CLUSTER_MAX, zone, 1749 &sc, priority, 0); 1750 1751 if (!zone_watermark_ok(zone, order, zone->pages_high, 1752 0, 0)) { 1753 end_zone = i; 1754 break; 1755 } 1756 } 1757 if (i < 0) 1758 goto out; 1759 1760 for (i = 0; i <= end_zone; i++) { 1761 struct zone *zone = pgdat->node_zones + i; 1762 1763 lru_pages += zone_lru_pages(zone); 1764 } 1765 1766 /* 1767 * Now scan the zone in the dma->highmem direction, stopping 1768 * at the last zone which needs scanning. 1769 * 1770 * We do this because the page allocator works in the opposite 1771 * direction. This prevents the page allocator from allocating 1772 * pages behind kswapd's direction of progress, which would 1773 * cause too much scanning of the lower zones. 1774 */ 1775 for (i = 0; i <= end_zone; i++) { 1776 struct zone *zone = pgdat->node_zones + i; 1777 int nr_slab; 1778 1779 if (!populated_zone(zone)) 1780 continue; 1781 1782 if (zone_is_all_unreclaimable(zone) && 1783 priority != DEF_PRIORITY) 1784 continue; 1785 1786 if (!zone_watermark_ok(zone, order, zone->pages_high, 1787 end_zone, 0)) 1788 all_zones_ok = 0; 1789 temp_priority[i] = priority; 1790 sc.nr_scanned = 0; 1791 note_zone_scanning_priority(zone, priority); 1792 /* 1793 * We put equal pressure on every zone, unless one 1794 * zone has way too many pages free already. 1795 */ 1796 if (!zone_watermark_ok(zone, order, 8*zone->pages_high, 1797 end_zone, 0)) 1798 nr_reclaimed += shrink_zone(priority, zone, &sc); 1799 reclaim_state->reclaimed_slab = 0; 1800 nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL, 1801 lru_pages); 1802 nr_reclaimed += reclaim_state->reclaimed_slab; 1803 total_scanned += sc.nr_scanned; 1804 if (zone_is_all_unreclaimable(zone)) 1805 continue; 1806 if (nr_slab == 0 && zone->pages_scanned >= 1807 (zone_lru_pages(zone) * 6)) 1808 zone_set_flag(zone, 1809 ZONE_ALL_UNRECLAIMABLE); 1810 /* 1811 * If we've done a decent amount of scanning and 1812 * the reclaim ratio is low, start doing writepage 1813 * even in laptop mode 1814 */ 1815 if (total_scanned > SWAP_CLUSTER_MAX * 2 && 1816 total_scanned > nr_reclaimed + nr_reclaimed / 2) 1817 sc.may_writepage = 1; 1818 } 1819 if (all_zones_ok) 1820 break; /* kswapd: all done */ 1821 /* 1822 * OK, kswapd is getting into trouble. Take a nap, then take 1823 * another pass across the zones. 1824 */ 1825 if (total_scanned && priority < DEF_PRIORITY - 2) 1826 congestion_wait(WRITE, HZ/10); 1827 1828 /* 1829 * We do this so kswapd doesn't build up large priorities for 1830 * example when it is freeing in parallel with allocators. It 1831 * matches the direct reclaim path behaviour in terms of impact 1832 * on zone->*_priority. 1833 */ 1834 if (nr_reclaimed >= SWAP_CLUSTER_MAX) 1835 break; 1836 } 1837out: 1838 /* 1839 * Note within each zone the priority level at which this zone was 1840 * brought into a happy state. So that the next thread which scans this 1841 * zone will start out at that priority level. 1842 */ 1843 for (i = 0; i < pgdat->nr_zones; i++) { 1844 struct zone *zone = pgdat->node_zones + i; 1845 1846 zone->prev_priority = temp_priority[i]; 1847 } 1848 if (!all_zones_ok) { 1849 cond_resched(); 1850 1851 try_to_freeze(); 1852 1853 goto loop_again; 1854 } 1855 1856 return nr_reclaimed; 1857} 1858 1859/* 1860 * The background pageout daemon, started as a kernel thread 1861 * from the init process. 1862 * 1863 * This basically trickles out pages so that we have _some_ 1864 * free memory available even if there is no other activity 1865 * that frees anything up. This is needed for things like routing 1866 * etc, where we otherwise might have all activity going on in 1867 * asynchronous contexts that cannot page things out. 1868 * 1869 * If there are applications that are active memory-allocators 1870 * (most normal use), this basically shouldn't matter. 1871 */ 1872static int kswapd(void *p) 1873{ 1874 unsigned long order; 1875 pg_data_t *pgdat = (pg_data_t*)p; 1876 struct task_struct *tsk = current; 1877 DEFINE_WAIT(wait); 1878 struct reclaim_state reclaim_state = { 1879 .reclaimed_slab = 0, 1880 }; 1881 node_to_cpumask_ptr(cpumask, pgdat->node_id); 1882 1883 if (!cpus_empty(*cpumask)) 1884 set_cpus_allowed_ptr(tsk, cpumask); 1885 current->reclaim_state = &reclaim_state; 1886 1887 /* 1888 * Tell the memory management that we're a "memory allocator", 1889 * and that if we need more memory we should get access to it 1890 * regardless (see "__alloc_pages()"). "kswapd" should 1891 * never get caught in the normal page freeing logic. 1892 * 1893 * (Kswapd normally doesn't need memory anyway, but sometimes 1894 * you need a small amount of memory in order to be able to 1895 * page out something else, and this flag essentially protects 1896 * us from recursively trying to free more memory as we're 1897 * trying to free the first piece of memory in the first place). 1898 */ 1899 tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD; 1900 set_freezable(); 1901 1902 order = 0; 1903 for ( ; ; ) { 1904 unsigned long new_order; 1905 1906 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 1907 new_order = pgdat->kswapd_max_order; 1908 pgdat->kswapd_max_order = 0; 1909 if (order < new_order) { 1910 /* 1911 * Don't sleep if someone wants a larger 'order' 1912 * allocation 1913 */ 1914 order = new_order; 1915 } else { 1916 if (!freezing(current)) 1917 schedule(); 1918 1919 order = pgdat->kswapd_max_order; 1920 } 1921 finish_wait(&pgdat->kswapd_wait, &wait); 1922 1923 if (!try_to_freeze()) { 1924 /* We can speed up thawing tasks if we don't call 1925 * balance_pgdat after returning from the refrigerator 1926 */ 1927 balance_pgdat(pgdat, order); 1928 } 1929 } 1930 return 0; 1931} 1932 1933/* 1934 * A zone is low on free memory, so wake its kswapd task to service it. 1935 */ 1936void wakeup_kswapd(struct zone *zone, int order) 1937{ 1938 pg_data_t *pgdat; 1939 1940 if (!populated_zone(zone)) 1941 return; 1942 1943 pgdat = zone->zone_pgdat; 1944 if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0)) 1945 return; 1946 if (pgdat->kswapd_max_order < order) 1947 pgdat->kswapd_max_order = order; 1948 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL)) 1949 return; 1950 if (!waitqueue_active(&pgdat->kswapd_wait)) 1951 return; 1952 wake_up_interruptible(&pgdat->kswapd_wait); 1953} 1954 1955unsigned long global_lru_pages(void) 1956{ 1957 return global_page_state(NR_ACTIVE_ANON) 1958 + global_page_state(NR_ACTIVE_FILE) 1959 + global_page_state(NR_INACTIVE_ANON) 1960 + global_page_state(NR_INACTIVE_FILE); 1961} 1962 1963#ifdef CONFIG_PM 1964/* 1965 * Helper function for shrink_all_memory(). Tries to reclaim 'nr_pages' pages 1966 * from LRU lists system-wide, for given pass and priority, and returns the 1967 * number of reclaimed pages 1968 * 1969 * For pass > 3 we also try to shrink the LRU lists that contain a few pages 1970 */ 1971static unsigned long shrink_all_zones(unsigned long nr_pages, int prio, 1972 int pass, struct scan_control *sc) 1973{ 1974 struct zone *zone; 1975 unsigned long nr_to_scan, ret = 0; 1976 enum lru_list l; 1977 1978 for_each_zone(zone) { 1979 1980 if (!populated_zone(zone)) 1981 continue; 1982 1983 if (zone_is_all_unreclaimable(zone) && prio != DEF_PRIORITY) 1984 continue; 1985 1986 for_each_evictable_lru(l) { 1987 /* For pass = 0, we don't shrink the active list */ 1988 if (pass == 0 && 1989 (l == LRU_ACTIVE || l == LRU_ACTIVE_FILE)) 1990 continue; 1991 1992 zone->lru[l].nr_scan += 1993 (zone_page_state(zone, NR_LRU_BASE + l) 1994 >> prio) + 1; 1995 if (zone->lru[l].nr_scan >= nr_pages || pass > 3) { 1996 zone->lru[l].nr_scan = 0; 1997 nr_to_scan = min(nr_pages, 1998 zone_page_state(zone, 1999 NR_LRU_BASE + l)); 2000 ret += shrink_list(l, nr_to_scan, zone, 2001 sc, prio); 2002 if (ret >= nr_pages) 2003 return ret; 2004 } 2005 } 2006 } 2007 2008 return ret; 2009} 2010 2011/* 2012 * Try to free `nr_pages' of memory, system-wide, and return the number of 2013 * freed pages. 2014 * 2015 * Rather than trying to age LRUs the aim is to preserve the overall 2016 * LRU order by reclaiming preferentially 2017 * inactive > active > active referenced > active mapped 2018 */ 2019unsigned long shrink_all_memory(unsigned long nr_pages) 2020{ 2021 unsigned long lru_pages, nr_slab; 2022 unsigned long ret = 0; 2023 int pass; 2024 struct reclaim_state reclaim_state; 2025 struct scan_control sc = { 2026 .gfp_mask = GFP_KERNEL, 2027 .may_swap = 0, 2028 .swap_cluster_max = nr_pages, 2029 .may_writepage = 1, 2030 .swappiness = vm_swappiness, 2031 .isolate_pages = isolate_pages_global, 2032 }; 2033 2034 current->reclaim_state = &reclaim_state; 2035 2036 lru_pages = global_lru_pages(); 2037 nr_slab = global_page_state(NR_SLAB_RECLAIMABLE); 2038 /* If slab caches are huge, it's better to hit them first */ 2039 while (nr_slab >= lru_pages) { 2040 reclaim_state.reclaimed_slab = 0; 2041 shrink_slab(nr_pages, sc.gfp_mask, lru_pages); 2042 if (!reclaim_state.reclaimed_slab) 2043 break; 2044 2045 ret += reclaim_state.reclaimed_slab; 2046 if (ret >= nr_pages) 2047 goto out; 2048 2049 nr_slab -= reclaim_state.reclaimed_slab; 2050 } 2051 2052 /* 2053 * We try to shrink LRUs in 5 passes: 2054 * 0 = Reclaim from inactive_list only 2055 * 1 = Reclaim from active list but don't reclaim mapped 2056 * 2 = 2nd pass of type 1 2057 * 3 = Reclaim mapped (normal reclaim) 2058 * 4 = 2nd pass of type 3 2059 */ 2060 for (pass = 0; pass < 5; pass++) { 2061 int prio; 2062 2063 /* Force reclaiming mapped pages in the passes #3 and #4 */ 2064 if (pass > 2) { 2065 sc.may_swap = 1; 2066 sc.swappiness = 100; 2067 } 2068 2069 for (prio = DEF_PRIORITY; prio >= 0; prio--) { 2070 unsigned long nr_to_scan = nr_pages - ret; 2071 2072 sc.nr_scanned = 0; 2073 ret += shrink_all_zones(nr_to_scan, prio, pass, &sc); 2074 if (ret >= nr_pages) 2075 goto out; 2076 2077 reclaim_state.reclaimed_slab = 0; 2078 shrink_slab(sc.nr_scanned, sc.gfp_mask, 2079 global_lru_pages()); 2080 ret += reclaim_state.reclaimed_slab; 2081 if (ret >= nr_pages) 2082 goto out; 2083 2084 if (sc.nr_scanned && prio < DEF_PRIORITY - 2) 2085 congestion_wait(WRITE, HZ / 10); 2086 } 2087 } 2088 2089 /* 2090 * If ret = 0, we could not shrink LRUs, but there may be something 2091 * in slab caches 2092 */ 2093 if (!ret) { 2094 do { 2095 reclaim_state.reclaimed_slab = 0; 2096 shrink_slab(nr_pages, sc.gfp_mask, global_lru_pages()); 2097 ret += reclaim_state.reclaimed_slab; 2098 } while (ret < nr_pages && reclaim_state.reclaimed_slab > 0); 2099 } 2100 2101out: 2102 current->reclaim_state = NULL; 2103 2104 return ret; 2105} 2106#endif 2107 2108/* It's optimal to keep kswapds on the same CPUs as their memory, but 2109 not required for correctness. So if the last cpu in a node goes 2110 away, we get changed to run anywhere: as the first one comes back, 2111 restore their cpu bindings. */ 2112static int __devinit cpu_callback(struct notifier_block *nfb, 2113 unsigned long action, void *hcpu) 2114{ 2115 int nid; 2116 2117 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) { 2118 for_each_node_state(nid, N_HIGH_MEMORY) { 2119 pg_data_t *pgdat = NODE_DATA(nid); 2120 node_to_cpumask_ptr(mask, pgdat->node_id); 2121 2122 if (any_online_cpu(*mask) < nr_cpu_ids) 2123 /* One of our CPUs online: restore mask */ 2124 set_cpus_allowed_ptr(pgdat->kswapd, mask); 2125 } 2126 } 2127 return NOTIFY_OK; 2128} 2129 2130/* 2131 * This kswapd start function will be called by init and node-hot-add. 2132 * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added. 2133 */ 2134int kswapd_run(int nid) 2135{ 2136 pg_data_t *pgdat = NODE_DATA(nid); 2137 int ret = 0; 2138 2139 if (pgdat->kswapd) 2140 return 0; 2141 2142 pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); 2143 if (IS_ERR(pgdat->kswapd)) { 2144 /* failure at boot is fatal */ 2145 BUG_ON(system_state == SYSTEM_BOOTING); 2146 printk("Failed to start kswapd on node %d\n",nid); 2147 ret = -1; 2148 } 2149 return ret; 2150} 2151 2152static int __init kswapd_init(void) 2153{ 2154 int nid; 2155 2156 swap_setup(); 2157 for_each_node_state(nid, N_HIGH_MEMORY) 2158 kswapd_run(nid); 2159 hotcpu_notifier(cpu_callback, 0); 2160 return 0; 2161} 2162 2163module_init(kswapd_init) 2164 2165#ifdef CONFIG_NUMA 2166/* 2167 * Zone reclaim mode 2168 * 2169 * If non-zero call zone_reclaim when the number of free pages falls below 2170 * the watermarks. 2171 */ 2172int zone_reclaim_mode __read_mostly; 2173 2174#define RECLAIM_OFF 0 2175#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */ 2176#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */ 2177#define RECLAIM_SWAP (1<<2) /* Swap pages out during reclaim */ 2178 2179/* 2180 * Priority for ZONE_RECLAIM. This determines the fraction of pages 2181 * of a node considered for each zone_reclaim. 4 scans 1/16th of 2182 * a zone. 2183 */ 2184#define ZONE_RECLAIM_PRIORITY 4 2185 2186/* 2187 * Percentage of pages in a zone that must be unmapped for zone_reclaim to 2188 * occur. 2189 */ 2190int sysctl_min_unmapped_ratio = 1; 2191 2192/* 2193 * If the number of slab pages in a zone grows beyond this percentage then 2194 * slab reclaim needs to occur. 2195 */ 2196int sysctl_min_slab_ratio = 5; 2197 2198/* 2199 * Try to free up some pages from this zone through reclaim. 2200 */ 2201static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) 2202{ 2203 /* Minimum pages needed in order to stay on node */ 2204 const unsigned long nr_pages = 1 << order; 2205 struct task_struct *p = current; 2206 struct reclaim_state reclaim_state; 2207 int priority; 2208 unsigned long nr_reclaimed = 0; 2209 struct scan_control sc = { 2210 .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE), 2211 .may_swap = !!(zone_reclaim_mode & RECLAIM_SWAP), 2212 .swap_cluster_max = max_t(unsigned long, nr_pages, 2213 SWAP_CLUSTER_MAX), 2214 .gfp_mask = gfp_mask, 2215 .swappiness = vm_swappiness, 2216 .isolate_pages = isolate_pages_global, 2217 }; 2218 unsigned long slab_reclaimable; 2219 2220 disable_swap_token(); 2221 cond_resched(); 2222 /* 2223 * We need to be able to allocate from the reserves for RECLAIM_SWAP 2224 * and we also need to be able to write out pages for RECLAIM_WRITE 2225 * and RECLAIM_SWAP. 2226 */ 2227 p->flags |= PF_MEMALLOC | PF_SWAPWRITE; 2228 reclaim_state.reclaimed_slab = 0; 2229 p->reclaim_state = &reclaim_state; 2230 2231 if (zone_page_state(zone, NR_FILE_PAGES) - 2232 zone_page_state(zone, NR_FILE_MAPPED) > 2233 zone->min_unmapped_pages) { 2234 /* 2235 * Free memory by calling shrink zone with increasing 2236 * priorities until we have enough memory freed. 2237 */ 2238 priority = ZONE_RECLAIM_PRIORITY; 2239 do { 2240 note_zone_scanning_priority(zone, priority); 2241 nr_reclaimed += shrink_zone(priority, zone, &sc); 2242 priority--; 2243 } while (priority >= 0 && nr_reclaimed < nr_pages); 2244 } 2245 2246 slab_reclaimable = zone_page_state(zone, NR_SLAB_RECLAIMABLE); 2247 if (slab_reclaimable > zone->min_slab_pages) { 2248 /* 2249 * shrink_slab() does not currently allow us to determine how 2250 * many pages were freed in this zone. So we take the current 2251 * number of slab pages and shake the slab until it is reduced 2252 * by the same nr_pages that we used for reclaiming unmapped 2253 * pages. 2254 * 2255 * Note that shrink_slab will free memory on all zones and may 2256 * take a long time. 2257 */ 2258 while (shrink_slab(sc.nr_scanned, gfp_mask, order) && 2259 zone_page_state(zone, NR_SLAB_RECLAIMABLE) > 2260 slab_reclaimable - nr_pages) 2261 ; 2262 2263 /* 2264 * Update nr_reclaimed by the number of slab pages we 2265 * reclaimed from this zone. 2266 */ 2267 nr_reclaimed += slab_reclaimable - 2268 zone_page_state(zone, NR_SLAB_RECLAIMABLE); 2269 } 2270 2271 p->reclaim_state = NULL; 2272 current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE); 2273 return nr_reclaimed >= nr_pages; 2274} 2275 2276int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) 2277{ 2278 int node_id; 2279 int ret; 2280 2281 /* 2282 * Zone reclaim reclaims unmapped file backed pages and 2283 * slab pages if we are over the defined limits. 2284 * 2285 * A small portion of unmapped file backed pages is needed for 2286 * file I/O otherwise pages read by file I/O will be immediately 2287 * thrown out if the zone is overallocated. So we do not reclaim 2288 * if less than a specified percentage of the zone is used by 2289 * unmapped file backed pages. 2290 */ 2291 if (zone_page_state(zone, NR_FILE_PAGES) - 2292 zone_page_state(zone, NR_FILE_MAPPED) <= zone->min_unmapped_pages 2293 && zone_page_state(zone, NR_SLAB_RECLAIMABLE) 2294 <= zone->min_slab_pages) 2295 return 0; 2296 2297 if (zone_is_all_unreclaimable(zone)) 2298 return 0; 2299 2300 /* 2301 * Do not scan if the allocation should not be delayed. 2302 */ 2303 if (!(gfp_mask & __GFP_WAIT) || (current->flags & PF_MEMALLOC)) 2304 return 0; 2305 2306 /* 2307 * Only run zone reclaim on the local zone or on zones that do not 2308 * have associated processors. This will favor the local processor 2309 * over remote processors and spread off node memory allocations 2310 * as wide as possible. 2311 */ 2312 node_id = zone_to_nid(zone); 2313 if (node_state(node_id, N_CPU) && node_id != numa_node_id()) 2314 return 0; 2315 2316 if (zone_test_and_set_flag(zone, ZONE_RECLAIM_LOCKED)) 2317 return 0; 2318 ret = __zone_reclaim(zone, gfp_mask, order); 2319 zone_clear_flag(zone, ZONE_RECLAIM_LOCKED); 2320 2321 return ret; 2322} 2323#endif 2324 2325#ifdef CONFIG_UNEVICTABLE_LRU 2326/* 2327 * page_evictable - test whether a page is evictable 2328 * @page: the page to test 2329 * @vma: the VMA in which the page is or will be mapped, may be NULL 2330 * 2331 * Test whether page is evictable--i.e., should be placed on active/inactive 2332 * lists vs unevictable list. 2333 * 2334 * Reasons page might not be evictable: 2335 * TODO - later patches 2336 */ 2337int page_evictable(struct page *page, struct vm_area_struct *vma) 2338{ 2339 2340 /* TODO: test page [!]evictable conditions */ 2341 2342 return 1; 2343} 2344#endif 2345