swap.c revision 315601809d124d046abd6c3ffa346d0dbd7aa29d
1/* 2 * linux/mm/swap.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7/* 8 * This file contains the default values for the operation of the 9 * Linux VM subsystem. Fine-tuning documentation can be found in 10 * Documentation/sysctl/vm.txt. 11 * Started 18.12.91 12 * Swap aging added 23.2.95, Stephen Tweedie. 13 * Buffermem limits added 12.3.98, Rik van Riel. 14 */ 15 16#include <linux/mm.h> 17#include <linux/sched.h> 18#include <linux/kernel_stat.h> 19#include <linux/swap.h> 20#include <linux/mman.h> 21#include <linux/pagemap.h> 22#include <linux/pagevec.h> 23#include <linux/init.h> 24#include <linux/module.h> 25#include <linux/mm_inline.h> 26#include <linux/buffer_head.h> /* for try_to_release_page() */ 27#include <linux/percpu_counter.h> 28#include <linux/percpu.h> 29#include <linux/cpu.h> 30#include <linux/notifier.h> 31#include <linux/backing-dev.h> 32#include <linux/memcontrol.h> 33#include <linux/gfp.h> 34 35#include "internal.h" 36 37/* How many pages do we try to swap or page in/out together? */ 38int page_cluster; 39 40static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs); 41static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); 42static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs); 43 44/* 45 * This path almost never happens for VM activity - pages are normally 46 * freed via pagevecs. But it gets used by networking. 47 */ 48static void __page_cache_release(struct page *page) 49{ 50 if (PageLRU(page)) { 51 unsigned long flags; 52 struct zone *zone = page_zone(page); 53 54 spin_lock_irqsave(&zone->lru_lock, flags); 55 VM_BUG_ON(!PageLRU(page)); 56 __ClearPageLRU(page); 57 del_page_from_lru(zone, page); 58 spin_unlock_irqrestore(&zone->lru_lock, flags); 59 } 60} 61 62static void __put_single_page(struct page *page) 63{ 64 __page_cache_release(page); 65 free_hot_cold_page(page, 0); 66} 67 68static void __put_compound_page(struct page *page) 69{ 70 compound_page_dtor *dtor; 71 72 __page_cache_release(page); 73 dtor = get_compound_page_dtor(page); 74 (*dtor)(page); 75} 76 77static void put_compound_page(struct page *page) 78{ 79 if (unlikely(PageTail(page))) { 80 /* __split_huge_page_refcount can run under us */ 81 struct page *page_head = page->first_page; 82 smp_rmb(); 83 /* 84 * If PageTail is still set after smp_rmb() we can be sure 85 * that the page->first_page we read wasn't a dangling pointer. 86 * See __split_huge_page_refcount() smp_wmb(). 87 */ 88 if (likely(PageTail(page) && get_page_unless_zero(page_head))) { 89 unsigned long flags; 90 /* 91 * Verify that our page_head wasn't converted 92 * to a a regular page before we got a 93 * reference on it. 94 */ 95 if (unlikely(!PageHead(page_head))) { 96 /* PageHead is cleared after PageTail */ 97 smp_rmb(); 98 VM_BUG_ON(PageTail(page)); 99 goto out_put_head; 100 } 101 /* 102 * Only run compound_lock on a valid PageHead, 103 * after having it pinned with 104 * get_page_unless_zero() above. 105 */ 106 smp_mb(); 107 /* page_head wasn't a dangling pointer */ 108 flags = compound_lock_irqsave(page_head); 109 if (unlikely(!PageTail(page))) { 110 /* __split_huge_page_refcount run before us */ 111 compound_unlock_irqrestore(page_head, flags); 112 VM_BUG_ON(PageHead(page_head)); 113 out_put_head: 114 if (put_page_testzero(page_head)) 115 __put_single_page(page_head); 116 out_put_single: 117 if (put_page_testzero(page)) 118 __put_single_page(page); 119 return; 120 } 121 VM_BUG_ON(page_head != page->first_page); 122 /* 123 * We can release the refcount taken by 124 * get_page_unless_zero now that 125 * split_huge_page_refcount is blocked on the 126 * compound_lock. 127 */ 128 if (put_page_testzero(page_head)) 129 VM_BUG_ON(1); 130 /* __split_huge_page_refcount will wait now */ 131 VM_BUG_ON(atomic_read(&page->_count) <= 0); 132 atomic_dec(&page->_count); 133 VM_BUG_ON(atomic_read(&page_head->_count) <= 0); 134 compound_unlock_irqrestore(page_head, flags); 135 if (put_page_testzero(page_head)) { 136 if (PageHead(page_head)) 137 __put_compound_page(page_head); 138 else 139 __put_single_page(page_head); 140 } 141 } else { 142 /* page_head is a dangling pointer */ 143 VM_BUG_ON(PageTail(page)); 144 goto out_put_single; 145 } 146 } else if (put_page_testzero(page)) { 147 if (PageHead(page)) 148 __put_compound_page(page); 149 else 150 __put_single_page(page); 151 } 152} 153 154void put_page(struct page *page) 155{ 156 if (unlikely(PageCompound(page))) 157 put_compound_page(page); 158 else if (put_page_testzero(page)) 159 __put_single_page(page); 160} 161EXPORT_SYMBOL(put_page); 162 163/** 164 * put_pages_list() - release a list of pages 165 * @pages: list of pages threaded on page->lru 166 * 167 * Release a list of pages which are strung together on page.lru. Currently 168 * used by read_cache_pages() and related error recovery code. 169 */ 170void put_pages_list(struct list_head *pages) 171{ 172 while (!list_empty(pages)) { 173 struct page *victim; 174 175 victim = list_entry(pages->prev, struct page, lru); 176 list_del(&victim->lru); 177 page_cache_release(victim); 178 } 179} 180EXPORT_SYMBOL(put_pages_list); 181 182/* 183 * pagevec_move_tail() must be called with IRQ disabled. 184 * Otherwise this may cause nasty races. 185 */ 186static void pagevec_move_tail(struct pagevec *pvec) 187{ 188 int i; 189 int pgmoved = 0; 190 struct zone *zone = NULL; 191 192 for (i = 0; i < pagevec_count(pvec); i++) { 193 struct page *page = pvec->pages[i]; 194 struct zone *pagezone = page_zone(page); 195 196 if (pagezone != zone) { 197 if (zone) 198 spin_unlock(&zone->lru_lock); 199 zone = pagezone; 200 spin_lock(&zone->lru_lock); 201 } 202 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 203 int lru = page_lru_base_type(page); 204 list_move_tail(&page->lru, &zone->lru[lru].list); 205 pgmoved++; 206 } 207 } 208 if (zone) 209 spin_unlock(&zone->lru_lock); 210 __count_vm_events(PGROTATED, pgmoved); 211 release_pages(pvec->pages, pvec->nr, pvec->cold); 212 pagevec_reinit(pvec); 213} 214 215/* 216 * Writeback is about to end against a page which has been marked for immediate 217 * reclaim. If it still appears to be reclaimable, move it to the tail of the 218 * inactive list. 219 */ 220void rotate_reclaimable_page(struct page *page) 221{ 222 if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) && 223 !PageUnevictable(page) && PageLRU(page)) { 224 struct pagevec *pvec; 225 unsigned long flags; 226 227 page_cache_get(page); 228 local_irq_save(flags); 229 pvec = &__get_cpu_var(lru_rotate_pvecs); 230 if (!pagevec_add(pvec, page)) 231 pagevec_move_tail(pvec); 232 local_irq_restore(flags); 233 } 234} 235 236static void update_page_reclaim_stat(struct zone *zone, struct page *page, 237 int file, int rotated) 238{ 239 struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat; 240 struct zone_reclaim_stat *memcg_reclaim_stat; 241 242 memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page); 243 244 reclaim_stat->recent_scanned[file]++; 245 if (rotated) 246 reclaim_stat->recent_rotated[file]++; 247 248 if (!memcg_reclaim_stat) 249 return; 250 251 memcg_reclaim_stat->recent_scanned[file]++; 252 if (rotated) 253 memcg_reclaim_stat->recent_rotated[file]++; 254} 255 256/* 257 * FIXME: speed this up? 258 */ 259void activate_page(struct page *page) 260{ 261 struct zone *zone = page_zone(page); 262 263 spin_lock_irq(&zone->lru_lock); 264 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 265 int file = page_is_file_cache(page); 266 int lru = page_lru_base_type(page); 267 del_page_from_lru_list(zone, page, lru); 268 269 SetPageActive(page); 270 lru += LRU_ACTIVE; 271 add_page_to_lru_list(zone, page, lru); 272 __count_vm_event(PGACTIVATE); 273 274 update_page_reclaim_stat(zone, page, file, 1); 275 } 276 spin_unlock_irq(&zone->lru_lock); 277} 278 279/* 280 * Mark a page as having seen activity. 281 * 282 * inactive,unreferenced -> inactive,referenced 283 * inactive,referenced -> active,unreferenced 284 * active,unreferenced -> active,referenced 285 */ 286void mark_page_accessed(struct page *page) 287{ 288 if (!PageActive(page) && !PageUnevictable(page) && 289 PageReferenced(page) && PageLRU(page)) { 290 activate_page(page); 291 ClearPageReferenced(page); 292 } else if (!PageReferenced(page)) { 293 SetPageReferenced(page); 294 } 295} 296 297EXPORT_SYMBOL(mark_page_accessed); 298 299void __lru_cache_add(struct page *page, enum lru_list lru) 300{ 301 struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru]; 302 303 page_cache_get(page); 304 if (!pagevec_add(pvec, page)) 305 ____pagevec_lru_add(pvec, lru); 306 put_cpu_var(lru_add_pvecs); 307} 308EXPORT_SYMBOL(__lru_cache_add); 309 310/** 311 * lru_cache_add_lru - add a page to a page list 312 * @page: the page to be added to the LRU. 313 * @lru: the LRU list to which the page is added. 314 */ 315void lru_cache_add_lru(struct page *page, enum lru_list lru) 316{ 317 if (PageActive(page)) { 318 VM_BUG_ON(PageUnevictable(page)); 319 ClearPageActive(page); 320 } else if (PageUnevictable(page)) { 321 VM_BUG_ON(PageActive(page)); 322 ClearPageUnevictable(page); 323 } 324 325 VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page)); 326 __lru_cache_add(page, lru); 327} 328 329/** 330 * add_page_to_unevictable_list - add a page to the unevictable list 331 * @page: the page to be added to the unevictable list 332 * 333 * Add page directly to its zone's unevictable list. To avoid races with 334 * tasks that might be making the page evictable, through eg. munlock, 335 * munmap or exit, while it's not on the lru, we want to add the page 336 * while it's locked or otherwise "invisible" to other tasks. This is 337 * difficult to do when using the pagevec cache, so bypass that. 338 */ 339void add_page_to_unevictable_list(struct page *page) 340{ 341 struct zone *zone = page_zone(page); 342 343 spin_lock_irq(&zone->lru_lock); 344 SetPageUnevictable(page); 345 SetPageLRU(page); 346 add_page_to_lru_list(zone, page, LRU_UNEVICTABLE); 347 spin_unlock_irq(&zone->lru_lock); 348} 349 350/* 351 * If the page can not be invalidated, it is moved to the 352 * inactive list to speed up its reclaim. It is moved to the 353 * head of the list, rather than the tail, to give the flusher 354 * threads some time to write it out, as this is much more 355 * effective than the single-page writeout from reclaim. 356 */ 357static void lru_deactivate(struct page *page, struct zone *zone) 358{ 359 int lru, file; 360 361 if (!PageLRU(page) || !PageActive(page)) 362 return; 363 364 /* Some processes are using the page */ 365 if (page_mapped(page)) 366 return; 367 368 file = page_is_file_cache(page); 369 lru = page_lru_base_type(page); 370 del_page_from_lru_list(zone, page, lru + LRU_ACTIVE); 371 ClearPageActive(page); 372 ClearPageReferenced(page); 373 add_page_to_lru_list(zone, page, lru); 374 __count_vm_event(PGDEACTIVATE); 375 376 update_page_reclaim_stat(zone, page, file, 0); 377} 378 379static void ____pagevec_lru_deactivate(struct pagevec *pvec) 380{ 381 int i; 382 struct zone *zone = NULL; 383 384 for (i = 0; i < pagevec_count(pvec); i++) { 385 struct page *page = pvec->pages[i]; 386 struct zone *pagezone = page_zone(page); 387 388 if (pagezone != zone) { 389 if (zone) 390 spin_unlock_irq(&zone->lru_lock); 391 zone = pagezone; 392 spin_lock_irq(&zone->lru_lock); 393 } 394 lru_deactivate(page, zone); 395 } 396 if (zone) 397 spin_unlock_irq(&zone->lru_lock); 398 399 release_pages(pvec->pages, pvec->nr, pvec->cold); 400 pagevec_reinit(pvec); 401} 402 403 404/* 405 * Drain pages out of the cpu's pagevecs. 406 * Either "cpu" is the current CPU, and preemption has already been 407 * disabled; or "cpu" is being hot-unplugged, and is already dead. 408 */ 409static void drain_cpu_pagevecs(int cpu) 410{ 411 struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu); 412 struct pagevec *pvec; 413 int lru; 414 415 for_each_lru(lru) { 416 pvec = &pvecs[lru - LRU_BASE]; 417 if (pagevec_count(pvec)) 418 ____pagevec_lru_add(pvec, lru); 419 } 420 421 pvec = &per_cpu(lru_rotate_pvecs, cpu); 422 if (pagevec_count(pvec)) { 423 unsigned long flags; 424 425 /* No harm done if a racing interrupt already did this */ 426 local_irq_save(flags); 427 pagevec_move_tail(pvec); 428 local_irq_restore(flags); 429 } 430 431 pvec = &per_cpu(lru_deactivate_pvecs, cpu); 432 if (pagevec_count(pvec)) 433 ____pagevec_lru_deactivate(pvec); 434} 435 436/** 437 * deactivate_page - forcefully deactivate a page 438 * @page: page to deactivate 439 * 440 * This function hints the VM that @page is a good reclaim candidate, 441 * for example if its invalidation fails due to the page being dirty 442 * or under writeback. 443 */ 444void deactivate_page(struct page *page) 445{ 446 if (likely(get_page_unless_zero(page))) { 447 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs); 448 449 if (!pagevec_add(pvec, page)) 450 ____pagevec_lru_deactivate(pvec); 451 put_cpu_var(lru_deactivate_pvecs); 452 } 453} 454 455void lru_add_drain(void) 456{ 457 drain_cpu_pagevecs(get_cpu()); 458 put_cpu(); 459} 460 461static void lru_add_drain_per_cpu(struct work_struct *dummy) 462{ 463 lru_add_drain(); 464} 465 466/* 467 * Returns 0 for success 468 */ 469int lru_add_drain_all(void) 470{ 471 return schedule_on_each_cpu(lru_add_drain_per_cpu); 472} 473 474/* 475 * Batched page_cache_release(). Decrement the reference count on all the 476 * passed pages. If it fell to zero then remove the page from the LRU and 477 * free it. 478 * 479 * Avoid taking zone->lru_lock if possible, but if it is taken, retain it 480 * for the remainder of the operation. 481 * 482 * The locking in this function is against shrink_inactive_list(): we recheck 483 * the page count inside the lock to see whether shrink_inactive_list() 484 * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() 485 * will free it. 486 */ 487void release_pages(struct page **pages, int nr, int cold) 488{ 489 int i; 490 struct pagevec pages_to_free; 491 struct zone *zone = NULL; 492 unsigned long uninitialized_var(flags); 493 494 pagevec_init(&pages_to_free, cold); 495 for (i = 0; i < nr; i++) { 496 struct page *page = pages[i]; 497 498 if (unlikely(PageCompound(page))) { 499 if (zone) { 500 spin_unlock_irqrestore(&zone->lru_lock, flags); 501 zone = NULL; 502 } 503 put_compound_page(page); 504 continue; 505 } 506 507 if (!put_page_testzero(page)) 508 continue; 509 510 if (PageLRU(page)) { 511 struct zone *pagezone = page_zone(page); 512 513 if (pagezone != zone) { 514 if (zone) 515 spin_unlock_irqrestore(&zone->lru_lock, 516 flags); 517 zone = pagezone; 518 spin_lock_irqsave(&zone->lru_lock, flags); 519 } 520 VM_BUG_ON(!PageLRU(page)); 521 __ClearPageLRU(page); 522 del_page_from_lru(zone, page); 523 } 524 525 if (!pagevec_add(&pages_to_free, page)) { 526 if (zone) { 527 spin_unlock_irqrestore(&zone->lru_lock, flags); 528 zone = NULL; 529 } 530 __pagevec_free(&pages_to_free); 531 pagevec_reinit(&pages_to_free); 532 } 533 } 534 if (zone) 535 spin_unlock_irqrestore(&zone->lru_lock, flags); 536 537 pagevec_free(&pages_to_free); 538} 539EXPORT_SYMBOL(release_pages); 540 541/* 542 * The pages which we're about to release may be in the deferred lru-addition 543 * queues. That would prevent them from really being freed right now. That's 544 * OK from a correctness point of view but is inefficient - those pages may be 545 * cache-warm and we want to give them back to the page allocator ASAP. 546 * 547 * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 548 * and __pagevec_lru_add_active() call release_pages() directly to avoid 549 * mutual recursion. 550 */ 551void __pagevec_release(struct pagevec *pvec) 552{ 553 lru_add_drain(); 554 release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 555 pagevec_reinit(pvec); 556} 557 558EXPORT_SYMBOL(__pagevec_release); 559 560/* used by __split_huge_page_refcount() */ 561void lru_add_page_tail(struct zone* zone, 562 struct page *page, struct page *page_tail) 563{ 564 int active; 565 enum lru_list lru; 566 const int file = 0; 567 struct list_head *head; 568 569 VM_BUG_ON(!PageHead(page)); 570 VM_BUG_ON(PageCompound(page_tail)); 571 VM_BUG_ON(PageLRU(page_tail)); 572 VM_BUG_ON(!spin_is_locked(&zone->lru_lock)); 573 574 SetPageLRU(page_tail); 575 576 if (page_evictable(page_tail, NULL)) { 577 if (PageActive(page)) { 578 SetPageActive(page_tail); 579 active = 1; 580 lru = LRU_ACTIVE_ANON; 581 } else { 582 active = 0; 583 lru = LRU_INACTIVE_ANON; 584 } 585 update_page_reclaim_stat(zone, page_tail, file, active); 586 if (likely(PageLRU(page))) 587 head = page->lru.prev; 588 else 589 head = &zone->lru[lru].list; 590 __add_page_to_lru_list(zone, page_tail, lru, head); 591 } else { 592 SetPageUnevictable(page_tail); 593 add_page_to_lru_list(zone, page_tail, LRU_UNEVICTABLE); 594 } 595} 596 597/* 598 * Add the passed pages to the LRU, then drop the caller's refcount 599 * on them. Reinitialises the caller's pagevec. 600 */ 601void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru) 602{ 603 int i; 604 struct zone *zone = NULL; 605 606 VM_BUG_ON(is_unevictable_lru(lru)); 607 608 for (i = 0; i < pagevec_count(pvec); i++) { 609 struct page *page = pvec->pages[i]; 610 struct zone *pagezone = page_zone(page); 611 int file; 612 int active; 613 614 if (pagezone != zone) { 615 if (zone) 616 spin_unlock_irq(&zone->lru_lock); 617 zone = pagezone; 618 spin_lock_irq(&zone->lru_lock); 619 } 620 VM_BUG_ON(PageActive(page)); 621 VM_BUG_ON(PageUnevictable(page)); 622 VM_BUG_ON(PageLRU(page)); 623 SetPageLRU(page); 624 active = is_active_lru(lru); 625 file = is_file_lru(lru); 626 if (active) 627 SetPageActive(page); 628 update_page_reclaim_stat(zone, page, file, active); 629 add_page_to_lru_list(zone, page, lru); 630 } 631 if (zone) 632 spin_unlock_irq(&zone->lru_lock); 633 release_pages(pvec->pages, pvec->nr, pvec->cold); 634 pagevec_reinit(pvec); 635} 636 637EXPORT_SYMBOL(____pagevec_lru_add); 638 639/* 640 * Try to drop buffers from the pages in a pagevec 641 */ 642void pagevec_strip(struct pagevec *pvec) 643{ 644 int i; 645 646 for (i = 0; i < pagevec_count(pvec); i++) { 647 struct page *page = pvec->pages[i]; 648 649 if (page_has_private(page) && trylock_page(page)) { 650 if (page_has_private(page)) 651 try_to_release_page(page, 0); 652 unlock_page(page); 653 } 654 } 655} 656 657/** 658 * pagevec_lookup - gang pagecache lookup 659 * @pvec: Where the resulting pages are placed 660 * @mapping: The address_space to search 661 * @start: The starting page index 662 * @nr_pages: The maximum number of pages 663 * 664 * pagevec_lookup() will search for and return a group of up to @nr_pages pages 665 * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a 666 * reference against the pages in @pvec. 667 * 668 * The search returns a group of mapping-contiguous pages with ascending 669 * indexes. There may be holes in the indices due to not-present pages. 670 * 671 * pagevec_lookup() returns the number of pages which were found. 672 */ 673unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, 674 pgoff_t start, unsigned nr_pages) 675{ 676 pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); 677 return pagevec_count(pvec); 678} 679 680EXPORT_SYMBOL(pagevec_lookup); 681 682unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 683 pgoff_t *index, int tag, unsigned nr_pages) 684{ 685 pvec->nr = find_get_pages_tag(mapping, index, tag, 686 nr_pages, pvec->pages); 687 return pagevec_count(pvec); 688} 689 690EXPORT_SYMBOL(pagevec_lookup_tag); 691 692/* 693 * Perform any setup for the swap system 694 */ 695void __init swap_setup(void) 696{ 697 unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT); 698 699#ifdef CONFIG_SWAP 700 bdi_init(swapper_space.backing_dev_info); 701#endif 702 703 /* Use a smaller cluster for small-memory machines */ 704 if (megs < 16) 705 page_cluster = 2; 706 else 707 page_cluster = 3; 708 /* 709 * Right now other parts of the system means that we 710 * _really_ don't want to cluster much more 711 */ 712} 713