swap.c revision a95a82e96c48270980dd248ccd5546f1b49e6f8a
1/* 2 * linux/mm/swap.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7/* 8 * This file contains the default values for the operation of the 9 * Linux VM subsystem. Fine-tuning documentation can be found in 10 * Documentation/sysctl/vm.txt. 11 * Started 18.12.91 12 * Swap aging added 23.2.95, Stephen Tweedie. 13 * Buffermem limits added 12.3.98, Rik van Riel. 14 */ 15 16#include <linux/mm.h> 17#include <linux/sched.h> 18#include <linux/kernel_stat.h> 19#include <linux/swap.h> 20#include <linux/mman.h> 21#include <linux/pagemap.h> 22#include <linux/pagevec.h> 23#include <linux/init.h> 24#include <linux/module.h> 25#include <linux/mm_inline.h> 26#include <linux/buffer_head.h> /* for try_to_release_page() */ 27#include <linux/percpu_counter.h> 28#include <linux/percpu.h> 29#include <linux/cpu.h> 30#include <linux/notifier.h> 31#include <linux/backing-dev.h> 32#include <linux/memcontrol.h> 33#include <linux/gfp.h> 34 35#include "internal.h" 36 37/* How many pages do we try to swap or page in/out together? */ 38int page_cluster; 39 40static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs); 41static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); 42 43/* 44 * This path almost never happens for VM activity - pages are normally 45 * freed via pagevecs. But it gets used by networking. 46 */ 47static void __page_cache_release(struct page *page) 48{ 49 if (PageLRU(page)) { 50 unsigned long flags; 51 struct zone *zone = page_zone(page); 52 53 spin_lock_irqsave(&zone->lru_lock, flags); 54 VM_BUG_ON(!PageLRU(page)); 55 __ClearPageLRU(page); 56 del_page_from_lru(zone, page); 57 spin_unlock_irqrestore(&zone->lru_lock, flags); 58 } 59} 60 61static void __put_single_page(struct page *page) 62{ 63 __page_cache_release(page); 64 free_hot_cold_page(page, 0); 65} 66 67static void __put_compound_page(struct page *page) 68{ 69 compound_page_dtor *dtor; 70 71 __page_cache_release(page); 72 dtor = get_compound_page_dtor(page); 73 (*dtor)(page); 74} 75 76static void put_compound_page(struct page *page) 77{ 78 if (unlikely(PageTail(page))) { 79 /* __split_huge_page_refcount can run under us */ 80 struct page *page_head = page->first_page; 81 smp_rmb(); 82 /* 83 * If PageTail is still set after smp_rmb() we can be sure 84 * that the page->first_page we read wasn't a dangling pointer. 85 * See __split_huge_page_refcount() smp_wmb(). 86 */ 87 if (likely(PageTail(page) && get_page_unless_zero(page_head))) { 88 unsigned long flags; 89 /* 90 * Verify that our page_head wasn't converted 91 * to a a regular page before we got a 92 * reference on it. 93 */ 94 if (unlikely(!PageHead(page_head))) { 95 /* PageHead is cleared after PageTail */ 96 smp_rmb(); 97 VM_BUG_ON(PageTail(page)); 98 goto out_put_head; 99 } 100 /* 101 * Only run compound_lock on a valid PageHead, 102 * after having it pinned with 103 * get_page_unless_zero() above. 104 */ 105 smp_mb(); 106 /* page_head wasn't a dangling pointer */ 107 flags = compound_lock_irqsave(page_head); 108 if (unlikely(!PageTail(page))) { 109 /* __split_huge_page_refcount run before us */ 110 compound_unlock_irqrestore(page_head, flags); 111 VM_BUG_ON(PageHead(page_head)); 112 out_put_head: 113 if (put_page_testzero(page_head)) 114 __put_single_page(page_head); 115 out_put_single: 116 if (put_page_testzero(page)) 117 __put_single_page(page); 118 return; 119 } 120 VM_BUG_ON(page_head != page->first_page); 121 /* 122 * We can release the refcount taken by 123 * get_page_unless_zero now that 124 * split_huge_page_refcount is blocked on the 125 * compound_lock. 126 */ 127 if (put_page_testzero(page_head)) 128 VM_BUG_ON(1); 129 /* __split_huge_page_refcount will wait now */ 130 VM_BUG_ON(atomic_read(&page->_count) <= 0); 131 atomic_dec(&page->_count); 132 VM_BUG_ON(atomic_read(&page_head->_count) <= 0); 133 compound_unlock_irqrestore(page_head, flags); 134 if (put_page_testzero(page_head)) { 135 if (PageHead(page_head)) 136 __put_compound_page(page_head); 137 else 138 __put_single_page(page_head); 139 } 140 } else { 141 /* page_head is a dangling pointer */ 142 VM_BUG_ON(PageTail(page)); 143 goto out_put_single; 144 } 145 } else if (put_page_testzero(page)) { 146 if (PageHead(page)) 147 __put_compound_page(page); 148 else 149 __put_single_page(page); 150 } 151} 152 153void put_page(struct page *page) 154{ 155 if (unlikely(PageCompound(page))) 156 put_compound_page(page); 157 else if (put_page_testzero(page)) 158 __put_single_page(page); 159} 160EXPORT_SYMBOL(put_page); 161 162/** 163 * put_pages_list() - release a list of pages 164 * @pages: list of pages threaded on page->lru 165 * 166 * Release a list of pages which are strung together on page.lru. Currently 167 * used by read_cache_pages() and related error recovery code. 168 */ 169void put_pages_list(struct list_head *pages) 170{ 171 while (!list_empty(pages)) { 172 struct page *victim; 173 174 victim = list_entry(pages->prev, struct page, lru); 175 list_del(&victim->lru); 176 page_cache_release(victim); 177 } 178} 179EXPORT_SYMBOL(put_pages_list); 180 181/* 182 * pagevec_move_tail() must be called with IRQ disabled. 183 * Otherwise this may cause nasty races. 184 */ 185static void pagevec_move_tail(struct pagevec *pvec) 186{ 187 int i; 188 int pgmoved = 0; 189 struct zone *zone = NULL; 190 191 for (i = 0; i < pagevec_count(pvec); i++) { 192 struct page *page = pvec->pages[i]; 193 struct zone *pagezone = page_zone(page); 194 195 if (pagezone != zone) { 196 if (zone) 197 spin_unlock(&zone->lru_lock); 198 zone = pagezone; 199 spin_lock(&zone->lru_lock); 200 } 201 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 202 int lru = page_lru_base_type(page); 203 list_move_tail(&page->lru, &zone->lru[lru].list); 204 pgmoved++; 205 } 206 } 207 if (zone) 208 spin_unlock(&zone->lru_lock); 209 __count_vm_events(PGROTATED, pgmoved); 210 release_pages(pvec->pages, pvec->nr, pvec->cold); 211 pagevec_reinit(pvec); 212} 213 214/* 215 * Writeback is about to end against a page which has been marked for immediate 216 * reclaim. If it still appears to be reclaimable, move it to the tail of the 217 * inactive list. 218 */ 219void rotate_reclaimable_page(struct page *page) 220{ 221 if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) && 222 !PageUnevictable(page) && PageLRU(page)) { 223 struct pagevec *pvec; 224 unsigned long flags; 225 226 page_cache_get(page); 227 local_irq_save(flags); 228 pvec = &__get_cpu_var(lru_rotate_pvecs); 229 if (!pagevec_add(pvec, page)) 230 pagevec_move_tail(pvec); 231 local_irq_restore(flags); 232 } 233} 234 235static void update_page_reclaim_stat(struct zone *zone, struct page *page, 236 int file, int rotated) 237{ 238 struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat; 239 struct zone_reclaim_stat *memcg_reclaim_stat; 240 241 memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page); 242 243 reclaim_stat->recent_scanned[file]++; 244 if (rotated) 245 reclaim_stat->recent_rotated[file]++; 246 247 if (!memcg_reclaim_stat) 248 return; 249 250 memcg_reclaim_stat->recent_scanned[file]++; 251 if (rotated) 252 memcg_reclaim_stat->recent_rotated[file]++; 253} 254 255/* 256 * FIXME: speed this up? 257 */ 258void activate_page(struct page *page) 259{ 260 struct zone *zone = page_zone(page); 261 262 spin_lock_irq(&zone->lru_lock); 263 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 264 int file = page_is_file_cache(page); 265 int lru = page_lru_base_type(page); 266 del_page_from_lru_list(zone, page, lru); 267 268 SetPageActive(page); 269 lru += LRU_ACTIVE; 270 add_page_to_lru_list(zone, page, lru); 271 __count_vm_event(PGACTIVATE); 272 273 update_page_reclaim_stat(zone, page, file, 1); 274 } 275 spin_unlock_irq(&zone->lru_lock); 276} 277 278/* 279 * Mark a page as having seen activity. 280 * 281 * inactive,unreferenced -> inactive,referenced 282 * inactive,referenced -> active,unreferenced 283 * active,unreferenced -> active,referenced 284 */ 285void mark_page_accessed(struct page *page) 286{ 287 if (!PageActive(page) && !PageUnevictable(page) && 288 PageReferenced(page) && PageLRU(page)) { 289 activate_page(page); 290 ClearPageReferenced(page); 291 } else if (!PageReferenced(page)) { 292 SetPageReferenced(page); 293 } 294} 295 296EXPORT_SYMBOL(mark_page_accessed); 297 298void __lru_cache_add(struct page *page, enum lru_list lru) 299{ 300 struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru]; 301 302 page_cache_get(page); 303 if (!pagevec_add(pvec, page)) 304 ____pagevec_lru_add(pvec, lru); 305 put_cpu_var(lru_add_pvecs); 306} 307EXPORT_SYMBOL(__lru_cache_add); 308 309/** 310 * lru_cache_add_lru - add a page to a page list 311 * @page: the page to be added to the LRU. 312 * @lru: the LRU list to which the page is added. 313 */ 314void lru_cache_add_lru(struct page *page, enum lru_list lru) 315{ 316 if (PageActive(page)) { 317 VM_BUG_ON(PageUnevictable(page)); 318 ClearPageActive(page); 319 } else if (PageUnevictable(page)) { 320 VM_BUG_ON(PageActive(page)); 321 ClearPageUnevictable(page); 322 } 323 324 VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page)); 325 __lru_cache_add(page, lru); 326} 327 328/** 329 * add_page_to_unevictable_list - add a page to the unevictable list 330 * @page: the page to be added to the unevictable list 331 * 332 * Add page directly to its zone's unevictable list. To avoid races with 333 * tasks that might be making the page evictable, through eg. munlock, 334 * munmap or exit, while it's not on the lru, we want to add the page 335 * while it's locked or otherwise "invisible" to other tasks. This is 336 * difficult to do when using the pagevec cache, so bypass that. 337 */ 338void add_page_to_unevictable_list(struct page *page) 339{ 340 struct zone *zone = page_zone(page); 341 342 spin_lock_irq(&zone->lru_lock); 343 SetPageUnevictable(page); 344 SetPageLRU(page); 345 add_page_to_lru_list(zone, page, LRU_UNEVICTABLE); 346 spin_unlock_irq(&zone->lru_lock); 347} 348 349/* 350 * Drain pages out of the cpu's pagevecs. 351 * Either "cpu" is the current CPU, and preemption has already been 352 * disabled; or "cpu" is being hot-unplugged, and is already dead. 353 */ 354static void drain_cpu_pagevecs(int cpu) 355{ 356 struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu); 357 struct pagevec *pvec; 358 int lru; 359 360 for_each_lru(lru) { 361 pvec = &pvecs[lru - LRU_BASE]; 362 if (pagevec_count(pvec)) 363 ____pagevec_lru_add(pvec, lru); 364 } 365 366 pvec = &per_cpu(lru_rotate_pvecs, cpu); 367 if (pagevec_count(pvec)) { 368 unsigned long flags; 369 370 /* No harm done if a racing interrupt already did this */ 371 local_irq_save(flags); 372 pagevec_move_tail(pvec); 373 local_irq_restore(flags); 374 } 375} 376 377void lru_add_drain(void) 378{ 379 drain_cpu_pagevecs(get_cpu()); 380 put_cpu(); 381} 382 383static void lru_add_drain_per_cpu(struct work_struct *dummy) 384{ 385 lru_add_drain(); 386} 387 388/* 389 * Returns 0 for success 390 */ 391int lru_add_drain_all(void) 392{ 393 return schedule_on_each_cpu(lru_add_drain_per_cpu); 394} 395 396/* 397 * Batched page_cache_release(). Decrement the reference count on all the 398 * passed pages. If it fell to zero then remove the page from the LRU and 399 * free it. 400 * 401 * Avoid taking zone->lru_lock if possible, but if it is taken, retain it 402 * for the remainder of the operation. 403 * 404 * The locking in this function is against shrink_inactive_list(): we recheck 405 * the page count inside the lock to see whether shrink_inactive_list() 406 * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() 407 * will free it. 408 */ 409void release_pages(struct page **pages, int nr, int cold) 410{ 411 int i; 412 struct pagevec pages_to_free; 413 struct zone *zone = NULL; 414 unsigned long uninitialized_var(flags); 415 416 pagevec_init(&pages_to_free, cold); 417 for (i = 0; i < nr; i++) { 418 struct page *page = pages[i]; 419 420 if (unlikely(PageCompound(page))) { 421 if (zone) { 422 spin_unlock_irqrestore(&zone->lru_lock, flags); 423 zone = NULL; 424 } 425 put_compound_page(page); 426 continue; 427 } 428 429 if (!put_page_testzero(page)) 430 continue; 431 432 if (PageLRU(page)) { 433 struct zone *pagezone = page_zone(page); 434 435 if (pagezone != zone) { 436 if (zone) 437 spin_unlock_irqrestore(&zone->lru_lock, 438 flags); 439 zone = pagezone; 440 spin_lock_irqsave(&zone->lru_lock, flags); 441 } 442 VM_BUG_ON(!PageLRU(page)); 443 __ClearPageLRU(page); 444 del_page_from_lru(zone, page); 445 } 446 447 if (!pagevec_add(&pages_to_free, page)) { 448 if (zone) { 449 spin_unlock_irqrestore(&zone->lru_lock, flags); 450 zone = NULL; 451 } 452 __pagevec_free(&pages_to_free); 453 pagevec_reinit(&pages_to_free); 454 } 455 } 456 if (zone) 457 spin_unlock_irqrestore(&zone->lru_lock, flags); 458 459 pagevec_free(&pages_to_free); 460} 461EXPORT_SYMBOL(release_pages); 462 463/* 464 * The pages which we're about to release may be in the deferred lru-addition 465 * queues. That would prevent them from really being freed right now. That's 466 * OK from a correctness point of view but is inefficient - those pages may be 467 * cache-warm and we want to give them back to the page allocator ASAP. 468 * 469 * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 470 * and __pagevec_lru_add_active() call release_pages() directly to avoid 471 * mutual recursion. 472 */ 473void __pagevec_release(struct pagevec *pvec) 474{ 475 lru_add_drain(); 476 release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 477 pagevec_reinit(pvec); 478} 479 480EXPORT_SYMBOL(__pagevec_release); 481 482/* 483 * Add the passed pages to the LRU, then drop the caller's refcount 484 * on them. Reinitialises the caller's pagevec. 485 */ 486void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru) 487{ 488 int i; 489 struct zone *zone = NULL; 490 491 VM_BUG_ON(is_unevictable_lru(lru)); 492 493 for (i = 0; i < pagevec_count(pvec); i++) { 494 struct page *page = pvec->pages[i]; 495 struct zone *pagezone = page_zone(page); 496 int file; 497 int active; 498 499 if (pagezone != zone) { 500 if (zone) 501 spin_unlock_irq(&zone->lru_lock); 502 zone = pagezone; 503 spin_lock_irq(&zone->lru_lock); 504 } 505 VM_BUG_ON(PageActive(page)); 506 VM_BUG_ON(PageUnevictable(page)); 507 VM_BUG_ON(PageLRU(page)); 508 SetPageLRU(page); 509 active = is_active_lru(lru); 510 file = is_file_lru(lru); 511 if (active) 512 SetPageActive(page); 513 update_page_reclaim_stat(zone, page, file, active); 514 add_page_to_lru_list(zone, page, lru); 515 } 516 if (zone) 517 spin_unlock_irq(&zone->lru_lock); 518 release_pages(pvec->pages, pvec->nr, pvec->cold); 519 pagevec_reinit(pvec); 520} 521 522EXPORT_SYMBOL(____pagevec_lru_add); 523 524/* 525 * Try to drop buffers from the pages in a pagevec 526 */ 527void pagevec_strip(struct pagevec *pvec) 528{ 529 int i; 530 531 for (i = 0; i < pagevec_count(pvec); i++) { 532 struct page *page = pvec->pages[i]; 533 534 if (page_has_private(page) && trylock_page(page)) { 535 if (page_has_private(page)) 536 try_to_release_page(page, 0); 537 unlock_page(page); 538 } 539 } 540} 541 542/** 543 * pagevec_lookup - gang pagecache lookup 544 * @pvec: Where the resulting pages are placed 545 * @mapping: The address_space to search 546 * @start: The starting page index 547 * @nr_pages: The maximum number of pages 548 * 549 * pagevec_lookup() will search for and return a group of up to @nr_pages pages 550 * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a 551 * reference against the pages in @pvec. 552 * 553 * The search returns a group of mapping-contiguous pages with ascending 554 * indexes. There may be holes in the indices due to not-present pages. 555 * 556 * pagevec_lookup() returns the number of pages which were found. 557 */ 558unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, 559 pgoff_t start, unsigned nr_pages) 560{ 561 pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); 562 return pagevec_count(pvec); 563} 564 565EXPORT_SYMBOL(pagevec_lookup); 566 567unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 568 pgoff_t *index, int tag, unsigned nr_pages) 569{ 570 pvec->nr = find_get_pages_tag(mapping, index, tag, 571 nr_pages, pvec->pages); 572 return pagevec_count(pvec); 573} 574 575EXPORT_SYMBOL(pagevec_lookup_tag); 576 577/* 578 * Perform any setup for the swap system 579 */ 580void __init swap_setup(void) 581{ 582 unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT); 583 584#ifdef CONFIG_SWAP 585 bdi_init(swapper_space.backing_dev_info); 586#endif 587 588 /* Use a smaller cluster for small-memory machines */ 589 if (megs < 16) 590 page_cluster = 2; 591 else 592 page_cluster = 3; 593 /* 594 * Right now other parts of the system means that we 595 * _really_ don't want to cluster much more 596 */ 597} 598