rmap.c revision a66979abad090b2765a6c6790c9fdeab996833f2
1/* 2 * mm/rmap.c - physical to virtual reverse mappings 3 * 4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 5 * Released under the General Public License (GPL). 6 * 7 * Simple, low overhead reverse mapping scheme. 8 * Please try to keep this thing as modular as possible. 9 * 10 * Provides methods for unmapping each kind of mapped page: 11 * the anon methods track anonymous pages, and 12 * the file methods track pages belonging to an inode. 13 * 14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001 15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 17 * Contributions by Hugh Dickins 2003, 2004 18 */ 19 20/* 21 * Lock ordering in mm: 22 * 23 * inode->i_mutex (while writing or truncating, not reading or faulting) 24 * inode->i_alloc_sem (vmtruncate_range) 25 * mm->mmap_sem 26 * page->flags PG_locked (lock_page) 27 * mapping->i_mmap_lock 28 * anon_vma->lock 29 * mm->page_table_lock or pte_lock 30 * zone->lru_lock (in mark_page_accessed, isolate_lru_page) 31 * swap_lock (in swap_duplicate, swap_info_get) 32 * mmlist_lock (in mmput, drain_mmlist and others) 33 * mapping->private_lock (in __set_page_dirty_buffers) 34 * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 35 * inode_wb_list_lock (in set_page_dirty's __mark_inode_dirty) 36 * sb_lock (within inode_lock in fs/fs-writeback.c) 37 * mapping->tree_lock (widely used, in set_page_dirty, 38 * in arch-dependent flush_dcache_mmap_lock, 39 * within inode_wb_list_lock in __sync_single_inode) 40 * 41 * (code doesn't rely on that order so it could be switched around) 42 * ->tasklist_lock 43 * anon_vma->lock (memory_failure, collect_procs_anon) 44 * pte map lock 45 */ 46 47#include <linux/mm.h> 48#include <linux/pagemap.h> 49#include <linux/swap.h> 50#include <linux/swapops.h> 51#include <linux/slab.h> 52#include <linux/init.h> 53#include <linux/ksm.h> 54#include <linux/rmap.h> 55#include <linux/rcupdate.h> 56#include <linux/module.h> 57#include <linux/memcontrol.h> 58#include <linux/mmu_notifier.h> 59#include <linux/migrate.h> 60#include <linux/hugetlb.h> 61 62#include <asm/tlbflush.h> 63 64#include "internal.h" 65 66static struct kmem_cache *anon_vma_cachep; 67static struct kmem_cache *anon_vma_chain_cachep; 68 69static inline struct anon_vma *anon_vma_alloc(void) 70{ 71 struct anon_vma *anon_vma; 72 73 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 74 if (anon_vma) { 75 atomic_set(&anon_vma->refcount, 1); 76 /* 77 * Initialise the anon_vma root to point to itself. If called 78 * from fork, the root will be reset to the parents anon_vma. 79 */ 80 anon_vma->root = anon_vma; 81 } 82 83 return anon_vma; 84} 85 86static inline void anon_vma_free(struct anon_vma *anon_vma) 87{ 88 VM_BUG_ON(atomic_read(&anon_vma->refcount)); 89 kmem_cache_free(anon_vma_cachep, anon_vma); 90} 91 92static inline struct anon_vma_chain *anon_vma_chain_alloc(void) 93{ 94 return kmem_cache_alloc(anon_vma_chain_cachep, GFP_KERNEL); 95} 96 97static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 98{ 99 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 100} 101 102/** 103 * anon_vma_prepare - attach an anon_vma to a memory region 104 * @vma: the memory region in question 105 * 106 * This makes sure the memory mapping described by 'vma' has 107 * an 'anon_vma' attached to it, so that we can associate the 108 * anonymous pages mapped into it with that anon_vma. 109 * 110 * The common case will be that we already have one, but if 111 * not we either need to find an adjacent mapping that we 112 * can re-use the anon_vma from (very common when the only 113 * reason for splitting a vma has been mprotect()), or we 114 * allocate a new one. 115 * 116 * Anon-vma allocations are very subtle, because we may have 117 * optimistically looked up an anon_vma in page_lock_anon_vma() 118 * and that may actually touch the spinlock even in the newly 119 * allocated vma (it depends on RCU to make sure that the 120 * anon_vma isn't actually destroyed). 121 * 122 * As a result, we need to do proper anon_vma locking even 123 * for the new allocation. At the same time, we do not want 124 * to do any locking for the common case of already having 125 * an anon_vma. 126 * 127 * This must be called with the mmap_sem held for reading. 128 */ 129int anon_vma_prepare(struct vm_area_struct *vma) 130{ 131 struct anon_vma *anon_vma = vma->anon_vma; 132 struct anon_vma_chain *avc; 133 134 might_sleep(); 135 if (unlikely(!anon_vma)) { 136 struct mm_struct *mm = vma->vm_mm; 137 struct anon_vma *allocated; 138 139 avc = anon_vma_chain_alloc(); 140 if (!avc) 141 goto out_enomem; 142 143 anon_vma = find_mergeable_anon_vma(vma); 144 allocated = NULL; 145 if (!anon_vma) { 146 anon_vma = anon_vma_alloc(); 147 if (unlikely(!anon_vma)) 148 goto out_enomem_free_avc; 149 allocated = anon_vma; 150 } 151 152 anon_vma_lock(anon_vma); 153 /* page_table_lock to protect against threads */ 154 spin_lock(&mm->page_table_lock); 155 if (likely(!vma->anon_vma)) { 156 vma->anon_vma = anon_vma; 157 avc->anon_vma = anon_vma; 158 avc->vma = vma; 159 list_add(&avc->same_vma, &vma->anon_vma_chain); 160 list_add_tail(&avc->same_anon_vma, &anon_vma->head); 161 allocated = NULL; 162 avc = NULL; 163 } 164 spin_unlock(&mm->page_table_lock); 165 anon_vma_unlock(anon_vma); 166 167 if (unlikely(allocated)) 168 put_anon_vma(allocated); 169 if (unlikely(avc)) 170 anon_vma_chain_free(avc); 171 } 172 return 0; 173 174 out_enomem_free_avc: 175 anon_vma_chain_free(avc); 176 out_enomem: 177 return -ENOMEM; 178} 179 180static void anon_vma_chain_link(struct vm_area_struct *vma, 181 struct anon_vma_chain *avc, 182 struct anon_vma *anon_vma) 183{ 184 avc->vma = vma; 185 avc->anon_vma = anon_vma; 186 list_add(&avc->same_vma, &vma->anon_vma_chain); 187 188 anon_vma_lock(anon_vma); 189 /* 190 * It's critical to add new vmas to the tail of the anon_vma, 191 * see comment in huge_memory.c:__split_huge_page(). 192 */ 193 list_add_tail(&avc->same_anon_vma, &anon_vma->head); 194 anon_vma_unlock(anon_vma); 195} 196 197/* 198 * Attach the anon_vmas from src to dst. 199 * Returns 0 on success, -ENOMEM on failure. 200 */ 201int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 202{ 203 struct anon_vma_chain *avc, *pavc; 204 205 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 206 avc = anon_vma_chain_alloc(); 207 if (!avc) 208 goto enomem_failure; 209 anon_vma_chain_link(dst, avc, pavc->anon_vma); 210 } 211 return 0; 212 213 enomem_failure: 214 unlink_anon_vmas(dst); 215 return -ENOMEM; 216} 217 218/* 219 * Attach vma to its own anon_vma, as well as to the anon_vmas that 220 * the corresponding VMA in the parent process is attached to. 221 * Returns 0 on success, non-zero on failure. 222 */ 223int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 224{ 225 struct anon_vma_chain *avc; 226 struct anon_vma *anon_vma; 227 228 /* Don't bother if the parent process has no anon_vma here. */ 229 if (!pvma->anon_vma) 230 return 0; 231 232 /* 233 * First, attach the new VMA to the parent VMA's anon_vmas, 234 * so rmap can find non-COWed pages in child processes. 235 */ 236 if (anon_vma_clone(vma, pvma)) 237 return -ENOMEM; 238 239 /* Then add our own anon_vma. */ 240 anon_vma = anon_vma_alloc(); 241 if (!anon_vma) 242 goto out_error; 243 avc = anon_vma_chain_alloc(); 244 if (!avc) 245 goto out_error_free_anon_vma; 246 247 /* 248 * The root anon_vma's spinlock is the lock actually used when we 249 * lock any of the anon_vmas in this anon_vma tree. 250 */ 251 anon_vma->root = pvma->anon_vma->root; 252 /* 253 * With refcounts, an anon_vma can stay around longer than the 254 * process it belongs to. The root anon_vma needs to be pinned until 255 * this anon_vma is freed, because the lock lives in the root. 256 */ 257 get_anon_vma(anon_vma->root); 258 /* Mark this anon_vma as the one where our new (COWed) pages go. */ 259 vma->anon_vma = anon_vma; 260 anon_vma_chain_link(vma, avc, anon_vma); 261 262 return 0; 263 264 out_error_free_anon_vma: 265 put_anon_vma(anon_vma); 266 out_error: 267 unlink_anon_vmas(vma); 268 return -ENOMEM; 269} 270 271static void anon_vma_unlink(struct anon_vma_chain *anon_vma_chain) 272{ 273 struct anon_vma *anon_vma = anon_vma_chain->anon_vma; 274 int empty; 275 276 /* If anon_vma_fork fails, we can get an empty anon_vma_chain. */ 277 if (!anon_vma) 278 return; 279 280 anon_vma_lock(anon_vma); 281 list_del(&anon_vma_chain->same_anon_vma); 282 283 /* We must garbage collect the anon_vma if it's empty */ 284 empty = list_empty(&anon_vma->head); 285 anon_vma_unlock(anon_vma); 286 287 if (empty) 288 put_anon_vma(anon_vma); 289} 290 291void unlink_anon_vmas(struct vm_area_struct *vma) 292{ 293 struct anon_vma_chain *avc, *next; 294 295 /* 296 * Unlink each anon_vma chained to the VMA. This list is ordered 297 * from newest to oldest, ensuring the root anon_vma gets freed last. 298 */ 299 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 300 anon_vma_unlink(avc); 301 list_del(&avc->same_vma); 302 anon_vma_chain_free(avc); 303 } 304} 305 306static void anon_vma_ctor(void *data) 307{ 308 struct anon_vma *anon_vma = data; 309 310 spin_lock_init(&anon_vma->lock); 311 atomic_set(&anon_vma->refcount, 0); 312 INIT_LIST_HEAD(&anon_vma->head); 313} 314 315void __init anon_vma_init(void) 316{ 317 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 318 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor); 319 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC); 320} 321 322/* 323 * Getting a lock on a stable anon_vma from a page off the LRU is 324 * tricky: page_lock_anon_vma rely on RCU to guard against the races. 325 */ 326struct anon_vma *__page_lock_anon_vma(struct page *page) 327{ 328 struct anon_vma *anon_vma, *root_anon_vma; 329 unsigned long anon_mapping; 330 331 rcu_read_lock(); 332 anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping); 333 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 334 goto out; 335 if (!page_mapped(page)) 336 goto out; 337 338 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 339 root_anon_vma = ACCESS_ONCE(anon_vma->root); 340 spin_lock(&root_anon_vma->lock); 341 342 /* 343 * If this page is still mapped, then its anon_vma cannot have been 344 * freed. But if it has been unmapped, we have no security against 345 * the anon_vma structure being freed and reused (for another anon_vma: 346 * SLAB_DESTROY_BY_RCU guarantees that - so the spin_lock above cannot 347 * corrupt): with anon_vma_prepare() or anon_vma_fork() redirecting 348 * anon_vma->root before page_unlock_anon_vma() is called to unlock. 349 */ 350 if (page_mapped(page)) 351 return anon_vma; 352 353 spin_unlock(&root_anon_vma->lock); 354out: 355 rcu_read_unlock(); 356 return NULL; 357} 358 359void page_unlock_anon_vma(struct anon_vma *anon_vma) 360 __releases(&anon_vma->root->lock) 361 __releases(RCU) 362{ 363 anon_vma_unlock(anon_vma); 364 rcu_read_unlock(); 365} 366 367/* 368 * At what user virtual address is page expected in @vma? 369 * Returns virtual address or -EFAULT if page's index/offset is not 370 * within the range mapped the @vma. 371 */ 372inline unsigned long 373vma_address(struct page *page, struct vm_area_struct *vma) 374{ 375 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 376 unsigned long address; 377 378 if (unlikely(is_vm_hugetlb_page(vma))) 379 pgoff = page->index << huge_page_order(page_hstate(page)); 380 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 381 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) { 382 /* page should be within @vma mapping range */ 383 return -EFAULT; 384 } 385 return address; 386} 387 388/* 389 * At what user virtual address is page expected in vma? 390 * Caller should check the page is actually part of the vma. 391 */ 392unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 393{ 394 if (PageAnon(page)) { 395 struct anon_vma *page__anon_vma = page_anon_vma(page); 396 /* 397 * Note: swapoff's unuse_vma() is more efficient with this 398 * check, and needs it to match anon_vma when KSM is active. 399 */ 400 if (!vma->anon_vma || !page__anon_vma || 401 vma->anon_vma->root != page__anon_vma->root) 402 return -EFAULT; 403 } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) { 404 if (!vma->vm_file || 405 vma->vm_file->f_mapping != page->mapping) 406 return -EFAULT; 407 } else 408 return -EFAULT; 409 return vma_address(page, vma); 410} 411 412/* 413 * Check that @page is mapped at @address into @mm. 414 * 415 * If @sync is false, page_check_address may perform a racy check to avoid 416 * the page table lock when the pte is not present (helpful when reclaiming 417 * highly shared pages). 418 * 419 * On success returns with pte mapped and locked. 420 */ 421pte_t *__page_check_address(struct page *page, struct mm_struct *mm, 422 unsigned long address, spinlock_t **ptlp, int sync) 423{ 424 pgd_t *pgd; 425 pud_t *pud; 426 pmd_t *pmd; 427 pte_t *pte; 428 spinlock_t *ptl; 429 430 if (unlikely(PageHuge(page))) { 431 pte = huge_pte_offset(mm, address); 432 ptl = &mm->page_table_lock; 433 goto check; 434 } 435 436 pgd = pgd_offset(mm, address); 437 if (!pgd_present(*pgd)) 438 return NULL; 439 440 pud = pud_offset(pgd, address); 441 if (!pud_present(*pud)) 442 return NULL; 443 444 pmd = pmd_offset(pud, address); 445 if (!pmd_present(*pmd)) 446 return NULL; 447 if (pmd_trans_huge(*pmd)) 448 return NULL; 449 450 pte = pte_offset_map(pmd, address); 451 /* Make a quick check before getting the lock */ 452 if (!sync && !pte_present(*pte)) { 453 pte_unmap(pte); 454 return NULL; 455 } 456 457 ptl = pte_lockptr(mm, pmd); 458check: 459 spin_lock(ptl); 460 if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) { 461 *ptlp = ptl; 462 return pte; 463 } 464 pte_unmap_unlock(pte, ptl); 465 return NULL; 466} 467 468/** 469 * page_mapped_in_vma - check whether a page is really mapped in a VMA 470 * @page: the page to test 471 * @vma: the VMA to test 472 * 473 * Returns 1 if the page is mapped into the page tables of the VMA, 0 474 * if the page is not mapped into the page tables of this VMA. Only 475 * valid for normal file or anonymous VMAs. 476 */ 477int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) 478{ 479 unsigned long address; 480 pte_t *pte; 481 spinlock_t *ptl; 482 483 address = vma_address(page, vma); 484 if (address == -EFAULT) /* out of vma range */ 485 return 0; 486 pte = page_check_address(page, vma->vm_mm, address, &ptl, 1); 487 if (!pte) /* the page is not in this mm */ 488 return 0; 489 pte_unmap_unlock(pte, ptl); 490 491 return 1; 492} 493 494/* 495 * Subfunctions of page_referenced: page_referenced_one called 496 * repeatedly from either page_referenced_anon or page_referenced_file. 497 */ 498int page_referenced_one(struct page *page, struct vm_area_struct *vma, 499 unsigned long address, unsigned int *mapcount, 500 unsigned long *vm_flags) 501{ 502 struct mm_struct *mm = vma->vm_mm; 503 int referenced = 0; 504 505 if (unlikely(PageTransHuge(page))) { 506 pmd_t *pmd; 507 508 spin_lock(&mm->page_table_lock); 509 /* 510 * rmap might return false positives; we must filter 511 * these out using page_check_address_pmd(). 512 */ 513 pmd = page_check_address_pmd(page, mm, address, 514 PAGE_CHECK_ADDRESS_PMD_FLAG); 515 if (!pmd) { 516 spin_unlock(&mm->page_table_lock); 517 goto out; 518 } 519 520 if (vma->vm_flags & VM_LOCKED) { 521 spin_unlock(&mm->page_table_lock); 522 *mapcount = 0; /* break early from loop */ 523 *vm_flags |= VM_LOCKED; 524 goto out; 525 } 526 527 /* go ahead even if the pmd is pmd_trans_splitting() */ 528 if (pmdp_clear_flush_young_notify(vma, address, pmd)) 529 referenced++; 530 spin_unlock(&mm->page_table_lock); 531 } else { 532 pte_t *pte; 533 spinlock_t *ptl; 534 535 /* 536 * rmap might return false positives; we must filter 537 * these out using page_check_address(). 538 */ 539 pte = page_check_address(page, mm, address, &ptl, 0); 540 if (!pte) 541 goto out; 542 543 if (vma->vm_flags & VM_LOCKED) { 544 pte_unmap_unlock(pte, ptl); 545 *mapcount = 0; /* break early from loop */ 546 *vm_flags |= VM_LOCKED; 547 goto out; 548 } 549 550 if (ptep_clear_flush_young_notify(vma, address, pte)) { 551 /* 552 * Don't treat a reference through a sequentially read 553 * mapping as such. If the page has been used in 554 * another mapping, we will catch it; if this other 555 * mapping is already gone, the unmap path will have 556 * set PG_referenced or activated the page. 557 */ 558 if (likely(!VM_SequentialReadHint(vma))) 559 referenced++; 560 } 561 pte_unmap_unlock(pte, ptl); 562 } 563 564 /* Pretend the page is referenced if the task has the 565 swap token and is in the middle of a page fault. */ 566 if (mm != current->mm && has_swap_token(mm) && 567 rwsem_is_locked(&mm->mmap_sem)) 568 referenced++; 569 570 (*mapcount)--; 571 572 if (referenced) 573 *vm_flags |= vma->vm_flags; 574out: 575 return referenced; 576} 577 578static int page_referenced_anon(struct page *page, 579 struct mem_cgroup *mem_cont, 580 unsigned long *vm_flags) 581{ 582 unsigned int mapcount; 583 struct anon_vma *anon_vma; 584 struct anon_vma_chain *avc; 585 int referenced = 0; 586 587 anon_vma = page_lock_anon_vma(page); 588 if (!anon_vma) 589 return referenced; 590 591 mapcount = page_mapcount(page); 592 list_for_each_entry(avc, &anon_vma->head, same_anon_vma) { 593 struct vm_area_struct *vma = avc->vma; 594 unsigned long address = vma_address(page, vma); 595 if (address == -EFAULT) 596 continue; 597 /* 598 * If we are reclaiming on behalf of a cgroup, skip 599 * counting on behalf of references from different 600 * cgroups 601 */ 602 if (mem_cont && !mm_match_cgroup(vma->vm_mm, mem_cont)) 603 continue; 604 referenced += page_referenced_one(page, vma, address, 605 &mapcount, vm_flags); 606 if (!mapcount) 607 break; 608 } 609 610 page_unlock_anon_vma(anon_vma); 611 return referenced; 612} 613 614/** 615 * page_referenced_file - referenced check for object-based rmap 616 * @page: the page we're checking references on. 617 * @mem_cont: target memory controller 618 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 619 * 620 * For an object-based mapped page, find all the places it is mapped and 621 * check/clear the referenced flag. This is done by following the page->mapping 622 * pointer, then walking the chain of vmas it holds. It returns the number 623 * of references it found. 624 * 625 * This function is only called from page_referenced for object-based pages. 626 */ 627static int page_referenced_file(struct page *page, 628 struct mem_cgroup *mem_cont, 629 unsigned long *vm_flags) 630{ 631 unsigned int mapcount; 632 struct address_space *mapping = page->mapping; 633 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 634 struct vm_area_struct *vma; 635 struct prio_tree_iter iter; 636 int referenced = 0; 637 638 /* 639 * The caller's checks on page->mapping and !PageAnon have made 640 * sure that this is a file page: the check for page->mapping 641 * excludes the case just before it gets set on an anon page. 642 */ 643 BUG_ON(PageAnon(page)); 644 645 /* 646 * The page lock not only makes sure that page->mapping cannot 647 * suddenly be NULLified by truncation, it makes sure that the 648 * structure at mapping cannot be freed and reused yet, 649 * so we can safely take mapping->i_mmap_lock. 650 */ 651 BUG_ON(!PageLocked(page)); 652 653 spin_lock(&mapping->i_mmap_lock); 654 655 /* 656 * i_mmap_lock does not stabilize mapcount at all, but mapcount 657 * is more likely to be accurate if we note it after spinning. 658 */ 659 mapcount = page_mapcount(page); 660 661 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 662 unsigned long address = vma_address(page, vma); 663 if (address == -EFAULT) 664 continue; 665 /* 666 * If we are reclaiming on behalf of a cgroup, skip 667 * counting on behalf of references from different 668 * cgroups 669 */ 670 if (mem_cont && !mm_match_cgroup(vma->vm_mm, mem_cont)) 671 continue; 672 referenced += page_referenced_one(page, vma, address, 673 &mapcount, vm_flags); 674 if (!mapcount) 675 break; 676 } 677 678 spin_unlock(&mapping->i_mmap_lock); 679 return referenced; 680} 681 682/** 683 * page_referenced - test if the page was referenced 684 * @page: the page to test 685 * @is_locked: caller holds lock on the page 686 * @mem_cont: target memory controller 687 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 688 * 689 * Quick test_and_clear_referenced for all mappings to a page, 690 * returns the number of ptes which referenced the page. 691 */ 692int page_referenced(struct page *page, 693 int is_locked, 694 struct mem_cgroup *mem_cont, 695 unsigned long *vm_flags) 696{ 697 int referenced = 0; 698 int we_locked = 0; 699 700 *vm_flags = 0; 701 if (page_mapped(page) && page_rmapping(page)) { 702 if (!is_locked && (!PageAnon(page) || PageKsm(page))) { 703 we_locked = trylock_page(page); 704 if (!we_locked) { 705 referenced++; 706 goto out; 707 } 708 } 709 if (unlikely(PageKsm(page))) 710 referenced += page_referenced_ksm(page, mem_cont, 711 vm_flags); 712 else if (PageAnon(page)) 713 referenced += page_referenced_anon(page, mem_cont, 714 vm_flags); 715 else if (page->mapping) 716 referenced += page_referenced_file(page, mem_cont, 717 vm_flags); 718 if (we_locked) 719 unlock_page(page); 720 } 721out: 722 if (page_test_and_clear_young(page)) 723 referenced++; 724 725 return referenced; 726} 727 728static int page_mkclean_one(struct page *page, struct vm_area_struct *vma, 729 unsigned long address) 730{ 731 struct mm_struct *mm = vma->vm_mm; 732 pte_t *pte; 733 spinlock_t *ptl; 734 int ret = 0; 735 736 pte = page_check_address(page, mm, address, &ptl, 1); 737 if (!pte) 738 goto out; 739 740 if (pte_dirty(*pte) || pte_write(*pte)) { 741 pte_t entry; 742 743 flush_cache_page(vma, address, pte_pfn(*pte)); 744 entry = ptep_clear_flush_notify(vma, address, pte); 745 entry = pte_wrprotect(entry); 746 entry = pte_mkclean(entry); 747 set_pte_at(mm, address, pte, entry); 748 ret = 1; 749 } 750 751 pte_unmap_unlock(pte, ptl); 752out: 753 return ret; 754} 755 756static int page_mkclean_file(struct address_space *mapping, struct page *page) 757{ 758 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 759 struct vm_area_struct *vma; 760 struct prio_tree_iter iter; 761 int ret = 0; 762 763 BUG_ON(PageAnon(page)); 764 765 spin_lock(&mapping->i_mmap_lock); 766 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 767 if (vma->vm_flags & VM_SHARED) { 768 unsigned long address = vma_address(page, vma); 769 if (address == -EFAULT) 770 continue; 771 ret += page_mkclean_one(page, vma, address); 772 } 773 } 774 spin_unlock(&mapping->i_mmap_lock); 775 return ret; 776} 777 778int page_mkclean(struct page *page) 779{ 780 int ret = 0; 781 782 BUG_ON(!PageLocked(page)); 783 784 if (page_mapped(page)) { 785 struct address_space *mapping = page_mapping(page); 786 if (mapping) { 787 ret = page_mkclean_file(mapping, page); 788 if (page_test_dirty(page)) { 789 page_clear_dirty(page, 1); 790 ret = 1; 791 } 792 } 793 } 794 795 return ret; 796} 797EXPORT_SYMBOL_GPL(page_mkclean); 798 799/** 800 * page_move_anon_rmap - move a page to our anon_vma 801 * @page: the page to move to our anon_vma 802 * @vma: the vma the page belongs to 803 * @address: the user virtual address mapped 804 * 805 * When a page belongs exclusively to one process after a COW event, 806 * that page can be moved into the anon_vma that belongs to just that 807 * process, so the rmap code will not search the parent or sibling 808 * processes. 809 */ 810void page_move_anon_rmap(struct page *page, 811 struct vm_area_struct *vma, unsigned long address) 812{ 813 struct anon_vma *anon_vma = vma->anon_vma; 814 815 VM_BUG_ON(!PageLocked(page)); 816 VM_BUG_ON(!anon_vma); 817 VM_BUG_ON(page->index != linear_page_index(vma, address)); 818 819 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 820 page->mapping = (struct address_space *) anon_vma; 821} 822 823/** 824 * __page_set_anon_rmap - set up new anonymous rmap 825 * @page: Page to add to rmap 826 * @vma: VM area to add page to. 827 * @address: User virtual address of the mapping 828 * @exclusive: the page is exclusively owned by the current process 829 */ 830static void __page_set_anon_rmap(struct page *page, 831 struct vm_area_struct *vma, unsigned long address, int exclusive) 832{ 833 struct anon_vma *anon_vma = vma->anon_vma; 834 835 BUG_ON(!anon_vma); 836 837 if (PageAnon(page)) 838 return; 839 840 /* 841 * If the page isn't exclusively mapped into this vma, 842 * we must use the _oldest_ possible anon_vma for the 843 * page mapping! 844 */ 845 if (!exclusive) 846 anon_vma = anon_vma->root; 847 848 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 849 page->mapping = (struct address_space *) anon_vma; 850 page->index = linear_page_index(vma, address); 851} 852 853/** 854 * __page_check_anon_rmap - sanity check anonymous rmap addition 855 * @page: the page to add the mapping to 856 * @vma: the vm area in which the mapping is added 857 * @address: the user virtual address mapped 858 */ 859static void __page_check_anon_rmap(struct page *page, 860 struct vm_area_struct *vma, unsigned long address) 861{ 862#ifdef CONFIG_DEBUG_VM 863 /* 864 * The page's anon-rmap details (mapping and index) are guaranteed to 865 * be set up correctly at this point. 866 * 867 * We have exclusion against page_add_anon_rmap because the caller 868 * always holds the page locked, except if called from page_dup_rmap, 869 * in which case the page is already known to be setup. 870 * 871 * We have exclusion against page_add_new_anon_rmap because those pages 872 * are initially only visible via the pagetables, and the pte is locked 873 * over the call to page_add_new_anon_rmap. 874 */ 875 BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root); 876 BUG_ON(page->index != linear_page_index(vma, address)); 877#endif 878} 879 880/** 881 * page_add_anon_rmap - add pte mapping to an anonymous page 882 * @page: the page to add the mapping to 883 * @vma: the vm area in which the mapping is added 884 * @address: the user virtual address mapped 885 * 886 * The caller needs to hold the pte lock, and the page must be locked in 887 * the anon_vma case: to serialize mapping,index checking after setting, 888 * and to ensure that PageAnon is not being upgraded racily to PageKsm 889 * (but PageKsm is never downgraded to PageAnon). 890 */ 891void page_add_anon_rmap(struct page *page, 892 struct vm_area_struct *vma, unsigned long address) 893{ 894 do_page_add_anon_rmap(page, vma, address, 0); 895} 896 897/* 898 * Special version of the above for do_swap_page, which often runs 899 * into pages that are exclusively owned by the current process. 900 * Everybody else should continue to use page_add_anon_rmap above. 901 */ 902void do_page_add_anon_rmap(struct page *page, 903 struct vm_area_struct *vma, unsigned long address, int exclusive) 904{ 905 int first = atomic_inc_and_test(&page->_mapcount); 906 if (first) { 907 if (!PageTransHuge(page)) 908 __inc_zone_page_state(page, NR_ANON_PAGES); 909 else 910 __inc_zone_page_state(page, 911 NR_ANON_TRANSPARENT_HUGEPAGES); 912 } 913 if (unlikely(PageKsm(page))) 914 return; 915 916 VM_BUG_ON(!PageLocked(page)); 917 VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); 918 if (first) 919 __page_set_anon_rmap(page, vma, address, exclusive); 920 else 921 __page_check_anon_rmap(page, vma, address); 922} 923 924/** 925 * page_add_new_anon_rmap - add pte mapping to a new anonymous page 926 * @page: the page to add the mapping to 927 * @vma: the vm area in which the mapping is added 928 * @address: the user virtual address mapped 929 * 930 * Same as page_add_anon_rmap but must only be called on *new* pages. 931 * This means the inc-and-test can be bypassed. 932 * Page does not have to be locked. 933 */ 934void page_add_new_anon_rmap(struct page *page, 935 struct vm_area_struct *vma, unsigned long address) 936{ 937 VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); 938 SetPageSwapBacked(page); 939 atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */ 940 if (!PageTransHuge(page)) 941 __inc_zone_page_state(page, NR_ANON_PAGES); 942 else 943 __inc_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 944 __page_set_anon_rmap(page, vma, address, 1); 945 if (page_evictable(page, vma)) 946 lru_cache_add_lru(page, LRU_ACTIVE_ANON); 947 else 948 add_page_to_unevictable_list(page); 949} 950 951/** 952 * page_add_file_rmap - add pte mapping to a file page 953 * @page: the page to add the mapping to 954 * 955 * The caller needs to hold the pte lock. 956 */ 957void page_add_file_rmap(struct page *page) 958{ 959 if (atomic_inc_and_test(&page->_mapcount)) { 960 __inc_zone_page_state(page, NR_FILE_MAPPED); 961 mem_cgroup_inc_page_stat(page, MEMCG_NR_FILE_MAPPED); 962 } 963} 964 965/** 966 * page_remove_rmap - take down pte mapping from a page 967 * @page: page to remove mapping from 968 * 969 * The caller needs to hold the pte lock. 970 */ 971void page_remove_rmap(struct page *page) 972{ 973 /* page still mapped by someone else? */ 974 if (!atomic_add_negative(-1, &page->_mapcount)) 975 return; 976 977 /* 978 * Now that the last pte has gone, s390 must transfer dirty 979 * flag from storage key to struct page. We can usually skip 980 * this if the page is anon, so about to be freed; but perhaps 981 * not if it's in swapcache - there might be another pte slot 982 * containing the swap entry, but page not yet written to swap. 983 */ 984 if ((!PageAnon(page) || PageSwapCache(page)) && page_test_dirty(page)) { 985 page_clear_dirty(page, 1); 986 set_page_dirty(page); 987 } 988 /* 989 * Hugepages are not counted in NR_ANON_PAGES nor NR_FILE_MAPPED 990 * and not charged by memcg for now. 991 */ 992 if (unlikely(PageHuge(page))) 993 return; 994 if (PageAnon(page)) { 995 mem_cgroup_uncharge_page(page); 996 if (!PageTransHuge(page)) 997 __dec_zone_page_state(page, NR_ANON_PAGES); 998 else 999 __dec_zone_page_state(page, 1000 NR_ANON_TRANSPARENT_HUGEPAGES); 1001 } else { 1002 __dec_zone_page_state(page, NR_FILE_MAPPED); 1003 mem_cgroup_dec_page_stat(page, MEMCG_NR_FILE_MAPPED); 1004 } 1005 /* 1006 * It would be tidy to reset the PageAnon mapping here, 1007 * but that might overwrite a racing page_add_anon_rmap 1008 * which increments mapcount after us but sets mapping 1009 * before us: so leave the reset to free_hot_cold_page, 1010 * and remember that it's only reliable while mapped. 1011 * Leaving it set also helps swapoff to reinstate ptes 1012 * faster for those pages still in swapcache. 1013 */ 1014} 1015 1016/* 1017 * Subfunctions of try_to_unmap: try_to_unmap_one called 1018 * repeatedly from either try_to_unmap_anon or try_to_unmap_file. 1019 */ 1020int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, 1021 unsigned long address, enum ttu_flags flags) 1022{ 1023 struct mm_struct *mm = vma->vm_mm; 1024 pte_t *pte; 1025 pte_t pteval; 1026 spinlock_t *ptl; 1027 int ret = SWAP_AGAIN; 1028 1029 pte = page_check_address(page, mm, address, &ptl, 0); 1030 if (!pte) 1031 goto out; 1032 1033 /* 1034 * If the page is mlock()d, we cannot swap it out. 1035 * If it's recently referenced (perhaps page_referenced 1036 * skipped over this mm) then we should reactivate it. 1037 */ 1038 if (!(flags & TTU_IGNORE_MLOCK)) { 1039 if (vma->vm_flags & VM_LOCKED) 1040 goto out_mlock; 1041 1042 if (TTU_ACTION(flags) == TTU_MUNLOCK) 1043 goto out_unmap; 1044 } 1045 if (!(flags & TTU_IGNORE_ACCESS)) { 1046 if (ptep_clear_flush_young_notify(vma, address, pte)) { 1047 ret = SWAP_FAIL; 1048 goto out_unmap; 1049 } 1050 } 1051 1052 /* Nuke the page table entry. */ 1053 flush_cache_page(vma, address, page_to_pfn(page)); 1054 pteval = ptep_clear_flush_notify(vma, address, pte); 1055 1056 /* Move the dirty bit to the physical page now the pte is gone. */ 1057 if (pte_dirty(pteval)) 1058 set_page_dirty(page); 1059 1060 /* Update high watermark before we lower rss */ 1061 update_hiwater_rss(mm); 1062 1063 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { 1064 if (PageAnon(page)) 1065 dec_mm_counter(mm, MM_ANONPAGES); 1066 else 1067 dec_mm_counter(mm, MM_FILEPAGES); 1068 set_pte_at(mm, address, pte, 1069 swp_entry_to_pte(make_hwpoison_entry(page))); 1070 } else if (PageAnon(page)) { 1071 swp_entry_t entry = { .val = page_private(page) }; 1072 1073 if (PageSwapCache(page)) { 1074 /* 1075 * Store the swap location in the pte. 1076 * See handle_pte_fault() ... 1077 */ 1078 if (swap_duplicate(entry) < 0) { 1079 set_pte_at(mm, address, pte, pteval); 1080 ret = SWAP_FAIL; 1081 goto out_unmap; 1082 } 1083 if (list_empty(&mm->mmlist)) { 1084 spin_lock(&mmlist_lock); 1085 if (list_empty(&mm->mmlist)) 1086 list_add(&mm->mmlist, &init_mm.mmlist); 1087 spin_unlock(&mmlist_lock); 1088 } 1089 dec_mm_counter(mm, MM_ANONPAGES); 1090 inc_mm_counter(mm, MM_SWAPENTS); 1091 } else if (PAGE_MIGRATION) { 1092 /* 1093 * Store the pfn of the page in a special migration 1094 * pte. do_swap_page() will wait until the migration 1095 * pte is removed and then restart fault handling. 1096 */ 1097 BUG_ON(TTU_ACTION(flags) != TTU_MIGRATION); 1098 entry = make_migration_entry(page, pte_write(pteval)); 1099 } 1100 set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); 1101 BUG_ON(pte_file(*pte)); 1102 } else if (PAGE_MIGRATION && (TTU_ACTION(flags) == TTU_MIGRATION)) { 1103 /* Establish migration entry for a file page */ 1104 swp_entry_t entry; 1105 entry = make_migration_entry(page, pte_write(pteval)); 1106 set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); 1107 } else 1108 dec_mm_counter(mm, MM_FILEPAGES); 1109 1110 page_remove_rmap(page); 1111 page_cache_release(page); 1112 1113out_unmap: 1114 pte_unmap_unlock(pte, ptl); 1115out: 1116 return ret; 1117 1118out_mlock: 1119 pte_unmap_unlock(pte, ptl); 1120 1121 1122 /* 1123 * We need mmap_sem locking, Otherwise VM_LOCKED check makes 1124 * unstable result and race. Plus, We can't wait here because 1125 * we now hold anon_vma->lock or mapping->i_mmap_lock. 1126 * if trylock failed, the page remain in evictable lru and later 1127 * vmscan could retry to move the page to unevictable lru if the 1128 * page is actually mlocked. 1129 */ 1130 if (down_read_trylock(&vma->vm_mm->mmap_sem)) { 1131 if (vma->vm_flags & VM_LOCKED) { 1132 mlock_vma_page(page); 1133 ret = SWAP_MLOCK; 1134 } 1135 up_read(&vma->vm_mm->mmap_sem); 1136 } 1137 return ret; 1138} 1139 1140/* 1141 * objrmap doesn't work for nonlinear VMAs because the assumption that 1142 * offset-into-file correlates with offset-into-virtual-addresses does not hold. 1143 * Consequently, given a particular page and its ->index, we cannot locate the 1144 * ptes which are mapping that page without an exhaustive linear search. 1145 * 1146 * So what this code does is a mini "virtual scan" of each nonlinear VMA which 1147 * maps the file to which the target page belongs. The ->vm_private_data field 1148 * holds the current cursor into that scan. Successive searches will circulate 1149 * around the vma's virtual address space. 1150 * 1151 * So as more replacement pressure is applied to the pages in a nonlinear VMA, 1152 * more scanning pressure is placed against them as well. Eventually pages 1153 * will become fully unmapped and are eligible for eviction. 1154 * 1155 * For very sparsely populated VMAs this is a little inefficient - chances are 1156 * there there won't be many ptes located within the scan cluster. In this case 1157 * maybe we could scan further - to the end of the pte page, perhaps. 1158 * 1159 * Mlocked pages: check VM_LOCKED under mmap_sem held for read, if we can 1160 * acquire it without blocking. If vma locked, mlock the pages in the cluster, 1161 * rather than unmapping them. If we encounter the "check_page" that vmscan is 1162 * trying to unmap, return SWAP_MLOCK, else default SWAP_AGAIN. 1163 */ 1164#define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE) 1165#define CLUSTER_MASK (~(CLUSTER_SIZE - 1)) 1166 1167static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, 1168 struct vm_area_struct *vma, struct page *check_page) 1169{ 1170 struct mm_struct *mm = vma->vm_mm; 1171 pgd_t *pgd; 1172 pud_t *pud; 1173 pmd_t *pmd; 1174 pte_t *pte; 1175 pte_t pteval; 1176 spinlock_t *ptl; 1177 struct page *page; 1178 unsigned long address; 1179 unsigned long end; 1180 int ret = SWAP_AGAIN; 1181 int locked_vma = 0; 1182 1183 address = (vma->vm_start + cursor) & CLUSTER_MASK; 1184 end = address + CLUSTER_SIZE; 1185 if (address < vma->vm_start) 1186 address = vma->vm_start; 1187 if (end > vma->vm_end) 1188 end = vma->vm_end; 1189 1190 pgd = pgd_offset(mm, address); 1191 if (!pgd_present(*pgd)) 1192 return ret; 1193 1194 pud = pud_offset(pgd, address); 1195 if (!pud_present(*pud)) 1196 return ret; 1197 1198 pmd = pmd_offset(pud, address); 1199 if (!pmd_present(*pmd)) 1200 return ret; 1201 1202 /* 1203 * If we can acquire the mmap_sem for read, and vma is VM_LOCKED, 1204 * keep the sem while scanning the cluster for mlocking pages. 1205 */ 1206 if (down_read_trylock(&vma->vm_mm->mmap_sem)) { 1207 locked_vma = (vma->vm_flags & VM_LOCKED); 1208 if (!locked_vma) 1209 up_read(&vma->vm_mm->mmap_sem); /* don't need it */ 1210 } 1211 1212 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 1213 1214 /* Update high watermark before we lower rss */ 1215 update_hiwater_rss(mm); 1216 1217 for (; address < end; pte++, address += PAGE_SIZE) { 1218 if (!pte_present(*pte)) 1219 continue; 1220 page = vm_normal_page(vma, address, *pte); 1221 BUG_ON(!page || PageAnon(page)); 1222 1223 if (locked_vma) { 1224 mlock_vma_page(page); /* no-op if already mlocked */ 1225 if (page == check_page) 1226 ret = SWAP_MLOCK; 1227 continue; /* don't unmap */ 1228 } 1229 1230 if (ptep_clear_flush_young_notify(vma, address, pte)) 1231 continue; 1232 1233 /* Nuke the page table entry. */ 1234 flush_cache_page(vma, address, pte_pfn(*pte)); 1235 pteval = ptep_clear_flush_notify(vma, address, pte); 1236 1237 /* If nonlinear, store the file page offset in the pte. */ 1238 if (page->index != linear_page_index(vma, address)) 1239 set_pte_at(mm, address, pte, pgoff_to_pte(page->index)); 1240 1241 /* Move the dirty bit to the physical page now the pte is gone. */ 1242 if (pte_dirty(pteval)) 1243 set_page_dirty(page); 1244 1245 page_remove_rmap(page); 1246 page_cache_release(page); 1247 dec_mm_counter(mm, MM_FILEPAGES); 1248 (*mapcount)--; 1249 } 1250 pte_unmap_unlock(pte - 1, ptl); 1251 if (locked_vma) 1252 up_read(&vma->vm_mm->mmap_sem); 1253 return ret; 1254} 1255 1256bool is_vma_temporary_stack(struct vm_area_struct *vma) 1257{ 1258 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 1259 1260 if (!maybe_stack) 1261 return false; 1262 1263 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1264 VM_STACK_INCOMPLETE_SETUP) 1265 return true; 1266 1267 return false; 1268} 1269 1270/** 1271 * try_to_unmap_anon - unmap or unlock anonymous page using the object-based 1272 * rmap method 1273 * @page: the page to unmap/unlock 1274 * @flags: action and flags 1275 * 1276 * Find all the mappings of a page using the mapping pointer and the vma chains 1277 * contained in the anon_vma struct it points to. 1278 * 1279 * This function is only called from try_to_unmap/try_to_munlock for 1280 * anonymous pages. 1281 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1282 * where the page was found will be held for write. So, we won't recheck 1283 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1284 * 'LOCKED. 1285 */ 1286static int try_to_unmap_anon(struct page *page, enum ttu_flags flags) 1287{ 1288 struct anon_vma *anon_vma; 1289 struct anon_vma_chain *avc; 1290 int ret = SWAP_AGAIN; 1291 1292 anon_vma = page_lock_anon_vma(page); 1293 if (!anon_vma) 1294 return ret; 1295 1296 list_for_each_entry(avc, &anon_vma->head, same_anon_vma) { 1297 struct vm_area_struct *vma = avc->vma; 1298 unsigned long address; 1299 1300 /* 1301 * During exec, a temporary VMA is setup and later moved. 1302 * The VMA is moved under the anon_vma lock but not the 1303 * page tables leading to a race where migration cannot 1304 * find the migration ptes. Rather than increasing the 1305 * locking requirements of exec(), migration skips 1306 * temporary VMAs until after exec() completes. 1307 */ 1308 if (PAGE_MIGRATION && (flags & TTU_MIGRATION) && 1309 is_vma_temporary_stack(vma)) 1310 continue; 1311 1312 address = vma_address(page, vma); 1313 if (address == -EFAULT) 1314 continue; 1315 ret = try_to_unmap_one(page, vma, address, flags); 1316 if (ret != SWAP_AGAIN || !page_mapped(page)) 1317 break; 1318 } 1319 1320 page_unlock_anon_vma(anon_vma); 1321 return ret; 1322} 1323 1324/** 1325 * try_to_unmap_file - unmap/unlock file page using the object-based rmap method 1326 * @page: the page to unmap/unlock 1327 * @flags: action and flags 1328 * 1329 * Find all the mappings of a page using the mapping pointer and the vma chains 1330 * contained in the address_space struct it points to. 1331 * 1332 * This function is only called from try_to_unmap/try_to_munlock for 1333 * object-based pages. 1334 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1335 * where the page was found will be held for write. So, we won't recheck 1336 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1337 * 'LOCKED. 1338 */ 1339static int try_to_unmap_file(struct page *page, enum ttu_flags flags) 1340{ 1341 struct address_space *mapping = page->mapping; 1342 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 1343 struct vm_area_struct *vma; 1344 struct prio_tree_iter iter; 1345 int ret = SWAP_AGAIN; 1346 unsigned long cursor; 1347 unsigned long max_nl_cursor = 0; 1348 unsigned long max_nl_size = 0; 1349 unsigned int mapcount; 1350 1351 spin_lock(&mapping->i_mmap_lock); 1352 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 1353 unsigned long address = vma_address(page, vma); 1354 if (address == -EFAULT) 1355 continue; 1356 ret = try_to_unmap_one(page, vma, address, flags); 1357 if (ret != SWAP_AGAIN || !page_mapped(page)) 1358 goto out; 1359 } 1360 1361 if (list_empty(&mapping->i_mmap_nonlinear)) 1362 goto out; 1363 1364 /* 1365 * We don't bother to try to find the munlocked page in nonlinears. 1366 * It's costly. Instead, later, page reclaim logic may call 1367 * try_to_unmap(TTU_MUNLOCK) and recover PG_mlocked lazily. 1368 */ 1369 if (TTU_ACTION(flags) == TTU_MUNLOCK) 1370 goto out; 1371 1372 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, 1373 shared.vm_set.list) { 1374 cursor = (unsigned long) vma->vm_private_data; 1375 if (cursor > max_nl_cursor) 1376 max_nl_cursor = cursor; 1377 cursor = vma->vm_end - vma->vm_start; 1378 if (cursor > max_nl_size) 1379 max_nl_size = cursor; 1380 } 1381 1382 if (max_nl_size == 0) { /* all nonlinears locked or reserved ? */ 1383 ret = SWAP_FAIL; 1384 goto out; 1385 } 1386 1387 /* 1388 * We don't try to search for this page in the nonlinear vmas, 1389 * and page_referenced wouldn't have found it anyway. Instead 1390 * just walk the nonlinear vmas trying to age and unmap some. 1391 * The mapcount of the page we came in with is irrelevant, 1392 * but even so use it as a guide to how hard we should try? 1393 */ 1394 mapcount = page_mapcount(page); 1395 if (!mapcount) 1396 goto out; 1397 cond_resched_lock(&mapping->i_mmap_lock); 1398 1399 max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK; 1400 if (max_nl_cursor == 0) 1401 max_nl_cursor = CLUSTER_SIZE; 1402 1403 do { 1404 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, 1405 shared.vm_set.list) { 1406 cursor = (unsigned long) vma->vm_private_data; 1407 while ( cursor < max_nl_cursor && 1408 cursor < vma->vm_end - vma->vm_start) { 1409 if (try_to_unmap_cluster(cursor, &mapcount, 1410 vma, page) == SWAP_MLOCK) 1411 ret = SWAP_MLOCK; 1412 cursor += CLUSTER_SIZE; 1413 vma->vm_private_data = (void *) cursor; 1414 if ((int)mapcount <= 0) 1415 goto out; 1416 } 1417 vma->vm_private_data = (void *) max_nl_cursor; 1418 } 1419 cond_resched_lock(&mapping->i_mmap_lock); 1420 max_nl_cursor += CLUSTER_SIZE; 1421 } while (max_nl_cursor <= max_nl_size); 1422 1423 /* 1424 * Don't loop forever (perhaps all the remaining pages are 1425 * in locked vmas). Reset cursor on all unreserved nonlinear 1426 * vmas, now forgetting on which ones it had fallen behind. 1427 */ 1428 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list) 1429 vma->vm_private_data = NULL; 1430out: 1431 spin_unlock(&mapping->i_mmap_lock); 1432 return ret; 1433} 1434 1435/** 1436 * try_to_unmap - try to remove all page table mappings to a page 1437 * @page: the page to get unmapped 1438 * @flags: action and flags 1439 * 1440 * Tries to remove all the page table entries which are mapping this 1441 * page, used in the pageout path. Caller must hold the page lock. 1442 * Return values are: 1443 * 1444 * SWAP_SUCCESS - we succeeded in removing all mappings 1445 * SWAP_AGAIN - we missed a mapping, try again later 1446 * SWAP_FAIL - the page is unswappable 1447 * SWAP_MLOCK - page is mlocked. 1448 */ 1449int try_to_unmap(struct page *page, enum ttu_flags flags) 1450{ 1451 int ret; 1452 1453 BUG_ON(!PageLocked(page)); 1454 VM_BUG_ON(!PageHuge(page) && PageTransHuge(page)); 1455 1456 if (unlikely(PageKsm(page))) 1457 ret = try_to_unmap_ksm(page, flags); 1458 else if (PageAnon(page)) 1459 ret = try_to_unmap_anon(page, flags); 1460 else 1461 ret = try_to_unmap_file(page, flags); 1462 if (ret != SWAP_MLOCK && !page_mapped(page)) 1463 ret = SWAP_SUCCESS; 1464 return ret; 1465} 1466 1467/** 1468 * try_to_munlock - try to munlock a page 1469 * @page: the page to be munlocked 1470 * 1471 * Called from munlock code. Checks all of the VMAs mapping the page 1472 * to make sure nobody else has this page mlocked. The page will be 1473 * returned with PG_mlocked cleared if no other vmas have it mlocked. 1474 * 1475 * Return values are: 1476 * 1477 * SWAP_AGAIN - no vma is holding page mlocked, or, 1478 * SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem 1479 * SWAP_FAIL - page cannot be located at present 1480 * SWAP_MLOCK - page is now mlocked. 1481 */ 1482int try_to_munlock(struct page *page) 1483{ 1484 VM_BUG_ON(!PageLocked(page) || PageLRU(page)); 1485 1486 if (unlikely(PageKsm(page))) 1487 return try_to_unmap_ksm(page, TTU_MUNLOCK); 1488 else if (PageAnon(page)) 1489 return try_to_unmap_anon(page, TTU_MUNLOCK); 1490 else 1491 return try_to_unmap_file(page, TTU_MUNLOCK); 1492} 1493 1494void __put_anon_vma(struct anon_vma *anon_vma) 1495{ 1496 struct anon_vma *root = anon_vma->root; 1497 1498 if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 1499 anon_vma_free(root); 1500 1501 anon_vma_free(anon_vma); 1502} 1503 1504#ifdef CONFIG_MIGRATION 1505/* 1506 * rmap_walk() and its helpers rmap_walk_anon() and rmap_walk_file(): 1507 * Called by migrate.c to remove migration ptes, but might be used more later. 1508 */ 1509static int rmap_walk_anon(struct page *page, int (*rmap_one)(struct page *, 1510 struct vm_area_struct *, unsigned long, void *), void *arg) 1511{ 1512 struct anon_vma *anon_vma; 1513 struct anon_vma_chain *avc; 1514 int ret = SWAP_AGAIN; 1515 1516 /* 1517 * Note: remove_migration_ptes() cannot use page_lock_anon_vma() 1518 * because that depends on page_mapped(); but not all its usages 1519 * are holding mmap_sem. Users without mmap_sem are required to 1520 * take a reference count to prevent the anon_vma disappearing 1521 */ 1522 anon_vma = page_anon_vma(page); 1523 if (!anon_vma) 1524 return ret; 1525 anon_vma_lock(anon_vma); 1526 list_for_each_entry(avc, &anon_vma->head, same_anon_vma) { 1527 struct vm_area_struct *vma = avc->vma; 1528 unsigned long address = vma_address(page, vma); 1529 if (address == -EFAULT) 1530 continue; 1531 ret = rmap_one(page, vma, address, arg); 1532 if (ret != SWAP_AGAIN) 1533 break; 1534 } 1535 anon_vma_unlock(anon_vma); 1536 return ret; 1537} 1538 1539static int rmap_walk_file(struct page *page, int (*rmap_one)(struct page *, 1540 struct vm_area_struct *, unsigned long, void *), void *arg) 1541{ 1542 struct address_space *mapping = page->mapping; 1543 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 1544 struct vm_area_struct *vma; 1545 struct prio_tree_iter iter; 1546 int ret = SWAP_AGAIN; 1547 1548 if (!mapping) 1549 return ret; 1550 spin_lock(&mapping->i_mmap_lock); 1551 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 1552 unsigned long address = vma_address(page, vma); 1553 if (address == -EFAULT) 1554 continue; 1555 ret = rmap_one(page, vma, address, arg); 1556 if (ret != SWAP_AGAIN) 1557 break; 1558 } 1559 /* 1560 * No nonlinear handling: being always shared, nonlinear vmas 1561 * never contain migration ptes. Decide what to do about this 1562 * limitation to linear when we need rmap_walk() on nonlinear. 1563 */ 1564 spin_unlock(&mapping->i_mmap_lock); 1565 return ret; 1566} 1567 1568int rmap_walk(struct page *page, int (*rmap_one)(struct page *, 1569 struct vm_area_struct *, unsigned long, void *), void *arg) 1570{ 1571 VM_BUG_ON(!PageLocked(page)); 1572 1573 if (unlikely(PageKsm(page))) 1574 return rmap_walk_ksm(page, rmap_one, arg); 1575 else if (PageAnon(page)) 1576 return rmap_walk_anon(page, rmap_one, arg); 1577 else 1578 return rmap_walk_file(page, rmap_one, arg); 1579} 1580#endif /* CONFIG_MIGRATION */ 1581 1582#ifdef CONFIG_HUGETLB_PAGE 1583/* 1584 * The following three functions are for anonymous (private mapped) hugepages. 1585 * Unlike common anonymous pages, anonymous hugepages have no accounting code 1586 * and no lru code, because we handle hugepages differently from common pages. 1587 */ 1588static void __hugepage_set_anon_rmap(struct page *page, 1589 struct vm_area_struct *vma, unsigned long address, int exclusive) 1590{ 1591 struct anon_vma *anon_vma = vma->anon_vma; 1592 1593 BUG_ON(!anon_vma); 1594 1595 if (PageAnon(page)) 1596 return; 1597 if (!exclusive) 1598 anon_vma = anon_vma->root; 1599 1600 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1601 page->mapping = (struct address_space *) anon_vma; 1602 page->index = linear_page_index(vma, address); 1603} 1604 1605void hugepage_add_anon_rmap(struct page *page, 1606 struct vm_area_struct *vma, unsigned long address) 1607{ 1608 struct anon_vma *anon_vma = vma->anon_vma; 1609 int first; 1610 1611 BUG_ON(!PageLocked(page)); 1612 BUG_ON(!anon_vma); 1613 BUG_ON(address < vma->vm_start || address >= vma->vm_end); 1614 first = atomic_inc_and_test(&page->_mapcount); 1615 if (first) 1616 __hugepage_set_anon_rmap(page, vma, address, 0); 1617} 1618 1619void hugepage_add_new_anon_rmap(struct page *page, 1620 struct vm_area_struct *vma, unsigned long address) 1621{ 1622 BUG_ON(address < vma->vm_start || address >= vma->vm_end); 1623 atomic_set(&page->_mapcount, 0); 1624 __hugepage_set_anon_rmap(page, vma, address, 1); 1625} 1626#endif /* CONFIG_HUGETLB_PAGE */ 1627