rmap.c revision 21d0d443cdc1658a8c1484fdcece4803f0f96d0e
1/* 2 * mm/rmap.c - physical to virtual reverse mappings 3 * 4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 5 * Released under the General Public License (GPL). 6 * 7 * Simple, low overhead reverse mapping scheme. 8 * Please try to keep this thing as modular as possible. 9 * 10 * Provides methods for unmapping each kind of mapped page: 11 * the anon methods track anonymous pages, and 12 * the file methods track pages belonging to an inode. 13 * 14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001 15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 17 * Contributions by Hugh Dickins 2003, 2004 18 */ 19 20/* 21 * Lock ordering in mm: 22 * 23 * inode->i_mutex (while writing or truncating, not reading or faulting) 24 * inode->i_alloc_sem (vmtruncate_range) 25 * mm->mmap_sem 26 * page->flags PG_locked (lock_page) 27 * mapping->i_mmap_lock 28 * anon_vma->lock 29 * mm->page_table_lock or pte_lock 30 * zone->lru_lock (in mark_page_accessed, isolate_lru_page) 31 * swap_lock (in swap_duplicate, swap_info_get) 32 * mmlist_lock (in mmput, drain_mmlist and others) 33 * mapping->private_lock (in __set_page_dirty_buffers) 34 * inode_lock (in set_page_dirty's __mark_inode_dirty) 35 * sb_lock (within inode_lock in fs/fs-writeback.c) 36 * mapping->tree_lock (widely used, in set_page_dirty, 37 * in arch-dependent flush_dcache_mmap_lock, 38 * within inode_lock in __sync_single_inode) 39 * 40 * (code doesn't rely on that order so it could be switched around) 41 * ->tasklist_lock 42 * anon_vma->lock (memory_failure, collect_procs_anon) 43 * pte map lock 44 */ 45 46#include <linux/mm.h> 47#include <linux/pagemap.h> 48#include <linux/swap.h> 49#include <linux/swapops.h> 50#include <linux/slab.h> 51#include <linux/init.h> 52#include <linux/ksm.h> 53#include <linux/rmap.h> 54#include <linux/rcupdate.h> 55#include <linux/module.h> 56#include <linux/memcontrol.h> 57#include <linux/mmu_notifier.h> 58#include <linux/migrate.h> 59 60#include <asm/tlbflush.h> 61 62#include "internal.h" 63 64static struct kmem_cache *anon_vma_cachep; 65static struct kmem_cache *anon_vma_chain_cachep; 66 67static inline struct anon_vma *anon_vma_alloc(void) 68{ 69 return kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 70} 71 72void anon_vma_free(struct anon_vma *anon_vma) 73{ 74 kmem_cache_free(anon_vma_cachep, anon_vma); 75} 76 77static inline struct anon_vma_chain *anon_vma_chain_alloc(void) 78{ 79 return kmem_cache_alloc(anon_vma_chain_cachep, GFP_KERNEL); 80} 81 82void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 83{ 84 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 85} 86 87/** 88 * anon_vma_prepare - attach an anon_vma to a memory region 89 * @vma: the memory region in question 90 * 91 * This makes sure the memory mapping described by 'vma' has 92 * an 'anon_vma' attached to it, so that we can associate the 93 * anonymous pages mapped into it with that anon_vma. 94 * 95 * The common case will be that we already have one, but if 96 * if not we either need to find an adjacent mapping that we 97 * can re-use the anon_vma from (very common when the only 98 * reason for splitting a vma has been mprotect()), or we 99 * allocate a new one. 100 * 101 * Anon-vma allocations are very subtle, because we may have 102 * optimistically looked up an anon_vma in page_lock_anon_vma() 103 * and that may actually touch the spinlock even in the newly 104 * allocated vma (it depends on RCU to make sure that the 105 * anon_vma isn't actually destroyed). 106 * 107 * As a result, we need to do proper anon_vma locking even 108 * for the new allocation. At the same time, we do not want 109 * to do any locking for the common case of already having 110 * an anon_vma. 111 * 112 * This must be called with the mmap_sem held for reading. 113 */ 114int anon_vma_prepare(struct vm_area_struct *vma) 115{ 116 struct anon_vma *anon_vma = vma->anon_vma; 117 struct anon_vma_chain *avc; 118 119 might_sleep(); 120 if (unlikely(!anon_vma)) { 121 struct mm_struct *mm = vma->vm_mm; 122 struct anon_vma *allocated; 123 124 avc = anon_vma_chain_alloc(); 125 if (!avc) 126 goto out_enomem; 127 128 anon_vma = find_mergeable_anon_vma(vma); 129 allocated = NULL; 130 if (!anon_vma) { 131 anon_vma = anon_vma_alloc(); 132 if (unlikely(!anon_vma)) 133 goto out_enomem_free_avc; 134 allocated = anon_vma; 135 /* 136 * This VMA had no anon_vma yet. This anon_vma is 137 * the root of any anon_vma tree that might form. 138 */ 139 anon_vma->root = anon_vma; 140 } 141 142 anon_vma_lock(anon_vma); 143 /* page_table_lock to protect against threads */ 144 spin_lock(&mm->page_table_lock); 145 if (likely(!vma->anon_vma)) { 146 vma->anon_vma = anon_vma; 147 avc->anon_vma = anon_vma; 148 avc->vma = vma; 149 list_add(&avc->same_vma, &vma->anon_vma_chain); 150 list_add_tail(&avc->same_anon_vma, &anon_vma->head); 151 allocated = NULL; 152 avc = NULL; 153 } 154 spin_unlock(&mm->page_table_lock); 155 anon_vma_unlock(anon_vma); 156 157 if (unlikely(allocated)) 158 anon_vma_free(allocated); 159 if (unlikely(avc)) 160 anon_vma_chain_free(avc); 161 } 162 return 0; 163 164 out_enomem_free_avc: 165 anon_vma_chain_free(avc); 166 out_enomem: 167 return -ENOMEM; 168} 169 170static void anon_vma_chain_link(struct vm_area_struct *vma, 171 struct anon_vma_chain *avc, 172 struct anon_vma *anon_vma) 173{ 174 avc->vma = vma; 175 avc->anon_vma = anon_vma; 176 list_add(&avc->same_vma, &vma->anon_vma_chain); 177 178 anon_vma_lock(anon_vma); 179 list_add_tail(&avc->same_anon_vma, &anon_vma->head); 180 anon_vma_unlock(anon_vma); 181} 182 183/* 184 * Attach the anon_vmas from src to dst. 185 * Returns 0 on success, -ENOMEM on failure. 186 */ 187int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 188{ 189 struct anon_vma_chain *avc, *pavc; 190 191 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 192 avc = anon_vma_chain_alloc(); 193 if (!avc) 194 goto enomem_failure; 195 anon_vma_chain_link(dst, avc, pavc->anon_vma); 196 } 197 return 0; 198 199 enomem_failure: 200 unlink_anon_vmas(dst); 201 return -ENOMEM; 202} 203 204/* 205 * Attach vma to its own anon_vma, as well as to the anon_vmas that 206 * the corresponding VMA in the parent process is attached to. 207 * Returns 0 on success, non-zero on failure. 208 */ 209int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 210{ 211 struct anon_vma_chain *avc; 212 struct anon_vma *anon_vma; 213 214 /* Don't bother if the parent process has no anon_vma here. */ 215 if (!pvma->anon_vma) 216 return 0; 217 218 /* 219 * First, attach the new VMA to the parent VMA's anon_vmas, 220 * so rmap can find non-COWed pages in child processes. 221 */ 222 if (anon_vma_clone(vma, pvma)) 223 return -ENOMEM; 224 225 /* Then add our own anon_vma. */ 226 anon_vma = anon_vma_alloc(); 227 if (!anon_vma) 228 goto out_error; 229 avc = anon_vma_chain_alloc(); 230 if (!avc) 231 goto out_error_free_anon_vma; 232 233 /* 234 * The root anon_vma's spinlock is the lock actually used when we 235 * lock any of the anon_vmas in this anon_vma tree. 236 */ 237 anon_vma->root = pvma->anon_vma->root; 238 /* 239 * With KSM refcounts, an anon_vma can stay around longer than the 240 * process it belongs to. The root anon_vma needs to be pinned 241 * until this anon_vma is freed, because the lock lives in the root. 242 */ 243 get_anon_vma(anon_vma->root); 244 /* Mark this anon_vma as the one where our new (COWed) pages go. */ 245 vma->anon_vma = anon_vma; 246 anon_vma_chain_link(vma, avc, anon_vma); 247 248 return 0; 249 250 out_error_free_anon_vma: 251 anon_vma_free(anon_vma); 252 out_error: 253 unlink_anon_vmas(vma); 254 return -ENOMEM; 255} 256 257static void anon_vma_unlink(struct anon_vma_chain *anon_vma_chain) 258{ 259 struct anon_vma *anon_vma = anon_vma_chain->anon_vma; 260 int empty; 261 262 /* If anon_vma_fork fails, we can get an empty anon_vma_chain. */ 263 if (!anon_vma) 264 return; 265 266 anon_vma_lock(anon_vma); 267 list_del(&anon_vma_chain->same_anon_vma); 268 269 /* We must garbage collect the anon_vma if it's empty */ 270 empty = list_empty(&anon_vma->head) && !anonvma_external_refcount(anon_vma); 271 anon_vma_unlock(anon_vma); 272 273 if (empty) { 274 /* We no longer need the root anon_vma */ 275 if (anon_vma->root != anon_vma) 276 drop_anon_vma(anon_vma->root); 277 anon_vma_free(anon_vma); 278 } 279} 280 281void unlink_anon_vmas(struct vm_area_struct *vma) 282{ 283 struct anon_vma_chain *avc, *next; 284 285 /* 286 * Unlink each anon_vma chained to the VMA. This list is ordered 287 * from newest to oldest, ensuring the root anon_vma gets freed last. 288 */ 289 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 290 anon_vma_unlink(avc); 291 list_del(&avc->same_vma); 292 anon_vma_chain_free(avc); 293 } 294} 295 296static void anon_vma_ctor(void *data) 297{ 298 struct anon_vma *anon_vma = data; 299 300 spin_lock_init(&anon_vma->lock); 301 anonvma_external_refcount_init(anon_vma); 302 INIT_LIST_HEAD(&anon_vma->head); 303} 304 305void __init anon_vma_init(void) 306{ 307 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 308 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor); 309 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC); 310} 311 312/* 313 * Getting a lock on a stable anon_vma from a page off the LRU is 314 * tricky: page_lock_anon_vma rely on RCU to guard against the races. 315 */ 316struct anon_vma *page_lock_anon_vma(struct page *page) 317{ 318 struct anon_vma *anon_vma; 319 unsigned long anon_mapping; 320 321 rcu_read_lock(); 322 anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping); 323 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 324 goto out; 325 if (!page_mapped(page)) 326 goto out; 327 328 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 329 anon_vma_lock(anon_vma); 330 return anon_vma; 331out: 332 rcu_read_unlock(); 333 return NULL; 334} 335 336void page_unlock_anon_vma(struct anon_vma *anon_vma) 337{ 338 anon_vma_unlock(anon_vma); 339 rcu_read_unlock(); 340} 341 342/* 343 * At what user virtual address is page expected in @vma? 344 * Returns virtual address or -EFAULT if page's index/offset is not 345 * within the range mapped the @vma. 346 */ 347static inline unsigned long 348vma_address(struct page *page, struct vm_area_struct *vma) 349{ 350 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 351 unsigned long address; 352 353 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 354 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) { 355 /* page should be within @vma mapping range */ 356 return -EFAULT; 357 } 358 return address; 359} 360 361/* 362 * At what user virtual address is page expected in vma? 363 * Caller should check the page is actually part of the vma. 364 */ 365unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 366{ 367 if (PageAnon(page)) { 368 if (vma->anon_vma->root != page_anon_vma(page)->root) 369 return -EFAULT; 370 } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) { 371 if (!vma->vm_file || 372 vma->vm_file->f_mapping != page->mapping) 373 return -EFAULT; 374 } else 375 return -EFAULT; 376 return vma_address(page, vma); 377} 378 379/* 380 * Check that @page is mapped at @address into @mm. 381 * 382 * If @sync is false, page_check_address may perform a racy check to avoid 383 * the page table lock when the pte is not present (helpful when reclaiming 384 * highly shared pages). 385 * 386 * On success returns with pte mapped and locked. 387 */ 388pte_t *page_check_address(struct page *page, struct mm_struct *mm, 389 unsigned long address, spinlock_t **ptlp, int sync) 390{ 391 pgd_t *pgd; 392 pud_t *pud; 393 pmd_t *pmd; 394 pte_t *pte; 395 spinlock_t *ptl; 396 397 pgd = pgd_offset(mm, address); 398 if (!pgd_present(*pgd)) 399 return NULL; 400 401 pud = pud_offset(pgd, address); 402 if (!pud_present(*pud)) 403 return NULL; 404 405 pmd = pmd_offset(pud, address); 406 if (!pmd_present(*pmd)) 407 return NULL; 408 409 pte = pte_offset_map(pmd, address); 410 /* Make a quick check before getting the lock */ 411 if (!sync && !pte_present(*pte)) { 412 pte_unmap(pte); 413 return NULL; 414 } 415 416 ptl = pte_lockptr(mm, pmd); 417 spin_lock(ptl); 418 if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) { 419 *ptlp = ptl; 420 return pte; 421 } 422 pte_unmap_unlock(pte, ptl); 423 return NULL; 424} 425 426/** 427 * page_mapped_in_vma - check whether a page is really mapped in a VMA 428 * @page: the page to test 429 * @vma: the VMA to test 430 * 431 * Returns 1 if the page is mapped into the page tables of the VMA, 0 432 * if the page is not mapped into the page tables of this VMA. Only 433 * valid for normal file or anonymous VMAs. 434 */ 435int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) 436{ 437 unsigned long address; 438 pte_t *pte; 439 spinlock_t *ptl; 440 441 address = vma_address(page, vma); 442 if (address == -EFAULT) /* out of vma range */ 443 return 0; 444 pte = page_check_address(page, vma->vm_mm, address, &ptl, 1); 445 if (!pte) /* the page is not in this mm */ 446 return 0; 447 pte_unmap_unlock(pte, ptl); 448 449 return 1; 450} 451 452/* 453 * Subfunctions of page_referenced: page_referenced_one called 454 * repeatedly from either page_referenced_anon or page_referenced_file. 455 */ 456int page_referenced_one(struct page *page, struct vm_area_struct *vma, 457 unsigned long address, unsigned int *mapcount, 458 unsigned long *vm_flags) 459{ 460 struct mm_struct *mm = vma->vm_mm; 461 pte_t *pte; 462 spinlock_t *ptl; 463 int referenced = 0; 464 465 pte = page_check_address(page, mm, address, &ptl, 0); 466 if (!pte) 467 goto out; 468 469 /* 470 * Don't want to elevate referenced for mlocked page that gets this far, 471 * in order that it progresses to try_to_unmap and is moved to the 472 * unevictable list. 473 */ 474 if (vma->vm_flags & VM_LOCKED) { 475 *mapcount = 1; /* break early from loop */ 476 *vm_flags |= VM_LOCKED; 477 goto out_unmap; 478 } 479 480 if (ptep_clear_flush_young_notify(vma, address, pte)) { 481 /* 482 * Don't treat a reference through a sequentially read 483 * mapping as such. If the page has been used in 484 * another mapping, we will catch it; if this other 485 * mapping is already gone, the unmap path will have 486 * set PG_referenced or activated the page. 487 */ 488 if (likely(!VM_SequentialReadHint(vma))) 489 referenced++; 490 } 491 492 /* Pretend the page is referenced if the task has the 493 swap token and is in the middle of a page fault. */ 494 if (mm != current->mm && has_swap_token(mm) && 495 rwsem_is_locked(&mm->mmap_sem)) 496 referenced++; 497 498out_unmap: 499 (*mapcount)--; 500 pte_unmap_unlock(pte, ptl); 501 502 if (referenced) 503 *vm_flags |= vma->vm_flags; 504out: 505 return referenced; 506} 507 508static int page_referenced_anon(struct page *page, 509 struct mem_cgroup *mem_cont, 510 unsigned long *vm_flags) 511{ 512 unsigned int mapcount; 513 struct anon_vma *anon_vma; 514 struct anon_vma_chain *avc; 515 int referenced = 0; 516 517 anon_vma = page_lock_anon_vma(page); 518 if (!anon_vma) 519 return referenced; 520 521 mapcount = page_mapcount(page); 522 list_for_each_entry(avc, &anon_vma->head, same_anon_vma) { 523 struct vm_area_struct *vma = avc->vma; 524 unsigned long address = vma_address(page, vma); 525 if (address == -EFAULT) 526 continue; 527 /* 528 * If we are reclaiming on behalf of a cgroup, skip 529 * counting on behalf of references from different 530 * cgroups 531 */ 532 if (mem_cont && !mm_match_cgroup(vma->vm_mm, mem_cont)) 533 continue; 534 referenced += page_referenced_one(page, vma, address, 535 &mapcount, vm_flags); 536 if (!mapcount) 537 break; 538 } 539 540 page_unlock_anon_vma(anon_vma); 541 return referenced; 542} 543 544/** 545 * page_referenced_file - referenced check for object-based rmap 546 * @page: the page we're checking references on. 547 * @mem_cont: target memory controller 548 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 549 * 550 * For an object-based mapped page, find all the places it is mapped and 551 * check/clear the referenced flag. This is done by following the page->mapping 552 * pointer, then walking the chain of vmas it holds. It returns the number 553 * of references it found. 554 * 555 * This function is only called from page_referenced for object-based pages. 556 */ 557static int page_referenced_file(struct page *page, 558 struct mem_cgroup *mem_cont, 559 unsigned long *vm_flags) 560{ 561 unsigned int mapcount; 562 struct address_space *mapping = page->mapping; 563 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 564 struct vm_area_struct *vma; 565 struct prio_tree_iter iter; 566 int referenced = 0; 567 568 /* 569 * The caller's checks on page->mapping and !PageAnon have made 570 * sure that this is a file page: the check for page->mapping 571 * excludes the case just before it gets set on an anon page. 572 */ 573 BUG_ON(PageAnon(page)); 574 575 /* 576 * The page lock not only makes sure that page->mapping cannot 577 * suddenly be NULLified by truncation, it makes sure that the 578 * structure at mapping cannot be freed and reused yet, 579 * so we can safely take mapping->i_mmap_lock. 580 */ 581 BUG_ON(!PageLocked(page)); 582 583 spin_lock(&mapping->i_mmap_lock); 584 585 /* 586 * i_mmap_lock does not stabilize mapcount at all, but mapcount 587 * is more likely to be accurate if we note it after spinning. 588 */ 589 mapcount = page_mapcount(page); 590 591 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 592 unsigned long address = vma_address(page, vma); 593 if (address == -EFAULT) 594 continue; 595 /* 596 * If we are reclaiming on behalf of a cgroup, skip 597 * counting on behalf of references from different 598 * cgroups 599 */ 600 if (mem_cont && !mm_match_cgroup(vma->vm_mm, mem_cont)) 601 continue; 602 referenced += page_referenced_one(page, vma, address, 603 &mapcount, vm_flags); 604 if (!mapcount) 605 break; 606 } 607 608 spin_unlock(&mapping->i_mmap_lock); 609 return referenced; 610} 611 612/** 613 * page_referenced - test if the page was referenced 614 * @page: the page to test 615 * @is_locked: caller holds lock on the page 616 * @mem_cont: target memory controller 617 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 618 * 619 * Quick test_and_clear_referenced for all mappings to a page, 620 * returns the number of ptes which referenced the page. 621 */ 622int page_referenced(struct page *page, 623 int is_locked, 624 struct mem_cgroup *mem_cont, 625 unsigned long *vm_flags) 626{ 627 int referenced = 0; 628 int we_locked = 0; 629 630 *vm_flags = 0; 631 if (page_mapped(page) && page_rmapping(page)) { 632 if (!is_locked && (!PageAnon(page) || PageKsm(page))) { 633 we_locked = trylock_page(page); 634 if (!we_locked) { 635 referenced++; 636 goto out; 637 } 638 } 639 if (unlikely(PageKsm(page))) 640 referenced += page_referenced_ksm(page, mem_cont, 641 vm_flags); 642 else if (PageAnon(page)) 643 referenced += page_referenced_anon(page, mem_cont, 644 vm_flags); 645 else if (page->mapping) 646 referenced += page_referenced_file(page, mem_cont, 647 vm_flags); 648 if (we_locked) 649 unlock_page(page); 650 } 651out: 652 if (page_test_and_clear_young(page)) 653 referenced++; 654 655 return referenced; 656} 657 658static int page_mkclean_one(struct page *page, struct vm_area_struct *vma, 659 unsigned long address) 660{ 661 struct mm_struct *mm = vma->vm_mm; 662 pte_t *pte; 663 spinlock_t *ptl; 664 int ret = 0; 665 666 pte = page_check_address(page, mm, address, &ptl, 1); 667 if (!pte) 668 goto out; 669 670 if (pte_dirty(*pte) || pte_write(*pte)) { 671 pte_t entry; 672 673 flush_cache_page(vma, address, pte_pfn(*pte)); 674 entry = ptep_clear_flush_notify(vma, address, pte); 675 entry = pte_wrprotect(entry); 676 entry = pte_mkclean(entry); 677 set_pte_at(mm, address, pte, entry); 678 ret = 1; 679 } 680 681 pte_unmap_unlock(pte, ptl); 682out: 683 return ret; 684} 685 686static int page_mkclean_file(struct address_space *mapping, struct page *page) 687{ 688 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 689 struct vm_area_struct *vma; 690 struct prio_tree_iter iter; 691 int ret = 0; 692 693 BUG_ON(PageAnon(page)); 694 695 spin_lock(&mapping->i_mmap_lock); 696 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 697 if (vma->vm_flags & VM_SHARED) { 698 unsigned long address = vma_address(page, vma); 699 if (address == -EFAULT) 700 continue; 701 ret += page_mkclean_one(page, vma, address); 702 } 703 } 704 spin_unlock(&mapping->i_mmap_lock); 705 return ret; 706} 707 708int page_mkclean(struct page *page) 709{ 710 int ret = 0; 711 712 BUG_ON(!PageLocked(page)); 713 714 if (page_mapped(page)) { 715 struct address_space *mapping = page_mapping(page); 716 if (mapping) { 717 ret = page_mkclean_file(mapping, page); 718 if (page_test_dirty(page)) { 719 page_clear_dirty(page); 720 ret = 1; 721 } 722 } 723 } 724 725 return ret; 726} 727EXPORT_SYMBOL_GPL(page_mkclean); 728 729/** 730 * page_move_anon_rmap - move a page to our anon_vma 731 * @page: the page to move to our anon_vma 732 * @vma: the vma the page belongs to 733 * @address: the user virtual address mapped 734 * 735 * When a page belongs exclusively to one process after a COW event, 736 * that page can be moved into the anon_vma that belongs to just that 737 * process, so the rmap code will not search the parent or sibling 738 * processes. 739 */ 740void page_move_anon_rmap(struct page *page, 741 struct vm_area_struct *vma, unsigned long address) 742{ 743 struct anon_vma *anon_vma = vma->anon_vma; 744 745 VM_BUG_ON(!PageLocked(page)); 746 VM_BUG_ON(!anon_vma); 747 VM_BUG_ON(page->index != linear_page_index(vma, address)); 748 749 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 750 page->mapping = (struct address_space *) anon_vma; 751} 752 753/** 754 * __page_set_anon_rmap - setup new anonymous rmap 755 * @page: the page to add the mapping to 756 * @vma: the vm area in which the mapping is added 757 * @address: the user virtual address mapped 758 * @exclusive: the page is exclusively owned by the current process 759 */ 760static void __page_set_anon_rmap(struct page *page, 761 struct vm_area_struct *vma, unsigned long address, int exclusive) 762{ 763 struct anon_vma *anon_vma = vma->anon_vma; 764 765 BUG_ON(!anon_vma); 766 767 /* 768 * If the page isn't exclusively mapped into this vma, 769 * we must use the _oldest_ possible anon_vma for the 770 * page mapping! 771 */ 772 if (!exclusive) { 773 if (PageAnon(page)) 774 return; 775 anon_vma = anon_vma->root; 776 } else { 777 /* 778 * In this case, swapped-out-but-not-discarded swap-cache 779 * is remapped. So, no need to update page->mapping here. 780 * We convice anon_vma poitned by page->mapping is not obsolete 781 * because vma->anon_vma is necessary to be a family of it. 782 */ 783 if (PageAnon(page)) 784 return; 785 } 786 787 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 788 page->mapping = (struct address_space *) anon_vma; 789 page->index = linear_page_index(vma, address); 790} 791 792/** 793 * __page_check_anon_rmap - sanity check anonymous rmap addition 794 * @page: the page to add the mapping to 795 * @vma: the vm area in which the mapping is added 796 * @address: the user virtual address mapped 797 */ 798static void __page_check_anon_rmap(struct page *page, 799 struct vm_area_struct *vma, unsigned long address) 800{ 801#ifdef CONFIG_DEBUG_VM 802 /* 803 * The page's anon-rmap details (mapping and index) are guaranteed to 804 * be set up correctly at this point. 805 * 806 * We have exclusion against page_add_anon_rmap because the caller 807 * always holds the page locked, except if called from page_dup_rmap, 808 * in which case the page is already known to be setup. 809 * 810 * We have exclusion against page_add_new_anon_rmap because those pages 811 * are initially only visible via the pagetables, and the pte is locked 812 * over the call to page_add_new_anon_rmap. 813 */ 814 BUG_ON(page->index != linear_page_index(vma, address)); 815#endif 816} 817 818/** 819 * page_add_anon_rmap - add pte mapping to an anonymous page 820 * @page: the page to add the mapping to 821 * @vma: the vm area in which the mapping is added 822 * @address: the user virtual address mapped 823 * 824 * The caller needs to hold the pte lock, and the page must be locked in 825 * the anon_vma case: to serialize mapping,index checking after setting, 826 * and to ensure that PageAnon is not being upgraded racily to PageKsm 827 * (but PageKsm is never downgraded to PageAnon). 828 */ 829void page_add_anon_rmap(struct page *page, 830 struct vm_area_struct *vma, unsigned long address) 831{ 832 int first = atomic_inc_and_test(&page->_mapcount); 833 if (first) 834 __inc_zone_page_state(page, NR_ANON_PAGES); 835 if (unlikely(PageKsm(page))) 836 return; 837 838 VM_BUG_ON(!PageLocked(page)); 839 VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); 840 if (first) 841 __page_set_anon_rmap(page, vma, address, 0); 842 else 843 __page_check_anon_rmap(page, vma, address); 844} 845 846/** 847 * page_add_new_anon_rmap - add pte mapping to a new anonymous page 848 * @page: the page to add the mapping to 849 * @vma: the vm area in which the mapping is added 850 * @address: the user virtual address mapped 851 * 852 * Same as page_add_anon_rmap but must only be called on *new* pages. 853 * This means the inc-and-test can be bypassed. 854 * Page does not have to be locked. 855 */ 856void page_add_new_anon_rmap(struct page *page, 857 struct vm_area_struct *vma, unsigned long address) 858{ 859 VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); 860 SetPageSwapBacked(page); 861 atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */ 862 __inc_zone_page_state(page, NR_ANON_PAGES); 863 __page_set_anon_rmap(page, vma, address, 1); 864 if (page_evictable(page, vma)) 865 lru_cache_add_lru(page, LRU_ACTIVE_ANON); 866 else 867 add_page_to_unevictable_list(page); 868} 869 870/** 871 * page_add_file_rmap - add pte mapping to a file page 872 * @page: the page to add the mapping to 873 * 874 * The caller needs to hold the pte lock. 875 */ 876void page_add_file_rmap(struct page *page) 877{ 878 if (atomic_inc_and_test(&page->_mapcount)) { 879 __inc_zone_page_state(page, NR_FILE_MAPPED); 880 mem_cgroup_update_file_mapped(page, 1); 881 } 882} 883 884/** 885 * page_remove_rmap - take down pte mapping from a page 886 * @page: page to remove mapping from 887 * 888 * The caller needs to hold the pte lock. 889 */ 890void page_remove_rmap(struct page *page) 891{ 892 /* page still mapped by someone else? */ 893 if (!atomic_add_negative(-1, &page->_mapcount)) 894 return; 895 896 /* 897 * Now that the last pte has gone, s390 must transfer dirty 898 * flag from storage key to struct page. We can usually skip 899 * this if the page is anon, so about to be freed; but perhaps 900 * not if it's in swapcache - there might be another pte slot 901 * containing the swap entry, but page not yet written to swap. 902 */ 903 if ((!PageAnon(page) || PageSwapCache(page)) && page_test_dirty(page)) { 904 page_clear_dirty(page); 905 set_page_dirty(page); 906 } 907 if (PageAnon(page)) { 908 mem_cgroup_uncharge_page(page); 909 __dec_zone_page_state(page, NR_ANON_PAGES); 910 } else { 911 __dec_zone_page_state(page, NR_FILE_MAPPED); 912 mem_cgroup_update_file_mapped(page, -1); 913 } 914 /* 915 * It would be tidy to reset the PageAnon mapping here, 916 * but that might overwrite a racing page_add_anon_rmap 917 * which increments mapcount after us but sets mapping 918 * before us: so leave the reset to free_hot_cold_page, 919 * and remember that it's only reliable while mapped. 920 * Leaving it set also helps swapoff to reinstate ptes 921 * faster for those pages still in swapcache. 922 */ 923} 924 925/* 926 * Subfunctions of try_to_unmap: try_to_unmap_one called 927 * repeatedly from either try_to_unmap_anon or try_to_unmap_file. 928 */ 929int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, 930 unsigned long address, enum ttu_flags flags) 931{ 932 struct mm_struct *mm = vma->vm_mm; 933 pte_t *pte; 934 pte_t pteval; 935 spinlock_t *ptl; 936 int ret = SWAP_AGAIN; 937 938 pte = page_check_address(page, mm, address, &ptl, 0); 939 if (!pte) 940 goto out; 941 942 /* 943 * If the page is mlock()d, we cannot swap it out. 944 * If it's recently referenced (perhaps page_referenced 945 * skipped over this mm) then we should reactivate it. 946 */ 947 if (!(flags & TTU_IGNORE_MLOCK)) { 948 if (vma->vm_flags & VM_LOCKED) 949 goto out_mlock; 950 951 if (TTU_ACTION(flags) == TTU_MUNLOCK) 952 goto out_unmap; 953 } 954 if (!(flags & TTU_IGNORE_ACCESS)) { 955 if (ptep_clear_flush_young_notify(vma, address, pte)) { 956 ret = SWAP_FAIL; 957 goto out_unmap; 958 } 959 } 960 961 /* Nuke the page table entry. */ 962 flush_cache_page(vma, address, page_to_pfn(page)); 963 pteval = ptep_clear_flush_notify(vma, address, pte); 964 965 /* Move the dirty bit to the physical page now the pte is gone. */ 966 if (pte_dirty(pteval)) 967 set_page_dirty(page); 968 969 /* Update high watermark before we lower rss */ 970 update_hiwater_rss(mm); 971 972 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { 973 if (PageAnon(page)) 974 dec_mm_counter(mm, MM_ANONPAGES); 975 else 976 dec_mm_counter(mm, MM_FILEPAGES); 977 set_pte_at(mm, address, pte, 978 swp_entry_to_pte(make_hwpoison_entry(page))); 979 } else if (PageAnon(page)) { 980 swp_entry_t entry = { .val = page_private(page) }; 981 982 if (PageSwapCache(page)) { 983 /* 984 * Store the swap location in the pte. 985 * See handle_pte_fault() ... 986 */ 987 if (swap_duplicate(entry) < 0) { 988 set_pte_at(mm, address, pte, pteval); 989 ret = SWAP_FAIL; 990 goto out_unmap; 991 } 992 if (list_empty(&mm->mmlist)) { 993 spin_lock(&mmlist_lock); 994 if (list_empty(&mm->mmlist)) 995 list_add(&mm->mmlist, &init_mm.mmlist); 996 spin_unlock(&mmlist_lock); 997 } 998 dec_mm_counter(mm, MM_ANONPAGES); 999 inc_mm_counter(mm, MM_SWAPENTS); 1000 } else if (PAGE_MIGRATION) { 1001 /* 1002 * Store the pfn of the page in a special migration 1003 * pte. do_swap_page() will wait until the migration 1004 * pte is removed and then restart fault handling. 1005 */ 1006 BUG_ON(TTU_ACTION(flags) != TTU_MIGRATION); 1007 entry = make_migration_entry(page, pte_write(pteval)); 1008 } 1009 set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); 1010 BUG_ON(pte_file(*pte)); 1011 } else if (PAGE_MIGRATION && (TTU_ACTION(flags) == TTU_MIGRATION)) { 1012 /* Establish migration entry for a file page */ 1013 swp_entry_t entry; 1014 entry = make_migration_entry(page, pte_write(pteval)); 1015 set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); 1016 } else 1017 dec_mm_counter(mm, MM_FILEPAGES); 1018 1019 page_remove_rmap(page); 1020 page_cache_release(page); 1021 1022out_unmap: 1023 pte_unmap_unlock(pte, ptl); 1024out: 1025 return ret; 1026 1027out_mlock: 1028 pte_unmap_unlock(pte, ptl); 1029 1030 1031 /* 1032 * We need mmap_sem locking, Otherwise VM_LOCKED check makes 1033 * unstable result and race. Plus, We can't wait here because 1034 * we now hold anon_vma->lock or mapping->i_mmap_lock. 1035 * if trylock failed, the page remain in evictable lru and later 1036 * vmscan could retry to move the page to unevictable lru if the 1037 * page is actually mlocked. 1038 */ 1039 if (down_read_trylock(&vma->vm_mm->mmap_sem)) { 1040 if (vma->vm_flags & VM_LOCKED) { 1041 mlock_vma_page(page); 1042 ret = SWAP_MLOCK; 1043 } 1044 up_read(&vma->vm_mm->mmap_sem); 1045 } 1046 return ret; 1047} 1048 1049/* 1050 * objrmap doesn't work for nonlinear VMAs because the assumption that 1051 * offset-into-file correlates with offset-into-virtual-addresses does not hold. 1052 * Consequently, given a particular page and its ->index, we cannot locate the 1053 * ptes which are mapping that page without an exhaustive linear search. 1054 * 1055 * So what this code does is a mini "virtual scan" of each nonlinear VMA which 1056 * maps the file to which the target page belongs. The ->vm_private_data field 1057 * holds the current cursor into that scan. Successive searches will circulate 1058 * around the vma's virtual address space. 1059 * 1060 * So as more replacement pressure is applied to the pages in a nonlinear VMA, 1061 * more scanning pressure is placed against them as well. Eventually pages 1062 * will become fully unmapped and are eligible for eviction. 1063 * 1064 * For very sparsely populated VMAs this is a little inefficient - chances are 1065 * there there won't be many ptes located within the scan cluster. In this case 1066 * maybe we could scan further - to the end of the pte page, perhaps. 1067 * 1068 * Mlocked pages: check VM_LOCKED under mmap_sem held for read, if we can 1069 * acquire it without blocking. If vma locked, mlock the pages in the cluster, 1070 * rather than unmapping them. If we encounter the "check_page" that vmscan is 1071 * trying to unmap, return SWAP_MLOCK, else default SWAP_AGAIN. 1072 */ 1073#define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE) 1074#define CLUSTER_MASK (~(CLUSTER_SIZE - 1)) 1075 1076static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, 1077 struct vm_area_struct *vma, struct page *check_page) 1078{ 1079 struct mm_struct *mm = vma->vm_mm; 1080 pgd_t *pgd; 1081 pud_t *pud; 1082 pmd_t *pmd; 1083 pte_t *pte; 1084 pte_t pteval; 1085 spinlock_t *ptl; 1086 struct page *page; 1087 unsigned long address; 1088 unsigned long end; 1089 int ret = SWAP_AGAIN; 1090 int locked_vma = 0; 1091 1092 address = (vma->vm_start + cursor) & CLUSTER_MASK; 1093 end = address + CLUSTER_SIZE; 1094 if (address < vma->vm_start) 1095 address = vma->vm_start; 1096 if (end > vma->vm_end) 1097 end = vma->vm_end; 1098 1099 pgd = pgd_offset(mm, address); 1100 if (!pgd_present(*pgd)) 1101 return ret; 1102 1103 pud = pud_offset(pgd, address); 1104 if (!pud_present(*pud)) 1105 return ret; 1106 1107 pmd = pmd_offset(pud, address); 1108 if (!pmd_present(*pmd)) 1109 return ret; 1110 1111 /* 1112 * If we can acquire the mmap_sem for read, and vma is VM_LOCKED, 1113 * keep the sem while scanning the cluster for mlocking pages. 1114 */ 1115 if (down_read_trylock(&vma->vm_mm->mmap_sem)) { 1116 locked_vma = (vma->vm_flags & VM_LOCKED); 1117 if (!locked_vma) 1118 up_read(&vma->vm_mm->mmap_sem); /* don't need it */ 1119 } 1120 1121 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 1122 1123 /* Update high watermark before we lower rss */ 1124 update_hiwater_rss(mm); 1125 1126 for (; address < end; pte++, address += PAGE_SIZE) { 1127 if (!pte_present(*pte)) 1128 continue; 1129 page = vm_normal_page(vma, address, *pte); 1130 BUG_ON(!page || PageAnon(page)); 1131 1132 if (locked_vma) { 1133 mlock_vma_page(page); /* no-op if already mlocked */ 1134 if (page == check_page) 1135 ret = SWAP_MLOCK; 1136 continue; /* don't unmap */ 1137 } 1138 1139 if (ptep_clear_flush_young_notify(vma, address, pte)) 1140 continue; 1141 1142 /* Nuke the page table entry. */ 1143 flush_cache_page(vma, address, pte_pfn(*pte)); 1144 pteval = ptep_clear_flush_notify(vma, address, pte); 1145 1146 /* If nonlinear, store the file page offset in the pte. */ 1147 if (page->index != linear_page_index(vma, address)) 1148 set_pte_at(mm, address, pte, pgoff_to_pte(page->index)); 1149 1150 /* Move the dirty bit to the physical page now the pte is gone. */ 1151 if (pte_dirty(pteval)) 1152 set_page_dirty(page); 1153 1154 page_remove_rmap(page); 1155 page_cache_release(page); 1156 dec_mm_counter(mm, MM_FILEPAGES); 1157 (*mapcount)--; 1158 } 1159 pte_unmap_unlock(pte - 1, ptl); 1160 if (locked_vma) 1161 up_read(&vma->vm_mm->mmap_sem); 1162 return ret; 1163} 1164 1165static bool is_vma_temporary_stack(struct vm_area_struct *vma) 1166{ 1167 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 1168 1169 if (!maybe_stack) 1170 return false; 1171 1172 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1173 VM_STACK_INCOMPLETE_SETUP) 1174 return true; 1175 1176 return false; 1177} 1178 1179/** 1180 * try_to_unmap_anon - unmap or unlock anonymous page using the object-based 1181 * rmap method 1182 * @page: the page to unmap/unlock 1183 * @flags: action and flags 1184 * 1185 * Find all the mappings of a page using the mapping pointer and the vma chains 1186 * contained in the anon_vma struct it points to. 1187 * 1188 * This function is only called from try_to_unmap/try_to_munlock for 1189 * anonymous pages. 1190 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1191 * where the page was found will be held for write. So, we won't recheck 1192 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1193 * 'LOCKED. 1194 */ 1195static int try_to_unmap_anon(struct page *page, enum ttu_flags flags) 1196{ 1197 struct anon_vma *anon_vma; 1198 struct anon_vma_chain *avc; 1199 int ret = SWAP_AGAIN; 1200 1201 anon_vma = page_lock_anon_vma(page); 1202 if (!anon_vma) 1203 return ret; 1204 1205 list_for_each_entry(avc, &anon_vma->head, same_anon_vma) { 1206 struct vm_area_struct *vma = avc->vma; 1207 unsigned long address; 1208 1209 /* 1210 * During exec, a temporary VMA is setup and later moved. 1211 * The VMA is moved under the anon_vma lock but not the 1212 * page tables leading to a race where migration cannot 1213 * find the migration ptes. Rather than increasing the 1214 * locking requirements of exec(), migration skips 1215 * temporary VMAs until after exec() completes. 1216 */ 1217 if (PAGE_MIGRATION && (flags & TTU_MIGRATION) && 1218 is_vma_temporary_stack(vma)) 1219 continue; 1220 1221 address = vma_address(page, vma); 1222 if (address == -EFAULT) 1223 continue; 1224 ret = try_to_unmap_one(page, vma, address, flags); 1225 if (ret != SWAP_AGAIN || !page_mapped(page)) 1226 break; 1227 } 1228 1229 page_unlock_anon_vma(anon_vma); 1230 return ret; 1231} 1232 1233/** 1234 * try_to_unmap_file - unmap/unlock file page using the object-based rmap method 1235 * @page: the page to unmap/unlock 1236 * @flags: action and flags 1237 * 1238 * Find all the mappings of a page using the mapping pointer and the vma chains 1239 * contained in the address_space struct it points to. 1240 * 1241 * This function is only called from try_to_unmap/try_to_munlock for 1242 * object-based pages. 1243 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1244 * where the page was found will be held for write. So, we won't recheck 1245 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1246 * 'LOCKED. 1247 */ 1248static int try_to_unmap_file(struct page *page, enum ttu_flags flags) 1249{ 1250 struct address_space *mapping = page->mapping; 1251 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 1252 struct vm_area_struct *vma; 1253 struct prio_tree_iter iter; 1254 int ret = SWAP_AGAIN; 1255 unsigned long cursor; 1256 unsigned long max_nl_cursor = 0; 1257 unsigned long max_nl_size = 0; 1258 unsigned int mapcount; 1259 1260 spin_lock(&mapping->i_mmap_lock); 1261 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 1262 unsigned long address = vma_address(page, vma); 1263 if (address == -EFAULT) 1264 continue; 1265 ret = try_to_unmap_one(page, vma, address, flags); 1266 if (ret != SWAP_AGAIN || !page_mapped(page)) 1267 goto out; 1268 } 1269 1270 if (list_empty(&mapping->i_mmap_nonlinear)) 1271 goto out; 1272 1273 /* 1274 * We don't bother to try to find the munlocked page in nonlinears. 1275 * It's costly. Instead, later, page reclaim logic may call 1276 * try_to_unmap(TTU_MUNLOCK) and recover PG_mlocked lazily. 1277 */ 1278 if (TTU_ACTION(flags) == TTU_MUNLOCK) 1279 goto out; 1280 1281 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, 1282 shared.vm_set.list) { 1283 cursor = (unsigned long) vma->vm_private_data; 1284 if (cursor > max_nl_cursor) 1285 max_nl_cursor = cursor; 1286 cursor = vma->vm_end - vma->vm_start; 1287 if (cursor > max_nl_size) 1288 max_nl_size = cursor; 1289 } 1290 1291 if (max_nl_size == 0) { /* all nonlinears locked or reserved ? */ 1292 ret = SWAP_FAIL; 1293 goto out; 1294 } 1295 1296 /* 1297 * We don't try to search for this page in the nonlinear vmas, 1298 * and page_referenced wouldn't have found it anyway. Instead 1299 * just walk the nonlinear vmas trying to age and unmap some. 1300 * The mapcount of the page we came in with is irrelevant, 1301 * but even so use it as a guide to how hard we should try? 1302 */ 1303 mapcount = page_mapcount(page); 1304 if (!mapcount) 1305 goto out; 1306 cond_resched_lock(&mapping->i_mmap_lock); 1307 1308 max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK; 1309 if (max_nl_cursor == 0) 1310 max_nl_cursor = CLUSTER_SIZE; 1311 1312 do { 1313 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, 1314 shared.vm_set.list) { 1315 cursor = (unsigned long) vma->vm_private_data; 1316 while ( cursor < max_nl_cursor && 1317 cursor < vma->vm_end - vma->vm_start) { 1318 if (try_to_unmap_cluster(cursor, &mapcount, 1319 vma, page) == SWAP_MLOCK) 1320 ret = SWAP_MLOCK; 1321 cursor += CLUSTER_SIZE; 1322 vma->vm_private_data = (void *) cursor; 1323 if ((int)mapcount <= 0) 1324 goto out; 1325 } 1326 vma->vm_private_data = (void *) max_nl_cursor; 1327 } 1328 cond_resched_lock(&mapping->i_mmap_lock); 1329 max_nl_cursor += CLUSTER_SIZE; 1330 } while (max_nl_cursor <= max_nl_size); 1331 1332 /* 1333 * Don't loop forever (perhaps all the remaining pages are 1334 * in locked vmas). Reset cursor on all unreserved nonlinear 1335 * vmas, now forgetting on which ones it had fallen behind. 1336 */ 1337 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list) 1338 vma->vm_private_data = NULL; 1339out: 1340 spin_unlock(&mapping->i_mmap_lock); 1341 return ret; 1342} 1343 1344/** 1345 * try_to_unmap - try to remove all page table mappings to a page 1346 * @page: the page to get unmapped 1347 * @flags: action and flags 1348 * 1349 * Tries to remove all the page table entries which are mapping this 1350 * page, used in the pageout path. Caller must hold the page lock. 1351 * Return values are: 1352 * 1353 * SWAP_SUCCESS - we succeeded in removing all mappings 1354 * SWAP_AGAIN - we missed a mapping, try again later 1355 * SWAP_FAIL - the page is unswappable 1356 * SWAP_MLOCK - page is mlocked. 1357 */ 1358int try_to_unmap(struct page *page, enum ttu_flags flags) 1359{ 1360 int ret; 1361 1362 BUG_ON(!PageLocked(page)); 1363 1364 if (unlikely(PageKsm(page))) 1365 ret = try_to_unmap_ksm(page, flags); 1366 else if (PageAnon(page)) 1367 ret = try_to_unmap_anon(page, flags); 1368 else 1369 ret = try_to_unmap_file(page, flags); 1370 if (ret != SWAP_MLOCK && !page_mapped(page)) 1371 ret = SWAP_SUCCESS; 1372 return ret; 1373} 1374 1375/** 1376 * try_to_munlock - try to munlock a page 1377 * @page: the page to be munlocked 1378 * 1379 * Called from munlock code. Checks all of the VMAs mapping the page 1380 * to make sure nobody else has this page mlocked. The page will be 1381 * returned with PG_mlocked cleared if no other vmas have it mlocked. 1382 * 1383 * Return values are: 1384 * 1385 * SWAP_AGAIN - no vma is holding page mlocked, or, 1386 * SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem 1387 * SWAP_FAIL - page cannot be located at present 1388 * SWAP_MLOCK - page is now mlocked. 1389 */ 1390int try_to_munlock(struct page *page) 1391{ 1392 VM_BUG_ON(!PageLocked(page) || PageLRU(page)); 1393 1394 if (unlikely(PageKsm(page))) 1395 return try_to_unmap_ksm(page, TTU_MUNLOCK); 1396 else if (PageAnon(page)) 1397 return try_to_unmap_anon(page, TTU_MUNLOCK); 1398 else 1399 return try_to_unmap_file(page, TTU_MUNLOCK); 1400} 1401 1402#if defined(CONFIG_KSM) || defined(CONFIG_MIGRATION) 1403/* 1404 * Drop an anon_vma refcount, freeing the anon_vma and anon_vma->root 1405 * if necessary. Be careful to do all the tests under the lock. Once 1406 * we know we are the last user, nobody else can get a reference and we 1407 * can do the freeing without the lock. 1408 */ 1409void drop_anon_vma(struct anon_vma *anon_vma) 1410{ 1411 if (atomic_dec_and_lock(&anon_vma->external_refcount, &anon_vma->root->lock)) { 1412 struct anon_vma *root = anon_vma->root; 1413 int empty = list_empty(&anon_vma->head); 1414 int last_root_user = 0; 1415 int root_empty = 0; 1416 1417 /* 1418 * The refcount on a non-root anon_vma got dropped. Drop 1419 * the refcount on the root and check if we need to free it. 1420 */ 1421 if (empty && anon_vma != root) { 1422 last_root_user = atomic_dec_and_test(&root->external_refcount); 1423 root_empty = list_empty(&root->head); 1424 } 1425 anon_vma_unlock(anon_vma); 1426 1427 if (empty) { 1428 anon_vma_free(anon_vma); 1429 if (root_empty && last_root_user) 1430 anon_vma_free(root); 1431 } 1432 } 1433} 1434#endif 1435 1436#ifdef CONFIG_MIGRATION 1437/* 1438 * rmap_walk() and its helpers rmap_walk_anon() and rmap_walk_file(): 1439 * Called by migrate.c to remove migration ptes, but might be used more later. 1440 */ 1441static int rmap_walk_anon(struct page *page, int (*rmap_one)(struct page *, 1442 struct vm_area_struct *, unsigned long, void *), void *arg) 1443{ 1444 struct anon_vma *anon_vma; 1445 struct anon_vma_chain *avc; 1446 int ret = SWAP_AGAIN; 1447 1448 /* 1449 * Note: remove_migration_ptes() cannot use page_lock_anon_vma() 1450 * because that depends on page_mapped(); but not all its usages 1451 * are holding mmap_sem. Users without mmap_sem are required to 1452 * take a reference count to prevent the anon_vma disappearing 1453 */ 1454 anon_vma = page_anon_vma(page); 1455 if (!anon_vma) 1456 return ret; 1457 anon_vma_lock(anon_vma); 1458 list_for_each_entry(avc, &anon_vma->head, same_anon_vma) { 1459 struct vm_area_struct *vma = avc->vma; 1460 unsigned long address = vma_address(page, vma); 1461 if (address == -EFAULT) 1462 continue; 1463 ret = rmap_one(page, vma, address, arg); 1464 if (ret != SWAP_AGAIN) 1465 break; 1466 } 1467 anon_vma_unlock(anon_vma); 1468 return ret; 1469} 1470 1471static int rmap_walk_file(struct page *page, int (*rmap_one)(struct page *, 1472 struct vm_area_struct *, unsigned long, void *), void *arg) 1473{ 1474 struct address_space *mapping = page->mapping; 1475 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 1476 struct vm_area_struct *vma; 1477 struct prio_tree_iter iter; 1478 int ret = SWAP_AGAIN; 1479 1480 if (!mapping) 1481 return ret; 1482 spin_lock(&mapping->i_mmap_lock); 1483 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 1484 unsigned long address = vma_address(page, vma); 1485 if (address == -EFAULT) 1486 continue; 1487 ret = rmap_one(page, vma, address, arg); 1488 if (ret != SWAP_AGAIN) 1489 break; 1490 } 1491 /* 1492 * No nonlinear handling: being always shared, nonlinear vmas 1493 * never contain migration ptes. Decide what to do about this 1494 * limitation to linear when we need rmap_walk() on nonlinear. 1495 */ 1496 spin_unlock(&mapping->i_mmap_lock); 1497 return ret; 1498} 1499 1500int rmap_walk(struct page *page, int (*rmap_one)(struct page *, 1501 struct vm_area_struct *, unsigned long, void *), void *arg) 1502{ 1503 VM_BUG_ON(!PageLocked(page)); 1504 1505 if (unlikely(PageKsm(page))) 1506 return rmap_walk_ksm(page, rmap_one, arg); 1507 else if (PageAnon(page)) 1508 return rmap_walk_anon(page, rmap_one, arg); 1509 else 1510 return rmap_walk_file(page, rmap_one, arg); 1511} 1512#endif /* CONFIG_MIGRATION */ 1513