1/* 2 * mm/rmap.c - physical to virtual reverse mappings 3 * 4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 5 * Released under the General Public License (GPL). 6 * 7 * Simple, low overhead reverse mapping scheme. 8 * Please try to keep this thing as modular as possible. 9 * 10 * Provides methods for unmapping each kind of mapped page: 11 * the anon methods track anonymous pages, and 12 * the file methods track pages belonging to an inode. 13 * 14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001 15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 17 * Contributions by Hugh Dickins 2003, 2004 18 */ 19 20/* 21 * Lock ordering in mm: 22 * 23 * inode->i_mutex (while writing or truncating, not reading or faulting) 24 * mm->mmap_sem 25 * page->flags PG_locked (lock_page) 26 * mapping->i_mmap_mutex 27 * anon_vma->rwsem 28 * mm->page_table_lock or pte_lock 29 * zone->lru_lock (in mark_page_accessed, isolate_lru_page) 30 * swap_lock (in swap_duplicate, swap_info_get) 31 * mmlist_lock (in mmput, drain_mmlist and others) 32 * mapping->private_lock (in __set_page_dirty_buffers) 33 * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 34 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 35 * sb_lock (within inode_lock in fs/fs-writeback.c) 36 * mapping->tree_lock (widely used, in set_page_dirty, 37 * in arch-dependent flush_dcache_mmap_lock, 38 * within bdi.wb->list_lock in __sync_single_inode) 39 * 40 * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon) 41 * ->tasklist_lock 42 * pte map lock 43 */ 44 45#include <linux/mm.h> 46#include <linux/pagemap.h> 47#include <linux/swap.h> 48#include <linux/swapops.h> 49#include <linux/slab.h> 50#include <linux/init.h> 51#include <linux/ksm.h> 52#include <linux/rmap.h> 53#include <linux/rcupdate.h> 54#include <linux/export.h> 55#include <linux/memcontrol.h> 56#include <linux/mmu_notifier.h> 57#include <linux/migrate.h> 58#include <linux/hugetlb.h> 59#include <linux/backing-dev.h> 60 61#include <asm/tlbflush.h> 62 63#include "internal.h" 64 65static struct kmem_cache *anon_vma_cachep; 66static struct kmem_cache *anon_vma_chain_cachep; 67 68static inline struct anon_vma *anon_vma_alloc(void) 69{ 70 struct anon_vma *anon_vma; 71 72 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 73 if (anon_vma) { 74 atomic_set(&anon_vma->refcount, 1); 75 /* 76 * Initialise the anon_vma root to point to itself. If called 77 * from fork, the root will be reset to the parents anon_vma. 78 */ 79 anon_vma->root = anon_vma; 80 } 81 82 return anon_vma; 83} 84 85static inline void anon_vma_free(struct anon_vma *anon_vma) 86{ 87 VM_BUG_ON(atomic_read(&anon_vma->refcount)); 88 89 /* 90 * Synchronize against page_lock_anon_vma_read() such that 91 * we can safely hold the lock without the anon_vma getting 92 * freed. 93 * 94 * Relies on the full mb implied by the atomic_dec_and_test() from 95 * put_anon_vma() against the acquire barrier implied by 96 * down_read_trylock() from page_lock_anon_vma_read(). This orders: 97 * 98 * page_lock_anon_vma_read() VS put_anon_vma() 99 * down_read_trylock() atomic_dec_and_test() 100 * LOCK MB 101 * atomic_read() rwsem_is_locked() 102 * 103 * LOCK should suffice since the actual taking of the lock must 104 * happen _before_ what follows. 105 */ 106 might_sleep(); 107 if (rwsem_is_locked(&anon_vma->root->rwsem)) { 108 anon_vma_lock_write(anon_vma); 109 anon_vma_unlock_write(anon_vma); 110 } 111 112 kmem_cache_free(anon_vma_cachep, anon_vma); 113} 114 115static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) 116{ 117 return kmem_cache_alloc(anon_vma_chain_cachep, gfp); 118} 119 120static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 121{ 122 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 123} 124 125static void anon_vma_chain_link(struct vm_area_struct *vma, 126 struct anon_vma_chain *avc, 127 struct anon_vma *anon_vma) 128{ 129 avc->vma = vma; 130 avc->anon_vma = anon_vma; 131 list_add(&avc->same_vma, &vma->anon_vma_chain); 132 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); 133} 134 135/** 136 * anon_vma_prepare - attach an anon_vma to a memory region 137 * @vma: the memory region in question 138 * 139 * This makes sure the memory mapping described by 'vma' has 140 * an 'anon_vma' attached to it, so that we can associate the 141 * anonymous pages mapped into it with that anon_vma. 142 * 143 * The common case will be that we already have one, but if 144 * not we either need to find an adjacent mapping that we 145 * can re-use the anon_vma from (very common when the only 146 * reason for splitting a vma has been mprotect()), or we 147 * allocate a new one. 148 * 149 * Anon-vma allocations are very subtle, because we may have 150 * optimistically looked up an anon_vma in page_lock_anon_vma_read() 151 * and that may actually touch the spinlock even in the newly 152 * allocated vma (it depends on RCU to make sure that the 153 * anon_vma isn't actually destroyed). 154 * 155 * As a result, we need to do proper anon_vma locking even 156 * for the new allocation. At the same time, we do not want 157 * to do any locking for the common case of already having 158 * an anon_vma. 159 * 160 * This must be called with the mmap_sem held for reading. 161 */ 162int anon_vma_prepare(struct vm_area_struct *vma) 163{ 164 struct anon_vma *anon_vma = vma->anon_vma; 165 struct anon_vma_chain *avc; 166 167 might_sleep(); 168 if (unlikely(!anon_vma)) { 169 struct mm_struct *mm = vma->vm_mm; 170 struct anon_vma *allocated; 171 172 avc = anon_vma_chain_alloc(GFP_KERNEL); 173 if (!avc) 174 goto out_enomem; 175 176 anon_vma = find_mergeable_anon_vma(vma); 177 allocated = NULL; 178 if (!anon_vma) { 179 anon_vma = anon_vma_alloc(); 180 if (unlikely(!anon_vma)) 181 goto out_enomem_free_avc; 182 allocated = anon_vma; 183 } 184 185 anon_vma_lock_write(anon_vma); 186 /* page_table_lock to protect against threads */ 187 spin_lock(&mm->page_table_lock); 188 if (likely(!vma->anon_vma)) { 189 vma->anon_vma = anon_vma; 190 anon_vma_chain_link(vma, avc, anon_vma); 191 allocated = NULL; 192 avc = NULL; 193 } 194 spin_unlock(&mm->page_table_lock); 195 anon_vma_unlock_write(anon_vma); 196 197 if (unlikely(allocated)) 198 put_anon_vma(allocated); 199 if (unlikely(avc)) 200 anon_vma_chain_free(avc); 201 } 202 return 0; 203 204 out_enomem_free_avc: 205 anon_vma_chain_free(avc); 206 out_enomem: 207 return -ENOMEM; 208} 209 210/* 211 * This is a useful helper function for locking the anon_vma root as 212 * we traverse the vma->anon_vma_chain, looping over anon_vma's that 213 * have the same vma. 214 * 215 * Such anon_vma's should have the same root, so you'd expect to see 216 * just a single mutex_lock for the whole traversal. 217 */ 218static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) 219{ 220 struct anon_vma *new_root = anon_vma->root; 221 if (new_root != root) { 222 if (WARN_ON_ONCE(root)) 223 up_write(&root->rwsem); 224 root = new_root; 225 down_write(&root->rwsem); 226 } 227 return root; 228} 229 230static inline void unlock_anon_vma_root(struct anon_vma *root) 231{ 232 if (root) 233 up_write(&root->rwsem); 234} 235 236/* 237 * Attach the anon_vmas from src to dst. 238 * Returns 0 on success, -ENOMEM on failure. 239 */ 240int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 241{ 242 struct anon_vma_chain *avc, *pavc; 243 struct anon_vma *root = NULL; 244 245 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 246 struct anon_vma *anon_vma; 247 248 avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN); 249 if (unlikely(!avc)) { 250 unlock_anon_vma_root(root); 251 root = NULL; 252 avc = anon_vma_chain_alloc(GFP_KERNEL); 253 if (!avc) 254 goto enomem_failure; 255 } 256 anon_vma = pavc->anon_vma; 257 root = lock_anon_vma_root(root, anon_vma); 258 anon_vma_chain_link(dst, avc, anon_vma); 259 } 260 unlock_anon_vma_root(root); 261 return 0; 262 263 enomem_failure: 264 unlink_anon_vmas(dst); 265 return -ENOMEM; 266} 267 268/* 269 * Attach vma to its own anon_vma, as well as to the anon_vmas that 270 * the corresponding VMA in the parent process is attached to. 271 * Returns 0 on success, non-zero on failure. 272 */ 273int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 274{ 275 struct anon_vma_chain *avc; 276 struct anon_vma *anon_vma; 277 int error; 278 279 /* Don't bother if the parent process has no anon_vma here. */ 280 if (!pvma->anon_vma) 281 return 0; 282 283 /* 284 * First, attach the new VMA to the parent VMA's anon_vmas, 285 * so rmap can find non-COWed pages in child processes. 286 */ 287 error = anon_vma_clone(vma, pvma); 288 if (error) 289 return error; 290 291 /* Then add our own anon_vma. */ 292 anon_vma = anon_vma_alloc(); 293 if (!anon_vma) 294 goto out_error; 295 avc = anon_vma_chain_alloc(GFP_KERNEL); 296 if (!avc) 297 goto out_error_free_anon_vma; 298 299 /* 300 * The root anon_vma's spinlock is the lock actually used when we 301 * lock any of the anon_vmas in this anon_vma tree. 302 */ 303 anon_vma->root = pvma->anon_vma->root; 304 /* 305 * With refcounts, an anon_vma can stay around longer than the 306 * process it belongs to. The root anon_vma needs to be pinned until 307 * this anon_vma is freed, because the lock lives in the root. 308 */ 309 get_anon_vma(anon_vma->root); 310 /* Mark this anon_vma as the one where our new (COWed) pages go. */ 311 vma->anon_vma = anon_vma; 312 anon_vma_lock_write(anon_vma); 313 anon_vma_chain_link(vma, avc, anon_vma); 314 anon_vma_unlock_write(anon_vma); 315 316 return 0; 317 318 out_error_free_anon_vma: 319 put_anon_vma(anon_vma); 320 out_error: 321 unlink_anon_vmas(vma); 322 return -ENOMEM; 323} 324 325void unlink_anon_vmas(struct vm_area_struct *vma) 326{ 327 struct anon_vma_chain *avc, *next; 328 struct anon_vma *root = NULL; 329 330 /* 331 * Unlink each anon_vma chained to the VMA. This list is ordered 332 * from newest to oldest, ensuring the root anon_vma gets freed last. 333 */ 334 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 335 struct anon_vma *anon_vma = avc->anon_vma; 336 337 root = lock_anon_vma_root(root, anon_vma); 338 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); 339 340 /* 341 * Leave empty anon_vmas on the list - we'll need 342 * to free them outside the lock. 343 */ 344 if (RB_EMPTY_ROOT(&anon_vma->rb_root)) 345 continue; 346 347 list_del(&avc->same_vma); 348 anon_vma_chain_free(avc); 349 } 350 unlock_anon_vma_root(root); 351 352 /* 353 * Iterate the list once more, it now only contains empty and unlinked 354 * anon_vmas, destroy them. Could not do before due to __put_anon_vma() 355 * needing to write-acquire the anon_vma->root->rwsem. 356 */ 357 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 358 struct anon_vma *anon_vma = avc->anon_vma; 359 360 put_anon_vma(anon_vma); 361 362 list_del(&avc->same_vma); 363 anon_vma_chain_free(avc); 364 } 365} 366 367static void anon_vma_ctor(void *data) 368{ 369 struct anon_vma *anon_vma = data; 370 371 init_rwsem(&anon_vma->rwsem); 372 atomic_set(&anon_vma->refcount, 0); 373 anon_vma->rb_root = RB_ROOT; 374} 375 376void __init anon_vma_init(void) 377{ 378 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 379 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor); 380 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC); 381} 382 383/* 384 * Getting a lock on a stable anon_vma from a page off the LRU is tricky! 385 * 386 * Since there is no serialization what so ever against page_remove_rmap() 387 * the best this function can do is return a locked anon_vma that might 388 * have been relevant to this page. 389 * 390 * The page might have been remapped to a different anon_vma or the anon_vma 391 * returned may already be freed (and even reused). 392 * 393 * In case it was remapped to a different anon_vma, the new anon_vma will be a 394 * child of the old anon_vma, and the anon_vma lifetime rules will therefore 395 * ensure that any anon_vma obtained from the page will still be valid for as 396 * long as we observe page_mapped() [ hence all those page_mapped() tests ]. 397 * 398 * All users of this function must be very careful when walking the anon_vma 399 * chain and verify that the page in question is indeed mapped in it 400 * [ something equivalent to page_mapped_in_vma() ]. 401 * 402 * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap() 403 * that the anon_vma pointer from page->mapping is valid if there is a 404 * mapcount, we can dereference the anon_vma after observing those. 405 */ 406struct anon_vma *page_get_anon_vma(struct page *page) 407{ 408 struct anon_vma *anon_vma = NULL; 409 unsigned long anon_mapping; 410 411 rcu_read_lock(); 412 anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping); 413 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 414 goto out; 415 if (!page_mapped(page)) 416 goto out; 417 418 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 419 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 420 anon_vma = NULL; 421 goto out; 422 } 423 424 /* 425 * If this page is still mapped, then its anon_vma cannot have been 426 * freed. But if it has been unmapped, we have no security against the 427 * anon_vma structure being freed and reused (for another anon_vma: 428 * SLAB_DESTROY_BY_RCU guarantees that - so the atomic_inc_not_zero() 429 * above cannot corrupt). 430 */ 431 if (!page_mapped(page)) { 432 rcu_read_unlock(); 433 put_anon_vma(anon_vma); 434 return NULL; 435 } 436out: 437 rcu_read_unlock(); 438 439 return anon_vma; 440} 441 442/* 443 * Similar to page_get_anon_vma() except it locks the anon_vma. 444 * 445 * Its a little more complex as it tries to keep the fast path to a single 446 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a 447 * reference like with page_get_anon_vma() and then block on the mutex. 448 */ 449struct anon_vma *page_lock_anon_vma_read(struct page *page) 450{ 451 struct anon_vma *anon_vma = NULL; 452 struct anon_vma *root_anon_vma; 453 unsigned long anon_mapping; 454 455 rcu_read_lock(); 456 anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping); 457 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 458 goto out; 459 if (!page_mapped(page)) 460 goto out; 461 462 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 463 root_anon_vma = ACCESS_ONCE(anon_vma->root); 464 if (down_read_trylock(&root_anon_vma->rwsem)) { 465 /* 466 * If the page is still mapped, then this anon_vma is still 467 * its anon_vma, and holding the mutex ensures that it will 468 * not go away, see anon_vma_free(). 469 */ 470 if (!page_mapped(page)) { 471 up_read(&root_anon_vma->rwsem); 472 anon_vma = NULL; 473 } 474 goto out; 475 } 476 477 /* trylock failed, we got to sleep */ 478 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 479 anon_vma = NULL; 480 goto out; 481 } 482 483 if (!page_mapped(page)) { 484 rcu_read_unlock(); 485 put_anon_vma(anon_vma); 486 return NULL; 487 } 488 489 /* we pinned the anon_vma, its safe to sleep */ 490 rcu_read_unlock(); 491 anon_vma_lock_read(anon_vma); 492 493 if (atomic_dec_and_test(&anon_vma->refcount)) { 494 /* 495 * Oops, we held the last refcount, release the lock 496 * and bail -- can't simply use put_anon_vma() because 497 * we'll deadlock on the anon_vma_lock_write() recursion. 498 */ 499 anon_vma_unlock_read(anon_vma); 500 __put_anon_vma(anon_vma); 501 anon_vma = NULL; 502 } 503 504 return anon_vma; 505 506out: 507 rcu_read_unlock(); 508 return anon_vma; 509} 510 511void page_unlock_anon_vma_read(struct anon_vma *anon_vma) 512{ 513 anon_vma_unlock_read(anon_vma); 514} 515 516/* 517 * At what user virtual address is page expected in @vma? 518 */ 519static inline unsigned long 520__vma_address(struct page *page, struct vm_area_struct *vma) 521{ 522 pgoff_t pgoff = page_to_pgoff(page); 523 return vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 524} 525 526inline unsigned long 527vma_address(struct page *page, struct vm_area_struct *vma) 528{ 529 unsigned long address = __vma_address(page, vma); 530 531 /* page should be within @vma mapping range */ 532 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); 533 534 return address; 535} 536 537/* 538 * At what user virtual address is page expected in vma? 539 * Caller should check the page is actually part of the vma. 540 */ 541unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 542{ 543 unsigned long address; 544 if (PageAnon(page)) { 545 struct anon_vma *page__anon_vma = page_anon_vma(page); 546 /* 547 * Note: swapoff's unuse_vma() is more efficient with this 548 * check, and needs it to match anon_vma when KSM is active. 549 */ 550 if (!vma->anon_vma || !page__anon_vma || 551 vma->anon_vma->root != page__anon_vma->root) 552 return -EFAULT; 553 } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) { 554 if (!vma->vm_file || 555 vma->vm_file->f_mapping != page->mapping) 556 return -EFAULT; 557 } else 558 return -EFAULT; 559 address = __vma_address(page, vma); 560 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) 561 return -EFAULT; 562 return address; 563} 564 565pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) 566{ 567 pgd_t *pgd; 568 pud_t *pud; 569 pmd_t *pmd = NULL; 570 pmd_t pmde; 571 572 pgd = pgd_offset(mm, address); 573 if (!pgd_present(*pgd)) 574 goto out; 575 576 pud = pud_offset(pgd, address); 577 if (!pud_present(*pud)) 578 goto out; 579 580 pmd = pmd_offset(pud, address); 581 /* 582 * Some THP functions use the sequence pmdp_clear_flush(), set_pmd_at() 583 * without holding anon_vma lock for write. So when looking for a 584 * genuine pmde (in which to find pte), test present and !THP together. 585 */ 586 pmde = ACCESS_ONCE(*pmd); 587 if (!pmd_present(pmde) || pmd_trans_huge(pmde)) 588 pmd = NULL; 589out: 590 return pmd; 591} 592 593/* 594 * Check that @page is mapped at @address into @mm. 595 * 596 * If @sync is false, page_check_address may perform a racy check to avoid 597 * the page table lock when the pte is not present (helpful when reclaiming 598 * highly shared pages). 599 * 600 * On success returns with pte mapped and locked. 601 */ 602pte_t *__page_check_address(struct page *page, struct mm_struct *mm, 603 unsigned long address, spinlock_t **ptlp, int sync) 604{ 605 pmd_t *pmd; 606 pte_t *pte; 607 spinlock_t *ptl; 608 609 if (unlikely(PageHuge(page))) { 610 /* when pud is not present, pte will be NULL */ 611 pte = huge_pte_offset(mm, address); 612 if (!pte) 613 return NULL; 614 615 ptl = huge_pte_lockptr(page_hstate(page), mm, pte); 616 goto check; 617 } 618 619 pmd = mm_find_pmd(mm, address); 620 if (!pmd) 621 return NULL; 622 623 pte = pte_offset_map(pmd, address); 624 /* Make a quick check before getting the lock */ 625 if (!sync && !pte_present(*pte)) { 626 pte_unmap(pte); 627 return NULL; 628 } 629 630 ptl = pte_lockptr(mm, pmd); 631check: 632 spin_lock(ptl); 633 if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) { 634 *ptlp = ptl; 635 return pte; 636 } 637 pte_unmap_unlock(pte, ptl); 638 return NULL; 639} 640 641/** 642 * page_mapped_in_vma - check whether a page is really mapped in a VMA 643 * @page: the page to test 644 * @vma: the VMA to test 645 * 646 * Returns 1 if the page is mapped into the page tables of the VMA, 0 647 * if the page is not mapped into the page tables of this VMA. Only 648 * valid for normal file or anonymous VMAs. 649 */ 650int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) 651{ 652 unsigned long address; 653 pte_t *pte; 654 spinlock_t *ptl; 655 656 address = __vma_address(page, vma); 657 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) 658 return 0; 659 pte = page_check_address(page, vma->vm_mm, address, &ptl, 1); 660 if (!pte) /* the page is not in this mm */ 661 return 0; 662 pte_unmap_unlock(pte, ptl); 663 664 return 1; 665} 666 667struct page_referenced_arg { 668 int mapcount; 669 int referenced; 670 unsigned long vm_flags; 671 struct mem_cgroup *memcg; 672}; 673/* 674 * arg: page_referenced_arg will be passed 675 */ 676static int page_referenced_one(struct page *page, struct vm_area_struct *vma, 677 unsigned long address, void *arg) 678{ 679 struct mm_struct *mm = vma->vm_mm; 680 spinlock_t *ptl; 681 int referenced = 0; 682 struct page_referenced_arg *pra = arg; 683 684 if (unlikely(PageTransHuge(page))) { 685 pmd_t *pmd; 686 687 /* 688 * rmap might return false positives; we must filter 689 * these out using page_check_address_pmd(). 690 */ 691 pmd = page_check_address_pmd(page, mm, address, 692 PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl); 693 if (!pmd) 694 return SWAP_AGAIN; 695 696 if (vma->vm_flags & VM_LOCKED) { 697 spin_unlock(ptl); 698 pra->vm_flags |= VM_LOCKED; 699 return SWAP_FAIL; /* To break the loop */ 700 } 701 702 /* go ahead even if the pmd is pmd_trans_splitting() */ 703 if (pmdp_clear_flush_young_notify(vma, address, pmd)) 704 referenced++; 705 spin_unlock(ptl); 706 } else { 707 pte_t *pte; 708 709 /* 710 * rmap might return false positives; we must filter 711 * these out using page_check_address(). 712 */ 713 pte = page_check_address(page, mm, address, &ptl, 0); 714 if (!pte) 715 return SWAP_AGAIN; 716 717 if (vma->vm_flags & VM_LOCKED) { 718 pte_unmap_unlock(pte, ptl); 719 pra->vm_flags |= VM_LOCKED; 720 return SWAP_FAIL; /* To break the loop */ 721 } 722 723 if (ptep_clear_flush_young_notify(vma, address, pte)) { 724 /* 725 * Don't treat a reference through a sequentially read 726 * mapping as such. If the page has been used in 727 * another mapping, we will catch it; if this other 728 * mapping is already gone, the unmap path will have 729 * set PG_referenced or activated the page. 730 */ 731 if (likely(!(vma->vm_flags & VM_SEQ_READ))) 732 referenced++; 733 } 734 pte_unmap_unlock(pte, ptl); 735 } 736 737 if (referenced) { 738 pra->referenced++; 739 pra->vm_flags |= vma->vm_flags; 740 } 741 742 pra->mapcount--; 743 if (!pra->mapcount) 744 return SWAP_SUCCESS; /* To break the loop */ 745 746 return SWAP_AGAIN; 747} 748 749static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg) 750{ 751 struct page_referenced_arg *pra = arg; 752 struct mem_cgroup *memcg = pra->memcg; 753 754 if (!mm_match_cgroup(vma->vm_mm, memcg)) 755 return true; 756 757 return false; 758} 759 760/** 761 * page_referenced - test if the page was referenced 762 * @page: the page to test 763 * @is_locked: caller holds lock on the page 764 * @memcg: target memory cgroup 765 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 766 * 767 * Quick test_and_clear_referenced for all mappings to a page, 768 * returns the number of ptes which referenced the page. 769 */ 770int page_referenced(struct page *page, 771 int is_locked, 772 struct mem_cgroup *memcg, 773 unsigned long *vm_flags) 774{ 775 int ret; 776 int we_locked = 0; 777 struct page_referenced_arg pra = { 778 .mapcount = page_mapcount(page), 779 .memcg = memcg, 780 }; 781 struct rmap_walk_control rwc = { 782 .rmap_one = page_referenced_one, 783 .arg = (void *)&pra, 784 .anon_lock = page_lock_anon_vma_read, 785 }; 786 787 *vm_flags = 0; 788 if (!page_mapped(page)) 789 return 0; 790 791 if (!page_rmapping(page)) 792 return 0; 793 794 if (!is_locked && (!PageAnon(page) || PageKsm(page))) { 795 we_locked = trylock_page(page); 796 if (!we_locked) 797 return 1; 798 } 799 800 /* 801 * If we are reclaiming on behalf of a cgroup, skip 802 * counting on behalf of references from different 803 * cgroups 804 */ 805 if (memcg) { 806 rwc.invalid_vma = invalid_page_referenced_vma; 807 } 808 809 ret = rmap_walk(page, &rwc); 810 *vm_flags = pra.vm_flags; 811 812 if (we_locked) 813 unlock_page(page); 814 815 return pra.referenced; 816} 817 818static int page_mkclean_one(struct page *page, struct vm_area_struct *vma, 819 unsigned long address, void *arg) 820{ 821 struct mm_struct *mm = vma->vm_mm; 822 pte_t *pte; 823 spinlock_t *ptl; 824 int ret = 0; 825 int *cleaned = arg; 826 827 pte = page_check_address(page, mm, address, &ptl, 1); 828 if (!pte) 829 goto out; 830 831 if (pte_dirty(*pte) || pte_write(*pte)) { 832 pte_t entry; 833 834 flush_cache_page(vma, address, pte_pfn(*pte)); 835 entry = ptep_clear_flush(vma, address, pte); 836 entry = pte_wrprotect(entry); 837 entry = pte_mkclean(entry); 838 set_pte_at(mm, address, pte, entry); 839 ret = 1; 840 } 841 842 pte_unmap_unlock(pte, ptl); 843 844 if (ret) { 845 mmu_notifier_invalidate_page(mm, address); 846 (*cleaned)++; 847 } 848out: 849 return SWAP_AGAIN; 850} 851 852static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) 853{ 854 if (vma->vm_flags & VM_SHARED) 855 return false; 856 857 return true; 858} 859 860int page_mkclean(struct page *page) 861{ 862 int cleaned = 0; 863 struct address_space *mapping; 864 struct rmap_walk_control rwc = { 865 .arg = (void *)&cleaned, 866 .rmap_one = page_mkclean_one, 867 .invalid_vma = invalid_mkclean_vma, 868 }; 869 870 BUG_ON(!PageLocked(page)); 871 872 if (!page_mapped(page)) 873 return 0; 874 875 mapping = page_mapping(page); 876 if (!mapping) 877 return 0; 878 879 rmap_walk(page, &rwc); 880 881 return cleaned; 882} 883EXPORT_SYMBOL_GPL(page_mkclean); 884 885/** 886 * page_move_anon_rmap - move a page to our anon_vma 887 * @page: the page to move to our anon_vma 888 * @vma: the vma the page belongs to 889 * @address: the user virtual address mapped 890 * 891 * When a page belongs exclusively to one process after a COW event, 892 * that page can be moved into the anon_vma that belongs to just that 893 * process, so the rmap code will not search the parent or sibling 894 * processes. 895 */ 896void page_move_anon_rmap(struct page *page, 897 struct vm_area_struct *vma, unsigned long address) 898{ 899 struct anon_vma *anon_vma = vma->anon_vma; 900 901 VM_BUG_ON_PAGE(!PageLocked(page), page); 902 VM_BUG_ON_VMA(!anon_vma, vma); 903 VM_BUG_ON_PAGE(page->index != linear_page_index(vma, address), page); 904 905 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 906 page->mapping = (struct address_space *) anon_vma; 907} 908 909/** 910 * __page_set_anon_rmap - set up new anonymous rmap 911 * @page: Page to add to rmap 912 * @vma: VM area to add page to. 913 * @address: User virtual address of the mapping 914 * @exclusive: the page is exclusively owned by the current process 915 */ 916static void __page_set_anon_rmap(struct page *page, 917 struct vm_area_struct *vma, unsigned long address, int exclusive) 918{ 919 struct anon_vma *anon_vma = vma->anon_vma; 920 921 BUG_ON(!anon_vma); 922 923 if (PageAnon(page)) 924 return; 925 926 /* 927 * If the page isn't exclusively mapped into this vma, 928 * we must use the _oldest_ possible anon_vma for the 929 * page mapping! 930 */ 931 if (!exclusive) 932 anon_vma = anon_vma->root; 933 934 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 935 page->mapping = (struct address_space *) anon_vma; 936 page->index = linear_page_index(vma, address); 937} 938 939/** 940 * __page_check_anon_rmap - sanity check anonymous rmap addition 941 * @page: the page to add the mapping to 942 * @vma: the vm area in which the mapping is added 943 * @address: the user virtual address mapped 944 */ 945static void __page_check_anon_rmap(struct page *page, 946 struct vm_area_struct *vma, unsigned long address) 947{ 948#ifdef CONFIG_DEBUG_VM 949 /* 950 * The page's anon-rmap details (mapping and index) are guaranteed to 951 * be set up correctly at this point. 952 * 953 * We have exclusion against page_add_anon_rmap because the caller 954 * always holds the page locked, except if called from page_dup_rmap, 955 * in which case the page is already known to be setup. 956 * 957 * We have exclusion against page_add_new_anon_rmap because those pages 958 * are initially only visible via the pagetables, and the pte is locked 959 * over the call to page_add_new_anon_rmap. 960 */ 961 BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root); 962 BUG_ON(page->index != linear_page_index(vma, address)); 963#endif 964} 965 966/** 967 * page_add_anon_rmap - add pte mapping to an anonymous page 968 * @page: the page to add the mapping to 969 * @vma: the vm area in which the mapping is added 970 * @address: the user virtual address mapped 971 * 972 * The caller needs to hold the pte lock, and the page must be locked in 973 * the anon_vma case: to serialize mapping,index checking after setting, 974 * and to ensure that PageAnon is not being upgraded racily to PageKsm 975 * (but PageKsm is never downgraded to PageAnon). 976 */ 977void page_add_anon_rmap(struct page *page, 978 struct vm_area_struct *vma, unsigned long address) 979{ 980 do_page_add_anon_rmap(page, vma, address, 0); 981} 982 983/* 984 * Special version of the above for do_swap_page, which often runs 985 * into pages that are exclusively owned by the current process. 986 * Everybody else should continue to use page_add_anon_rmap above. 987 */ 988void do_page_add_anon_rmap(struct page *page, 989 struct vm_area_struct *vma, unsigned long address, int exclusive) 990{ 991 int first = atomic_inc_and_test(&page->_mapcount); 992 if (first) { 993 /* 994 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 995 * these counters are not modified in interrupt context, and 996 * pte lock(a spinlock) is held, which implies preemption 997 * disabled. 998 */ 999 if (PageTransHuge(page)) 1000 __inc_zone_page_state(page, 1001 NR_ANON_TRANSPARENT_HUGEPAGES); 1002 __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, 1003 hpage_nr_pages(page)); 1004 } 1005 if (unlikely(PageKsm(page))) 1006 return; 1007 1008 VM_BUG_ON_PAGE(!PageLocked(page), page); 1009 /* address might be in next vma when migration races vma_adjust */ 1010 if (first) 1011 __page_set_anon_rmap(page, vma, address, exclusive); 1012 else 1013 __page_check_anon_rmap(page, vma, address); 1014} 1015 1016/** 1017 * page_add_new_anon_rmap - add pte mapping to a new anonymous page 1018 * @page: the page to add the mapping to 1019 * @vma: the vm area in which the mapping is added 1020 * @address: the user virtual address mapped 1021 * 1022 * Same as page_add_anon_rmap but must only be called on *new* pages. 1023 * This means the inc-and-test can be bypassed. 1024 * Page does not have to be locked. 1025 */ 1026void page_add_new_anon_rmap(struct page *page, 1027 struct vm_area_struct *vma, unsigned long address) 1028{ 1029 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); 1030 SetPageSwapBacked(page); 1031 atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */ 1032 if (PageTransHuge(page)) 1033 __inc_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 1034 __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, 1035 hpage_nr_pages(page)); 1036 __page_set_anon_rmap(page, vma, address, 1); 1037} 1038 1039/** 1040 * page_add_file_rmap - add pte mapping to a file page 1041 * @page: the page to add the mapping to 1042 * 1043 * The caller needs to hold the pte lock. 1044 */ 1045void page_add_file_rmap(struct page *page) 1046{ 1047 struct mem_cgroup *memcg; 1048 unsigned long flags; 1049 bool locked; 1050 1051 memcg = mem_cgroup_begin_page_stat(page, &locked, &flags); 1052 if (atomic_inc_and_test(&page->_mapcount)) { 1053 __inc_zone_page_state(page, NR_FILE_MAPPED); 1054 mem_cgroup_inc_page_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED); 1055 } 1056 mem_cgroup_end_page_stat(memcg, locked, flags); 1057} 1058 1059static void page_remove_file_rmap(struct page *page) 1060{ 1061 struct mem_cgroup *memcg; 1062 unsigned long flags; 1063 bool locked; 1064 1065 memcg = mem_cgroup_begin_page_stat(page, &locked, &flags); 1066 1067 /* page still mapped by someone else? */ 1068 if (!atomic_add_negative(-1, &page->_mapcount)) 1069 goto out; 1070 1071 /* Hugepages are not counted in NR_FILE_MAPPED for now. */ 1072 if (unlikely(PageHuge(page))) 1073 goto out; 1074 1075 /* 1076 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1077 * these counters are not modified in interrupt context, and 1078 * pte lock(a spinlock) is held, which implies preemption disabled. 1079 */ 1080 __dec_zone_page_state(page, NR_FILE_MAPPED); 1081 mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED); 1082 1083 if (unlikely(PageMlocked(page))) 1084 clear_page_mlock(page); 1085out: 1086 mem_cgroup_end_page_stat(memcg, locked, flags); 1087} 1088 1089/** 1090 * page_remove_rmap - take down pte mapping from a page 1091 * @page: page to remove mapping from 1092 * 1093 * The caller needs to hold the pte lock. 1094 */ 1095void page_remove_rmap(struct page *page) 1096{ 1097 if (!PageAnon(page)) { 1098 page_remove_file_rmap(page); 1099 return; 1100 } 1101 1102 /* page still mapped by someone else? */ 1103 if (!atomic_add_negative(-1, &page->_mapcount)) 1104 return; 1105 1106 /* Hugepages are not counted in NR_ANON_PAGES for now. */ 1107 if (unlikely(PageHuge(page))) 1108 return; 1109 1110 /* 1111 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1112 * these counters are not modified in interrupt context, and 1113 * pte lock(a spinlock) is held, which implies preemption disabled. 1114 */ 1115 if (PageTransHuge(page)) 1116 __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 1117 1118 __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, 1119 -hpage_nr_pages(page)); 1120 1121 if (unlikely(PageMlocked(page))) 1122 clear_page_mlock(page); 1123 1124 /* 1125 * It would be tidy to reset the PageAnon mapping here, 1126 * but that might overwrite a racing page_add_anon_rmap 1127 * which increments mapcount after us but sets mapping 1128 * before us: so leave the reset to free_hot_cold_page, 1129 * and remember that it's only reliable while mapped. 1130 * Leaving it set also helps swapoff to reinstate ptes 1131 * faster for those pages still in swapcache. 1132 */ 1133} 1134 1135/* 1136 * @arg: enum ttu_flags will be passed to this argument 1137 */ 1138static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, 1139 unsigned long address, void *arg) 1140{ 1141 struct mm_struct *mm = vma->vm_mm; 1142 pte_t *pte; 1143 pte_t pteval; 1144 spinlock_t *ptl; 1145 int ret = SWAP_AGAIN; 1146 enum ttu_flags flags = (enum ttu_flags)arg; 1147 1148 pte = page_check_address(page, mm, address, &ptl, 0); 1149 if (!pte) 1150 goto out; 1151 1152 /* 1153 * If the page is mlock()d, we cannot swap it out. 1154 * If it's recently referenced (perhaps page_referenced 1155 * skipped over this mm) then we should reactivate it. 1156 */ 1157 if (!(flags & TTU_IGNORE_MLOCK)) { 1158 if (vma->vm_flags & VM_LOCKED) 1159 goto out_mlock; 1160 1161 if (flags & TTU_MUNLOCK) 1162 goto out_unmap; 1163 } 1164 if (!(flags & TTU_IGNORE_ACCESS)) { 1165 if (ptep_clear_flush_young_notify(vma, address, pte)) { 1166 ret = SWAP_FAIL; 1167 goto out_unmap; 1168 } 1169 } 1170 1171 /* Nuke the page table entry. */ 1172 flush_cache_page(vma, address, page_to_pfn(page)); 1173 pteval = ptep_clear_flush(vma, address, pte); 1174 1175 /* Move the dirty bit to the physical page now the pte is gone. */ 1176 if (pte_dirty(pteval)) 1177 set_page_dirty(page); 1178 1179 /* Update high watermark before we lower rss */ 1180 update_hiwater_rss(mm); 1181 1182 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { 1183 if (!PageHuge(page)) { 1184 if (PageAnon(page)) 1185 dec_mm_counter(mm, MM_ANONPAGES); 1186 else 1187 dec_mm_counter(mm, MM_FILEPAGES); 1188 } 1189 set_pte_at(mm, address, pte, 1190 swp_entry_to_pte(make_hwpoison_entry(page))); 1191 } else if (pte_unused(pteval)) { 1192 /* 1193 * The guest indicated that the page content is of no 1194 * interest anymore. Simply discard the pte, vmscan 1195 * will take care of the rest. 1196 */ 1197 if (PageAnon(page)) 1198 dec_mm_counter(mm, MM_ANONPAGES); 1199 else 1200 dec_mm_counter(mm, MM_FILEPAGES); 1201 } else if (PageAnon(page)) { 1202 swp_entry_t entry = { .val = page_private(page) }; 1203 pte_t swp_pte; 1204 1205 if (PageSwapCache(page)) { 1206 /* 1207 * Store the swap location in the pte. 1208 * See handle_pte_fault() ... 1209 */ 1210 if (swap_duplicate(entry) < 0) { 1211 set_pte_at(mm, address, pte, pteval); 1212 ret = SWAP_FAIL; 1213 goto out_unmap; 1214 } 1215 if (list_empty(&mm->mmlist)) { 1216 spin_lock(&mmlist_lock); 1217 if (list_empty(&mm->mmlist)) 1218 list_add(&mm->mmlist, &init_mm.mmlist); 1219 spin_unlock(&mmlist_lock); 1220 } 1221 dec_mm_counter(mm, MM_ANONPAGES); 1222 inc_mm_counter(mm, MM_SWAPENTS); 1223 } else if (IS_ENABLED(CONFIG_MIGRATION)) { 1224 /* 1225 * Store the pfn of the page in a special migration 1226 * pte. do_swap_page() will wait until the migration 1227 * pte is removed and then restart fault handling. 1228 */ 1229 BUG_ON(!(flags & TTU_MIGRATION)); 1230 entry = make_migration_entry(page, pte_write(pteval)); 1231 } 1232 swp_pte = swp_entry_to_pte(entry); 1233 if (pte_soft_dirty(pteval)) 1234 swp_pte = pte_swp_mksoft_dirty(swp_pte); 1235 set_pte_at(mm, address, pte, swp_pte); 1236 BUG_ON(pte_file(*pte)); 1237 } else if (IS_ENABLED(CONFIG_MIGRATION) && 1238 (flags & TTU_MIGRATION)) { 1239 /* Establish migration entry for a file page */ 1240 swp_entry_t entry; 1241 entry = make_migration_entry(page, pte_write(pteval)); 1242 set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); 1243 } else 1244 dec_mm_counter(mm, MM_FILEPAGES); 1245 1246 page_remove_rmap(page); 1247 page_cache_release(page); 1248 1249out_unmap: 1250 pte_unmap_unlock(pte, ptl); 1251 if (ret != SWAP_FAIL && !(flags & TTU_MUNLOCK)) 1252 mmu_notifier_invalidate_page(mm, address); 1253out: 1254 return ret; 1255 1256out_mlock: 1257 pte_unmap_unlock(pte, ptl); 1258 1259 1260 /* 1261 * We need mmap_sem locking, Otherwise VM_LOCKED check makes 1262 * unstable result and race. Plus, We can't wait here because 1263 * we now hold anon_vma->rwsem or mapping->i_mmap_mutex. 1264 * if trylock failed, the page remain in evictable lru and later 1265 * vmscan could retry to move the page to unevictable lru if the 1266 * page is actually mlocked. 1267 */ 1268 if (down_read_trylock(&vma->vm_mm->mmap_sem)) { 1269 if (vma->vm_flags & VM_LOCKED) { 1270 mlock_vma_page(page); 1271 ret = SWAP_MLOCK; 1272 } 1273 up_read(&vma->vm_mm->mmap_sem); 1274 } 1275 return ret; 1276} 1277 1278/* 1279 * objrmap doesn't work for nonlinear VMAs because the assumption that 1280 * offset-into-file correlates with offset-into-virtual-addresses does not hold. 1281 * Consequently, given a particular page and its ->index, we cannot locate the 1282 * ptes which are mapping that page without an exhaustive linear search. 1283 * 1284 * So what this code does is a mini "virtual scan" of each nonlinear VMA which 1285 * maps the file to which the target page belongs. The ->vm_private_data field 1286 * holds the current cursor into that scan. Successive searches will circulate 1287 * around the vma's virtual address space. 1288 * 1289 * So as more replacement pressure is applied to the pages in a nonlinear VMA, 1290 * more scanning pressure is placed against them as well. Eventually pages 1291 * will become fully unmapped and are eligible for eviction. 1292 * 1293 * For very sparsely populated VMAs this is a little inefficient - chances are 1294 * there there won't be many ptes located within the scan cluster. In this case 1295 * maybe we could scan further - to the end of the pte page, perhaps. 1296 * 1297 * Mlocked pages: check VM_LOCKED under mmap_sem held for read, if we can 1298 * acquire it without blocking. If vma locked, mlock the pages in the cluster, 1299 * rather than unmapping them. If we encounter the "check_page" that vmscan is 1300 * trying to unmap, return SWAP_MLOCK, else default SWAP_AGAIN. 1301 */ 1302#define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE) 1303#define CLUSTER_MASK (~(CLUSTER_SIZE - 1)) 1304 1305static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, 1306 struct vm_area_struct *vma, struct page *check_page) 1307{ 1308 struct mm_struct *mm = vma->vm_mm; 1309 pmd_t *pmd; 1310 pte_t *pte; 1311 pte_t pteval; 1312 spinlock_t *ptl; 1313 struct page *page; 1314 unsigned long address; 1315 unsigned long mmun_start; /* For mmu_notifiers */ 1316 unsigned long mmun_end; /* For mmu_notifiers */ 1317 unsigned long end; 1318 int ret = SWAP_AGAIN; 1319 int locked_vma = 0; 1320 1321 address = (vma->vm_start + cursor) & CLUSTER_MASK; 1322 end = address + CLUSTER_SIZE; 1323 if (address < vma->vm_start) 1324 address = vma->vm_start; 1325 if (end > vma->vm_end) 1326 end = vma->vm_end; 1327 1328 pmd = mm_find_pmd(mm, address); 1329 if (!pmd) 1330 return ret; 1331 1332 mmun_start = address; 1333 mmun_end = end; 1334 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1335 1336 /* 1337 * If we can acquire the mmap_sem for read, and vma is VM_LOCKED, 1338 * keep the sem while scanning the cluster for mlocking pages. 1339 */ 1340 if (down_read_trylock(&vma->vm_mm->mmap_sem)) { 1341 locked_vma = (vma->vm_flags & VM_LOCKED); 1342 if (!locked_vma) 1343 up_read(&vma->vm_mm->mmap_sem); /* don't need it */ 1344 } 1345 1346 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 1347 1348 /* Update high watermark before we lower rss */ 1349 update_hiwater_rss(mm); 1350 1351 for (; address < end; pte++, address += PAGE_SIZE) { 1352 if (!pte_present(*pte)) 1353 continue; 1354 page = vm_normal_page(vma, address, *pte); 1355 BUG_ON(!page || PageAnon(page)); 1356 1357 if (locked_vma) { 1358 if (page == check_page) { 1359 /* we know we have check_page locked */ 1360 mlock_vma_page(page); 1361 ret = SWAP_MLOCK; 1362 } else if (trylock_page(page)) { 1363 /* 1364 * If we can lock the page, perform mlock. 1365 * Otherwise leave the page alone, it will be 1366 * eventually encountered again later. 1367 */ 1368 mlock_vma_page(page); 1369 unlock_page(page); 1370 } 1371 continue; /* don't unmap */ 1372 } 1373 1374 /* 1375 * No need for _notify because we're within an 1376 * mmu_notifier_invalidate_range_ {start|end} scope. 1377 */ 1378 if (ptep_clear_flush_young(vma, address, pte)) 1379 continue; 1380 1381 /* Nuke the page table entry. */ 1382 flush_cache_page(vma, address, pte_pfn(*pte)); 1383 pteval = ptep_clear_flush(vma, address, pte); 1384 1385 /* If nonlinear, store the file page offset in the pte. */ 1386 if (page->index != linear_page_index(vma, address)) { 1387 pte_t ptfile = pgoff_to_pte(page->index); 1388 if (pte_soft_dirty(pteval)) 1389 ptfile = pte_file_mksoft_dirty(ptfile); 1390 set_pte_at(mm, address, pte, ptfile); 1391 } 1392 1393 /* Move the dirty bit to the physical page now the pte is gone. */ 1394 if (pte_dirty(pteval)) 1395 set_page_dirty(page); 1396 1397 page_remove_rmap(page); 1398 page_cache_release(page); 1399 dec_mm_counter(mm, MM_FILEPAGES); 1400 (*mapcount)--; 1401 } 1402 pte_unmap_unlock(pte - 1, ptl); 1403 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1404 if (locked_vma) 1405 up_read(&vma->vm_mm->mmap_sem); 1406 return ret; 1407} 1408 1409static int try_to_unmap_nonlinear(struct page *page, 1410 struct address_space *mapping, void *arg) 1411{ 1412 struct vm_area_struct *vma; 1413 int ret = SWAP_AGAIN; 1414 unsigned long cursor; 1415 unsigned long max_nl_cursor = 0; 1416 unsigned long max_nl_size = 0; 1417 unsigned int mapcount; 1418 1419 list_for_each_entry(vma, 1420 &mapping->i_mmap_nonlinear, shared.nonlinear) { 1421 1422 cursor = (unsigned long) vma->vm_private_data; 1423 if (cursor > max_nl_cursor) 1424 max_nl_cursor = cursor; 1425 cursor = vma->vm_end - vma->vm_start; 1426 if (cursor > max_nl_size) 1427 max_nl_size = cursor; 1428 } 1429 1430 if (max_nl_size == 0) { /* all nonlinears locked or reserved ? */ 1431 return SWAP_FAIL; 1432 } 1433 1434 /* 1435 * We don't try to search for this page in the nonlinear vmas, 1436 * and page_referenced wouldn't have found it anyway. Instead 1437 * just walk the nonlinear vmas trying to age and unmap some. 1438 * The mapcount of the page we came in with is irrelevant, 1439 * but even so use it as a guide to how hard we should try? 1440 */ 1441 mapcount = page_mapcount(page); 1442 if (!mapcount) 1443 return ret; 1444 1445 cond_resched(); 1446 1447 max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK; 1448 if (max_nl_cursor == 0) 1449 max_nl_cursor = CLUSTER_SIZE; 1450 1451 do { 1452 list_for_each_entry(vma, 1453 &mapping->i_mmap_nonlinear, shared.nonlinear) { 1454 1455 cursor = (unsigned long) vma->vm_private_data; 1456 while (cursor < max_nl_cursor && 1457 cursor < vma->vm_end - vma->vm_start) { 1458 if (try_to_unmap_cluster(cursor, &mapcount, 1459 vma, page) == SWAP_MLOCK) 1460 ret = SWAP_MLOCK; 1461 cursor += CLUSTER_SIZE; 1462 vma->vm_private_data = (void *) cursor; 1463 if ((int)mapcount <= 0) 1464 return ret; 1465 } 1466 vma->vm_private_data = (void *) max_nl_cursor; 1467 } 1468 cond_resched(); 1469 max_nl_cursor += CLUSTER_SIZE; 1470 } while (max_nl_cursor <= max_nl_size); 1471 1472 /* 1473 * Don't loop forever (perhaps all the remaining pages are 1474 * in locked vmas). Reset cursor on all unreserved nonlinear 1475 * vmas, now forgetting on which ones it had fallen behind. 1476 */ 1477 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.nonlinear) 1478 vma->vm_private_data = NULL; 1479 1480 return ret; 1481} 1482 1483bool is_vma_temporary_stack(struct vm_area_struct *vma) 1484{ 1485 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 1486 1487 if (!maybe_stack) 1488 return false; 1489 1490 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1491 VM_STACK_INCOMPLETE_SETUP) 1492 return true; 1493 1494 return false; 1495} 1496 1497static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) 1498{ 1499 return is_vma_temporary_stack(vma); 1500} 1501 1502static int page_not_mapped(struct page *page) 1503{ 1504 return !page_mapped(page); 1505}; 1506 1507/** 1508 * try_to_unmap - try to remove all page table mappings to a page 1509 * @page: the page to get unmapped 1510 * @flags: action and flags 1511 * 1512 * Tries to remove all the page table entries which are mapping this 1513 * page, used in the pageout path. Caller must hold the page lock. 1514 * Return values are: 1515 * 1516 * SWAP_SUCCESS - we succeeded in removing all mappings 1517 * SWAP_AGAIN - we missed a mapping, try again later 1518 * SWAP_FAIL - the page is unswappable 1519 * SWAP_MLOCK - page is mlocked. 1520 */ 1521int try_to_unmap(struct page *page, enum ttu_flags flags) 1522{ 1523 int ret; 1524 struct rmap_walk_control rwc = { 1525 .rmap_one = try_to_unmap_one, 1526 .arg = (void *)flags, 1527 .done = page_not_mapped, 1528 .file_nonlinear = try_to_unmap_nonlinear, 1529 .anon_lock = page_lock_anon_vma_read, 1530 }; 1531 1532 VM_BUG_ON_PAGE(!PageHuge(page) && PageTransHuge(page), page); 1533 1534 /* 1535 * During exec, a temporary VMA is setup and later moved. 1536 * The VMA is moved under the anon_vma lock but not the 1537 * page tables leading to a race where migration cannot 1538 * find the migration ptes. Rather than increasing the 1539 * locking requirements of exec(), migration skips 1540 * temporary VMAs until after exec() completes. 1541 */ 1542 if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page)) 1543 rwc.invalid_vma = invalid_migration_vma; 1544 1545 ret = rmap_walk(page, &rwc); 1546 1547 if (ret != SWAP_MLOCK && !page_mapped(page)) 1548 ret = SWAP_SUCCESS; 1549 return ret; 1550} 1551 1552/** 1553 * try_to_munlock - try to munlock a page 1554 * @page: the page to be munlocked 1555 * 1556 * Called from munlock code. Checks all of the VMAs mapping the page 1557 * to make sure nobody else has this page mlocked. The page will be 1558 * returned with PG_mlocked cleared if no other vmas have it mlocked. 1559 * 1560 * Return values are: 1561 * 1562 * SWAP_AGAIN - no vma is holding page mlocked, or, 1563 * SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem 1564 * SWAP_FAIL - page cannot be located at present 1565 * SWAP_MLOCK - page is now mlocked. 1566 */ 1567int try_to_munlock(struct page *page) 1568{ 1569 int ret; 1570 struct rmap_walk_control rwc = { 1571 .rmap_one = try_to_unmap_one, 1572 .arg = (void *)TTU_MUNLOCK, 1573 .done = page_not_mapped, 1574 /* 1575 * We don't bother to try to find the munlocked page in 1576 * nonlinears. It's costly. Instead, later, page reclaim logic 1577 * may call try_to_unmap() and recover PG_mlocked lazily. 1578 */ 1579 .file_nonlinear = NULL, 1580 .anon_lock = page_lock_anon_vma_read, 1581 1582 }; 1583 1584 VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page); 1585 1586 ret = rmap_walk(page, &rwc); 1587 return ret; 1588} 1589 1590void __put_anon_vma(struct anon_vma *anon_vma) 1591{ 1592 struct anon_vma *root = anon_vma->root; 1593 1594 anon_vma_free(anon_vma); 1595 if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 1596 anon_vma_free(root); 1597} 1598 1599static struct anon_vma *rmap_walk_anon_lock(struct page *page, 1600 struct rmap_walk_control *rwc) 1601{ 1602 struct anon_vma *anon_vma; 1603 1604 if (rwc->anon_lock) 1605 return rwc->anon_lock(page); 1606 1607 /* 1608 * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read() 1609 * because that depends on page_mapped(); but not all its usages 1610 * are holding mmap_sem. Users without mmap_sem are required to 1611 * take a reference count to prevent the anon_vma disappearing 1612 */ 1613 anon_vma = page_anon_vma(page); 1614 if (!anon_vma) 1615 return NULL; 1616 1617 anon_vma_lock_read(anon_vma); 1618 return anon_vma; 1619} 1620 1621/* 1622 * rmap_walk_anon - do something to anonymous page using the object-based 1623 * rmap method 1624 * @page: the page to be handled 1625 * @rwc: control variable according to each walk type 1626 * 1627 * Find all the mappings of a page using the mapping pointer and the vma chains 1628 * contained in the anon_vma struct it points to. 1629 * 1630 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1631 * where the page was found will be held for write. So, we won't recheck 1632 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1633 * LOCKED. 1634 */ 1635static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc) 1636{ 1637 struct anon_vma *anon_vma; 1638 pgoff_t pgoff = page_to_pgoff(page); 1639 struct anon_vma_chain *avc; 1640 int ret = SWAP_AGAIN; 1641 1642 anon_vma = rmap_walk_anon_lock(page, rwc); 1643 if (!anon_vma) 1644 return ret; 1645 1646 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) { 1647 struct vm_area_struct *vma = avc->vma; 1648 unsigned long address = vma_address(page, vma); 1649 1650 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 1651 continue; 1652 1653 ret = rwc->rmap_one(page, vma, address, rwc->arg); 1654 if (ret != SWAP_AGAIN) 1655 break; 1656 if (rwc->done && rwc->done(page)) 1657 break; 1658 } 1659 anon_vma_unlock_read(anon_vma); 1660 return ret; 1661} 1662 1663/* 1664 * rmap_walk_file - do something to file page using the object-based rmap method 1665 * @page: the page to be handled 1666 * @rwc: control variable according to each walk type 1667 * 1668 * Find all the mappings of a page using the mapping pointer and the vma chains 1669 * contained in the address_space struct it points to. 1670 * 1671 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1672 * where the page was found will be held for write. So, we won't recheck 1673 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1674 * LOCKED. 1675 */ 1676static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc) 1677{ 1678 struct address_space *mapping = page->mapping; 1679 pgoff_t pgoff = page_to_pgoff(page); 1680 struct vm_area_struct *vma; 1681 int ret = SWAP_AGAIN; 1682 1683 /* 1684 * The page lock not only makes sure that page->mapping cannot 1685 * suddenly be NULLified by truncation, it makes sure that the 1686 * structure at mapping cannot be freed and reused yet, 1687 * so we can safely take mapping->i_mmap_mutex. 1688 */ 1689 VM_BUG_ON_PAGE(!PageLocked(page), page); 1690 1691 if (!mapping) 1692 return ret; 1693 mutex_lock(&mapping->i_mmap_mutex); 1694 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 1695 unsigned long address = vma_address(page, vma); 1696 1697 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 1698 continue; 1699 1700 ret = rwc->rmap_one(page, vma, address, rwc->arg); 1701 if (ret != SWAP_AGAIN) 1702 goto done; 1703 if (rwc->done && rwc->done(page)) 1704 goto done; 1705 } 1706 1707 if (!rwc->file_nonlinear) 1708 goto done; 1709 1710 if (list_empty(&mapping->i_mmap_nonlinear)) 1711 goto done; 1712 1713 ret = rwc->file_nonlinear(page, mapping, rwc->arg); 1714 1715done: 1716 mutex_unlock(&mapping->i_mmap_mutex); 1717 return ret; 1718} 1719 1720int rmap_walk(struct page *page, struct rmap_walk_control *rwc) 1721{ 1722 if (unlikely(PageKsm(page))) 1723 return rmap_walk_ksm(page, rwc); 1724 else if (PageAnon(page)) 1725 return rmap_walk_anon(page, rwc); 1726 else 1727 return rmap_walk_file(page, rwc); 1728} 1729 1730#ifdef CONFIG_HUGETLB_PAGE 1731/* 1732 * The following three functions are for anonymous (private mapped) hugepages. 1733 * Unlike common anonymous pages, anonymous hugepages have no accounting code 1734 * and no lru code, because we handle hugepages differently from common pages. 1735 */ 1736static void __hugepage_set_anon_rmap(struct page *page, 1737 struct vm_area_struct *vma, unsigned long address, int exclusive) 1738{ 1739 struct anon_vma *anon_vma = vma->anon_vma; 1740 1741 BUG_ON(!anon_vma); 1742 1743 if (PageAnon(page)) 1744 return; 1745 if (!exclusive) 1746 anon_vma = anon_vma->root; 1747 1748 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1749 page->mapping = (struct address_space *) anon_vma; 1750 page->index = linear_page_index(vma, address); 1751} 1752 1753void hugepage_add_anon_rmap(struct page *page, 1754 struct vm_area_struct *vma, unsigned long address) 1755{ 1756 struct anon_vma *anon_vma = vma->anon_vma; 1757 int first; 1758 1759 BUG_ON(!PageLocked(page)); 1760 BUG_ON(!anon_vma); 1761 /* address might be in next vma when migration races vma_adjust */ 1762 first = atomic_inc_and_test(&page->_mapcount); 1763 if (first) 1764 __hugepage_set_anon_rmap(page, vma, address, 0); 1765} 1766 1767void hugepage_add_new_anon_rmap(struct page *page, 1768 struct vm_area_struct *vma, unsigned long address) 1769{ 1770 BUG_ON(address < vma->vm_start || address >= vma->vm_end); 1771 atomic_set(&page->_mapcount, 0); 1772 __hugepage_set_anon_rmap(page, vma, address, 1); 1773} 1774#endif /* CONFIG_HUGETLB_PAGE */ 1775