memory.c revision 83b964bbf82eb13a8f31bb49ca420787fe01f7a6
1/* 2 * linux/mm/memory.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7/* 8 * demand-loading started 01.12.91 - seems it is high on the list of 9 * things wanted, and it should be easy to implement. - Linus 10 */ 11 12/* 13 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared 14 * pages started 02.12.91, seems to work. - Linus. 15 * 16 * Tested sharing by executing about 30 /bin/sh: under the old kernel it 17 * would have taken more than the 6M I have free, but it worked well as 18 * far as I could see. 19 * 20 * Also corrected some "invalidate()"s - I wasn't doing enough of them. 21 */ 22 23/* 24 * Real VM (paging to/from disk) started 18.12.91. Much more work and 25 * thought has to go into this. Oh, well.. 26 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. 27 * Found it. Everything seems to work now. 28 * 20.12.91 - Ok, making the swap-device changeable like the root. 29 */ 30 31/* 32 * 05.04.94 - Multi-page memory management added for v1.1. 33 * Idea by Alex Bligh (alex@cconcepts.co.uk) 34 * 35 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG 36 * (Gerhard.Wichert@pdb.siemens.de) 37 * 38 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) 39 */ 40 41#include <linux/kernel_stat.h> 42#include <linux/mm.h> 43#include <linux/hugetlb.h> 44#include <linux/mman.h> 45#include <linux/swap.h> 46#include <linux/highmem.h> 47#include <linux/pagemap.h> 48#include <linux/ksm.h> 49#include <linux/rmap.h> 50#include <linux/module.h> 51#include <linux/delayacct.h> 52#include <linux/init.h> 53#include <linux/writeback.h> 54#include <linux/memcontrol.h> 55#include <linux/mmu_notifier.h> 56#include <linux/kallsyms.h> 57#include <linux/swapops.h> 58#include <linux/elf.h> 59#include <linux/gfp.h> 60 61#include <asm/io.h> 62#include <asm/pgalloc.h> 63#include <asm/uaccess.h> 64#include <asm/tlb.h> 65#include <asm/tlbflush.h> 66#include <asm/pgtable.h> 67 68#include "internal.h" 69 70#ifndef CONFIG_NEED_MULTIPLE_NODES 71/* use the per-pgdat data instead for discontigmem - mbligh */ 72unsigned long max_mapnr; 73struct page *mem_map; 74 75EXPORT_SYMBOL(max_mapnr); 76EXPORT_SYMBOL(mem_map); 77#endif 78 79unsigned long num_physpages; 80/* 81 * A number of key systems in x86 including ioremap() rely on the assumption 82 * that high_memory defines the upper bound on direct map memory, then end 83 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and 84 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL 85 * and ZONE_HIGHMEM. 86 */ 87void * high_memory; 88 89EXPORT_SYMBOL(num_physpages); 90EXPORT_SYMBOL(high_memory); 91 92/* 93 * Randomize the address space (stacks, mmaps, brk, etc.). 94 * 95 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, 96 * as ancient (libc5 based) binaries can segfault. ) 97 */ 98int randomize_va_space __read_mostly = 99#ifdef CONFIG_COMPAT_BRK 100 1; 101#else 102 2; 103#endif 104 105static int __init disable_randmaps(char *s) 106{ 107 randomize_va_space = 0; 108 return 1; 109} 110__setup("norandmaps", disable_randmaps); 111 112unsigned long zero_pfn __read_mostly; 113unsigned long highest_memmap_pfn __read_mostly; 114 115/* 116 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init() 117 */ 118static int __init init_zero_pfn(void) 119{ 120 zero_pfn = page_to_pfn(ZERO_PAGE(0)); 121 return 0; 122} 123core_initcall(init_zero_pfn); 124 125 126#if defined(SPLIT_RSS_COUNTING) 127 128static void __sync_task_rss_stat(struct task_struct *task, struct mm_struct *mm) 129{ 130 int i; 131 132 for (i = 0; i < NR_MM_COUNTERS; i++) { 133 if (task->rss_stat.count[i]) { 134 add_mm_counter(mm, i, task->rss_stat.count[i]); 135 task->rss_stat.count[i] = 0; 136 } 137 } 138 task->rss_stat.events = 0; 139} 140 141static void add_mm_counter_fast(struct mm_struct *mm, int member, int val) 142{ 143 struct task_struct *task = current; 144 145 if (likely(task->mm == mm)) 146 task->rss_stat.count[member] += val; 147 else 148 add_mm_counter(mm, member, val); 149} 150#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1) 151#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) 152 153/* sync counter once per 64 page faults */ 154#define TASK_RSS_EVENTS_THRESH (64) 155static void check_sync_rss_stat(struct task_struct *task) 156{ 157 if (unlikely(task != current)) 158 return; 159 if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH)) 160 __sync_task_rss_stat(task, task->mm); 161} 162 163unsigned long get_mm_counter(struct mm_struct *mm, int member) 164{ 165 long val = 0; 166 167 /* 168 * Don't use task->mm here...for avoiding to use task_get_mm().. 169 * The caller must guarantee task->mm is not invalid. 170 */ 171 val = atomic_long_read(&mm->rss_stat.count[member]); 172 /* 173 * counter is updated in asynchronous manner and may go to minus. 174 * But it's never be expected number for users. 175 */ 176 if (val < 0) 177 return 0; 178 return (unsigned long)val; 179} 180 181void sync_mm_rss(struct task_struct *task, struct mm_struct *mm) 182{ 183 __sync_task_rss_stat(task, mm); 184} 185#else 186 187#define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member) 188#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member) 189 190static void check_sync_rss_stat(struct task_struct *task) 191{ 192} 193 194#endif 195 196/* 197 * If a p?d_bad entry is found while walking page tables, report 198 * the error, before resetting entry to p?d_none. Usually (but 199 * very seldom) called out from the p?d_none_or_clear_bad macros. 200 */ 201 202void pgd_clear_bad(pgd_t *pgd) 203{ 204 pgd_ERROR(*pgd); 205 pgd_clear(pgd); 206} 207 208void pud_clear_bad(pud_t *pud) 209{ 210 pud_ERROR(*pud); 211 pud_clear(pud); 212} 213 214void pmd_clear_bad(pmd_t *pmd) 215{ 216 pmd_ERROR(*pmd); 217 pmd_clear(pmd); 218} 219 220/* 221 * Note: this doesn't free the actual pages themselves. That 222 * has been handled earlier when unmapping all the memory regions. 223 */ 224static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, 225 unsigned long addr) 226{ 227 pgtable_t token = pmd_pgtable(*pmd); 228 pmd_clear(pmd); 229 pte_free_tlb(tlb, token, addr); 230 tlb->mm->nr_ptes--; 231} 232 233static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 234 unsigned long addr, unsigned long end, 235 unsigned long floor, unsigned long ceiling) 236{ 237 pmd_t *pmd; 238 unsigned long next; 239 unsigned long start; 240 241 start = addr; 242 pmd = pmd_offset(pud, addr); 243 do { 244 next = pmd_addr_end(addr, end); 245 if (pmd_none_or_clear_bad(pmd)) 246 continue; 247 free_pte_range(tlb, pmd, addr); 248 } while (pmd++, addr = next, addr != end); 249 250 start &= PUD_MASK; 251 if (start < floor) 252 return; 253 if (ceiling) { 254 ceiling &= PUD_MASK; 255 if (!ceiling) 256 return; 257 } 258 if (end - 1 > ceiling - 1) 259 return; 260 261 pmd = pmd_offset(pud, start); 262 pud_clear(pud); 263 pmd_free_tlb(tlb, pmd, start); 264} 265 266static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, 267 unsigned long addr, unsigned long end, 268 unsigned long floor, unsigned long ceiling) 269{ 270 pud_t *pud; 271 unsigned long next; 272 unsigned long start; 273 274 start = addr; 275 pud = pud_offset(pgd, addr); 276 do { 277 next = pud_addr_end(addr, end); 278 if (pud_none_or_clear_bad(pud)) 279 continue; 280 free_pmd_range(tlb, pud, addr, next, floor, ceiling); 281 } while (pud++, addr = next, addr != end); 282 283 start &= PGDIR_MASK; 284 if (start < floor) 285 return; 286 if (ceiling) { 287 ceiling &= PGDIR_MASK; 288 if (!ceiling) 289 return; 290 } 291 if (end - 1 > ceiling - 1) 292 return; 293 294 pud = pud_offset(pgd, start); 295 pgd_clear(pgd); 296 pud_free_tlb(tlb, pud, start); 297} 298 299/* 300 * This function frees user-level page tables of a process. 301 * 302 * Must be called with pagetable lock held. 303 */ 304void free_pgd_range(struct mmu_gather *tlb, 305 unsigned long addr, unsigned long end, 306 unsigned long floor, unsigned long ceiling) 307{ 308 pgd_t *pgd; 309 unsigned long next; 310 311 /* 312 * The next few lines have given us lots of grief... 313 * 314 * Why are we testing PMD* at this top level? Because often 315 * there will be no work to do at all, and we'd prefer not to 316 * go all the way down to the bottom just to discover that. 317 * 318 * Why all these "- 1"s? Because 0 represents both the bottom 319 * of the address space and the top of it (using -1 for the 320 * top wouldn't help much: the masks would do the wrong thing). 321 * The rule is that addr 0 and floor 0 refer to the bottom of 322 * the address space, but end 0 and ceiling 0 refer to the top 323 * Comparisons need to use "end - 1" and "ceiling - 1" (though 324 * that end 0 case should be mythical). 325 * 326 * Wherever addr is brought up or ceiling brought down, we must 327 * be careful to reject "the opposite 0" before it confuses the 328 * subsequent tests. But what about where end is brought down 329 * by PMD_SIZE below? no, end can't go down to 0 there. 330 * 331 * Whereas we round start (addr) and ceiling down, by different 332 * masks at different levels, in order to test whether a table 333 * now has no other vmas using it, so can be freed, we don't 334 * bother to round floor or end up - the tests don't need that. 335 */ 336 337 addr &= PMD_MASK; 338 if (addr < floor) { 339 addr += PMD_SIZE; 340 if (!addr) 341 return; 342 } 343 if (ceiling) { 344 ceiling &= PMD_MASK; 345 if (!ceiling) 346 return; 347 } 348 if (end - 1 > ceiling - 1) 349 end -= PMD_SIZE; 350 if (addr > end - 1) 351 return; 352 353 pgd = pgd_offset(tlb->mm, addr); 354 do { 355 next = pgd_addr_end(addr, end); 356 if (pgd_none_or_clear_bad(pgd)) 357 continue; 358 free_pud_range(tlb, pgd, addr, next, floor, ceiling); 359 } while (pgd++, addr = next, addr != end); 360} 361 362void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, 363 unsigned long floor, unsigned long ceiling) 364{ 365 while (vma) { 366 struct vm_area_struct *next = vma->vm_next; 367 unsigned long addr = vma->vm_start; 368 369 /* 370 * Hide vma from rmap and truncate_pagecache before freeing 371 * pgtables 372 */ 373 unlink_anon_vmas(vma); 374 unlink_file_vma(vma); 375 376 if (is_vm_hugetlb_page(vma)) { 377 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, 378 floor, next? next->vm_start: ceiling); 379 } else { 380 /* 381 * Optimization: gather nearby vmas into one call down 382 */ 383 while (next && next->vm_start <= vma->vm_end + PMD_SIZE 384 && !is_vm_hugetlb_page(next)) { 385 vma = next; 386 next = vma->vm_next; 387 unlink_anon_vmas(vma); 388 unlink_file_vma(vma); 389 } 390 free_pgd_range(tlb, addr, vma->vm_end, 391 floor, next? next->vm_start: ceiling); 392 } 393 vma = next; 394 } 395} 396 397int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 398 pmd_t *pmd, unsigned long address) 399{ 400 pgtable_t new = pte_alloc_one(mm, address); 401 int wait_split_huge_page; 402 if (!new) 403 return -ENOMEM; 404 405 /* 406 * Ensure all pte setup (eg. pte page lock and page clearing) are 407 * visible before the pte is made visible to other CPUs by being 408 * put into page tables. 409 * 410 * The other side of the story is the pointer chasing in the page 411 * table walking code (when walking the page table without locking; 412 * ie. most of the time). Fortunately, these data accesses consist 413 * of a chain of data-dependent loads, meaning most CPUs (alpha 414 * being the notable exception) will already guarantee loads are 415 * seen in-order. See the alpha page table accessors for the 416 * smp_read_barrier_depends() barriers in page table walking code. 417 */ 418 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ 419 420 spin_lock(&mm->page_table_lock); 421 wait_split_huge_page = 0; 422 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 423 mm->nr_ptes++; 424 pmd_populate(mm, pmd, new); 425 new = NULL; 426 } else if (unlikely(pmd_trans_splitting(*pmd))) 427 wait_split_huge_page = 1; 428 spin_unlock(&mm->page_table_lock); 429 if (new) 430 pte_free(mm, new); 431 if (wait_split_huge_page) 432 wait_split_huge_page(vma->anon_vma, pmd); 433 return 0; 434} 435 436int __pte_alloc_kernel(pmd_t *pmd, unsigned long address) 437{ 438 pte_t *new = pte_alloc_one_kernel(&init_mm, address); 439 if (!new) 440 return -ENOMEM; 441 442 smp_wmb(); /* See comment in __pte_alloc */ 443 444 spin_lock(&init_mm.page_table_lock); 445 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 446 pmd_populate_kernel(&init_mm, pmd, new); 447 new = NULL; 448 } else 449 VM_BUG_ON(pmd_trans_splitting(*pmd)); 450 spin_unlock(&init_mm.page_table_lock); 451 if (new) 452 pte_free_kernel(&init_mm, new); 453 return 0; 454} 455 456static inline void init_rss_vec(int *rss) 457{ 458 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS); 459} 460 461static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) 462{ 463 int i; 464 465 if (current->mm == mm) 466 sync_mm_rss(current, mm); 467 for (i = 0; i < NR_MM_COUNTERS; i++) 468 if (rss[i]) 469 add_mm_counter(mm, i, rss[i]); 470} 471 472/* 473 * This function is called to print an error when a bad pte 474 * is found. For example, we might have a PFN-mapped pte in 475 * a region that doesn't allow it. 476 * 477 * The calling function must still handle the error. 478 */ 479static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, 480 pte_t pte, struct page *page) 481{ 482 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); 483 pud_t *pud = pud_offset(pgd, addr); 484 pmd_t *pmd = pmd_offset(pud, addr); 485 struct address_space *mapping; 486 pgoff_t index; 487 static unsigned long resume; 488 static unsigned long nr_shown; 489 static unsigned long nr_unshown; 490 491 /* 492 * Allow a burst of 60 reports, then keep quiet for that minute; 493 * or allow a steady drip of one report per second. 494 */ 495 if (nr_shown == 60) { 496 if (time_before(jiffies, resume)) { 497 nr_unshown++; 498 return; 499 } 500 if (nr_unshown) { 501 printk(KERN_ALERT 502 "BUG: Bad page map: %lu messages suppressed\n", 503 nr_unshown); 504 nr_unshown = 0; 505 } 506 nr_shown = 0; 507 } 508 if (nr_shown++ == 0) 509 resume = jiffies + 60 * HZ; 510 511 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; 512 index = linear_page_index(vma, addr); 513 514 printk(KERN_ALERT 515 "BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n", 516 current->comm, 517 (long long)pte_val(pte), (long long)pmd_val(*pmd)); 518 if (page) 519 dump_page(page); 520 printk(KERN_ALERT 521 "addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n", 522 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); 523 /* 524 * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y 525 */ 526 if (vma->vm_ops) 527 print_symbol(KERN_ALERT "vma->vm_ops->fault: %s\n", 528 (unsigned long)vma->vm_ops->fault); 529 if (vma->vm_file && vma->vm_file->f_op) 530 print_symbol(KERN_ALERT "vma->vm_file->f_op->mmap: %s\n", 531 (unsigned long)vma->vm_file->f_op->mmap); 532 dump_stack(); 533 add_taint(TAINT_BAD_PAGE); 534} 535 536static inline int is_cow_mapping(unsigned int flags) 537{ 538 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 539} 540 541#ifndef is_zero_pfn 542static inline int is_zero_pfn(unsigned long pfn) 543{ 544 return pfn == zero_pfn; 545} 546#endif 547 548#ifndef my_zero_pfn 549static inline unsigned long my_zero_pfn(unsigned long addr) 550{ 551 return zero_pfn; 552} 553#endif 554 555/* 556 * vm_normal_page -- This function gets the "struct page" associated with a pte. 557 * 558 * "Special" mappings do not wish to be associated with a "struct page" (either 559 * it doesn't exist, or it exists but they don't want to touch it). In this 560 * case, NULL is returned here. "Normal" mappings do have a struct page. 561 * 562 * There are 2 broad cases. Firstly, an architecture may define a pte_special() 563 * pte bit, in which case this function is trivial. Secondly, an architecture 564 * may not have a spare pte bit, which requires a more complicated scheme, 565 * described below. 566 * 567 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a 568 * special mapping (even if there are underlying and valid "struct pages"). 569 * COWed pages of a VM_PFNMAP are always normal. 570 * 571 * The way we recognize COWed pages within VM_PFNMAP mappings is through the 572 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit 573 * set, and the vm_pgoff will point to the first PFN mapped: thus every special 574 * mapping will always honor the rule 575 * 576 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) 577 * 578 * And for normal mappings this is false. 579 * 580 * This restricts such mappings to be a linear translation from virtual address 581 * to pfn. To get around this restriction, we allow arbitrary mappings so long 582 * as the vma is not a COW mapping; in that case, we know that all ptes are 583 * special (because none can have been COWed). 584 * 585 * 586 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP. 587 * 588 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct 589 * page" backing, however the difference is that _all_ pages with a struct 590 * page (that is, those where pfn_valid is true) are refcounted and considered 591 * normal pages by the VM. The disadvantage is that pages are refcounted 592 * (which can be slower and simply not an option for some PFNMAP users). The 593 * advantage is that we don't have to follow the strict linearity rule of 594 * PFNMAP mappings in order to support COWable mappings. 595 * 596 */ 597#ifdef __HAVE_ARCH_PTE_SPECIAL 598# define HAVE_PTE_SPECIAL 1 599#else 600# define HAVE_PTE_SPECIAL 0 601#endif 602struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 603 pte_t pte) 604{ 605 unsigned long pfn = pte_pfn(pte); 606 607 if (HAVE_PTE_SPECIAL) { 608 if (likely(!pte_special(pte))) 609 goto check_pfn; 610 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) 611 return NULL; 612 if (!is_zero_pfn(pfn)) 613 print_bad_pte(vma, addr, pte, NULL); 614 return NULL; 615 } 616 617 /* !HAVE_PTE_SPECIAL case follows: */ 618 619 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 620 if (vma->vm_flags & VM_MIXEDMAP) { 621 if (!pfn_valid(pfn)) 622 return NULL; 623 goto out; 624 } else { 625 unsigned long off; 626 off = (addr - vma->vm_start) >> PAGE_SHIFT; 627 if (pfn == vma->vm_pgoff + off) 628 return NULL; 629 if (!is_cow_mapping(vma->vm_flags)) 630 return NULL; 631 } 632 } 633 634 if (is_zero_pfn(pfn)) 635 return NULL; 636check_pfn: 637 if (unlikely(pfn > highest_memmap_pfn)) { 638 print_bad_pte(vma, addr, pte, NULL); 639 return NULL; 640 } 641 642 /* 643 * NOTE! We still have PageReserved() pages in the page tables. 644 * eg. VDSO mappings can cause them to exist. 645 */ 646out: 647 return pfn_to_page(pfn); 648} 649 650/* 651 * copy one vm_area from one task to the other. Assumes the page tables 652 * already present in the new task to be cleared in the whole range 653 * covered by this vma. 654 */ 655 656static inline unsigned long 657copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, 658 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, 659 unsigned long addr, int *rss) 660{ 661 unsigned long vm_flags = vma->vm_flags; 662 pte_t pte = *src_pte; 663 struct page *page; 664 665 /* pte contains position in swap or file, so copy. */ 666 if (unlikely(!pte_present(pte))) { 667 if (!pte_file(pte)) { 668 swp_entry_t entry = pte_to_swp_entry(pte); 669 670 if (swap_duplicate(entry) < 0) 671 return entry.val; 672 673 /* make sure dst_mm is on swapoff's mmlist. */ 674 if (unlikely(list_empty(&dst_mm->mmlist))) { 675 spin_lock(&mmlist_lock); 676 if (list_empty(&dst_mm->mmlist)) 677 list_add(&dst_mm->mmlist, 678 &src_mm->mmlist); 679 spin_unlock(&mmlist_lock); 680 } 681 if (likely(!non_swap_entry(entry))) 682 rss[MM_SWAPENTS]++; 683 else if (is_write_migration_entry(entry) && 684 is_cow_mapping(vm_flags)) { 685 /* 686 * COW mappings require pages in both parent 687 * and child to be set to read. 688 */ 689 make_migration_entry_read(&entry); 690 pte = swp_entry_to_pte(entry); 691 set_pte_at(src_mm, addr, src_pte, pte); 692 } 693 } 694 goto out_set_pte; 695 } 696 697 /* 698 * If it's a COW mapping, write protect it both 699 * in the parent and the child 700 */ 701 if (is_cow_mapping(vm_flags)) { 702 ptep_set_wrprotect(src_mm, addr, src_pte); 703 pte = pte_wrprotect(pte); 704 } 705 706 /* 707 * If it's a shared mapping, mark it clean in 708 * the child 709 */ 710 if (vm_flags & VM_SHARED) 711 pte = pte_mkclean(pte); 712 pte = pte_mkold(pte); 713 714 page = vm_normal_page(vma, addr, pte); 715 if (page) { 716 get_page(page); 717 page_dup_rmap(page); 718 if (PageAnon(page)) 719 rss[MM_ANONPAGES]++; 720 else 721 rss[MM_FILEPAGES]++; 722 } 723 724out_set_pte: 725 set_pte_at(dst_mm, addr, dst_pte, pte); 726 return 0; 727} 728 729int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 730 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, 731 unsigned long addr, unsigned long end) 732{ 733 pte_t *orig_src_pte, *orig_dst_pte; 734 pte_t *src_pte, *dst_pte; 735 spinlock_t *src_ptl, *dst_ptl; 736 int progress = 0; 737 int rss[NR_MM_COUNTERS]; 738 swp_entry_t entry = (swp_entry_t){0}; 739 740again: 741 init_rss_vec(rss); 742 743 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); 744 if (!dst_pte) 745 return -ENOMEM; 746 src_pte = pte_offset_map(src_pmd, addr); 747 src_ptl = pte_lockptr(src_mm, src_pmd); 748 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 749 orig_src_pte = src_pte; 750 orig_dst_pte = dst_pte; 751 arch_enter_lazy_mmu_mode(); 752 753 do { 754 /* 755 * We are holding two locks at this point - either of them 756 * could generate latencies in another task on another CPU. 757 */ 758 if (progress >= 32) { 759 progress = 0; 760 if (need_resched() || 761 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl)) 762 break; 763 } 764 if (pte_none(*src_pte)) { 765 progress++; 766 continue; 767 } 768 entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, 769 vma, addr, rss); 770 if (entry.val) 771 break; 772 progress += 8; 773 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 774 775 arch_leave_lazy_mmu_mode(); 776 spin_unlock(src_ptl); 777 pte_unmap(orig_src_pte); 778 add_mm_rss_vec(dst_mm, rss); 779 pte_unmap_unlock(orig_dst_pte, dst_ptl); 780 cond_resched(); 781 782 if (entry.val) { 783 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) 784 return -ENOMEM; 785 progress = 0; 786 } 787 if (addr != end) 788 goto again; 789 return 0; 790} 791 792static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 793 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma, 794 unsigned long addr, unsigned long end) 795{ 796 pmd_t *src_pmd, *dst_pmd; 797 unsigned long next; 798 799 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); 800 if (!dst_pmd) 801 return -ENOMEM; 802 src_pmd = pmd_offset(src_pud, addr); 803 do { 804 next = pmd_addr_end(addr, end); 805 if (pmd_trans_huge(*src_pmd)) { 806 int err; 807 VM_BUG_ON(next-addr != HPAGE_PMD_SIZE); 808 err = copy_huge_pmd(dst_mm, src_mm, 809 dst_pmd, src_pmd, addr, vma); 810 if (err == -ENOMEM) 811 return -ENOMEM; 812 if (!err) 813 continue; 814 /* fall through */ 815 } 816 if (pmd_none_or_clear_bad(src_pmd)) 817 continue; 818 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd, 819 vma, addr, next)) 820 return -ENOMEM; 821 } while (dst_pmd++, src_pmd++, addr = next, addr != end); 822 return 0; 823} 824 825static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 826 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma, 827 unsigned long addr, unsigned long end) 828{ 829 pud_t *src_pud, *dst_pud; 830 unsigned long next; 831 832 dst_pud = pud_alloc(dst_mm, dst_pgd, addr); 833 if (!dst_pud) 834 return -ENOMEM; 835 src_pud = pud_offset(src_pgd, addr); 836 do { 837 next = pud_addr_end(addr, end); 838 if (pud_none_or_clear_bad(src_pud)) 839 continue; 840 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud, 841 vma, addr, next)) 842 return -ENOMEM; 843 } while (dst_pud++, src_pud++, addr = next, addr != end); 844 return 0; 845} 846 847int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 848 struct vm_area_struct *vma) 849{ 850 pgd_t *src_pgd, *dst_pgd; 851 unsigned long next; 852 unsigned long addr = vma->vm_start; 853 unsigned long end = vma->vm_end; 854 int ret; 855 856 /* 857 * Don't copy ptes where a page fault will fill them correctly. 858 * Fork becomes much lighter when there are big shared or private 859 * readonly mappings. The tradeoff is that copy_page_range is more 860 * efficient than faulting. 861 */ 862 if (!(vma->vm_flags & (VM_HUGETLB|VM_NONLINEAR|VM_PFNMAP|VM_INSERTPAGE))) { 863 if (!vma->anon_vma) 864 return 0; 865 } 866 867 if (is_vm_hugetlb_page(vma)) 868 return copy_hugetlb_page_range(dst_mm, src_mm, vma); 869 870 if (unlikely(is_pfn_mapping(vma))) { 871 /* 872 * We do not free on error cases below as remove_vma 873 * gets called on error from higher level routine 874 */ 875 ret = track_pfn_vma_copy(vma); 876 if (ret) 877 return ret; 878 } 879 880 /* 881 * We need to invalidate the secondary MMU mappings only when 882 * there could be a permission downgrade on the ptes of the 883 * parent mm. And a permission downgrade will only happen if 884 * is_cow_mapping() returns true. 885 */ 886 if (is_cow_mapping(vma->vm_flags)) 887 mmu_notifier_invalidate_range_start(src_mm, addr, end); 888 889 ret = 0; 890 dst_pgd = pgd_offset(dst_mm, addr); 891 src_pgd = pgd_offset(src_mm, addr); 892 do { 893 next = pgd_addr_end(addr, end); 894 if (pgd_none_or_clear_bad(src_pgd)) 895 continue; 896 if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd, 897 vma, addr, next))) { 898 ret = -ENOMEM; 899 break; 900 } 901 } while (dst_pgd++, src_pgd++, addr = next, addr != end); 902 903 if (is_cow_mapping(vma->vm_flags)) 904 mmu_notifier_invalidate_range_end(src_mm, 905 vma->vm_start, end); 906 return ret; 907} 908 909static unsigned long zap_pte_range(struct mmu_gather *tlb, 910 struct vm_area_struct *vma, pmd_t *pmd, 911 unsigned long addr, unsigned long end, 912 long *zap_work, struct zap_details *details) 913{ 914 struct mm_struct *mm = tlb->mm; 915 pte_t *pte; 916 spinlock_t *ptl; 917 int rss[NR_MM_COUNTERS]; 918 919 init_rss_vec(rss); 920 921 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 922 arch_enter_lazy_mmu_mode(); 923 do { 924 pte_t ptent = *pte; 925 if (pte_none(ptent)) { 926 (*zap_work)--; 927 continue; 928 } 929 930 (*zap_work) -= PAGE_SIZE; 931 932 if (pte_present(ptent)) { 933 struct page *page; 934 935 page = vm_normal_page(vma, addr, ptent); 936 if (unlikely(details) && page) { 937 /* 938 * unmap_shared_mapping_pages() wants to 939 * invalidate cache without truncating: 940 * unmap shared but keep private pages. 941 */ 942 if (details->check_mapping && 943 details->check_mapping != page->mapping) 944 continue; 945 /* 946 * Each page->index must be checked when 947 * invalidating or truncating nonlinear. 948 */ 949 if (details->nonlinear_vma && 950 (page->index < details->first_index || 951 page->index > details->last_index)) 952 continue; 953 } 954 ptent = ptep_get_and_clear_full(mm, addr, pte, 955 tlb->fullmm); 956 tlb_remove_tlb_entry(tlb, pte, addr); 957 if (unlikely(!page)) 958 continue; 959 if (unlikely(details) && details->nonlinear_vma 960 && linear_page_index(details->nonlinear_vma, 961 addr) != page->index) 962 set_pte_at(mm, addr, pte, 963 pgoff_to_pte(page->index)); 964 if (PageAnon(page)) 965 rss[MM_ANONPAGES]--; 966 else { 967 if (pte_dirty(ptent)) 968 set_page_dirty(page); 969 if (pte_young(ptent) && 970 likely(!VM_SequentialReadHint(vma))) 971 mark_page_accessed(page); 972 rss[MM_FILEPAGES]--; 973 } 974 page_remove_rmap(page); 975 if (unlikely(page_mapcount(page) < 0)) 976 print_bad_pte(vma, addr, ptent, page); 977 tlb_remove_page(tlb, page); 978 continue; 979 } 980 /* 981 * If details->check_mapping, we leave swap entries; 982 * if details->nonlinear_vma, we leave file entries. 983 */ 984 if (unlikely(details)) 985 continue; 986 if (pte_file(ptent)) { 987 if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) 988 print_bad_pte(vma, addr, ptent, NULL); 989 } else { 990 swp_entry_t entry = pte_to_swp_entry(ptent); 991 992 if (!non_swap_entry(entry)) 993 rss[MM_SWAPENTS]--; 994 if (unlikely(!free_swap_and_cache(entry))) 995 print_bad_pte(vma, addr, ptent, NULL); 996 } 997 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 998 } while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0)); 999 1000 add_mm_rss_vec(mm, rss); 1001 arch_leave_lazy_mmu_mode(); 1002 pte_unmap_unlock(pte - 1, ptl); 1003 1004 return addr; 1005} 1006 1007static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, 1008 struct vm_area_struct *vma, pud_t *pud, 1009 unsigned long addr, unsigned long end, 1010 long *zap_work, struct zap_details *details) 1011{ 1012 pmd_t *pmd; 1013 unsigned long next; 1014 1015 pmd = pmd_offset(pud, addr); 1016 do { 1017 next = pmd_addr_end(addr, end); 1018 if (pmd_trans_huge(*pmd)) { 1019 if (next-addr != HPAGE_PMD_SIZE) { 1020 VM_BUG_ON(!rwsem_is_locked(&tlb->mm->mmap_sem)); 1021 split_huge_page_pmd(vma->vm_mm, pmd); 1022 } else if (zap_huge_pmd(tlb, vma, pmd)) { 1023 (*zap_work)--; 1024 continue; 1025 } 1026 /* fall through */ 1027 } 1028 if (pmd_none_or_clear_bad(pmd)) { 1029 (*zap_work)--; 1030 continue; 1031 } 1032 next = zap_pte_range(tlb, vma, pmd, addr, next, 1033 zap_work, details); 1034 } while (pmd++, addr = next, (addr != end && *zap_work > 0)); 1035 1036 return addr; 1037} 1038 1039static inline unsigned long zap_pud_range(struct mmu_gather *tlb, 1040 struct vm_area_struct *vma, pgd_t *pgd, 1041 unsigned long addr, unsigned long end, 1042 long *zap_work, struct zap_details *details) 1043{ 1044 pud_t *pud; 1045 unsigned long next; 1046 1047 pud = pud_offset(pgd, addr); 1048 do { 1049 next = pud_addr_end(addr, end); 1050 if (pud_none_or_clear_bad(pud)) { 1051 (*zap_work)--; 1052 continue; 1053 } 1054 next = zap_pmd_range(tlb, vma, pud, addr, next, 1055 zap_work, details); 1056 } while (pud++, addr = next, (addr != end && *zap_work > 0)); 1057 1058 return addr; 1059} 1060 1061static unsigned long unmap_page_range(struct mmu_gather *tlb, 1062 struct vm_area_struct *vma, 1063 unsigned long addr, unsigned long end, 1064 long *zap_work, struct zap_details *details) 1065{ 1066 pgd_t *pgd; 1067 unsigned long next; 1068 1069 if (details && !details->check_mapping && !details->nonlinear_vma) 1070 details = NULL; 1071 1072 BUG_ON(addr >= end); 1073 mem_cgroup_uncharge_start(); 1074 tlb_start_vma(tlb, vma); 1075 pgd = pgd_offset(vma->vm_mm, addr); 1076 do { 1077 next = pgd_addr_end(addr, end); 1078 if (pgd_none_or_clear_bad(pgd)) { 1079 (*zap_work)--; 1080 continue; 1081 } 1082 next = zap_pud_range(tlb, vma, pgd, addr, next, 1083 zap_work, details); 1084 } while (pgd++, addr = next, (addr != end && *zap_work > 0)); 1085 tlb_end_vma(tlb, vma); 1086 mem_cgroup_uncharge_end(); 1087 1088 return addr; 1089} 1090 1091#ifdef CONFIG_PREEMPT 1092# define ZAP_BLOCK_SIZE (8 * PAGE_SIZE) 1093#else 1094/* No preempt: go for improved straight-line efficiency */ 1095# define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE) 1096#endif 1097 1098/** 1099 * unmap_vmas - unmap a range of memory covered by a list of vma's 1100 * @tlbp: address of the caller's struct mmu_gather 1101 * @vma: the starting vma 1102 * @start_addr: virtual address at which to start unmapping 1103 * @end_addr: virtual address at which to end unmapping 1104 * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here 1105 * @details: details of nonlinear truncation or shared cache invalidation 1106 * 1107 * Returns the end address of the unmapping (restart addr if interrupted). 1108 * 1109 * Unmap all pages in the vma list. 1110 * 1111 * We aim to not hold locks for too long (for scheduling latency reasons). 1112 * So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to 1113 * return the ending mmu_gather to the caller. 1114 * 1115 * Only addresses between `start' and `end' will be unmapped. 1116 * 1117 * The VMA list must be sorted in ascending virtual address order. 1118 * 1119 * unmap_vmas() assumes that the caller will flush the whole unmapped address 1120 * range after unmap_vmas() returns. So the only responsibility here is to 1121 * ensure that any thus-far unmapped pages are flushed before unmap_vmas() 1122 * drops the lock and schedules. 1123 */ 1124unsigned long unmap_vmas(struct mmu_gather **tlbp, 1125 struct vm_area_struct *vma, unsigned long start_addr, 1126 unsigned long end_addr, unsigned long *nr_accounted, 1127 struct zap_details *details) 1128{ 1129 long zap_work = ZAP_BLOCK_SIZE; 1130 unsigned long tlb_start = 0; /* For tlb_finish_mmu */ 1131 int tlb_start_valid = 0; 1132 unsigned long start = start_addr; 1133 spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL; 1134 int fullmm = (*tlbp)->fullmm; 1135 struct mm_struct *mm = vma->vm_mm; 1136 1137 mmu_notifier_invalidate_range_start(mm, start_addr, end_addr); 1138 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) { 1139 unsigned long end; 1140 1141 start = max(vma->vm_start, start_addr); 1142 if (start >= vma->vm_end) 1143 continue; 1144 end = min(vma->vm_end, end_addr); 1145 if (end <= vma->vm_start) 1146 continue; 1147 1148 if (vma->vm_flags & VM_ACCOUNT) 1149 *nr_accounted += (end - start) >> PAGE_SHIFT; 1150 1151 if (unlikely(is_pfn_mapping(vma))) 1152 untrack_pfn_vma(vma, 0, 0); 1153 1154 while (start != end) { 1155 if (!tlb_start_valid) { 1156 tlb_start = start; 1157 tlb_start_valid = 1; 1158 } 1159 1160 if (unlikely(is_vm_hugetlb_page(vma))) { 1161 /* 1162 * It is undesirable to test vma->vm_file as it 1163 * should be non-null for valid hugetlb area. 1164 * However, vm_file will be NULL in the error 1165 * cleanup path of do_mmap_pgoff. When 1166 * hugetlbfs ->mmap method fails, 1167 * do_mmap_pgoff() nullifies vma->vm_file 1168 * before calling this function to clean up. 1169 * Since no pte has actually been setup, it is 1170 * safe to do nothing in this case. 1171 */ 1172 if (vma->vm_file) { 1173 unmap_hugepage_range(vma, start, end, NULL); 1174 zap_work -= (end - start) / 1175 pages_per_huge_page(hstate_vma(vma)); 1176 } 1177 1178 start = end; 1179 } else 1180 start = unmap_page_range(*tlbp, vma, 1181 start, end, &zap_work, details); 1182 1183 if (zap_work > 0) { 1184 BUG_ON(start != end); 1185 break; 1186 } 1187 1188 tlb_finish_mmu(*tlbp, tlb_start, start); 1189 1190 if (need_resched() || 1191 (i_mmap_lock && spin_needbreak(i_mmap_lock))) { 1192 if (i_mmap_lock) { 1193 *tlbp = NULL; 1194 goto out; 1195 } 1196 cond_resched(); 1197 } 1198 1199 *tlbp = tlb_gather_mmu(vma->vm_mm, fullmm); 1200 tlb_start_valid = 0; 1201 zap_work = ZAP_BLOCK_SIZE; 1202 } 1203 } 1204out: 1205 mmu_notifier_invalidate_range_end(mm, start_addr, end_addr); 1206 return start; /* which is now the end (or restart) address */ 1207} 1208 1209/** 1210 * zap_page_range - remove user pages in a given range 1211 * @vma: vm_area_struct holding the applicable pages 1212 * @address: starting address of pages to zap 1213 * @size: number of bytes to zap 1214 * @details: details of nonlinear truncation or shared cache invalidation 1215 */ 1216unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address, 1217 unsigned long size, struct zap_details *details) 1218{ 1219 struct mm_struct *mm = vma->vm_mm; 1220 struct mmu_gather *tlb; 1221 unsigned long end = address + size; 1222 unsigned long nr_accounted = 0; 1223 1224 lru_add_drain(); 1225 tlb = tlb_gather_mmu(mm, 0); 1226 update_hiwater_rss(mm); 1227 end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details); 1228 if (tlb) 1229 tlb_finish_mmu(tlb, address, end); 1230 return end; 1231} 1232 1233/** 1234 * zap_vma_ptes - remove ptes mapping the vma 1235 * @vma: vm_area_struct holding ptes to be zapped 1236 * @address: starting address of pages to zap 1237 * @size: number of bytes to zap 1238 * 1239 * This function only unmaps ptes assigned to VM_PFNMAP vmas. 1240 * 1241 * The entire address range must be fully contained within the vma. 1242 * 1243 * Returns 0 if successful. 1244 */ 1245int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 1246 unsigned long size) 1247{ 1248 if (address < vma->vm_start || address + size > vma->vm_end || 1249 !(vma->vm_flags & VM_PFNMAP)) 1250 return -1; 1251 zap_page_range(vma, address, size, NULL); 1252 return 0; 1253} 1254EXPORT_SYMBOL_GPL(zap_vma_ptes); 1255 1256/** 1257 * follow_page - look up a page descriptor from a user-virtual address 1258 * @vma: vm_area_struct mapping @address 1259 * @address: virtual address to look up 1260 * @flags: flags modifying lookup behaviour 1261 * 1262 * @flags can have FOLL_ flags set, defined in <linux/mm.h> 1263 * 1264 * Returns the mapped (struct page *), %NULL if no mapping exists, or 1265 * an error pointer if there is a mapping to something not represented 1266 * by a page descriptor (see also vm_normal_page()). 1267 */ 1268struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 1269 unsigned int flags) 1270{ 1271 pgd_t *pgd; 1272 pud_t *pud; 1273 pmd_t *pmd; 1274 pte_t *ptep, pte; 1275 spinlock_t *ptl; 1276 struct page *page; 1277 struct mm_struct *mm = vma->vm_mm; 1278 1279 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); 1280 if (!IS_ERR(page)) { 1281 BUG_ON(flags & FOLL_GET); 1282 goto out; 1283 } 1284 1285 page = NULL; 1286 pgd = pgd_offset(mm, address); 1287 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 1288 goto no_page_table; 1289 1290 pud = pud_offset(pgd, address); 1291 if (pud_none(*pud)) 1292 goto no_page_table; 1293 if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) { 1294 BUG_ON(flags & FOLL_GET); 1295 page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE); 1296 goto out; 1297 } 1298 if (unlikely(pud_bad(*pud))) 1299 goto no_page_table; 1300 1301 pmd = pmd_offset(pud, address); 1302 if (pmd_none(*pmd)) 1303 goto no_page_table; 1304 if (pmd_huge(*pmd) && vma->vm_flags & VM_HUGETLB) { 1305 BUG_ON(flags & FOLL_GET); 1306 page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE); 1307 goto out; 1308 } 1309 if (pmd_trans_huge(*pmd)) { 1310 if (flags & FOLL_SPLIT) { 1311 split_huge_page_pmd(mm, pmd); 1312 goto split_fallthrough; 1313 } 1314 spin_lock(&mm->page_table_lock); 1315 if (likely(pmd_trans_huge(*pmd))) { 1316 if (unlikely(pmd_trans_splitting(*pmd))) { 1317 spin_unlock(&mm->page_table_lock); 1318 wait_split_huge_page(vma->anon_vma, pmd); 1319 } else { 1320 page = follow_trans_huge_pmd(mm, address, 1321 pmd, flags); 1322 spin_unlock(&mm->page_table_lock); 1323 goto out; 1324 } 1325 } else 1326 spin_unlock(&mm->page_table_lock); 1327 /* fall through */ 1328 } 1329split_fallthrough: 1330 if (unlikely(pmd_bad(*pmd))) 1331 goto no_page_table; 1332 1333 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 1334 1335 pte = *ptep; 1336 if (!pte_present(pte)) 1337 goto no_page; 1338 if ((flags & FOLL_WRITE) && !pte_write(pte)) 1339 goto unlock; 1340 1341 page = vm_normal_page(vma, address, pte); 1342 if (unlikely(!page)) { 1343 if ((flags & FOLL_DUMP) || 1344 !is_zero_pfn(pte_pfn(pte))) 1345 goto bad_page; 1346 page = pte_page(pte); 1347 } 1348 1349 if (flags & FOLL_GET) 1350 get_page(page); 1351 if (flags & FOLL_TOUCH) { 1352 if ((flags & FOLL_WRITE) && 1353 !pte_dirty(pte) && !PageDirty(page)) 1354 set_page_dirty(page); 1355 /* 1356 * pte_mkyoung() would be more correct here, but atomic care 1357 * is needed to avoid losing the dirty bit: it is easier to use 1358 * mark_page_accessed(). 1359 */ 1360 mark_page_accessed(page); 1361 } 1362 if (flags & FOLL_MLOCK) { 1363 /* 1364 * The preliminary mapping check is mainly to avoid the 1365 * pointless overhead of lock_page on the ZERO_PAGE 1366 * which might bounce very badly if there is contention. 1367 * 1368 * If the page is already locked, we don't need to 1369 * handle it now - vmscan will handle it later if and 1370 * when it attempts to reclaim the page. 1371 */ 1372 if (page->mapping && trylock_page(page)) { 1373 lru_add_drain(); /* push cached pages to LRU */ 1374 /* 1375 * Because we lock page here and migration is 1376 * blocked by the pte's page reference, we need 1377 * only check for file-cache page truncation. 1378 */ 1379 if (page->mapping) 1380 mlock_vma_page(page); 1381 unlock_page(page); 1382 } 1383 } 1384unlock: 1385 pte_unmap_unlock(ptep, ptl); 1386out: 1387 return page; 1388 1389bad_page: 1390 pte_unmap_unlock(ptep, ptl); 1391 return ERR_PTR(-EFAULT); 1392 1393no_page: 1394 pte_unmap_unlock(ptep, ptl); 1395 if (!pte_none(pte)) 1396 return page; 1397 1398no_page_table: 1399 /* 1400 * When core dumping an enormous anonymous area that nobody 1401 * has touched so far, we don't want to allocate unnecessary pages or 1402 * page tables. Return error instead of NULL to skip handle_mm_fault, 1403 * then get_dump_page() will return NULL to leave a hole in the dump. 1404 * But we can only make this optimization where a hole would surely 1405 * be zero-filled if handle_mm_fault() actually did handle it. 1406 */ 1407 if ((flags & FOLL_DUMP) && 1408 (!vma->vm_ops || !vma->vm_ops->fault)) 1409 return ERR_PTR(-EFAULT); 1410 return page; 1411} 1412 1413/** 1414 * __get_user_pages() - pin user pages in memory 1415 * @tsk: task_struct of target task 1416 * @mm: mm_struct of target mm 1417 * @start: starting user address 1418 * @nr_pages: number of pages from start to pin 1419 * @gup_flags: flags modifying pin behaviour 1420 * @pages: array that receives pointers to the pages pinned. 1421 * Should be at least nr_pages long. Or NULL, if caller 1422 * only intends to ensure the pages are faulted in. 1423 * @vmas: array of pointers to vmas corresponding to each page. 1424 * Or NULL if the caller does not require them. 1425 * @nonblocking: whether waiting for disk IO or mmap_sem contention 1426 * 1427 * Returns number of pages pinned. This may be fewer than the number 1428 * requested. If nr_pages is 0 or negative, returns 0. If no pages 1429 * were pinned, returns -errno. Each page returned must be released 1430 * with a put_page() call when it is finished with. vmas will only 1431 * remain valid while mmap_sem is held. 1432 * 1433 * Must be called with mmap_sem held for read or write. 1434 * 1435 * __get_user_pages walks a process's page tables and takes a reference to 1436 * each struct page that each user address corresponds to at a given 1437 * instant. That is, it takes the page that would be accessed if a user 1438 * thread accesses the given user virtual address at that instant. 1439 * 1440 * This does not guarantee that the page exists in the user mappings when 1441 * __get_user_pages returns, and there may even be a completely different 1442 * page there in some cases (eg. if mmapped pagecache has been invalidated 1443 * and subsequently re faulted). However it does guarantee that the page 1444 * won't be freed completely. And mostly callers simply care that the page 1445 * contains data that was valid *at some point in time*. Typically, an IO 1446 * or similar operation cannot guarantee anything stronger anyway because 1447 * locks can't be held over the syscall boundary. 1448 * 1449 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If 1450 * the page is written to, set_page_dirty (or set_page_dirty_lock, as 1451 * appropriate) must be called after the page is finished with, and 1452 * before put_page is called. 1453 * 1454 * If @nonblocking != NULL, __get_user_pages will not wait for disk IO 1455 * or mmap_sem contention, and if waiting is needed to pin all pages, 1456 * *@nonblocking will be set to 0. 1457 * 1458 * In most cases, get_user_pages or get_user_pages_fast should be used 1459 * instead of __get_user_pages. __get_user_pages should be used only if 1460 * you need some special @gup_flags. 1461 */ 1462int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1463 unsigned long start, int nr_pages, unsigned int gup_flags, 1464 struct page **pages, struct vm_area_struct **vmas, 1465 int *nonblocking) 1466{ 1467 int i; 1468 unsigned long vm_flags; 1469 1470 if (nr_pages <= 0) 1471 return 0; 1472 1473 VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET)); 1474 1475 /* 1476 * Require read or write permissions. 1477 * If FOLL_FORCE is set, we only require the "MAY" flags. 1478 */ 1479 vm_flags = (gup_flags & FOLL_WRITE) ? 1480 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 1481 vm_flags &= (gup_flags & FOLL_FORCE) ? 1482 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 1483 i = 0; 1484 1485 do { 1486 struct vm_area_struct *vma; 1487 1488 vma = find_extend_vma(mm, start); 1489 if (!vma && in_gate_area(tsk->mm, start)) { 1490 unsigned long pg = start & PAGE_MASK; 1491 struct vm_area_struct *gate_vma = get_gate_vma(tsk->mm); 1492 pgd_t *pgd; 1493 pud_t *pud; 1494 pmd_t *pmd; 1495 pte_t *pte; 1496 1497 /* user gate pages are read-only */ 1498 if (gup_flags & FOLL_WRITE) 1499 return i ? : -EFAULT; 1500 if (pg > TASK_SIZE) 1501 pgd = pgd_offset_k(pg); 1502 else 1503 pgd = pgd_offset_gate(mm, pg); 1504 BUG_ON(pgd_none(*pgd)); 1505 pud = pud_offset(pgd, pg); 1506 BUG_ON(pud_none(*pud)); 1507 pmd = pmd_offset(pud, pg); 1508 if (pmd_none(*pmd)) 1509 return i ? : -EFAULT; 1510 VM_BUG_ON(pmd_trans_huge(*pmd)); 1511 pte = pte_offset_map(pmd, pg); 1512 if (pte_none(*pte)) { 1513 pte_unmap(pte); 1514 return i ? : -EFAULT; 1515 } 1516 if (pages) { 1517 struct page *page; 1518 1519 page = vm_normal_page(gate_vma, start, *pte); 1520 if (!page) { 1521 if (!(gup_flags & FOLL_DUMP) && 1522 is_zero_pfn(pte_pfn(*pte))) 1523 page = pte_page(*pte); 1524 else { 1525 pte_unmap(pte); 1526 return i ? : -EFAULT; 1527 } 1528 } 1529 pages[i] = page; 1530 get_page(page); 1531 } 1532 pte_unmap(pte); 1533 if (vmas) 1534 vmas[i] = gate_vma; 1535 i++; 1536 start += PAGE_SIZE; 1537 nr_pages--; 1538 continue; 1539 } 1540 1541 if (!vma || 1542 (vma->vm_flags & (VM_IO | VM_PFNMAP)) || 1543 !(vm_flags & vma->vm_flags)) 1544 return i ? : -EFAULT; 1545 1546 if (is_vm_hugetlb_page(vma)) { 1547 i = follow_hugetlb_page(mm, vma, pages, vmas, 1548 &start, &nr_pages, i, gup_flags); 1549 continue; 1550 } 1551 1552 do { 1553 struct page *page; 1554 unsigned int foll_flags = gup_flags; 1555 1556 /* 1557 * If we have a pending SIGKILL, don't keep faulting 1558 * pages and potentially allocating memory. 1559 */ 1560 if (unlikely(fatal_signal_pending(current))) 1561 return i ? i : -ERESTARTSYS; 1562 1563 cond_resched(); 1564 while (!(page = follow_page(vma, start, foll_flags))) { 1565 int ret; 1566 unsigned int fault_flags = 0; 1567 1568 if (foll_flags & FOLL_WRITE) 1569 fault_flags |= FAULT_FLAG_WRITE; 1570 if (nonblocking) 1571 fault_flags |= FAULT_FLAG_ALLOW_RETRY; 1572 1573 ret = handle_mm_fault(mm, vma, start, 1574 fault_flags); 1575 1576 if (ret & VM_FAULT_ERROR) { 1577 if (ret & VM_FAULT_OOM) 1578 return i ? i : -ENOMEM; 1579 if (ret & (VM_FAULT_HWPOISON | 1580 VM_FAULT_HWPOISON_LARGE)) { 1581 if (i) 1582 return i; 1583 else if (gup_flags & FOLL_HWPOISON) 1584 return -EHWPOISON; 1585 else 1586 return -EFAULT; 1587 } 1588 if (ret & VM_FAULT_SIGBUS) 1589 return i ? i : -EFAULT; 1590 BUG(); 1591 } 1592 if (ret & VM_FAULT_MAJOR) 1593 tsk->maj_flt++; 1594 else 1595 tsk->min_flt++; 1596 1597 if (ret & VM_FAULT_RETRY) { 1598 *nonblocking = 0; 1599 return i; 1600 } 1601 1602 /* 1603 * The VM_FAULT_WRITE bit tells us that 1604 * do_wp_page has broken COW when necessary, 1605 * even if maybe_mkwrite decided not to set 1606 * pte_write. We can thus safely do subsequent 1607 * page lookups as if they were reads. But only 1608 * do so when looping for pte_write is futile: 1609 * in some cases userspace may also be wanting 1610 * to write to the gotten user page, which a 1611 * read fault here might prevent (a readonly 1612 * page might get reCOWed by userspace write). 1613 */ 1614 if ((ret & VM_FAULT_WRITE) && 1615 !(vma->vm_flags & VM_WRITE)) 1616 foll_flags &= ~FOLL_WRITE; 1617 1618 cond_resched(); 1619 } 1620 if (IS_ERR(page)) 1621 return i ? i : PTR_ERR(page); 1622 if (pages) { 1623 pages[i] = page; 1624 1625 flush_anon_page(vma, page, start); 1626 flush_dcache_page(page); 1627 } 1628 if (vmas) 1629 vmas[i] = vma; 1630 i++; 1631 start += PAGE_SIZE; 1632 nr_pages--; 1633 } while (nr_pages && start < vma->vm_end); 1634 } while (nr_pages); 1635 return i; 1636} 1637EXPORT_SYMBOL(__get_user_pages); 1638 1639/** 1640 * get_user_pages() - pin user pages in memory 1641 * @tsk: task_struct of target task 1642 * @mm: mm_struct of target mm 1643 * @start: starting user address 1644 * @nr_pages: number of pages from start to pin 1645 * @write: whether pages will be written to by the caller 1646 * @force: whether to force write access even if user mapping is 1647 * readonly. This will result in the page being COWed even 1648 * in MAP_SHARED mappings. You do not want this. 1649 * @pages: array that receives pointers to the pages pinned. 1650 * Should be at least nr_pages long. Or NULL, if caller 1651 * only intends to ensure the pages are faulted in. 1652 * @vmas: array of pointers to vmas corresponding to each page. 1653 * Or NULL if the caller does not require them. 1654 * 1655 * Returns number of pages pinned. This may be fewer than the number 1656 * requested. If nr_pages is 0 or negative, returns 0. If no pages 1657 * were pinned, returns -errno. Each page returned must be released 1658 * with a put_page() call when it is finished with. vmas will only 1659 * remain valid while mmap_sem is held. 1660 * 1661 * Must be called with mmap_sem held for read or write. 1662 * 1663 * get_user_pages walks a process's page tables and takes a reference to 1664 * each struct page that each user address corresponds to at a given 1665 * instant. That is, it takes the page that would be accessed if a user 1666 * thread accesses the given user virtual address at that instant. 1667 * 1668 * This does not guarantee that the page exists in the user mappings when 1669 * get_user_pages returns, and there may even be a completely different 1670 * page there in some cases (eg. if mmapped pagecache has been invalidated 1671 * and subsequently re faulted). However it does guarantee that the page 1672 * won't be freed completely. And mostly callers simply care that the page 1673 * contains data that was valid *at some point in time*. Typically, an IO 1674 * or similar operation cannot guarantee anything stronger anyway because 1675 * locks can't be held over the syscall boundary. 1676 * 1677 * If write=0, the page must not be written to. If the page is written to, 1678 * set_page_dirty (or set_page_dirty_lock, as appropriate) must be called 1679 * after the page is finished with, and before put_page is called. 1680 * 1681 * get_user_pages is typically used for fewer-copy IO operations, to get a 1682 * handle on the memory by some means other than accesses via the user virtual 1683 * addresses. The pages may be submitted for DMA to devices or accessed via 1684 * their kernel linear mapping (via the kmap APIs). Care should be taken to 1685 * use the correct cache flushing APIs. 1686 * 1687 * See also get_user_pages_fast, for performance critical applications. 1688 */ 1689int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1690 unsigned long start, int nr_pages, int write, int force, 1691 struct page **pages, struct vm_area_struct **vmas) 1692{ 1693 int flags = FOLL_TOUCH; 1694 1695 if (pages) 1696 flags |= FOLL_GET; 1697 if (write) 1698 flags |= FOLL_WRITE; 1699 if (force) 1700 flags |= FOLL_FORCE; 1701 1702 return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas, 1703 NULL); 1704} 1705EXPORT_SYMBOL(get_user_pages); 1706 1707/** 1708 * get_dump_page() - pin user page in memory while writing it to core dump 1709 * @addr: user address 1710 * 1711 * Returns struct page pointer of user page pinned for dump, 1712 * to be freed afterwards by page_cache_release() or put_page(). 1713 * 1714 * Returns NULL on any kind of failure - a hole must then be inserted into 1715 * the corefile, to preserve alignment with its headers; and also returns 1716 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - 1717 * allowing a hole to be left in the corefile to save diskspace. 1718 * 1719 * Called without mmap_sem, but after all other threads have been killed. 1720 */ 1721#ifdef CONFIG_ELF_CORE 1722struct page *get_dump_page(unsigned long addr) 1723{ 1724 struct vm_area_struct *vma; 1725 struct page *page; 1726 1727 if (__get_user_pages(current, current->mm, addr, 1, 1728 FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, 1729 NULL) < 1) 1730 return NULL; 1731 flush_cache_page(vma, addr, page_to_pfn(page)); 1732 return page; 1733} 1734#endif /* CONFIG_ELF_CORE */ 1735 1736pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, 1737 spinlock_t **ptl) 1738{ 1739 pgd_t * pgd = pgd_offset(mm, addr); 1740 pud_t * pud = pud_alloc(mm, pgd, addr); 1741 if (pud) { 1742 pmd_t * pmd = pmd_alloc(mm, pud, addr); 1743 if (pmd) { 1744 VM_BUG_ON(pmd_trans_huge(*pmd)); 1745 return pte_alloc_map_lock(mm, pmd, addr, ptl); 1746 } 1747 } 1748 return NULL; 1749} 1750 1751/* 1752 * This is the old fallback for page remapping. 1753 * 1754 * For historical reasons, it only allows reserved pages. Only 1755 * old drivers should use this, and they needed to mark their 1756 * pages reserved for the old functions anyway. 1757 */ 1758static int insert_page(struct vm_area_struct *vma, unsigned long addr, 1759 struct page *page, pgprot_t prot) 1760{ 1761 struct mm_struct *mm = vma->vm_mm; 1762 int retval; 1763 pte_t *pte; 1764 spinlock_t *ptl; 1765 1766 retval = -EINVAL; 1767 if (PageAnon(page)) 1768 goto out; 1769 retval = -ENOMEM; 1770 flush_dcache_page(page); 1771 pte = get_locked_pte(mm, addr, &ptl); 1772 if (!pte) 1773 goto out; 1774 retval = -EBUSY; 1775 if (!pte_none(*pte)) 1776 goto out_unlock; 1777 1778 /* Ok, finally just insert the thing.. */ 1779 get_page(page); 1780 inc_mm_counter_fast(mm, MM_FILEPAGES); 1781 page_add_file_rmap(page); 1782 set_pte_at(mm, addr, pte, mk_pte(page, prot)); 1783 1784 retval = 0; 1785 pte_unmap_unlock(pte, ptl); 1786 return retval; 1787out_unlock: 1788 pte_unmap_unlock(pte, ptl); 1789out: 1790 return retval; 1791} 1792 1793/** 1794 * vm_insert_page - insert single page into user vma 1795 * @vma: user vma to map to 1796 * @addr: target user address of this page 1797 * @page: source kernel page 1798 * 1799 * This allows drivers to insert individual pages they've allocated 1800 * into a user vma. 1801 * 1802 * The page has to be a nice clean _individual_ kernel allocation. 1803 * If you allocate a compound page, you need to have marked it as 1804 * such (__GFP_COMP), or manually just split the page up yourself 1805 * (see split_page()). 1806 * 1807 * NOTE! Traditionally this was done with "remap_pfn_range()" which 1808 * took an arbitrary page protection parameter. This doesn't allow 1809 * that. Your vma protection will have to be set up correctly, which 1810 * means that if you want a shared writable mapping, you'd better 1811 * ask for a shared writable mapping! 1812 * 1813 * The page does not need to be reserved. 1814 */ 1815int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 1816 struct page *page) 1817{ 1818 if (addr < vma->vm_start || addr >= vma->vm_end) 1819 return -EFAULT; 1820 if (!page_count(page)) 1821 return -EINVAL; 1822 vma->vm_flags |= VM_INSERTPAGE; 1823 return insert_page(vma, addr, page, vma->vm_page_prot); 1824} 1825EXPORT_SYMBOL(vm_insert_page); 1826 1827static int insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1828 unsigned long pfn, pgprot_t prot) 1829{ 1830 struct mm_struct *mm = vma->vm_mm; 1831 int retval; 1832 pte_t *pte, entry; 1833 spinlock_t *ptl; 1834 1835 retval = -ENOMEM; 1836 pte = get_locked_pte(mm, addr, &ptl); 1837 if (!pte) 1838 goto out; 1839 retval = -EBUSY; 1840 if (!pte_none(*pte)) 1841 goto out_unlock; 1842 1843 /* Ok, finally just insert the thing.. */ 1844 entry = pte_mkspecial(pfn_pte(pfn, prot)); 1845 set_pte_at(mm, addr, pte, entry); 1846 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */ 1847 1848 retval = 0; 1849out_unlock: 1850 pte_unmap_unlock(pte, ptl); 1851out: 1852 return retval; 1853} 1854 1855/** 1856 * vm_insert_pfn - insert single pfn into user vma 1857 * @vma: user vma to map to 1858 * @addr: target user address of this page 1859 * @pfn: source kernel pfn 1860 * 1861 * Similar to vm_inert_page, this allows drivers to insert individual pages 1862 * they've allocated into a user vma. Same comments apply. 1863 * 1864 * This function should only be called from a vm_ops->fault handler, and 1865 * in that case the handler should return NULL. 1866 * 1867 * vma cannot be a COW mapping. 1868 * 1869 * As this is called only for pages that do not currently exist, we 1870 * do not need to flush old virtual caches or the TLB. 1871 */ 1872int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1873 unsigned long pfn) 1874{ 1875 int ret; 1876 pgprot_t pgprot = vma->vm_page_prot; 1877 /* 1878 * Technically, architectures with pte_special can avoid all these 1879 * restrictions (same for remap_pfn_range). However we would like 1880 * consistency in testing and feature parity among all, so we should 1881 * try to keep these invariants in place for everybody. 1882 */ 1883 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1884 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1885 (VM_PFNMAP|VM_MIXEDMAP)); 1886 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1887 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); 1888 1889 if (addr < vma->vm_start || addr >= vma->vm_end) 1890 return -EFAULT; 1891 if (track_pfn_vma_new(vma, &pgprot, pfn, PAGE_SIZE)) 1892 return -EINVAL; 1893 1894 ret = insert_pfn(vma, addr, pfn, pgprot); 1895 1896 if (ret) 1897 untrack_pfn_vma(vma, pfn, PAGE_SIZE); 1898 1899 return ret; 1900} 1901EXPORT_SYMBOL(vm_insert_pfn); 1902 1903int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 1904 unsigned long pfn) 1905{ 1906 BUG_ON(!(vma->vm_flags & VM_MIXEDMAP)); 1907 1908 if (addr < vma->vm_start || addr >= vma->vm_end) 1909 return -EFAULT; 1910 1911 /* 1912 * If we don't have pte special, then we have to use the pfn_valid() 1913 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must* 1914 * refcount the page if pfn_valid is true (hence insert_page rather 1915 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP 1916 * without pte special, it would there be refcounted as a normal page. 1917 */ 1918 if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) { 1919 struct page *page; 1920 1921 page = pfn_to_page(pfn); 1922 return insert_page(vma, addr, page, vma->vm_page_prot); 1923 } 1924 return insert_pfn(vma, addr, pfn, vma->vm_page_prot); 1925} 1926EXPORT_SYMBOL(vm_insert_mixed); 1927 1928/* 1929 * maps a range of physical memory into the requested pages. the old 1930 * mappings are removed. any references to nonexistent pages results 1931 * in null mappings (currently treated as "copy-on-access") 1932 */ 1933static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, 1934 unsigned long addr, unsigned long end, 1935 unsigned long pfn, pgprot_t prot) 1936{ 1937 pte_t *pte; 1938 spinlock_t *ptl; 1939 1940 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); 1941 if (!pte) 1942 return -ENOMEM; 1943 arch_enter_lazy_mmu_mode(); 1944 do { 1945 BUG_ON(!pte_none(*pte)); 1946 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); 1947 pfn++; 1948 } while (pte++, addr += PAGE_SIZE, addr != end); 1949 arch_leave_lazy_mmu_mode(); 1950 pte_unmap_unlock(pte - 1, ptl); 1951 return 0; 1952} 1953 1954static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, 1955 unsigned long addr, unsigned long end, 1956 unsigned long pfn, pgprot_t prot) 1957{ 1958 pmd_t *pmd; 1959 unsigned long next; 1960 1961 pfn -= addr >> PAGE_SHIFT; 1962 pmd = pmd_alloc(mm, pud, addr); 1963 if (!pmd) 1964 return -ENOMEM; 1965 VM_BUG_ON(pmd_trans_huge(*pmd)); 1966 do { 1967 next = pmd_addr_end(addr, end); 1968 if (remap_pte_range(mm, pmd, addr, next, 1969 pfn + (addr >> PAGE_SHIFT), prot)) 1970 return -ENOMEM; 1971 } while (pmd++, addr = next, addr != end); 1972 return 0; 1973} 1974 1975static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd, 1976 unsigned long addr, unsigned long end, 1977 unsigned long pfn, pgprot_t prot) 1978{ 1979 pud_t *pud; 1980 unsigned long next; 1981 1982 pfn -= addr >> PAGE_SHIFT; 1983 pud = pud_alloc(mm, pgd, addr); 1984 if (!pud) 1985 return -ENOMEM; 1986 do { 1987 next = pud_addr_end(addr, end); 1988 if (remap_pmd_range(mm, pud, addr, next, 1989 pfn + (addr >> PAGE_SHIFT), prot)) 1990 return -ENOMEM; 1991 } while (pud++, addr = next, addr != end); 1992 return 0; 1993} 1994 1995/** 1996 * remap_pfn_range - remap kernel memory to userspace 1997 * @vma: user vma to map to 1998 * @addr: target user address to start at 1999 * @pfn: physical address of kernel memory 2000 * @size: size of map area 2001 * @prot: page protection flags for this mapping 2002 * 2003 * Note: this is only safe if the mm semaphore is held when called. 2004 */ 2005int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 2006 unsigned long pfn, unsigned long size, pgprot_t prot) 2007{ 2008 pgd_t *pgd; 2009 unsigned long next; 2010 unsigned long end = addr + PAGE_ALIGN(size); 2011 struct mm_struct *mm = vma->vm_mm; 2012 int err; 2013 2014 /* 2015 * Physically remapped pages are special. Tell the 2016 * rest of the world about it: 2017 * VM_IO tells people not to look at these pages 2018 * (accesses can have side effects). 2019 * VM_RESERVED is specified all over the place, because 2020 * in 2.4 it kept swapout's vma scan off this vma; but 2021 * in 2.6 the LRU scan won't even find its pages, so this 2022 * flag means no more than count its pages in reserved_vm, 2023 * and omit it from core dump, even when VM_IO turned off. 2024 * VM_PFNMAP tells the core MM that the base pages are just 2025 * raw PFN mappings, and do not have a "struct page" associated 2026 * with them. 2027 * 2028 * There's a horrible special case to handle copy-on-write 2029 * behaviour that some programs depend on. We mark the "original" 2030 * un-COW'ed pages by matching them up with "vma->vm_pgoff". 2031 */ 2032 if (addr == vma->vm_start && end == vma->vm_end) { 2033 vma->vm_pgoff = pfn; 2034 vma->vm_flags |= VM_PFN_AT_MMAP; 2035 } else if (is_cow_mapping(vma->vm_flags)) 2036 return -EINVAL; 2037 2038 vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; 2039 2040 err = track_pfn_vma_new(vma, &prot, pfn, PAGE_ALIGN(size)); 2041 if (err) { 2042 /* 2043 * To indicate that track_pfn related cleanup is not 2044 * needed from higher level routine calling unmap_vmas 2045 */ 2046 vma->vm_flags &= ~(VM_IO | VM_RESERVED | VM_PFNMAP); 2047 vma->vm_flags &= ~VM_PFN_AT_MMAP; 2048 return -EINVAL; 2049 } 2050 2051 BUG_ON(addr >= end); 2052 pfn -= addr >> PAGE_SHIFT; 2053 pgd = pgd_offset(mm, addr); 2054 flush_cache_range(vma, addr, end); 2055 do { 2056 next = pgd_addr_end(addr, end); 2057 err = remap_pud_range(mm, pgd, addr, next, 2058 pfn + (addr >> PAGE_SHIFT), prot); 2059 if (err) 2060 break; 2061 } while (pgd++, addr = next, addr != end); 2062 2063 if (err) 2064 untrack_pfn_vma(vma, pfn, PAGE_ALIGN(size)); 2065 2066 return err; 2067} 2068EXPORT_SYMBOL(remap_pfn_range); 2069 2070static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, 2071 unsigned long addr, unsigned long end, 2072 pte_fn_t fn, void *data) 2073{ 2074 pte_t *pte; 2075 int err; 2076 pgtable_t token; 2077 spinlock_t *uninitialized_var(ptl); 2078 2079 pte = (mm == &init_mm) ? 2080 pte_alloc_kernel(pmd, addr) : 2081 pte_alloc_map_lock(mm, pmd, addr, &ptl); 2082 if (!pte) 2083 return -ENOMEM; 2084 2085 BUG_ON(pmd_huge(*pmd)); 2086 2087 arch_enter_lazy_mmu_mode(); 2088 2089 token = pmd_pgtable(*pmd); 2090 2091 do { 2092 err = fn(pte++, token, addr, data); 2093 if (err) 2094 break; 2095 } while (addr += PAGE_SIZE, addr != end); 2096 2097 arch_leave_lazy_mmu_mode(); 2098 2099 if (mm != &init_mm) 2100 pte_unmap_unlock(pte-1, ptl); 2101 return err; 2102} 2103 2104static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, 2105 unsigned long addr, unsigned long end, 2106 pte_fn_t fn, void *data) 2107{ 2108 pmd_t *pmd; 2109 unsigned long next; 2110 int err; 2111 2112 BUG_ON(pud_huge(*pud)); 2113 2114 pmd = pmd_alloc(mm, pud, addr); 2115 if (!pmd) 2116 return -ENOMEM; 2117 do { 2118 next = pmd_addr_end(addr, end); 2119 err = apply_to_pte_range(mm, pmd, addr, next, fn, data); 2120 if (err) 2121 break; 2122 } while (pmd++, addr = next, addr != end); 2123 return err; 2124} 2125 2126static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd, 2127 unsigned long addr, unsigned long end, 2128 pte_fn_t fn, void *data) 2129{ 2130 pud_t *pud; 2131 unsigned long next; 2132 int err; 2133 2134 pud = pud_alloc(mm, pgd, addr); 2135 if (!pud) 2136 return -ENOMEM; 2137 do { 2138 next = pud_addr_end(addr, end); 2139 err = apply_to_pmd_range(mm, pud, addr, next, fn, data); 2140 if (err) 2141 break; 2142 } while (pud++, addr = next, addr != end); 2143 return err; 2144} 2145 2146/* 2147 * Scan a region of virtual memory, filling in page tables as necessary 2148 * and calling a provided function on each leaf page table. 2149 */ 2150int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 2151 unsigned long size, pte_fn_t fn, void *data) 2152{ 2153 pgd_t *pgd; 2154 unsigned long next; 2155 unsigned long end = addr + size; 2156 int err; 2157 2158 BUG_ON(addr >= end); 2159 pgd = pgd_offset(mm, addr); 2160 do { 2161 next = pgd_addr_end(addr, end); 2162 err = apply_to_pud_range(mm, pgd, addr, next, fn, data); 2163 if (err) 2164 break; 2165 } while (pgd++, addr = next, addr != end); 2166 2167 return err; 2168} 2169EXPORT_SYMBOL_GPL(apply_to_page_range); 2170 2171/* 2172 * handle_pte_fault chooses page fault handler according to an entry 2173 * which was read non-atomically. Before making any commitment, on 2174 * those architectures or configurations (e.g. i386 with PAE) which 2175 * might give a mix of unmatched parts, do_swap_page and do_nonlinear_fault 2176 * must check under lock before unmapping the pte and proceeding 2177 * (but do_wp_page is only called after already making such a check; 2178 * and do_anonymous_page can safely check later on). 2179 */ 2180static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd, 2181 pte_t *page_table, pte_t orig_pte) 2182{ 2183 int same = 1; 2184#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) 2185 if (sizeof(pte_t) > sizeof(unsigned long)) { 2186 spinlock_t *ptl = pte_lockptr(mm, pmd); 2187 spin_lock(ptl); 2188 same = pte_same(*page_table, orig_pte); 2189 spin_unlock(ptl); 2190 } 2191#endif 2192 pte_unmap(page_table); 2193 return same; 2194} 2195 2196static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) 2197{ 2198 /* 2199 * If the source page was a PFN mapping, we don't have 2200 * a "struct page" for it. We do a best-effort copy by 2201 * just copying from the original user address. If that 2202 * fails, we just zero-fill it. Live with it. 2203 */ 2204 if (unlikely(!src)) { 2205 void *kaddr = kmap_atomic(dst, KM_USER0); 2206 void __user *uaddr = (void __user *)(va & PAGE_MASK); 2207 2208 /* 2209 * This really shouldn't fail, because the page is there 2210 * in the page tables. But it might just be unreadable, 2211 * in which case we just give up and fill the result with 2212 * zeroes. 2213 */ 2214 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) 2215 clear_page(kaddr); 2216 kunmap_atomic(kaddr, KM_USER0); 2217 flush_dcache_page(dst); 2218 } else 2219 copy_user_highpage(dst, src, va, vma); 2220} 2221 2222/* 2223 * This routine handles present pages, when users try to write 2224 * to a shared page. It is done by copying the page to a new address 2225 * and decrementing the shared-page counter for the old page. 2226 * 2227 * Note that this routine assumes that the protection checks have been 2228 * done by the caller (the low-level page fault routine in most cases). 2229 * Thus we can safely just mark it writable once we've done any necessary 2230 * COW. 2231 * 2232 * We also mark the page dirty at this point even though the page will 2233 * change only once the write actually happens. This avoids a few races, 2234 * and potentially makes it more efficient. 2235 * 2236 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2237 * but allow concurrent faults), with pte both mapped and locked. 2238 * We return with mmap_sem still held, but pte unmapped and unlocked. 2239 */ 2240static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 2241 unsigned long address, pte_t *page_table, pmd_t *pmd, 2242 spinlock_t *ptl, pte_t orig_pte) 2243 __releases(ptl) 2244{ 2245 struct page *old_page, *new_page; 2246 pte_t entry; 2247 int ret = 0; 2248 int page_mkwrite = 0; 2249 struct page *dirty_page = NULL; 2250 2251 old_page = vm_normal_page(vma, address, orig_pte); 2252 if (!old_page) { 2253 /* 2254 * VM_MIXEDMAP !pfn_valid() case 2255 * 2256 * We should not cow pages in a shared writeable mapping. 2257 * Just mark the pages writable as we can't do any dirty 2258 * accounting on raw pfn maps. 2259 */ 2260 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2261 (VM_WRITE|VM_SHARED)) 2262 goto reuse; 2263 goto gotten; 2264 } 2265 2266 /* 2267 * Take out anonymous pages first, anonymous shared vmas are 2268 * not dirty accountable. 2269 */ 2270 if (PageAnon(old_page) && !PageKsm(old_page)) { 2271 if (!trylock_page(old_page)) { 2272 page_cache_get(old_page); 2273 pte_unmap_unlock(page_table, ptl); 2274 lock_page(old_page); 2275 page_table = pte_offset_map_lock(mm, pmd, address, 2276 &ptl); 2277 if (!pte_same(*page_table, orig_pte)) { 2278 unlock_page(old_page); 2279 goto unlock; 2280 } 2281 page_cache_release(old_page); 2282 } 2283 if (reuse_swap_page(old_page)) { 2284 /* 2285 * The page is all ours. Move it to our anon_vma so 2286 * the rmap code will not search our parent or siblings. 2287 * Protected against the rmap code by the page lock. 2288 */ 2289 page_move_anon_rmap(old_page, vma, address); 2290 unlock_page(old_page); 2291 goto reuse; 2292 } 2293 unlock_page(old_page); 2294 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2295 (VM_WRITE|VM_SHARED))) { 2296 /* 2297 * Only catch write-faults on shared writable pages, 2298 * read-only shared pages can get COWed by 2299 * get_user_pages(.write=1, .force=1). 2300 */ 2301 if (vma->vm_ops && vma->vm_ops->page_mkwrite) { 2302 struct vm_fault vmf; 2303 int tmp; 2304 2305 vmf.virtual_address = (void __user *)(address & 2306 PAGE_MASK); 2307 vmf.pgoff = old_page->index; 2308 vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 2309 vmf.page = old_page; 2310 2311 /* 2312 * Notify the address space that the page is about to 2313 * become writable so that it can prohibit this or wait 2314 * for the page to get into an appropriate state. 2315 * 2316 * We do this without the lock held, so that it can 2317 * sleep if it needs to. 2318 */ 2319 page_cache_get(old_page); 2320 pte_unmap_unlock(page_table, ptl); 2321 2322 tmp = vma->vm_ops->page_mkwrite(vma, &vmf); 2323 if (unlikely(tmp & 2324 (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 2325 ret = tmp; 2326 goto unwritable_page; 2327 } 2328 if (unlikely(!(tmp & VM_FAULT_LOCKED))) { 2329 lock_page(old_page); 2330 if (!old_page->mapping) { 2331 ret = 0; /* retry the fault */ 2332 unlock_page(old_page); 2333 goto unwritable_page; 2334 } 2335 } else 2336 VM_BUG_ON(!PageLocked(old_page)); 2337 2338 /* 2339 * Since we dropped the lock we need to revalidate 2340 * the PTE as someone else may have changed it. If 2341 * they did, we just return, as we can count on the 2342 * MMU to tell us if they didn't also make it writable. 2343 */ 2344 page_table = pte_offset_map_lock(mm, pmd, address, 2345 &ptl); 2346 if (!pte_same(*page_table, orig_pte)) { 2347 unlock_page(old_page); 2348 goto unlock; 2349 } 2350 2351 page_mkwrite = 1; 2352 } 2353 dirty_page = old_page; 2354 get_page(dirty_page); 2355 2356reuse: 2357 flush_cache_page(vma, address, pte_pfn(orig_pte)); 2358 entry = pte_mkyoung(orig_pte); 2359 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2360 if (ptep_set_access_flags(vma, address, page_table, entry,1)) 2361 update_mmu_cache(vma, address, page_table); 2362 pte_unmap_unlock(page_table, ptl); 2363 ret |= VM_FAULT_WRITE; 2364 2365 if (!dirty_page) 2366 return ret; 2367 2368 /* 2369 * Yes, Virginia, this is actually required to prevent a race 2370 * with clear_page_dirty_for_io() from clearing the page dirty 2371 * bit after it clear all dirty ptes, but before a racing 2372 * do_wp_page installs a dirty pte. 2373 * 2374 * __do_fault is protected similarly. 2375 */ 2376 if (!page_mkwrite) { 2377 wait_on_page_locked(dirty_page); 2378 set_page_dirty_balance(dirty_page, page_mkwrite); 2379 } 2380 put_page(dirty_page); 2381 if (page_mkwrite) { 2382 struct address_space *mapping = dirty_page->mapping; 2383 2384 set_page_dirty(dirty_page); 2385 unlock_page(dirty_page); 2386 page_cache_release(dirty_page); 2387 if (mapping) { 2388 /* 2389 * Some device drivers do not set page.mapping 2390 * but still dirty their pages 2391 */ 2392 balance_dirty_pages_ratelimited(mapping); 2393 } 2394 } 2395 2396 /* file_update_time outside page_lock */ 2397 if (vma->vm_file) 2398 file_update_time(vma->vm_file); 2399 2400 return ret; 2401 } 2402 2403 /* 2404 * Ok, we need to copy. Oh, well.. 2405 */ 2406 page_cache_get(old_page); 2407gotten: 2408 pte_unmap_unlock(page_table, ptl); 2409 2410 if (unlikely(anon_vma_prepare(vma))) 2411 goto oom; 2412 2413 if (is_zero_pfn(pte_pfn(orig_pte))) { 2414 new_page = alloc_zeroed_user_highpage_movable(vma, address); 2415 if (!new_page) 2416 goto oom; 2417 } else { 2418 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); 2419 if (!new_page) 2420 goto oom; 2421 cow_user_page(new_page, old_page, address, vma); 2422 } 2423 __SetPageUptodate(new_page); 2424 2425 if (mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL)) 2426 goto oom_free_new; 2427 2428 /* 2429 * Re-check the pte - we dropped the lock 2430 */ 2431 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2432 if (likely(pte_same(*page_table, orig_pte))) { 2433 if (old_page) { 2434 if (!PageAnon(old_page)) { 2435 dec_mm_counter_fast(mm, MM_FILEPAGES); 2436 inc_mm_counter_fast(mm, MM_ANONPAGES); 2437 } 2438 } else 2439 inc_mm_counter_fast(mm, MM_ANONPAGES); 2440 flush_cache_page(vma, address, pte_pfn(orig_pte)); 2441 entry = mk_pte(new_page, vma->vm_page_prot); 2442 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2443 /* 2444 * Clear the pte entry and flush it first, before updating the 2445 * pte with the new entry. This will avoid a race condition 2446 * seen in the presence of one thread doing SMC and another 2447 * thread doing COW. 2448 */ 2449 ptep_clear_flush(vma, address, page_table); 2450 page_add_new_anon_rmap(new_page, vma, address); 2451 /* 2452 * We call the notify macro here because, when using secondary 2453 * mmu page tables (such as kvm shadow page tables), we want the 2454 * new page to be mapped directly into the secondary page table. 2455 */ 2456 set_pte_at_notify(mm, address, page_table, entry); 2457 update_mmu_cache(vma, address, page_table); 2458 if (old_page) { 2459 /* 2460 * Only after switching the pte to the new page may 2461 * we remove the mapcount here. Otherwise another 2462 * process may come and find the rmap count decremented 2463 * before the pte is switched to the new page, and 2464 * "reuse" the old page writing into it while our pte 2465 * here still points into it and can be read by other 2466 * threads. 2467 * 2468 * The critical issue is to order this 2469 * page_remove_rmap with the ptp_clear_flush above. 2470 * Those stores are ordered by (if nothing else,) 2471 * the barrier present in the atomic_add_negative 2472 * in page_remove_rmap. 2473 * 2474 * Then the TLB flush in ptep_clear_flush ensures that 2475 * no process can access the old page before the 2476 * decremented mapcount is visible. And the old page 2477 * cannot be reused until after the decremented 2478 * mapcount is visible. So transitively, TLBs to 2479 * old page will be flushed before it can be reused. 2480 */ 2481 page_remove_rmap(old_page); 2482 } 2483 2484 /* Free the old page.. */ 2485 new_page = old_page; 2486 ret |= VM_FAULT_WRITE; 2487 } else 2488 mem_cgroup_uncharge_page(new_page); 2489 2490 if (new_page) 2491 page_cache_release(new_page); 2492unlock: 2493 pte_unmap_unlock(page_table, ptl); 2494 if (old_page) { 2495 /* 2496 * Don't let another task, with possibly unlocked vma, 2497 * keep the mlocked page. 2498 */ 2499 if ((ret & VM_FAULT_WRITE) && (vma->vm_flags & VM_LOCKED)) { 2500 lock_page(old_page); /* LRU manipulation */ 2501 munlock_vma_page(old_page); 2502 unlock_page(old_page); 2503 } 2504 page_cache_release(old_page); 2505 } 2506 return ret; 2507oom_free_new: 2508 page_cache_release(new_page); 2509oom: 2510 if (old_page) { 2511 if (page_mkwrite) { 2512 unlock_page(old_page); 2513 page_cache_release(old_page); 2514 } 2515 page_cache_release(old_page); 2516 } 2517 return VM_FAULT_OOM; 2518 2519unwritable_page: 2520 page_cache_release(old_page); 2521 return ret; 2522} 2523 2524/* 2525 * Helper functions for unmap_mapping_range(). 2526 * 2527 * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __ 2528 * 2529 * We have to restart searching the prio_tree whenever we drop the lock, 2530 * since the iterator is only valid while the lock is held, and anyway 2531 * a later vma might be split and reinserted earlier while lock dropped. 2532 * 2533 * The list of nonlinear vmas could be handled more efficiently, using 2534 * a placeholder, but handle it in the same way until a need is shown. 2535 * It is important to search the prio_tree before nonlinear list: a vma 2536 * may become nonlinear and be shifted from prio_tree to nonlinear list 2537 * while the lock is dropped; but never shifted from list to prio_tree. 2538 * 2539 * In order to make forward progress despite restarting the search, 2540 * vm_truncate_count is used to mark a vma as now dealt with, so we can 2541 * quickly skip it next time around. Since the prio_tree search only 2542 * shows us those vmas affected by unmapping the range in question, we 2543 * can't efficiently keep all vmas in step with mapping->truncate_count: 2544 * so instead reset them all whenever it wraps back to 0 (then go to 1). 2545 * mapping->truncate_count and vma->vm_truncate_count are protected by 2546 * i_mmap_lock. 2547 * 2548 * In order to make forward progress despite repeatedly restarting some 2549 * large vma, note the restart_addr from unmap_vmas when it breaks out: 2550 * and restart from that address when we reach that vma again. It might 2551 * have been split or merged, shrunk or extended, but never shifted: so 2552 * restart_addr remains valid so long as it remains in the vma's range. 2553 * unmap_mapping_range forces truncate_count to leap over page-aligned 2554 * values so we can save vma's restart_addr in its truncate_count field. 2555 */ 2556#define is_restart_addr(truncate_count) (!((truncate_count) & ~PAGE_MASK)) 2557 2558static void reset_vma_truncate_counts(struct address_space *mapping) 2559{ 2560 struct vm_area_struct *vma; 2561 struct prio_tree_iter iter; 2562 2563 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX) 2564 vma->vm_truncate_count = 0; 2565 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list) 2566 vma->vm_truncate_count = 0; 2567} 2568 2569static int unmap_mapping_range_vma(struct vm_area_struct *vma, 2570 unsigned long start_addr, unsigned long end_addr, 2571 struct zap_details *details) 2572{ 2573 unsigned long restart_addr; 2574 int need_break; 2575 2576 /* 2577 * files that support invalidating or truncating portions of the 2578 * file from under mmaped areas must have their ->fault function 2579 * return a locked page (and set VM_FAULT_LOCKED in the return). 2580 * This provides synchronisation against concurrent unmapping here. 2581 */ 2582 2583again: 2584 restart_addr = vma->vm_truncate_count; 2585 if (is_restart_addr(restart_addr) && start_addr < restart_addr) { 2586 start_addr = restart_addr; 2587 if (start_addr >= end_addr) { 2588 /* Top of vma has been split off since last time */ 2589 vma->vm_truncate_count = details->truncate_count; 2590 return 0; 2591 } 2592 } 2593 2594 restart_addr = zap_page_range(vma, start_addr, 2595 end_addr - start_addr, details); 2596 need_break = need_resched() || spin_needbreak(details->i_mmap_lock); 2597 2598 if (restart_addr >= end_addr) { 2599 /* We have now completed this vma: mark it so */ 2600 vma->vm_truncate_count = details->truncate_count; 2601 if (!need_break) 2602 return 0; 2603 } else { 2604 /* Note restart_addr in vma's truncate_count field */ 2605 vma->vm_truncate_count = restart_addr; 2606 if (!need_break) 2607 goto again; 2608 } 2609 2610 spin_unlock(details->i_mmap_lock); 2611 cond_resched(); 2612 spin_lock(details->i_mmap_lock); 2613 return -EINTR; 2614} 2615 2616static inline void unmap_mapping_range_tree(struct prio_tree_root *root, 2617 struct zap_details *details) 2618{ 2619 struct vm_area_struct *vma; 2620 struct prio_tree_iter iter; 2621 pgoff_t vba, vea, zba, zea; 2622 2623restart: 2624 vma_prio_tree_foreach(vma, &iter, root, 2625 details->first_index, details->last_index) { 2626 /* Skip quickly over those we have already dealt with */ 2627 if (vma->vm_truncate_count == details->truncate_count) 2628 continue; 2629 2630 vba = vma->vm_pgoff; 2631 vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1; 2632 /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */ 2633 zba = details->first_index; 2634 if (zba < vba) 2635 zba = vba; 2636 zea = details->last_index; 2637 if (zea > vea) 2638 zea = vea; 2639 2640 if (unmap_mapping_range_vma(vma, 2641 ((zba - vba) << PAGE_SHIFT) + vma->vm_start, 2642 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, 2643 details) < 0) 2644 goto restart; 2645 } 2646} 2647 2648static inline void unmap_mapping_range_list(struct list_head *head, 2649 struct zap_details *details) 2650{ 2651 struct vm_area_struct *vma; 2652 2653 /* 2654 * In nonlinear VMAs there is no correspondence between virtual address 2655 * offset and file offset. So we must perform an exhaustive search 2656 * across *all* the pages in each nonlinear VMA, not just the pages 2657 * whose virtual address lies outside the file truncation point. 2658 */ 2659restart: 2660 list_for_each_entry(vma, head, shared.vm_set.list) { 2661 /* Skip quickly over those we have already dealt with */ 2662 if (vma->vm_truncate_count == details->truncate_count) 2663 continue; 2664 details->nonlinear_vma = vma; 2665 if (unmap_mapping_range_vma(vma, vma->vm_start, 2666 vma->vm_end, details) < 0) 2667 goto restart; 2668 } 2669} 2670 2671/** 2672 * unmap_mapping_range - unmap the portion of all mmaps in the specified address_space corresponding to the specified page range in the underlying file. 2673 * @mapping: the address space containing mmaps to be unmapped. 2674 * @holebegin: byte in first page to unmap, relative to the start of 2675 * the underlying file. This will be rounded down to a PAGE_SIZE 2676 * boundary. Note that this is different from truncate_pagecache(), which 2677 * must keep the partial page. In contrast, we must get rid of 2678 * partial pages. 2679 * @holelen: size of prospective hole in bytes. This will be rounded 2680 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the 2681 * end of the file. 2682 * @even_cows: 1 when truncating a file, unmap even private COWed pages; 2683 * but 0 when invalidating pagecache, don't throw away private data. 2684 */ 2685void unmap_mapping_range(struct address_space *mapping, 2686 loff_t const holebegin, loff_t const holelen, int even_cows) 2687{ 2688 struct zap_details details; 2689 pgoff_t hba = holebegin >> PAGE_SHIFT; 2690 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2691 2692 /* Check for overflow. */ 2693 if (sizeof(holelen) > sizeof(hlen)) { 2694 long long holeend = 2695 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2696 if (holeend & ~(long long)ULONG_MAX) 2697 hlen = ULONG_MAX - hba + 1; 2698 } 2699 2700 details.check_mapping = even_cows? NULL: mapping; 2701 details.nonlinear_vma = NULL; 2702 details.first_index = hba; 2703 details.last_index = hba + hlen - 1; 2704 if (details.last_index < details.first_index) 2705 details.last_index = ULONG_MAX; 2706 details.i_mmap_lock = &mapping->i_mmap_lock; 2707 2708 mutex_lock(&mapping->unmap_mutex); 2709 spin_lock(&mapping->i_mmap_lock); 2710 2711 /* Protect against endless unmapping loops */ 2712 mapping->truncate_count++; 2713 if (unlikely(is_restart_addr(mapping->truncate_count))) { 2714 if (mapping->truncate_count == 0) 2715 reset_vma_truncate_counts(mapping); 2716 mapping->truncate_count++; 2717 } 2718 details.truncate_count = mapping->truncate_count; 2719 2720 if (unlikely(!prio_tree_empty(&mapping->i_mmap))) 2721 unmap_mapping_range_tree(&mapping->i_mmap, &details); 2722 if (unlikely(!list_empty(&mapping->i_mmap_nonlinear))) 2723 unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details); 2724 spin_unlock(&mapping->i_mmap_lock); 2725 mutex_unlock(&mapping->unmap_mutex); 2726} 2727EXPORT_SYMBOL(unmap_mapping_range); 2728 2729int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end) 2730{ 2731 struct address_space *mapping = inode->i_mapping; 2732 2733 /* 2734 * If the underlying filesystem is not going to provide 2735 * a way to truncate a range of blocks (punch a hole) - 2736 * we should return failure right now. 2737 */ 2738 if (!inode->i_op->truncate_range) 2739 return -ENOSYS; 2740 2741 mutex_lock(&inode->i_mutex); 2742 down_write(&inode->i_alloc_sem); 2743 unmap_mapping_range(mapping, offset, (end - offset), 1); 2744 truncate_inode_pages_range(mapping, offset, end); 2745 unmap_mapping_range(mapping, offset, (end - offset), 1); 2746 inode->i_op->truncate_range(inode, offset, end); 2747 up_write(&inode->i_alloc_sem); 2748 mutex_unlock(&inode->i_mutex); 2749 2750 return 0; 2751} 2752 2753/* 2754 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2755 * but allow concurrent faults), and pte mapped but not yet locked. 2756 * We return with mmap_sem still held, but pte unmapped and unlocked. 2757 */ 2758static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, 2759 unsigned long address, pte_t *page_table, pmd_t *pmd, 2760 unsigned int flags, pte_t orig_pte) 2761{ 2762 spinlock_t *ptl; 2763 struct page *page, *swapcache = NULL; 2764 swp_entry_t entry; 2765 pte_t pte; 2766 int locked; 2767 struct mem_cgroup *ptr = NULL; 2768 int exclusive = 0; 2769 int ret = 0; 2770 2771 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 2772 goto out; 2773 2774 entry = pte_to_swp_entry(orig_pte); 2775 if (unlikely(non_swap_entry(entry))) { 2776 if (is_migration_entry(entry)) { 2777 migration_entry_wait(mm, pmd, address); 2778 } else if (is_hwpoison_entry(entry)) { 2779 ret = VM_FAULT_HWPOISON; 2780 } else { 2781 print_bad_pte(vma, address, orig_pte, NULL); 2782 ret = VM_FAULT_SIGBUS; 2783 } 2784 goto out; 2785 } 2786 delayacct_set_flag(DELAYACCT_PF_SWAPIN); 2787 page = lookup_swap_cache(entry); 2788 if (!page) { 2789 grab_swap_token(mm); /* Contend for token _before_ read-in */ 2790 page = swapin_readahead(entry, 2791 GFP_HIGHUSER_MOVABLE, vma, address); 2792 if (!page) { 2793 /* 2794 * Back out if somebody else faulted in this pte 2795 * while we released the pte lock. 2796 */ 2797 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2798 if (likely(pte_same(*page_table, orig_pte))) 2799 ret = VM_FAULT_OOM; 2800 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2801 goto unlock; 2802 } 2803 2804 /* Had to read the page from swap area: Major fault */ 2805 ret = VM_FAULT_MAJOR; 2806 count_vm_event(PGMAJFAULT); 2807 } else if (PageHWPoison(page)) { 2808 /* 2809 * hwpoisoned dirty swapcache pages are kept for killing 2810 * owner processes (which may be unknown at hwpoison time) 2811 */ 2812 ret = VM_FAULT_HWPOISON; 2813 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2814 goto out_release; 2815 } 2816 2817 locked = lock_page_or_retry(page, mm, flags); 2818 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2819 if (!locked) { 2820 ret |= VM_FAULT_RETRY; 2821 goto out_release; 2822 } 2823 2824 /* 2825 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not 2826 * release the swapcache from under us. The page pin, and pte_same 2827 * test below, are not enough to exclude that. Even if it is still 2828 * swapcache, we need to check that the page's swap has not changed. 2829 */ 2830 if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val)) 2831 goto out_page; 2832 2833 if (ksm_might_need_to_copy(page, vma, address)) { 2834 swapcache = page; 2835 page = ksm_does_need_to_copy(page, vma, address); 2836 2837 if (unlikely(!page)) { 2838 ret = VM_FAULT_OOM; 2839 page = swapcache; 2840 swapcache = NULL; 2841 goto out_page; 2842 } 2843 } 2844 2845 if (mem_cgroup_try_charge_swapin(mm, page, GFP_KERNEL, &ptr)) { 2846 ret = VM_FAULT_OOM; 2847 goto out_page; 2848 } 2849 2850 /* 2851 * Back out if somebody else already faulted in this pte. 2852 */ 2853 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2854 if (unlikely(!pte_same(*page_table, orig_pte))) 2855 goto out_nomap; 2856 2857 if (unlikely(!PageUptodate(page))) { 2858 ret = VM_FAULT_SIGBUS; 2859 goto out_nomap; 2860 } 2861 2862 /* 2863 * The page isn't present yet, go ahead with the fault. 2864 * 2865 * Be careful about the sequence of operations here. 2866 * To get its accounting right, reuse_swap_page() must be called 2867 * while the page is counted on swap but not yet in mapcount i.e. 2868 * before page_add_anon_rmap() and swap_free(); try_to_free_swap() 2869 * must be called after the swap_free(), or it will never succeed. 2870 * Because delete_from_swap_page() may be called by reuse_swap_page(), 2871 * mem_cgroup_commit_charge_swapin() may not be able to find swp_entry 2872 * in page->private. In this case, a record in swap_cgroup is silently 2873 * discarded at swap_free(). 2874 */ 2875 2876 inc_mm_counter_fast(mm, MM_ANONPAGES); 2877 dec_mm_counter_fast(mm, MM_SWAPENTS); 2878 pte = mk_pte(page, vma->vm_page_prot); 2879 if ((flags & FAULT_FLAG_WRITE) && reuse_swap_page(page)) { 2880 pte = maybe_mkwrite(pte_mkdirty(pte), vma); 2881 flags &= ~FAULT_FLAG_WRITE; 2882 ret |= VM_FAULT_WRITE; 2883 exclusive = 1; 2884 } 2885 flush_icache_page(vma, page); 2886 set_pte_at(mm, address, page_table, pte); 2887 do_page_add_anon_rmap(page, vma, address, exclusive); 2888 /* It's better to call commit-charge after rmap is established */ 2889 mem_cgroup_commit_charge_swapin(page, ptr); 2890 2891 swap_free(entry); 2892 if (vm_swap_full() || (vma->vm_flags & VM_LOCKED) || PageMlocked(page)) 2893 try_to_free_swap(page); 2894 unlock_page(page); 2895 if (swapcache) { 2896 /* 2897 * Hold the lock to avoid the swap entry to be reused 2898 * until we take the PT lock for the pte_same() check 2899 * (to avoid false positives from pte_same). For 2900 * further safety release the lock after the swap_free 2901 * so that the swap count won't change under a 2902 * parallel locked swapcache. 2903 */ 2904 unlock_page(swapcache); 2905 page_cache_release(swapcache); 2906 } 2907 2908 if (flags & FAULT_FLAG_WRITE) { 2909 ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte); 2910 if (ret & VM_FAULT_ERROR) 2911 ret &= VM_FAULT_ERROR; 2912 goto out; 2913 } 2914 2915 /* No need to invalidate - it was non-present before */ 2916 update_mmu_cache(vma, address, page_table); 2917unlock: 2918 pte_unmap_unlock(page_table, ptl); 2919out: 2920 return ret; 2921out_nomap: 2922 mem_cgroup_cancel_charge_swapin(ptr); 2923 pte_unmap_unlock(page_table, ptl); 2924out_page: 2925 unlock_page(page); 2926out_release: 2927 page_cache_release(page); 2928 if (swapcache) { 2929 unlock_page(swapcache); 2930 page_cache_release(swapcache); 2931 } 2932 return ret; 2933} 2934 2935/* 2936 * This is like a special single-page "expand_{down|up}wards()", 2937 * except we must first make sure that 'address{-|+}PAGE_SIZE' 2938 * doesn't hit another vma. 2939 */ 2940static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned long address) 2941{ 2942 address &= PAGE_MASK; 2943 if ((vma->vm_flags & VM_GROWSDOWN) && address == vma->vm_start) { 2944 struct vm_area_struct *prev = vma->vm_prev; 2945 2946 /* 2947 * Is there a mapping abutting this one below? 2948 * 2949 * That's only ok if it's the same stack mapping 2950 * that has gotten split.. 2951 */ 2952 if (prev && prev->vm_end == address) 2953 return prev->vm_flags & VM_GROWSDOWN ? 0 : -ENOMEM; 2954 2955 expand_stack(vma, address - PAGE_SIZE); 2956 } 2957 if ((vma->vm_flags & VM_GROWSUP) && address + PAGE_SIZE == vma->vm_end) { 2958 struct vm_area_struct *next = vma->vm_next; 2959 2960 /* As VM_GROWSDOWN but s/below/above/ */ 2961 if (next && next->vm_start == address + PAGE_SIZE) 2962 return next->vm_flags & VM_GROWSUP ? 0 : -ENOMEM; 2963 2964 expand_upwards(vma, address + PAGE_SIZE); 2965 } 2966 return 0; 2967} 2968 2969/* 2970 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2971 * but allow concurrent faults), and pte mapped but not yet locked. 2972 * We return with mmap_sem still held, but pte unmapped and unlocked. 2973 */ 2974static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 2975 unsigned long address, pte_t *page_table, pmd_t *pmd, 2976 unsigned int flags) 2977{ 2978 struct page *page; 2979 spinlock_t *ptl; 2980 pte_t entry; 2981 2982 pte_unmap(page_table); 2983 2984 /* Check if we need to add a guard page to the stack */ 2985 if (check_stack_guard_page(vma, address) < 0) 2986 return VM_FAULT_SIGBUS; 2987 2988 /* Use the zero-page for reads */ 2989 if (!(flags & FAULT_FLAG_WRITE)) { 2990 entry = pte_mkspecial(pfn_pte(my_zero_pfn(address), 2991 vma->vm_page_prot)); 2992 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2993 if (!pte_none(*page_table)) 2994 goto unlock; 2995 goto setpte; 2996 } 2997 2998 /* Allocate our own private page. */ 2999 if (unlikely(anon_vma_prepare(vma))) 3000 goto oom; 3001 page = alloc_zeroed_user_highpage_movable(vma, address); 3002 if (!page) 3003 goto oom; 3004 __SetPageUptodate(page); 3005 3006 if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) 3007 goto oom_free_page; 3008 3009 entry = mk_pte(page, vma->vm_page_prot); 3010 if (vma->vm_flags & VM_WRITE) 3011 entry = pte_mkwrite(pte_mkdirty(entry)); 3012 3013 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3014 if (!pte_none(*page_table)) 3015 goto release; 3016 3017 inc_mm_counter_fast(mm, MM_ANONPAGES); 3018 page_add_new_anon_rmap(page, vma, address); 3019setpte: 3020 set_pte_at(mm, address, page_table, entry); 3021 3022 /* No need to invalidate - it was non-present before */ 3023 update_mmu_cache(vma, address, page_table); 3024unlock: 3025 pte_unmap_unlock(page_table, ptl); 3026 return 0; 3027release: 3028 mem_cgroup_uncharge_page(page); 3029 page_cache_release(page); 3030 goto unlock; 3031oom_free_page: 3032 page_cache_release(page); 3033oom: 3034 return VM_FAULT_OOM; 3035} 3036 3037/* 3038 * __do_fault() tries to create a new page mapping. It aggressively 3039 * tries to share with existing pages, but makes a separate copy if 3040 * the FAULT_FLAG_WRITE is set in the flags parameter in order to avoid 3041 * the next page fault. 3042 * 3043 * As this is called only for pages that do not currently exist, we 3044 * do not need to flush old virtual caches or the TLB. 3045 * 3046 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3047 * but allow concurrent faults), and pte neither mapped nor locked. 3048 * We return with mmap_sem still held, but pte unmapped and unlocked. 3049 */ 3050static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3051 unsigned long address, pmd_t *pmd, 3052 pgoff_t pgoff, unsigned int flags, pte_t orig_pte) 3053{ 3054 pte_t *page_table; 3055 spinlock_t *ptl; 3056 struct page *page; 3057 pte_t entry; 3058 int anon = 0; 3059 int charged = 0; 3060 struct page *dirty_page = NULL; 3061 struct vm_fault vmf; 3062 int ret; 3063 int page_mkwrite = 0; 3064 3065 vmf.virtual_address = (void __user *)(address & PAGE_MASK); 3066 vmf.pgoff = pgoff; 3067 vmf.flags = flags; 3068 vmf.page = NULL; 3069 3070 ret = vma->vm_ops->fault(vma, &vmf); 3071 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | 3072 VM_FAULT_RETRY))) 3073 return ret; 3074 3075 if (unlikely(PageHWPoison(vmf.page))) { 3076 if (ret & VM_FAULT_LOCKED) 3077 unlock_page(vmf.page); 3078 return VM_FAULT_HWPOISON; 3079 } 3080 3081 /* 3082 * For consistency in subsequent calls, make the faulted page always 3083 * locked. 3084 */ 3085 if (unlikely(!(ret & VM_FAULT_LOCKED))) 3086 lock_page(vmf.page); 3087 else 3088 VM_BUG_ON(!PageLocked(vmf.page)); 3089 3090 /* 3091 * Should we do an early C-O-W break? 3092 */ 3093 page = vmf.page; 3094 if (flags & FAULT_FLAG_WRITE) { 3095 if (!(vma->vm_flags & VM_SHARED)) { 3096 anon = 1; 3097 if (unlikely(anon_vma_prepare(vma))) { 3098 ret = VM_FAULT_OOM; 3099 goto out; 3100 } 3101 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, 3102 vma, address); 3103 if (!page) { 3104 ret = VM_FAULT_OOM; 3105 goto out; 3106 } 3107 if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) { 3108 ret = VM_FAULT_OOM; 3109 page_cache_release(page); 3110 goto out; 3111 } 3112 charged = 1; 3113 copy_user_highpage(page, vmf.page, address, vma); 3114 __SetPageUptodate(page); 3115 } else { 3116 /* 3117 * If the page will be shareable, see if the backing 3118 * address space wants to know that the page is about 3119 * to become writable 3120 */ 3121 if (vma->vm_ops->page_mkwrite) { 3122 int tmp; 3123 3124 unlock_page(page); 3125 vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 3126 tmp = vma->vm_ops->page_mkwrite(vma, &vmf); 3127 if (unlikely(tmp & 3128 (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 3129 ret = tmp; 3130 goto unwritable_page; 3131 } 3132 if (unlikely(!(tmp & VM_FAULT_LOCKED))) { 3133 lock_page(page); 3134 if (!page->mapping) { 3135 ret = 0; /* retry the fault */ 3136 unlock_page(page); 3137 goto unwritable_page; 3138 } 3139 } else 3140 VM_BUG_ON(!PageLocked(page)); 3141 page_mkwrite = 1; 3142 } 3143 } 3144 3145 } 3146 3147 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3148 3149 /* 3150 * This silly early PAGE_DIRTY setting removes a race 3151 * due to the bad i386 page protection. But it's valid 3152 * for other architectures too. 3153 * 3154 * Note that if FAULT_FLAG_WRITE is set, we either now have 3155 * an exclusive copy of the page, or this is a shared mapping, 3156 * so we can make it writable and dirty to avoid having to 3157 * handle that later. 3158 */ 3159 /* Only go through if we didn't race with anybody else... */ 3160 if (likely(pte_same(*page_table, orig_pte))) { 3161 flush_icache_page(vma, page); 3162 entry = mk_pte(page, vma->vm_page_prot); 3163 if (flags & FAULT_FLAG_WRITE) 3164 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 3165 if (anon) { 3166 inc_mm_counter_fast(mm, MM_ANONPAGES); 3167 page_add_new_anon_rmap(page, vma, address); 3168 } else { 3169 inc_mm_counter_fast(mm, MM_FILEPAGES); 3170 page_add_file_rmap(page); 3171 if (flags & FAULT_FLAG_WRITE) { 3172 dirty_page = page; 3173 get_page(dirty_page); 3174 } 3175 } 3176 set_pte_at(mm, address, page_table, entry); 3177 3178 /* no need to invalidate: a not-present page won't be cached */ 3179 update_mmu_cache(vma, address, page_table); 3180 } else { 3181 if (charged) 3182 mem_cgroup_uncharge_page(page); 3183 if (anon) 3184 page_cache_release(page); 3185 else 3186 anon = 1; /* no anon but release faulted_page */ 3187 } 3188 3189 pte_unmap_unlock(page_table, ptl); 3190 3191out: 3192 if (dirty_page) { 3193 struct address_space *mapping = page->mapping; 3194 3195 if (set_page_dirty(dirty_page)) 3196 page_mkwrite = 1; 3197 unlock_page(dirty_page); 3198 put_page(dirty_page); 3199 if (page_mkwrite && mapping) { 3200 /* 3201 * Some device drivers do not set page.mapping but still 3202 * dirty their pages 3203 */ 3204 balance_dirty_pages_ratelimited(mapping); 3205 } 3206 3207 /* file_update_time outside page_lock */ 3208 if (vma->vm_file) 3209 file_update_time(vma->vm_file); 3210 } else { 3211 unlock_page(vmf.page); 3212 if (anon) 3213 page_cache_release(vmf.page); 3214 } 3215 3216 return ret; 3217 3218unwritable_page: 3219 page_cache_release(page); 3220 return ret; 3221} 3222 3223static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3224 unsigned long address, pte_t *page_table, pmd_t *pmd, 3225 unsigned int flags, pte_t orig_pte) 3226{ 3227 pgoff_t pgoff = (((address & PAGE_MASK) 3228 - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 3229 3230 pte_unmap(page_table); 3231 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 3232} 3233 3234/* 3235 * Fault of a previously existing named mapping. Repopulate the pte 3236 * from the encoded file_pte if possible. This enables swappable 3237 * nonlinear vmas. 3238 * 3239 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3240 * but allow concurrent faults), and pte mapped but not yet locked. 3241 * We return with mmap_sem still held, but pte unmapped and unlocked. 3242 */ 3243static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3244 unsigned long address, pte_t *page_table, pmd_t *pmd, 3245 unsigned int flags, pte_t orig_pte) 3246{ 3247 pgoff_t pgoff; 3248 3249 flags |= FAULT_FLAG_NONLINEAR; 3250 3251 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 3252 return 0; 3253 3254 if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) { 3255 /* 3256 * Page table corrupted: show pte and kill process. 3257 */ 3258 print_bad_pte(vma, address, orig_pte, NULL); 3259 return VM_FAULT_SIGBUS; 3260 } 3261 3262 pgoff = pte_to_pgoff(orig_pte); 3263 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 3264} 3265 3266/* 3267 * These routines also need to handle stuff like marking pages dirty 3268 * and/or accessed for architectures that don't do it in hardware (most 3269 * RISC architectures). The early dirtying is also good on the i386. 3270 * 3271 * There is also a hook called "update_mmu_cache()" that architectures 3272 * with external mmu caches can use to update those (ie the Sparc or 3273 * PowerPC hashed page tables that act as extended TLBs). 3274 * 3275 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3276 * but allow concurrent faults), and pte mapped but not yet locked. 3277 * We return with mmap_sem still held, but pte unmapped and unlocked. 3278 */ 3279int handle_pte_fault(struct mm_struct *mm, 3280 struct vm_area_struct *vma, unsigned long address, 3281 pte_t *pte, pmd_t *pmd, unsigned int flags) 3282{ 3283 pte_t entry; 3284 spinlock_t *ptl; 3285 3286 entry = *pte; 3287 if (!pte_present(entry)) { 3288 if (pte_none(entry)) { 3289 if (vma->vm_ops) { 3290 if (likely(vma->vm_ops->fault)) 3291 return do_linear_fault(mm, vma, address, 3292 pte, pmd, flags, entry); 3293 } 3294 return do_anonymous_page(mm, vma, address, 3295 pte, pmd, flags); 3296 } 3297 if (pte_file(entry)) 3298 return do_nonlinear_fault(mm, vma, address, 3299 pte, pmd, flags, entry); 3300 return do_swap_page(mm, vma, address, 3301 pte, pmd, flags, entry); 3302 } 3303 3304 ptl = pte_lockptr(mm, pmd); 3305 spin_lock(ptl); 3306 if (unlikely(!pte_same(*pte, entry))) 3307 goto unlock; 3308 if (flags & FAULT_FLAG_WRITE) { 3309 if (!pte_write(entry)) 3310 return do_wp_page(mm, vma, address, 3311 pte, pmd, ptl, entry); 3312 entry = pte_mkdirty(entry); 3313 } 3314 entry = pte_mkyoung(entry); 3315 if (ptep_set_access_flags(vma, address, pte, entry, flags & FAULT_FLAG_WRITE)) { 3316 update_mmu_cache(vma, address, pte); 3317 } else { 3318 /* 3319 * This is needed only for protection faults but the arch code 3320 * is not yet telling us if this is a protection fault or not. 3321 * This still avoids useless tlb flushes for .text page faults 3322 * with threads. 3323 */ 3324 if (flags & FAULT_FLAG_WRITE) 3325 flush_tlb_fix_spurious_fault(vma, address); 3326 } 3327unlock: 3328 pte_unmap_unlock(pte, ptl); 3329 return 0; 3330} 3331 3332/* 3333 * By the time we get here, we already hold the mm semaphore 3334 */ 3335int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3336 unsigned long address, unsigned int flags) 3337{ 3338 pgd_t *pgd; 3339 pud_t *pud; 3340 pmd_t *pmd; 3341 pte_t *pte; 3342 3343 __set_current_state(TASK_RUNNING); 3344 3345 count_vm_event(PGFAULT); 3346 3347 /* do counter updates before entering really critical section. */ 3348 check_sync_rss_stat(current); 3349 3350 if (unlikely(is_vm_hugetlb_page(vma))) 3351 return hugetlb_fault(mm, vma, address, flags); 3352 3353 pgd = pgd_offset(mm, address); 3354 pud = pud_alloc(mm, pgd, address); 3355 if (!pud) 3356 return VM_FAULT_OOM; 3357 pmd = pmd_alloc(mm, pud, address); 3358 if (!pmd) 3359 return VM_FAULT_OOM; 3360 if (pmd_none(*pmd) && transparent_hugepage_enabled(vma)) { 3361 if (!vma->vm_ops) 3362 return do_huge_pmd_anonymous_page(mm, vma, address, 3363 pmd, flags); 3364 } else { 3365 pmd_t orig_pmd = *pmd; 3366 barrier(); 3367 if (pmd_trans_huge(orig_pmd)) { 3368 if (flags & FAULT_FLAG_WRITE && 3369 !pmd_write(orig_pmd) && 3370 !pmd_trans_splitting(orig_pmd)) 3371 return do_huge_pmd_wp_page(mm, vma, address, 3372 pmd, orig_pmd); 3373 return 0; 3374 } 3375 } 3376 3377 /* 3378 * Use __pte_alloc instead of pte_alloc_map, because we can't 3379 * run pte_offset_map on the pmd, if an huge pmd could 3380 * materialize from under us from a different thread. 3381 */ 3382 if (unlikely(__pte_alloc(mm, vma, pmd, address))) 3383 return VM_FAULT_OOM; 3384 /* if an huge pmd materialized from under us just retry later */ 3385 if (unlikely(pmd_trans_huge(*pmd))) 3386 return 0; 3387 /* 3388 * A regular pmd is established and it can't morph into a huge pmd 3389 * from under us anymore at this point because we hold the mmap_sem 3390 * read mode and khugepaged takes it in write mode. So now it's 3391 * safe to run pte_offset_map(). 3392 */ 3393 pte = pte_offset_map(pmd, address); 3394 3395 return handle_pte_fault(mm, vma, address, pte, pmd, flags); 3396} 3397 3398#ifndef __PAGETABLE_PUD_FOLDED 3399/* 3400 * Allocate page upper directory. 3401 * We've already handled the fast-path in-line. 3402 */ 3403int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 3404{ 3405 pud_t *new = pud_alloc_one(mm, address); 3406 if (!new) 3407 return -ENOMEM; 3408 3409 smp_wmb(); /* See comment in __pte_alloc */ 3410 3411 spin_lock(&mm->page_table_lock); 3412 if (pgd_present(*pgd)) /* Another has populated it */ 3413 pud_free(mm, new); 3414 else 3415 pgd_populate(mm, pgd, new); 3416 spin_unlock(&mm->page_table_lock); 3417 return 0; 3418} 3419#endif /* __PAGETABLE_PUD_FOLDED */ 3420 3421#ifndef __PAGETABLE_PMD_FOLDED 3422/* 3423 * Allocate page middle directory. 3424 * We've already handled the fast-path in-line. 3425 */ 3426int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 3427{ 3428 pmd_t *new = pmd_alloc_one(mm, address); 3429 if (!new) 3430 return -ENOMEM; 3431 3432 smp_wmb(); /* See comment in __pte_alloc */ 3433 3434 spin_lock(&mm->page_table_lock); 3435#ifndef __ARCH_HAS_4LEVEL_HACK 3436 if (pud_present(*pud)) /* Another has populated it */ 3437 pmd_free(mm, new); 3438 else 3439 pud_populate(mm, pud, new); 3440#else 3441 if (pgd_present(*pud)) /* Another has populated it */ 3442 pmd_free(mm, new); 3443 else 3444 pgd_populate(mm, pud, new); 3445#endif /* __ARCH_HAS_4LEVEL_HACK */ 3446 spin_unlock(&mm->page_table_lock); 3447 return 0; 3448} 3449#endif /* __PAGETABLE_PMD_FOLDED */ 3450 3451int make_pages_present(unsigned long addr, unsigned long end) 3452{ 3453 int ret, len, write; 3454 struct vm_area_struct * vma; 3455 3456 vma = find_vma(current->mm, addr); 3457 if (!vma) 3458 return -ENOMEM; 3459 /* 3460 * We want to touch writable mappings with a write fault in order 3461 * to break COW, except for shared mappings because these don't COW 3462 * and we would not want to dirty them for nothing. 3463 */ 3464 write = (vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE; 3465 BUG_ON(addr >= end); 3466 BUG_ON(end > vma->vm_end); 3467 len = DIV_ROUND_UP(end, PAGE_SIZE) - addr/PAGE_SIZE; 3468 ret = get_user_pages(current, current->mm, addr, 3469 len, write, 0, NULL, NULL); 3470 if (ret < 0) 3471 return ret; 3472 return ret == len ? 0 : -EFAULT; 3473} 3474 3475#if !defined(__HAVE_ARCH_GATE_AREA) 3476 3477#if defined(AT_SYSINFO_EHDR) 3478static struct vm_area_struct gate_vma; 3479 3480static int __init gate_vma_init(void) 3481{ 3482 gate_vma.vm_mm = NULL; 3483 gate_vma.vm_start = FIXADDR_USER_START; 3484 gate_vma.vm_end = FIXADDR_USER_END; 3485 gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC; 3486 gate_vma.vm_page_prot = __P101; 3487 /* 3488 * Make sure the vDSO gets into every core dump. 3489 * Dumping its contents makes post-mortem fully interpretable later 3490 * without matching up the same kernel and hardware config to see 3491 * what PC values meant. 3492 */ 3493 gate_vma.vm_flags |= VM_ALWAYSDUMP; 3494 return 0; 3495} 3496__initcall(gate_vma_init); 3497#endif 3498 3499struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 3500{ 3501#ifdef AT_SYSINFO_EHDR 3502 return &gate_vma; 3503#else 3504 return NULL; 3505#endif 3506} 3507 3508int in_gate_area_no_task(unsigned long addr) 3509{ 3510#ifdef AT_SYSINFO_EHDR 3511 if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END)) 3512 return 1; 3513#endif 3514 return 0; 3515} 3516 3517#endif /* __HAVE_ARCH_GATE_AREA */ 3518 3519static int __follow_pte(struct mm_struct *mm, unsigned long address, 3520 pte_t **ptepp, spinlock_t **ptlp) 3521{ 3522 pgd_t *pgd; 3523 pud_t *pud; 3524 pmd_t *pmd; 3525 pte_t *ptep; 3526 3527 pgd = pgd_offset(mm, address); 3528 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 3529 goto out; 3530 3531 pud = pud_offset(pgd, address); 3532 if (pud_none(*pud) || unlikely(pud_bad(*pud))) 3533 goto out; 3534 3535 pmd = pmd_offset(pud, address); 3536 VM_BUG_ON(pmd_trans_huge(*pmd)); 3537 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) 3538 goto out; 3539 3540 /* We cannot handle huge page PFN maps. Luckily they don't exist. */ 3541 if (pmd_huge(*pmd)) 3542 goto out; 3543 3544 ptep = pte_offset_map_lock(mm, pmd, address, ptlp); 3545 if (!ptep) 3546 goto out; 3547 if (!pte_present(*ptep)) 3548 goto unlock; 3549 *ptepp = ptep; 3550 return 0; 3551unlock: 3552 pte_unmap_unlock(ptep, *ptlp); 3553out: 3554 return -EINVAL; 3555} 3556 3557static inline int follow_pte(struct mm_struct *mm, unsigned long address, 3558 pte_t **ptepp, spinlock_t **ptlp) 3559{ 3560 int res; 3561 3562 /* (void) is needed to make gcc happy */ 3563 (void) __cond_lock(*ptlp, 3564 !(res = __follow_pte(mm, address, ptepp, ptlp))); 3565 return res; 3566} 3567 3568/** 3569 * follow_pfn - look up PFN at a user virtual address 3570 * @vma: memory mapping 3571 * @address: user virtual address 3572 * @pfn: location to store found PFN 3573 * 3574 * Only IO mappings and raw PFN mappings are allowed. 3575 * 3576 * Returns zero and the pfn at @pfn on success, -ve otherwise. 3577 */ 3578int follow_pfn(struct vm_area_struct *vma, unsigned long address, 3579 unsigned long *pfn) 3580{ 3581 int ret = -EINVAL; 3582 spinlock_t *ptl; 3583 pte_t *ptep; 3584 3585 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3586 return ret; 3587 3588 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); 3589 if (ret) 3590 return ret; 3591 *pfn = pte_pfn(*ptep); 3592 pte_unmap_unlock(ptep, ptl); 3593 return 0; 3594} 3595EXPORT_SYMBOL(follow_pfn); 3596 3597#ifdef CONFIG_HAVE_IOREMAP_PROT 3598int follow_phys(struct vm_area_struct *vma, 3599 unsigned long address, unsigned int flags, 3600 unsigned long *prot, resource_size_t *phys) 3601{ 3602 int ret = -EINVAL; 3603 pte_t *ptep, pte; 3604 spinlock_t *ptl; 3605 3606 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3607 goto out; 3608 3609 if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) 3610 goto out; 3611 pte = *ptep; 3612 3613 if ((flags & FOLL_WRITE) && !pte_write(pte)) 3614 goto unlock; 3615 3616 *prot = pgprot_val(pte_pgprot(pte)); 3617 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; 3618 3619 ret = 0; 3620unlock: 3621 pte_unmap_unlock(ptep, ptl); 3622out: 3623 return ret; 3624} 3625 3626int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 3627 void *buf, int len, int write) 3628{ 3629 resource_size_t phys_addr; 3630 unsigned long prot = 0; 3631 void __iomem *maddr; 3632 int offset = addr & (PAGE_SIZE-1); 3633 3634 if (follow_phys(vma, addr, write, &prot, &phys_addr)) 3635 return -EINVAL; 3636 3637 maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); 3638 if (write) 3639 memcpy_toio(maddr + offset, buf, len); 3640 else 3641 memcpy_fromio(buf, maddr + offset, len); 3642 iounmap(maddr); 3643 3644 return len; 3645} 3646#endif 3647 3648/* 3649 * Access another process' address space. 3650 * Source/target buffer must be kernel space, 3651 * Do not walk the page table directly, use get_user_pages 3652 */ 3653int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write) 3654{ 3655 struct mm_struct *mm; 3656 struct vm_area_struct *vma; 3657 void *old_buf = buf; 3658 3659 mm = get_task_mm(tsk); 3660 if (!mm) 3661 return 0; 3662 3663 down_read(&mm->mmap_sem); 3664 /* ignore errors, just check how much was successfully transferred */ 3665 while (len) { 3666 int bytes, ret, offset; 3667 void *maddr; 3668 struct page *page = NULL; 3669 3670 ret = get_user_pages(tsk, mm, addr, 1, 3671 write, 1, &page, &vma); 3672 if (ret <= 0) { 3673 /* 3674 * Check if this is a VM_IO | VM_PFNMAP VMA, which 3675 * we can access using slightly different code. 3676 */ 3677#ifdef CONFIG_HAVE_IOREMAP_PROT 3678 vma = find_vma(mm, addr); 3679 if (!vma) 3680 break; 3681 if (vma->vm_ops && vma->vm_ops->access) 3682 ret = vma->vm_ops->access(vma, addr, buf, 3683 len, write); 3684 if (ret <= 0) 3685#endif 3686 break; 3687 bytes = ret; 3688 } else { 3689 bytes = len; 3690 offset = addr & (PAGE_SIZE-1); 3691 if (bytes > PAGE_SIZE-offset) 3692 bytes = PAGE_SIZE-offset; 3693 3694 maddr = kmap(page); 3695 if (write) { 3696 copy_to_user_page(vma, page, addr, 3697 maddr + offset, buf, bytes); 3698 set_page_dirty_lock(page); 3699 } else { 3700 copy_from_user_page(vma, page, addr, 3701 buf, maddr + offset, bytes); 3702 } 3703 kunmap(page); 3704 page_cache_release(page); 3705 } 3706 len -= bytes; 3707 buf += bytes; 3708 addr += bytes; 3709 } 3710 up_read(&mm->mmap_sem); 3711 mmput(mm); 3712 3713 return buf - old_buf; 3714} 3715 3716/* 3717 * Print the name of a VMA. 3718 */ 3719void print_vma_addr(char *prefix, unsigned long ip) 3720{ 3721 struct mm_struct *mm = current->mm; 3722 struct vm_area_struct *vma; 3723 3724 /* 3725 * Do not print if we are in atomic 3726 * contexts (in exception stacks, etc.): 3727 */ 3728 if (preempt_count()) 3729 return; 3730 3731 down_read(&mm->mmap_sem); 3732 vma = find_vma(mm, ip); 3733 if (vma && vma->vm_file) { 3734 struct file *f = vma->vm_file; 3735 char *buf = (char *)__get_free_page(GFP_KERNEL); 3736 if (buf) { 3737 char *p, *s; 3738 3739 p = d_path(&f->f_path, buf, PAGE_SIZE); 3740 if (IS_ERR(p)) 3741 p = "?"; 3742 s = strrchr(p, '/'); 3743 if (s) 3744 p = s+1; 3745 printk("%s%s[%lx+%lx]", prefix, p, 3746 vma->vm_start, 3747 vma->vm_end - vma->vm_start); 3748 free_page((unsigned long)buf); 3749 } 3750 } 3751 up_read(¤t->mm->mmap_sem); 3752} 3753 3754#ifdef CONFIG_PROVE_LOCKING 3755void might_fault(void) 3756{ 3757 /* 3758 * Some code (nfs/sunrpc) uses socket ops on kernel memory while 3759 * holding the mmap_sem, this is safe because kernel memory doesn't 3760 * get paged out, therefore we'll never actually fault, and the 3761 * below annotations will generate false positives. 3762 */ 3763 if (segment_eq(get_fs(), KERNEL_DS)) 3764 return; 3765 3766 might_sleep(); 3767 /* 3768 * it would be nicer only to annotate paths which are not under 3769 * pagefault_disable, however that requires a larger audit and 3770 * providing helpers like get_user_atomic. 3771 */ 3772 if (!in_atomic() && current->mm) 3773 might_lock_read(¤t->mm->mmap_sem); 3774} 3775EXPORT_SYMBOL(might_fault); 3776#endif 3777 3778#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 3779static void clear_gigantic_page(struct page *page, 3780 unsigned long addr, 3781 unsigned int pages_per_huge_page) 3782{ 3783 int i; 3784 struct page *p = page; 3785 3786 might_sleep(); 3787 for (i = 0; i < pages_per_huge_page; 3788 i++, p = mem_map_next(p, page, i)) { 3789 cond_resched(); 3790 clear_user_highpage(p, addr + i * PAGE_SIZE); 3791 } 3792} 3793void clear_huge_page(struct page *page, 3794 unsigned long addr, unsigned int pages_per_huge_page) 3795{ 3796 int i; 3797 3798 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 3799 clear_gigantic_page(page, addr, pages_per_huge_page); 3800 return; 3801 } 3802 3803 might_sleep(); 3804 for (i = 0; i < pages_per_huge_page; i++) { 3805 cond_resched(); 3806 clear_user_highpage(page + i, addr + i * PAGE_SIZE); 3807 } 3808} 3809 3810static void copy_user_gigantic_page(struct page *dst, struct page *src, 3811 unsigned long addr, 3812 struct vm_area_struct *vma, 3813 unsigned int pages_per_huge_page) 3814{ 3815 int i; 3816 struct page *dst_base = dst; 3817 struct page *src_base = src; 3818 3819 for (i = 0; i < pages_per_huge_page; ) { 3820 cond_resched(); 3821 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma); 3822 3823 i++; 3824 dst = mem_map_next(dst, dst_base, i); 3825 src = mem_map_next(src, src_base, i); 3826 } 3827} 3828 3829void copy_user_huge_page(struct page *dst, struct page *src, 3830 unsigned long addr, struct vm_area_struct *vma, 3831 unsigned int pages_per_huge_page) 3832{ 3833 int i; 3834 3835 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 3836 copy_user_gigantic_page(dst, src, addr, vma, 3837 pages_per_huge_page); 3838 return; 3839 } 3840 3841 might_sleep(); 3842 for (i = 0; i < pages_per_huge_page; i++) { 3843 cond_resched(); 3844 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma); 3845 } 3846} 3847#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 3848