memory.c revision 3b6748e2dd69906af3835db4dc9d1c8a3ee4c68c
1/* 2 * linux/mm/memory.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7/* 8 * demand-loading started 01.12.91 - seems it is high on the list of 9 * things wanted, and it should be easy to implement. - Linus 10 */ 11 12/* 13 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared 14 * pages started 02.12.91, seems to work. - Linus. 15 * 16 * Tested sharing by executing about 30 /bin/sh: under the old kernel it 17 * would have taken more than the 6M I have free, but it worked well as 18 * far as I could see. 19 * 20 * Also corrected some "invalidate()"s - I wasn't doing enough of them. 21 */ 22 23/* 24 * Real VM (paging to/from disk) started 18.12.91. Much more work and 25 * thought has to go into this. Oh, well.. 26 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. 27 * Found it. Everything seems to work now. 28 * 20.12.91 - Ok, making the swap-device changeable like the root. 29 */ 30 31/* 32 * 05.04.94 - Multi-page memory management added for v1.1. 33 * Idea by Alex Bligh (alex@cconcepts.co.uk) 34 * 35 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG 36 * (Gerhard.Wichert@pdb.siemens.de) 37 * 38 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) 39 */ 40 41#include <linux/kernel_stat.h> 42#include <linux/mm.h> 43#include <linux/hugetlb.h> 44#include <linux/mman.h> 45#include <linux/swap.h> 46#include <linux/highmem.h> 47#include <linux/pagemap.h> 48#include <linux/rmap.h> 49#include <linux/module.h> 50#include <linux/delayacct.h> 51#include <linux/init.h> 52#include <linux/writeback.h> 53#include <linux/memcontrol.h> 54#include <linux/mmu_notifier.h> 55#include <linux/kallsyms.h> 56#include <linux/swapops.h> 57#include <linux/elf.h> 58 59#include <asm/pgalloc.h> 60#include <asm/uaccess.h> 61#include <asm/tlb.h> 62#include <asm/tlbflush.h> 63#include <asm/pgtable.h> 64 65#include "internal.h" 66 67#ifndef CONFIG_NEED_MULTIPLE_NODES 68/* use the per-pgdat data instead for discontigmem - mbligh */ 69unsigned long max_mapnr; 70struct page *mem_map; 71 72EXPORT_SYMBOL(max_mapnr); 73EXPORT_SYMBOL(mem_map); 74#endif 75 76unsigned long num_physpages; 77/* 78 * A number of key systems in x86 including ioremap() rely on the assumption 79 * that high_memory defines the upper bound on direct map memory, then end 80 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and 81 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL 82 * and ZONE_HIGHMEM. 83 */ 84void * high_memory; 85 86EXPORT_SYMBOL(num_physpages); 87EXPORT_SYMBOL(high_memory); 88 89/* 90 * Randomize the address space (stacks, mmaps, brk, etc.). 91 * 92 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, 93 * as ancient (libc5 based) binaries can segfault. ) 94 */ 95int randomize_va_space __read_mostly = 96#ifdef CONFIG_COMPAT_BRK 97 1; 98#else 99 2; 100#endif 101 102static int __init disable_randmaps(char *s) 103{ 104 randomize_va_space = 0; 105 return 1; 106} 107__setup("norandmaps", disable_randmaps); 108 109 110/* 111 * If a p?d_bad entry is found while walking page tables, report 112 * the error, before resetting entry to p?d_none. Usually (but 113 * very seldom) called out from the p?d_none_or_clear_bad macros. 114 */ 115 116void pgd_clear_bad(pgd_t *pgd) 117{ 118 pgd_ERROR(*pgd); 119 pgd_clear(pgd); 120} 121 122void pud_clear_bad(pud_t *pud) 123{ 124 pud_ERROR(*pud); 125 pud_clear(pud); 126} 127 128void pmd_clear_bad(pmd_t *pmd) 129{ 130 pmd_ERROR(*pmd); 131 pmd_clear(pmd); 132} 133 134/* 135 * Note: this doesn't free the actual pages themselves. That 136 * has been handled earlier when unmapping all the memory regions. 137 */ 138static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd) 139{ 140 pgtable_t token = pmd_pgtable(*pmd); 141 pmd_clear(pmd); 142 pte_free_tlb(tlb, token); 143 tlb->mm->nr_ptes--; 144} 145 146static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 147 unsigned long addr, unsigned long end, 148 unsigned long floor, unsigned long ceiling) 149{ 150 pmd_t *pmd; 151 unsigned long next; 152 unsigned long start; 153 154 start = addr; 155 pmd = pmd_offset(pud, addr); 156 do { 157 next = pmd_addr_end(addr, end); 158 if (pmd_none_or_clear_bad(pmd)) 159 continue; 160 free_pte_range(tlb, pmd); 161 } while (pmd++, addr = next, addr != end); 162 163 start &= PUD_MASK; 164 if (start < floor) 165 return; 166 if (ceiling) { 167 ceiling &= PUD_MASK; 168 if (!ceiling) 169 return; 170 } 171 if (end - 1 > ceiling - 1) 172 return; 173 174 pmd = pmd_offset(pud, start); 175 pud_clear(pud); 176 pmd_free_tlb(tlb, pmd); 177} 178 179static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, 180 unsigned long addr, unsigned long end, 181 unsigned long floor, unsigned long ceiling) 182{ 183 pud_t *pud; 184 unsigned long next; 185 unsigned long start; 186 187 start = addr; 188 pud = pud_offset(pgd, addr); 189 do { 190 next = pud_addr_end(addr, end); 191 if (pud_none_or_clear_bad(pud)) 192 continue; 193 free_pmd_range(tlb, pud, addr, next, floor, ceiling); 194 } while (pud++, addr = next, addr != end); 195 196 start &= PGDIR_MASK; 197 if (start < floor) 198 return; 199 if (ceiling) { 200 ceiling &= PGDIR_MASK; 201 if (!ceiling) 202 return; 203 } 204 if (end - 1 > ceiling - 1) 205 return; 206 207 pud = pud_offset(pgd, start); 208 pgd_clear(pgd); 209 pud_free_tlb(tlb, pud); 210} 211 212/* 213 * This function frees user-level page tables of a process. 214 * 215 * Must be called with pagetable lock held. 216 */ 217void free_pgd_range(struct mmu_gather *tlb, 218 unsigned long addr, unsigned long end, 219 unsigned long floor, unsigned long ceiling) 220{ 221 pgd_t *pgd; 222 unsigned long next; 223 unsigned long start; 224 225 /* 226 * The next few lines have given us lots of grief... 227 * 228 * Why are we testing PMD* at this top level? Because often 229 * there will be no work to do at all, and we'd prefer not to 230 * go all the way down to the bottom just to discover that. 231 * 232 * Why all these "- 1"s? Because 0 represents both the bottom 233 * of the address space and the top of it (using -1 for the 234 * top wouldn't help much: the masks would do the wrong thing). 235 * The rule is that addr 0 and floor 0 refer to the bottom of 236 * the address space, but end 0 and ceiling 0 refer to the top 237 * Comparisons need to use "end - 1" and "ceiling - 1" (though 238 * that end 0 case should be mythical). 239 * 240 * Wherever addr is brought up or ceiling brought down, we must 241 * be careful to reject "the opposite 0" before it confuses the 242 * subsequent tests. But what about where end is brought down 243 * by PMD_SIZE below? no, end can't go down to 0 there. 244 * 245 * Whereas we round start (addr) and ceiling down, by different 246 * masks at different levels, in order to test whether a table 247 * now has no other vmas using it, so can be freed, we don't 248 * bother to round floor or end up - the tests don't need that. 249 */ 250 251 addr &= PMD_MASK; 252 if (addr < floor) { 253 addr += PMD_SIZE; 254 if (!addr) 255 return; 256 } 257 if (ceiling) { 258 ceiling &= PMD_MASK; 259 if (!ceiling) 260 return; 261 } 262 if (end - 1 > ceiling - 1) 263 end -= PMD_SIZE; 264 if (addr > end - 1) 265 return; 266 267 start = addr; 268 pgd = pgd_offset(tlb->mm, addr); 269 do { 270 next = pgd_addr_end(addr, end); 271 if (pgd_none_or_clear_bad(pgd)) 272 continue; 273 free_pud_range(tlb, pgd, addr, next, floor, ceiling); 274 } while (pgd++, addr = next, addr != end); 275} 276 277void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, 278 unsigned long floor, unsigned long ceiling) 279{ 280 while (vma) { 281 struct vm_area_struct *next = vma->vm_next; 282 unsigned long addr = vma->vm_start; 283 284 /* 285 * Hide vma from rmap and vmtruncate before freeing pgtables 286 */ 287 anon_vma_unlink(vma); 288 unlink_file_vma(vma); 289 290 if (is_vm_hugetlb_page(vma)) { 291 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, 292 floor, next? next->vm_start: ceiling); 293 } else { 294 /* 295 * Optimization: gather nearby vmas into one call down 296 */ 297 while (next && next->vm_start <= vma->vm_end + PMD_SIZE 298 && !is_vm_hugetlb_page(next)) { 299 vma = next; 300 next = vma->vm_next; 301 anon_vma_unlink(vma); 302 unlink_file_vma(vma); 303 } 304 free_pgd_range(tlb, addr, vma->vm_end, 305 floor, next? next->vm_start: ceiling); 306 } 307 vma = next; 308 } 309} 310 311int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address) 312{ 313 pgtable_t new = pte_alloc_one(mm, address); 314 if (!new) 315 return -ENOMEM; 316 317 /* 318 * Ensure all pte setup (eg. pte page lock and page clearing) are 319 * visible before the pte is made visible to other CPUs by being 320 * put into page tables. 321 * 322 * The other side of the story is the pointer chasing in the page 323 * table walking code (when walking the page table without locking; 324 * ie. most of the time). Fortunately, these data accesses consist 325 * of a chain of data-dependent loads, meaning most CPUs (alpha 326 * being the notable exception) will already guarantee loads are 327 * seen in-order. See the alpha page table accessors for the 328 * smp_read_barrier_depends() barriers in page table walking code. 329 */ 330 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ 331 332 spin_lock(&mm->page_table_lock); 333 if (!pmd_present(*pmd)) { /* Has another populated it ? */ 334 mm->nr_ptes++; 335 pmd_populate(mm, pmd, new); 336 new = NULL; 337 } 338 spin_unlock(&mm->page_table_lock); 339 if (new) 340 pte_free(mm, new); 341 return 0; 342} 343 344int __pte_alloc_kernel(pmd_t *pmd, unsigned long address) 345{ 346 pte_t *new = pte_alloc_one_kernel(&init_mm, address); 347 if (!new) 348 return -ENOMEM; 349 350 smp_wmb(); /* See comment in __pte_alloc */ 351 352 spin_lock(&init_mm.page_table_lock); 353 if (!pmd_present(*pmd)) { /* Has another populated it ? */ 354 pmd_populate_kernel(&init_mm, pmd, new); 355 new = NULL; 356 } 357 spin_unlock(&init_mm.page_table_lock); 358 if (new) 359 pte_free_kernel(&init_mm, new); 360 return 0; 361} 362 363static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss) 364{ 365 if (file_rss) 366 add_mm_counter(mm, file_rss, file_rss); 367 if (anon_rss) 368 add_mm_counter(mm, anon_rss, anon_rss); 369} 370 371/* 372 * This function is called to print an error when a bad pte 373 * is found. For example, we might have a PFN-mapped pte in 374 * a region that doesn't allow it. 375 * 376 * The calling function must still handle the error. 377 */ 378static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, 379 pte_t pte, struct page *page) 380{ 381 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); 382 pud_t *pud = pud_offset(pgd, addr); 383 pmd_t *pmd = pmd_offset(pud, addr); 384 struct address_space *mapping; 385 pgoff_t index; 386 static unsigned long resume; 387 static unsigned long nr_shown; 388 static unsigned long nr_unshown; 389 390 /* 391 * Allow a burst of 60 reports, then keep quiet for that minute; 392 * or allow a steady drip of one report per second. 393 */ 394 if (nr_shown == 60) { 395 if (time_before(jiffies, resume)) { 396 nr_unshown++; 397 return; 398 } 399 if (nr_unshown) { 400 printk(KERN_ALERT 401 "BUG: Bad page map: %lu messages suppressed\n", 402 nr_unshown); 403 nr_unshown = 0; 404 } 405 nr_shown = 0; 406 } 407 if (nr_shown++ == 0) 408 resume = jiffies + 60 * HZ; 409 410 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; 411 index = linear_page_index(vma, addr); 412 413 printk(KERN_ALERT 414 "BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n", 415 current->comm, 416 (long long)pte_val(pte), (long long)pmd_val(*pmd)); 417 if (page) { 418 printk(KERN_ALERT 419 "page:%p flags:%p count:%d mapcount:%d mapping:%p index:%lx\n", 420 page, (void *)page->flags, page_count(page), 421 page_mapcount(page), page->mapping, page->index); 422 } 423 printk(KERN_ALERT 424 "addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n", 425 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); 426 /* 427 * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y 428 */ 429 if (vma->vm_ops) 430 print_symbol(KERN_ALERT "vma->vm_ops->fault: %s\n", 431 (unsigned long)vma->vm_ops->fault); 432 if (vma->vm_file && vma->vm_file->f_op) 433 print_symbol(KERN_ALERT "vma->vm_file->f_op->mmap: %s\n", 434 (unsigned long)vma->vm_file->f_op->mmap); 435 dump_stack(); 436 add_taint(TAINT_BAD_PAGE); 437} 438 439static inline int is_cow_mapping(unsigned int flags) 440{ 441 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 442} 443 444/* 445 * vm_normal_page -- This function gets the "struct page" associated with a pte. 446 * 447 * "Special" mappings do not wish to be associated with a "struct page" (either 448 * it doesn't exist, or it exists but they don't want to touch it). In this 449 * case, NULL is returned here. "Normal" mappings do have a struct page. 450 * 451 * There are 2 broad cases. Firstly, an architecture may define a pte_special() 452 * pte bit, in which case this function is trivial. Secondly, an architecture 453 * may not have a spare pte bit, which requires a more complicated scheme, 454 * described below. 455 * 456 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a 457 * special mapping (even if there are underlying and valid "struct pages"). 458 * COWed pages of a VM_PFNMAP are always normal. 459 * 460 * The way we recognize COWed pages within VM_PFNMAP mappings is through the 461 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit 462 * set, and the vm_pgoff will point to the first PFN mapped: thus every special 463 * mapping will always honor the rule 464 * 465 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) 466 * 467 * And for normal mappings this is false. 468 * 469 * This restricts such mappings to be a linear translation from virtual address 470 * to pfn. To get around this restriction, we allow arbitrary mappings so long 471 * as the vma is not a COW mapping; in that case, we know that all ptes are 472 * special (because none can have been COWed). 473 * 474 * 475 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP. 476 * 477 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct 478 * page" backing, however the difference is that _all_ pages with a struct 479 * page (that is, those where pfn_valid is true) are refcounted and considered 480 * normal pages by the VM. The disadvantage is that pages are refcounted 481 * (which can be slower and simply not an option for some PFNMAP users). The 482 * advantage is that we don't have to follow the strict linearity rule of 483 * PFNMAP mappings in order to support COWable mappings. 484 * 485 */ 486#ifdef __HAVE_ARCH_PTE_SPECIAL 487# define HAVE_PTE_SPECIAL 1 488#else 489# define HAVE_PTE_SPECIAL 0 490#endif 491struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 492 pte_t pte) 493{ 494 unsigned long pfn = pte_pfn(pte); 495 496 if (HAVE_PTE_SPECIAL) { 497 if (likely(!pte_special(pte))) 498 goto check_pfn; 499 if (!(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))) 500 print_bad_pte(vma, addr, pte, NULL); 501 return NULL; 502 } 503 504 /* !HAVE_PTE_SPECIAL case follows: */ 505 506 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 507 if (vma->vm_flags & VM_MIXEDMAP) { 508 if (!pfn_valid(pfn)) 509 return NULL; 510 goto out; 511 } else { 512 unsigned long off; 513 off = (addr - vma->vm_start) >> PAGE_SHIFT; 514 if (pfn == vma->vm_pgoff + off) 515 return NULL; 516 if (!is_cow_mapping(vma->vm_flags)) 517 return NULL; 518 } 519 } 520 521check_pfn: 522 if (unlikely(pfn > highest_memmap_pfn)) { 523 print_bad_pte(vma, addr, pte, NULL); 524 return NULL; 525 } 526 527 /* 528 * NOTE! We still have PageReserved() pages in the page tables. 529 * eg. VDSO mappings can cause them to exist. 530 */ 531out: 532 return pfn_to_page(pfn); 533} 534 535/* 536 * copy one vm_area from one task to the other. Assumes the page tables 537 * already present in the new task to be cleared in the whole range 538 * covered by this vma. 539 */ 540 541static inline void 542copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, 543 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, 544 unsigned long addr, int *rss) 545{ 546 unsigned long vm_flags = vma->vm_flags; 547 pte_t pte = *src_pte; 548 struct page *page; 549 550 /* pte contains position in swap or file, so copy. */ 551 if (unlikely(!pte_present(pte))) { 552 if (!pte_file(pte)) { 553 swp_entry_t entry = pte_to_swp_entry(pte); 554 555 swap_duplicate(entry); 556 /* make sure dst_mm is on swapoff's mmlist. */ 557 if (unlikely(list_empty(&dst_mm->mmlist))) { 558 spin_lock(&mmlist_lock); 559 if (list_empty(&dst_mm->mmlist)) 560 list_add(&dst_mm->mmlist, 561 &src_mm->mmlist); 562 spin_unlock(&mmlist_lock); 563 } 564 if (is_write_migration_entry(entry) && 565 is_cow_mapping(vm_flags)) { 566 /* 567 * COW mappings require pages in both parent 568 * and child to be set to read. 569 */ 570 make_migration_entry_read(&entry); 571 pte = swp_entry_to_pte(entry); 572 set_pte_at(src_mm, addr, src_pte, pte); 573 } 574 } 575 goto out_set_pte; 576 } 577 578 /* 579 * If it's a COW mapping, write protect it both 580 * in the parent and the child 581 */ 582 if (is_cow_mapping(vm_flags)) { 583 ptep_set_wrprotect(src_mm, addr, src_pte); 584 pte = pte_wrprotect(pte); 585 } 586 587 /* 588 * If it's a shared mapping, mark it clean in 589 * the child 590 */ 591 if (vm_flags & VM_SHARED) 592 pte = pte_mkclean(pte); 593 pte = pte_mkold(pte); 594 595 page = vm_normal_page(vma, addr, pte); 596 if (page) { 597 get_page(page); 598 page_dup_rmap(page, vma, addr); 599 rss[!!PageAnon(page)]++; 600 } 601 602out_set_pte: 603 set_pte_at(dst_mm, addr, dst_pte, pte); 604} 605 606static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 607 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, 608 unsigned long addr, unsigned long end) 609{ 610 pte_t *src_pte, *dst_pte; 611 spinlock_t *src_ptl, *dst_ptl; 612 int progress = 0; 613 int rss[2]; 614 615again: 616 rss[1] = rss[0] = 0; 617 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); 618 if (!dst_pte) 619 return -ENOMEM; 620 src_pte = pte_offset_map_nested(src_pmd, addr); 621 src_ptl = pte_lockptr(src_mm, src_pmd); 622 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 623 arch_enter_lazy_mmu_mode(); 624 625 do { 626 /* 627 * We are holding two locks at this point - either of them 628 * could generate latencies in another task on another CPU. 629 */ 630 if (progress >= 32) { 631 progress = 0; 632 if (need_resched() || 633 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl)) 634 break; 635 } 636 if (pte_none(*src_pte)) { 637 progress++; 638 continue; 639 } 640 copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss); 641 progress += 8; 642 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 643 644 arch_leave_lazy_mmu_mode(); 645 spin_unlock(src_ptl); 646 pte_unmap_nested(src_pte - 1); 647 add_mm_rss(dst_mm, rss[0], rss[1]); 648 pte_unmap_unlock(dst_pte - 1, dst_ptl); 649 cond_resched(); 650 if (addr != end) 651 goto again; 652 return 0; 653} 654 655static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 656 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma, 657 unsigned long addr, unsigned long end) 658{ 659 pmd_t *src_pmd, *dst_pmd; 660 unsigned long next; 661 662 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); 663 if (!dst_pmd) 664 return -ENOMEM; 665 src_pmd = pmd_offset(src_pud, addr); 666 do { 667 next = pmd_addr_end(addr, end); 668 if (pmd_none_or_clear_bad(src_pmd)) 669 continue; 670 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd, 671 vma, addr, next)) 672 return -ENOMEM; 673 } while (dst_pmd++, src_pmd++, addr = next, addr != end); 674 return 0; 675} 676 677static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 678 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma, 679 unsigned long addr, unsigned long end) 680{ 681 pud_t *src_pud, *dst_pud; 682 unsigned long next; 683 684 dst_pud = pud_alloc(dst_mm, dst_pgd, addr); 685 if (!dst_pud) 686 return -ENOMEM; 687 src_pud = pud_offset(src_pgd, addr); 688 do { 689 next = pud_addr_end(addr, end); 690 if (pud_none_or_clear_bad(src_pud)) 691 continue; 692 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud, 693 vma, addr, next)) 694 return -ENOMEM; 695 } while (dst_pud++, src_pud++, addr = next, addr != end); 696 return 0; 697} 698 699int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 700 struct vm_area_struct *vma) 701{ 702 pgd_t *src_pgd, *dst_pgd; 703 unsigned long next; 704 unsigned long addr = vma->vm_start; 705 unsigned long end = vma->vm_end; 706 int ret; 707 708 /* 709 * Don't copy ptes where a page fault will fill them correctly. 710 * Fork becomes much lighter when there are big shared or private 711 * readonly mappings. The tradeoff is that copy_page_range is more 712 * efficient than faulting. 713 */ 714 if (!(vma->vm_flags & (VM_HUGETLB|VM_NONLINEAR|VM_PFNMAP|VM_INSERTPAGE))) { 715 if (!vma->anon_vma) 716 return 0; 717 } 718 719 if (is_vm_hugetlb_page(vma)) 720 return copy_hugetlb_page_range(dst_mm, src_mm, vma); 721 722 if (unlikely(is_pfn_mapping(vma))) { 723 /* 724 * We do not free on error cases below as remove_vma 725 * gets called on error from higher level routine 726 */ 727 ret = track_pfn_vma_copy(vma); 728 if (ret) 729 return ret; 730 } 731 732 /* 733 * We need to invalidate the secondary MMU mappings only when 734 * there could be a permission downgrade on the ptes of the 735 * parent mm. And a permission downgrade will only happen if 736 * is_cow_mapping() returns true. 737 */ 738 if (is_cow_mapping(vma->vm_flags)) 739 mmu_notifier_invalidate_range_start(src_mm, addr, end); 740 741 ret = 0; 742 dst_pgd = pgd_offset(dst_mm, addr); 743 src_pgd = pgd_offset(src_mm, addr); 744 do { 745 next = pgd_addr_end(addr, end); 746 if (pgd_none_or_clear_bad(src_pgd)) 747 continue; 748 if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd, 749 vma, addr, next))) { 750 ret = -ENOMEM; 751 break; 752 } 753 } while (dst_pgd++, src_pgd++, addr = next, addr != end); 754 755 if (is_cow_mapping(vma->vm_flags)) 756 mmu_notifier_invalidate_range_end(src_mm, 757 vma->vm_start, end); 758 return ret; 759} 760 761static unsigned long zap_pte_range(struct mmu_gather *tlb, 762 struct vm_area_struct *vma, pmd_t *pmd, 763 unsigned long addr, unsigned long end, 764 long *zap_work, struct zap_details *details) 765{ 766 struct mm_struct *mm = tlb->mm; 767 pte_t *pte; 768 spinlock_t *ptl; 769 int file_rss = 0; 770 int anon_rss = 0; 771 772 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 773 arch_enter_lazy_mmu_mode(); 774 do { 775 pte_t ptent = *pte; 776 if (pte_none(ptent)) { 777 (*zap_work)--; 778 continue; 779 } 780 781 (*zap_work) -= PAGE_SIZE; 782 783 if (pte_present(ptent)) { 784 struct page *page; 785 786 page = vm_normal_page(vma, addr, ptent); 787 if (unlikely(details) && page) { 788 /* 789 * unmap_shared_mapping_pages() wants to 790 * invalidate cache without truncating: 791 * unmap shared but keep private pages. 792 */ 793 if (details->check_mapping && 794 details->check_mapping != page->mapping) 795 continue; 796 /* 797 * Each page->index must be checked when 798 * invalidating or truncating nonlinear. 799 */ 800 if (details->nonlinear_vma && 801 (page->index < details->first_index || 802 page->index > details->last_index)) 803 continue; 804 } 805 ptent = ptep_get_and_clear_full(mm, addr, pte, 806 tlb->fullmm); 807 tlb_remove_tlb_entry(tlb, pte, addr); 808 if (unlikely(!page)) 809 continue; 810 if (unlikely(details) && details->nonlinear_vma 811 && linear_page_index(details->nonlinear_vma, 812 addr) != page->index) 813 set_pte_at(mm, addr, pte, 814 pgoff_to_pte(page->index)); 815 if (PageAnon(page)) 816 anon_rss--; 817 else { 818 if (pte_dirty(ptent)) 819 set_page_dirty(page); 820 if (pte_young(ptent) && 821 likely(!VM_SequentialReadHint(vma))) 822 mark_page_accessed(page); 823 file_rss--; 824 } 825 page_remove_rmap(page); 826 if (unlikely(page_mapcount(page) < 0)) 827 print_bad_pte(vma, addr, ptent, page); 828 tlb_remove_page(tlb, page); 829 continue; 830 } 831 /* 832 * If details->check_mapping, we leave swap entries; 833 * if details->nonlinear_vma, we leave file entries. 834 */ 835 if (unlikely(details)) 836 continue; 837 if (pte_file(ptent)) { 838 if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) 839 print_bad_pte(vma, addr, ptent, NULL); 840 } else if 841 (unlikely(!free_swap_and_cache(pte_to_swp_entry(ptent)))) 842 print_bad_pte(vma, addr, ptent, NULL); 843 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 844 } while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0)); 845 846 add_mm_rss(mm, file_rss, anon_rss); 847 arch_leave_lazy_mmu_mode(); 848 pte_unmap_unlock(pte - 1, ptl); 849 850 return addr; 851} 852 853static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, 854 struct vm_area_struct *vma, pud_t *pud, 855 unsigned long addr, unsigned long end, 856 long *zap_work, struct zap_details *details) 857{ 858 pmd_t *pmd; 859 unsigned long next; 860 861 pmd = pmd_offset(pud, addr); 862 do { 863 next = pmd_addr_end(addr, end); 864 if (pmd_none_or_clear_bad(pmd)) { 865 (*zap_work)--; 866 continue; 867 } 868 next = zap_pte_range(tlb, vma, pmd, addr, next, 869 zap_work, details); 870 } while (pmd++, addr = next, (addr != end && *zap_work > 0)); 871 872 return addr; 873} 874 875static inline unsigned long zap_pud_range(struct mmu_gather *tlb, 876 struct vm_area_struct *vma, pgd_t *pgd, 877 unsigned long addr, unsigned long end, 878 long *zap_work, struct zap_details *details) 879{ 880 pud_t *pud; 881 unsigned long next; 882 883 pud = pud_offset(pgd, addr); 884 do { 885 next = pud_addr_end(addr, end); 886 if (pud_none_or_clear_bad(pud)) { 887 (*zap_work)--; 888 continue; 889 } 890 next = zap_pmd_range(tlb, vma, pud, addr, next, 891 zap_work, details); 892 } while (pud++, addr = next, (addr != end && *zap_work > 0)); 893 894 return addr; 895} 896 897static unsigned long unmap_page_range(struct mmu_gather *tlb, 898 struct vm_area_struct *vma, 899 unsigned long addr, unsigned long end, 900 long *zap_work, struct zap_details *details) 901{ 902 pgd_t *pgd; 903 unsigned long next; 904 905 if (details && !details->check_mapping && !details->nonlinear_vma) 906 details = NULL; 907 908 BUG_ON(addr >= end); 909 tlb_start_vma(tlb, vma); 910 pgd = pgd_offset(vma->vm_mm, addr); 911 do { 912 next = pgd_addr_end(addr, end); 913 if (pgd_none_or_clear_bad(pgd)) { 914 (*zap_work)--; 915 continue; 916 } 917 next = zap_pud_range(tlb, vma, pgd, addr, next, 918 zap_work, details); 919 } while (pgd++, addr = next, (addr != end && *zap_work > 0)); 920 tlb_end_vma(tlb, vma); 921 922 return addr; 923} 924 925#ifdef CONFIG_PREEMPT 926# define ZAP_BLOCK_SIZE (8 * PAGE_SIZE) 927#else 928/* No preempt: go for improved straight-line efficiency */ 929# define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE) 930#endif 931 932/** 933 * unmap_vmas - unmap a range of memory covered by a list of vma's 934 * @tlbp: address of the caller's struct mmu_gather 935 * @vma: the starting vma 936 * @start_addr: virtual address at which to start unmapping 937 * @end_addr: virtual address at which to end unmapping 938 * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here 939 * @details: details of nonlinear truncation or shared cache invalidation 940 * 941 * Returns the end address of the unmapping (restart addr if interrupted). 942 * 943 * Unmap all pages in the vma list. 944 * 945 * We aim to not hold locks for too long (for scheduling latency reasons). 946 * So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to 947 * return the ending mmu_gather to the caller. 948 * 949 * Only addresses between `start' and `end' will be unmapped. 950 * 951 * The VMA list must be sorted in ascending virtual address order. 952 * 953 * unmap_vmas() assumes that the caller will flush the whole unmapped address 954 * range after unmap_vmas() returns. So the only responsibility here is to 955 * ensure that any thus-far unmapped pages are flushed before unmap_vmas() 956 * drops the lock and schedules. 957 */ 958unsigned long unmap_vmas(struct mmu_gather **tlbp, 959 struct vm_area_struct *vma, unsigned long start_addr, 960 unsigned long end_addr, unsigned long *nr_accounted, 961 struct zap_details *details) 962{ 963 long zap_work = ZAP_BLOCK_SIZE; 964 unsigned long tlb_start = 0; /* For tlb_finish_mmu */ 965 int tlb_start_valid = 0; 966 unsigned long start = start_addr; 967 spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL; 968 int fullmm = (*tlbp)->fullmm; 969 struct mm_struct *mm = vma->vm_mm; 970 971 mmu_notifier_invalidate_range_start(mm, start_addr, end_addr); 972 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) { 973 unsigned long end; 974 975 start = max(vma->vm_start, start_addr); 976 if (start >= vma->vm_end) 977 continue; 978 end = min(vma->vm_end, end_addr); 979 if (end <= vma->vm_start) 980 continue; 981 982 if (vma->vm_flags & VM_ACCOUNT) 983 *nr_accounted += (end - start) >> PAGE_SHIFT; 984 985 if (unlikely(is_pfn_mapping(vma))) 986 untrack_pfn_vma(vma, 0, 0); 987 988 while (start != end) { 989 if (!tlb_start_valid) { 990 tlb_start = start; 991 tlb_start_valid = 1; 992 } 993 994 if (unlikely(is_vm_hugetlb_page(vma))) { 995 /* 996 * It is undesirable to test vma->vm_file as it 997 * should be non-null for valid hugetlb area. 998 * However, vm_file will be NULL in the error 999 * cleanup path of do_mmap_pgoff. When 1000 * hugetlbfs ->mmap method fails, 1001 * do_mmap_pgoff() nullifies vma->vm_file 1002 * before calling this function to clean up. 1003 * Since no pte has actually been setup, it is 1004 * safe to do nothing in this case. 1005 */ 1006 if (vma->vm_file) { 1007 unmap_hugepage_range(vma, start, end, NULL); 1008 zap_work -= (end - start) / 1009 pages_per_huge_page(hstate_vma(vma)); 1010 } 1011 1012 start = end; 1013 } else 1014 start = unmap_page_range(*tlbp, vma, 1015 start, end, &zap_work, details); 1016 1017 if (zap_work > 0) { 1018 BUG_ON(start != end); 1019 break; 1020 } 1021 1022 tlb_finish_mmu(*tlbp, tlb_start, start); 1023 1024 if (need_resched() || 1025 (i_mmap_lock && spin_needbreak(i_mmap_lock))) { 1026 if (i_mmap_lock) { 1027 *tlbp = NULL; 1028 goto out; 1029 } 1030 cond_resched(); 1031 } 1032 1033 *tlbp = tlb_gather_mmu(vma->vm_mm, fullmm); 1034 tlb_start_valid = 0; 1035 zap_work = ZAP_BLOCK_SIZE; 1036 } 1037 } 1038out: 1039 mmu_notifier_invalidate_range_end(mm, start_addr, end_addr); 1040 return start; /* which is now the end (or restart) address */ 1041} 1042 1043/** 1044 * zap_page_range - remove user pages in a given range 1045 * @vma: vm_area_struct holding the applicable pages 1046 * @address: starting address of pages to zap 1047 * @size: number of bytes to zap 1048 * @details: details of nonlinear truncation or shared cache invalidation 1049 */ 1050unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address, 1051 unsigned long size, struct zap_details *details) 1052{ 1053 struct mm_struct *mm = vma->vm_mm; 1054 struct mmu_gather *tlb; 1055 unsigned long end = address + size; 1056 unsigned long nr_accounted = 0; 1057 1058 lru_add_drain(); 1059 tlb = tlb_gather_mmu(mm, 0); 1060 update_hiwater_rss(mm); 1061 end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details); 1062 if (tlb) 1063 tlb_finish_mmu(tlb, address, end); 1064 return end; 1065} 1066 1067/** 1068 * zap_vma_ptes - remove ptes mapping the vma 1069 * @vma: vm_area_struct holding ptes to be zapped 1070 * @address: starting address of pages to zap 1071 * @size: number of bytes to zap 1072 * 1073 * This function only unmaps ptes assigned to VM_PFNMAP vmas. 1074 * 1075 * The entire address range must be fully contained within the vma. 1076 * 1077 * Returns 0 if successful. 1078 */ 1079int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 1080 unsigned long size) 1081{ 1082 if (address < vma->vm_start || address + size > vma->vm_end || 1083 !(vma->vm_flags & VM_PFNMAP)) 1084 return -1; 1085 zap_page_range(vma, address, size, NULL); 1086 return 0; 1087} 1088EXPORT_SYMBOL_GPL(zap_vma_ptes); 1089 1090/* 1091 * Do a quick page-table lookup for a single page. 1092 */ 1093struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 1094 unsigned int flags) 1095{ 1096 pgd_t *pgd; 1097 pud_t *pud; 1098 pmd_t *pmd; 1099 pte_t *ptep, pte; 1100 spinlock_t *ptl; 1101 struct page *page; 1102 struct mm_struct *mm = vma->vm_mm; 1103 1104 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); 1105 if (!IS_ERR(page)) { 1106 BUG_ON(flags & FOLL_GET); 1107 goto out; 1108 } 1109 1110 page = NULL; 1111 pgd = pgd_offset(mm, address); 1112 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 1113 goto no_page_table; 1114 1115 pud = pud_offset(pgd, address); 1116 if (pud_none(*pud)) 1117 goto no_page_table; 1118 if (pud_huge(*pud)) { 1119 BUG_ON(flags & FOLL_GET); 1120 page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE); 1121 goto out; 1122 } 1123 if (unlikely(pud_bad(*pud))) 1124 goto no_page_table; 1125 1126 pmd = pmd_offset(pud, address); 1127 if (pmd_none(*pmd)) 1128 goto no_page_table; 1129 if (pmd_huge(*pmd)) { 1130 BUG_ON(flags & FOLL_GET); 1131 page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE); 1132 goto out; 1133 } 1134 if (unlikely(pmd_bad(*pmd))) 1135 goto no_page_table; 1136 1137 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 1138 1139 pte = *ptep; 1140 if (!pte_present(pte)) 1141 goto no_page; 1142 if ((flags & FOLL_WRITE) && !pte_write(pte)) 1143 goto unlock; 1144 page = vm_normal_page(vma, address, pte); 1145 if (unlikely(!page)) 1146 goto bad_page; 1147 1148 if (flags & FOLL_GET) 1149 get_page(page); 1150 if (flags & FOLL_TOUCH) { 1151 if ((flags & FOLL_WRITE) && 1152 !pte_dirty(pte) && !PageDirty(page)) 1153 set_page_dirty(page); 1154 /* 1155 * pte_mkyoung() would be more correct here, but atomic care 1156 * is needed to avoid losing the dirty bit: it is easier to use 1157 * mark_page_accessed(). 1158 */ 1159 mark_page_accessed(page); 1160 } 1161unlock: 1162 pte_unmap_unlock(ptep, ptl); 1163out: 1164 return page; 1165 1166bad_page: 1167 pte_unmap_unlock(ptep, ptl); 1168 return ERR_PTR(-EFAULT); 1169 1170no_page: 1171 pte_unmap_unlock(ptep, ptl); 1172 if (!pte_none(pte)) 1173 return page; 1174 /* Fall through to ZERO_PAGE handling */ 1175no_page_table: 1176 /* 1177 * When core dumping an enormous anonymous area that nobody 1178 * has touched so far, we don't want to allocate page tables. 1179 */ 1180 if (flags & FOLL_ANON) { 1181 page = ZERO_PAGE(0); 1182 if (flags & FOLL_GET) 1183 get_page(page); 1184 BUG_ON(flags & FOLL_WRITE); 1185 } 1186 return page; 1187} 1188 1189/* Can we do the FOLL_ANON optimization? */ 1190static inline int use_zero_page(struct vm_area_struct *vma) 1191{ 1192 /* 1193 * We don't want to optimize FOLL_ANON for make_pages_present() 1194 * when it tries to page in a VM_LOCKED region. As to VM_SHARED, 1195 * we want to get the page from the page tables to make sure 1196 * that we serialize and update with any other user of that 1197 * mapping. 1198 */ 1199 if (vma->vm_flags & (VM_LOCKED | VM_SHARED)) 1200 return 0; 1201 /* 1202 * And if we have a fault routine, it's not an anonymous region. 1203 */ 1204 return !vma->vm_ops || !vma->vm_ops->fault; 1205} 1206 1207 1208 1209int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1210 unsigned long start, int len, int flags, 1211 struct page **pages, struct vm_area_struct **vmas) 1212{ 1213 int i; 1214 unsigned int vm_flags = 0; 1215 int write = !!(flags & GUP_FLAGS_WRITE); 1216 int force = !!(flags & GUP_FLAGS_FORCE); 1217 int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS); 1218 int ignore_sigkill = !!(flags & GUP_FLAGS_IGNORE_SIGKILL); 1219 1220 if (len <= 0) 1221 return 0; 1222 /* 1223 * Require read or write permissions. 1224 * If 'force' is set, we only require the "MAY" flags. 1225 */ 1226 vm_flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 1227 vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 1228 i = 0; 1229 1230 do { 1231 struct vm_area_struct *vma; 1232 unsigned int foll_flags; 1233 1234 vma = find_extend_vma(mm, start); 1235 if (!vma && in_gate_area(tsk, start)) { 1236 unsigned long pg = start & PAGE_MASK; 1237 struct vm_area_struct *gate_vma = get_gate_vma(tsk); 1238 pgd_t *pgd; 1239 pud_t *pud; 1240 pmd_t *pmd; 1241 pte_t *pte; 1242 1243 /* user gate pages are read-only */ 1244 if (!ignore && write) 1245 return i ? : -EFAULT; 1246 if (pg > TASK_SIZE) 1247 pgd = pgd_offset_k(pg); 1248 else 1249 pgd = pgd_offset_gate(mm, pg); 1250 BUG_ON(pgd_none(*pgd)); 1251 pud = pud_offset(pgd, pg); 1252 BUG_ON(pud_none(*pud)); 1253 pmd = pmd_offset(pud, pg); 1254 if (pmd_none(*pmd)) 1255 return i ? : -EFAULT; 1256 pte = pte_offset_map(pmd, pg); 1257 if (pte_none(*pte)) { 1258 pte_unmap(pte); 1259 return i ? : -EFAULT; 1260 } 1261 if (pages) { 1262 struct page *page = vm_normal_page(gate_vma, start, *pte); 1263 pages[i] = page; 1264 if (page) 1265 get_page(page); 1266 } 1267 pte_unmap(pte); 1268 if (vmas) 1269 vmas[i] = gate_vma; 1270 i++; 1271 start += PAGE_SIZE; 1272 len--; 1273 continue; 1274 } 1275 1276 if (!vma || 1277 (vma->vm_flags & (VM_IO | VM_PFNMAP)) || 1278 (!ignore && !(vm_flags & vma->vm_flags))) 1279 return i ? : -EFAULT; 1280 1281 if (is_vm_hugetlb_page(vma)) { 1282 i = follow_hugetlb_page(mm, vma, pages, vmas, 1283 &start, &len, i, write); 1284 continue; 1285 } 1286 1287 foll_flags = FOLL_TOUCH; 1288 if (pages) 1289 foll_flags |= FOLL_GET; 1290 if (!write && use_zero_page(vma)) 1291 foll_flags |= FOLL_ANON; 1292 1293 do { 1294 struct page *page; 1295 1296 /* 1297 * If we have a pending SIGKILL, don't keep faulting 1298 * pages and potentially allocating memory, unless 1299 * current is handling munlock--e.g., on exit. In 1300 * that case, we are not allocating memory. Rather, 1301 * we're only unlocking already resident/mapped pages. 1302 */ 1303 if (unlikely(!ignore_sigkill && 1304 fatal_signal_pending(current))) 1305 return i ? i : -ERESTARTSYS; 1306 1307 if (write) 1308 foll_flags |= FOLL_WRITE; 1309 1310 cond_resched(); 1311 while (!(page = follow_page(vma, start, foll_flags))) { 1312 int ret; 1313 ret = handle_mm_fault(mm, vma, start, 1314 foll_flags & FOLL_WRITE); 1315 if (ret & VM_FAULT_ERROR) { 1316 if (ret & VM_FAULT_OOM) 1317 return i ? i : -ENOMEM; 1318 else if (ret & VM_FAULT_SIGBUS) 1319 return i ? i : -EFAULT; 1320 BUG(); 1321 } 1322 if (ret & VM_FAULT_MAJOR) 1323 tsk->maj_flt++; 1324 else 1325 tsk->min_flt++; 1326 1327 /* 1328 * The VM_FAULT_WRITE bit tells us that 1329 * do_wp_page has broken COW when necessary, 1330 * even if maybe_mkwrite decided not to set 1331 * pte_write. We can thus safely do subsequent 1332 * page lookups as if they were reads. But only 1333 * do so when looping for pte_write is futile: 1334 * in some cases userspace may also be wanting 1335 * to write to the gotten user page, which a 1336 * read fault here might prevent (a readonly 1337 * page might get reCOWed by userspace write). 1338 */ 1339 if ((ret & VM_FAULT_WRITE) && 1340 !(vma->vm_flags & VM_WRITE)) 1341 foll_flags &= ~FOLL_WRITE; 1342 1343 cond_resched(); 1344 } 1345 if (IS_ERR(page)) 1346 return i ? i : PTR_ERR(page); 1347 if (pages) { 1348 pages[i] = page; 1349 1350 flush_anon_page(vma, page, start); 1351 flush_dcache_page(page); 1352 } 1353 if (vmas) 1354 vmas[i] = vma; 1355 i++; 1356 start += PAGE_SIZE; 1357 len--; 1358 } while (len && start < vma->vm_end); 1359 } while (len); 1360 return i; 1361} 1362 1363/** 1364 * get_user_pages() - pin user pages in memory 1365 * @tsk: task_struct of target task 1366 * @mm: mm_struct of target mm 1367 * @start: starting user address 1368 * @len: number of pages from start to pin 1369 * @write: whether pages will be written to by the caller 1370 * @force: whether to force write access even if user mapping is 1371 * readonly. This will result in the page being COWed even 1372 * in MAP_SHARED mappings. You do not want this. 1373 * @pages: array that receives pointers to the pages pinned. 1374 * Should be at least nr_pages long. Or NULL, if caller 1375 * only intends to ensure the pages are faulted in. 1376 * @vmas: array of pointers to vmas corresponding to each page. 1377 * Or NULL if the caller does not require them. 1378 * 1379 * Returns number of pages pinned. This may be fewer than the number 1380 * requested. If len is 0 or negative, returns 0. If no pages 1381 * were pinned, returns -errno. Each page returned must be released 1382 * with a put_page() call when it is finished with. vmas will only 1383 * remain valid while mmap_sem is held. 1384 * 1385 * Must be called with mmap_sem held for read or write. 1386 * 1387 * get_user_pages walks a process's page tables and takes a reference to 1388 * each struct page that each user address corresponds to at a given 1389 * instant. That is, it takes the page that would be accessed if a user 1390 * thread accesses the given user virtual address at that instant. 1391 * 1392 * This does not guarantee that the page exists in the user mappings when 1393 * get_user_pages returns, and there may even be a completely different 1394 * page there in some cases (eg. if mmapped pagecache has been invalidated 1395 * and subsequently re faulted). However it does guarantee that the page 1396 * won't be freed completely. And mostly callers simply care that the page 1397 * contains data that was valid *at some point in time*. Typically, an IO 1398 * or similar operation cannot guarantee anything stronger anyway because 1399 * locks can't be held over the syscall boundary. 1400 * 1401 * If write=0, the page must not be written to. If the page is written to, 1402 * set_page_dirty (or set_page_dirty_lock, as appropriate) must be called 1403 * after the page is finished with, and before put_page is called. 1404 * 1405 * get_user_pages is typically used for fewer-copy IO operations, to get a 1406 * handle on the memory by some means other than accesses via the user virtual 1407 * addresses. The pages may be submitted for DMA to devices or accessed via 1408 * their kernel linear mapping (via the kmap APIs). Care should be taken to 1409 * use the correct cache flushing APIs. 1410 * 1411 * See also get_user_pages_fast, for performance critical applications. 1412 */ 1413int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1414 unsigned long start, int len, int write, int force, 1415 struct page **pages, struct vm_area_struct **vmas) 1416{ 1417 int flags = 0; 1418 1419 if (write) 1420 flags |= GUP_FLAGS_WRITE; 1421 if (force) 1422 flags |= GUP_FLAGS_FORCE; 1423 1424 return __get_user_pages(tsk, mm, 1425 start, len, flags, 1426 pages, vmas); 1427} 1428 1429EXPORT_SYMBOL(get_user_pages); 1430 1431pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, 1432 spinlock_t **ptl) 1433{ 1434 pgd_t * pgd = pgd_offset(mm, addr); 1435 pud_t * pud = pud_alloc(mm, pgd, addr); 1436 if (pud) { 1437 pmd_t * pmd = pmd_alloc(mm, pud, addr); 1438 if (pmd) 1439 return pte_alloc_map_lock(mm, pmd, addr, ptl); 1440 } 1441 return NULL; 1442} 1443 1444/* 1445 * This is the old fallback for page remapping. 1446 * 1447 * For historical reasons, it only allows reserved pages. Only 1448 * old drivers should use this, and they needed to mark their 1449 * pages reserved for the old functions anyway. 1450 */ 1451static int insert_page(struct vm_area_struct *vma, unsigned long addr, 1452 struct page *page, pgprot_t prot) 1453{ 1454 struct mm_struct *mm = vma->vm_mm; 1455 int retval; 1456 pte_t *pte; 1457 spinlock_t *ptl; 1458 1459 retval = -EINVAL; 1460 if (PageAnon(page)) 1461 goto out; 1462 retval = -ENOMEM; 1463 flush_dcache_page(page); 1464 pte = get_locked_pte(mm, addr, &ptl); 1465 if (!pte) 1466 goto out; 1467 retval = -EBUSY; 1468 if (!pte_none(*pte)) 1469 goto out_unlock; 1470 1471 /* Ok, finally just insert the thing.. */ 1472 get_page(page); 1473 inc_mm_counter(mm, file_rss); 1474 page_add_file_rmap(page); 1475 set_pte_at(mm, addr, pte, mk_pte(page, prot)); 1476 1477 retval = 0; 1478 pte_unmap_unlock(pte, ptl); 1479 return retval; 1480out_unlock: 1481 pte_unmap_unlock(pte, ptl); 1482out: 1483 return retval; 1484} 1485 1486/** 1487 * vm_insert_page - insert single page into user vma 1488 * @vma: user vma to map to 1489 * @addr: target user address of this page 1490 * @page: source kernel page 1491 * 1492 * This allows drivers to insert individual pages they've allocated 1493 * into a user vma. 1494 * 1495 * The page has to be a nice clean _individual_ kernel allocation. 1496 * If you allocate a compound page, you need to have marked it as 1497 * such (__GFP_COMP), or manually just split the page up yourself 1498 * (see split_page()). 1499 * 1500 * NOTE! Traditionally this was done with "remap_pfn_range()" which 1501 * took an arbitrary page protection parameter. This doesn't allow 1502 * that. Your vma protection will have to be set up correctly, which 1503 * means that if you want a shared writable mapping, you'd better 1504 * ask for a shared writable mapping! 1505 * 1506 * The page does not need to be reserved. 1507 */ 1508int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 1509 struct page *page) 1510{ 1511 if (addr < vma->vm_start || addr >= vma->vm_end) 1512 return -EFAULT; 1513 if (!page_count(page)) 1514 return -EINVAL; 1515 vma->vm_flags |= VM_INSERTPAGE; 1516 return insert_page(vma, addr, page, vma->vm_page_prot); 1517} 1518EXPORT_SYMBOL(vm_insert_page); 1519 1520static int insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1521 unsigned long pfn, pgprot_t prot) 1522{ 1523 struct mm_struct *mm = vma->vm_mm; 1524 int retval; 1525 pte_t *pte, entry; 1526 spinlock_t *ptl; 1527 1528 retval = -ENOMEM; 1529 pte = get_locked_pte(mm, addr, &ptl); 1530 if (!pte) 1531 goto out; 1532 retval = -EBUSY; 1533 if (!pte_none(*pte)) 1534 goto out_unlock; 1535 1536 /* Ok, finally just insert the thing.. */ 1537 entry = pte_mkspecial(pfn_pte(pfn, prot)); 1538 set_pte_at(mm, addr, pte, entry); 1539 update_mmu_cache(vma, addr, entry); /* XXX: why not for insert_page? */ 1540 1541 retval = 0; 1542out_unlock: 1543 pte_unmap_unlock(pte, ptl); 1544out: 1545 return retval; 1546} 1547 1548/** 1549 * vm_insert_pfn - insert single pfn into user vma 1550 * @vma: user vma to map to 1551 * @addr: target user address of this page 1552 * @pfn: source kernel pfn 1553 * 1554 * Similar to vm_inert_page, this allows drivers to insert individual pages 1555 * they've allocated into a user vma. Same comments apply. 1556 * 1557 * This function should only be called from a vm_ops->fault handler, and 1558 * in that case the handler should return NULL. 1559 * 1560 * vma cannot be a COW mapping. 1561 * 1562 * As this is called only for pages that do not currently exist, we 1563 * do not need to flush old virtual caches or the TLB. 1564 */ 1565int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1566 unsigned long pfn) 1567{ 1568 int ret; 1569 pgprot_t pgprot = vma->vm_page_prot; 1570 /* 1571 * Technically, architectures with pte_special can avoid all these 1572 * restrictions (same for remap_pfn_range). However we would like 1573 * consistency in testing and feature parity among all, so we should 1574 * try to keep these invariants in place for everybody. 1575 */ 1576 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1577 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1578 (VM_PFNMAP|VM_MIXEDMAP)); 1579 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1580 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); 1581 1582 if (addr < vma->vm_start || addr >= vma->vm_end) 1583 return -EFAULT; 1584 if (track_pfn_vma_new(vma, &pgprot, pfn, PAGE_SIZE)) 1585 return -EINVAL; 1586 1587 ret = insert_pfn(vma, addr, pfn, pgprot); 1588 1589 if (ret) 1590 untrack_pfn_vma(vma, pfn, PAGE_SIZE); 1591 1592 return ret; 1593} 1594EXPORT_SYMBOL(vm_insert_pfn); 1595 1596int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 1597 unsigned long pfn) 1598{ 1599 BUG_ON(!(vma->vm_flags & VM_MIXEDMAP)); 1600 1601 if (addr < vma->vm_start || addr >= vma->vm_end) 1602 return -EFAULT; 1603 1604 /* 1605 * If we don't have pte special, then we have to use the pfn_valid() 1606 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must* 1607 * refcount the page if pfn_valid is true (hence insert_page rather 1608 * than insert_pfn). 1609 */ 1610 if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) { 1611 struct page *page; 1612 1613 page = pfn_to_page(pfn); 1614 return insert_page(vma, addr, page, vma->vm_page_prot); 1615 } 1616 return insert_pfn(vma, addr, pfn, vma->vm_page_prot); 1617} 1618EXPORT_SYMBOL(vm_insert_mixed); 1619 1620/* 1621 * maps a range of physical memory into the requested pages. the old 1622 * mappings are removed. any references to nonexistent pages results 1623 * in null mappings (currently treated as "copy-on-access") 1624 */ 1625static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, 1626 unsigned long addr, unsigned long end, 1627 unsigned long pfn, pgprot_t prot) 1628{ 1629 pte_t *pte; 1630 spinlock_t *ptl; 1631 1632 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); 1633 if (!pte) 1634 return -ENOMEM; 1635 arch_enter_lazy_mmu_mode(); 1636 do { 1637 BUG_ON(!pte_none(*pte)); 1638 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); 1639 pfn++; 1640 } while (pte++, addr += PAGE_SIZE, addr != end); 1641 arch_leave_lazy_mmu_mode(); 1642 pte_unmap_unlock(pte - 1, ptl); 1643 return 0; 1644} 1645 1646static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, 1647 unsigned long addr, unsigned long end, 1648 unsigned long pfn, pgprot_t prot) 1649{ 1650 pmd_t *pmd; 1651 unsigned long next; 1652 1653 pfn -= addr >> PAGE_SHIFT; 1654 pmd = pmd_alloc(mm, pud, addr); 1655 if (!pmd) 1656 return -ENOMEM; 1657 do { 1658 next = pmd_addr_end(addr, end); 1659 if (remap_pte_range(mm, pmd, addr, next, 1660 pfn + (addr >> PAGE_SHIFT), prot)) 1661 return -ENOMEM; 1662 } while (pmd++, addr = next, addr != end); 1663 return 0; 1664} 1665 1666static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd, 1667 unsigned long addr, unsigned long end, 1668 unsigned long pfn, pgprot_t prot) 1669{ 1670 pud_t *pud; 1671 unsigned long next; 1672 1673 pfn -= addr >> PAGE_SHIFT; 1674 pud = pud_alloc(mm, pgd, addr); 1675 if (!pud) 1676 return -ENOMEM; 1677 do { 1678 next = pud_addr_end(addr, end); 1679 if (remap_pmd_range(mm, pud, addr, next, 1680 pfn + (addr >> PAGE_SHIFT), prot)) 1681 return -ENOMEM; 1682 } while (pud++, addr = next, addr != end); 1683 return 0; 1684} 1685 1686/** 1687 * remap_pfn_range - remap kernel memory to userspace 1688 * @vma: user vma to map to 1689 * @addr: target user address to start at 1690 * @pfn: physical address of kernel memory 1691 * @size: size of map area 1692 * @prot: page protection flags for this mapping 1693 * 1694 * Note: this is only safe if the mm semaphore is held when called. 1695 */ 1696int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 1697 unsigned long pfn, unsigned long size, pgprot_t prot) 1698{ 1699 pgd_t *pgd; 1700 unsigned long next; 1701 unsigned long end = addr + PAGE_ALIGN(size); 1702 struct mm_struct *mm = vma->vm_mm; 1703 int err; 1704 1705 /* 1706 * Physically remapped pages are special. Tell the 1707 * rest of the world about it: 1708 * VM_IO tells people not to look at these pages 1709 * (accesses can have side effects). 1710 * VM_RESERVED is specified all over the place, because 1711 * in 2.4 it kept swapout's vma scan off this vma; but 1712 * in 2.6 the LRU scan won't even find its pages, so this 1713 * flag means no more than count its pages in reserved_vm, 1714 * and omit it from core dump, even when VM_IO turned off. 1715 * VM_PFNMAP tells the core MM that the base pages are just 1716 * raw PFN mappings, and do not have a "struct page" associated 1717 * with them. 1718 * 1719 * There's a horrible special case to handle copy-on-write 1720 * behaviour that some programs depend on. We mark the "original" 1721 * un-COW'ed pages by matching them up with "vma->vm_pgoff". 1722 */ 1723 if (addr == vma->vm_start && end == vma->vm_end) { 1724 vma->vm_pgoff = pfn; 1725 vma->vm_flags |= VM_PFN_AT_MMAP; 1726 } else if (is_cow_mapping(vma->vm_flags)) 1727 return -EINVAL; 1728 1729 vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; 1730 1731 err = track_pfn_vma_new(vma, &prot, pfn, PAGE_ALIGN(size)); 1732 if (err) { 1733 /* 1734 * To indicate that track_pfn related cleanup is not 1735 * needed from higher level routine calling unmap_vmas 1736 */ 1737 vma->vm_flags &= ~(VM_IO | VM_RESERVED | VM_PFNMAP); 1738 vma->vm_flags &= ~VM_PFN_AT_MMAP; 1739 return -EINVAL; 1740 } 1741 1742 BUG_ON(addr >= end); 1743 pfn -= addr >> PAGE_SHIFT; 1744 pgd = pgd_offset(mm, addr); 1745 flush_cache_range(vma, addr, end); 1746 do { 1747 next = pgd_addr_end(addr, end); 1748 err = remap_pud_range(mm, pgd, addr, next, 1749 pfn + (addr >> PAGE_SHIFT), prot); 1750 if (err) 1751 break; 1752 } while (pgd++, addr = next, addr != end); 1753 1754 if (err) 1755 untrack_pfn_vma(vma, pfn, PAGE_ALIGN(size)); 1756 1757 return err; 1758} 1759EXPORT_SYMBOL(remap_pfn_range); 1760 1761static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, 1762 unsigned long addr, unsigned long end, 1763 pte_fn_t fn, void *data) 1764{ 1765 pte_t *pte; 1766 int err; 1767 pgtable_t token; 1768 spinlock_t *uninitialized_var(ptl); 1769 1770 pte = (mm == &init_mm) ? 1771 pte_alloc_kernel(pmd, addr) : 1772 pte_alloc_map_lock(mm, pmd, addr, &ptl); 1773 if (!pte) 1774 return -ENOMEM; 1775 1776 BUG_ON(pmd_huge(*pmd)); 1777 1778 arch_enter_lazy_mmu_mode(); 1779 1780 token = pmd_pgtable(*pmd); 1781 1782 do { 1783 err = fn(pte, token, addr, data); 1784 if (err) 1785 break; 1786 } while (pte++, addr += PAGE_SIZE, addr != end); 1787 1788 arch_leave_lazy_mmu_mode(); 1789 1790 if (mm != &init_mm) 1791 pte_unmap_unlock(pte-1, ptl); 1792 return err; 1793} 1794 1795static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, 1796 unsigned long addr, unsigned long end, 1797 pte_fn_t fn, void *data) 1798{ 1799 pmd_t *pmd; 1800 unsigned long next; 1801 int err; 1802 1803 BUG_ON(pud_huge(*pud)); 1804 1805 pmd = pmd_alloc(mm, pud, addr); 1806 if (!pmd) 1807 return -ENOMEM; 1808 do { 1809 next = pmd_addr_end(addr, end); 1810 err = apply_to_pte_range(mm, pmd, addr, next, fn, data); 1811 if (err) 1812 break; 1813 } while (pmd++, addr = next, addr != end); 1814 return err; 1815} 1816 1817static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd, 1818 unsigned long addr, unsigned long end, 1819 pte_fn_t fn, void *data) 1820{ 1821 pud_t *pud; 1822 unsigned long next; 1823 int err; 1824 1825 pud = pud_alloc(mm, pgd, addr); 1826 if (!pud) 1827 return -ENOMEM; 1828 do { 1829 next = pud_addr_end(addr, end); 1830 err = apply_to_pmd_range(mm, pud, addr, next, fn, data); 1831 if (err) 1832 break; 1833 } while (pud++, addr = next, addr != end); 1834 return err; 1835} 1836 1837/* 1838 * Scan a region of virtual memory, filling in page tables as necessary 1839 * and calling a provided function on each leaf page table. 1840 */ 1841int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 1842 unsigned long size, pte_fn_t fn, void *data) 1843{ 1844 pgd_t *pgd; 1845 unsigned long next; 1846 unsigned long start = addr, end = addr + size; 1847 int err; 1848 1849 BUG_ON(addr >= end); 1850 mmu_notifier_invalidate_range_start(mm, start, end); 1851 pgd = pgd_offset(mm, addr); 1852 do { 1853 next = pgd_addr_end(addr, end); 1854 err = apply_to_pud_range(mm, pgd, addr, next, fn, data); 1855 if (err) 1856 break; 1857 } while (pgd++, addr = next, addr != end); 1858 mmu_notifier_invalidate_range_end(mm, start, end); 1859 return err; 1860} 1861EXPORT_SYMBOL_GPL(apply_to_page_range); 1862 1863/* 1864 * handle_pte_fault chooses page fault handler according to an entry 1865 * which was read non-atomically. Before making any commitment, on 1866 * those architectures or configurations (e.g. i386 with PAE) which 1867 * might give a mix of unmatched parts, do_swap_page and do_file_page 1868 * must check under lock before unmapping the pte and proceeding 1869 * (but do_wp_page is only called after already making such a check; 1870 * and do_anonymous_page and do_no_page can safely check later on). 1871 */ 1872static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd, 1873 pte_t *page_table, pte_t orig_pte) 1874{ 1875 int same = 1; 1876#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) 1877 if (sizeof(pte_t) > sizeof(unsigned long)) { 1878 spinlock_t *ptl = pte_lockptr(mm, pmd); 1879 spin_lock(ptl); 1880 same = pte_same(*page_table, orig_pte); 1881 spin_unlock(ptl); 1882 } 1883#endif 1884 pte_unmap(page_table); 1885 return same; 1886} 1887 1888/* 1889 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when 1890 * servicing faults for write access. In the normal case, do always want 1891 * pte_mkwrite. But get_user_pages can cause write faults for mappings 1892 * that do not have writing enabled, when used by access_process_vm. 1893 */ 1894static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 1895{ 1896 if (likely(vma->vm_flags & VM_WRITE)) 1897 pte = pte_mkwrite(pte); 1898 return pte; 1899} 1900 1901static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) 1902{ 1903 /* 1904 * If the source page was a PFN mapping, we don't have 1905 * a "struct page" for it. We do a best-effort copy by 1906 * just copying from the original user address. If that 1907 * fails, we just zero-fill it. Live with it. 1908 */ 1909 if (unlikely(!src)) { 1910 void *kaddr = kmap_atomic(dst, KM_USER0); 1911 void __user *uaddr = (void __user *)(va & PAGE_MASK); 1912 1913 /* 1914 * This really shouldn't fail, because the page is there 1915 * in the page tables. But it might just be unreadable, 1916 * in which case we just give up and fill the result with 1917 * zeroes. 1918 */ 1919 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) 1920 memset(kaddr, 0, PAGE_SIZE); 1921 kunmap_atomic(kaddr, KM_USER0); 1922 flush_dcache_page(dst); 1923 } else 1924 copy_user_highpage(dst, src, va, vma); 1925} 1926 1927/* 1928 * This routine handles present pages, when users try to write 1929 * to a shared page. It is done by copying the page to a new address 1930 * and decrementing the shared-page counter for the old page. 1931 * 1932 * Note that this routine assumes that the protection checks have been 1933 * done by the caller (the low-level page fault routine in most cases). 1934 * Thus we can safely just mark it writable once we've done any necessary 1935 * COW. 1936 * 1937 * We also mark the page dirty at this point even though the page will 1938 * change only once the write actually happens. This avoids a few races, 1939 * and potentially makes it more efficient. 1940 * 1941 * We enter with non-exclusive mmap_sem (to exclude vma changes, 1942 * but allow concurrent faults), with pte both mapped and locked. 1943 * We return with mmap_sem still held, but pte unmapped and unlocked. 1944 */ 1945static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 1946 unsigned long address, pte_t *page_table, pmd_t *pmd, 1947 spinlock_t *ptl, pte_t orig_pte) 1948{ 1949 struct page *old_page, *new_page; 1950 pte_t entry; 1951 int reuse = 0, ret = 0; 1952 int page_mkwrite = 0; 1953 struct page *dirty_page = NULL; 1954 1955 old_page = vm_normal_page(vma, address, orig_pte); 1956 if (!old_page) { 1957 /* 1958 * VM_MIXEDMAP !pfn_valid() case 1959 * 1960 * We should not cow pages in a shared writeable mapping. 1961 * Just mark the pages writable as we can't do any dirty 1962 * accounting on raw pfn maps. 1963 */ 1964 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 1965 (VM_WRITE|VM_SHARED)) 1966 goto reuse; 1967 goto gotten; 1968 } 1969 1970 /* 1971 * Take out anonymous pages first, anonymous shared vmas are 1972 * not dirty accountable. 1973 */ 1974 if (PageAnon(old_page)) { 1975 if (!trylock_page(old_page)) { 1976 page_cache_get(old_page); 1977 pte_unmap_unlock(page_table, ptl); 1978 lock_page(old_page); 1979 page_table = pte_offset_map_lock(mm, pmd, address, 1980 &ptl); 1981 if (!pte_same(*page_table, orig_pte)) { 1982 unlock_page(old_page); 1983 page_cache_release(old_page); 1984 goto unlock; 1985 } 1986 page_cache_release(old_page); 1987 } 1988 reuse = reuse_swap_page(old_page); 1989 unlock_page(old_page); 1990 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 1991 (VM_WRITE|VM_SHARED))) { 1992 /* 1993 * Only catch write-faults on shared writable pages, 1994 * read-only shared pages can get COWed by 1995 * get_user_pages(.write=1, .force=1). 1996 */ 1997 if (vma->vm_ops && vma->vm_ops->page_mkwrite) { 1998 struct vm_fault vmf; 1999 int tmp; 2000 2001 vmf.virtual_address = (void __user *)(address & 2002 PAGE_MASK); 2003 vmf.pgoff = old_page->index; 2004 vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 2005 vmf.page = old_page; 2006 2007 /* 2008 * Notify the address space that the page is about to 2009 * become writable so that it can prohibit this or wait 2010 * for the page to get into an appropriate state. 2011 * 2012 * We do this without the lock held, so that it can 2013 * sleep if it needs to. 2014 */ 2015 page_cache_get(old_page); 2016 pte_unmap_unlock(page_table, ptl); 2017 2018 tmp = vma->vm_ops->page_mkwrite(vma, &vmf); 2019 if (unlikely(tmp & 2020 (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 2021 ret = tmp; 2022 goto unwritable_page; 2023 } 2024 if (unlikely(!(tmp & VM_FAULT_LOCKED))) { 2025 lock_page(old_page); 2026 if (!old_page->mapping) { 2027 ret = 0; /* retry the fault */ 2028 unlock_page(old_page); 2029 goto unwritable_page; 2030 } 2031 } else 2032 VM_BUG_ON(!PageLocked(old_page)); 2033 2034 /* 2035 * Since we dropped the lock we need to revalidate 2036 * the PTE as someone else may have changed it. If 2037 * they did, we just return, as we can count on the 2038 * MMU to tell us if they didn't also make it writable. 2039 */ 2040 page_table = pte_offset_map_lock(mm, pmd, address, 2041 &ptl); 2042 if (!pte_same(*page_table, orig_pte)) { 2043 unlock_page(old_page); 2044 page_cache_release(old_page); 2045 goto unlock; 2046 } 2047 2048 page_mkwrite = 1; 2049 } 2050 dirty_page = old_page; 2051 get_page(dirty_page); 2052 reuse = 1; 2053 } 2054 2055 if (reuse) { 2056reuse: 2057 flush_cache_page(vma, address, pte_pfn(orig_pte)); 2058 entry = pte_mkyoung(orig_pte); 2059 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2060 if (ptep_set_access_flags(vma, address, page_table, entry,1)) 2061 update_mmu_cache(vma, address, entry); 2062 ret |= VM_FAULT_WRITE; 2063 goto unlock; 2064 } 2065 2066 /* 2067 * Ok, we need to copy. Oh, well.. 2068 */ 2069 page_cache_get(old_page); 2070gotten: 2071 pte_unmap_unlock(page_table, ptl); 2072 2073 if (unlikely(anon_vma_prepare(vma))) 2074 goto oom; 2075 VM_BUG_ON(old_page == ZERO_PAGE(0)); 2076 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); 2077 if (!new_page) 2078 goto oom; 2079 /* 2080 * Don't let another task, with possibly unlocked vma, 2081 * keep the mlocked page. 2082 */ 2083 if ((vma->vm_flags & VM_LOCKED) && old_page) { 2084 lock_page(old_page); /* for LRU manipulation */ 2085 clear_page_mlock(old_page); 2086 unlock_page(old_page); 2087 } 2088 cow_user_page(new_page, old_page, address, vma); 2089 __SetPageUptodate(new_page); 2090 2091 if (mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL)) 2092 goto oom_free_new; 2093 2094 /* 2095 * Re-check the pte - we dropped the lock 2096 */ 2097 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2098 if (likely(pte_same(*page_table, orig_pte))) { 2099 if (old_page) { 2100 if (!PageAnon(old_page)) { 2101 dec_mm_counter(mm, file_rss); 2102 inc_mm_counter(mm, anon_rss); 2103 } 2104 } else 2105 inc_mm_counter(mm, anon_rss); 2106 flush_cache_page(vma, address, pte_pfn(orig_pte)); 2107 entry = mk_pte(new_page, vma->vm_page_prot); 2108 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2109 /* 2110 * Clear the pte entry and flush it first, before updating the 2111 * pte with the new entry. This will avoid a race condition 2112 * seen in the presence of one thread doing SMC and another 2113 * thread doing COW. 2114 */ 2115 ptep_clear_flush_notify(vma, address, page_table); 2116 page_add_new_anon_rmap(new_page, vma, address); 2117 set_pte_at(mm, address, page_table, entry); 2118 update_mmu_cache(vma, address, entry); 2119 if (old_page) { 2120 /* 2121 * Only after switching the pte to the new page may 2122 * we remove the mapcount here. Otherwise another 2123 * process may come and find the rmap count decremented 2124 * before the pte is switched to the new page, and 2125 * "reuse" the old page writing into it while our pte 2126 * here still points into it and can be read by other 2127 * threads. 2128 * 2129 * The critical issue is to order this 2130 * page_remove_rmap with the ptp_clear_flush above. 2131 * Those stores are ordered by (if nothing else,) 2132 * the barrier present in the atomic_add_negative 2133 * in page_remove_rmap. 2134 * 2135 * Then the TLB flush in ptep_clear_flush ensures that 2136 * no process can access the old page before the 2137 * decremented mapcount is visible. And the old page 2138 * cannot be reused until after the decremented 2139 * mapcount is visible. So transitively, TLBs to 2140 * old page will be flushed before it can be reused. 2141 */ 2142 page_remove_rmap(old_page); 2143 } 2144 2145 /* Free the old page.. */ 2146 new_page = old_page; 2147 ret |= VM_FAULT_WRITE; 2148 } else 2149 mem_cgroup_uncharge_page(new_page); 2150 2151 if (new_page) 2152 page_cache_release(new_page); 2153 if (old_page) 2154 page_cache_release(old_page); 2155unlock: 2156 pte_unmap_unlock(page_table, ptl); 2157 if (dirty_page) { 2158 /* 2159 * Yes, Virginia, this is actually required to prevent a race 2160 * with clear_page_dirty_for_io() from clearing the page dirty 2161 * bit after it clear all dirty ptes, but before a racing 2162 * do_wp_page installs a dirty pte. 2163 * 2164 * do_no_page is protected similarly. 2165 */ 2166 if (!page_mkwrite) { 2167 wait_on_page_locked(dirty_page); 2168 set_page_dirty_balance(dirty_page, page_mkwrite); 2169 } 2170 put_page(dirty_page); 2171 if (page_mkwrite) { 2172 struct address_space *mapping = dirty_page->mapping; 2173 2174 set_page_dirty(dirty_page); 2175 unlock_page(dirty_page); 2176 page_cache_release(dirty_page); 2177 if (mapping) { 2178 /* 2179 * Some device drivers do not set page.mapping 2180 * but still dirty their pages 2181 */ 2182 balance_dirty_pages_ratelimited(mapping); 2183 } 2184 } 2185 2186 /* file_update_time outside page_lock */ 2187 if (vma->vm_file) 2188 file_update_time(vma->vm_file); 2189 } 2190 return ret; 2191oom_free_new: 2192 page_cache_release(new_page); 2193oom: 2194 if (old_page) { 2195 if (page_mkwrite) { 2196 unlock_page(old_page); 2197 page_cache_release(old_page); 2198 } 2199 page_cache_release(old_page); 2200 } 2201 return VM_FAULT_OOM; 2202 2203unwritable_page: 2204 page_cache_release(old_page); 2205 return ret; 2206} 2207 2208/* 2209 * Helper functions for unmap_mapping_range(). 2210 * 2211 * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __ 2212 * 2213 * We have to restart searching the prio_tree whenever we drop the lock, 2214 * since the iterator is only valid while the lock is held, and anyway 2215 * a later vma might be split and reinserted earlier while lock dropped. 2216 * 2217 * The list of nonlinear vmas could be handled more efficiently, using 2218 * a placeholder, but handle it in the same way until a need is shown. 2219 * It is important to search the prio_tree before nonlinear list: a vma 2220 * may become nonlinear and be shifted from prio_tree to nonlinear list 2221 * while the lock is dropped; but never shifted from list to prio_tree. 2222 * 2223 * In order to make forward progress despite restarting the search, 2224 * vm_truncate_count is used to mark a vma as now dealt with, so we can 2225 * quickly skip it next time around. Since the prio_tree search only 2226 * shows us those vmas affected by unmapping the range in question, we 2227 * can't efficiently keep all vmas in step with mapping->truncate_count: 2228 * so instead reset them all whenever it wraps back to 0 (then go to 1). 2229 * mapping->truncate_count and vma->vm_truncate_count are protected by 2230 * i_mmap_lock. 2231 * 2232 * In order to make forward progress despite repeatedly restarting some 2233 * large vma, note the restart_addr from unmap_vmas when it breaks out: 2234 * and restart from that address when we reach that vma again. It might 2235 * have been split or merged, shrunk or extended, but never shifted: so 2236 * restart_addr remains valid so long as it remains in the vma's range. 2237 * unmap_mapping_range forces truncate_count to leap over page-aligned 2238 * values so we can save vma's restart_addr in its truncate_count field. 2239 */ 2240#define is_restart_addr(truncate_count) (!((truncate_count) & ~PAGE_MASK)) 2241 2242static void reset_vma_truncate_counts(struct address_space *mapping) 2243{ 2244 struct vm_area_struct *vma; 2245 struct prio_tree_iter iter; 2246 2247 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX) 2248 vma->vm_truncate_count = 0; 2249 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list) 2250 vma->vm_truncate_count = 0; 2251} 2252 2253static int unmap_mapping_range_vma(struct vm_area_struct *vma, 2254 unsigned long start_addr, unsigned long end_addr, 2255 struct zap_details *details) 2256{ 2257 unsigned long restart_addr; 2258 int need_break; 2259 2260 /* 2261 * files that support invalidating or truncating portions of the 2262 * file from under mmaped areas must have their ->fault function 2263 * return a locked page (and set VM_FAULT_LOCKED in the return). 2264 * This provides synchronisation against concurrent unmapping here. 2265 */ 2266 2267again: 2268 restart_addr = vma->vm_truncate_count; 2269 if (is_restart_addr(restart_addr) && start_addr < restart_addr) { 2270 start_addr = restart_addr; 2271 if (start_addr >= end_addr) { 2272 /* Top of vma has been split off since last time */ 2273 vma->vm_truncate_count = details->truncate_count; 2274 return 0; 2275 } 2276 } 2277 2278 restart_addr = zap_page_range(vma, start_addr, 2279 end_addr - start_addr, details); 2280 need_break = need_resched() || spin_needbreak(details->i_mmap_lock); 2281 2282 if (restart_addr >= end_addr) { 2283 /* We have now completed this vma: mark it so */ 2284 vma->vm_truncate_count = details->truncate_count; 2285 if (!need_break) 2286 return 0; 2287 } else { 2288 /* Note restart_addr in vma's truncate_count field */ 2289 vma->vm_truncate_count = restart_addr; 2290 if (!need_break) 2291 goto again; 2292 } 2293 2294 spin_unlock(details->i_mmap_lock); 2295 cond_resched(); 2296 spin_lock(details->i_mmap_lock); 2297 return -EINTR; 2298} 2299 2300static inline void unmap_mapping_range_tree(struct prio_tree_root *root, 2301 struct zap_details *details) 2302{ 2303 struct vm_area_struct *vma; 2304 struct prio_tree_iter iter; 2305 pgoff_t vba, vea, zba, zea; 2306 2307restart: 2308 vma_prio_tree_foreach(vma, &iter, root, 2309 details->first_index, details->last_index) { 2310 /* Skip quickly over those we have already dealt with */ 2311 if (vma->vm_truncate_count == details->truncate_count) 2312 continue; 2313 2314 vba = vma->vm_pgoff; 2315 vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1; 2316 /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */ 2317 zba = details->first_index; 2318 if (zba < vba) 2319 zba = vba; 2320 zea = details->last_index; 2321 if (zea > vea) 2322 zea = vea; 2323 2324 if (unmap_mapping_range_vma(vma, 2325 ((zba - vba) << PAGE_SHIFT) + vma->vm_start, 2326 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, 2327 details) < 0) 2328 goto restart; 2329 } 2330} 2331 2332static inline void unmap_mapping_range_list(struct list_head *head, 2333 struct zap_details *details) 2334{ 2335 struct vm_area_struct *vma; 2336 2337 /* 2338 * In nonlinear VMAs there is no correspondence between virtual address 2339 * offset and file offset. So we must perform an exhaustive search 2340 * across *all* the pages in each nonlinear VMA, not just the pages 2341 * whose virtual address lies outside the file truncation point. 2342 */ 2343restart: 2344 list_for_each_entry(vma, head, shared.vm_set.list) { 2345 /* Skip quickly over those we have already dealt with */ 2346 if (vma->vm_truncate_count == details->truncate_count) 2347 continue; 2348 details->nonlinear_vma = vma; 2349 if (unmap_mapping_range_vma(vma, vma->vm_start, 2350 vma->vm_end, details) < 0) 2351 goto restart; 2352 } 2353} 2354 2355/** 2356 * unmap_mapping_range - unmap the portion of all mmaps in the specified address_space corresponding to the specified page range in the underlying file. 2357 * @mapping: the address space containing mmaps to be unmapped. 2358 * @holebegin: byte in first page to unmap, relative to the start of 2359 * the underlying file. This will be rounded down to a PAGE_SIZE 2360 * boundary. Note that this is different from vmtruncate(), which 2361 * must keep the partial page. In contrast, we must get rid of 2362 * partial pages. 2363 * @holelen: size of prospective hole in bytes. This will be rounded 2364 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the 2365 * end of the file. 2366 * @even_cows: 1 when truncating a file, unmap even private COWed pages; 2367 * but 0 when invalidating pagecache, don't throw away private data. 2368 */ 2369void unmap_mapping_range(struct address_space *mapping, 2370 loff_t const holebegin, loff_t const holelen, int even_cows) 2371{ 2372 struct zap_details details; 2373 pgoff_t hba = holebegin >> PAGE_SHIFT; 2374 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2375 2376 /* Check for overflow. */ 2377 if (sizeof(holelen) > sizeof(hlen)) { 2378 long long holeend = 2379 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2380 if (holeend & ~(long long)ULONG_MAX) 2381 hlen = ULONG_MAX - hba + 1; 2382 } 2383 2384 details.check_mapping = even_cows? NULL: mapping; 2385 details.nonlinear_vma = NULL; 2386 details.first_index = hba; 2387 details.last_index = hba + hlen - 1; 2388 if (details.last_index < details.first_index) 2389 details.last_index = ULONG_MAX; 2390 details.i_mmap_lock = &mapping->i_mmap_lock; 2391 2392 spin_lock(&mapping->i_mmap_lock); 2393 2394 /* Protect against endless unmapping loops */ 2395 mapping->truncate_count++; 2396 if (unlikely(is_restart_addr(mapping->truncate_count))) { 2397 if (mapping->truncate_count == 0) 2398 reset_vma_truncate_counts(mapping); 2399 mapping->truncate_count++; 2400 } 2401 details.truncate_count = mapping->truncate_count; 2402 2403 if (unlikely(!prio_tree_empty(&mapping->i_mmap))) 2404 unmap_mapping_range_tree(&mapping->i_mmap, &details); 2405 if (unlikely(!list_empty(&mapping->i_mmap_nonlinear))) 2406 unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details); 2407 spin_unlock(&mapping->i_mmap_lock); 2408} 2409EXPORT_SYMBOL(unmap_mapping_range); 2410 2411/** 2412 * vmtruncate - unmap mappings "freed" by truncate() syscall 2413 * @inode: inode of the file used 2414 * @offset: file offset to start truncating 2415 * 2416 * NOTE! We have to be ready to update the memory sharing 2417 * between the file and the memory map for a potential last 2418 * incomplete page. Ugly, but necessary. 2419 */ 2420int vmtruncate(struct inode * inode, loff_t offset) 2421{ 2422 if (inode->i_size < offset) { 2423 unsigned long limit; 2424 2425 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; 2426 if (limit != RLIM_INFINITY && offset > limit) 2427 goto out_sig; 2428 if (offset > inode->i_sb->s_maxbytes) 2429 goto out_big; 2430 i_size_write(inode, offset); 2431 } else { 2432 struct address_space *mapping = inode->i_mapping; 2433 2434 /* 2435 * truncation of in-use swapfiles is disallowed - it would 2436 * cause subsequent swapout to scribble on the now-freed 2437 * blocks. 2438 */ 2439 if (IS_SWAPFILE(inode)) 2440 return -ETXTBSY; 2441 i_size_write(inode, offset); 2442 2443 /* 2444 * unmap_mapping_range is called twice, first simply for 2445 * efficiency so that truncate_inode_pages does fewer 2446 * single-page unmaps. However after this first call, and 2447 * before truncate_inode_pages finishes, it is possible for 2448 * private pages to be COWed, which remain after 2449 * truncate_inode_pages finishes, hence the second 2450 * unmap_mapping_range call must be made for correctness. 2451 */ 2452 unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1); 2453 truncate_inode_pages(mapping, offset); 2454 unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1); 2455 } 2456 2457 if (inode->i_op->truncate) 2458 inode->i_op->truncate(inode); 2459 return 0; 2460 2461out_sig: 2462 send_sig(SIGXFSZ, current, 0); 2463out_big: 2464 return -EFBIG; 2465} 2466EXPORT_SYMBOL(vmtruncate); 2467 2468int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end) 2469{ 2470 struct address_space *mapping = inode->i_mapping; 2471 2472 /* 2473 * If the underlying filesystem is not going to provide 2474 * a way to truncate a range of blocks (punch a hole) - 2475 * we should return failure right now. 2476 */ 2477 if (!inode->i_op->truncate_range) 2478 return -ENOSYS; 2479 2480 mutex_lock(&inode->i_mutex); 2481 down_write(&inode->i_alloc_sem); 2482 unmap_mapping_range(mapping, offset, (end - offset), 1); 2483 truncate_inode_pages_range(mapping, offset, end); 2484 unmap_mapping_range(mapping, offset, (end - offset), 1); 2485 inode->i_op->truncate_range(inode, offset, end); 2486 up_write(&inode->i_alloc_sem); 2487 mutex_unlock(&inode->i_mutex); 2488 2489 return 0; 2490} 2491 2492/* 2493 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2494 * but allow concurrent faults), and pte mapped but not yet locked. 2495 * We return with mmap_sem still held, but pte unmapped and unlocked. 2496 */ 2497static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, 2498 unsigned long address, pte_t *page_table, pmd_t *pmd, 2499 int write_access, pte_t orig_pte) 2500{ 2501 spinlock_t *ptl; 2502 struct page *page; 2503 swp_entry_t entry; 2504 pte_t pte; 2505 struct mem_cgroup *ptr = NULL; 2506 int ret = 0; 2507 2508 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 2509 goto out; 2510 2511 entry = pte_to_swp_entry(orig_pte); 2512 if (is_migration_entry(entry)) { 2513 migration_entry_wait(mm, pmd, address); 2514 goto out; 2515 } 2516 delayacct_set_flag(DELAYACCT_PF_SWAPIN); 2517 page = lookup_swap_cache(entry); 2518 if (!page) { 2519 grab_swap_token(); /* Contend for token _before_ read-in */ 2520 page = swapin_readahead(entry, 2521 GFP_HIGHUSER_MOVABLE, vma, address); 2522 if (!page) { 2523 /* 2524 * Back out if somebody else faulted in this pte 2525 * while we released the pte lock. 2526 */ 2527 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2528 if (likely(pte_same(*page_table, orig_pte))) 2529 ret = VM_FAULT_OOM; 2530 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2531 goto unlock; 2532 } 2533 2534 /* Had to read the page from swap area: Major fault */ 2535 ret = VM_FAULT_MAJOR; 2536 count_vm_event(PGMAJFAULT); 2537 } 2538 2539 lock_page(page); 2540 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2541 2542 if (mem_cgroup_try_charge_swapin(mm, page, GFP_KERNEL, &ptr)) { 2543 ret = VM_FAULT_OOM; 2544 goto out_page; 2545 } 2546 2547 /* 2548 * Back out if somebody else already faulted in this pte. 2549 */ 2550 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2551 if (unlikely(!pte_same(*page_table, orig_pte))) 2552 goto out_nomap; 2553 2554 if (unlikely(!PageUptodate(page))) { 2555 ret = VM_FAULT_SIGBUS; 2556 goto out_nomap; 2557 } 2558 2559 /* 2560 * The page isn't present yet, go ahead with the fault. 2561 * 2562 * Be careful about the sequence of operations here. 2563 * To get its accounting right, reuse_swap_page() must be called 2564 * while the page is counted on swap but not yet in mapcount i.e. 2565 * before page_add_anon_rmap() and swap_free(); try_to_free_swap() 2566 * must be called after the swap_free(), or it will never succeed. 2567 * Because delete_from_swap_page() may be called by reuse_swap_page(), 2568 * mem_cgroup_commit_charge_swapin() may not be able to find swp_entry 2569 * in page->private. In this case, a record in swap_cgroup is silently 2570 * discarded at swap_free(). 2571 */ 2572 2573 inc_mm_counter(mm, anon_rss); 2574 pte = mk_pte(page, vma->vm_page_prot); 2575 if (write_access && reuse_swap_page(page)) { 2576 pte = maybe_mkwrite(pte_mkdirty(pte), vma); 2577 write_access = 0; 2578 } 2579 flush_icache_page(vma, page); 2580 set_pte_at(mm, address, page_table, pte); 2581 page_add_anon_rmap(page, vma, address); 2582 /* It's better to call commit-charge after rmap is established */ 2583 mem_cgroup_commit_charge_swapin(page, ptr); 2584 2585 swap_free(entry); 2586 if (vm_swap_full() || (vma->vm_flags & VM_LOCKED) || PageMlocked(page)) 2587 try_to_free_swap(page); 2588 unlock_page(page); 2589 2590 if (write_access) { 2591 ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte); 2592 if (ret & VM_FAULT_ERROR) 2593 ret &= VM_FAULT_ERROR; 2594 goto out; 2595 } 2596 2597 /* No need to invalidate - it was non-present before */ 2598 update_mmu_cache(vma, address, pte); 2599unlock: 2600 pte_unmap_unlock(page_table, ptl); 2601out: 2602 return ret; 2603out_nomap: 2604 mem_cgroup_cancel_charge_swapin(ptr); 2605 pte_unmap_unlock(page_table, ptl); 2606out_page: 2607 unlock_page(page); 2608 page_cache_release(page); 2609 return ret; 2610} 2611 2612/* 2613 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2614 * but allow concurrent faults), and pte mapped but not yet locked. 2615 * We return with mmap_sem still held, but pte unmapped and unlocked. 2616 */ 2617static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 2618 unsigned long address, pte_t *page_table, pmd_t *pmd, 2619 int write_access) 2620{ 2621 struct page *page; 2622 spinlock_t *ptl; 2623 pte_t entry; 2624 2625 /* Allocate our own private page. */ 2626 pte_unmap(page_table); 2627 2628 if (unlikely(anon_vma_prepare(vma))) 2629 goto oom; 2630 page = alloc_zeroed_user_highpage_movable(vma, address); 2631 if (!page) 2632 goto oom; 2633 __SetPageUptodate(page); 2634 2635 if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) 2636 goto oom_free_page; 2637 2638 entry = mk_pte(page, vma->vm_page_prot); 2639 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2640 2641 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2642 if (!pte_none(*page_table)) 2643 goto release; 2644 inc_mm_counter(mm, anon_rss); 2645 page_add_new_anon_rmap(page, vma, address); 2646 set_pte_at(mm, address, page_table, entry); 2647 2648 /* No need to invalidate - it was non-present before */ 2649 update_mmu_cache(vma, address, entry); 2650unlock: 2651 pte_unmap_unlock(page_table, ptl); 2652 return 0; 2653release: 2654 mem_cgroup_uncharge_page(page); 2655 page_cache_release(page); 2656 goto unlock; 2657oom_free_page: 2658 page_cache_release(page); 2659oom: 2660 return VM_FAULT_OOM; 2661} 2662 2663/* 2664 * __do_fault() tries to create a new page mapping. It aggressively 2665 * tries to share with existing pages, but makes a separate copy if 2666 * the FAULT_FLAG_WRITE is set in the flags parameter in order to avoid 2667 * the next page fault. 2668 * 2669 * As this is called only for pages that do not currently exist, we 2670 * do not need to flush old virtual caches or the TLB. 2671 * 2672 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2673 * but allow concurrent faults), and pte neither mapped nor locked. 2674 * We return with mmap_sem still held, but pte unmapped and unlocked. 2675 */ 2676static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, 2677 unsigned long address, pmd_t *pmd, 2678 pgoff_t pgoff, unsigned int flags, pte_t orig_pte) 2679{ 2680 pte_t *page_table; 2681 spinlock_t *ptl; 2682 struct page *page; 2683 pte_t entry; 2684 int anon = 0; 2685 int charged = 0; 2686 struct page *dirty_page = NULL; 2687 struct vm_fault vmf; 2688 int ret; 2689 int page_mkwrite = 0; 2690 2691 vmf.virtual_address = (void __user *)(address & PAGE_MASK); 2692 vmf.pgoff = pgoff; 2693 vmf.flags = flags; 2694 vmf.page = NULL; 2695 2696 ret = vma->vm_ops->fault(vma, &vmf); 2697 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) 2698 return ret; 2699 2700 /* 2701 * For consistency in subsequent calls, make the faulted page always 2702 * locked. 2703 */ 2704 if (unlikely(!(ret & VM_FAULT_LOCKED))) 2705 lock_page(vmf.page); 2706 else 2707 VM_BUG_ON(!PageLocked(vmf.page)); 2708 2709 /* 2710 * Should we do an early C-O-W break? 2711 */ 2712 page = vmf.page; 2713 if (flags & FAULT_FLAG_WRITE) { 2714 if (!(vma->vm_flags & VM_SHARED)) { 2715 anon = 1; 2716 if (unlikely(anon_vma_prepare(vma))) { 2717 ret = VM_FAULT_OOM; 2718 goto out; 2719 } 2720 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, 2721 vma, address); 2722 if (!page) { 2723 ret = VM_FAULT_OOM; 2724 goto out; 2725 } 2726 if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) { 2727 ret = VM_FAULT_OOM; 2728 page_cache_release(page); 2729 goto out; 2730 } 2731 charged = 1; 2732 /* 2733 * Don't let another task, with possibly unlocked vma, 2734 * keep the mlocked page. 2735 */ 2736 if (vma->vm_flags & VM_LOCKED) 2737 clear_page_mlock(vmf.page); 2738 copy_user_highpage(page, vmf.page, address, vma); 2739 __SetPageUptodate(page); 2740 } else { 2741 /* 2742 * If the page will be shareable, see if the backing 2743 * address space wants to know that the page is about 2744 * to become writable 2745 */ 2746 if (vma->vm_ops->page_mkwrite) { 2747 int tmp; 2748 2749 unlock_page(page); 2750 vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 2751 tmp = vma->vm_ops->page_mkwrite(vma, &vmf); 2752 if (unlikely(tmp & 2753 (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 2754 ret = tmp; 2755 goto unwritable_page; 2756 } 2757 if (unlikely(!(tmp & VM_FAULT_LOCKED))) { 2758 lock_page(page); 2759 if (!page->mapping) { 2760 ret = 0; /* retry the fault */ 2761 unlock_page(page); 2762 goto unwritable_page; 2763 } 2764 } else 2765 VM_BUG_ON(!PageLocked(page)); 2766 page_mkwrite = 1; 2767 } 2768 } 2769 2770 } 2771 2772 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2773 2774 /* 2775 * This silly early PAGE_DIRTY setting removes a race 2776 * due to the bad i386 page protection. But it's valid 2777 * for other architectures too. 2778 * 2779 * Note that if write_access is true, we either now have 2780 * an exclusive copy of the page, or this is a shared mapping, 2781 * so we can make it writable and dirty to avoid having to 2782 * handle that later. 2783 */ 2784 /* Only go through if we didn't race with anybody else... */ 2785 if (likely(pte_same(*page_table, orig_pte))) { 2786 flush_icache_page(vma, page); 2787 entry = mk_pte(page, vma->vm_page_prot); 2788 if (flags & FAULT_FLAG_WRITE) 2789 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2790 if (anon) { 2791 inc_mm_counter(mm, anon_rss); 2792 page_add_new_anon_rmap(page, vma, address); 2793 } else { 2794 inc_mm_counter(mm, file_rss); 2795 page_add_file_rmap(page); 2796 if (flags & FAULT_FLAG_WRITE) { 2797 dirty_page = page; 2798 get_page(dirty_page); 2799 } 2800 } 2801 set_pte_at(mm, address, page_table, entry); 2802 2803 /* no need to invalidate: a not-present page won't be cached */ 2804 update_mmu_cache(vma, address, entry); 2805 } else { 2806 if (charged) 2807 mem_cgroup_uncharge_page(page); 2808 if (anon) 2809 page_cache_release(page); 2810 else 2811 anon = 1; /* no anon but release faulted_page */ 2812 } 2813 2814 pte_unmap_unlock(page_table, ptl); 2815 2816out: 2817 if (dirty_page) { 2818 struct address_space *mapping = page->mapping; 2819 2820 if (set_page_dirty(dirty_page)) 2821 page_mkwrite = 1; 2822 unlock_page(dirty_page); 2823 put_page(dirty_page); 2824 if (page_mkwrite && mapping) { 2825 /* 2826 * Some device drivers do not set page.mapping but still 2827 * dirty their pages 2828 */ 2829 balance_dirty_pages_ratelimited(mapping); 2830 } 2831 2832 /* file_update_time outside page_lock */ 2833 if (vma->vm_file) 2834 file_update_time(vma->vm_file); 2835 } else { 2836 unlock_page(vmf.page); 2837 if (anon) 2838 page_cache_release(vmf.page); 2839 } 2840 2841 return ret; 2842 2843unwritable_page: 2844 page_cache_release(page); 2845 return ret; 2846} 2847 2848static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 2849 unsigned long address, pte_t *page_table, pmd_t *pmd, 2850 int write_access, pte_t orig_pte) 2851{ 2852 pgoff_t pgoff = (((address & PAGE_MASK) 2853 - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2854 unsigned int flags = (write_access ? FAULT_FLAG_WRITE : 0); 2855 2856 pte_unmap(page_table); 2857 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 2858} 2859 2860/* 2861 * Fault of a previously existing named mapping. Repopulate the pte 2862 * from the encoded file_pte if possible. This enables swappable 2863 * nonlinear vmas. 2864 * 2865 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2866 * but allow concurrent faults), and pte mapped but not yet locked. 2867 * We return with mmap_sem still held, but pte unmapped and unlocked. 2868 */ 2869static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 2870 unsigned long address, pte_t *page_table, pmd_t *pmd, 2871 int write_access, pte_t orig_pte) 2872{ 2873 unsigned int flags = FAULT_FLAG_NONLINEAR | 2874 (write_access ? FAULT_FLAG_WRITE : 0); 2875 pgoff_t pgoff; 2876 2877 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 2878 return 0; 2879 2880 if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) { 2881 /* 2882 * Page table corrupted: show pte and kill process. 2883 */ 2884 print_bad_pte(vma, address, orig_pte, NULL); 2885 return VM_FAULT_OOM; 2886 } 2887 2888 pgoff = pte_to_pgoff(orig_pte); 2889 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 2890} 2891 2892/* 2893 * These routines also need to handle stuff like marking pages dirty 2894 * and/or accessed for architectures that don't do it in hardware (most 2895 * RISC architectures). The early dirtying is also good on the i386. 2896 * 2897 * There is also a hook called "update_mmu_cache()" that architectures 2898 * with external mmu caches can use to update those (ie the Sparc or 2899 * PowerPC hashed page tables that act as extended TLBs). 2900 * 2901 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2902 * but allow concurrent faults), and pte mapped but not yet locked. 2903 * We return with mmap_sem still held, but pte unmapped and unlocked. 2904 */ 2905static inline int handle_pte_fault(struct mm_struct *mm, 2906 struct vm_area_struct *vma, unsigned long address, 2907 pte_t *pte, pmd_t *pmd, int write_access) 2908{ 2909 pte_t entry; 2910 spinlock_t *ptl; 2911 2912 entry = *pte; 2913 if (!pte_present(entry)) { 2914 if (pte_none(entry)) { 2915 if (vma->vm_ops) { 2916 if (likely(vma->vm_ops->fault)) 2917 return do_linear_fault(mm, vma, address, 2918 pte, pmd, write_access, entry); 2919 } 2920 return do_anonymous_page(mm, vma, address, 2921 pte, pmd, write_access); 2922 } 2923 if (pte_file(entry)) 2924 return do_nonlinear_fault(mm, vma, address, 2925 pte, pmd, write_access, entry); 2926 return do_swap_page(mm, vma, address, 2927 pte, pmd, write_access, entry); 2928 } 2929 2930 ptl = pte_lockptr(mm, pmd); 2931 spin_lock(ptl); 2932 if (unlikely(!pte_same(*pte, entry))) 2933 goto unlock; 2934 if (write_access) { 2935 if (!pte_write(entry)) 2936 return do_wp_page(mm, vma, address, 2937 pte, pmd, ptl, entry); 2938 entry = pte_mkdirty(entry); 2939 } 2940 entry = pte_mkyoung(entry); 2941 if (ptep_set_access_flags(vma, address, pte, entry, write_access)) { 2942 update_mmu_cache(vma, address, entry); 2943 } else { 2944 /* 2945 * This is needed only for protection faults but the arch code 2946 * is not yet telling us if this is a protection fault or not. 2947 * This still avoids useless tlb flushes for .text page faults 2948 * with threads. 2949 */ 2950 if (write_access) 2951 flush_tlb_page(vma, address); 2952 } 2953unlock: 2954 pte_unmap_unlock(pte, ptl); 2955 return 0; 2956} 2957 2958/* 2959 * By the time we get here, we already hold the mm semaphore 2960 */ 2961int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, 2962 unsigned long address, int write_access) 2963{ 2964 pgd_t *pgd; 2965 pud_t *pud; 2966 pmd_t *pmd; 2967 pte_t *pte; 2968 2969 __set_current_state(TASK_RUNNING); 2970 2971 count_vm_event(PGFAULT); 2972 2973 if (unlikely(is_vm_hugetlb_page(vma))) 2974 return hugetlb_fault(mm, vma, address, write_access); 2975 2976 pgd = pgd_offset(mm, address); 2977 pud = pud_alloc(mm, pgd, address); 2978 if (!pud) 2979 return VM_FAULT_OOM; 2980 pmd = pmd_alloc(mm, pud, address); 2981 if (!pmd) 2982 return VM_FAULT_OOM; 2983 pte = pte_alloc_map(mm, pmd, address); 2984 if (!pte) 2985 return VM_FAULT_OOM; 2986 2987 return handle_pte_fault(mm, vma, address, pte, pmd, write_access); 2988} 2989 2990#ifndef __PAGETABLE_PUD_FOLDED 2991/* 2992 * Allocate page upper directory. 2993 * We've already handled the fast-path in-line. 2994 */ 2995int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 2996{ 2997 pud_t *new = pud_alloc_one(mm, address); 2998 if (!new) 2999 return -ENOMEM; 3000 3001 smp_wmb(); /* See comment in __pte_alloc */ 3002 3003 spin_lock(&mm->page_table_lock); 3004 if (pgd_present(*pgd)) /* Another has populated it */ 3005 pud_free(mm, new); 3006 else 3007 pgd_populate(mm, pgd, new); 3008 spin_unlock(&mm->page_table_lock); 3009 return 0; 3010} 3011#endif /* __PAGETABLE_PUD_FOLDED */ 3012 3013#ifndef __PAGETABLE_PMD_FOLDED 3014/* 3015 * Allocate page middle directory. 3016 * We've already handled the fast-path in-line. 3017 */ 3018int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 3019{ 3020 pmd_t *new = pmd_alloc_one(mm, address); 3021 if (!new) 3022 return -ENOMEM; 3023 3024 smp_wmb(); /* See comment in __pte_alloc */ 3025 3026 spin_lock(&mm->page_table_lock); 3027#ifndef __ARCH_HAS_4LEVEL_HACK 3028 if (pud_present(*pud)) /* Another has populated it */ 3029 pmd_free(mm, new); 3030 else 3031 pud_populate(mm, pud, new); 3032#else 3033 if (pgd_present(*pud)) /* Another has populated it */ 3034 pmd_free(mm, new); 3035 else 3036 pgd_populate(mm, pud, new); 3037#endif /* __ARCH_HAS_4LEVEL_HACK */ 3038 spin_unlock(&mm->page_table_lock); 3039 return 0; 3040} 3041#endif /* __PAGETABLE_PMD_FOLDED */ 3042 3043int make_pages_present(unsigned long addr, unsigned long end) 3044{ 3045 int ret, len, write; 3046 struct vm_area_struct * vma; 3047 3048 vma = find_vma(current->mm, addr); 3049 if (!vma) 3050 return -ENOMEM; 3051 write = (vma->vm_flags & VM_WRITE) != 0; 3052 BUG_ON(addr >= end); 3053 BUG_ON(end > vma->vm_end); 3054 len = DIV_ROUND_UP(end, PAGE_SIZE) - addr/PAGE_SIZE; 3055 ret = get_user_pages(current, current->mm, addr, 3056 len, write, 0, NULL, NULL); 3057 if (ret < 0) 3058 return ret; 3059 return ret == len ? 0 : -EFAULT; 3060} 3061 3062#if !defined(__HAVE_ARCH_GATE_AREA) 3063 3064#if defined(AT_SYSINFO_EHDR) 3065static struct vm_area_struct gate_vma; 3066 3067static int __init gate_vma_init(void) 3068{ 3069 gate_vma.vm_mm = NULL; 3070 gate_vma.vm_start = FIXADDR_USER_START; 3071 gate_vma.vm_end = FIXADDR_USER_END; 3072 gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC; 3073 gate_vma.vm_page_prot = __P101; 3074 /* 3075 * Make sure the vDSO gets into every core dump. 3076 * Dumping its contents makes post-mortem fully interpretable later 3077 * without matching up the same kernel and hardware config to see 3078 * what PC values meant. 3079 */ 3080 gate_vma.vm_flags |= VM_ALWAYSDUMP; 3081 return 0; 3082} 3083__initcall(gate_vma_init); 3084#endif 3085 3086struct vm_area_struct *get_gate_vma(struct task_struct *tsk) 3087{ 3088#ifdef AT_SYSINFO_EHDR 3089 return &gate_vma; 3090#else 3091 return NULL; 3092#endif 3093} 3094 3095int in_gate_area_no_task(unsigned long addr) 3096{ 3097#ifdef AT_SYSINFO_EHDR 3098 if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END)) 3099 return 1; 3100#endif 3101 return 0; 3102} 3103 3104#endif /* __HAVE_ARCH_GATE_AREA */ 3105 3106static int follow_pte(struct mm_struct *mm, unsigned long address, 3107 pte_t **ptepp, spinlock_t **ptlp) 3108{ 3109 pgd_t *pgd; 3110 pud_t *pud; 3111 pmd_t *pmd; 3112 pte_t *ptep; 3113 3114 pgd = pgd_offset(mm, address); 3115 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 3116 goto out; 3117 3118 pud = pud_offset(pgd, address); 3119 if (pud_none(*pud) || unlikely(pud_bad(*pud))) 3120 goto out; 3121 3122 pmd = pmd_offset(pud, address); 3123 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) 3124 goto out; 3125 3126 /* We cannot handle huge page PFN maps. Luckily they don't exist. */ 3127 if (pmd_huge(*pmd)) 3128 goto out; 3129 3130 ptep = pte_offset_map_lock(mm, pmd, address, ptlp); 3131 if (!ptep) 3132 goto out; 3133 if (!pte_present(*ptep)) 3134 goto unlock; 3135 *ptepp = ptep; 3136 return 0; 3137unlock: 3138 pte_unmap_unlock(ptep, *ptlp); 3139out: 3140 return -EINVAL; 3141} 3142 3143/** 3144 * follow_pfn - look up PFN at a user virtual address 3145 * @vma: memory mapping 3146 * @address: user virtual address 3147 * @pfn: location to store found PFN 3148 * 3149 * Only IO mappings and raw PFN mappings are allowed. 3150 * 3151 * Returns zero and the pfn at @pfn on success, -ve otherwise. 3152 */ 3153int follow_pfn(struct vm_area_struct *vma, unsigned long address, 3154 unsigned long *pfn) 3155{ 3156 int ret = -EINVAL; 3157 spinlock_t *ptl; 3158 pte_t *ptep; 3159 3160 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3161 return ret; 3162 3163 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); 3164 if (ret) 3165 return ret; 3166 *pfn = pte_pfn(*ptep); 3167 pte_unmap_unlock(ptep, ptl); 3168 return 0; 3169} 3170EXPORT_SYMBOL(follow_pfn); 3171 3172#ifdef CONFIG_HAVE_IOREMAP_PROT 3173int follow_phys(struct vm_area_struct *vma, 3174 unsigned long address, unsigned int flags, 3175 unsigned long *prot, resource_size_t *phys) 3176{ 3177 int ret = -EINVAL; 3178 pte_t *ptep, pte; 3179 spinlock_t *ptl; 3180 3181 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3182 goto out; 3183 3184 if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) 3185 goto out; 3186 pte = *ptep; 3187 3188 if ((flags & FOLL_WRITE) && !pte_write(pte)) 3189 goto unlock; 3190 3191 *prot = pgprot_val(pte_pgprot(pte)); 3192 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; 3193 3194 ret = 0; 3195unlock: 3196 pte_unmap_unlock(ptep, ptl); 3197out: 3198 return ret; 3199} 3200 3201int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 3202 void *buf, int len, int write) 3203{ 3204 resource_size_t phys_addr; 3205 unsigned long prot = 0; 3206 void __iomem *maddr; 3207 int offset = addr & (PAGE_SIZE-1); 3208 3209 if (follow_phys(vma, addr, write, &prot, &phys_addr)) 3210 return -EINVAL; 3211 3212 maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); 3213 if (write) 3214 memcpy_toio(maddr + offset, buf, len); 3215 else 3216 memcpy_fromio(buf, maddr + offset, len); 3217 iounmap(maddr); 3218 3219 return len; 3220} 3221#endif 3222 3223/* 3224 * Access another process' address space. 3225 * Source/target buffer must be kernel space, 3226 * Do not walk the page table directly, use get_user_pages 3227 */ 3228int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write) 3229{ 3230 struct mm_struct *mm; 3231 struct vm_area_struct *vma; 3232 void *old_buf = buf; 3233 3234 mm = get_task_mm(tsk); 3235 if (!mm) 3236 return 0; 3237 3238 down_read(&mm->mmap_sem); 3239 /* ignore errors, just check how much was successfully transferred */ 3240 while (len) { 3241 int bytes, ret, offset; 3242 void *maddr; 3243 struct page *page = NULL; 3244 3245 ret = get_user_pages(tsk, mm, addr, 1, 3246 write, 1, &page, &vma); 3247 if (ret <= 0) { 3248 /* 3249 * Check if this is a VM_IO | VM_PFNMAP VMA, which 3250 * we can access using slightly different code. 3251 */ 3252#ifdef CONFIG_HAVE_IOREMAP_PROT 3253 vma = find_vma(mm, addr); 3254 if (!vma) 3255 break; 3256 if (vma->vm_ops && vma->vm_ops->access) 3257 ret = vma->vm_ops->access(vma, addr, buf, 3258 len, write); 3259 if (ret <= 0) 3260#endif 3261 break; 3262 bytes = ret; 3263 } else { 3264 bytes = len; 3265 offset = addr & (PAGE_SIZE-1); 3266 if (bytes > PAGE_SIZE-offset) 3267 bytes = PAGE_SIZE-offset; 3268 3269 maddr = kmap(page); 3270 if (write) { 3271 copy_to_user_page(vma, page, addr, 3272 maddr + offset, buf, bytes); 3273 set_page_dirty_lock(page); 3274 } else { 3275 copy_from_user_page(vma, page, addr, 3276 buf, maddr + offset, bytes); 3277 } 3278 kunmap(page); 3279 page_cache_release(page); 3280 } 3281 len -= bytes; 3282 buf += bytes; 3283 addr += bytes; 3284 } 3285 up_read(&mm->mmap_sem); 3286 mmput(mm); 3287 3288 return buf - old_buf; 3289} 3290 3291/* 3292 * Print the name of a VMA. 3293 */ 3294void print_vma_addr(char *prefix, unsigned long ip) 3295{ 3296 struct mm_struct *mm = current->mm; 3297 struct vm_area_struct *vma; 3298 3299 /* 3300 * Do not print if we are in atomic 3301 * contexts (in exception stacks, etc.): 3302 */ 3303 if (preempt_count()) 3304 return; 3305 3306 down_read(&mm->mmap_sem); 3307 vma = find_vma(mm, ip); 3308 if (vma && vma->vm_file) { 3309 struct file *f = vma->vm_file; 3310 char *buf = (char *)__get_free_page(GFP_KERNEL); 3311 if (buf) { 3312 char *p, *s; 3313 3314 p = d_path(&f->f_path, buf, PAGE_SIZE); 3315 if (IS_ERR(p)) 3316 p = "?"; 3317 s = strrchr(p, '/'); 3318 if (s) 3319 p = s+1; 3320 printk("%s%s[%lx+%lx]", prefix, p, 3321 vma->vm_start, 3322 vma->vm_end - vma->vm_start); 3323 free_page((unsigned long)buf); 3324 } 3325 } 3326 up_read(¤t->mm->mmap_sem); 3327} 3328 3329#ifdef CONFIG_PROVE_LOCKING 3330void might_fault(void) 3331{ 3332 /* 3333 * Some code (nfs/sunrpc) uses socket ops on kernel memory while 3334 * holding the mmap_sem, this is safe because kernel memory doesn't 3335 * get paged out, therefore we'll never actually fault, and the 3336 * below annotations will generate false positives. 3337 */ 3338 if (segment_eq(get_fs(), KERNEL_DS)) 3339 return; 3340 3341 might_sleep(); 3342 /* 3343 * it would be nicer only to annotate paths which are not under 3344 * pagefault_disable, however that requires a larger audit and 3345 * providing helpers like get_user_atomic. 3346 */ 3347 if (!in_atomic() && current->mm) 3348 might_lock_read(¤t->mm->mmap_sem); 3349} 3350EXPORT_SYMBOL(might_fault); 3351#endif 3352