memory.c revision a137e1cc6d6e7d315fef03962a2a5a113348b13b
1/* 2 * linux/mm/memory.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7/* 8 * demand-loading started 01.12.91 - seems it is high on the list of 9 * things wanted, and it should be easy to implement. - Linus 10 */ 11 12/* 13 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared 14 * pages started 02.12.91, seems to work. - Linus. 15 * 16 * Tested sharing by executing about 30 /bin/sh: under the old kernel it 17 * would have taken more than the 6M I have free, but it worked well as 18 * far as I could see. 19 * 20 * Also corrected some "invalidate()"s - I wasn't doing enough of them. 21 */ 22 23/* 24 * Real VM (paging to/from disk) started 18.12.91. Much more work and 25 * thought has to go into this. Oh, well.. 26 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. 27 * Found it. Everything seems to work now. 28 * 20.12.91 - Ok, making the swap-device changeable like the root. 29 */ 30 31/* 32 * 05.04.94 - Multi-page memory management added for v1.1. 33 * Idea by Alex Bligh (alex@cconcepts.co.uk) 34 * 35 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG 36 * (Gerhard.Wichert@pdb.siemens.de) 37 * 38 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) 39 */ 40 41#include <linux/kernel_stat.h> 42#include <linux/mm.h> 43#include <linux/hugetlb.h> 44#include <linux/mman.h> 45#include <linux/swap.h> 46#include <linux/highmem.h> 47#include <linux/pagemap.h> 48#include <linux/rmap.h> 49#include <linux/module.h> 50#include <linux/delayacct.h> 51#include <linux/init.h> 52#include <linux/writeback.h> 53#include <linux/memcontrol.h> 54 55#include <asm/pgalloc.h> 56#include <asm/uaccess.h> 57#include <asm/tlb.h> 58#include <asm/tlbflush.h> 59#include <asm/pgtable.h> 60 61#include <linux/swapops.h> 62#include <linux/elf.h> 63 64#include "internal.h" 65 66#ifndef CONFIG_NEED_MULTIPLE_NODES 67/* use the per-pgdat data instead for discontigmem - mbligh */ 68unsigned long max_mapnr; 69struct page *mem_map; 70 71EXPORT_SYMBOL(max_mapnr); 72EXPORT_SYMBOL(mem_map); 73#endif 74 75unsigned long num_physpages; 76/* 77 * A number of key systems in x86 including ioremap() rely on the assumption 78 * that high_memory defines the upper bound on direct map memory, then end 79 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and 80 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL 81 * and ZONE_HIGHMEM. 82 */ 83void * high_memory; 84 85EXPORT_SYMBOL(num_physpages); 86EXPORT_SYMBOL(high_memory); 87 88/* 89 * Randomize the address space (stacks, mmaps, brk, etc.). 90 * 91 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, 92 * as ancient (libc5 based) binaries can segfault. ) 93 */ 94int randomize_va_space __read_mostly = 95#ifdef CONFIG_COMPAT_BRK 96 1; 97#else 98 2; 99#endif 100 101static int __init disable_randmaps(char *s) 102{ 103 randomize_va_space = 0; 104 return 1; 105} 106__setup("norandmaps", disable_randmaps); 107 108 109/* 110 * If a p?d_bad entry is found while walking page tables, report 111 * the error, before resetting entry to p?d_none. Usually (but 112 * very seldom) called out from the p?d_none_or_clear_bad macros. 113 */ 114 115void pgd_clear_bad(pgd_t *pgd) 116{ 117 pgd_ERROR(*pgd); 118 pgd_clear(pgd); 119} 120 121void pud_clear_bad(pud_t *pud) 122{ 123 pud_ERROR(*pud); 124 pud_clear(pud); 125} 126 127void pmd_clear_bad(pmd_t *pmd) 128{ 129 pmd_ERROR(*pmd); 130 pmd_clear(pmd); 131} 132 133/* 134 * Note: this doesn't free the actual pages themselves. That 135 * has been handled earlier when unmapping all the memory regions. 136 */ 137static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd) 138{ 139 pgtable_t token = pmd_pgtable(*pmd); 140 pmd_clear(pmd); 141 pte_free_tlb(tlb, token); 142 tlb->mm->nr_ptes--; 143} 144 145static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 146 unsigned long addr, unsigned long end, 147 unsigned long floor, unsigned long ceiling) 148{ 149 pmd_t *pmd; 150 unsigned long next; 151 unsigned long start; 152 153 start = addr; 154 pmd = pmd_offset(pud, addr); 155 do { 156 next = pmd_addr_end(addr, end); 157 if (pmd_none_or_clear_bad(pmd)) 158 continue; 159 free_pte_range(tlb, pmd); 160 } while (pmd++, addr = next, addr != end); 161 162 start &= PUD_MASK; 163 if (start < floor) 164 return; 165 if (ceiling) { 166 ceiling &= PUD_MASK; 167 if (!ceiling) 168 return; 169 } 170 if (end - 1 > ceiling - 1) 171 return; 172 173 pmd = pmd_offset(pud, start); 174 pud_clear(pud); 175 pmd_free_tlb(tlb, pmd); 176} 177 178static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, 179 unsigned long addr, unsigned long end, 180 unsigned long floor, unsigned long ceiling) 181{ 182 pud_t *pud; 183 unsigned long next; 184 unsigned long start; 185 186 start = addr; 187 pud = pud_offset(pgd, addr); 188 do { 189 next = pud_addr_end(addr, end); 190 if (pud_none_or_clear_bad(pud)) 191 continue; 192 free_pmd_range(tlb, pud, addr, next, floor, ceiling); 193 } while (pud++, addr = next, addr != end); 194 195 start &= PGDIR_MASK; 196 if (start < floor) 197 return; 198 if (ceiling) { 199 ceiling &= PGDIR_MASK; 200 if (!ceiling) 201 return; 202 } 203 if (end - 1 > ceiling - 1) 204 return; 205 206 pud = pud_offset(pgd, start); 207 pgd_clear(pgd); 208 pud_free_tlb(tlb, pud); 209} 210 211/* 212 * This function frees user-level page tables of a process. 213 * 214 * Must be called with pagetable lock held. 215 */ 216void free_pgd_range(struct mmu_gather *tlb, 217 unsigned long addr, unsigned long end, 218 unsigned long floor, unsigned long ceiling) 219{ 220 pgd_t *pgd; 221 unsigned long next; 222 unsigned long start; 223 224 /* 225 * The next few lines have given us lots of grief... 226 * 227 * Why are we testing PMD* at this top level? Because often 228 * there will be no work to do at all, and we'd prefer not to 229 * go all the way down to the bottom just to discover that. 230 * 231 * Why all these "- 1"s? Because 0 represents both the bottom 232 * of the address space and the top of it (using -1 for the 233 * top wouldn't help much: the masks would do the wrong thing). 234 * The rule is that addr 0 and floor 0 refer to the bottom of 235 * the address space, but end 0 and ceiling 0 refer to the top 236 * Comparisons need to use "end - 1" and "ceiling - 1" (though 237 * that end 0 case should be mythical). 238 * 239 * Wherever addr is brought up or ceiling brought down, we must 240 * be careful to reject "the opposite 0" before it confuses the 241 * subsequent tests. But what about where end is brought down 242 * by PMD_SIZE below? no, end can't go down to 0 there. 243 * 244 * Whereas we round start (addr) and ceiling down, by different 245 * masks at different levels, in order to test whether a table 246 * now has no other vmas using it, so can be freed, we don't 247 * bother to round floor or end up - the tests don't need that. 248 */ 249 250 addr &= PMD_MASK; 251 if (addr < floor) { 252 addr += PMD_SIZE; 253 if (!addr) 254 return; 255 } 256 if (ceiling) { 257 ceiling &= PMD_MASK; 258 if (!ceiling) 259 return; 260 } 261 if (end - 1 > ceiling - 1) 262 end -= PMD_SIZE; 263 if (addr > end - 1) 264 return; 265 266 start = addr; 267 pgd = pgd_offset(tlb->mm, addr); 268 do { 269 next = pgd_addr_end(addr, end); 270 if (pgd_none_or_clear_bad(pgd)) 271 continue; 272 free_pud_range(tlb, pgd, addr, next, floor, ceiling); 273 } while (pgd++, addr = next, addr != end); 274} 275 276void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, 277 unsigned long floor, unsigned long ceiling) 278{ 279 while (vma) { 280 struct vm_area_struct *next = vma->vm_next; 281 unsigned long addr = vma->vm_start; 282 283 /* 284 * Hide vma from rmap and vmtruncate before freeing pgtables 285 */ 286 anon_vma_unlink(vma); 287 unlink_file_vma(vma); 288 289 if (is_vm_hugetlb_page(vma)) { 290 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, 291 floor, next? next->vm_start: ceiling); 292 } else { 293 /* 294 * Optimization: gather nearby vmas into one call down 295 */ 296 while (next && next->vm_start <= vma->vm_end + PMD_SIZE 297 && !is_vm_hugetlb_page(next)) { 298 vma = next; 299 next = vma->vm_next; 300 anon_vma_unlink(vma); 301 unlink_file_vma(vma); 302 } 303 free_pgd_range(tlb, addr, vma->vm_end, 304 floor, next? next->vm_start: ceiling); 305 } 306 vma = next; 307 } 308} 309 310int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address) 311{ 312 pgtable_t new = pte_alloc_one(mm, address); 313 if (!new) 314 return -ENOMEM; 315 316 /* 317 * Ensure all pte setup (eg. pte page lock and page clearing) are 318 * visible before the pte is made visible to other CPUs by being 319 * put into page tables. 320 * 321 * The other side of the story is the pointer chasing in the page 322 * table walking code (when walking the page table without locking; 323 * ie. most of the time). Fortunately, these data accesses consist 324 * of a chain of data-dependent loads, meaning most CPUs (alpha 325 * being the notable exception) will already guarantee loads are 326 * seen in-order. See the alpha page table accessors for the 327 * smp_read_barrier_depends() barriers in page table walking code. 328 */ 329 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ 330 331 spin_lock(&mm->page_table_lock); 332 if (!pmd_present(*pmd)) { /* Has another populated it ? */ 333 mm->nr_ptes++; 334 pmd_populate(mm, pmd, new); 335 new = NULL; 336 } 337 spin_unlock(&mm->page_table_lock); 338 if (new) 339 pte_free(mm, new); 340 return 0; 341} 342 343int __pte_alloc_kernel(pmd_t *pmd, unsigned long address) 344{ 345 pte_t *new = pte_alloc_one_kernel(&init_mm, address); 346 if (!new) 347 return -ENOMEM; 348 349 smp_wmb(); /* See comment in __pte_alloc */ 350 351 spin_lock(&init_mm.page_table_lock); 352 if (!pmd_present(*pmd)) { /* Has another populated it ? */ 353 pmd_populate_kernel(&init_mm, pmd, new); 354 new = NULL; 355 } 356 spin_unlock(&init_mm.page_table_lock); 357 if (new) 358 pte_free_kernel(&init_mm, new); 359 return 0; 360} 361 362static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss) 363{ 364 if (file_rss) 365 add_mm_counter(mm, file_rss, file_rss); 366 if (anon_rss) 367 add_mm_counter(mm, anon_rss, anon_rss); 368} 369 370/* 371 * This function is called to print an error when a bad pte 372 * is found. For example, we might have a PFN-mapped pte in 373 * a region that doesn't allow it. 374 * 375 * The calling function must still handle the error. 376 */ 377void print_bad_pte(struct vm_area_struct *vma, pte_t pte, unsigned long vaddr) 378{ 379 printk(KERN_ERR "Bad pte = %08llx, process = %s, " 380 "vm_flags = %lx, vaddr = %lx\n", 381 (long long)pte_val(pte), 382 (vma->vm_mm == current->mm ? current->comm : "???"), 383 vma->vm_flags, vaddr); 384 dump_stack(); 385} 386 387static inline int is_cow_mapping(unsigned int flags) 388{ 389 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 390} 391 392/* 393 * vm_normal_page -- This function gets the "struct page" associated with a pte. 394 * 395 * "Special" mappings do not wish to be associated with a "struct page" (either 396 * it doesn't exist, or it exists but they don't want to touch it). In this 397 * case, NULL is returned here. "Normal" mappings do have a struct page. 398 * 399 * There are 2 broad cases. Firstly, an architecture may define a pte_special() 400 * pte bit, in which case this function is trivial. Secondly, an architecture 401 * may not have a spare pte bit, which requires a more complicated scheme, 402 * described below. 403 * 404 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a 405 * special mapping (even if there are underlying and valid "struct pages"). 406 * COWed pages of a VM_PFNMAP are always normal. 407 * 408 * The way we recognize COWed pages within VM_PFNMAP mappings is through the 409 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit 410 * set, and the vm_pgoff will point to the first PFN mapped: thus every special 411 * mapping will always honor the rule 412 * 413 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) 414 * 415 * And for normal mappings this is false. 416 * 417 * This restricts such mappings to be a linear translation from virtual address 418 * to pfn. To get around this restriction, we allow arbitrary mappings so long 419 * as the vma is not a COW mapping; in that case, we know that all ptes are 420 * special (because none can have been COWed). 421 * 422 * 423 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP. 424 * 425 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct 426 * page" backing, however the difference is that _all_ pages with a struct 427 * page (that is, those where pfn_valid is true) are refcounted and considered 428 * normal pages by the VM. The disadvantage is that pages are refcounted 429 * (which can be slower and simply not an option for some PFNMAP users). The 430 * advantage is that we don't have to follow the strict linearity rule of 431 * PFNMAP mappings in order to support COWable mappings. 432 * 433 */ 434#ifdef __HAVE_ARCH_PTE_SPECIAL 435# define HAVE_PTE_SPECIAL 1 436#else 437# define HAVE_PTE_SPECIAL 0 438#endif 439struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 440 pte_t pte) 441{ 442 unsigned long pfn; 443 444 if (HAVE_PTE_SPECIAL) { 445 if (likely(!pte_special(pte))) { 446 VM_BUG_ON(!pfn_valid(pte_pfn(pte))); 447 return pte_page(pte); 448 } 449 VM_BUG_ON(!(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))); 450 return NULL; 451 } 452 453 /* !HAVE_PTE_SPECIAL case follows: */ 454 455 pfn = pte_pfn(pte); 456 457 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 458 if (vma->vm_flags & VM_MIXEDMAP) { 459 if (!pfn_valid(pfn)) 460 return NULL; 461 goto out; 462 } else { 463 unsigned long off; 464 off = (addr - vma->vm_start) >> PAGE_SHIFT; 465 if (pfn == vma->vm_pgoff + off) 466 return NULL; 467 if (!is_cow_mapping(vma->vm_flags)) 468 return NULL; 469 } 470 } 471 472 VM_BUG_ON(!pfn_valid(pfn)); 473 474 /* 475 * NOTE! We still have PageReserved() pages in the page tables. 476 * 477 * eg. VDSO mappings can cause them to exist. 478 */ 479out: 480 return pfn_to_page(pfn); 481} 482 483/* 484 * copy one vm_area from one task to the other. Assumes the page tables 485 * already present in the new task to be cleared in the whole range 486 * covered by this vma. 487 */ 488 489static inline void 490copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, 491 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, 492 unsigned long addr, int *rss) 493{ 494 unsigned long vm_flags = vma->vm_flags; 495 pte_t pte = *src_pte; 496 struct page *page; 497 498 /* pte contains position in swap or file, so copy. */ 499 if (unlikely(!pte_present(pte))) { 500 if (!pte_file(pte)) { 501 swp_entry_t entry = pte_to_swp_entry(pte); 502 503 swap_duplicate(entry); 504 /* make sure dst_mm is on swapoff's mmlist. */ 505 if (unlikely(list_empty(&dst_mm->mmlist))) { 506 spin_lock(&mmlist_lock); 507 if (list_empty(&dst_mm->mmlist)) 508 list_add(&dst_mm->mmlist, 509 &src_mm->mmlist); 510 spin_unlock(&mmlist_lock); 511 } 512 if (is_write_migration_entry(entry) && 513 is_cow_mapping(vm_flags)) { 514 /* 515 * COW mappings require pages in both parent 516 * and child to be set to read. 517 */ 518 make_migration_entry_read(&entry); 519 pte = swp_entry_to_pte(entry); 520 set_pte_at(src_mm, addr, src_pte, pte); 521 } 522 } 523 goto out_set_pte; 524 } 525 526 /* 527 * If it's a COW mapping, write protect it both 528 * in the parent and the child 529 */ 530 if (is_cow_mapping(vm_flags)) { 531 ptep_set_wrprotect(src_mm, addr, src_pte); 532 pte = pte_wrprotect(pte); 533 } 534 535 /* 536 * If it's a shared mapping, mark it clean in 537 * the child 538 */ 539 if (vm_flags & VM_SHARED) 540 pte = pte_mkclean(pte); 541 pte = pte_mkold(pte); 542 543 page = vm_normal_page(vma, addr, pte); 544 if (page) { 545 get_page(page); 546 page_dup_rmap(page, vma, addr); 547 rss[!!PageAnon(page)]++; 548 } 549 550out_set_pte: 551 set_pte_at(dst_mm, addr, dst_pte, pte); 552} 553 554static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 555 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, 556 unsigned long addr, unsigned long end) 557{ 558 pte_t *src_pte, *dst_pte; 559 spinlock_t *src_ptl, *dst_ptl; 560 int progress = 0; 561 int rss[2]; 562 563again: 564 rss[1] = rss[0] = 0; 565 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); 566 if (!dst_pte) 567 return -ENOMEM; 568 src_pte = pte_offset_map_nested(src_pmd, addr); 569 src_ptl = pte_lockptr(src_mm, src_pmd); 570 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 571 arch_enter_lazy_mmu_mode(); 572 573 do { 574 /* 575 * We are holding two locks at this point - either of them 576 * could generate latencies in another task on another CPU. 577 */ 578 if (progress >= 32) { 579 progress = 0; 580 if (need_resched() || 581 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl)) 582 break; 583 } 584 if (pte_none(*src_pte)) { 585 progress++; 586 continue; 587 } 588 copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss); 589 progress += 8; 590 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 591 592 arch_leave_lazy_mmu_mode(); 593 spin_unlock(src_ptl); 594 pte_unmap_nested(src_pte - 1); 595 add_mm_rss(dst_mm, rss[0], rss[1]); 596 pte_unmap_unlock(dst_pte - 1, dst_ptl); 597 cond_resched(); 598 if (addr != end) 599 goto again; 600 return 0; 601} 602 603static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 604 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma, 605 unsigned long addr, unsigned long end) 606{ 607 pmd_t *src_pmd, *dst_pmd; 608 unsigned long next; 609 610 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); 611 if (!dst_pmd) 612 return -ENOMEM; 613 src_pmd = pmd_offset(src_pud, addr); 614 do { 615 next = pmd_addr_end(addr, end); 616 if (pmd_none_or_clear_bad(src_pmd)) 617 continue; 618 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd, 619 vma, addr, next)) 620 return -ENOMEM; 621 } while (dst_pmd++, src_pmd++, addr = next, addr != end); 622 return 0; 623} 624 625static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 626 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma, 627 unsigned long addr, unsigned long end) 628{ 629 pud_t *src_pud, *dst_pud; 630 unsigned long next; 631 632 dst_pud = pud_alloc(dst_mm, dst_pgd, addr); 633 if (!dst_pud) 634 return -ENOMEM; 635 src_pud = pud_offset(src_pgd, addr); 636 do { 637 next = pud_addr_end(addr, end); 638 if (pud_none_or_clear_bad(src_pud)) 639 continue; 640 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud, 641 vma, addr, next)) 642 return -ENOMEM; 643 } while (dst_pud++, src_pud++, addr = next, addr != end); 644 return 0; 645} 646 647int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 648 struct vm_area_struct *vma) 649{ 650 pgd_t *src_pgd, *dst_pgd; 651 unsigned long next; 652 unsigned long addr = vma->vm_start; 653 unsigned long end = vma->vm_end; 654 655 /* 656 * Don't copy ptes where a page fault will fill them correctly. 657 * Fork becomes much lighter when there are big shared or private 658 * readonly mappings. The tradeoff is that copy_page_range is more 659 * efficient than faulting. 660 */ 661 if (!(vma->vm_flags & (VM_HUGETLB|VM_NONLINEAR|VM_PFNMAP|VM_INSERTPAGE))) { 662 if (!vma->anon_vma) 663 return 0; 664 } 665 666 if (is_vm_hugetlb_page(vma)) 667 return copy_hugetlb_page_range(dst_mm, src_mm, vma); 668 669 dst_pgd = pgd_offset(dst_mm, addr); 670 src_pgd = pgd_offset(src_mm, addr); 671 do { 672 next = pgd_addr_end(addr, end); 673 if (pgd_none_or_clear_bad(src_pgd)) 674 continue; 675 if (copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd, 676 vma, addr, next)) 677 return -ENOMEM; 678 } while (dst_pgd++, src_pgd++, addr = next, addr != end); 679 return 0; 680} 681 682static unsigned long zap_pte_range(struct mmu_gather *tlb, 683 struct vm_area_struct *vma, pmd_t *pmd, 684 unsigned long addr, unsigned long end, 685 long *zap_work, struct zap_details *details) 686{ 687 struct mm_struct *mm = tlb->mm; 688 pte_t *pte; 689 spinlock_t *ptl; 690 int file_rss = 0; 691 int anon_rss = 0; 692 693 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 694 arch_enter_lazy_mmu_mode(); 695 do { 696 pte_t ptent = *pte; 697 if (pte_none(ptent)) { 698 (*zap_work)--; 699 continue; 700 } 701 702 (*zap_work) -= PAGE_SIZE; 703 704 if (pte_present(ptent)) { 705 struct page *page; 706 707 page = vm_normal_page(vma, addr, ptent); 708 if (unlikely(details) && page) { 709 /* 710 * unmap_shared_mapping_pages() wants to 711 * invalidate cache without truncating: 712 * unmap shared but keep private pages. 713 */ 714 if (details->check_mapping && 715 details->check_mapping != page->mapping) 716 continue; 717 /* 718 * Each page->index must be checked when 719 * invalidating or truncating nonlinear. 720 */ 721 if (details->nonlinear_vma && 722 (page->index < details->first_index || 723 page->index > details->last_index)) 724 continue; 725 } 726 ptent = ptep_get_and_clear_full(mm, addr, pte, 727 tlb->fullmm); 728 tlb_remove_tlb_entry(tlb, pte, addr); 729 if (unlikely(!page)) 730 continue; 731 if (unlikely(details) && details->nonlinear_vma 732 && linear_page_index(details->nonlinear_vma, 733 addr) != page->index) 734 set_pte_at(mm, addr, pte, 735 pgoff_to_pte(page->index)); 736 if (PageAnon(page)) 737 anon_rss--; 738 else { 739 if (pte_dirty(ptent)) 740 set_page_dirty(page); 741 if (pte_young(ptent)) 742 SetPageReferenced(page); 743 file_rss--; 744 } 745 page_remove_rmap(page, vma); 746 tlb_remove_page(tlb, page); 747 continue; 748 } 749 /* 750 * If details->check_mapping, we leave swap entries; 751 * if details->nonlinear_vma, we leave file entries. 752 */ 753 if (unlikely(details)) 754 continue; 755 if (!pte_file(ptent)) 756 free_swap_and_cache(pte_to_swp_entry(ptent)); 757 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 758 } while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0)); 759 760 add_mm_rss(mm, file_rss, anon_rss); 761 arch_leave_lazy_mmu_mode(); 762 pte_unmap_unlock(pte - 1, ptl); 763 764 return addr; 765} 766 767static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, 768 struct vm_area_struct *vma, pud_t *pud, 769 unsigned long addr, unsigned long end, 770 long *zap_work, struct zap_details *details) 771{ 772 pmd_t *pmd; 773 unsigned long next; 774 775 pmd = pmd_offset(pud, addr); 776 do { 777 next = pmd_addr_end(addr, end); 778 if (pmd_none_or_clear_bad(pmd)) { 779 (*zap_work)--; 780 continue; 781 } 782 next = zap_pte_range(tlb, vma, pmd, addr, next, 783 zap_work, details); 784 } while (pmd++, addr = next, (addr != end && *zap_work > 0)); 785 786 return addr; 787} 788 789static inline unsigned long zap_pud_range(struct mmu_gather *tlb, 790 struct vm_area_struct *vma, pgd_t *pgd, 791 unsigned long addr, unsigned long end, 792 long *zap_work, struct zap_details *details) 793{ 794 pud_t *pud; 795 unsigned long next; 796 797 pud = pud_offset(pgd, addr); 798 do { 799 next = pud_addr_end(addr, end); 800 if (pud_none_or_clear_bad(pud)) { 801 (*zap_work)--; 802 continue; 803 } 804 next = zap_pmd_range(tlb, vma, pud, addr, next, 805 zap_work, details); 806 } while (pud++, addr = next, (addr != end && *zap_work > 0)); 807 808 return addr; 809} 810 811static unsigned long unmap_page_range(struct mmu_gather *tlb, 812 struct vm_area_struct *vma, 813 unsigned long addr, unsigned long end, 814 long *zap_work, struct zap_details *details) 815{ 816 pgd_t *pgd; 817 unsigned long next; 818 819 if (details && !details->check_mapping && !details->nonlinear_vma) 820 details = NULL; 821 822 BUG_ON(addr >= end); 823 tlb_start_vma(tlb, vma); 824 pgd = pgd_offset(vma->vm_mm, addr); 825 do { 826 next = pgd_addr_end(addr, end); 827 if (pgd_none_or_clear_bad(pgd)) { 828 (*zap_work)--; 829 continue; 830 } 831 next = zap_pud_range(tlb, vma, pgd, addr, next, 832 zap_work, details); 833 } while (pgd++, addr = next, (addr != end && *zap_work > 0)); 834 tlb_end_vma(tlb, vma); 835 836 return addr; 837} 838 839#ifdef CONFIG_PREEMPT 840# define ZAP_BLOCK_SIZE (8 * PAGE_SIZE) 841#else 842/* No preempt: go for improved straight-line efficiency */ 843# define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE) 844#endif 845 846/** 847 * unmap_vmas - unmap a range of memory covered by a list of vma's 848 * @tlbp: address of the caller's struct mmu_gather 849 * @vma: the starting vma 850 * @start_addr: virtual address at which to start unmapping 851 * @end_addr: virtual address at which to end unmapping 852 * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here 853 * @details: details of nonlinear truncation or shared cache invalidation 854 * 855 * Returns the end address of the unmapping (restart addr if interrupted). 856 * 857 * Unmap all pages in the vma list. 858 * 859 * We aim to not hold locks for too long (for scheduling latency reasons). 860 * So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to 861 * return the ending mmu_gather to the caller. 862 * 863 * Only addresses between `start' and `end' will be unmapped. 864 * 865 * The VMA list must be sorted in ascending virtual address order. 866 * 867 * unmap_vmas() assumes that the caller will flush the whole unmapped address 868 * range after unmap_vmas() returns. So the only responsibility here is to 869 * ensure that any thus-far unmapped pages are flushed before unmap_vmas() 870 * drops the lock and schedules. 871 */ 872unsigned long unmap_vmas(struct mmu_gather **tlbp, 873 struct vm_area_struct *vma, unsigned long start_addr, 874 unsigned long end_addr, unsigned long *nr_accounted, 875 struct zap_details *details) 876{ 877 long zap_work = ZAP_BLOCK_SIZE; 878 unsigned long tlb_start = 0; /* For tlb_finish_mmu */ 879 int tlb_start_valid = 0; 880 unsigned long start = start_addr; 881 spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL; 882 int fullmm = (*tlbp)->fullmm; 883 884 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) { 885 unsigned long end; 886 887 start = max(vma->vm_start, start_addr); 888 if (start >= vma->vm_end) 889 continue; 890 end = min(vma->vm_end, end_addr); 891 if (end <= vma->vm_start) 892 continue; 893 894 if (vma->vm_flags & VM_ACCOUNT) 895 *nr_accounted += (end - start) >> PAGE_SHIFT; 896 897 while (start != end) { 898 if (!tlb_start_valid) { 899 tlb_start = start; 900 tlb_start_valid = 1; 901 } 902 903 if (unlikely(is_vm_hugetlb_page(vma))) { 904 /* 905 * It is undesirable to test vma->vm_file as it 906 * should be non-null for valid hugetlb area. 907 * However, vm_file will be NULL in the error 908 * cleanup path of do_mmap_pgoff. When 909 * hugetlbfs ->mmap method fails, 910 * do_mmap_pgoff() nullifies vma->vm_file 911 * before calling this function to clean up. 912 * Since no pte has actually been setup, it is 913 * safe to do nothing in this case. 914 */ 915 if (vma->vm_file) { 916 unmap_hugepage_range(vma, start, end, NULL); 917 zap_work -= (end - start) / 918 pages_per_huge_page(hstate_vma(vma)); 919 } 920 921 start = end; 922 } else 923 start = unmap_page_range(*tlbp, vma, 924 start, end, &zap_work, details); 925 926 if (zap_work > 0) { 927 BUG_ON(start != end); 928 break; 929 } 930 931 tlb_finish_mmu(*tlbp, tlb_start, start); 932 933 if (need_resched() || 934 (i_mmap_lock && spin_needbreak(i_mmap_lock))) { 935 if (i_mmap_lock) { 936 *tlbp = NULL; 937 goto out; 938 } 939 cond_resched(); 940 } 941 942 *tlbp = tlb_gather_mmu(vma->vm_mm, fullmm); 943 tlb_start_valid = 0; 944 zap_work = ZAP_BLOCK_SIZE; 945 } 946 } 947out: 948 return start; /* which is now the end (or restart) address */ 949} 950 951/** 952 * zap_page_range - remove user pages in a given range 953 * @vma: vm_area_struct holding the applicable pages 954 * @address: starting address of pages to zap 955 * @size: number of bytes to zap 956 * @details: details of nonlinear truncation or shared cache invalidation 957 */ 958unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address, 959 unsigned long size, struct zap_details *details) 960{ 961 struct mm_struct *mm = vma->vm_mm; 962 struct mmu_gather *tlb; 963 unsigned long end = address + size; 964 unsigned long nr_accounted = 0; 965 966 lru_add_drain(); 967 tlb = tlb_gather_mmu(mm, 0); 968 update_hiwater_rss(mm); 969 end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details); 970 if (tlb) 971 tlb_finish_mmu(tlb, address, end); 972 return end; 973} 974 975/* 976 * Do a quick page-table lookup for a single page. 977 */ 978struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 979 unsigned int flags) 980{ 981 pgd_t *pgd; 982 pud_t *pud; 983 pmd_t *pmd; 984 pte_t *ptep, pte; 985 spinlock_t *ptl; 986 struct page *page; 987 struct mm_struct *mm = vma->vm_mm; 988 989 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); 990 if (!IS_ERR(page)) { 991 BUG_ON(flags & FOLL_GET); 992 goto out; 993 } 994 995 page = NULL; 996 pgd = pgd_offset(mm, address); 997 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 998 goto no_page_table; 999 1000 pud = pud_offset(pgd, address); 1001 if (pud_none(*pud) || unlikely(pud_bad(*pud))) 1002 goto no_page_table; 1003 1004 pmd = pmd_offset(pud, address); 1005 if (pmd_none(*pmd)) 1006 goto no_page_table; 1007 1008 if (pmd_huge(*pmd)) { 1009 BUG_ON(flags & FOLL_GET); 1010 page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE); 1011 goto out; 1012 } 1013 1014 if (unlikely(pmd_bad(*pmd))) 1015 goto no_page_table; 1016 1017 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 1018 1019 pte = *ptep; 1020 if (!pte_present(pte)) 1021 goto no_page; 1022 if ((flags & FOLL_WRITE) && !pte_write(pte)) 1023 goto unlock; 1024 page = vm_normal_page(vma, address, pte); 1025 if (unlikely(!page)) 1026 goto bad_page; 1027 1028 if (flags & FOLL_GET) 1029 get_page(page); 1030 if (flags & FOLL_TOUCH) { 1031 if ((flags & FOLL_WRITE) && 1032 !pte_dirty(pte) && !PageDirty(page)) 1033 set_page_dirty(page); 1034 mark_page_accessed(page); 1035 } 1036unlock: 1037 pte_unmap_unlock(ptep, ptl); 1038out: 1039 return page; 1040 1041bad_page: 1042 pte_unmap_unlock(ptep, ptl); 1043 return ERR_PTR(-EFAULT); 1044 1045no_page: 1046 pte_unmap_unlock(ptep, ptl); 1047 if (!pte_none(pte)) 1048 return page; 1049 /* Fall through to ZERO_PAGE handling */ 1050no_page_table: 1051 /* 1052 * When core dumping an enormous anonymous area that nobody 1053 * has touched so far, we don't want to allocate page tables. 1054 */ 1055 if (flags & FOLL_ANON) { 1056 page = ZERO_PAGE(0); 1057 if (flags & FOLL_GET) 1058 get_page(page); 1059 BUG_ON(flags & FOLL_WRITE); 1060 } 1061 return page; 1062} 1063 1064/* Can we do the FOLL_ANON optimization? */ 1065static inline int use_zero_page(struct vm_area_struct *vma) 1066{ 1067 /* 1068 * We don't want to optimize FOLL_ANON for make_pages_present() 1069 * when it tries to page in a VM_LOCKED region. As to VM_SHARED, 1070 * we want to get the page from the page tables to make sure 1071 * that we serialize and update with any other user of that 1072 * mapping. 1073 */ 1074 if (vma->vm_flags & (VM_LOCKED | VM_SHARED)) 1075 return 0; 1076 /* 1077 * And if we have a fault routine, it's not an anonymous region. 1078 */ 1079 return !vma->vm_ops || !vma->vm_ops->fault; 1080} 1081 1082int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1083 unsigned long start, int len, int write, int force, 1084 struct page **pages, struct vm_area_struct **vmas) 1085{ 1086 int i; 1087 unsigned int vm_flags; 1088 1089 if (len <= 0) 1090 return 0; 1091 /* 1092 * Require read or write permissions. 1093 * If 'force' is set, we only require the "MAY" flags. 1094 */ 1095 vm_flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 1096 vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 1097 i = 0; 1098 1099 do { 1100 struct vm_area_struct *vma; 1101 unsigned int foll_flags; 1102 1103 vma = find_extend_vma(mm, start); 1104 if (!vma && in_gate_area(tsk, start)) { 1105 unsigned long pg = start & PAGE_MASK; 1106 struct vm_area_struct *gate_vma = get_gate_vma(tsk); 1107 pgd_t *pgd; 1108 pud_t *pud; 1109 pmd_t *pmd; 1110 pte_t *pte; 1111 if (write) /* user gate pages are read-only */ 1112 return i ? : -EFAULT; 1113 if (pg > TASK_SIZE) 1114 pgd = pgd_offset_k(pg); 1115 else 1116 pgd = pgd_offset_gate(mm, pg); 1117 BUG_ON(pgd_none(*pgd)); 1118 pud = pud_offset(pgd, pg); 1119 BUG_ON(pud_none(*pud)); 1120 pmd = pmd_offset(pud, pg); 1121 if (pmd_none(*pmd)) 1122 return i ? : -EFAULT; 1123 pte = pte_offset_map(pmd, pg); 1124 if (pte_none(*pte)) { 1125 pte_unmap(pte); 1126 return i ? : -EFAULT; 1127 } 1128 if (pages) { 1129 struct page *page = vm_normal_page(gate_vma, start, *pte); 1130 pages[i] = page; 1131 if (page) 1132 get_page(page); 1133 } 1134 pte_unmap(pte); 1135 if (vmas) 1136 vmas[i] = gate_vma; 1137 i++; 1138 start += PAGE_SIZE; 1139 len--; 1140 continue; 1141 } 1142 1143 if (!vma || (vma->vm_flags & (VM_IO | VM_PFNMAP)) 1144 || !(vm_flags & vma->vm_flags)) 1145 return i ? : -EFAULT; 1146 1147 if (is_vm_hugetlb_page(vma)) { 1148 i = follow_hugetlb_page(mm, vma, pages, vmas, 1149 &start, &len, i, write); 1150 continue; 1151 } 1152 1153 foll_flags = FOLL_TOUCH; 1154 if (pages) 1155 foll_flags |= FOLL_GET; 1156 if (!write && use_zero_page(vma)) 1157 foll_flags |= FOLL_ANON; 1158 1159 do { 1160 struct page *page; 1161 1162 /* 1163 * If tsk is ooming, cut off its access to large memory 1164 * allocations. It has a pending SIGKILL, but it can't 1165 * be processed until returning to user space. 1166 */ 1167 if (unlikely(test_tsk_thread_flag(tsk, TIF_MEMDIE))) 1168 return i ? i : -ENOMEM; 1169 1170 if (write) 1171 foll_flags |= FOLL_WRITE; 1172 1173 cond_resched(); 1174 while (!(page = follow_page(vma, start, foll_flags))) { 1175 int ret; 1176 ret = handle_mm_fault(mm, vma, start, 1177 foll_flags & FOLL_WRITE); 1178 if (ret & VM_FAULT_ERROR) { 1179 if (ret & VM_FAULT_OOM) 1180 return i ? i : -ENOMEM; 1181 else if (ret & VM_FAULT_SIGBUS) 1182 return i ? i : -EFAULT; 1183 BUG(); 1184 } 1185 if (ret & VM_FAULT_MAJOR) 1186 tsk->maj_flt++; 1187 else 1188 tsk->min_flt++; 1189 1190 /* 1191 * The VM_FAULT_WRITE bit tells us that 1192 * do_wp_page has broken COW when necessary, 1193 * even if maybe_mkwrite decided not to set 1194 * pte_write. We can thus safely do subsequent 1195 * page lookups as if they were reads. 1196 */ 1197 if (ret & VM_FAULT_WRITE) 1198 foll_flags &= ~FOLL_WRITE; 1199 1200 cond_resched(); 1201 } 1202 if (IS_ERR(page)) 1203 return i ? i : PTR_ERR(page); 1204 if (pages) { 1205 pages[i] = page; 1206 1207 flush_anon_page(vma, page, start); 1208 flush_dcache_page(page); 1209 } 1210 if (vmas) 1211 vmas[i] = vma; 1212 i++; 1213 start += PAGE_SIZE; 1214 len--; 1215 } while (len && start < vma->vm_end); 1216 } while (len); 1217 return i; 1218} 1219EXPORT_SYMBOL(get_user_pages); 1220 1221pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, 1222 spinlock_t **ptl) 1223{ 1224 pgd_t * pgd = pgd_offset(mm, addr); 1225 pud_t * pud = pud_alloc(mm, pgd, addr); 1226 if (pud) { 1227 pmd_t * pmd = pmd_alloc(mm, pud, addr); 1228 if (pmd) 1229 return pte_alloc_map_lock(mm, pmd, addr, ptl); 1230 } 1231 return NULL; 1232} 1233 1234/* 1235 * This is the old fallback for page remapping. 1236 * 1237 * For historical reasons, it only allows reserved pages. Only 1238 * old drivers should use this, and they needed to mark their 1239 * pages reserved for the old functions anyway. 1240 */ 1241static int insert_page(struct vm_area_struct *vma, unsigned long addr, 1242 struct page *page, pgprot_t prot) 1243{ 1244 struct mm_struct *mm = vma->vm_mm; 1245 int retval; 1246 pte_t *pte; 1247 spinlock_t *ptl; 1248 1249 retval = mem_cgroup_charge(page, mm, GFP_KERNEL); 1250 if (retval) 1251 goto out; 1252 1253 retval = -EINVAL; 1254 if (PageAnon(page)) 1255 goto out_uncharge; 1256 retval = -ENOMEM; 1257 flush_dcache_page(page); 1258 pte = get_locked_pte(mm, addr, &ptl); 1259 if (!pte) 1260 goto out_uncharge; 1261 retval = -EBUSY; 1262 if (!pte_none(*pte)) 1263 goto out_unlock; 1264 1265 /* Ok, finally just insert the thing.. */ 1266 get_page(page); 1267 inc_mm_counter(mm, file_rss); 1268 page_add_file_rmap(page); 1269 set_pte_at(mm, addr, pte, mk_pte(page, prot)); 1270 1271 retval = 0; 1272 pte_unmap_unlock(pte, ptl); 1273 return retval; 1274out_unlock: 1275 pte_unmap_unlock(pte, ptl); 1276out_uncharge: 1277 mem_cgroup_uncharge_page(page); 1278out: 1279 return retval; 1280} 1281 1282/** 1283 * vm_insert_page - insert single page into user vma 1284 * @vma: user vma to map to 1285 * @addr: target user address of this page 1286 * @page: source kernel page 1287 * 1288 * This allows drivers to insert individual pages they've allocated 1289 * into a user vma. 1290 * 1291 * The page has to be a nice clean _individual_ kernel allocation. 1292 * If you allocate a compound page, you need to have marked it as 1293 * such (__GFP_COMP), or manually just split the page up yourself 1294 * (see split_page()). 1295 * 1296 * NOTE! Traditionally this was done with "remap_pfn_range()" which 1297 * took an arbitrary page protection parameter. This doesn't allow 1298 * that. Your vma protection will have to be set up correctly, which 1299 * means that if you want a shared writable mapping, you'd better 1300 * ask for a shared writable mapping! 1301 * 1302 * The page does not need to be reserved. 1303 */ 1304int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 1305 struct page *page) 1306{ 1307 if (addr < vma->vm_start || addr >= vma->vm_end) 1308 return -EFAULT; 1309 if (!page_count(page)) 1310 return -EINVAL; 1311 vma->vm_flags |= VM_INSERTPAGE; 1312 return insert_page(vma, addr, page, vma->vm_page_prot); 1313} 1314EXPORT_SYMBOL(vm_insert_page); 1315 1316static int insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1317 unsigned long pfn, pgprot_t prot) 1318{ 1319 struct mm_struct *mm = vma->vm_mm; 1320 int retval; 1321 pte_t *pte, entry; 1322 spinlock_t *ptl; 1323 1324 retval = -ENOMEM; 1325 pte = get_locked_pte(mm, addr, &ptl); 1326 if (!pte) 1327 goto out; 1328 retval = -EBUSY; 1329 if (!pte_none(*pte)) 1330 goto out_unlock; 1331 1332 /* Ok, finally just insert the thing.. */ 1333 entry = pte_mkspecial(pfn_pte(pfn, prot)); 1334 set_pte_at(mm, addr, pte, entry); 1335 update_mmu_cache(vma, addr, entry); /* XXX: why not for insert_page? */ 1336 1337 retval = 0; 1338out_unlock: 1339 pte_unmap_unlock(pte, ptl); 1340out: 1341 return retval; 1342} 1343 1344/** 1345 * vm_insert_pfn - insert single pfn into user vma 1346 * @vma: user vma to map to 1347 * @addr: target user address of this page 1348 * @pfn: source kernel pfn 1349 * 1350 * Similar to vm_inert_page, this allows drivers to insert individual pages 1351 * they've allocated into a user vma. Same comments apply. 1352 * 1353 * This function should only be called from a vm_ops->fault handler, and 1354 * in that case the handler should return NULL. 1355 * 1356 * vma cannot be a COW mapping. 1357 * 1358 * As this is called only for pages that do not currently exist, we 1359 * do not need to flush old virtual caches or the TLB. 1360 */ 1361int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1362 unsigned long pfn) 1363{ 1364 /* 1365 * Technically, architectures with pte_special can avoid all these 1366 * restrictions (same for remap_pfn_range). However we would like 1367 * consistency in testing and feature parity among all, so we should 1368 * try to keep these invariants in place for everybody. 1369 */ 1370 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1371 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1372 (VM_PFNMAP|VM_MIXEDMAP)); 1373 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1374 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); 1375 1376 if (addr < vma->vm_start || addr >= vma->vm_end) 1377 return -EFAULT; 1378 return insert_pfn(vma, addr, pfn, vma->vm_page_prot); 1379} 1380EXPORT_SYMBOL(vm_insert_pfn); 1381 1382int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 1383 unsigned long pfn) 1384{ 1385 BUG_ON(!(vma->vm_flags & VM_MIXEDMAP)); 1386 1387 if (addr < vma->vm_start || addr >= vma->vm_end) 1388 return -EFAULT; 1389 1390 /* 1391 * If we don't have pte special, then we have to use the pfn_valid() 1392 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must* 1393 * refcount the page if pfn_valid is true (hence insert_page rather 1394 * than insert_pfn). 1395 */ 1396 if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) { 1397 struct page *page; 1398 1399 page = pfn_to_page(pfn); 1400 return insert_page(vma, addr, page, vma->vm_page_prot); 1401 } 1402 return insert_pfn(vma, addr, pfn, vma->vm_page_prot); 1403} 1404EXPORT_SYMBOL(vm_insert_mixed); 1405 1406/* 1407 * maps a range of physical memory into the requested pages. the old 1408 * mappings are removed. any references to nonexistent pages results 1409 * in null mappings (currently treated as "copy-on-access") 1410 */ 1411static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, 1412 unsigned long addr, unsigned long end, 1413 unsigned long pfn, pgprot_t prot) 1414{ 1415 pte_t *pte; 1416 spinlock_t *ptl; 1417 1418 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); 1419 if (!pte) 1420 return -ENOMEM; 1421 arch_enter_lazy_mmu_mode(); 1422 do { 1423 BUG_ON(!pte_none(*pte)); 1424 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); 1425 pfn++; 1426 } while (pte++, addr += PAGE_SIZE, addr != end); 1427 arch_leave_lazy_mmu_mode(); 1428 pte_unmap_unlock(pte - 1, ptl); 1429 return 0; 1430} 1431 1432static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, 1433 unsigned long addr, unsigned long end, 1434 unsigned long pfn, pgprot_t prot) 1435{ 1436 pmd_t *pmd; 1437 unsigned long next; 1438 1439 pfn -= addr >> PAGE_SHIFT; 1440 pmd = pmd_alloc(mm, pud, addr); 1441 if (!pmd) 1442 return -ENOMEM; 1443 do { 1444 next = pmd_addr_end(addr, end); 1445 if (remap_pte_range(mm, pmd, addr, next, 1446 pfn + (addr >> PAGE_SHIFT), prot)) 1447 return -ENOMEM; 1448 } while (pmd++, addr = next, addr != end); 1449 return 0; 1450} 1451 1452static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd, 1453 unsigned long addr, unsigned long end, 1454 unsigned long pfn, pgprot_t prot) 1455{ 1456 pud_t *pud; 1457 unsigned long next; 1458 1459 pfn -= addr >> PAGE_SHIFT; 1460 pud = pud_alloc(mm, pgd, addr); 1461 if (!pud) 1462 return -ENOMEM; 1463 do { 1464 next = pud_addr_end(addr, end); 1465 if (remap_pmd_range(mm, pud, addr, next, 1466 pfn + (addr >> PAGE_SHIFT), prot)) 1467 return -ENOMEM; 1468 } while (pud++, addr = next, addr != end); 1469 return 0; 1470} 1471 1472/** 1473 * remap_pfn_range - remap kernel memory to userspace 1474 * @vma: user vma to map to 1475 * @addr: target user address to start at 1476 * @pfn: physical address of kernel memory 1477 * @size: size of map area 1478 * @prot: page protection flags for this mapping 1479 * 1480 * Note: this is only safe if the mm semaphore is held when called. 1481 */ 1482int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 1483 unsigned long pfn, unsigned long size, pgprot_t prot) 1484{ 1485 pgd_t *pgd; 1486 unsigned long next; 1487 unsigned long end = addr + PAGE_ALIGN(size); 1488 struct mm_struct *mm = vma->vm_mm; 1489 int err; 1490 1491 /* 1492 * Physically remapped pages are special. Tell the 1493 * rest of the world about it: 1494 * VM_IO tells people not to look at these pages 1495 * (accesses can have side effects). 1496 * VM_RESERVED is specified all over the place, because 1497 * in 2.4 it kept swapout's vma scan off this vma; but 1498 * in 2.6 the LRU scan won't even find its pages, so this 1499 * flag means no more than count its pages in reserved_vm, 1500 * and omit it from core dump, even when VM_IO turned off. 1501 * VM_PFNMAP tells the core MM that the base pages are just 1502 * raw PFN mappings, and do not have a "struct page" associated 1503 * with them. 1504 * 1505 * There's a horrible special case to handle copy-on-write 1506 * behaviour that some programs depend on. We mark the "original" 1507 * un-COW'ed pages by matching them up with "vma->vm_pgoff". 1508 */ 1509 if (is_cow_mapping(vma->vm_flags)) { 1510 if (addr != vma->vm_start || end != vma->vm_end) 1511 return -EINVAL; 1512 vma->vm_pgoff = pfn; 1513 } 1514 1515 vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; 1516 1517 BUG_ON(addr >= end); 1518 pfn -= addr >> PAGE_SHIFT; 1519 pgd = pgd_offset(mm, addr); 1520 flush_cache_range(vma, addr, end); 1521 do { 1522 next = pgd_addr_end(addr, end); 1523 err = remap_pud_range(mm, pgd, addr, next, 1524 pfn + (addr >> PAGE_SHIFT), prot); 1525 if (err) 1526 break; 1527 } while (pgd++, addr = next, addr != end); 1528 return err; 1529} 1530EXPORT_SYMBOL(remap_pfn_range); 1531 1532static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, 1533 unsigned long addr, unsigned long end, 1534 pte_fn_t fn, void *data) 1535{ 1536 pte_t *pte; 1537 int err; 1538 pgtable_t token; 1539 spinlock_t *uninitialized_var(ptl); 1540 1541 pte = (mm == &init_mm) ? 1542 pte_alloc_kernel(pmd, addr) : 1543 pte_alloc_map_lock(mm, pmd, addr, &ptl); 1544 if (!pte) 1545 return -ENOMEM; 1546 1547 BUG_ON(pmd_huge(*pmd)); 1548 1549 token = pmd_pgtable(*pmd); 1550 1551 do { 1552 err = fn(pte, token, addr, data); 1553 if (err) 1554 break; 1555 } while (pte++, addr += PAGE_SIZE, addr != end); 1556 1557 if (mm != &init_mm) 1558 pte_unmap_unlock(pte-1, ptl); 1559 return err; 1560} 1561 1562static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, 1563 unsigned long addr, unsigned long end, 1564 pte_fn_t fn, void *data) 1565{ 1566 pmd_t *pmd; 1567 unsigned long next; 1568 int err; 1569 1570 pmd = pmd_alloc(mm, pud, addr); 1571 if (!pmd) 1572 return -ENOMEM; 1573 do { 1574 next = pmd_addr_end(addr, end); 1575 err = apply_to_pte_range(mm, pmd, addr, next, fn, data); 1576 if (err) 1577 break; 1578 } while (pmd++, addr = next, addr != end); 1579 return err; 1580} 1581 1582static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd, 1583 unsigned long addr, unsigned long end, 1584 pte_fn_t fn, void *data) 1585{ 1586 pud_t *pud; 1587 unsigned long next; 1588 int err; 1589 1590 pud = pud_alloc(mm, pgd, addr); 1591 if (!pud) 1592 return -ENOMEM; 1593 do { 1594 next = pud_addr_end(addr, end); 1595 err = apply_to_pmd_range(mm, pud, addr, next, fn, data); 1596 if (err) 1597 break; 1598 } while (pud++, addr = next, addr != end); 1599 return err; 1600} 1601 1602/* 1603 * Scan a region of virtual memory, filling in page tables as necessary 1604 * and calling a provided function on each leaf page table. 1605 */ 1606int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 1607 unsigned long size, pte_fn_t fn, void *data) 1608{ 1609 pgd_t *pgd; 1610 unsigned long next; 1611 unsigned long end = addr + size; 1612 int err; 1613 1614 BUG_ON(addr >= end); 1615 pgd = pgd_offset(mm, addr); 1616 do { 1617 next = pgd_addr_end(addr, end); 1618 err = apply_to_pud_range(mm, pgd, addr, next, fn, data); 1619 if (err) 1620 break; 1621 } while (pgd++, addr = next, addr != end); 1622 return err; 1623} 1624EXPORT_SYMBOL_GPL(apply_to_page_range); 1625 1626/* 1627 * handle_pte_fault chooses page fault handler according to an entry 1628 * which was read non-atomically. Before making any commitment, on 1629 * those architectures or configurations (e.g. i386 with PAE) which 1630 * might give a mix of unmatched parts, do_swap_page and do_file_page 1631 * must check under lock before unmapping the pte and proceeding 1632 * (but do_wp_page is only called after already making such a check; 1633 * and do_anonymous_page and do_no_page can safely check later on). 1634 */ 1635static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd, 1636 pte_t *page_table, pte_t orig_pte) 1637{ 1638 int same = 1; 1639#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) 1640 if (sizeof(pte_t) > sizeof(unsigned long)) { 1641 spinlock_t *ptl = pte_lockptr(mm, pmd); 1642 spin_lock(ptl); 1643 same = pte_same(*page_table, orig_pte); 1644 spin_unlock(ptl); 1645 } 1646#endif 1647 pte_unmap(page_table); 1648 return same; 1649} 1650 1651/* 1652 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when 1653 * servicing faults for write access. In the normal case, do always want 1654 * pte_mkwrite. But get_user_pages can cause write faults for mappings 1655 * that do not have writing enabled, when used by access_process_vm. 1656 */ 1657static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 1658{ 1659 if (likely(vma->vm_flags & VM_WRITE)) 1660 pte = pte_mkwrite(pte); 1661 return pte; 1662} 1663 1664static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) 1665{ 1666 /* 1667 * If the source page was a PFN mapping, we don't have 1668 * a "struct page" for it. We do a best-effort copy by 1669 * just copying from the original user address. If that 1670 * fails, we just zero-fill it. Live with it. 1671 */ 1672 if (unlikely(!src)) { 1673 void *kaddr = kmap_atomic(dst, KM_USER0); 1674 void __user *uaddr = (void __user *)(va & PAGE_MASK); 1675 1676 /* 1677 * This really shouldn't fail, because the page is there 1678 * in the page tables. But it might just be unreadable, 1679 * in which case we just give up and fill the result with 1680 * zeroes. 1681 */ 1682 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) 1683 memset(kaddr, 0, PAGE_SIZE); 1684 kunmap_atomic(kaddr, KM_USER0); 1685 flush_dcache_page(dst); 1686 } else 1687 copy_user_highpage(dst, src, va, vma); 1688} 1689 1690/* 1691 * This routine handles present pages, when users try to write 1692 * to a shared page. It is done by copying the page to a new address 1693 * and decrementing the shared-page counter for the old page. 1694 * 1695 * Note that this routine assumes that the protection checks have been 1696 * done by the caller (the low-level page fault routine in most cases). 1697 * Thus we can safely just mark it writable once we've done any necessary 1698 * COW. 1699 * 1700 * We also mark the page dirty at this point even though the page will 1701 * change only once the write actually happens. This avoids a few races, 1702 * and potentially makes it more efficient. 1703 * 1704 * We enter with non-exclusive mmap_sem (to exclude vma changes, 1705 * but allow concurrent faults), with pte both mapped and locked. 1706 * We return with mmap_sem still held, but pte unmapped and unlocked. 1707 */ 1708static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 1709 unsigned long address, pte_t *page_table, pmd_t *pmd, 1710 spinlock_t *ptl, pte_t orig_pte) 1711{ 1712 struct page *old_page, *new_page; 1713 pte_t entry; 1714 int reuse = 0, ret = 0; 1715 int page_mkwrite = 0; 1716 struct page *dirty_page = NULL; 1717 1718 old_page = vm_normal_page(vma, address, orig_pte); 1719 if (!old_page) { 1720 /* 1721 * VM_MIXEDMAP !pfn_valid() case 1722 * 1723 * We should not cow pages in a shared writeable mapping. 1724 * Just mark the pages writable as we can't do any dirty 1725 * accounting on raw pfn maps. 1726 */ 1727 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 1728 (VM_WRITE|VM_SHARED)) 1729 goto reuse; 1730 goto gotten; 1731 } 1732 1733 /* 1734 * Take out anonymous pages first, anonymous shared vmas are 1735 * not dirty accountable. 1736 */ 1737 if (PageAnon(old_page)) { 1738 if (!TestSetPageLocked(old_page)) { 1739 reuse = can_share_swap_page(old_page); 1740 unlock_page(old_page); 1741 } 1742 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 1743 (VM_WRITE|VM_SHARED))) { 1744 /* 1745 * Only catch write-faults on shared writable pages, 1746 * read-only shared pages can get COWed by 1747 * get_user_pages(.write=1, .force=1). 1748 */ 1749 if (vma->vm_ops && vma->vm_ops->page_mkwrite) { 1750 /* 1751 * Notify the address space that the page is about to 1752 * become writable so that it can prohibit this or wait 1753 * for the page to get into an appropriate state. 1754 * 1755 * We do this without the lock held, so that it can 1756 * sleep if it needs to. 1757 */ 1758 page_cache_get(old_page); 1759 pte_unmap_unlock(page_table, ptl); 1760 1761 if (vma->vm_ops->page_mkwrite(vma, old_page) < 0) 1762 goto unwritable_page; 1763 1764 /* 1765 * Since we dropped the lock we need to revalidate 1766 * the PTE as someone else may have changed it. If 1767 * they did, we just return, as we can count on the 1768 * MMU to tell us if they didn't also make it writable. 1769 */ 1770 page_table = pte_offset_map_lock(mm, pmd, address, 1771 &ptl); 1772 page_cache_release(old_page); 1773 if (!pte_same(*page_table, orig_pte)) 1774 goto unlock; 1775 1776 page_mkwrite = 1; 1777 } 1778 dirty_page = old_page; 1779 get_page(dirty_page); 1780 reuse = 1; 1781 } 1782 1783 if (reuse) { 1784reuse: 1785 flush_cache_page(vma, address, pte_pfn(orig_pte)); 1786 entry = pte_mkyoung(orig_pte); 1787 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 1788 if (ptep_set_access_flags(vma, address, page_table, entry,1)) 1789 update_mmu_cache(vma, address, entry); 1790 ret |= VM_FAULT_WRITE; 1791 goto unlock; 1792 } 1793 1794 /* 1795 * Ok, we need to copy. Oh, well.. 1796 */ 1797 page_cache_get(old_page); 1798gotten: 1799 pte_unmap_unlock(page_table, ptl); 1800 1801 if (unlikely(anon_vma_prepare(vma))) 1802 goto oom; 1803 VM_BUG_ON(old_page == ZERO_PAGE(0)); 1804 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); 1805 if (!new_page) 1806 goto oom; 1807 cow_user_page(new_page, old_page, address, vma); 1808 __SetPageUptodate(new_page); 1809 1810 if (mem_cgroup_charge(new_page, mm, GFP_KERNEL)) 1811 goto oom_free_new; 1812 1813 /* 1814 * Re-check the pte - we dropped the lock 1815 */ 1816 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 1817 if (likely(pte_same(*page_table, orig_pte))) { 1818 if (old_page) { 1819 if (!PageAnon(old_page)) { 1820 dec_mm_counter(mm, file_rss); 1821 inc_mm_counter(mm, anon_rss); 1822 } 1823 } else 1824 inc_mm_counter(mm, anon_rss); 1825 flush_cache_page(vma, address, pte_pfn(orig_pte)); 1826 entry = mk_pte(new_page, vma->vm_page_prot); 1827 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 1828 /* 1829 * Clear the pte entry and flush it first, before updating the 1830 * pte with the new entry. This will avoid a race condition 1831 * seen in the presence of one thread doing SMC and another 1832 * thread doing COW. 1833 */ 1834 ptep_clear_flush(vma, address, page_table); 1835 set_pte_at(mm, address, page_table, entry); 1836 update_mmu_cache(vma, address, entry); 1837 lru_cache_add_active(new_page); 1838 page_add_new_anon_rmap(new_page, vma, address); 1839 1840 if (old_page) { 1841 /* 1842 * Only after switching the pte to the new page may 1843 * we remove the mapcount here. Otherwise another 1844 * process may come and find the rmap count decremented 1845 * before the pte is switched to the new page, and 1846 * "reuse" the old page writing into it while our pte 1847 * here still points into it and can be read by other 1848 * threads. 1849 * 1850 * The critical issue is to order this 1851 * page_remove_rmap with the ptp_clear_flush above. 1852 * Those stores are ordered by (if nothing else,) 1853 * the barrier present in the atomic_add_negative 1854 * in page_remove_rmap. 1855 * 1856 * Then the TLB flush in ptep_clear_flush ensures that 1857 * no process can access the old page before the 1858 * decremented mapcount is visible. And the old page 1859 * cannot be reused until after the decremented 1860 * mapcount is visible. So transitively, TLBs to 1861 * old page will be flushed before it can be reused. 1862 */ 1863 page_remove_rmap(old_page, vma); 1864 } 1865 1866 /* Free the old page.. */ 1867 new_page = old_page; 1868 ret |= VM_FAULT_WRITE; 1869 } else 1870 mem_cgroup_uncharge_page(new_page); 1871 1872 if (new_page) 1873 page_cache_release(new_page); 1874 if (old_page) 1875 page_cache_release(old_page); 1876unlock: 1877 pte_unmap_unlock(page_table, ptl); 1878 if (dirty_page) { 1879 if (vma->vm_file) 1880 file_update_time(vma->vm_file); 1881 1882 /* 1883 * Yes, Virginia, this is actually required to prevent a race 1884 * with clear_page_dirty_for_io() from clearing the page dirty 1885 * bit after it clear all dirty ptes, but before a racing 1886 * do_wp_page installs a dirty pte. 1887 * 1888 * do_no_page is protected similarly. 1889 */ 1890 wait_on_page_locked(dirty_page); 1891 set_page_dirty_balance(dirty_page, page_mkwrite); 1892 put_page(dirty_page); 1893 } 1894 return ret; 1895oom_free_new: 1896 page_cache_release(new_page); 1897oom: 1898 if (old_page) 1899 page_cache_release(old_page); 1900 return VM_FAULT_OOM; 1901 1902unwritable_page: 1903 page_cache_release(old_page); 1904 return VM_FAULT_SIGBUS; 1905} 1906 1907/* 1908 * Helper functions for unmap_mapping_range(). 1909 * 1910 * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __ 1911 * 1912 * We have to restart searching the prio_tree whenever we drop the lock, 1913 * since the iterator is only valid while the lock is held, and anyway 1914 * a later vma might be split and reinserted earlier while lock dropped. 1915 * 1916 * The list of nonlinear vmas could be handled more efficiently, using 1917 * a placeholder, but handle it in the same way until a need is shown. 1918 * It is important to search the prio_tree before nonlinear list: a vma 1919 * may become nonlinear and be shifted from prio_tree to nonlinear list 1920 * while the lock is dropped; but never shifted from list to prio_tree. 1921 * 1922 * In order to make forward progress despite restarting the search, 1923 * vm_truncate_count is used to mark a vma as now dealt with, so we can 1924 * quickly skip it next time around. Since the prio_tree search only 1925 * shows us those vmas affected by unmapping the range in question, we 1926 * can't efficiently keep all vmas in step with mapping->truncate_count: 1927 * so instead reset them all whenever it wraps back to 0 (then go to 1). 1928 * mapping->truncate_count and vma->vm_truncate_count are protected by 1929 * i_mmap_lock. 1930 * 1931 * In order to make forward progress despite repeatedly restarting some 1932 * large vma, note the restart_addr from unmap_vmas when it breaks out: 1933 * and restart from that address when we reach that vma again. It might 1934 * have been split or merged, shrunk or extended, but never shifted: so 1935 * restart_addr remains valid so long as it remains in the vma's range. 1936 * unmap_mapping_range forces truncate_count to leap over page-aligned 1937 * values so we can save vma's restart_addr in its truncate_count field. 1938 */ 1939#define is_restart_addr(truncate_count) (!((truncate_count) & ~PAGE_MASK)) 1940 1941static void reset_vma_truncate_counts(struct address_space *mapping) 1942{ 1943 struct vm_area_struct *vma; 1944 struct prio_tree_iter iter; 1945 1946 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX) 1947 vma->vm_truncate_count = 0; 1948 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list) 1949 vma->vm_truncate_count = 0; 1950} 1951 1952static int unmap_mapping_range_vma(struct vm_area_struct *vma, 1953 unsigned long start_addr, unsigned long end_addr, 1954 struct zap_details *details) 1955{ 1956 unsigned long restart_addr; 1957 int need_break; 1958 1959 /* 1960 * files that support invalidating or truncating portions of the 1961 * file from under mmaped areas must have their ->fault function 1962 * return a locked page (and set VM_FAULT_LOCKED in the return). 1963 * This provides synchronisation against concurrent unmapping here. 1964 */ 1965 1966again: 1967 restart_addr = vma->vm_truncate_count; 1968 if (is_restart_addr(restart_addr) && start_addr < restart_addr) { 1969 start_addr = restart_addr; 1970 if (start_addr >= end_addr) { 1971 /* Top of vma has been split off since last time */ 1972 vma->vm_truncate_count = details->truncate_count; 1973 return 0; 1974 } 1975 } 1976 1977 restart_addr = zap_page_range(vma, start_addr, 1978 end_addr - start_addr, details); 1979 need_break = need_resched() || spin_needbreak(details->i_mmap_lock); 1980 1981 if (restart_addr >= end_addr) { 1982 /* We have now completed this vma: mark it so */ 1983 vma->vm_truncate_count = details->truncate_count; 1984 if (!need_break) 1985 return 0; 1986 } else { 1987 /* Note restart_addr in vma's truncate_count field */ 1988 vma->vm_truncate_count = restart_addr; 1989 if (!need_break) 1990 goto again; 1991 } 1992 1993 spin_unlock(details->i_mmap_lock); 1994 cond_resched(); 1995 spin_lock(details->i_mmap_lock); 1996 return -EINTR; 1997} 1998 1999static inline void unmap_mapping_range_tree(struct prio_tree_root *root, 2000 struct zap_details *details) 2001{ 2002 struct vm_area_struct *vma; 2003 struct prio_tree_iter iter; 2004 pgoff_t vba, vea, zba, zea; 2005 2006restart: 2007 vma_prio_tree_foreach(vma, &iter, root, 2008 details->first_index, details->last_index) { 2009 /* Skip quickly over those we have already dealt with */ 2010 if (vma->vm_truncate_count == details->truncate_count) 2011 continue; 2012 2013 vba = vma->vm_pgoff; 2014 vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1; 2015 /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */ 2016 zba = details->first_index; 2017 if (zba < vba) 2018 zba = vba; 2019 zea = details->last_index; 2020 if (zea > vea) 2021 zea = vea; 2022 2023 if (unmap_mapping_range_vma(vma, 2024 ((zba - vba) << PAGE_SHIFT) + vma->vm_start, 2025 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, 2026 details) < 0) 2027 goto restart; 2028 } 2029} 2030 2031static inline void unmap_mapping_range_list(struct list_head *head, 2032 struct zap_details *details) 2033{ 2034 struct vm_area_struct *vma; 2035 2036 /* 2037 * In nonlinear VMAs there is no correspondence between virtual address 2038 * offset and file offset. So we must perform an exhaustive search 2039 * across *all* the pages in each nonlinear VMA, not just the pages 2040 * whose virtual address lies outside the file truncation point. 2041 */ 2042restart: 2043 list_for_each_entry(vma, head, shared.vm_set.list) { 2044 /* Skip quickly over those we have already dealt with */ 2045 if (vma->vm_truncate_count == details->truncate_count) 2046 continue; 2047 details->nonlinear_vma = vma; 2048 if (unmap_mapping_range_vma(vma, vma->vm_start, 2049 vma->vm_end, details) < 0) 2050 goto restart; 2051 } 2052} 2053 2054/** 2055 * unmap_mapping_range - unmap the portion of all mmaps in the specified address_space corresponding to the specified page range in the underlying file. 2056 * @mapping: the address space containing mmaps to be unmapped. 2057 * @holebegin: byte in first page to unmap, relative to the start of 2058 * the underlying file. This will be rounded down to a PAGE_SIZE 2059 * boundary. Note that this is different from vmtruncate(), which 2060 * must keep the partial page. In contrast, we must get rid of 2061 * partial pages. 2062 * @holelen: size of prospective hole in bytes. This will be rounded 2063 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the 2064 * end of the file. 2065 * @even_cows: 1 when truncating a file, unmap even private COWed pages; 2066 * but 0 when invalidating pagecache, don't throw away private data. 2067 */ 2068void unmap_mapping_range(struct address_space *mapping, 2069 loff_t const holebegin, loff_t const holelen, int even_cows) 2070{ 2071 struct zap_details details; 2072 pgoff_t hba = holebegin >> PAGE_SHIFT; 2073 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2074 2075 /* Check for overflow. */ 2076 if (sizeof(holelen) > sizeof(hlen)) { 2077 long long holeend = 2078 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2079 if (holeend & ~(long long)ULONG_MAX) 2080 hlen = ULONG_MAX - hba + 1; 2081 } 2082 2083 details.check_mapping = even_cows? NULL: mapping; 2084 details.nonlinear_vma = NULL; 2085 details.first_index = hba; 2086 details.last_index = hba + hlen - 1; 2087 if (details.last_index < details.first_index) 2088 details.last_index = ULONG_MAX; 2089 details.i_mmap_lock = &mapping->i_mmap_lock; 2090 2091 spin_lock(&mapping->i_mmap_lock); 2092 2093 /* Protect against endless unmapping loops */ 2094 mapping->truncate_count++; 2095 if (unlikely(is_restart_addr(mapping->truncate_count))) { 2096 if (mapping->truncate_count == 0) 2097 reset_vma_truncate_counts(mapping); 2098 mapping->truncate_count++; 2099 } 2100 details.truncate_count = mapping->truncate_count; 2101 2102 if (unlikely(!prio_tree_empty(&mapping->i_mmap))) 2103 unmap_mapping_range_tree(&mapping->i_mmap, &details); 2104 if (unlikely(!list_empty(&mapping->i_mmap_nonlinear))) 2105 unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details); 2106 spin_unlock(&mapping->i_mmap_lock); 2107} 2108EXPORT_SYMBOL(unmap_mapping_range); 2109 2110/** 2111 * vmtruncate - unmap mappings "freed" by truncate() syscall 2112 * @inode: inode of the file used 2113 * @offset: file offset to start truncating 2114 * 2115 * NOTE! We have to be ready to update the memory sharing 2116 * between the file and the memory map for a potential last 2117 * incomplete page. Ugly, but necessary. 2118 */ 2119int vmtruncate(struct inode * inode, loff_t offset) 2120{ 2121 if (inode->i_size < offset) { 2122 unsigned long limit; 2123 2124 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; 2125 if (limit != RLIM_INFINITY && offset > limit) 2126 goto out_sig; 2127 if (offset > inode->i_sb->s_maxbytes) 2128 goto out_big; 2129 i_size_write(inode, offset); 2130 } else { 2131 struct address_space *mapping = inode->i_mapping; 2132 2133 /* 2134 * truncation of in-use swapfiles is disallowed - it would 2135 * cause subsequent swapout to scribble on the now-freed 2136 * blocks. 2137 */ 2138 if (IS_SWAPFILE(inode)) 2139 return -ETXTBSY; 2140 i_size_write(inode, offset); 2141 2142 /* 2143 * unmap_mapping_range is called twice, first simply for 2144 * efficiency so that truncate_inode_pages does fewer 2145 * single-page unmaps. However after this first call, and 2146 * before truncate_inode_pages finishes, it is possible for 2147 * private pages to be COWed, which remain after 2148 * truncate_inode_pages finishes, hence the second 2149 * unmap_mapping_range call must be made for correctness. 2150 */ 2151 unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1); 2152 truncate_inode_pages(mapping, offset); 2153 unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1); 2154 } 2155 2156 if (inode->i_op && inode->i_op->truncate) 2157 inode->i_op->truncate(inode); 2158 return 0; 2159 2160out_sig: 2161 send_sig(SIGXFSZ, current, 0); 2162out_big: 2163 return -EFBIG; 2164} 2165EXPORT_SYMBOL(vmtruncate); 2166 2167int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end) 2168{ 2169 struct address_space *mapping = inode->i_mapping; 2170 2171 /* 2172 * If the underlying filesystem is not going to provide 2173 * a way to truncate a range of blocks (punch a hole) - 2174 * we should return failure right now. 2175 */ 2176 if (!inode->i_op || !inode->i_op->truncate_range) 2177 return -ENOSYS; 2178 2179 mutex_lock(&inode->i_mutex); 2180 down_write(&inode->i_alloc_sem); 2181 unmap_mapping_range(mapping, offset, (end - offset), 1); 2182 truncate_inode_pages_range(mapping, offset, end); 2183 unmap_mapping_range(mapping, offset, (end - offset), 1); 2184 inode->i_op->truncate_range(inode, offset, end); 2185 up_write(&inode->i_alloc_sem); 2186 mutex_unlock(&inode->i_mutex); 2187 2188 return 0; 2189} 2190 2191/* 2192 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2193 * but allow concurrent faults), and pte mapped but not yet locked. 2194 * We return with mmap_sem still held, but pte unmapped and unlocked. 2195 */ 2196static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, 2197 unsigned long address, pte_t *page_table, pmd_t *pmd, 2198 int write_access, pte_t orig_pte) 2199{ 2200 spinlock_t *ptl; 2201 struct page *page; 2202 swp_entry_t entry; 2203 pte_t pte; 2204 int ret = 0; 2205 2206 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 2207 goto out; 2208 2209 entry = pte_to_swp_entry(orig_pte); 2210 if (is_migration_entry(entry)) { 2211 migration_entry_wait(mm, pmd, address); 2212 goto out; 2213 } 2214 delayacct_set_flag(DELAYACCT_PF_SWAPIN); 2215 page = lookup_swap_cache(entry); 2216 if (!page) { 2217 grab_swap_token(); /* Contend for token _before_ read-in */ 2218 page = swapin_readahead(entry, 2219 GFP_HIGHUSER_MOVABLE, vma, address); 2220 if (!page) { 2221 /* 2222 * Back out if somebody else faulted in this pte 2223 * while we released the pte lock. 2224 */ 2225 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2226 if (likely(pte_same(*page_table, orig_pte))) 2227 ret = VM_FAULT_OOM; 2228 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2229 goto unlock; 2230 } 2231 2232 /* Had to read the page from swap area: Major fault */ 2233 ret = VM_FAULT_MAJOR; 2234 count_vm_event(PGMAJFAULT); 2235 } 2236 2237 if (mem_cgroup_charge(page, mm, GFP_KERNEL)) { 2238 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2239 ret = VM_FAULT_OOM; 2240 goto out; 2241 } 2242 2243 mark_page_accessed(page); 2244 lock_page(page); 2245 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2246 2247 /* 2248 * Back out if somebody else already faulted in this pte. 2249 */ 2250 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2251 if (unlikely(!pte_same(*page_table, orig_pte))) 2252 goto out_nomap; 2253 2254 if (unlikely(!PageUptodate(page))) { 2255 ret = VM_FAULT_SIGBUS; 2256 goto out_nomap; 2257 } 2258 2259 /* The page isn't present yet, go ahead with the fault. */ 2260 2261 inc_mm_counter(mm, anon_rss); 2262 pte = mk_pte(page, vma->vm_page_prot); 2263 if (write_access && can_share_swap_page(page)) { 2264 pte = maybe_mkwrite(pte_mkdirty(pte), vma); 2265 write_access = 0; 2266 } 2267 2268 flush_icache_page(vma, page); 2269 set_pte_at(mm, address, page_table, pte); 2270 page_add_anon_rmap(page, vma, address); 2271 2272 swap_free(entry); 2273 if (vm_swap_full()) 2274 remove_exclusive_swap_page(page); 2275 unlock_page(page); 2276 2277 if (write_access) { 2278 ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte); 2279 if (ret & VM_FAULT_ERROR) 2280 ret &= VM_FAULT_ERROR; 2281 goto out; 2282 } 2283 2284 /* No need to invalidate - it was non-present before */ 2285 update_mmu_cache(vma, address, pte); 2286unlock: 2287 pte_unmap_unlock(page_table, ptl); 2288out: 2289 return ret; 2290out_nomap: 2291 mem_cgroup_uncharge_page(page); 2292 pte_unmap_unlock(page_table, ptl); 2293 unlock_page(page); 2294 page_cache_release(page); 2295 return ret; 2296} 2297 2298/* 2299 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2300 * but allow concurrent faults), and pte mapped but not yet locked. 2301 * We return with mmap_sem still held, but pte unmapped and unlocked. 2302 */ 2303static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 2304 unsigned long address, pte_t *page_table, pmd_t *pmd, 2305 int write_access) 2306{ 2307 struct page *page; 2308 spinlock_t *ptl; 2309 pte_t entry; 2310 2311 /* Allocate our own private page. */ 2312 pte_unmap(page_table); 2313 2314 if (unlikely(anon_vma_prepare(vma))) 2315 goto oom; 2316 page = alloc_zeroed_user_highpage_movable(vma, address); 2317 if (!page) 2318 goto oom; 2319 __SetPageUptodate(page); 2320 2321 if (mem_cgroup_charge(page, mm, GFP_KERNEL)) 2322 goto oom_free_page; 2323 2324 entry = mk_pte(page, vma->vm_page_prot); 2325 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2326 2327 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2328 if (!pte_none(*page_table)) 2329 goto release; 2330 inc_mm_counter(mm, anon_rss); 2331 lru_cache_add_active(page); 2332 page_add_new_anon_rmap(page, vma, address); 2333 set_pte_at(mm, address, page_table, entry); 2334 2335 /* No need to invalidate - it was non-present before */ 2336 update_mmu_cache(vma, address, entry); 2337unlock: 2338 pte_unmap_unlock(page_table, ptl); 2339 return 0; 2340release: 2341 mem_cgroup_uncharge_page(page); 2342 page_cache_release(page); 2343 goto unlock; 2344oom_free_page: 2345 page_cache_release(page); 2346oom: 2347 return VM_FAULT_OOM; 2348} 2349 2350/* 2351 * __do_fault() tries to create a new page mapping. It aggressively 2352 * tries to share with existing pages, but makes a separate copy if 2353 * the FAULT_FLAG_WRITE is set in the flags parameter in order to avoid 2354 * the next page fault. 2355 * 2356 * As this is called only for pages that do not currently exist, we 2357 * do not need to flush old virtual caches or the TLB. 2358 * 2359 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2360 * but allow concurrent faults), and pte neither mapped nor locked. 2361 * We return with mmap_sem still held, but pte unmapped and unlocked. 2362 */ 2363static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, 2364 unsigned long address, pmd_t *pmd, 2365 pgoff_t pgoff, unsigned int flags, pte_t orig_pte) 2366{ 2367 pte_t *page_table; 2368 spinlock_t *ptl; 2369 struct page *page; 2370 pte_t entry; 2371 int anon = 0; 2372 struct page *dirty_page = NULL; 2373 struct vm_fault vmf; 2374 int ret; 2375 int page_mkwrite = 0; 2376 2377 vmf.virtual_address = (void __user *)(address & PAGE_MASK); 2378 vmf.pgoff = pgoff; 2379 vmf.flags = flags; 2380 vmf.page = NULL; 2381 2382 ret = vma->vm_ops->fault(vma, &vmf); 2383 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) 2384 return ret; 2385 2386 /* 2387 * For consistency in subsequent calls, make the faulted page always 2388 * locked. 2389 */ 2390 if (unlikely(!(ret & VM_FAULT_LOCKED))) 2391 lock_page(vmf.page); 2392 else 2393 VM_BUG_ON(!PageLocked(vmf.page)); 2394 2395 /* 2396 * Should we do an early C-O-W break? 2397 */ 2398 page = vmf.page; 2399 if (flags & FAULT_FLAG_WRITE) { 2400 if (!(vma->vm_flags & VM_SHARED)) { 2401 anon = 1; 2402 if (unlikely(anon_vma_prepare(vma))) { 2403 ret = VM_FAULT_OOM; 2404 goto out; 2405 } 2406 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, 2407 vma, address); 2408 if (!page) { 2409 ret = VM_FAULT_OOM; 2410 goto out; 2411 } 2412 copy_user_highpage(page, vmf.page, address, vma); 2413 __SetPageUptodate(page); 2414 } else { 2415 /* 2416 * If the page will be shareable, see if the backing 2417 * address space wants to know that the page is about 2418 * to become writable 2419 */ 2420 if (vma->vm_ops->page_mkwrite) { 2421 unlock_page(page); 2422 if (vma->vm_ops->page_mkwrite(vma, page) < 0) { 2423 ret = VM_FAULT_SIGBUS; 2424 anon = 1; /* no anon but release vmf.page */ 2425 goto out_unlocked; 2426 } 2427 lock_page(page); 2428 /* 2429 * XXX: this is not quite right (racy vs 2430 * invalidate) to unlock and relock the page 2431 * like this, however a better fix requires 2432 * reworking page_mkwrite locking API, which 2433 * is better done later. 2434 */ 2435 if (!page->mapping) { 2436 ret = 0; 2437 anon = 1; /* no anon but release vmf.page */ 2438 goto out; 2439 } 2440 page_mkwrite = 1; 2441 } 2442 } 2443 2444 } 2445 2446 if (mem_cgroup_charge(page, mm, GFP_KERNEL)) { 2447 ret = VM_FAULT_OOM; 2448 goto out; 2449 } 2450 2451 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2452 2453 /* 2454 * This silly early PAGE_DIRTY setting removes a race 2455 * due to the bad i386 page protection. But it's valid 2456 * for other architectures too. 2457 * 2458 * Note that if write_access is true, we either now have 2459 * an exclusive copy of the page, or this is a shared mapping, 2460 * so we can make it writable and dirty to avoid having to 2461 * handle that later. 2462 */ 2463 /* Only go through if we didn't race with anybody else... */ 2464 if (likely(pte_same(*page_table, orig_pte))) { 2465 flush_icache_page(vma, page); 2466 entry = mk_pte(page, vma->vm_page_prot); 2467 if (flags & FAULT_FLAG_WRITE) 2468 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2469 set_pte_at(mm, address, page_table, entry); 2470 if (anon) { 2471 inc_mm_counter(mm, anon_rss); 2472 lru_cache_add_active(page); 2473 page_add_new_anon_rmap(page, vma, address); 2474 } else { 2475 inc_mm_counter(mm, file_rss); 2476 page_add_file_rmap(page); 2477 if (flags & FAULT_FLAG_WRITE) { 2478 dirty_page = page; 2479 get_page(dirty_page); 2480 } 2481 } 2482 2483 /* no need to invalidate: a not-present page won't be cached */ 2484 update_mmu_cache(vma, address, entry); 2485 } else { 2486 mem_cgroup_uncharge_page(page); 2487 if (anon) 2488 page_cache_release(page); 2489 else 2490 anon = 1; /* no anon but release faulted_page */ 2491 } 2492 2493 pte_unmap_unlock(page_table, ptl); 2494 2495out: 2496 unlock_page(vmf.page); 2497out_unlocked: 2498 if (anon) 2499 page_cache_release(vmf.page); 2500 else if (dirty_page) { 2501 if (vma->vm_file) 2502 file_update_time(vma->vm_file); 2503 2504 set_page_dirty_balance(dirty_page, page_mkwrite); 2505 put_page(dirty_page); 2506 } 2507 2508 return ret; 2509} 2510 2511static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 2512 unsigned long address, pte_t *page_table, pmd_t *pmd, 2513 int write_access, pte_t orig_pte) 2514{ 2515 pgoff_t pgoff = (((address & PAGE_MASK) 2516 - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2517 unsigned int flags = (write_access ? FAULT_FLAG_WRITE : 0); 2518 2519 pte_unmap(page_table); 2520 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 2521} 2522 2523/* 2524 * Fault of a previously existing named mapping. Repopulate the pte 2525 * from the encoded file_pte if possible. This enables swappable 2526 * nonlinear vmas. 2527 * 2528 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2529 * but allow concurrent faults), and pte mapped but not yet locked. 2530 * We return with mmap_sem still held, but pte unmapped and unlocked. 2531 */ 2532static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 2533 unsigned long address, pte_t *page_table, pmd_t *pmd, 2534 int write_access, pte_t orig_pte) 2535{ 2536 unsigned int flags = FAULT_FLAG_NONLINEAR | 2537 (write_access ? FAULT_FLAG_WRITE : 0); 2538 pgoff_t pgoff; 2539 2540 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 2541 return 0; 2542 2543 if (unlikely(!(vma->vm_flags & VM_NONLINEAR) || 2544 !(vma->vm_flags & VM_CAN_NONLINEAR))) { 2545 /* 2546 * Page table corrupted: show pte and kill process. 2547 */ 2548 print_bad_pte(vma, orig_pte, address); 2549 return VM_FAULT_OOM; 2550 } 2551 2552 pgoff = pte_to_pgoff(orig_pte); 2553 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 2554} 2555 2556/* 2557 * These routines also need to handle stuff like marking pages dirty 2558 * and/or accessed for architectures that don't do it in hardware (most 2559 * RISC architectures). The early dirtying is also good on the i386. 2560 * 2561 * There is also a hook called "update_mmu_cache()" that architectures 2562 * with external mmu caches can use to update those (ie the Sparc or 2563 * PowerPC hashed page tables that act as extended TLBs). 2564 * 2565 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2566 * but allow concurrent faults), and pte mapped but not yet locked. 2567 * We return with mmap_sem still held, but pte unmapped and unlocked. 2568 */ 2569static inline int handle_pte_fault(struct mm_struct *mm, 2570 struct vm_area_struct *vma, unsigned long address, 2571 pte_t *pte, pmd_t *pmd, int write_access) 2572{ 2573 pte_t entry; 2574 spinlock_t *ptl; 2575 2576 entry = *pte; 2577 if (!pte_present(entry)) { 2578 if (pte_none(entry)) { 2579 if (vma->vm_ops) { 2580 if (likely(vma->vm_ops->fault)) 2581 return do_linear_fault(mm, vma, address, 2582 pte, pmd, write_access, entry); 2583 } 2584 return do_anonymous_page(mm, vma, address, 2585 pte, pmd, write_access); 2586 } 2587 if (pte_file(entry)) 2588 return do_nonlinear_fault(mm, vma, address, 2589 pte, pmd, write_access, entry); 2590 return do_swap_page(mm, vma, address, 2591 pte, pmd, write_access, entry); 2592 } 2593 2594 ptl = pte_lockptr(mm, pmd); 2595 spin_lock(ptl); 2596 if (unlikely(!pte_same(*pte, entry))) 2597 goto unlock; 2598 if (write_access) { 2599 if (!pte_write(entry)) 2600 return do_wp_page(mm, vma, address, 2601 pte, pmd, ptl, entry); 2602 entry = pte_mkdirty(entry); 2603 } 2604 entry = pte_mkyoung(entry); 2605 if (ptep_set_access_flags(vma, address, pte, entry, write_access)) { 2606 update_mmu_cache(vma, address, entry); 2607 } else { 2608 /* 2609 * This is needed only for protection faults but the arch code 2610 * is not yet telling us if this is a protection fault or not. 2611 * This still avoids useless tlb flushes for .text page faults 2612 * with threads. 2613 */ 2614 if (write_access) 2615 flush_tlb_page(vma, address); 2616 } 2617unlock: 2618 pte_unmap_unlock(pte, ptl); 2619 return 0; 2620} 2621 2622/* 2623 * By the time we get here, we already hold the mm semaphore 2624 */ 2625int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, 2626 unsigned long address, int write_access) 2627{ 2628 pgd_t *pgd; 2629 pud_t *pud; 2630 pmd_t *pmd; 2631 pte_t *pte; 2632 2633 __set_current_state(TASK_RUNNING); 2634 2635 count_vm_event(PGFAULT); 2636 2637 if (unlikely(is_vm_hugetlb_page(vma))) 2638 return hugetlb_fault(mm, vma, address, write_access); 2639 2640 pgd = pgd_offset(mm, address); 2641 pud = pud_alloc(mm, pgd, address); 2642 if (!pud) 2643 return VM_FAULT_OOM; 2644 pmd = pmd_alloc(mm, pud, address); 2645 if (!pmd) 2646 return VM_FAULT_OOM; 2647 pte = pte_alloc_map(mm, pmd, address); 2648 if (!pte) 2649 return VM_FAULT_OOM; 2650 2651 return handle_pte_fault(mm, vma, address, pte, pmd, write_access); 2652} 2653 2654#ifndef __PAGETABLE_PUD_FOLDED 2655/* 2656 * Allocate page upper directory. 2657 * We've already handled the fast-path in-line. 2658 */ 2659int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 2660{ 2661 pud_t *new = pud_alloc_one(mm, address); 2662 if (!new) 2663 return -ENOMEM; 2664 2665 smp_wmb(); /* See comment in __pte_alloc */ 2666 2667 spin_lock(&mm->page_table_lock); 2668 if (pgd_present(*pgd)) /* Another has populated it */ 2669 pud_free(mm, new); 2670 else 2671 pgd_populate(mm, pgd, new); 2672 spin_unlock(&mm->page_table_lock); 2673 return 0; 2674} 2675#endif /* __PAGETABLE_PUD_FOLDED */ 2676 2677#ifndef __PAGETABLE_PMD_FOLDED 2678/* 2679 * Allocate page middle directory. 2680 * We've already handled the fast-path in-line. 2681 */ 2682int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 2683{ 2684 pmd_t *new = pmd_alloc_one(mm, address); 2685 if (!new) 2686 return -ENOMEM; 2687 2688 smp_wmb(); /* See comment in __pte_alloc */ 2689 2690 spin_lock(&mm->page_table_lock); 2691#ifndef __ARCH_HAS_4LEVEL_HACK 2692 if (pud_present(*pud)) /* Another has populated it */ 2693 pmd_free(mm, new); 2694 else 2695 pud_populate(mm, pud, new); 2696#else 2697 if (pgd_present(*pud)) /* Another has populated it */ 2698 pmd_free(mm, new); 2699 else 2700 pgd_populate(mm, pud, new); 2701#endif /* __ARCH_HAS_4LEVEL_HACK */ 2702 spin_unlock(&mm->page_table_lock); 2703 return 0; 2704} 2705#endif /* __PAGETABLE_PMD_FOLDED */ 2706 2707int make_pages_present(unsigned long addr, unsigned long end) 2708{ 2709 int ret, len, write; 2710 struct vm_area_struct * vma; 2711 2712 vma = find_vma(current->mm, addr); 2713 if (!vma) 2714 return -1; 2715 write = (vma->vm_flags & VM_WRITE) != 0; 2716 BUG_ON(addr >= end); 2717 BUG_ON(end > vma->vm_end); 2718 len = DIV_ROUND_UP(end, PAGE_SIZE) - addr/PAGE_SIZE; 2719 ret = get_user_pages(current, current->mm, addr, 2720 len, write, 0, NULL, NULL); 2721 if (ret < 0) 2722 return ret; 2723 return ret == len ? 0 : -1; 2724} 2725 2726#if !defined(__HAVE_ARCH_GATE_AREA) 2727 2728#if defined(AT_SYSINFO_EHDR) 2729static struct vm_area_struct gate_vma; 2730 2731static int __init gate_vma_init(void) 2732{ 2733 gate_vma.vm_mm = NULL; 2734 gate_vma.vm_start = FIXADDR_USER_START; 2735 gate_vma.vm_end = FIXADDR_USER_END; 2736 gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC; 2737 gate_vma.vm_page_prot = __P101; 2738 /* 2739 * Make sure the vDSO gets into every core dump. 2740 * Dumping its contents makes post-mortem fully interpretable later 2741 * without matching up the same kernel and hardware config to see 2742 * what PC values meant. 2743 */ 2744 gate_vma.vm_flags |= VM_ALWAYSDUMP; 2745 return 0; 2746} 2747__initcall(gate_vma_init); 2748#endif 2749 2750struct vm_area_struct *get_gate_vma(struct task_struct *tsk) 2751{ 2752#ifdef AT_SYSINFO_EHDR 2753 return &gate_vma; 2754#else 2755 return NULL; 2756#endif 2757} 2758 2759int in_gate_area_no_task(unsigned long addr) 2760{ 2761#ifdef AT_SYSINFO_EHDR 2762 if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END)) 2763 return 1; 2764#endif 2765 return 0; 2766} 2767 2768#endif /* __HAVE_ARCH_GATE_AREA */ 2769 2770#ifdef CONFIG_HAVE_IOREMAP_PROT 2771static resource_size_t follow_phys(struct vm_area_struct *vma, 2772 unsigned long address, unsigned int flags, 2773 unsigned long *prot) 2774{ 2775 pgd_t *pgd; 2776 pud_t *pud; 2777 pmd_t *pmd; 2778 pte_t *ptep, pte; 2779 spinlock_t *ptl; 2780 resource_size_t phys_addr = 0; 2781 struct mm_struct *mm = vma->vm_mm; 2782 2783 VM_BUG_ON(!(vma->vm_flags & (VM_IO | VM_PFNMAP))); 2784 2785 pgd = pgd_offset(mm, address); 2786 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 2787 goto no_page_table; 2788 2789 pud = pud_offset(pgd, address); 2790 if (pud_none(*pud) || unlikely(pud_bad(*pud))) 2791 goto no_page_table; 2792 2793 pmd = pmd_offset(pud, address); 2794 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) 2795 goto no_page_table; 2796 2797 /* We cannot handle huge page PFN maps. Luckily they don't exist. */ 2798 if (pmd_huge(*pmd)) 2799 goto no_page_table; 2800 2801 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 2802 if (!ptep) 2803 goto out; 2804 2805 pte = *ptep; 2806 if (!pte_present(pte)) 2807 goto unlock; 2808 if ((flags & FOLL_WRITE) && !pte_write(pte)) 2809 goto unlock; 2810 phys_addr = pte_pfn(pte); 2811 phys_addr <<= PAGE_SHIFT; /* Shift here to avoid overflow on PAE */ 2812 2813 *prot = pgprot_val(pte_pgprot(pte)); 2814 2815unlock: 2816 pte_unmap_unlock(ptep, ptl); 2817out: 2818 return phys_addr; 2819no_page_table: 2820 return 0; 2821} 2822 2823int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 2824 void *buf, int len, int write) 2825{ 2826 resource_size_t phys_addr; 2827 unsigned long prot = 0; 2828 void *maddr; 2829 int offset = addr & (PAGE_SIZE-1); 2830 2831 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 2832 return -EINVAL; 2833 2834 phys_addr = follow_phys(vma, addr, write, &prot); 2835 2836 if (!phys_addr) 2837 return -EINVAL; 2838 2839 maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); 2840 if (write) 2841 memcpy_toio(maddr + offset, buf, len); 2842 else 2843 memcpy_fromio(buf, maddr + offset, len); 2844 iounmap(maddr); 2845 2846 return len; 2847} 2848#endif 2849 2850/* 2851 * Access another process' address space. 2852 * Source/target buffer must be kernel space, 2853 * Do not walk the page table directly, use get_user_pages 2854 */ 2855int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write) 2856{ 2857 struct mm_struct *mm; 2858 struct vm_area_struct *vma; 2859 void *old_buf = buf; 2860 2861 mm = get_task_mm(tsk); 2862 if (!mm) 2863 return 0; 2864 2865 down_read(&mm->mmap_sem); 2866 /* ignore errors, just check how much was successfully transferred */ 2867 while (len) { 2868 int bytes, ret, offset; 2869 void *maddr; 2870 struct page *page = NULL; 2871 2872 ret = get_user_pages(tsk, mm, addr, 1, 2873 write, 1, &page, &vma); 2874 if (ret <= 0) { 2875 /* 2876 * Check if this is a VM_IO | VM_PFNMAP VMA, which 2877 * we can access using slightly different code. 2878 */ 2879#ifdef CONFIG_HAVE_IOREMAP_PROT 2880 vma = find_vma(mm, addr); 2881 if (!vma) 2882 break; 2883 if (vma->vm_ops && vma->vm_ops->access) 2884 ret = vma->vm_ops->access(vma, addr, buf, 2885 len, write); 2886 if (ret <= 0) 2887#endif 2888 break; 2889 bytes = ret; 2890 } else { 2891 bytes = len; 2892 offset = addr & (PAGE_SIZE-1); 2893 if (bytes > PAGE_SIZE-offset) 2894 bytes = PAGE_SIZE-offset; 2895 2896 maddr = kmap(page); 2897 if (write) { 2898 copy_to_user_page(vma, page, addr, 2899 maddr + offset, buf, bytes); 2900 set_page_dirty_lock(page); 2901 } else { 2902 copy_from_user_page(vma, page, addr, 2903 buf, maddr + offset, bytes); 2904 } 2905 kunmap(page); 2906 page_cache_release(page); 2907 } 2908 len -= bytes; 2909 buf += bytes; 2910 addr += bytes; 2911 } 2912 up_read(&mm->mmap_sem); 2913 mmput(mm); 2914 2915 return buf - old_buf; 2916} 2917 2918/* 2919 * Print the name of a VMA. 2920 */ 2921void print_vma_addr(char *prefix, unsigned long ip) 2922{ 2923 struct mm_struct *mm = current->mm; 2924 struct vm_area_struct *vma; 2925 2926 /* 2927 * Do not print if we are in atomic 2928 * contexts (in exception stacks, etc.): 2929 */ 2930 if (preempt_count()) 2931 return; 2932 2933 down_read(&mm->mmap_sem); 2934 vma = find_vma(mm, ip); 2935 if (vma && vma->vm_file) { 2936 struct file *f = vma->vm_file; 2937 char *buf = (char *)__get_free_page(GFP_KERNEL); 2938 if (buf) { 2939 char *p, *s; 2940 2941 p = d_path(&f->f_path, buf, PAGE_SIZE); 2942 if (IS_ERR(p)) 2943 p = "?"; 2944 s = strrchr(p, '/'); 2945 if (s) 2946 p = s+1; 2947 printk("%s%s[%lx+%lx]", prefix, p, 2948 vma->vm_start, 2949 vma->vm_end - vma->vm_start); 2950 free_page((unsigned long)buf); 2951 } 2952 } 2953 up_read(¤t->mm->mmap_sem); 2954} 2955