nommu.c revision 41836382ebb415d68d3ebc4525e78e871fe58baf
1/* 2 * linux/mm/nommu.c 3 * 4 * Replacement code for mm functions to support CPU's that don't 5 * have any form of memory management unit (thus no virtual memory). 6 * 7 * See Documentation/nommu-mmap.txt 8 * 9 * Copyright (c) 2004-2005 David Howells <dhowells@redhat.com> 10 * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com> 11 * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org> 12 * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com> 13 * Copyright (c) 2007 Paul Mundt <lethal@linux-sh.org> 14 */ 15 16#include <linux/module.h> 17#include <linux/mm.h> 18#include <linux/mman.h> 19#include <linux/swap.h> 20#include <linux/file.h> 21#include <linux/highmem.h> 22#include <linux/pagemap.h> 23#include <linux/slab.h> 24#include <linux/vmalloc.h> 25#include <linux/tracehook.h> 26#include <linux/blkdev.h> 27#include <linux/backing-dev.h> 28#include <linux/mount.h> 29#include <linux/personality.h> 30#include <linux/security.h> 31#include <linux/syscalls.h> 32 33#include <asm/uaccess.h> 34#include <asm/tlb.h> 35#include <asm/tlbflush.h> 36 37#include "internal.h" 38 39void *high_memory; 40struct page *mem_map; 41unsigned long max_mapnr; 42unsigned long num_physpages; 43atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0); 44int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ 45int sysctl_overcommit_ratio = 50; /* default is 50% */ 46int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; 47int heap_stack_gap = 0; 48 49EXPORT_SYMBOL(mem_map); 50EXPORT_SYMBOL(num_physpages); 51 52/* list of shareable VMAs */ 53struct rb_root nommu_vma_tree = RB_ROOT; 54DECLARE_RWSEM(nommu_vma_sem); 55 56struct vm_operations_struct generic_file_vm_ops = { 57}; 58 59/* 60 * Handle all mappings that got truncated by a "truncate()" 61 * system call. 62 * 63 * NOTE! We have to be ready to update the memory sharing 64 * between the file and the memory map for a potential last 65 * incomplete page. Ugly, but necessary. 66 */ 67int vmtruncate(struct inode *inode, loff_t offset) 68{ 69 struct address_space *mapping = inode->i_mapping; 70 unsigned long limit; 71 72 if (inode->i_size < offset) 73 goto do_expand; 74 i_size_write(inode, offset); 75 76 truncate_inode_pages(mapping, offset); 77 goto out_truncate; 78 79do_expand: 80 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; 81 if (limit != RLIM_INFINITY && offset > limit) 82 goto out_sig; 83 if (offset > inode->i_sb->s_maxbytes) 84 goto out; 85 i_size_write(inode, offset); 86 87out_truncate: 88 if (inode->i_op->truncate) 89 inode->i_op->truncate(inode); 90 return 0; 91out_sig: 92 send_sig(SIGXFSZ, current, 0); 93out: 94 return -EFBIG; 95} 96 97EXPORT_SYMBOL(vmtruncate); 98 99/* 100 * Return the total memory allocated for this pointer, not 101 * just what the caller asked for. 102 * 103 * Doesn't have to be accurate, i.e. may have races. 104 */ 105unsigned int kobjsize(const void *objp) 106{ 107 struct page *page; 108 109 /* 110 * If the object we have should not have ksize performed on it, 111 * return size of 0 112 */ 113 if (!objp || !virt_addr_valid(objp)) 114 return 0; 115 116 page = virt_to_head_page(objp); 117 118 /* 119 * If the allocator sets PageSlab, we know the pointer came from 120 * kmalloc(). 121 */ 122 if (PageSlab(page)) 123 return ksize(objp); 124 125 /* 126 * The ksize() function is only guaranteed to work for pointers 127 * returned by kmalloc(). So handle arbitrary pointers here. 128 */ 129 return PAGE_SIZE << compound_order(page); 130} 131 132int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 133 unsigned long start, int len, int flags, 134 struct page **pages, struct vm_area_struct **vmas) 135{ 136 struct vm_area_struct *vma; 137 unsigned long vm_flags; 138 int i; 139 int write = !!(flags & GUP_FLAGS_WRITE); 140 int force = !!(flags & GUP_FLAGS_FORCE); 141 int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS); 142 143 /* calculate required read or write permissions. 144 * - if 'force' is set, we only require the "MAY" flags. 145 */ 146 vm_flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 147 vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 148 149 for (i = 0; i < len; i++) { 150 vma = find_vma(mm, start); 151 if (!vma) 152 goto finish_or_fault; 153 154 /* protect what we can, including chardevs */ 155 if (vma->vm_flags & (VM_IO | VM_PFNMAP) || 156 (!ignore && !(vm_flags & vma->vm_flags))) 157 goto finish_or_fault; 158 159 if (pages) { 160 pages[i] = virt_to_page(start); 161 if (pages[i]) 162 page_cache_get(pages[i]); 163 } 164 if (vmas) 165 vmas[i] = vma; 166 start += PAGE_SIZE; 167 } 168 169 return i; 170 171finish_or_fault: 172 return i ? : -EFAULT; 173} 174 175 176/* 177 * get a list of pages in an address range belonging to the specified process 178 * and indicate the VMA that covers each page 179 * - this is potentially dodgy as we may end incrementing the page count of a 180 * slab page or a secondary page from a compound page 181 * - don't permit access to VMAs that don't support it, such as I/O mappings 182 */ 183int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 184 unsigned long start, int len, int write, int force, 185 struct page **pages, struct vm_area_struct **vmas) 186{ 187 int flags = 0; 188 189 if (write) 190 flags |= GUP_FLAGS_WRITE; 191 if (force) 192 flags |= GUP_FLAGS_FORCE; 193 194 return __get_user_pages(tsk, mm, 195 start, len, flags, 196 pages, vmas); 197} 198EXPORT_SYMBOL(get_user_pages); 199 200DEFINE_RWLOCK(vmlist_lock); 201struct vm_struct *vmlist; 202 203void vfree(const void *addr) 204{ 205 kfree(addr); 206} 207EXPORT_SYMBOL(vfree); 208 209void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) 210{ 211 /* 212 * You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc() 213 * returns only a logical address. 214 */ 215 return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM); 216} 217EXPORT_SYMBOL(__vmalloc); 218 219void *vmalloc_user(unsigned long size) 220{ 221 void *ret; 222 223 ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, 224 PAGE_KERNEL); 225 if (ret) { 226 struct vm_area_struct *vma; 227 228 down_write(¤t->mm->mmap_sem); 229 vma = find_vma(current->mm, (unsigned long)ret); 230 if (vma) 231 vma->vm_flags |= VM_USERMAP; 232 up_write(¤t->mm->mmap_sem); 233 } 234 235 return ret; 236} 237EXPORT_SYMBOL(vmalloc_user); 238 239struct page *vmalloc_to_page(const void *addr) 240{ 241 return virt_to_page(addr); 242} 243EXPORT_SYMBOL(vmalloc_to_page); 244 245unsigned long vmalloc_to_pfn(const void *addr) 246{ 247 return page_to_pfn(virt_to_page(addr)); 248} 249EXPORT_SYMBOL(vmalloc_to_pfn); 250 251long vread(char *buf, char *addr, unsigned long count) 252{ 253 memcpy(buf, addr, count); 254 return count; 255} 256 257long vwrite(char *buf, char *addr, unsigned long count) 258{ 259 /* Don't allow overflow */ 260 if ((unsigned long) addr + count < count) 261 count = -(unsigned long) addr; 262 263 memcpy(addr, buf, count); 264 return(count); 265} 266 267/* 268 * vmalloc - allocate virtually continguos memory 269 * 270 * @size: allocation size 271 * 272 * Allocate enough pages to cover @size from the page level 273 * allocator and map them into continguos kernel virtual space. 274 * 275 * For tight control over page level allocator and protection flags 276 * use __vmalloc() instead. 277 */ 278void *vmalloc(unsigned long size) 279{ 280 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL); 281} 282EXPORT_SYMBOL(vmalloc); 283 284void *vmalloc_node(unsigned long size, int node) 285{ 286 return vmalloc(size); 287} 288EXPORT_SYMBOL(vmalloc_node); 289 290#ifndef PAGE_KERNEL_EXEC 291# define PAGE_KERNEL_EXEC PAGE_KERNEL 292#endif 293 294/** 295 * vmalloc_exec - allocate virtually contiguous, executable memory 296 * @size: allocation size 297 * 298 * Kernel-internal function to allocate enough pages to cover @size 299 * the page level allocator and map them into contiguous and 300 * executable kernel virtual space. 301 * 302 * For tight control over page level allocator and protection flags 303 * use __vmalloc() instead. 304 */ 305 306void *vmalloc_exec(unsigned long size) 307{ 308 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC); 309} 310 311/** 312 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 313 * @size: allocation size 314 * 315 * Allocate enough 32bit PA addressable pages to cover @size from the 316 * page level allocator and map them into continguos kernel virtual space. 317 */ 318void *vmalloc_32(unsigned long size) 319{ 320 return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL); 321} 322EXPORT_SYMBOL(vmalloc_32); 323 324/** 325 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 326 * @size: allocation size 327 * 328 * The resulting memory area is 32bit addressable and zeroed so it can be 329 * mapped to userspace without leaking data. 330 * 331 * VM_USERMAP is set on the corresponding VMA so that subsequent calls to 332 * remap_vmalloc_range() are permissible. 333 */ 334void *vmalloc_32_user(unsigned long size) 335{ 336 /* 337 * We'll have to sort out the ZONE_DMA bits for 64-bit, 338 * but for now this can simply use vmalloc_user() directly. 339 */ 340 return vmalloc_user(size); 341} 342EXPORT_SYMBOL(vmalloc_32_user); 343 344void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot) 345{ 346 BUG(); 347 return NULL; 348} 349EXPORT_SYMBOL(vmap); 350 351void vunmap(const void *addr) 352{ 353 BUG(); 354} 355EXPORT_SYMBOL(vunmap); 356 357/* 358 * Implement a stub for vmalloc_sync_all() if the architecture chose not to 359 * have one. 360 */ 361void __attribute__((weak)) vmalloc_sync_all(void) 362{ 363} 364 365int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 366 struct page *page) 367{ 368 return -EINVAL; 369} 370EXPORT_SYMBOL(vm_insert_page); 371 372/* 373 * sys_brk() for the most part doesn't need the global kernel 374 * lock, except when an application is doing something nasty 375 * like trying to un-brk an area that has already been mapped 376 * to a regular file. in this case, the unmapping will need 377 * to invoke file system routines that need the global lock. 378 */ 379asmlinkage unsigned long sys_brk(unsigned long brk) 380{ 381 struct mm_struct *mm = current->mm; 382 383 if (brk < mm->start_brk || brk > mm->context.end_brk) 384 return mm->brk; 385 386 if (mm->brk == brk) 387 return mm->brk; 388 389 /* 390 * Always allow shrinking brk 391 */ 392 if (brk <= mm->brk) { 393 mm->brk = brk; 394 return brk; 395 } 396 397 /* 398 * Ok, looks good - let it rip. 399 */ 400 return mm->brk = brk; 401} 402 403#ifdef DEBUG 404static void show_process_blocks(void) 405{ 406 struct vm_list_struct *vml; 407 408 printk("Process blocks %d:", current->pid); 409 410 for (vml = ¤t->mm->context.vmlist; vml; vml = vml->next) { 411 printk(" %p: %p", vml, vml->vma); 412 if (vml->vma) 413 printk(" (%d @%lx #%d)", 414 kobjsize((void *) vml->vma->vm_start), 415 vml->vma->vm_start, 416 atomic_read(&vml->vma->vm_usage)); 417 printk(vml->next ? " ->" : ".\n"); 418 } 419} 420#endif /* DEBUG */ 421 422/* 423 * add a VMA into a process's mm_struct in the appropriate place in the list 424 * - should be called with mm->mmap_sem held writelocked 425 */ 426static void add_vma_to_mm(struct mm_struct *mm, struct vm_list_struct *vml) 427{ 428 struct vm_list_struct **ppv; 429 430 for (ppv = ¤t->mm->context.vmlist; *ppv; ppv = &(*ppv)->next) 431 if ((*ppv)->vma->vm_start > vml->vma->vm_start) 432 break; 433 434 vml->next = *ppv; 435 *ppv = vml; 436} 437 438/* 439 * look up the first VMA in which addr resides, NULL if none 440 * - should be called with mm->mmap_sem at least held readlocked 441 */ 442struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 443{ 444 struct vm_list_struct *loop, *vml; 445 446 /* search the vm_start ordered list */ 447 vml = NULL; 448 for (loop = mm->context.vmlist; loop; loop = loop->next) { 449 if (loop->vma->vm_start > addr) 450 break; 451 vml = loop; 452 } 453 454 if (vml && vml->vma->vm_end > addr) 455 return vml->vma; 456 457 return NULL; 458} 459EXPORT_SYMBOL(find_vma); 460 461/* 462 * find a VMA 463 * - we don't extend stack VMAs under NOMMU conditions 464 */ 465struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) 466{ 467 return find_vma(mm, addr); 468} 469 470int expand_stack(struct vm_area_struct *vma, unsigned long address) 471{ 472 return -ENOMEM; 473} 474 475/* 476 * look up the first VMA exactly that exactly matches addr 477 * - should be called with mm->mmap_sem at least held readlocked 478 */ 479static inline struct vm_area_struct *find_vma_exact(struct mm_struct *mm, 480 unsigned long addr) 481{ 482 struct vm_list_struct *vml; 483 484 /* search the vm_start ordered list */ 485 for (vml = mm->context.vmlist; vml; vml = vml->next) { 486 if (vml->vma->vm_start == addr) 487 return vml->vma; 488 if (vml->vma->vm_start > addr) 489 break; 490 } 491 492 return NULL; 493} 494 495/* 496 * find a VMA in the global tree 497 */ 498static inline struct vm_area_struct *find_nommu_vma(unsigned long start) 499{ 500 struct vm_area_struct *vma; 501 struct rb_node *n = nommu_vma_tree.rb_node; 502 503 while (n) { 504 vma = rb_entry(n, struct vm_area_struct, vm_rb); 505 506 if (start < vma->vm_start) 507 n = n->rb_left; 508 else if (start > vma->vm_start) 509 n = n->rb_right; 510 else 511 return vma; 512 } 513 514 return NULL; 515} 516 517/* 518 * add a VMA in the global tree 519 */ 520static void add_nommu_vma(struct vm_area_struct *vma) 521{ 522 struct vm_area_struct *pvma; 523 struct address_space *mapping; 524 struct rb_node **p = &nommu_vma_tree.rb_node; 525 struct rb_node *parent = NULL; 526 527 /* add the VMA to the mapping */ 528 if (vma->vm_file) { 529 mapping = vma->vm_file->f_mapping; 530 531 flush_dcache_mmap_lock(mapping); 532 vma_prio_tree_insert(vma, &mapping->i_mmap); 533 flush_dcache_mmap_unlock(mapping); 534 } 535 536 /* add the VMA to the master list */ 537 while (*p) { 538 parent = *p; 539 pvma = rb_entry(parent, struct vm_area_struct, vm_rb); 540 541 if (vma->vm_start < pvma->vm_start) { 542 p = &(*p)->rb_left; 543 } 544 else if (vma->vm_start > pvma->vm_start) { 545 p = &(*p)->rb_right; 546 } 547 else { 548 /* mappings are at the same address - this can only 549 * happen for shared-mem chardevs and shared file 550 * mappings backed by ramfs/tmpfs */ 551 BUG_ON(!(pvma->vm_flags & VM_SHARED)); 552 553 if (vma < pvma) 554 p = &(*p)->rb_left; 555 else if (vma > pvma) 556 p = &(*p)->rb_right; 557 else 558 BUG(); 559 } 560 } 561 562 rb_link_node(&vma->vm_rb, parent, p); 563 rb_insert_color(&vma->vm_rb, &nommu_vma_tree); 564} 565 566/* 567 * delete a VMA from the global list 568 */ 569static void delete_nommu_vma(struct vm_area_struct *vma) 570{ 571 struct address_space *mapping; 572 573 /* remove the VMA from the mapping */ 574 if (vma->vm_file) { 575 mapping = vma->vm_file->f_mapping; 576 577 flush_dcache_mmap_lock(mapping); 578 vma_prio_tree_remove(vma, &mapping->i_mmap); 579 flush_dcache_mmap_unlock(mapping); 580 } 581 582 /* remove from the master list */ 583 rb_erase(&vma->vm_rb, &nommu_vma_tree); 584} 585 586/* 587 * determine whether a mapping should be permitted and, if so, what sort of 588 * mapping we're capable of supporting 589 */ 590static int validate_mmap_request(struct file *file, 591 unsigned long addr, 592 unsigned long len, 593 unsigned long prot, 594 unsigned long flags, 595 unsigned long pgoff, 596 unsigned long *_capabilities) 597{ 598 unsigned long capabilities; 599 unsigned long reqprot = prot; 600 int ret; 601 602 /* do the simple checks first */ 603 if (flags & MAP_FIXED || addr) { 604 printk(KERN_DEBUG 605 "%d: Can't do fixed-address/overlay mmap of RAM\n", 606 current->pid); 607 return -EINVAL; 608 } 609 610 if ((flags & MAP_TYPE) != MAP_PRIVATE && 611 (flags & MAP_TYPE) != MAP_SHARED) 612 return -EINVAL; 613 614 if (!len) 615 return -EINVAL; 616 617 /* Careful about overflows.. */ 618 len = PAGE_ALIGN(len); 619 if (!len || len > TASK_SIZE) 620 return -ENOMEM; 621 622 /* offset overflow? */ 623 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) 624 return -EOVERFLOW; 625 626 if (file) { 627 /* validate file mapping requests */ 628 struct address_space *mapping; 629 630 /* files must support mmap */ 631 if (!file->f_op || !file->f_op->mmap) 632 return -ENODEV; 633 634 /* work out if what we've got could possibly be shared 635 * - we support chardevs that provide their own "memory" 636 * - we support files/blockdevs that are memory backed 637 */ 638 mapping = file->f_mapping; 639 if (!mapping) 640 mapping = file->f_path.dentry->d_inode->i_mapping; 641 642 capabilities = 0; 643 if (mapping && mapping->backing_dev_info) 644 capabilities = mapping->backing_dev_info->capabilities; 645 646 if (!capabilities) { 647 /* no explicit capabilities set, so assume some 648 * defaults */ 649 switch (file->f_path.dentry->d_inode->i_mode & S_IFMT) { 650 case S_IFREG: 651 case S_IFBLK: 652 capabilities = BDI_CAP_MAP_COPY; 653 break; 654 655 case S_IFCHR: 656 capabilities = 657 BDI_CAP_MAP_DIRECT | 658 BDI_CAP_READ_MAP | 659 BDI_CAP_WRITE_MAP; 660 break; 661 662 default: 663 return -EINVAL; 664 } 665 } 666 667 /* eliminate any capabilities that we can't support on this 668 * device */ 669 if (!file->f_op->get_unmapped_area) 670 capabilities &= ~BDI_CAP_MAP_DIRECT; 671 if (!file->f_op->read) 672 capabilities &= ~BDI_CAP_MAP_COPY; 673 674 if (flags & MAP_SHARED) { 675 /* do checks for writing, appending and locking */ 676 if ((prot & PROT_WRITE) && 677 !(file->f_mode & FMODE_WRITE)) 678 return -EACCES; 679 680 if (IS_APPEND(file->f_path.dentry->d_inode) && 681 (file->f_mode & FMODE_WRITE)) 682 return -EACCES; 683 684 if (locks_verify_locked(file->f_path.dentry->d_inode)) 685 return -EAGAIN; 686 687 if (!(capabilities & BDI_CAP_MAP_DIRECT)) 688 return -ENODEV; 689 690 if (((prot & PROT_READ) && !(capabilities & BDI_CAP_READ_MAP)) || 691 ((prot & PROT_WRITE) && !(capabilities & BDI_CAP_WRITE_MAP)) || 692 ((prot & PROT_EXEC) && !(capabilities & BDI_CAP_EXEC_MAP)) 693 ) { 694 printk("MAP_SHARED not completely supported on !MMU\n"); 695 return -EINVAL; 696 } 697 698 /* we mustn't privatise shared mappings */ 699 capabilities &= ~BDI_CAP_MAP_COPY; 700 } 701 else { 702 /* we're going to read the file into private memory we 703 * allocate */ 704 if (!(capabilities & BDI_CAP_MAP_COPY)) 705 return -ENODEV; 706 707 /* we don't permit a private writable mapping to be 708 * shared with the backing device */ 709 if (prot & PROT_WRITE) 710 capabilities &= ~BDI_CAP_MAP_DIRECT; 711 } 712 713 /* handle executable mappings and implied executable 714 * mappings */ 715 if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) { 716 if (prot & PROT_EXEC) 717 return -EPERM; 718 } 719 else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) { 720 /* handle implication of PROT_EXEC by PROT_READ */ 721 if (current->personality & READ_IMPLIES_EXEC) { 722 if (capabilities & BDI_CAP_EXEC_MAP) 723 prot |= PROT_EXEC; 724 } 725 } 726 else if ((prot & PROT_READ) && 727 (prot & PROT_EXEC) && 728 !(capabilities & BDI_CAP_EXEC_MAP) 729 ) { 730 /* backing file is not executable, try to copy */ 731 capabilities &= ~BDI_CAP_MAP_DIRECT; 732 } 733 } 734 else { 735 /* anonymous mappings are always memory backed and can be 736 * privately mapped 737 */ 738 capabilities = BDI_CAP_MAP_COPY; 739 740 /* handle PROT_EXEC implication by PROT_READ */ 741 if ((prot & PROT_READ) && 742 (current->personality & READ_IMPLIES_EXEC)) 743 prot |= PROT_EXEC; 744 } 745 746 /* allow the security API to have its say */ 747 ret = security_file_mmap(file, reqprot, prot, flags, addr, 0); 748 if (ret < 0) 749 return ret; 750 751 /* looks okay */ 752 *_capabilities = capabilities; 753 return 0; 754} 755 756/* 757 * we've determined that we can make the mapping, now translate what we 758 * now know into VMA flags 759 */ 760static unsigned long determine_vm_flags(struct file *file, 761 unsigned long prot, 762 unsigned long flags, 763 unsigned long capabilities) 764{ 765 unsigned long vm_flags; 766 767 vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags); 768 vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 769 /* vm_flags |= mm->def_flags; */ 770 771 if (!(capabilities & BDI_CAP_MAP_DIRECT)) { 772 /* attempt to share read-only copies of mapped file chunks */ 773 if (file && !(prot & PROT_WRITE)) 774 vm_flags |= VM_MAYSHARE; 775 } 776 else { 777 /* overlay a shareable mapping on the backing device or inode 778 * if possible - used for chardevs, ramfs/tmpfs/shmfs and 779 * romfs/cramfs */ 780 if (flags & MAP_SHARED) 781 vm_flags |= VM_MAYSHARE | VM_SHARED; 782 else if ((((vm_flags & capabilities) ^ vm_flags) & BDI_CAP_VMFLAGS) == 0) 783 vm_flags |= VM_MAYSHARE; 784 } 785 786 /* refuse to let anyone share private mappings with this process if 787 * it's being traced - otherwise breakpoints set in it may interfere 788 * with another untraced process 789 */ 790 if ((flags & MAP_PRIVATE) && tracehook_expect_breakpoints(current)) 791 vm_flags &= ~VM_MAYSHARE; 792 793 return vm_flags; 794} 795 796/* 797 * set up a shared mapping on a file 798 */ 799static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len) 800{ 801 int ret; 802 803 ret = vma->vm_file->f_op->mmap(vma->vm_file, vma); 804 if (ret != -ENOSYS) 805 return ret; 806 807 /* getting an ENOSYS error indicates that direct mmap isn't 808 * possible (as opposed to tried but failed) so we'll fall 809 * through to making a private copy of the data and mapping 810 * that if we can */ 811 return -ENODEV; 812} 813 814/* 815 * set up a private mapping or an anonymous shared mapping 816 */ 817static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) 818{ 819 void *base; 820 int ret; 821 822 /* invoke the file's mapping function so that it can keep track of 823 * shared mappings on devices or memory 824 * - VM_MAYSHARE will be set if it may attempt to share 825 */ 826 if (vma->vm_file) { 827 ret = vma->vm_file->f_op->mmap(vma->vm_file, vma); 828 if (ret != -ENOSYS) { 829 /* shouldn't return success if we're not sharing */ 830 BUG_ON(ret == 0 && !(vma->vm_flags & VM_MAYSHARE)); 831 return ret; /* success or a real error */ 832 } 833 834 /* getting an ENOSYS error indicates that direct mmap isn't 835 * possible (as opposed to tried but failed) so we'll try to 836 * make a private copy of the data and map that instead */ 837 } 838 839 /* allocate some memory to hold the mapping 840 * - note that this may not return a page-aligned address if the object 841 * we're allocating is smaller than a page 842 */ 843 base = kmalloc(len, GFP_KERNEL|__GFP_COMP); 844 if (!base) 845 goto enomem; 846 847 vma->vm_start = (unsigned long) base; 848 vma->vm_end = vma->vm_start + len; 849 vma->vm_flags |= VM_MAPPED_COPY; 850 851#ifdef WARN_ON_SLACK 852 if (len + WARN_ON_SLACK <= kobjsize(result)) 853 printk("Allocation of %lu bytes from process %d has %lu bytes of slack\n", 854 len, current->pid, kobjsize(result) - len); 855#endif 856 857 if (vma->vm_file) { 858 /* read the contents of a file into the copy */ 859 mm_segment_t old_fs; 860 loff_t fpos; 861 862 fpos = vma->vm_pgoff; 863 fpos <<= PAGE_SHIFT; 864 865 old_fs = get_fs(); 866 set_fs(KERNEL_DS); 867 ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos); 868 set_fs(old_fs); 869 870 if (ret < 0) 871 goto error_free; 872 873 /* clear the last little bit */ 874 if (ret < len) 875 memset(base + ret, 0, len - ret); 876 877 } else { 878 /* if it's an anonymous mapping, then just clear it */ 879 memset(base, 0, len); 880 } 881 882 return 0; 883 884error_free: 885 kfree(base); 886 vma->vm_start = 0; 887 return ret; 888 889enomem: 890 printk("Allocation of length %lu from process %d failed\n", 891 len, current->pid); 892 show_free_areas(); 893 return -ENOMEM; 894} 895 896/* 897 * handle mapping creation for uClinux 898 */ 899unsigned long do_mmap_pgoff(struct file *file, 900 unsigned long addr, 901 unsigned long len, 902 unsigned long prot, 903 unsigned long flags, 904 unsigned long pgoff) 905{ 906 struct vm_list_struct *vml = NULL; 907 struct vm_area_struct *vma = NULL; 908 struct rb_node *rb; 909 unsigned long capabilities, vm_flags; 910 void *result; 911 int ret; 912 913 if (!(flags & MAP_FIXED)) 914 addr = round_hint_to_min(addr); 915 916 /* decide whether we should attempt the mapping, and if so what sort of 917 * mapping */ 918 ret = validate_mmap_request(file, addr, len, prot, flags, pgoff, 919 &capabilities); 920 if (ret < 0) 921 return ret; 922 923 /* we've determined that we can make the mapping, now translate what we 924 * now know into VMA flags */ 925 vm_flags = determine_vm_flags(file, prot, flags, capabilities); 926 927 /* we're going to need to record the mapping if it works */ 928 vml = kzalloc(sizeof(struct vm_list_struct), GFP_KERNEL); 929 if (!vml) 930 goto error_getting_vml; 931 932 down_write(&nommu_vma_sem); 933 934 /* if we want to share, we need to check for VMAs created by other 935 * mmap() calls that overlap with our proposed mapping 936 * - we can only share with an exact match on most regular files 937 * - shared mappings on character devices and memory backed files are 938 * permitted to overlap inexactly as far as we are concerned for in 939 * these cases, sharing is handled in the driver or filesystem rather 940 * than here 941 */ 942 if (vm_flags & VM_MAYSHARE) { 943 unsigned long pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; 944 unsigned long vmpglen; 945 946 /* suppress VMA sharing for shared regions */ 947 if (vm_flags & VM_SHARED && 948 capabilities & BDI_CAP_MAP_DIRECT) 949 goto dont_share_VMAs; 950 951 for (rb = rb_first(&nommu_vma_tree); rb; rb = rb_next(rb)) { 952 vma = rb_entry(rb, struct vm_area_struct, vm_rb); 953 954 if (!(vma->vm_flags & VM_MAYSHARE)) 955 continue; 956 957 /* search for overlapping mappings on the same file */ 958 if (vma->vm_file->f_path.dentry->d_inode != file->f_path.dentry->d_inode) 959 continue; 960 961 if (vma->vm_pgoff >= pgoff + pglen) 962 continue; 963 964 vmpglen = vma->vm_end - vma->vm_start + PAGE_SIZE - 1; 965 vmpglen >>= PAGE_SHIFT; 966 if (pgoff >= vma->vm_pgoff + vmpglen) 967 continue; 968 969 /* handle inexactly overlapping matches between mappings */ 970 if (vma->vm_pgoff != pgoff || vmpglen != pglen) { 971 if (!(capabilities & BDI_CAP_MAP_DIRECT)) 972 goto sharing_violation; 973 continue; 974 } 975 976 /* we've found a VMA we can share */ 977 atomic_inc(&vma->vm_usage); 978 979 vml->vma = vma; 980 result = (void *) vma->vm_start; 981 goto shared; 982 } 983 984 dont_share_VMAs: 985 vma = NULL; 986 987 /* obtain the address at which to make a shared mapping 988 * - this is the hook for quasi-memory character devices to 989 * tell us the location of a shared mapping 990 */ 991 if (file && file->f_op->get_unmapped_area) { 992 addr = file->f_op->get_unmapped_area(file, addr, len, 993 pgoff, flags); 994 if (IS_ERR((void *) addr)) { 995 ret = addr; 996 if (ret != (unsigned long) -ENOSYS) 997 goto error; 998 999 /* the driver refused to tell us where to site 1000 * the mapping so we'll have to attempt to copy 1001 * it */ 1002 ret = (unsigned long) -ENODEV; 1003 if (!(capabilities & BDI_CAP_MAP_COPY)) 1004 goto error; 1005 1006 capabilities &= ~BDI_CAP_MAP_DIRECT; 1007 } 1008 } 1009 } 1010 1011 /* we're going to need a VMA struct as well */ 1012 vma = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL); 1013 if (!vma) 1014 goto error_getting_vma; 1015 1016 INIT_LIST_HEAD(&vma->anon_vma_node); 1017 atomic_set(&vma->vm_usage, 1); 1018 if (file) { 1019 get_file(file); 1020 if (vm_flags & VM_EXECUTABLE) { 1021 added_exe_file_vma(current->mm); 1022 vma->vm_mm = current->mm; 1023 } 1024 } 1025 vma->vm_file = file; 1026 vma->vm_flags = vm_flags; 1027 vma->vm_start = addr; 1028 vma->vm_end = addr + len; 1029 vma->vm_pgoff = pgoff; 1030 1031 vml->vma = vma; 1032 1033 /* set up the mapping */ 1034 if (file && vma->vm_flags & VM_SHARED) 1035 ret = do_mmap_shared_file(vma, len); 1036 else 1037 ret = do_mmap_private(vma, len); 1038 if (ret < 0) 1039 goto error; 1040 1041 /* okay... we have a mapping; now we have to register it */ 1042 result = (void *) vma->vm_start; 1043 1044 current->mm->total_vm += len >> PAGE_SHIFT; 1045 1046 add_nommu_vma(vma); 1047 1048 shared: 1049 add_vma_to_mm(current->mm, vml); 1050 1051 up_write(&nommu_vma_sem); 1052 1053 if (prot & PROT_EXEC) 1054 flush_icache_range((unsigned long) result, 1055 (unsigned long) result + len); 1056 1057#ifdef DEBUG 1058 printk("do_mmap:\n"); 1059 show_process_blocks(); 1060#endif 1061 1062 return (unsigned long) result; 1063 1064 error: 1065 up_write(&nommu_vma_sem); 1066 kfree(vml); 1067 if (vma) { 1068 if (vma->vm_file) { 1069 fput(vma->vm_file); 1070 if (vma->vm_flags & VM_EXECUTABLE) 1071 removed_exe_file_vma(vma->vm_mm); 1072 } 1073 kfree(vma); 1074 } 1075 return ret; 1076 1077 sharing_violation: 1078 up_write(&nommu_vma_sem); 1079 printk("Attempt to share mismatched mappings\n"); 1080 kfree(vml); 1081 return -EINVAL; 1082 1083 error_getting_vma: 1084 up_write(&nommu_vma_sem); 1085 kfree(vml); 1086 printk("Allocation of vma for %lu byte allocation from process %d failed\n", 1087 len, current->pid); 1088 show_free_areas(); 1089 return -ENOMEM; 1090 1091 error_getting_vml: 1092 printk("Allocation of vml for %lu byte allocation from process %d failed\n", 1093 len, current->pid); 1094 show_free_areas(); 1095 return -ENOMEM; 1096} 1097EXPORT_SYMBOL(do_mmap_pgoff); 1098 1099/* 1100 * handle mapping disposal for uClinux 1101 */ 1102static void put_vma(struct mm_struct *mm, struct vm_area_struct *vma) 1103{ 1104 if (vma) { 1105 down_write(&nommu_vma_sem); 1106 1107 if (atomic_dec_and_test(&vma->vm_usage)) { 1108 delete_nommu_vma(vma); 1109 1110 if (vma->vm_ops && vma->vm_ops->close) 1111 vma->vm_ops->close(vma); 1112 1113 /* IO memory and memory shared directly out of the pagecache from 1114 * ramfs/tmpfs mustn't be released here */ 1115 if (vma->vm_flags & VM_MAPPED_COPY) 1116 kfree((void *) vma->vm_start); 1117 1118 if (vma->vm_file) { 1119 fput(vma->vm_file); 1120 if (vma->vm_flags & VM_EXECUTABLE) 1121 removed_exe_file_vma(mm); 1122 } 1123 kfree(vma); 1124 } 1125 1126 up_write(&nommu_vma_sem); 1127 } 1128} 1129 1130/* 1131 * release a mapping 1132 * - under NOMMU conditions the parameters must match exactly to the mapping to 1133 * be removed 1134 */ 1135int do_munmap(struct mm_struct *mm, unsigned long addr, size_t len) 1136{ 1137 struct vm_list_struct *vml, **parent; 1138 unsigned long end = addr + len; 1139 1140#ifdef DEBUG 1141 printk("do_munmap:\n"); 1142#endif 1143 1144 for (parent = &mm->context.vmlist; *parent; parent = &(*parent)->next) { 1145 if ((*parent)->vma->vm_start > addr) 1146 break; 1147 if ((*parent)->vma->vm_start == addr && 1148 ((len == 0) || ((*parent)->vma->vm_end == end))) 1149 goto found; 1150 } 1151 1152 printk("munmap of non-mmaped memory by process %d (%s): %p\n", 1153 current->pid, current->comm, (void *) addr); 1154 return -EINVAL; 1155 1156 found: 1157 vml = *parent; 1158 1159 put_vma(mm, vml->vma); 1160 1161 *parent = vml->next; 1162 kfree(vml); 1163 1164 update_hiwater_vm(mm); 1165 mm->total_vm -= len >> PAGE_SHIFT; 1166 1167#ifdef DEBUG 1168 show_process_blocks(); 1169#endif 1170 1171 return 0; 1172} 1173EXPORT_SYMBOL(do_munmap); 1174 1175asmlinkage long sys_munmap(unsigned long addr, size_t len) 1176{ 1177 int ret; 1178 struct mm_struct *mm = current->mm; 1179 1180 down_write(&mm->mmap_sem); 1181 ret = do_munmap(mm, addr, len); 1182 up_write(&mm->mmap_sem); 1183 return ret; 1184} 1185 1186/* 1187 * Release all mappings 1188 */ 1189void exit_mmap(struct mm_struct * mm) 1190{ 1191 struct vm_list_struct *tmp; 1192 1193 if (mm) { 1194#ifdef DEBUG 1195 printk("Exit_mmap:\n"); 1196#endif 1197 1198 mm->total_vm = 0; 1199 1200 while ((tmp = mm->context.vmlist)) { 1201 mm->context.vmlist = tmp->next; 1202 put_vma(mm, tmp->vma); 1203 kfree(tmp); 1204 } 1205 1206#ifdef DEBUG 1207 show_process_blocks(); 1208#endif 1209 } 1210} 1211 1212unsigned long do_brk(unsigned long addr, unsigned long len) 1213{ 1214 return -ENOMEM; 1215} 1216 1217/* 1218 * expand (or shrink) an existing mapping, potentially moving it at the same 1219 * time (controlled by the MREMAP_MAYMOVE flag and available VM space) 1220 * 1221 * under NOMMU conditions, we only permit changing a mapping's size, and only 1222 * as long as it stays within the hole allocated by the kmalloc() call in 1223 * do_mmap_pgoff() and the block is not shareable 1224 * 1225 * MREMAP_FIXED is not supported under NOMMU conditions 1226 */ 1227unsigned long do_mremap(unsigned long addr, 1228 unsigned long old_len, unsigned long new_len, 1229 unsigned long flags, unsigned long new_addr) 1230{ 1231 struct vm_area_struct *vma; 1232 1233 /* insanity checks first */ 1234 if (new_len == 0) 1235 return (unsigned long) -EINVAL; 1236 1237 if (flags & MREMAP_FIXED && new_addr != addr) 1238 return (unsigned long) -EINVAL; 1239 1240 vma = find_vma_exact(current->mm, addr); 1241 if (!vma) 1242 return (unsigned long) -EINVAL; 1243 1244 if (vma->vm_end != vma->vm_start + old_len) 1245 return (unsigned long) -EFAULT; 1246 1247 if (vma->vm_flags & VM_MAYSHARE) 1248 return (unsigned long) -EPERM; 1249 1250 if (new_len > kobjsize((void *) addr)) 1251 return (unsigned long) -ENOMEM; 1252 1253 /* all checks complete - do it */ 1254 vma->vm_end = vma->vm_start + new_len; 1255 1256 return vma->vm_start; 1257} 1258EXPORT_SYMBOL(do_mremap); 1259 1260asmlinkage unsigned long sys_mremap(unsigned long addr, 1261 unsigned long old_len, unsigned long new_len, 1262 unsigned long flags, unsigned long new_addr) 1263{ 1264 unsigned long ret; 1265 1266 down_write(¤t->mm->mmap_sem); 1267 ret = do_mremap(addr, old_len, new_len, flags, new_addr); 1268 up_write(¤t->mm->mmap_sem); 1269 return ret; 1270} 1271 1272struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 1273 unsigned int foll_flags) 1274{ 1275 return NULL; 1276} 1277 1278int remap_pfn_range(struct vm_area_struct *vma, unsigned long from, 1279 unsigned long to, unsigned long size, pgprot_t prot) 1280{ 1281 vma->vm_start = vma->vm_pgoff << PAGE_SHIFT; 1282 return 0; 1283} 1284EXPORT_SYMBOL(remap_pfn_range); 1285 1286int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 1287 unsigned long pgoff) 1288{ 1289 unsigned int size = vma->vm_end - vma->vm_start; 1290 1291 if (!(vma->vm_flags & VM_USERMAP)) 1292 return -EINVAL; 1293 1294 vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT)); 1295 vma->vm_end = vma->vm_start + size; 1296 1297 return 0; 1298} 1299EXPORT_SYMBOL(remap_vmalloc_range); 1300 1301void swap_unplug_io_fn(struct backing_dev_info *bdi, struct page *page) 1302{ 1303} 1304 1305unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr, 1306 unsigned long len, unsigned long pgoff, unsigned long flags) 1307{ 1308 return -ENOMEM; 1309} 1310 1311void arch_unmap_area(struct mm_struct *mm, unsigned long addr) 1312{ 1313} 1314 1315void unmap_mapping_range(struct address_space *mapping, 1316 loff_t const holebegin, loff_t const holelen, 1317 int even_cows) 1318{ 1319} 1320EXPORT_SYMBOL(unmap_mapping_range); 1321 1322/* 1323 * ask for an unmapped area at which to create a mapping on a file 1324 */ 1325unsigned long get_unmapped_area(struct file *file, unsigned long addr, 1326 unsigned long len, unsigned long pgoff, 1327 unsigned long flags) 1328{ 1329 unsigned long (*get_area)(struct file *, unsigned long, unsigned long, 1330 unsigned long, unsigned long); 1331 1332 get_area = current->mm->get_unmapped_area; 1333 if (file && file->f_op && file->f_op->get_unmapped_area) 1334 get_area = file->f_op->get_unmapped_area; 1335 1336 if (!get_area) 1337 return -ENOSYS; 1338 1339 return get_area(file, addr, len, pgoff, flags); 1340} 1341EXPORT_SYMBOL(get_unmapped_area); 1342 1343/* 1344 * Check that a process has enough memory to allocate a new virtual 1345 * mapping. 0 means there is enough memory for the allocation to 1346 * succeed and -ENOMEM implies there is not. 1347 * 1348 * We currently support three overcommit policies, which are set via the 1349 * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting 1350 * 1351 * Strict overcommit modes added 2002 Feb 26 by Alan Cox. 1352 * Additional code 2002 Jul 20 by Robert Love. 1353 * 1354 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise. 1355 * 1356 * Note this is a helper function intended to be used by LSMs which 1357 * wish to use this logic. 1358 */ 1359int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) 1360{ 1361 unsigned long free, allowed; 1362 1363 vm_acct_memory(pages); 1364 1365 /* 1366 * Sometimes we want to use more memory than we have 1367 */ 1368 if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS) 1369 return 0; 1370 1371 if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { 1372 unsigned long n; 1373 1374 free = global_page_state(NR_FILE_PAGES); 1375 free += nr_swap_pages; 1376 1377 /* 1378 * Any slabs which are created with the 1379 * SLAB_RECLAIM_ACCOUNT flag claim to have contents 1380 * which are reclaimable, under pressure. The dentry 1381 * cache and most inode caches should fall into this 1382 */ 1383 free += global_page_state(NR_SLAB_RECLAIMABLE); 1384 1385 /* 1386 * Leave the last 3% for root 1387 */ 1388 if (!cap_sys_admin) 1389 free -= free / 32; 1390 1391 if (free > pages) 1392 return 0; 1393 1394 /* 1395 * nr_free_pages() is very expensive on large systems, 1396 * only call if we're about to fail. 1397 */ 1398 n = nr_free_pages(); 1399 1400 /* 1401 * Leave reserved pages. The pages are not for anonymous pages. 1402 */ 1403 if (n <= totalreserve_pages) 1404 goto error; 1405 else 1406 n -= totalreserve_pages; 1407 1408 /* 1409 * Leave the last 3% for root 1410 */ 1411 if (!cap_sys_admin) 1412 n -= n / 32; 1413 free += n; 1414 1415 if (free > pages) 1416 return 0; 1417 1418 goto error; 1419 } 1420 1421 allowed = totalram_pages * sysctl_overcommit_ratio / 100; 1422 /* 1423 * Leave the last 3% for root 1424 */ 1425 if (!cap_sys_admin) 1426 allowed -= allowed / 32; 1427 allowed += total_swap_pages; 1428 1429 /* Don't let a single process grow too big: 1430 leave 3% of the size of this process for other processes */ 1431 if (mm) 1432 allowed -= mm->total_vm / 32; 1433 1434 /* 1435 * cast `allowed' as a signed long because vm_committed_space 1436 * sometimes has a negative value 1437 */ 1438 if (atomic_long_read(&vm_committed_space) < (long)allowed) 1439 return 0; 1440error: 1441 vm_unacct_memory(pages); 1442 1443 return -ENOMEM; 1444} 1445 1446int in_gate_area_no_task(unsigned long addr) 1447{ 1448 return 0; 1449} 1450 1451int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1452{ 1453 BUG(); 1454 return 0; 1455} 1456EXPORT_SYMBOL(filemap_fault); 1457 1458/* 1459 * Access another process' address space. 1460 * - source/target buffer must be kernel space 1461 */ 1462int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write) 1463{ 1464 struct vm_area_struct *vma; 1465 struct mm_struct *mm; 1466 1467 if (addr + len < addr) 1468 return 0; 1469 1470 mm = get_task_mm(tsk); 1471 if (!mm) 1472 return 0; 1473 1474 down_read(&mm->mmap_sem); 1475 1476 /* the access must start within one of the target process's mappings */ 1477 vma = find_vma(mm, addr); 1478 if (vma) { 1479 /* don't overrun this mapping */ 1480 if (addr + len >= vma->vm_end) 1481 len = vma->vm_end - addr; 1482 1483 /* only read or write mappings where it is permitted */ 1484 if (write && vma->vm_flags & VM_MAYWRITE) 1485 len -= copy_to_user((void *) addr, buf, len); 1486 else if (!write && vma->vm_flags & VM_MAYREAD) 1487 len -= copy_from_user(buf, (void *) addr, len); 1488 else 1489 len = 0; 1490 } else { 1491 len = 0; 1492 } 1493 1494 up_read(&mm->mmap_sem); 1495 mmput(mm); 1496 return len; 1497} 1498