11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * linux/mm/nommu.c 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Replacement code for mm functions to support CPU's that don't 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * have any form of memory management unit (thus no virtual memory). 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * See Documentation/nommu-mmap.txt 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 98feae13110d60cc6287afabc2887366b0eb226c2David Howells * Copyright (c) 2004-2008 David Howells <dhowells@redhat.com> 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com> 1329c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16b95f1b31b75588306e32b2afd32166cad48f670bPaul Gortmaker#include <linux/export.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mman.h> 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/swap.h> 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/file.h> 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/highmem.h> 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/pagemap.h> 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/vmalloc.h> 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/blkdev.h> 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/backing-dev.h> 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mount.h> 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/personality.h> 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/security.h> 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/syscalls.h> 31120a795da07c9a02221ca23464c28a7c6ad7de1dAl Viro#include <linux/audit.h> 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/uaccess.h> 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/tlb.h> 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/tlbflush.h> 36eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt#include <asm/mmu_context.h> 378feae13110d60cc6287afabc2887366b0eb226c2David Howells#include "internal.h" 388feae13110d60cc6287afabc2887366b0eb226c2David Howells 398feae13110d60cc6287afabc2887366b0eb226c2David Howells#if 0 408feae13110d60cc6287afabc2887366b0eb226c2David Howells#define kenter(FMT, ...) \ 418feae13110d60cc6287afabc2887366b0eb226c2David Howells printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__) 428feae13110d60cc6287afabc2887366b0eb226c2David Howells#define kleave(FMT, ...) \ 438feae13110d60cc6287afabc2887366b0eb226c2David Howells printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__) 448feae13110d60cc6287afabc2887366b0eb226c2David Howells#define kdebug(FMT, ...) \ 458feae13110d60cc6287afabc2887366b0eb226c2David Howells printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__) 468feae13110d60cc6287afabc2887366b0eb226c2David Howells#else 478feae13110d60cc6287afabc2887366b0eb226c2David Howells#define kenter(FMT, ...) \ 488feae13110d60cc6287afabc2887366b0eb226c2David Howells no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__) 498feae13110d60cc6287afabc2887366b0eb226c2David Howells#define kleave(FMT, ...) \ 508feae13110d60cc6287afabc2887366b0eb226c2David Howells no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__) 518feae13110d60cc6287afabc2887366b0eb226c2David Howells#define kdebug(FMT, ...) \ 528feae13110d60cc6287afabc2887366b0eb226c2David Howells no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__) 538feae13110d60cc6287afabc2887366b0eb226c2David Howells#endif 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid *high_memory; 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct page *mem_map; 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long max_mapnr; 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long num_physpages; 594266c97a3ef4604561a22212eb0eab8a3c338971Hugh Dickinsunsigned long highest_memmap_pfn; 6000a62ce91e554198ef28234c91c36f850f5a3bc9KOSAKI Motohirostruct percpu_counter vm_committed_as; 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint sysctl_overcommit_ratio = 50; /* default is 50% */ 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; 64fc4d5c292b68ef02514d2072dcbf82d090c34875David Howellsint sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS; 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint heap_stack_gap = 0; 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6733e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howellsatomic_long_t mmap_pages_allocated; 688feae13110d60cc6287afabc2887366b0eb226c2David Howells 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(mem_map); 706a04de6dbe1772d98fddf5099738d6f508e86e21Wu, BryanEXPORT_SYMBOL(num_physpages); 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 728feae13110d60cc6287afabc2887366b0eb226c2David Howells/* list of mapped, potentially shareable regions */ 738feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic struct kmem_cache *vm_region_jar; 748feae13110d60cc6287afabc2887366b0eb226c2David Howellsstruct rb_root nommu_region_tree = RB_ROOT; 758feae13110d60cc6287afabc2887366b0eb226c2David HowellsDECLARE_RWSEM(nommu_region_sem); 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 77f0f37e2f77731b3473fa6bd5ee53255d9a9cdb40Alexey Dobriyanconst struct vm_operations_struct generic_file_vm_ops = { 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Return the total memory allocated for this pointer, not 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * just what the caller asked for. 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Doesn't have to be accurate, i.e. may have races. 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned int kobjsize(const void *objp) 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page *page; 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 904016a1390d07f15b267eecb20e76a48fd5c524efMichael Hennerich /* 914016a1390d07f15b267eecb20e76a48fd5c524efMichael Hennerich * If the object we have should not have ksize performed on it, 924016a1390d07f15b267eecb20e76a48fd5c524efMichael Hennerich * return size of 0 934016a1390d07f15b267eecb20e76a48fd5c524efMichael Hennerich */ 945a1603be58f11edb1b30cb1e40cfbdd4439289d0Paul Mundt if (!objp || !virt_addr_valid(objp)) 956cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt return 0; 966cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt 976cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt page = virt_to_head_page(objp); 986cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt 996cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt /* 1006cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt * If the allocator sets PageSlab, we know the pointer came from 1016cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt * kmalloc(). 1026cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt */ 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (PageSlab(page)) 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ksize(objp); 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1066cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt /* 107ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt * If it's not a compound page, see if we have a matching VMA 108ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt * region. This test is intentionally done in reverse order, 109ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt * so if there's no VMA, we still fall through and hand back 110ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt * PAGE_SIZE for 0-order pages. 111ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt */ 112ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt if (!PageCompound(page)) { 113ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt struct vm_area_struct *vma; 114ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt 115ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt vma = find_vma(current->mm, (unsigned long)objp); 116ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt if (vma) 117ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt return vma->vm_end - vma->vm_start; 118ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt } 119ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt 120ab2e83ead4eca9e045daac4cbf66eb1e7a244bb2Paul Mundt /* 1216cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt * The ksize() function is only guaranteed to work for pointers 1225a1603be58f11edb1b30cb1e40cfbdd4439289d0Paul Mundt * returned by kmalloc(). So handle arbitrary pointers here. 1236cfd53fc03670c7a544a56d441eb1a6cc800d72bPaul Mundt */ 1245a1603be58f11edb1b30cb1e40cfbdd4439289d0Paul Mundt return PAGE_SIZE << compound_order(page); 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 127b291f000393f5a0b679012b39d79fbc85c018233Nick Pigginint __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1284266c97a3ef4604561a22212eb0eab8a3c338971Hugh Dickins unsigned long start, int nr_pages, unsigned int foll_flags, 12953a7706d5ed8f1a53ba062b318773160cc476ddeMichel Lespinasse struct page **pages, struct vm_area_struct **vmas, 13053a7706d5ed8f1a53ba062b318773160cc476ddeMichel Lespinasse int *retry) 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 132910e46da4b4e93d56ffea318c64afa41868d5e6dSonic Zhang struct vm_area_struct *vma; 1337b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells unsigned long vm_flags; 1347b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells int i; 1357b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells 1367b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells /* calculate required read or write permissions. 13758fa879e1e640a1856f736b418984ebeccee1c95Hugh Dickins * If FOLL_FORCE is set, we only require the "MAY" flags. 1387b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells */ 13958fa879e1e640a1856f736b418984ebeccee1c95Hugh Dickins vm_flags = (foll_flags & FOLL_WRITE) ? 14058fa879e1e640a1856f736b418984ebeccee1c95Hugh Dickins (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 14158fa879e1e640a1856f736b418984ebeccee1c95Hugh Dickins vm_flags &= (foll_flags & FOLL_FORCE) ? 14258fa879e1e640a1856f736b418984ebeccee1c95Hugh Dickins (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1449d73777e500929b71dcfed16eec05f6760e345a6Peter Zijlstra for (i = 0; i < nr_pages; i++) { 1457561e8ca0dfaf6fca3feef982830de3b65300e5bDavid Howells vma = find_vma(mm, start); 1467b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells if (!vma) 1477b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells goto finish_or_fault; 1487b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells 1497b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells /* protect what we can, including chardevs */ 1501c3aff1ceec2cc86810e2690e67873ff0c505862Hugh Dickins if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) || 1511c3aff1ceec2cc86810e2690e67873ff0c505862Hugh Dickins !(vm_flags & vma->vm_flags)) 1527b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells goto finish_or_fault; 153910e46da4b4e93d56ffea318c64afa41868d5e6dSonic Zhang 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pages) { 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pages[i] = virt_to_page(start); 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pages[i]) 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page_cache_get(pages[i]); 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (vmas) 160910e46da4b4e93d56ffea318c64afa41868d5e6dSonic Zhang vmas[i] = vma; 161e1ee65d85904c5dd4b9cea1b15d5e85e20eae8a1David Howells start = (start + PAGE_SIZE) & PAGE_MASK; 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1637b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells 1647b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells return i; 1657b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells 1667b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howellsfinish_or_fault: 1677b4d5b8b39fd3701ed3693a89f2bd8f6ef49bce2David Howells return i ? : -EFAULT; 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 169b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin 170b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin/* 171b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin * get a list of pages in an address range belonging to the specified process 172b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin * and indicate the VMA that covers each page 173b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin * - this is potentially dodgy as we may end incrementing the page count of a 174b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin * slab page or a secondary page from a compound page 175b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin * - don't permit access to VMAs that don't support it, such as I/O mappings 176b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin */ 177b291f000393f5a0b679012b39d79fbc85c018233Nick Pigginint get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1789d73777e500929b71dcfed16eec05f6760e345a6Peter Zijlstra unsigned long start, int nr_pages, int write, int force, 179b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin struct page **pages, struct vm_area_struct **vmas) 180b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin{ 181b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin int flags = 0; 182b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin 183b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin if (write) 18458fa879e1e640a1856f736b418984ebeccee1c95Hugh Dickins flags |= FOLL_WRITE; 185b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin if (force) 18658fa879e1e640a1856f736b418984ebeccee1c95Hugh Dickins flags |= FOLL_FORCE; 187b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin 18853a7706d5ed8f1a53ba062b318773160cc476ddeMichel Lespinasse return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas, 18953a7706d5ed8f1a53ba062b318773160cc476ddeMichel Lespinasse NULL); 190b291f000393f5a0b679012b39d79fbc85c018233Nick Piggin} 19166aa2b4b1cf9a61f1550251c56fc6f0d48287591Greg UngererEXPORT_SYMBOL(get_user_pages); 19266aa2b4b1cf9a61f1550251c56fc6f0d48287591Greg Ungerer 193dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt/** 194dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt * follow_pfn - look up PFN at a user virtual address 195dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt * @vma: memory mapping 196dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt * @address: user virtual address 197dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt * @pfn: location to store found PFN 198dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt * 199dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt * Only IO mappings and raw PFN mappings are allowed. 200dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt * 201dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt * Returns zero and the pfn at @pfn on success, -ve otherwise. 202dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt */ 203dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundtint follow_pfn(struct vm_area_struct *vma, unsigned long address, 204dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt unsigned long *pfn) 205dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt{ 206dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 207dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt return -EINVAL; 208dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt 209dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt *pfn = address >> PAGE_SHIFT; 210dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt return 0; 211dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt} 212dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul MundtEXPORT_SYMBOL(follow_pfn); 213dfc2f91ac29f5ef50e74bf15a1a6b6aa6b952e62Paul Mundt 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsDEFINE_RWLOCK(vmlist_lock); 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct vm_struct *vmlist; 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 217b3bdda02aa547a0753b4fdbc105e86ef9046b30bChristoph Lametervoid vfree(const void *addr) 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kfree(addr); 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 221b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(vfree); 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 223dd0fc66fb33cd610bc1a5db8a5e232d34879b4d7Al Virovoid *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 2268518609deeacebafd71855f87cc411adb0c3be4eRobert P. J. Day * You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc() 2278518609deeacebafd71855f87cc411adb0c3be4eRobert P. J. Day * returns only a logical address. 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 22984097518d1ecd2330f9488e4c2d09953a3340e74Nick Piggin return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM); 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 231b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(__vmalloc); 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 233f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundtvoid *vmalloc_user(unsigned long size) 234f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt{ 235f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt void *ret; 236f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt 237f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, 238f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt PAGE_KERNEL); 239f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt if (ret) { 240f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt struct vm_area_struct *vma; 241f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt 242f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt down_write(¤t->mm->mmap_sem); 243f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt vma = find_vma(current->mm, (unsigned long)ret); 244f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt if (vma) 245f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt vma->vm_flags |= VM_USERMAP; 246f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt up_write(¤t->mm->mmap_sem); 247f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt } 248f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt 249f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt return ret; 250f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt} 251f905bc447c303fefcb180c7e8b641746ffa6cf87Paul MundtEXPORT_SYMBOL(vmalloc_user); 252f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt 253b3bdda02aa547a0753b4fdbc105e86ef9046b30bChristoph Lameterstruct page *vmalloc_to_page(const void *addr) 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return virt_to_page(addr); 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 257b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(vmalloc_to_page); 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 259b3bdda02aa547a0753b4fdbc105e86ef9046b30bChristoph Lameterunsigned long vmalloc_to_pfn(const void *addr) 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return page_to_pfn(virt_to_page(addr)); 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 263b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(vmalloc_to_pfn); 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldslong vread(char *buf, char *addr, unsigned long count) 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memcpy(buf, addr, count); 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return count; 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldslong vwrite(char *buf, char *addr, unsigned long count) 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Don't allow overflow */ 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((unsigned long) addr + count < count) 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds count = -(unsigned long) addr; 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memcpy(addr, buf, count); 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return(count); 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * vmalloc - allocate virtually continguos memory 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @size: allocation size 2851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 2861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Allocate enough pages to cover @size from the page level 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allocator and map them into continguos kernel virtual space. 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 289c1c8897f830c66649b6866a0cbe21c263466295eMichael Opdenacker * For tight control over page level allocator and protection flags 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * use __vmalloc() instead. 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid *vmalloc(unsigned long size) 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL); 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 296f61388822a6040ff462c5f7260daa0f1017f2db0Andrew MortonEXPORT_SYMBOL(vmalloc); 297f61388822a6040ff462c5f7260daa0f1017f2db0Andrew Morton 298e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young/* 299e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * vzalloc - allocate virtually continguos memory with zero fill 300e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * 301e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * @size: allocation size 302e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * 303e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * Allocate enough pages to cover @size from the page level 304e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * allocator and map them into continguos kernel virtual space. 305e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * The memory allocated is set to zero. 306e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * 307e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * For tight control over page level allocator and protection flags 308e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * use __vmalloc() instead. 309e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young */ 310e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Youngvoid *vzalloc(unsigned long size) 311e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young{ 312e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, 313e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young PAGE_KERNEL); 314e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young} 315e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave YoungEXPORT_SYMBOL(vzalloc); 316e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young 317e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young/** 318e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * vmalloc_node - allocate memory on a specific node 319e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * @size: allocation size 320e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * @node: numa node 321e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * 322e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * Allocate enough pages to cover @size from the page level 323e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * allocator and map them into contiguous kernel virtual space. 324e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * 325e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * For tight control over page level allocator and protection flags 326e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * use __vmalloc() instead. 327e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young */ 328f61388822a6040ff462c5f7260daa0f1017f2db0Andrew Mortonvoid *vmalloc_node(unsigned long size, int node) 329f61388822a6040ff462c5f7260daa0f1017f2db0Andrew Morton{ 330f61388822a6040ff462c5f7260daa0f1017f2db0Andrew Morton return vmalloc(size); 331f61388822a6040ff462c5f7260daa0f1017f2db0Andrew Morton} 3329a14f653dfe349c0916e6a78c413effa2fa3f001Paul MundtEXPORT_SYMBOL(vmalloc_node); 333e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young 334e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young/** 335e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * vzalloc_node - allocate memory on a specific node with zero fill 336e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * @size: allocation size 337e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * @node: numa node 338e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * 339e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * Allocate enough pages to cover @size from the page level 340e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * allocator and map them into contiguous kernel virtual space. 341e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * The memory allocated is set to zero. 342e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * 343e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * For tight control over page level allocator and protection flags 344e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young * use __vmalloc() instead. 345e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young */ 346e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Youngvoid *vzalloc_node(unsigned long size, int node) 347e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young{ 348e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young return vzalloc(size); 349e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave Young} 350e1ca7788dec6773b1a2bce51b7141948f2b8bccfDave YoungEXPORT_SYMBOL(vzalloc_node); 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3521af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt#ifndef PAGE_KERNEL_EXEC 3531af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt# define PAGE_KERNEL_EXEC PAGE_KERNEL 3541af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt#endif 3551af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt 3561af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt/** 3571af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * vmalloc_exec - allocate virtually contiguous, executable memory 3581af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * @size: allocation size 3591af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * 3601af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * Kernel-internal function to allocate enough pages to cover @size 3611af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * the page level allocator and map them into contiguous and 3621af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * executable kernel virtual space. 3631af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * 3641af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * For tight control over page level allocator and protection flags 3651af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt * use __vmalloc() instead. 3661af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt */ 3671af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt 3681af446edfe3239b2b731f3458b3c285c397464ccPaul Mundtvoid *vmalloc_exec(unsigned long size) 3691af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt{ 3701af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC); 3711af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt} 3721af446edfe3239b2b731f3458b3c285c397464ccPaul Mundt 373b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt/** 374b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @size: allocation size 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Allocate enough 32bit PA addressable pages to cover @size from the 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * page level allocator and map them into continguos kernel virtual space. 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid *vmalloc_32(unsigned long size) 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL); 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 384b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(vmalloc_32); 385b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt 386b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt/** 387b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 388b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt * @size: allocation size 389b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt * 390b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt * The resulting memory area is 32bit addressable and zeroed so it can be 391b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt * mapped to userspace without leaking data. 392f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt * 393f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt * VM_USERMAP is set on the corresponding VMA so that subsequent calls to 394f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt * remap_vmalloc_range() are permissible. 395b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt */ 396b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundtvoid *vmalloc_32_user(unsigned long size) 397b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt{ 398f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt /* 399f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt * We'll have to sort out the ZONE_DMA bits for 64-bit, 400f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt * but for now this can simply use vmalloc_user() directly. 401f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt */ 402f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt return vmalloc_user(size); 403b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt} 404b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(vmalloc_32_user); 4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot) 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG(); 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 411b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(vmap); 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 413b3bdda02aa547a0753b4fdbc105e86ef9046b30bChristoph Lametervoid vunmap(const void *addr) 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BUG(); 4161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 417b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(vunmap); 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 419eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundtvoid *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot) 420eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt{ 421eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt BUG(); 422eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt return NULL; 423eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt} 424eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul MundtEXPORT_SYMBOL(vm_map_ram); 425eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt 426eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundtvoid vm_unmap_ram(const void *mem, unsigned int count) 427eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt{ 428eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt BUG(); 429eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt} 430eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul MundtEXPORT_SYMBOL(vm_unmap_ram); 431eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt 432eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundtvoid vm_unmap_aliases(void) 433eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt{ 434eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt} 435eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul MundtEXPORT_SYMBOL_GPL(vm_unmap_aliases); 436eb6434d9e79a72d35d68811efd68fe8bab8f5bafPaul Mundt 4371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 4381eeb66a1bb973534dc3d064920a5ca683823372eChristoph Hellwig * Implement a stub for vmalloc_sync_all() if the architecture chose not to 4391eeb66a1bb973534dc3d064920a5ca683823372eChristoph Hellwig * have one. 4401eeb66a1bb973534dc3d064920a5ca683823372eChristoph Hellwig */ 4411eeb66a1bb973534dc3d064920a5ca683823372eChristoph Hellwigvoid __attribute__((weak)) vmalloc_sync_all(void) 4421eeb66a1bb973534dc3d064920a5ca683823372eChristoph Hellwig{ 4431eeb66a1bb973534dc3d064920a5ca683823372eChristoph Hellwig} 4441eeb66a1bb973534dc3d064920a5ca683823372eChristoph Hellwig 44529c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt/** 44629c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * alloc_vm_area - allocate a range of kernel address space 44729c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * @size: size of the area 44829c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * 44929c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * Returns: NULL on failure, vm_struct on success 45029c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * 45129c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * This function reserves a range of kernel address space, and 45229c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * allocates pagetables to map that range. No actual mappings 45329c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * are created. If the kernel address space is not shared 45429c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * between processes, it syncs the pagetable across all 45529c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt * processes. 45629c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt */ 457cd12909cb576d37311fe35868780e82d5007d0c8David Vrabelstruct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) 45829c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt{ 45929c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt BUG(); 46029c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt return NULL; 46129c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt} 46229c185e5c681ca00d863d161eda7eadb93e32ee5Paul MundtEXPORT_SYMBOL_GPL(alloc_vm_area); 46329c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt 46429c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundtvoid free_vm_area(struct vm_struct *area) 46529c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt{ 46629c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt BUG(); 46729c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt} 46829c185e5c681ca00d863d161eda7eadb93e32ee5Paul MundtEXPORT_SYMBOL_GPL(free_vm_area); 46929c185e5c681ca00d863d161eda7eadb93e32ee5Paul Mundt 470b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundtint vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 471b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt struct page *page) 472b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt{ 473b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt return -EINVAL; 474b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt} 475b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(vm_insert_page); 476b50731732f926d6c49fd0724616a7344c31cd5cfPaul Mundt 4771eeb66a1bb973534dc3d064920a5ca683823372eChristoph Hellwig/* 4781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * sys_brk() for the most part doesn't need the global kernel 4791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * lock, except when an application is doing something nasty 4801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * like trying to un-brk an area that has already been mapped 4811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * to a regular file. in this case, the unmapping will need 4821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * to invoke file system routines that need the global lock. 4831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 4846a6160a7b5c27b3c38651baef92a14fa7072b3c1Heiko CarstensSYSCALL_DEFINE1(brk, unsigned long, brk) 4851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct mm_struct *mm = current->mm; 4871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (brk < mm->start_brk || brk > mm->context.end_brk) 4891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return mm->brk; 4901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (mm->brk == brk) 4921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return mm->brk; 4931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 4951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Always allow shrinking brk 4961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 4971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (brk <= mm->brk) { 4981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mm->brk = brk; 4991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return brk; 5001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 5011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 5031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Ok, looks good - let it rip. 5041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 505cfe79c00a2f4f687eed8b7534d1d3d3d35540c29Mike Frysinger flush_icache_range(mm->brk, brk); 5061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return mm->brk = brk; 5071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5098feae13110d60cc6287afabc2887366b0eb226c2David Howells/* 5108feae13110d60cc6287afabc2887366b0eb226c2David Howells * initialise the VMA and region record slabs 5118feae13110d60cc6287afabc2887366b0eb226c2David Howells */ 5128feae13110d60cc6287afabc2887366b0eb226c2David Howellsvoid __init mmap_init(void) 5131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 51400a62ce91e554198ef28234c91c36f850f5a3bc9KOSAKI Motohiro int ret; 51500a62ce91e554198ef28234c91c36f850f5a3bc9KOSAKI Motohiro 51600a62ce91e554198ef28234c91c36f850f5a3bc9KOSAKI Motohiro ret = percpu_counter_init(&vm_committed_as, 0); 51700a62ce91e554198ef28234c91c36f850f5a3bc9KOSAKI Motohiro VM_BUG_ON(ret); 51833e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC); 5191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5213034097a5017dd9281b1f795e80af9859627850eDavid Howells/* 5228feae13110d60cc6287afabc2887366b0eb226c2David Howells * validate the region tree 5238feae13110d60cc6287afabc2887366b0eb226c2David Howells * - the caller must hold the region lock 5243034097a5017dd9281b1f795e80af9859627850eDavid Howells */ 5258feae13110d60cc6287afabc2887366b0eb226c2David Howells#ifdef CONFIG_DEBUG_NOMMU_REGIONS 5268feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic noinline void validate_nommu_regions(void) 5273034097a5017dd9281b1f795e80af9859627850eDavid Howells{ 5288feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_region *region, *last; 5298feae13110d60cc6287afabc2887366b0eb226c2David Howells struct rb_node *p, *lastp; 5303034097a5017dd9281b1f795e80af9859627850eDavid Howells 5318feae13110d60cc6287afabc2887366b0eb226c2David Howells lastp = rb_first(&nommu_region_tree); 5328feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!lastp) 5338feae13110d60cc6287afabc2887366b0eb226c2David Howells return; 5348feae13110d60cc6287afabc2887366b0eb226c2David Howells 5358feae13110d60cc6287afabc2887366b0eb226c2David Howells last = rb_entry(lastp, struct vm_region, vm_rb); 53633e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells BUG_ON(unlikely(last->vm_end <= last->vm_start)); 53733e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells BUG_ON(unlikely(last->vm_top < last->vm_end)); 5388feae13110d60cc6287afabc2887366b0eb226c2David Howells 5398feae13110d60cc6287afabc2887366b0eb226c2David Howells while ((p = rb_next(lastp))) { 5408feae13110d60cc6287afabc2887366b0eb226c2David Howells region = rb_entry(p, struct vm_region, vm_rb); 5418feae13110d60cc6287afabc2887366b0eb226c2David Howells last = rb_entry(lastp, struct vm_region, vm_rb); 5428feae13110d60cc6287afabc2887366b0eb226c2David Howells 54333e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells BUG_ON(unlikely(region->vm_end <= region->vm_start)); 54433e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells BUG_ON(unlikely(region->vm_top < region->vm_end)); 54533e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells BUG_ON(unlikely(region->vm_start < last->vm_top)); 5463034097a5017dd9281b1f795e80af9859627850eDavid Howells 5478feae13110d60cc6287afabc2887366b0eb226c2David Howells lastp = p; 5488feae13110d60cc6287afabc2887366b0eb226c2David Howells } 5493034097a5017dd9281b1f795e80af9859627850eDavid Howells} 5508feae13110d60cc6287afabc2887366b0eb226c2David Howells#else 55133e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howellsstatic void validate_nommu_regions(void) 55233e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells{ 55333e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells} 5548feae13110d60cc6287afabc2887366b0eb226c2David Howells#endif 5553034097a5017dd9281b1f795e80af9859627850eDavid Howells 5563034097a5017dd9281b1f795e80af9859627850eDavid Howells/* 5578feae13110d60cc6287afabc2887366b0eb226c2David Howells * add a region into the global tree 5583034097a5017dd9281b1f795e80af9859627850eDavid Howells */ 5598feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic void add_nommu_region(struct vm_region *region) 5603034097a5017dd9281b1f795e80af9859627850eDavid Howells{ 5618feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_region *pregion; 5628feae13110d60cc6287afabc2887366b0eb226c2David Howells struct rb_node **p, *parent; 5633034097a5017dd9281b1f795e80af9859627850eDavid Howells 5648feae13110d60cc6287afabc2887366b0eb226c2David Howells validate_nommu_regions(); 5658feae13110d60cc6287afabc2887366b0eb226c2David Howells 5668feae13110d60cc6287afabc2887366b0eb226c2David Howells parent = NULL; 5678feae13110d60cc6287afabc2887366b0eb226c2David Howells p = &nommu_region_tree.rb_node; 5688feae13110d60cc6287afabc2887366b0eb226c2David Howells while (*p) { 5698feae13110d60cc6287afabc2887366b0eb226c2David Howells parent = *p; 5708feae13110d60cc6287afabc2887366b0eb226c2David Howells pregion = rb_entry(parent, struct vm_region, vm_rb); 5718feae13110d60cc6287afabc2887366b0eb226c2David Howells if (region->vm_start < pregion->vm_start) 5728feae13110d60cc6287afabc2887366b0eb226c2David Howells p = &(*p)->rb_left; 5738feae13110d60cc6287afabc2887366b0eb226c2David Howells else if (region->vm_start > pregion->vm_start) 5748feae13110d60cc6287afabc2887366b0eb226c2David Howells p = &(*p)->rb_right; 5758feae13110d60cc6287afabc2887366b0eb226c2David Howells else if (pregion == region) 5768feae13110d60cc6287afabc2887366b0eb226c2David Howells return; 5778feae13110d60cc6287afabc2887366b0eb226c2David Howells else 5788feae13110d60cc6287afabc2887366b0eb226c2David Howells BUG(); 5793034097a5017dd9281b1f795e80af9859627850eDavid Howells } 5803034097a5017dd9281b1f795e80af9859627850eDavid Howells 5818feae13110d60cc6287afabc2887366b0eb226c2David Howells rb_link_node(®ion->vm_rb, parent, p); 5828feae13110d60cc6287afabc2887366b0eb226c2David Howells rb_insert_color(®ion->vm_rb, &nommu_region_tree); 5833034097a5017dd9281b1f795e80af9859627850eDavid Howells 5848feae13110d60cc6287afabc2887366b0eb226c2David Howells validate_nommu_regions(); 5853034097a5017dd9281b1f795e80af9859627850eDavid Howells} 5863034097a5017dd9281b1f795e80af9859627850eDavid Howells 5873034097a5017dd9281b1f795e80af9859627850eDavid Howells/* 5888feae13110d60cc6287afabc2887366b0eb226c2David Howells * delete a region from the global tree 589930e652a21a08986b03d1f370f933057dc0db2dcDavid Howells */ 5908feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic void delete_nommu_region(struct vm_region *region) 591930e652a21a08986b03d1f370f933057dc0db2dcDavid Howells{ 5928feae13110d60cc6287afabc2887366b0eb226c2David Howells BUG_ON(!nommu_region_tree.rb_node); 593930e652a21a08986b03d1f370f933057dc0db2dcDavid Howells 5948feae13110d60cc6287afabc2887366b0eb226c2David Howells validate_nommu_regions(); 5958feae13110d60cc6287afabc2887366b0eb226c2David Howells rb_erase(®ion->vm_rb, &nommu_region_tree); 5968feae13110d60cc6287afabc2887366b0eb226c2David Howells validate_nommu_regions(); 59757c8f63e8e7a4a95d7fcc49e3953341fb4039899Greg Ungerer} 59857c8f63e8e7a4a95d7fcc49e3953341fb4039899Greg Ungerer 599930e652a21a08986b03d1f370f933057dc0db2dcDavid Howells/* 6008feae13110d60cc6287afabc2887366b0eb226c2David Howells * free a contiguous series of pages 6016fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells */ 6028feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic void free_page_series(unsigned long from, unsigned long to) 6036fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells{ 6048feae13110d60cc6287afabc2887366b0eb226c2David Howells for (; from < to; from += PAGE_SIZE) { 6058feae13110d60cc6287afabc2887366b0eb226c2David Howells struct page *page = virt_to_page(from); 6068feae13110d60cc6287afabc2887366b0eb226c2David Howells 6078feae13110d60cc6287afabc2887366b0eb226c2David Howells kdebug("- free %lx", from); 60833e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells atomic_long_dec(&mmap_pages_allocated); 6098feae13110d60cc6287afabc2887366b0eb226c2David Howells if (page_count(page) != 1) 61033e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells kdebug("free page %p: refcount not one: %d", 61133e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells page, page_count(page)); 6128feae13110d60cc6287afabc2887366b0eb226c2David Howells put_page(page); 6136fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells } 6146fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells} 6156fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells 6166fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells/* 6178feae13110d60cc6287afabc2887366b0eb226c2David Howells * release a reference to a region 61833e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells * - the caller must hold the region semaphore for writing, which this releases 619dd8632a12e500a684478fea0951f380478d56fedPaul Mundt * - the region may not have been added to the tree yet, in which case vm_top 6208feae13110d60cc6287afabc2887366b0eb226c2David Howells * will equal vm_start 6213034097a5017dd9281b1f795e80af9859627850eDavid Howells */ 6228feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic void __put_nommu_region(struct vm_region *region) 6238feae13110d60cc6287afabc2887366b0eb226c2David Howells __releases(nommu_region_sem) 6241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 6251e2ae599d37e60958c03ca5e46b1f657619a30cdDavid Howells kenter("%p{%d}", region, region->vm_usage); 6261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6278feae13110d60cc6287afabc2887366b0eb226c2David Howells BUG_ON(!nommu_region_tree.rb_node); 6281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6291e2ae599d37e60958c03ca5e46b1f657619a30cdDavid Howells if (--region->vm_usage == 0) { 630dd8632a12e500a684478fea0951f380478d56fedPaul Mundt if (region->vm_top > region->vm_start) 6318feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_nommu_region(region); 6328feae13110d60cc6287afabc2887366b0eb226c2David Howells up_write(&nommu_region_sem); 6338feae13110d60cc6287afabc2887366b0eb226c2David Howells 6348feae13110d60cc6287afabc2887366b0eb226c2David Howells if (region->vm_file) 6358feae13110d60cc6287afabc2887366b0eb226c2David Howells fput(region->vm_file); 6368feae13110d60cc6287afabc2887366b0eb226c2David Howells 6378feae13110d60cc6287afabc2887366b0eb226c2David Howells /* IO memory and memory shared directly out of the pagecache 6388feae13110d60cc6287afabc2887366b0eb226c2David Howells * from ramfs/tmpfs mustn't be released here */ 6398feae13110d60cc6287afabc2887366b0eb226c2David Howells if (region->vm_flags & VM_MAPPED_COPY) { 6408feae13110d60cc6287afabc2887366b0eb226c2David Howells kdebug("free series"); 641dd8632a12e500a684478fea0951f380478d56fedPaul Mundt free_page_series(region->vm_start, region->vm_top); 6428feae13110d60cc6287afabc2887366b0eb226c2David Howells } 6438feae13110d60cc6287afabc2887366b0eb226c2David Howells kmem_cache_free(vm_region_jar, region); 6448feae13110d60cc6287afabc2887366b0eb226c2David Howells } else { 6458feae13110d60cc6287afabc2887366b0eb226c2David Howells up_write(&nommu_region_sem); 6461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6478feae13110d60cc6287afabc2887366b0eb226c2David Howells} 6481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6498feae13110d60cc6287afabc2887366b0eb226c2David Howells/* 6508feae13110d60cc6287afabc2887366b0eb226c2David Howells * release a reference to a region 6518feae13110d60cc6287afabc2887366b0eb226c2David Howells */ 6528feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic void put_nommu_region(struct vm_region *region) 6538feae13110d60cc6287afabc2887366b0eb226c2David Howells{ 6548feae13110d60cc6287afabc2887366b0eb226c2David Howells down_write(&nommu_region_sem); 6558feae13110d60cc6287afabc2887366b0eb226c2David Howells __put_nommu_region(region); 6561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 6571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6583034097a5017dd9281b1f795e80af9859627850eDavid Howells/* 659eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt * update protection on a vma 660eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt */ 661eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidtstatic void protect_vma(struct vm_area_struct *vma, unsigned long flags) 662eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt{ 663eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt#ifdef CONFIG_MPU 664eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt struct mm_struct *mm = vma->vm_mm; 665eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt long start = vma->vm_start & PAGE_MASK; 666eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt while (start < vma->vm_end) { 667eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt protect_page(mm, start, flags); 668eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt start += PAGE_SIZE; 669eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt } 670eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt update_protections(mm); 671eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt#endif 672eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt} 673eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt 674eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt/* 6758feae13110d60cc6287afabc2887366b0eb226c2David Howells * add a VMA into a process's mm_struct in the appropriate place in the list 6768feae13110d60cc6287afabc2887366b0eb226c2David Howells * and tree and add to the address space's page tree also if not an anonymous 6778feae13110d60cc6287afabc2887366b0eb226c2David Howells * page 6788feae13110d60cc6287afabc2887366b0eb226c2David Howells * - should be called with mm->mmap_sem held writelocked 6793034097a5017dd9281b1f795e80af9859627850eDavid Howells */ 6808feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) 6811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 6826038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim struct vm_area_struct *pvma, *prev; 6831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct address_space *mapping; 6846038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim struct rb_node **p, *parent, *rb_prev; 6858feae13110d60cc6287afabc2887366b0eb226c2David Howells 6868feae13110d60cc6287afabc2887366b0eb226c2David Howells kenter(",%p", vma); 6878feae13110d60cc6287afabc2887366b0eb226c2David Howells 6888feae13110d60cc6287afabc2887366b0eb226c2David Howells BUG_ON(!vma->vm_region); 6898feae13110d60cc6287afabc2887366b0eb226c2David Howells 6908feae13110d60cc6287afabc2887366b0eb226c2David Howells mm->map_count++; 6918feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_mm = mm; 6921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 693eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt protect_vma(vma, vma->vm_flags); 694eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt 6951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* add the VMA to the mapping */ 6961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (vma->vm_file) { 6971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mapping = vma->vm_file->f_mapping; 6981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 699918e556ec214ed2f584e4cac56d7b29e4bb6bf27David Howells mutex_lock(&mapping->i_mmap_mutex); 7001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_dcache_mmap_lock(mapping); 7011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vma_prio_tree_insert(vma, &mapping->i_mmap); 7021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_dcache_mmap_unlock(mapping); 703918e556ec214ed2f584e4cac56d7b29e4bb6bf27David Howells mutex_unlock(&mapping->i_mmap_mutex); 7041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7068feae13110d60cc6287afabc2887366b0eb226c2David Howells /* add the VMA to the tree */ 7076038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim parent = rb_prev = NULL; 7088feae13110d60cc6287afabc2887366b0eb226c2David Howells p = &mm->mm_rb.rb_node; 7091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (*p) { 7101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds parent = *p; 7111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pvma = rb_entry(parent, struct vm_area_struct, vm_rb); 7121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7138feae13110d60cc6287afabc2887366b0eb226c2David Howells /* sort by: start addr, end addr, VMA struct addr in that order 7148feae13110d60cc6287afabc2887366b0eb226c2David Howells * (the latter is necessary as we may get identical VMAs) */ 7158feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_start < pvma->vm_start) 7161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds p = &(*p)->rb_left; 7176038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim else if (vma->vm_start > pvma->vm_start) { 7186038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim rb_prev = parent; 7191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds p = &(*p)->rb_right; 7206038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim } else if (vma->vm_end < pvma->vm_end) 7218feae13110d60cc6287afabc2887366b0eb226c2David Howells p = &(*p)->rb_left; 7226038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim else if (vma->vm_end > pvma->vm_end) { 7236038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim rb_prev = parent; 7248feae13110d60cc6287afabc2887366b0eb226c2David Howells p = &(*p)->rb_right; 7256038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim } else if (vma < pvma) 7268feae13110d60cc6287afabc2887366b0eb226c2David Howells p = &(*p)->rb_left; 7276038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim else if (vma > pvma) { 7286038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim rb_prev = parent; 7298feae13110d60cc6287afabc2887366b0eb226c2David Howells p = &(*p)->rb_right; 7306038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim } else 7318feae13110d60cc6287afabc2887366b0eb226c2David Howells BUG(); 7321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds rb_link_node(&vma->vm_rb, parent, p); 7358feae13110d60cc6287afabc2887366b0eb226c2David Howells rb_insert_color(&vma->vm_rb, &mm->mm_rb); 7368feae13110d60cc6287afabc2887366b0eb226c2David Howells 7378feae13110d60cc6287afabc2887366b0eb226c2David Howells /* add VMA to the VMA list also */ 7386038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim prev = NULL; 7396038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim if (rb_prev) 7406038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); 7418feae13110d60cc6287afabc2887366b0eb226c2David Howells 7426038def0d11b322019d0dbb43f2a611247dfbdb6Namhyung Kim __vma_link_list(mm, vma, prev, parent); 7431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 7441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7453034097a5017dd9281b1f795e80af9859627850eDavid Howells/* 7468feae13110d60cc6287afabc2887366b0eb226c2David Howells * delete a VMA from its owning mm_struct and address space 7473034097a5017dd9281b1f795e80af9859627850eDavid Howells */ 7488feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic void delete_vma_from_mm(struct vm_area_struct *vma) 7491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 7501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct address_space *mapping; 7518feae13110d60cc6287afabc2887366b0eb226c2David Howells struct mm_struct *mm = vma->vm_mm; 7528feae13110d60cc6287afabc2887366b0eb226c2David Howells 7538feae13110d60cc6287afabc2887366b0eb226c2David Howells kenter("%p", vma); 7548feae13110d60cc6287afabc2887366b0eb226c2David Howells 755eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt protect_vma(vma, 0); 756eb8cdec4a984fde123a91250dcc9e0bddf5eafdcBernd Schmidt 7578feae13110d60cc6287afabc2887366b0eb226c2David Howells mm->map_count--; 7588feae13110d60cc6287afabc2887366b0eb226c2David Howells if (mm->mmap_cache == vma) 7598feae13110d60cc6287afabc2887366b0eb226c2David Howells mm->mmap_cache = NULL; 7601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* remove the VMA from the mapping */ 7621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (vma->vm_file) { 7631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mapping = vma->vm_file->f_mapping; 7641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 765918e556ec214ed2f584e4cac56d7b29e4bb6bf27David Howells mutex_lock(&mapping->i_mmap_mutex); 7661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_dcache_mmap_lock(mapping); 7671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vma_prio_tree_remove(vma, &mapping->i_mmap); 7681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_dcache_mmap_unlock(mapping); 769918e556ec214ed2f584e4cac56d7b29e4bb6bf27David Howells mutex_unlock(&mapping->i_mmap_mutex); 7701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7728feae13110d60cc6287afabc2887366b0eb226c2David Howells /* remove from the MM's tree and list */ 7738feae13110d60cc6287afabc2887366b0eb226c2David Howells rb_erase(&vma->vm_rb, &mm->mm_rb); 774b951bf2c4693bfc9744e11293be859209f65f579Namhyung Kim 775b951bf2c4693bfc9744e11293be859209f65f579Namhyung Kim if (vma->vm_prev) 776b951bf2c4693bfc9744e11293be859209f65f579Namhyung Kim vma->vm_prev->vm_next = vma->vm_next; 777b951bf2c4693bfc9744e11293be859209f65f579Namhyung Kim else 778b951bf2c4693bfc9744e11293be859209f65f579Namhyung Kim mm->mmap = vma->vm_next; 779b951bf2c4693bfc9744e11293be859209f65f579Namhyung Kim 780b951bf2c4693bfc9744e11293be859209f65f579Namhyung Kim if (vma->vm_next) 781b951bf2c4693bfc9744e11293be859209f65f579Namhyung Kim vma->vm_next->vm_prev = vma->vm_prev; 7828feae13110d60cc6287afabc2887366b0eb226c2David Howells} 7838feae13110d60cc6287afabc2887366b0eb226c2David Howells 7848feae13110d60cc6287afabc2887366b0eb226c2David Howells/* 7858feae13110d60cc6287afabc2887366b0eb226c2David Howells * destroy a VMA record 7868feae13110d60cc6287afabc2887366b0eb226c2David Howells */ 7878feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) 7888feae13110d60cc6287afabc2887366b0eb226c2David Howells{ 7898feae13110d60cc6287afabc2887366b0eb226c2David Howells kenter("%p", vma); 7908feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_ops && vma->vm_ops->close) 7918feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_ops->close(vma); 7928feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_file) { 7938feae13110d60cc6287afabc2887366b0eb226c2David Howells fput(vma->vm_file); 7948feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_flags & VM_EXECUTABLE) 7958feae13110d60cc6287afabc2887366b0eb226c2David Howells removed_exe_file_vma(mm); 7968feae13110d60cc6287afabc2887366b0eb226c2David Howells } 7978feae13110d60cc6287afabc2887366b0eb226c2David Howells put_nommu_region(vma->vm_region); 7988feae13110d60cc6287afabc2887366b0eb226c2David Howells kmem_cache_free(vm_area_cachep, vma); 7998feae13110d60cc6287afabc2887366b0eb226c2David Howells} 8008feae13110d60cc6287afabc2887366b0eb226c2David Howells 8018feae13110d60cc6287afabc2887366b0eb226c2David Howells/* 8028feae13110d60cc6287afabc2887366b0eb226c2David Howells * look up the first VMA in which addr resides, NULL if none 8038feae13110d60cc6287afabc2887366b0eb226c2David Howells * - should be called with mm->mmap_sem at least held readlocked 8048feae13110d60cc6287afabc2887366b0eb226c2David Howells */ 8058feae13110d60cc6287afabc2887366b0eb226c2David Howellsstruct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 8068feae13110d60cc6287afabc2887366b0eb226c2David Howells{ 8078feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_area_struct *vma; 8088feae13110d60cc6287afabc2887366b0eb226c2David Howells 8098feae13110d60cc6287afabc2887366b0eb226c2David Howells /* check the cache first */ 8108feae13110d60cc6287afabc2887366b0eb226c2David Howells vma = mm->mmap_cache; 8118feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma && vma->vm_start <= addr && vma->vm_end > addr) 8128feae13110d60cc6287afabc2887366b0eb226c2David Howells return vma; 8138feae13110d60cc6287afabc2887366b0eb226c2David Howells 814e922c4c5360980bfeb862b3ec307d36bb344dcaeNamhyung Kim /* trawl the list (there may be multiple mappings in which addr 8158feae13110d60cc6287afabc2887366b0eb226c2David Howells * resides) */ 816e922c4c5360980bfeb862b3ec307d36bb344dcaeNamhyung Kim for (vma = mm->mmap; vma; vma = vma->vm_next) { 8178feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_start > addr) 8188feae13110d60cc6287afabc2887366b0eb226c2David Howells return NULL; 8198feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_end > addr) { 8208feae13110d60cc6287afabc2887366b0eb226c2David Howells mm->mmap_cache = vma; 8218feae13110d60cc6287afabc2887366b0eb226c2David Howells return vma; 8228feae13110d60cc6287afabc2887366b0eb226c2David Howells } 8238feae13110d60cc6287afabc2887366b0eb226c2David Howells } 8248feae13110d60cc6287afabc2887366b0eb226c2David Howells 8258feae13110d60cc6287afabc2887366b0eb226c2David Howells return NULL; 8268feae13110d60cc6287afabc2887366b0eb226c2David Howells} 8278feae13110d60cc6287afabc2887366b0eb226c2David HowellsEXPORT_SYMBOL(find_vma); 8288feae13110d60cc6287afabc2887366b0eb226c2David Howells 8298feae13110d60cc6287afabc2887366b0eb226c2David Howells/* 8308feae13110d60cc6287afabc2887366b0eb226c2David Howells * find a VMA 8318feae13110d60cc6287afabc2887366b0eb226c2David Howells * - we don't extend stack VMAs under NOMMU conditions 8328feae13110d60cc6287afabc2887366b0eb226c2David Howells */ 8338feae13110d60cc6287afabc2887366b0eb226c2David Howellsstruct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) 8348feae13110d60cc6287afabc2887366b0eb226c2David Howells{ 8357561e8ca0dfaf6fca3feef982830de3b65300e5bDavid Howells return find_vma(mm, addr); 8368feae13110d60cc6287afabc2887366b0eb226c2David Howells} 8378feae13110d60cc6287afabc2887366b0eb226c2David Howells 8388feae13110d60cc6287afabc2887366b0eb226c2David Howells/* 8398feae13110d60cc6287afabc2887366b0eb226c2David Howells * expand a stack to a given address 8408feae13110d60cc6287afabc2887366b0eb226c2David Howells * - not supported under NOMMU conditions 8418feae13110d60cc6287afabc2887366b0eb226c2David Howells */ 8428feae13110d60cc6287afabc2887366b0eb226c2David Howellsint expand_stack(struct vm_area_struct *vma, unsigned long address) 8438feae13110d60cc6287afabc2887366b0eb226c2David Howells{ 8448feae13110d60cc6287afabc2887366b0eb226c2David Howells return -ENOMEM; 8458feae13110d60cc6287afabc2887366b0eb226c2David Howells} 8468feae13110d60cc6287afabc2887366b0eb226c2David Howells 8478feae13110d60cc6287afabc2887366b0eb226c2David Howells/* 8488feae13110d60cc6287afabc2887366b0eb226c2David Howells * look up the first VMA exactly that exactly matches addr 8498feae13110d60cc6287afabc2887366b0eb226c2David Howells * - should be called with mm->mmap_sem at least held readlocked 8508feae13110d60cc6287afabc2887366b0eb226c2David Howells */ 8518feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic struct vm_area_struct *find_vma_exact(struct mm_struct *mm, 8528feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long addr, 8538feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long len) 8548feae13110d60cc6287afabc2887366b0eb226c2David Howells{ 8558feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_area_struct *vma; 8568feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long end = addr + len; 8578feae13110d60cc6287afabc2887366b0eb226c2David Howells 8588feae13110d60cc6287afabc2887366b0eb226c2David Howells /* check the cache first */ 8598feae13110d60cc6287afabc2887366b0eb226c2David Howells vma = mm->mmap_cache; 8608feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma && vma->vm_start == addr && vma->vm_end == end) 8618feae13110d60cc6287afabc2887366b0eb226c2David Howells return vma; 8628feae13110d60cc6287afabc2887366b0eb226c2David Howells 863e922c4c5360980bfeb862b3ec307d36bb344dcaeNamhyung Kim /* trawl the list (there may be multiple mappings in which addr 8648feae13110d60cc6287afabc2887366b0eb226c2David Howells * resides) */ 865e922c4c5360980bfeb862b3ec307d36bb344dcaeNamhyung Kim for (vma = mm->mmap; vma; vma = vma->vm_next) { 8668feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_start < addr) 8678feae13110d60cc6287afabc2887366b0eb226c2David Howells continue; 8688feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_start > addr) 8698feae13110d60cc6287afabc2887366b0eb226c2David Howells return NULL; 8708feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_end == end) { 8718feae13110d60cc6287afabc2887366b0eb226c2David Howells mm->mmap_cache = vma; 8728feae13110d60cc6287afabc2887366b0eb226c2David Howells return vma; 8738feae13110d60cc6287afabc2887366b0eb226c2David Howells } 8748feae13110d60cc6287afabc2887366b0eb226c2David Howells } 8758feae13110d60cc6287afabc2887366b0eb226c2David Howells 8768feae13110d60cc6287afabc2887366b0eb226c2David Howells return NULL; 8771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 8781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 8801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * determine whether a mapping should be permitted and, if so, what sort of 8811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mapping we're capable of supporting 8821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int validate_mmap_request(struct file *file, 8841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long addr, 8851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long len, 8861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long prot, 8871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags, 8881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long pgoff, 8891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long *_capabilities) 8901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 8918feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long capabilities, rlen; 8921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long reqprot = prot; 8931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret; 8941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* do the simple checks first */ 89606aab5a3084e1d825384fa353e6df4c7949c8683David Howells if (flags & MAP_FIXED) { 8971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds printk(KERN_DEBUG 8981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "%d: Can't do fixed-address/overlay mmap of RAM\n", 8991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds current->pid); 9001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINVAL; 9011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 9021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((flags & MAP_TYPE) != MAP_PRIVATE && 9041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (flags & MAP_TYPE) != MAP_SHARED) 9051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINVAL; 9061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 907f81cff0d4067e41fd7383d9c013cc82da7c169d2Mike Frysinger if (!len) 9081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINVAL; 9091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 910f81cff0d4067e41fd7383d9c013cc82da7c169d2Mike Frysinger /* Careful about overflows.. */ 9118feae13110d60cc6287afabc2887366b0eb226c2David Howells rlen = PAGE_ALIGN(len); 9128feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!rlen || rlen > TASK_SIZE) 913f81cff0d4067e41fd7383d9c013cc82da7c169d2Mike Frysinger return -ENOMEM; 914f81cff0d4067e41fd7383d9c013cc82da7c169d2Mike Frysinger 9151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* offset overflow? */ 9168feae13110d60cc6287afabc2887366b0eb226c2David Howells if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff) 917f81cff0d4067e41fd7383d9c013cc82da7c169d2Mike Frysinger return -EOVERFLOW; 9181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (file) { 9201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* validate file mapping requests */ 9211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct address_space *mapping; 9221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* files must support mmap */ 9241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!file->f_op || !file->f_op->mmap) 9251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENODEV; 9261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* work out if what we've got could possibly be shared 9281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - we support chardevs that provide their own "memory" 9291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - we support files/blockdevs that are memory backed 9301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 9311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mapping = file->f_mapping; 9321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!mapping) 933e9536ae7205d255bc94616b72910fc6e16c861feJosef Sipek mapping = file->f_path.dentry->d_inode->i_mapping; 9341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities = 0; 9361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (mapping && mapping->backing_dev_info) 9371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities = mapping->backing_dev_info->capabilities; 9381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!capabilities) { 9401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* no explicit capabilities set, so assume some 9411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * defaults */ 942e9536ae7205d255bc94616b72910fc6e16c861feJosef Sipek switch (file->f_path.dentry->d_inode->i_mode & S_IFMT) { 9431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case S_IFREG: 9441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case S_IFBLK: 9451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities = BDI_CAP_MAP_COPY; 9461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 9471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case S_IFCHR: 9491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities = 9501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BDI_CAP_MAP_DIRECT | 9511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BDI_CAP_READ_MAP | 9521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds BDI_CAP_WRITE_MAP; 9531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 9541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds default: 9561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINVAL; 9571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 9581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 9591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* eliminate any capabilities that we can't support on this 9611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * device */ 9621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!file->f_op->get_unmapped_area) 9631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities &= ~BDI_CAP_MAP_DIRECT; 9641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!file->f_op->read) 9651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities &= ~BDI_CAP_MAP_COPY; 9661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 96728d7a6ae92c099d81cbea08c20be0d2cf7ccd7caGraff Yang /* The file shall have been opened with read permission. */ 96828d7a6ae92c099d81cbea08c20be0d2cf7ccd7caGraff Yang if (!(file->f_mode & FMODE_READ)) 96928d7a6ae92c099d81cbea08c20be0d2cf7ccd7caGraff Yang return -EACCES; 97028d7a6ae92c099d81cbea08c20be0d2cf7ccd7caGraff Yang 9711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (flags & MAP_SHARED) { 9721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* do checks for writing, appending and locking */ 9731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((prot & PROT_WRITE) && 9741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds !(file->f_mode & FMODE_WRITE)) 9751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EACCES; 9761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 977e9536ae7205d255bc94616b72910fc6e16c861feJosef Sipek if (IS_APPEND(file->f_path.dentry->d_inode) && 9781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (file->f_mode & FMODE_WRITE)) 9791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EACCES; 9801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 981e9536ae7205d255bc94616b72910fc6e16c861feJosef Sipek if (locks_verify_locked(file->f_path.dentry->d_inode)) 9821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EAGAIN; 9831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(capabilities & BDI_CAP_MAP_DIRECT)) 9851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENODEV; 9861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* we mustn't privatise shared mappings */ 9881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities &= ~BDI_CAP_MAP_COPY; 9891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 9901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else { 9911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* we're going to read the file into private memory we 9921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allocate */ 9931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(capabilities & BDI_CAP_MAP_COPY)) 9941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENODEV; 9951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* we don't permit a private writable mapping to be 9971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * shared with the backing device */ 9981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (prot & PROT_WRITE) 9991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities &= ~BDI_CAP_MAP_DIRECT; 10001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 10011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10023c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt if (capabilities & BDI_CAP_MAP_DIRECT) { 10033c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt if (((prot & PROT_READ) && !(capabilities & BDI_CAP_READ_MAP)) || 10043c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt ((prot & PROT_WRITE) && !(capabilities & BDI_CAP_WRITE_MAP)) || 10053c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt ((prot & PROT_EXEC) && !(capabilities & BDI_CAP_EXEC_MAP)) 10063c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt ) { 10073c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt capabilities &= ~BDI_CAP_MAP_DIRECT; 10083c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt if (flags & MAP_SHARED) { 10093c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt printk(KERN_WARNING 10103c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt "MAP_SHARED not completely supported on !MMU\n"); 10113c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt return -EINVAL; 10123c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt } 10133c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt } 10143c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt } 10153c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt 10161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* handle executable mappings and implied executable 10171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mappings */ 1018e9536ae7205d255bc94616b72910fc6e16c861feJosef Sipek if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) { 10191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (prot & PROT_EXEC) 10201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EPERM; 10211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 10221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) { 10231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* handle implication of PROT_EXEC by PROT_READ */ 10241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (current->personality & READ_IMPLIES_EXEC) { 10251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (capabilities & BDI_CAP_EXEC_MAP) 10261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds prot |= PROT_EXEC; 10271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 10281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 10291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else if ((prot & PROT_READ) && 10301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (prot & PROT_EXEC) && 10311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds !(capabilities & BDI_CAP_EXEC_MAP) 10321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ) { 10331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* backing file is not executable, try to copy */ 10341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities &= ~BDI_CAP_MAP_DIRECT; 10351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 10361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 10371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else { 10381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* anonymous mappings are always memory backed and can be 10391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * privately mapped 10401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 10411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities = BDI_CAP_MAP_COPY; 10421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* handle PROT_EXEC implication by PROT_READ */ 10441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((prot & PROT_READ) && 10451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (current->personality & READ_IMPLIES_EXEC)) 10461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds prot |= PROT_EXEC; 10471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 10481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* allow the security API to have its say */ 1050ed0321895182ffb6ecf210e066d87911b270d587Eric Paris ret = security_file_mmap(file, reqprot, prot, flags, addr, 0); 10511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret < 0) 10521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 10531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* looks okay */ 10551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *_capabilities = capabilities; 10561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 10571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 10581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 10601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * we've determined that we can make the mapping, now translate what we 10611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * now know into VMA flags 10621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 10631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic unsigned long determine_vm_flags(struct file *file, 10641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long prot, 10651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags, 10661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long capabilities) 10671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 10681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long vm_flags; 10691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags); 10711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* vm_flags |= mm->def_flags; */ 10721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(capabilities & BDI_CAP_MAP_DIRECT)) { 10741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* attempt to share read-only copies of mapped file chunks */ 10753c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 10761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (file && !(prot & PROT_WRITE)) 10771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_flags |= VM_MAYSHARE; 10783c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt } else { 10791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* overlay a shareable mapping on the backing device or inode 10801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * if possible - used for chardevs, ramfs/tmpfs/shmfs and 10811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * romfs/cramfs */ 10823c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt vm_flags |= VM_MAYSHARE | (capabilities & BDI_CAP_VMFLAGS); 10831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (flags & MAP_SHARED) 10843c7b204547bc3d342a4e31196fe14803581d279fBernd Schmidt vm_flags |= VM_SHARED; 10851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 10861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* refuse to let anyone share private mappings with this process if 10881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * it's being traced - otherwise breakpoints set in it may interfere 10891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * with another untraced process 10901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1091a288eecce5253cc1565d400a52b9b476a157e040Tejun Heo if ((flags & MAP_PRIVATE) && current->ptrace) 10921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_flags &= ~VM_MAYSHARE; 10931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return vm_flags; 10951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 10961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 10988feae13110d60cc6287afabc2887366b0eb226c2David Howells * set up a shared mapping on a file (the driver or filesystem provides and 10998feae13110d60cc6287afabc2887366b0eb226c2David Howells * pins the storage) 11001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 11018feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic int do_mmap_shared_file(struct vm_area_struct *vma) 11021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 11031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret; 11041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = vma->vm_file->f_op->mmap(vma->vm_file, vma); 1106dd8632a12e500a684478fea0951f380478d56fedPaul Mundt if (ret == 0) { 1107dd8632a12e500a684478fea0951f380478d56fedPaul Mundt vma->vm_region->vm_top = vma->vm_region->vm_end; 1108645d83c5db970a1c57225e155113b4aa2451e920David Howells return 0; 1109dd8632a12e500a684478fea0951f380478d56fedPaul Mundt } 11101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret != -ENOSYS) 11111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 11121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11133fa30460ea502133a18a07b14452cd660906f16fDavid Howells /* getting -ENOSYS indicates that direct mmap isn't possible (as 11143fa30460ea502133a18a07b14452cd660906f16fDavid Howells * opposed to tried but failed) so we can only give a suitable error as 11153fa30460ea502133a18a07b14452cd660906f16fDavid Howells * it's not possible to make a private copy if MAP_SHARED was given */ 11161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENODEV; 11171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 11181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 11201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * set up a private mapping or an anonymous shared mapping 11211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 11228feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic int do_mmap_private(struct vm_area_struct *vma, 11238feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_region *region, 1124645d83c5db970a1c57225e155113b4aa2451e920David Howells unsigned long len, 1125645d83c5db970a1c57225e155113b4aa2451e920David Howells unsigned long capabilities) 11261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 11278feae13110d60cc6287afabc2887366b0eb226c2David Howells struct page *pages; 1128f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu unsigned long total, point, n; 11291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *base; 11308feae13110d60cc6287afabc2887366b0eb226c2David Howells int ret, order; 11311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* invoke the file's mapping function so that it can keep track of 11331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * shared mappings on devices or memory 11341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - VM_MAYSHARE will be set if it may attempt to share 11351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1136645d83c5db970a1c57225e155113b4aa2451e920David Howells if (capabilities & BDI_CAP_MAP_DIRECT) { 11371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = vma->vm_file->f_op->mmap(vma->vm_file, vma); 1138dd8632a12e500a684478fea0951f380478d56fedPaul Mundt if (ret == 0) { 11391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* shouldn't return success if we're not sharing */ 1140dd8632a12e500a684478fea0951f380478d56fedPaul Mundt BUG_ON(!(vma->vm_flags & VM_MAYSHARE)); 1141dd8632a12e500a684478fea0951f380478d56fedPaul Mundt vma->vm_region->vm_top = vma->vm_region->vm_end; 1142645d83c5db970a1c57225e155113b4aa2451e920David Howells return 0; 11431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1144dd8632a12e500a684478fea0951f380478d56fedPaul Mundt if (ret != -ENOSYS) 1145dd8632a12e500a684478fea0951f380478d56fedPaul Mundt return ret; 11461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* getting an ENOSYS error indicates that direct mmap isn't 11481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * possible (as opposed to tried but failed) so we'll try to 11491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * make a private copy of the data and map that instead */ 11501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 11511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11528feae13110d60cc6287afabc2887366b0eb226c2David Howells 11531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* allocate some memory to hold the mapping 11541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - note that this may not return a page-aligned address if the object 11551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * we're allocating is smaller than a page 11561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1157f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu order = get_order(len); 11588feae13110d60cc6287afabc2887366b0eb226c2David Howells kdebug("alloc order %d for %lx", order, len); 11598feae13110d60cc6287afabc2887366b0eb226c2David Howells 11608feae13110d60cc6287afabc2887366b0eb226c2David Howells pages = alloc_pages(GFP_KERNEL, order); 11618feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!pages) 11621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto enomem; 11631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11648feae13110d60cc6287afabc2887366b0eb226c2David Howells total = 1 << order; 116533e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells atomic_long_add(total, &mmap_pages_allocated); 11668feae13110d60cc6287afabc2887366b0eb226c2David Howells 1167f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu point = len >> PAGE_SHIFT; 1168dd8632a12e500a684478fea0951f380478d56fedPaul Mundt 1169dd8632a12e500a684478fea0951f380478d56fedPaul Mundt /* we allocated a power-of-2 sized page set, so we may want to trim off 1170dd8632a12e500a684478fea0951f380478d56fedPaul Mundt * the excess */ 1171dd8632a12e500a684478fea0951f380478d56fedPaul Mundt if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages) { 1172dd8632a12e500a684478fea0951f380478d56fedPaul Mundt while (total > point) { 1173dd8632a12e500a684478fea0951f380478d56fedPaul Mundt order = ilog2(total - point); 1174dd8632a12e500a684478fea0951f380478d56fedPaul Mundt n = 1 << order; 1175dd8632a12e500a684478fea0951f380478d56fedPaul Mundt kdebug("shave %lu/%lu @%lu", n, total - point, total); 117633e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells atomic_long_sub(n, &mmap_pages_allocated); 1177dd8632a12e500a684478fea0951f380478d56fedPaul Mundt total -= n; 1178dd8632a12e500a684478fea0951f380478d56fedPaul Mundt set_page_refcounted(pages + total); 1179dd8632a12e500a684478fea0951f380478d56fedPaul Mundt __free_pages(pages + total, order); 1180dd8632a12e500a684478fea0951f380478d56fedPaul Mundt } 11818feae13110d60cc6287afabc2887366b0eb226c2David Howells } 11828feae13110d60cc6287afabc2887366b0eb226c2David Howells 11838feae13110d60cc6287afabc2887366b0eb226c2David Howells for (point = 1; point < total; point++) 11848feae13110d60cc6287afabc2887366b0eb226c2David Howells set_page_refcounted(&pages[point]); 11851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11868feae13110d60cc6287afabc2887366b0eb226c2David Howells base = page_address(pages); 11878feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY; 11888feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_start = (unsigned long) base; 1189f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu region->vm_end = region->vm_start + len; 1190dd8632a12e500a684478fea0951f380478d56fedPaul Mundt region->vm_top = region->vm_start + (total << PAGE_SHIFT); 11918feae13110d60cc6287afabc2887366b0eb226c2David Howells 11928feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_start = region->vm_start; 11938feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_end = region->vm_start + len; 11941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (vma->vm_file) { 11961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* read the contents of a file into the copy */ 11971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mm_segment_t old_fs; 11981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds loff_t fpos; 11991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fpos = vma->vm_pgoff; 12011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fpos <<= PAGE_SHIFT; 12021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds old_fs = get_fs(); 12041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_fs(KERNEL_DS); 1205f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos); 12061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_fs(old_fs); 12071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret < 0) 12091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto error_free; 12101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* clear the last little bit */ 1212f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu if (ret < len) 1213f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu memset(base + ret, 0, len - ret); 12141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 12161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 12181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldserror_free: 12207223bb4a829628bdf37d544ed4363d99bac1ade6Namhyung Kim free_page_series(region->vm_start, region->vm_top); 12218feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_start = vma->vm_start = 0; 12228feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_end = vma->vm_end = 0; 1223dd8632a12e500a684478fea0951f380478d56fedPaul Mundt region->vm_top = 0; 12241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 12251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsenomem: 122705ae6fa31874eda2484da13c5dc4ddee8a47a0a4Greg Ungerer printk("Allocation of length %lu from process %d (%s) failed\n", 122805ae6fa31874eda2484da13c5dc4ddee8a47a0a4Greg Ungerer len, current->pid, current->comm); 12297bf02ea22c6cdd09e2d3f1d3c3fe366b834ae9afDavid Rientjes show_free_areas(0); 12301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 12311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 12321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 12341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * handle mapping creation for uClinux 12351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 12361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long do_mmap_pgoff(struct file *file, 12371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long addr, 12381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long len, 12391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long prot, 12401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags, 12411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long pgoff) 12421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 12438feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_area_struct *vma; 12448feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_region *region; 12451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct rb_node *rb; 12468feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long capabilities, vm_flags, result; 12471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret; 12481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12498feae13110d60cc6287afabc2887366b0eb226c2David Howells kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff); 12508feae13110d60cc6287afabc2887366b0eb226c2David Howells 12511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* decide whether we should attempt the mapping, and if so what sort of 12521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mapping */ 12531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = validate_mmap_request(file, addr, len, prot, flags, pgoff, 12541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds &capabilities); 12558feae13110d60cc6287afabc2887366b0eb226c2David Howells if (ret < 0) { 12568feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = %d [val]", ret); 12571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 12588feae13110d60cc6287afabc2887366b0eb226c2David Howells } 12591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 126006aab5a3084e1d825384fa353e6df4c7949c8683David Howells /* we ignore the address hint */ 126106aab5a3084e1d825384fa353e6df4c7949c8683David Howells addr = 0; 1262f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu len = PAGE_ALIGN(len); 126306aab5a3084e1d825384fa353e6df4c7949c8683David Howells 12641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* we've determined that we can make the mapping, now translate what we 12651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * now know into VMA flags */ 12661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_flags = determine_vm_flags(file, prot, flags, capabilities); 12671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12688feae13110d60cc6287afabc2887366b0eb226c2David Howells /* we're going to need to record the mapping */ 12698feae13110d60cc6287afabc2887366b0eb226c2David Howells region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); 12708feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!region) 12718feae13110d60cc6287afabc2887366b0eb226c2David Howells goto error_getting_region; 12728feae13110d60cc6287afabc2887366b0eb226c2David Howells 12738feae13110d60cc6287afabc2887366b0eb226c2David Howells vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL); 12748feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!vma) 12758feae13110d60cc6287afabc2887366b0eb226c2David Howells goto error_getting_vma; 12761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12771e2ae599d37e60958c03ca5e46b1f657619a30cdDavid Howells region->vm_usage = 1; 12788feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_flags = vm_flags; 12798feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_pgoff = pgoff; 12808feae13110d60cc6287afabc2887366b0eb226c2David Howells 12815beb49305251e5669852ed541e8e2f2f7696c53eRik van Riel INIT_LIST_HEAD(&vma->anon_vma_chain); 12828feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_flags = vm_flags; 12838feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_pgoff = pgoff; 12841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12858feae13110d60cc6287afabc2887366b0eb226c2David Howells if (file) { 12868feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_file = file; 12878feae13110d60cc6287afabc2887366b0eb226c2David Howells get_file(file); 12888feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_file = file; 12898feae13110d60cc6287afabc2887366b0eb226c2David Howells get_file(file); 12908feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vm_flags & VM_EXECUTABLE) { 12918feae13110d60cc6287afabc2887366b0eb226c2David Howells added_exe_file_vma(current->mm); 12928feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_mm = current->mm; 12938feae13110d60cc6287afabc2887366b0eb226c2David Howells } 12948feae13110d60cc6287afabc2887366b0eb226c2David Howells } 12958feae13110d60cc6287afabc2887366b0eb226c2David Howells 12968feae13110d60cc6287afabc2887366b0eb226c2David Howells down_write(&nommu_region_sem); 12978feae13110d60cc6287afabc2887366b0eb226c2David Howells 12988feae13110d60cc6287afabc2887366b0eb226c2David Howells /* if we want to share, we need to check for regions created by other 12991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mmap() calls that overlap with our proposed mapping 13008feae13110d60cc6287afabc2887366b0eb226c2David Howells * - we can only share with a superset match on most regular files 13011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - shared mappings on character devices and memory backed files are 13021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * permitted to overlap inexactly as far as we are concerned for in 13031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * these cases, sharing is handled in the driver or filesystem rather 13041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * than here 13051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 13061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (vm_flags & VM_MAYSHARE) { 13078feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_region *pregion; 13088feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long pglen, rpglen, pgend, rpgend, start; 13091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13108feae13110d60cc6287afabc2887366b0eb226c2David Howells pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; 13118feae13110d60cc6287afabc2887366b0eb226c2David Howells pgend = pgoff + pglen; 1312165b239270be610a1e3999cb0d2e4e2c1f2a8fd4David Howells 13138feae13110d60cc6287afabc2887366b0eb226c2David Howells for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) { 13148feae13110d60cc6287afabc2887366b0eb226c2David Howells pregion = rb_entry(rb, struct vm_region, vm_rb); 13151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13168feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!(pregion->vm_flags & VM_MAYSHARE)) 13171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 13181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* search for overlapping mappings on the same file */ 13208feae13110d60cc6287afabc2887366b0eb226c2David Howells if (pregion->vm_file->f_path.dentry->d_inode != 13218feae13110d60cc6287afabc2887366b0eb226c2David Howells file->f_path.dentry->d_inode) 13221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 13231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13248feae13110d60cc6287afabc2887366b0eb226c2David Howells if (pregion->vm_pgoff >= pgend) 13251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 13261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13278feae13110d60cc6287afabc2887366b0eb226c2David Howells rpglen = pregion->vm_end - pregion->vm_start; 13288feae13110d60cc6287afabc2887366b0eb226c2David Howells rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT; 13298feae13110d60cc6287afabc2887366b0eb226c2David Howells rpgend = pregion->vm_pgoff + rpglen; 13308feae13110d60cc6287afabc2887366b0eb226c2David Howells if (pgoff >= rpgend) 13311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 13321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13338feae13110d60cc6287afabc2887366b0eb226c2David Howells /* handle inexactly overlapping matches between 13348feae13110d60cc6287afabc2887366b0eb226c2David Howells * mappings */ 13358feae13110d60cc6287afabc2887366b0eb226c2David Howells if ((pregion->vm_pgoff != pgoff || rpglen != pglen) && 13368feae13110d60cc6287afabc2887366b0eb226c2David Howells !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) { 13378feae13110d60cc6287afabc2887366b0eb226c2David Howells /* new mapping is not a subset of the region */ 13381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(capabilities & BDI_CAP_MAP_DIRECT)) 13391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto sharing_violation; 13401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 13411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 13421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13438feae13110d60cc6287afabc2887366b0eb226c2David Howells /* we've found a region we can share */ 13441e2ae599d37e60958c03ca5e46b1f657619a30cdDavid Howells pregion->vm_usage++; 13458feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_region = pregion; 13468feae13110d60cc6287afabc2887366b0eb226c2David Howells start = pregion->vm_start; 13478feae13110d60cc6287afabc2887366b0eb226c2David Howells start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT; 13488feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_start = start; 13498feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_end = start + len; 13508feae13110d60cc6287afabc2887366b0eb226c2David Howells 13518feae13110d60cc6287afabc2887366b0eb226c2David Howells if (pregion->vm_flags & VM_MAPPED_COPY) { 13528feae13110d60cc6287afabc2887366b0eb226c2David Howells kdebug("share copy"); 13538feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_flags |= VM_MAPPED_COPY; 13548feae13110d60cc6287afabc2887366b0eb226c2David Howells } else { 13558feae13110d60cc6287afabc2887366b0eb226c2David Howells kdebug("share mmap"); 13568feae13110d60cc6287afabc2887366b0eb226c2David Howells ret = do_mmap_shared_file(vma); 13578feae13110d60cc6287afabc2887366b0eb226c2David Howells if (ret < 0) { 13588feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_region = NULL; 13598feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_start = 0; 13608feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_end = 0; 13611e2ae599d37e60958c03ca5e46b1f657619a30cdDavid Howells pregion->vm_usage--; 13628feae13110d60cc6287afabc2887366b0eb226c2David Howells pregion = NULL; 13638feae13110d60cc6287afabc2887366b0eb226c2David Howells goto error_just_free; 13648feae13110d60cc6287afabc2887366b0eb226c2David Howells } 13658feae13110d60cc6287afabc2887366b0eb226c2David Howells } 13668feae13110d60cc6287afabc2887366b0eb226c2David Howells fput(region->vm_file); 13678feae13110d60cc6287afabc2887366b0eb226c2David Howells kmem_cache_free(vm_region_jar, region); 13688feae13110d60cc6287afabc2887366b0eb226c2David Howells region = pregion; 13698feae13110d60cc6287afabc2887366b0eb226c2David Howells result = start; 13708feae13110d60cc6287afabc2887366b0eb226c2David Howells goto share; 13711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 13721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* obtain the address at which to make a shared mapping 13741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - this is the hook for quasi-memory character devices to 13751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * tell us the location of a shared mapping 13761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1377645d83c5db970a1c57225e155113b4aa2451e920David Howells if (capabilities & BDI_CAP_MAP_DIRECT) { 13781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds addr = file->f_op->get_unmapped_area(file, addr, len, 13791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pgoff, flags); 1380bb005a59e08733bb416126dc184f73120fc6366bNamhyung Kim if (IS_ERR_VALUE(addr)) { 13811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = addr; 1382bb005a59e08733bb416126dc184f73120fc6366bNamhyung Kim if (ret != -ENOSYS) 13838feae13110d60cc6287afabc2887366b0eb226c2David Howells goto error_just_free; 13841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* the driver refused to tell us where to site 13861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the mapping so we'll have to attempt to copy 13871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * it */ 1388bb005a59e08733bb416126dc184f73120fc6366bNamhyung Kim ret = -ENODEV; 13891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(capabilities & BDI_CAP_MAP_COPY)) 13908feae13110d60cc6287afabc2887366b0eb226c2David Howells goto error_just_free; 13911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds capabilities &= ~BDI_CAP_MAP_DIRECT; 13938feae13110d60cc6287afabc2887366b0eb226c2David Howells } else { 13948feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_start = region->vm_start = addr; 13958feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_end = region->vm_end = addr + len; 13961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 13971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 13981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 13991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 14008feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_region = region; 14011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1402645d83c5db970a1c57225e155113b4aa2451e920David Howells /* set up the mapping 1403645d83c5db970a1c57225e155113b4aa2451e920David Howells * - the region is filled in if BDI_CAP_MAP_DIRECT is still set 1404645d83c5db970a1c57225e155113b4aa2451e920David Howells */ 14051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (file && vma->vm_flags & VM_SHARED) 14068feae13110d60cc6287afabc2887366b0eb226c2David Howells ret = do_mmap_shared_file(vma); 14071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else 1408645d83c5db970a1c57225e155113b4aa2451e920David Howells ret = do_mmap_private(vma, region, len, capabilities); 14091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret < 0) 1410645d83c5db970a1c57225e155113b4aa2451e920David Howells goto error_just_free; 1411645d83c5db970a1c57225e155113b4aa2451e920David Howells add_nommu_region(region); 14128feae13110d60cc6287afabc2887366b0eb226c2David Howells 1413ea637639591def87a54cea811cbac796980cb30dJie Zhang /* clear anonymous mappings that don't ask for uninitialized data */ 1414ea637639591def87a54cea811cbac796980cb30dJie Zhang if (!vma->vm_file && !(flags & MAP_UNINITIALIZED)) 1415ea637639591def87a54cea811cbac796980cb30dJie Zhang memset((void *)region->vm_start, 0, 1416ea637639591def87a54cea811cbac796980cb30dJie Zhang region->vm_end - region->vm_start); 1417ea637639591def87a54cea811cbac796980cb30dJie Zhang 14181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* okay... we have a mapping; now we have to register it */ 14198feae13110d60cc6287afabc2887366b0eb226c2David Howells result = vma->vm_start; 14201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 14211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds current->mm->total_vm += len >> PAGE_SHIFT; 14221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 14238feae13110d60cc6287afabc2887366b0eb226c2David Howellsshare: 14248feae13110d60cc6287afabc2887366b0eb226c2David Howells add_vma_to_mm(current->mm, vma); 14251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1426cfe79c00a2f4f687eed8b7534d1d3d3d35540c29Mike Frysinger /* we flush the region from the icache only when the first executable 1427cfe79c00a2f4f687eed8b7534d1d3d3d35540c29Mike Frysinger * mapping of it is made */ 1428cfe79c00a2f4f687eed8b7534d1d3d3d35540c29Mike Frysinger if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) { 1429cfe79c00a2f4f687eed8b7534d1d3d3d35540c29Mike Frysinger flush_icache_range(region->vm_start, region->vm_end); 1430cfe79c00a2f4f687eed8b7534d1d3d3d35540c29Mike Frysinger region->vm_icache_flushed = true; 1431cfe79c00a2f4f687eed8b7534d1d3d3d35540c29Mike Frysinger } 14321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1433cfe79c00a2f4f687eed8b7534d1d3d3d35540c29Mike Frysinger up_write(&nommu_region_sem); 14341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 14358feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = %lx", result); 14368feae13110d60cc6287afabc2887366b0eb226c2David Howells return result; 14371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 14388feae13110d60cc6287afabc2887366b0eb226c2David Howellserror_just_free: 14398feae13110d60cc6287afabc2887366b0eb226c2David Howells up_write(&nommu_region_sem); 14408feae13110d60cc6287afabc2887366b0eb226c2David Howellserror: 144189a8640279f8bb78aaf778d1fc5c4a6778f18064David Howells if (region->vm_file) 144289a8640279f8bb78aaf778d1fc5c4a6778f18064David Howells fput(region->vm_file); 14438feae13110d60cc6287afabc2887366b0eb226c2David Howells kmem_cache_free(vm_region_jar, region); 144489a8640279f8bb78aaf778d1fc5c4a6778f18064David Howells if (vma->vm_file) 144589a8640279f8bb78aaf778d1fc5c4a6778f18064David Howells fput(vma->vm_file); 14468feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_flags & VM_EXECUTABLE) 14478feae13110d60cc6287afabc2887366b0eb226c2David Howells removed_exe_file_vma(vma->vm_mm); 14488feae13110d60cc6287afabc2887366b0eb226c2David Howells kmem_cache_free(vm_area_cachep, vma); 14498feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = %d", ret); 14508feae13110d60cc6287afabc2887366b0eb226c2David Howells return ret; 14518feae13110d60cc6287afabc2887366b0eb226c2David Howells 14528feae13110d60cc6287afabc2887366b0eb226c2David Howellssharing_violation: 14538feae13110d60cc6287afabc2887366b0eb226c2David Howells up_write(&nommu_region_sem); 14548feae13110d60cc6287afabc2887366b0eb226c2David Howells printk(KERN_WARNING "Attempt to share mismatched mappings\n"); 14558feae13110d60cc6287afabc2887366b0eb226c2David Howells ret = -EINVAL; 14568feae13110d60cc6287afabc2887366b0eb226c2David Howells goto error; 14571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 14588feae13110d60cc6287afabc2887366b0eb226c2David Howellserror_getting_vma: 14598feae13110d60cc6287afabc2887366b0eb226c2David Howells kmem_cache_free(vm_region_jar, region); 14608feae13110d60cc6287afabc2887366b0eb226c2David Howells printk(KERN_WARNING "Allocation of vma for %lu byte allocation" 14618feae13110d60cc6287afabc2887366b0eb226c2David Howells " from process %d failed\n", 14621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len, current->pid); 14637bf02ea22c6cdd09e2d3f1d3c3fe366b834ae9afDavid Rientjes show_free_areas(0); 14641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 14651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 14668feae13110d60cc6287afabc2887366b0eb226c2David Howellserror_getting_region: 14678feae13110d60cc6287afabc2887366b0eb226c2David Howells printk(KERN_WARNING "Allocation of vm region for %lu byte allocation" 14688feae13110d60cc6287afabc2887366b0eb226c2David Howells " from process %d failed\n", 14691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len, current->pid); 14707bf02ea22c6cdd09e2d3f1d3c3fe366b834ae9afDavid Rientjes show_free_areas(0); 14711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 14721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1473b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(do_mmap_pgoff); 14741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 147566f0dc481e5b802ab363b979fc1753410c7d82b5Hugh DickinsSYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 147666f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins unsigned long, prot, unsigned long, flags, 147766f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins unsigned long, fd, unsigned long, pgoff) 147866f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins{ 147966f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins struct file *file = NULL; 148066f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins unsigned long retval = -EBADF; 148166f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins 1482120a795da07c9a02221ca23464c28a7c6ad7de1dAl Viro audit_mmap_fd(fd, flags); 148366f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins if (!(flags & MAP_ANONYMOUS)) { 148466f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins file = fget(fd); 148566f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins if (!file) 148666f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins goto out; 148766f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins } 148866f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins 148966f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE); 149066f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins 149166f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins down_write(¤t->mm->mmap_sem); 149266f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins retval = do_mmap_pgoff(file, addr, len, prot, flags, pgoff); 149366f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins up_write(¤t->mm->mmap_sem); 149466f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins 149566f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins if (file) 149666f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins fput(file); 149766f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickinsout: 149866f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins return retval; 149966f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins} 150066f0dc481e5b802ab363b979fc1753410c7d82b5Hugh Dickins 1501a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig#ifdef __ARCH_WANT_SYS_OLD_MMAP 1502a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwigstruct mmap_arg_struct { 1503a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig unsigned long addr; 1504a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig unsigned long len; 1505a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig unsigned long prot; 1506a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig unsigned long flags; 1507a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig unsigned long fd; 1508a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig unsigned long offset; 1509a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig}; 1510a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig 1511a4679373cf4ee0e7792dc56205365732b725c2c1Christoph HellwigSYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) 1512a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig{ 1513a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig struct mmap_arg_struct a; 1514a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig 1515a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig if (copy_from_user(&a, arg, sizeof(a))) 1516a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig return -EFAULT; 1517a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig if (a.offset & ~PAGE_MASK) 1518a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig return -EINVAL; 1519a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig 1520a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, 1521a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig a.offset >> PAGE_SHIFT); 1522a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig} 1523a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig#endif /* __ARCH_WANT_SYS_OLD_MMAP */ 1524a4679373cf4ee0e7792dc56205365732b725c2c1Christoph Hellwig 15251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 15268feae13110d60cc6287afabc2887366b0eb226c2David Howells * split a vma into two pieces at address 'addr', a new vma is allocated either 15278feae13110d60cc6287afabc2887366b0eb226c2David Howells * for the first part or the tail. 15281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 15298feae13110d60cc6287afabc2887366b0eb226c2David Howellsint split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 15308feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long addr, int new_below) 15311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 15328feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_area_struct *new; 15338feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_region *region; 15348feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long npages; 15351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 15368feae13110d60cc6287afabc2887366b0eb226c2David Howells kenter(""); 15371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1538779c10232ceb11c1b259232c4845cfb2850287b7David Howells /* we're only permitted to split anonymous regions (these should have 1539779c10232ceb11c1b259232c4845cfb2850287b7David Howells * only a single usage on the region) */ 1540779c10232ceb11c1b259232c4845cfb2850287b7David Howells if (vma->vm_file) 15418feae13110d60cc6287afabc2887366b0eb226c2David Howells return -ENOMEM; 15421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 15438feae13110d60cc6287afabc2887366b0eb226c2David Howells if (mm->map_count >= sysctl_max_map_count) 15448feae13110d60cc6287afabc2887366b0eb226c2David Howells return -ENOMEM; 15451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 15468feae13110d60cc6287afabc2887366b0eb226c2David Howells region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL); 15478feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!region) 15488feae13110d60cc6287afabc2887366b0eb226c2David Howells return -ENOMEM; 15491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 15508feae13110d60cc6287afabc2887366b0eb226c2David Howells new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 15518feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!new) { 15528feae13110d60cc6287afabc2887366b0eb226c2David Howells kmem_cache_free(vm_region_jar, region); 15538feae13110d60cc6287afabc2887366b0eb226c2David Howells return -ENOMEM; 15548feae13110d60cc6287afabc2887366b0eb226c2David Howells } 15558feae13110d60cc6287afabc2887366b0eb226c2David Howells 15568feae13110d60cc6287afabc2887366b0eb226c2David Howells /* most fields are the same, copy all, and then fixup */ 15578feae13110d60cc6287afabc2887366b0eb226c2David Howells *new = *vma; 15588feae13110d60cc6287afabc2887366b0eb226c2David Howells *region = *vma->vm_region; 15598feae13110d60cc6287afabc2887366b0eb226c2David Howells new->vm_region = region; 15608feae13110d60cc6287afabc2887366b0eb226c2David Howells 15618feae13110d60cc6287afabc2887366b0eb226c2David Howells npages = (addr - vma->vm_start) >> PAGE_SHIFT; 15628feae13110d60cc6287afabc2887366b0eb226c2David Howells 15638feae13110d60cc6287afabc2887366b0eb226c2David Howells if (new_below) { 1564dd8632a12e500a684478fea0951f380478d56fedPaul Mundt region->vm_top = region->vm_end = new->vm_end = addr; 15658feae13110d60cc6287afabc2887366b0eb226c2David Howells } else { 15668feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_start = new->vm_start = addr; 15678feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_pgoff = new->vm_pgoff += npages; 15681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 15698feae13110d60cc6287afabc2887366b0eb226c2David Howells 15708feae13110d60cc6287afabc2887366b0eb226c2David Howells if (new->vm_ops && new->vm_ops->open) 15718feae13110d60cc6287afabc2887366b0eb226c2David Howells new->vm_ops->open(new); 15728feae13110d60cc6287afabc2887366b0eb226c2David Howells 15738feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_vma_from_mm(vma); 15748feae13110d60cc6287afabc2887366b0eb226c2David Howells down_write(&nommu_region_sem); 15758feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_nommu_region(vma->vm_region); 15768feae13110d60cc6287afabc2887366b0eb226c2David Howells if (new_below) { 15778feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_region->vm_start = vma->vm_start = addr; 15788feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_region->vm_pgoff = vma->vm_pgoff += npages; 15798feae13110d60cc6287afabc2887366b0eb226c2David Howells } else { 15808feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_region->vm_end = vma->vm_end = addr; 1581dd8632a12e500a684478fea0951f380478d56fedPaul Mundt vma->vm_region->vm_top = addr; 15828feae13110d60cc6287afabc2887366b0eb226c2David Howells } 15838feae13110d60cc6287afabc2887366b0eb226c2David Howells add_nommu_region(vma->vm_region); 15848feae13110d60cc6287afabc2887366b0eb226c2David Howells add_nommu_region(new->vm_region); 15858feae13110d60cc6287afabc2887366b0eb226c2David Howells up_write(&nommu_region_sem); 15868feae13110d60cc6287afabc2887366b0eb226c2David Howells add_vma_to_mm(mm, vma); 15878feae13110d60cc6287afabc2887366b0eb226c2David Howells add_vma_to_mm(mm, new); 15888feae13110d60cc6287afabc2887366b0eb226c2David Howells return 0; 15891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 15901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 15913034097a5017dd9281b1f795e80af9859627850eDavid Howells/* 15928feae13110d60cc6287afabc2887366b0eb226c2David Howells * shrink a VMA by removing the specified chunk from either the beginning or 15938feae13110d60cc6287afabc2887366b0eb226c2David Howells * the end 15943034097a5017dd9281b1f795e80af9859627850eDavid Howells */ 15958feae13110d60cc6287afabc2887366b0eb226c2David Howellsstatic int shrink_vma(struct mm_struct *mm, 15968feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_area_struct *vma, 15978feae13110d60cc6287afabc2887366b0eb226c2David Howells unsigned long from, unsigned long to) 15981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 15998feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_region *region; 16001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16018feae13110d60cc6287afabc2887366b0eb226c2David Howells kenter(""); 16021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16038feae13110d60cc6287afabc2887366b0eb226c2David Howells /* adjust the VMA's pointers, which may reposition it in the MM's tree 16048feae13110d60cc6287afabc2887366b0eb226c2David Howells * and list */ 16058feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_vma_from_mm(vma); 16068feae13110d60cc6287afabc2887366b0eb226c2David Howells if (from > vma->vm_start) 16078feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_end = from; 16088feae13110d60cc6287afabc2887366b0eb226c2David Howells else 16098feae13110d60cc6287afabc2887366b0eb226c2David Howells vma->vm_start = to; 16108feae13110d60cc6287afabc2887366b0eb226c2David Howells add_vma_to_mm(mm, vma); 16111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16128feae13110d60cc6287afabc2887366b0eb226c2David Howells /* cut the backing region down to size */ 16138feae13110d60cc6287afabc2887366b0eb226c2David Howells region = vma->vm_region; 16141e2ae599d37e60958c03ca5e46b1f657619a30cdDavid Howells BUG_ON(region->vm_usage != 1); 16158feae13110d60cc6287afabc2887366b0eb226c2David Howells 16168feae13110d60cc6287afabc2887366b0eb226c2David Howells down_write(&nommu_region_sem); 16178feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_nommu_region(region); 1618dd8632a12e500a684478fea0951f380478d56fedPaul Mundt if (from > region->vm_start) { 1619dd8632a12e500a684478fea0951f380478d56fedPaul Mundt to = region->vm_top; 1620dd8632a12e500a684478fea0951f380478d56fedPaul Mundt region->vm_top = region->vm_end = from; 1621dd8632a12e500a684478fea0951f380478d56fedPaul Mundt } else { 16228feae13110d60cc6287afabc2887366b0eb226c2David Howells region->vm_start = to; 1623dd8632a12e500a684478fea0951f380478d56fedPaul Mundt } 16248feae13110d60cc6287afabc2887366b0eb226c2David Howells add_nommu_region(region); 16258feae13110d60cc6287afabc2887366b0eb226c2David Howells up_write(&nommu_region_sem); 16268feae13110d60cc6287afabc2887366b0eb226c2David Howells 16278feae13110d60cc6287afabc2887366b0eb226c2David Howells free_page_series(from, to); 16288feae13110d60cc6287afabc2887366b0eb226c2David Howells return 0; 16298feae13110d60cc6287afabc2887366b0eb226c2David Howells} 16301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16318feae13110d60cc6287afabc2887366b0eb226c2David Howells/* 16328feae13110d60cc6287afabc2887366b0eb226c2David Howells * release a mapping 16338feae13110d60cc6287afabc2887366b0eb226c2David Howells * - under NOMMU conditions the chunk to be unmapped must be backed by a single 16348feae13110d60cc6287afabc2887366b0eb226c2David Howells * VMA, though it need not cover the whole VMA 16358feae13110d60cc6287afabc2887366b0eb226c2David Howells */ 16368feae13110d60cc6287afabc2887366b0eb226c2David Howellsint do_munmap(struct mm_struct *mm, unsigned long start, size_t len) 16378feae13110d60cc6287afabc2887366b0eb226c2David Howells{ 16388feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_area_struct *vma; 1639f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu unsigned long end; 16408feae13110d60cc6287afabc2887366b0eb226c2David Howells int ret; 16411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16428feae13110d60cc6287afabc2887366b0eb226c2David Howells kenter(",%lx,%zx", start, len); 16431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1644f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu len = PAGE_ALIGN(len); 16458feae13110d60cc6287afabc2887366b0eb226c2David Howells if (len == 0) 16468feae13110d60cc6287afabc2887366b0eb226c2David Howells return -EINVAL; 1647365e9c87a982c03d0af3886e29d877f581b59611Hugh Dickins 1648f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu end = start + len; 1649f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu 16508feae13110d60cc6287afabc2887366b0eb226c2David Howells /* find the first potentially overlapping VMA */ 16518feae13110d60cc6287afabc2887366b0eb226c2David Howells vma = find_vma(mm, start); 16528feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!vma) { 165333e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells static int limit = 0; 165433e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells if (limit < 5) { 165533e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells printk(KERN_WARNING 165633e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells "munmap of memory not mmapped by process %d" 165733e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells " (%s): 0x%lx-0x%lx\n", 165833e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells current->pid, current->comm, 165933e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells start, start + len - 1); 166033e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells limit++; 166133e5d76979cf01e3834814fe0aea569d1d602c1aDavid Howells } 16628feae13110d60cc6287afabc2887366b0eb226c2David Howells return -EINVAL; 16638feae13110d60cc6287afabc2887366b0eb226c2David Howells } 16641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16658feae13110d60cc6287afabc2887366b0eb226c2David Howells /* we're allowed to split an anonymous VMA but not a file-backed one */ 16668feae13110d60cc6287afabc2887366b0eb226c2David Howells if (vma->vm_file) { 16678feae13110d60cc6287afabc2887366b0eb226c2David Howells do { 16688feae13110d60cc6287afabc2887366b0eb226c2David Howells if (start > vma->vm_start) { 16698feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = -EINVAL [miss]"); 16708feae13110d60cc6287afabc2887366b0eb226c2David Howells return -EINVAL; 16718feae13110d60cc6287afabc2887366b0eb226c2David Howells } 16728feae13110d60cc6287afabc2887366b0eb226c2David Howells if (end == vma->vm_end) 16738feae13110d60cc6287afabc2887366b0eb226c2David Howells goto erase_whole_vma; 1674d75a310c42c616c168953ed45c1091074f97828cNamhyung Kim vma = vma->vm_next; 1675d75a310c42c616c168953ed45c1091074f97828cNamhyung Kim } while (vma); 16768feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = -EINVAL [split file]"); 16778feae13110d60cc6287afabc2887366b0eb226c2David Howells return -EINVAL; 16788feae13110d60cc6287afabc2887366b0eb226c2David Howells } else { 16798feae13110d60cc6287afabc2887366b0eb226c2David Howells /* the chunk must be a subset of the VMA found */ 16808feae13110d60cc6287afabc2887366b0eb226c2David Howells if (start == vma->vm_start && end == vma->vm_end) 16818feae13110d60cc6287afabc2887366b0eb226c2David Howells goto erase_whole_vma; 16828feae13110d60cc6287afabc2887366b0eb226c2David Howells if (start < vma->vm_start || end > vma->vm_end) { 16838feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = -EINVAL [superset]"); 16848feae13110d60cc6287afabc2887366b0eb226c2David Howells return -EINVAL; 16858feae13110d60cc6287afabc2887366b0eb226c2David Howells } 16868feae13110d60cc6287afabc2887366b0eb226c2David Howells if (start & ~PAGE_MASK) { 16878feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = -EINVAL [unaligned start]"); 16888feae13110d60cc6287afabc2887366b0eb226c2David Howells return -EINVAL; 16898feae13110d60cc6287afabc2887366b0eb226c2David Howells } 16908feae13110d60cc6287afabc2887366b0eb226c2David Howells if (end != vma->vm_end && end & ~PAGE_MASK) { 16918feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = -EINVAL [unaligned split]"); 16928feae13110d60cc6287afabc2887366b0eb226c2David Howells return -EINVAL; 16938feae13110d60cc6287afabc2887366b0eb226c2David Howells } 16948feae13110d60cc6287afabc2887366b0eb226c2David Howells if (start != vma->vm_start && end != vma->vm_end) { 16958feae13110d60cc6287afabc2887366b0eb226c2David Howells ret = split_vma(mm, vma, start, 1); 16968feae13110d60cc6287afabc2887366b0eb226c2David Howells if (ret < 0) { 16978feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = %d [split]", ret); 16988feae13110d60cc6287afabc2887366b0eb226c2David Howells return ret; 16998feae13110d60cc6287afabc2887366b0eb226c2David Howells } 17008feae13110d60cc6287afabc2887366b0eb226c2David Howells } 17018feae13110d60cc6287afabc2887366b0eb226c2David Howells return shrink_vma(mm, vma, start, end); 17028feae13110d60cc6287afabc2887366b0eb226c2David Howells } 17031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17048feae13110d60cc6287afabc2887366b0eb226c2David Howellserase_whole_vma: 17058feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_vma_from_mm(vma); 17068feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_vma(mm, vma); 17078feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(" = 0"); 17081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 17091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1710b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(do_munmap); 17111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17126a6160a7b5c27b3c38651baef92a14fa7072b3c1Heiko CarstensSYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) 17133034097a5017dd9281b1f795e80af9859627850eDavid Howells{ 17143034097a5017dd9281b1f795e80af9859627850eDavid Howells int ret; 17153034097a5017dd9281b1f795e80af9859627850eDavid Howells struct mm_struct *mm = current->mm; 17163034097a5017dd9281b1f795e80af9859627850eDavid Howells 17173034097a5017dd9281b1f795e80af9859627850eDavid Howells down_write(&mm->mmap_sem); 17183034097a5017dd9281b1f795e80af9859627850eDavid Howells ret = do_munmap(mm, addr, len); 17193034097a5017dd9281b1f795e80af9859627850eDavid Howells up_write(&mm->mmap_sem); 17203034097a5017dd9281b1f795e80af9859627850eDavid Howells return ret; 17213034097a5017dd9281b1f795e80af9859627850eDavid Howells} 17223034097a5017dd9281b1f795e80af9859627850eDavid Howells 17233034097a5017dd9281b1f795e80af9859627850eDavid Howells/* 17248feae13110d60cc6287afabc2887366b0eb226c2David Howells * release all the mappings made in a process's VM space 17253034097a5017dd9281b1f795e80af9859627850eDavid Howells */ 17268feae13110d60cc6287afabc2887366b0eb226c2David Howellsvoid exit_mmap(struct mm_struct *mm) 17271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 17288feae13110d60cc6287afabc2887366b0eb226c2David Howells struct vm_area_struct *vma; 17291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17308feae13110d60cc6287afabc2887366b0eb226c2David Howells if (!mm) 17318feae13110d60cc6287afabc2887366b0eb226c2David Howells return; 17321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17338feae13110d60cc6287afabc2887366b0eb226c2David Howells kenter(""); 17341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17358feae13110d60cc6287afabc2887366b0eb226c2David Howells mm->total_vm = 0; 17361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17378feae13110d60cc6287afabc2887366b0eb226c2David Howells while ((vma = mm->mmap)) { 17388feae13110d60cc6287afabc2887366b0eb226c2David Howells mm->mmap = vma->vm_next; 17398feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_vma_from_mm(vma); 17408feae13110d60cc6287afabc2887366b0eb226c2David Howells delete_vma(mm, vma); 174104c3496152394d17e3bc2316f9731ee3e8a026bcSteven J. Magnani cond_resched(); 17421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 17438feae13110d60cc6287afabc2887366b0eb226c2David Howells 17448feae13110d60cc6287afabc2887366b0eb226c2David Howells kleave(""); 17451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 17461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long do_brk(unsigned long addr, unsigned long len) 17481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 17491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 17501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 17511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 17536fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells * expand (or shrink) an existing mapping, potentially moving it at the same 17546fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells * time (controlled by the MREMAP_MAYMOVE flag and available VM space) 17551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 17566fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells * under NOMMU conditions, we only permit changing a mapping's size, and only 17578feae13110d60cc6287afabc2887366b0eb226c2David Howells * as long as it stays within the region allocated by do_mmap_private() and the 17588feae13110d60cc6287afabc2887366b0eb226c2David Howells * block is not shareable 17591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 17606fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells * MREMAP_FIXED is not supported under NOMMU conditions 17611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 17621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long do_mremap(unsigned long addr, 17631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long old_len, unsigned long new_len, 17641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags, unsigned long new_addr) 17651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 17666fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells struct vm_area_struct *vma; 17671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* insanity checks first */ 1769f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu old_len = PAGE_ALIGN(old_len); 1770f67d9b1576c1c6e02100f8b27f4e9d66bbeb4d49Bob Liu new_len = PAGE_ALIGN(new_len); 17718feae13110d60cc6287afabc2887366b0eb226c2David Howells if (old_len == 0 || new_len == 0) 17721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return (unsigned long) -EINVAL; 17731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17748feae13110d60cc6287afabc2887366b0eb226c2David Howells if (addr & ~PAGE_MASK) 17758feae13110d60cc6287afabc2887366b0eb226c2David Howells return -EINVAL; 17768feae13110d60cc6287afabc2887366b0eb226c2David Howells 17771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (flags & MREMAP_FIXED && new_addr != addr) 17781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return (unsigned long) -EINVAL; 17791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17808feae13110d60cc6287afabc2887366b0eb226c2David Howells vma = find_vma_exact(current->mm, addr, old_len); 17816fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells if (!vma) 17826fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells return (unsigned long) -EINVAL; 17831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17846fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells if (vma->vm_end != vma->vm_start + old_len) 17851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return (unsigned long) -EFAULT; 17861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17876fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells if (vma->vm_flags & VM_MAYSHARE) 17881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return (unsigned long) -EPERM; 17891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17908feae13110d60cc6287afabc2887366b0eb226c2David Howells if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start) 17911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return (unsigned long) -ENOMEM; 17921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* all checks complete - do it */ 17946fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells vma->vm_end = vma->vm_start + new_len; 17956fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells return vma->vm_start; 17966fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells} 1797b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(do_mremap); 17986fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells 17996a6160a7b5c27b3c38651baef92a14fa7072b3c1Heiko CarstensSYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, 18006a6160a7b5c27b3c38651baef92a14fa7072b3c1Heiko Carstens unsigned long, new_len, unsigned long, flags, 18016a6160a7b5c27b3c38651baef92a14fa7072b3c1Heiko Carstens unsigned long, new_addr) 18026fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells{ 18036fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells unsigned long ret; 18046fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells 18056fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells down_write(¤t->mm->mmap_sem); 18066fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells ret = do_mremap(addr, old_len, new_len, flags, new_addr); 18076fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells up_write(¤t->mm->mmap_sem); 18086fa5f80bc34da1a49b42117602b44441402cac2fDavid Howells return ret; 18091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 18101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18116aab341e0a28aff100a09831c5300a2994b8b986Linus Torvaldsstruct page *follow_page(struct vm_area_struct *vma, unsigned long address, 1812deceb6cd17e6dfafe4c4f81b1b4153bc41b2cb70Hugh Dickins unsigned int foll_flags) 18131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 18141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 18151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 18161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18178f3b1327aa454bc8283e96bca7669c3c88b83f79Bob Liuint remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 18188f3b1327aa454bc8283e96bca7669c3c88b83f79Bob Liu unsigned long pfn, unsigned long size, pgprot_t prot) 18191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 18208f3b1327aa454bc8283e96bca7669c3c88b83f79Bob Liu if (addr != (pfn << PAGE_SHIFT)) 18218f3b1327aa454bc8283e96bca7669c3c88b83f79Bob Liu return -EINVAL; 18228f3b1327aa454bc8283e96bca7669c3c88b83f79Bob Liu 18238f3b1327aa454bc8283e96bca7669c3c88b83f79Bob Liu vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; 182466aa2b4b1cf9a61f1550251c56fc6f0d48287591Greg Ungerer return 0; 18251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 182622c4af4092fc2e037ce2e2922023fc222cf0c443Luke YangEXPORT_SYMBOL(remap_pfn_range); 18271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1828f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundtint remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 1829f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt unsigned long pgoff) 1830f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt{ 1831f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt unsigned int size = vma->vm_end - vma->vm_start; 1832f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt 1833f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt if (!(vma->vm_flags & VM_USERMAP)) 1834f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt return -EINVAL; 1835f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt 1836f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT)); 1837f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt vma->vm_end = vma->vm_start + size; 1838f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt 1839f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt return 0; 1840f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt} 1841f905bc447c303fefcb180c7e8b641746ffa6cf87Paul MundtEXPORT_SYMBOL(remap_vmalloc_range); 1842f905bc447c303fefcb180c7e8b641746ffa6cf87Paul Mundt 18431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long arch_get_unmapped_area(struct file *file, unsigned long addr, 18441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long len, unsigned long pgoff, unsigned long flags) 18451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 18461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 18471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 18481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18491363c3cd8603a913a27e2995dccbd70d5312d8e6Wolfgang Wandervoid arch_unmap_area(struct mm_struct *mm, unsigned long addr) 18501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 18511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 18521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid unmap_mapping_range(struct address_space *mapping, 18541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds loff_t const holebegin, loff_t const holelen, 18551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int even_cows) 18561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 18571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 185822c4af4092fc2e037ce2e2922023fc222cf0c443Luke YangEXPORT_SYMBOL(unmap_mapping_range); 18591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 18611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Check that a process has enough memory to allocate a new virtual 18621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * mapping. 0 means there is enough memory for the allocation to 18631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * succeed and -ENOMEM implies there is not. 18641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We currently support three overcommit policies, which are set via the 18661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting 18671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Strict overcommit modes added 2002 Feb 26 by Alan Cox. 18691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Additional code 2002 Jul 20 by Robert Love. 18701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise. 18721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note this is a helper function intended to be used by LSMs which 18741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * wish to use this logic. 18751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 187634b4e4aa3c470ce8fa2bd78abb1741b4b58baad7Alan Coxint __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) 18771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 18781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long free, allowed; 18791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_acct_memory(pages); 18811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 18831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Sometimes we want to use more memory than we have 18841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 18851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS) 18861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 18871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { 1889c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink free = global_page_state(NR_FREE_PAGES); 1890c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink free += global_page_state(NR_FILE_PAGES); 1891c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink 1892c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink /* 1893c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink * shmem pages shouldn't be counted as free in this 1894c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink * case, they can't be purged, only swapped out, and 1895c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink * that won't affect the overall amount of available 1896c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink * memory in the system. 1897c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink */ 1898c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink free -= global_page_state(NR_SHMEM); 18991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds free += nr_swap_pages; 19011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 19031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Any slabs which are created with the 19041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * SLAB_RECLAIM_ACCOUNT flag claim to have contents 19051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * which are reclaimable, under pressure. The dentry 19061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * cache and most inode caches should fall into this 19071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1908972d1a7b140569084439a81265a0f15b74e924e0Christoph Lameter free += global_page_state(NR_SLAB_RECLAIMABLE); 19091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 1911d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI * Leave reserved pages. The pages are not for anonymous pages. 1912d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI */ 1913c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink if (free <= totalreserve_pages) 1914d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI goto error; 1915d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI else 1916c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink free -= totalreserve_pages; 1917d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI 1918d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI /* 1919d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI * Leave the last 3% for root 1920d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI */ 19211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!cap_sys_admin) 1922c15bef3099c346f2124367bff46954b59e13c3eeDmitry Fink free -= free / 32; 19231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (free > pages) 19251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 1926d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI 1927d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKI goto error; 19281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 19291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds allowed = totalram_pages * sysctl_overcommit_ratio / 100; 19311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 19321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Leave the last 3% for root 19331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 19341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!cap_sys_admin) 19351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds allowed -= allowed / 32; 19361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds allowed += total_swap_pages; 19371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Don't let a single process grow too big: 19391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds leave 3% of the size of this process for other processes */ 1940731572d39fcd3498702eda4600db4c43d51e0b26Alan Cox if (mm) 1941731572d39fcd3498702eda4600db4c43d51e0b26Alan Cox allowed -= mm->total_vm / 32; 19421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 194300a62ce91e554198ef28234c91c36f850f5a3bc9KOSAKI Motohiro if (percpu_counter_read_positive(&vm_committed_as) < allowed) 19441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 194500a62ce91e554198ef28234c91c36f850f5a3bc9KOSAKI Motohiro 1946d5ddc79bcaab6975e7671805c3578407dc33b764Hideo AOKIerror: 19471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_unacct_memory(pages); 19481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 19501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 19511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1952cae5d39032acf26c265f6b1dc73d7ce6ff4bc387Stephen Wilsonint in_gate_area_no_mm(unsigned long addr) 19531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 19541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 19551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1956b0e15190ead07056ab0c3844a499ff35e66d27ccDavid Howells 1957d0217ac04ca6591841e5665f518e38064f4e65bdNick Pigginint filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1958b0e15190ead07056ab0c3844a499ff35e66d27ccDavid Howells{ 1959b0e15190ead07056ab0c3844a499ff35e66d27ccDavid Howells BUG(); 1960d0217ac04ca6591841e5665f518e38064f4e65bdNick Piggin return 0; 1961b0e15190ead07056ab0c3844a499ff35e66d27ccDavid Howells} 1962b50731732f926d6c49fd0724616a7344c31cd5cfPaul MundtEXPORT_SYMBOL(filemap_fault); 19630ec76a110f432e98277e464b82ace8dd66571689David Howells 1964f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysingerstatic int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, 1965f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger unsigned long addr, void *buf, int len, int write) 19660ec76a110f432e98277e464b82ace8dd66571689David Howells{ 19670ec76a110f432e98277e464b82ace8dd66571689David Howells struct vm_area_struct *vma; 19680ec76a110f432e98277e464b82ace8dd66571689David Howells 19690ec76a110f432e98277e464b82ace8dd66571689David Howells down_read(&mm->mmap_sem); 19700ec76a110f432e98277e464b82ace8dd66571689David Howells 19710ec76a110f432e98277e464b82ace8dd66571689David Howells /* the access must start within one of the target process's mappings */ 19720159b141d8b1f9b9f9cffacae47bec1e05c63b8bDavid Howells vma = find_vma(mm, addr); 19730159b141d8b1f9b9f9cffacae47bec1e05c63b8bDavid Howells if (vma) { 19740ec76a110f432e98277e464b82ace8dd66571689David Howells /* don't overrun this mapping */ 19750ec76a110f432e98277e464b82ace8dd66571689David Howells if (addr + len >= vma->vm_end) 19760ec76a110f432e98277e464b82ace8dd66571689David Howells len = vma->vm_end - addr; 19770ec76a110f432e98277e464b82ace8dd66571689David Howells 19780ec76a110f432e98277e464b82ace8dd66571689David Howells /* only read or write mappings where it is permitted */ 1979d00c7b993712e4bb16d0012b35b654e40159b327David Howells if (write && vma->vm_flags & VM_MAYWRITE) 19807959722b951cffcd61a0a35229d007deeed8c2ddJie Zhang copy_to_user_page(vma, NULL, addr, 19817959722b951cffcd61a0a35229d007deeed8c2ddJie Zhang (void *) addr, buf, len); 1982d00c7b993712e4bb16d0012b35b654e40159b327David Howells else if (!write && vma->vm_flags & VM_MAYREAD) 19837959722b951cffcd61a0a35229d007deeed8c2ddJie Zhang copy_from_user_page(vma, NULL, addr, 19847959722b951cffcd61a0a35229d007deeed8c2ddJie Zhang buf, (void *) addr, len); 19850ec76a110f432e98277e464b82ace8dd66571689David Howells else 19860ec76a110f432e98277e464b82ace8dd66571689David Howells len = 0; 19870ec76a110f432e98277e464b82ace8dd66571689David Howells } else { 19880ec76a110f432e98277e464b82ace8dd66571689David Howells len = 0; 19890ec76a110f432e98277e464b82ace8dd66571689David Howells } 19900ec76a110f432e98277e464b82ace8dd66571689David Howells 19910ec76a110f432e98277e464b82ace8dd66571689David Howells up_read(&mm->mmap_sem); 1992f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger 1993f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger return len; 1994f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger} 1995f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger 1996f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger/** 1997f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * @access_remote_vm - access another process' address space 1998f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * @mm: the mm_struct of the target address space 1999f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * @addr: start address to access 2000f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * @buf: source or destination buffer 2001f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * @len: number of bytes to transfer 2002f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * @write: whether the access is a write 2003f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * 2004f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * The caller must hold a reference on @mm. 2005f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger */ 2006f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysingerint access_remote_vm(struct mm_struct *mm, unsigned long addr, 2007f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger void *buf, int len, int write) 2008f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger{ 2009f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger return __access_remote_vm(NULL, mm, addr, buf, len, write); 2010f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger} 2011f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger 2012f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger/* 2013f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * Access another process' address space. 2014f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger * - source/target buffer must be kernel space 2015f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger */ 2016f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysingerint access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write) 2017f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger{ 2018f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger struct mm_struct *mm; 2019f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger 2020f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger if (addr + len < addr) 2021f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger return 0; 2022f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger 2023f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger mm = get_task_mm(tsk); 2024f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger if (!mm) 2025f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger return 0; 2026f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger 2027f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger len = __access_remote_vm(tsk, mm, addr, buf, len, write); 2028f55f199b7d76a01e7ce9d1c3bb004327e075c327Mike Frysinger 20290ec76a110f432e98277e464b82ace8dd66571689David Howells mmput(mm); 20300ec76a110f432e98277e464b82ace8dd66571689David Howells return len; 20310ec76a110f432e98277e464b82ace8dd66571689David Howells} 20327e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 20337e6608724c640924aad1d556d17df33ebaa6124dDavid Howells/** 20347e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode 20357e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * @inode: The inode to check 20367e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * @size: The current filesize of the inode 20377e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * @newsize: The proposed filesize of the inode 20387e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * 20397e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * Check the shared mappings on an inode on behalf of a shrinking truncate to 20407e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * make sure that that any outstanding VMAs aren't broken and then shrink the 20417e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't 20427e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * automatically grant mappings that are too large. 20437e6608724c640924aad1d556d17df33ebaa6124dDavid Howells */ 20447e6608724c640924aad1d556d17df33ebaa6124dDavid Howellsint nommu_shrink_inode_mappings(struct inode *inode, size_t size, 20457e6608724c640924aad1d556d17df33ebaa6124dDavid Howells size_t newsize) 20467e6608724c640924aad1d556d17df33ebaa6124dDavid Howells{ 20477e6608724c640924aad1d556d17df33ebaa6124dDavid Howells struct vm_area_struct *vma; 20487e6608724c640924aad1d556d17df33ebaa6124dDavid Howells struct prio_tree_iter iter; 20497e6608724c640924aad1d556d17df33ebaa6124dDavid Howells struct vm_region *region; 20507e6608724c640924aad1d556d17df33ebaa6124dDavid Howells pgoff_t low, high; 20517e6608724c640924aad1d556d17df33ebaa6124dDavid Howells size_t r_size, r_top; 20527e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 20537e6608724c640924aad1d556d17df33ebaa6124dDavid Howells low = newsize >> PAGE_SHIFT; 20547e6608724c640924aad1d556d17df33ebaa6124dDavid Howells high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 20557e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 20567e6608724c640924aad1d556d17df33ebaa6124dDavid Howells down_write(&nommu_region_sem); 2057918e556ec214ed2f584e4cac56d7b29e4bb6bf27David Howells mutex_lock(&inode->i_mapping->i_mmap_mutex); 20587e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 20597e6608724c640924aad1d556d17df33ebaa6124dDavid Howells /* search for VMAs that fall within the dead zone */ 20607e6608724c640924aad1d556d17df33ebaa6124dDavid Howells vma_prio_tree_foreach(vma, &iter, &inode->i_mapping->i_mmap, 20617e6608724c640924aad1d556d17df33ebaa6124dDavid Howells low, high) { 20627e6608724c640924aad1d556d17df33ebaa6124dDavid Howells /* found one - only interested if it's shared out of the page 20637e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * cache */ 20647e6608724c640924aad1d556d17df33ebaa6124dDavid Howells if (vma->vm_flags & VM_SHARED) { 2065918e556ec214ed2f584e4cac56d7b29e4bb6bf27David Howells mutex_unlock(&inode->i_mapping->i_mmap_mutex); 20667e6608724c640924aad1d556d17df33ebaa6124dDavid Howells up_write(&nommu_region_sem); 20677e6608724c640924aad1d556d17df33ebaa6124dDavid Howells return -ETXTBSY; /* not quite true, but near enough */ 20687e6608724c640924aad1d556d17df33ebaa6124dDavid Howells } 20697e6608724c640924aad1d556d17df33ebaa6124dDavid Howells } 20707e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 20717e6608724c640924aad1d556d17df33ebaa6124dDavid Howells /* reduce any regions that overlap the dead zone - if in existence, 20727e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * these will be pointed to by VMAs that don't overlap the dead zone 20737e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * 20747e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * we don't check for any regions that start beyond the EOF as there 20757e6608724c640924aad1d556d17df33ebaa6124dDavid Howells * shouldn't be any 20767e6608724c640924aad1d556d17df33ebaa6124dDavid Howells */ 20777e6608724c640924aad1d556d17df33ebaa6124dDavid Howells vma_prio_tree_foreach(vma, &iter, &inode->i_mapping->i_mmap, 20787e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 0, ULONG_MAX) { 20797e6608724c640924aad1d556d17df33ebaa6124dDavid Howells if (!(vma->vm_flags & VM_SHARED)) 20807e6608724c640924aad1d556d17df33ebaa6124dDavid Howells continue; 20817e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 20827e6608724c640924aad1d556d17df33ebaa6124dDavid Howells region = vma->vm_region; 20837e6608724c640924aad1d556d17df33ebaa6124dDavid Howells r_size = region->vm_top - region->vm_start; 20847e6608724c640924aad1d556d17df33ebaa6124dDavid Howells r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size; 20857e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 20867e6608724c640924aad1d556d17df33ebaa6124dDavid Howells if (r_top > newsize) { 20877e6608724c640924aad1d556d17df33ebaa6124dDavid Howells region->vm_top -= r_top - newsize; 20887e6608724c640924aad1d556d17df33ebaa6124dDavid Howells if (region->vm_end > region->vm_top) 20897e6608724c640924aad1d556d17df33ebaa6124dDavid Howells region->vm_end = region->vm_top; 20907e6608724c640924aad1d556d17df33ebaa6124dDavid Howells } 20917e6608724c640924aad1d556d17df33ebaa6124dDavid Howells } 20927e6608724c640924aad1d556d17df33ebaa6124dDavid Howells 2093918e556ec214ed2f584e4cac56d7b29e4bb6bf27David Howells mutex_unlock(&inode->i_mapping->i_mmap_mutex); 20947e6608724c640924aad1d556d17df33ebaa6124dDavid Howells up_write(&nommu_region_sem); 20957e6608724c640924aad1d556d17df33ebaa6124dDavid Howells return 0; 20967e6608724c640924aad1d556d17df33ebaa6124dDavid Howells} 2097