1fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras/* 2fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * Copyright 2007-2008 Paul Mackerras, IBM Corp. 3fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * 4fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * This program is free software; you can redistribute it and/or 5fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * modify it under the terms of the GNU General Public License 6fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * as published by the Free Software Foundation; either version 7fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * 2 of the License, or (at your option) any later version. 8fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras */ 9fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 10fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <linux/errno.h> 11fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <linux/kernel.h> 12fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <linux/gfp.h> 13fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <linux/types.h> 14fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <linux/mm.h> 15fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <linux/hugetlb.h> 16fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 17fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <asm/pgtable.h> 18fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <asm/uaccess.h> 19fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras#include <asm/tlbflush.h> 20fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 21fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras/* 22fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * Free all pages allocated for subpage protection maps and pointers. 23fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * Also makes sure that the subpage_prot_table structure is 24fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * reinitialized for the next user. 25fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras */ 26d28513bc7f675d28b479db666d572e078ecf182dDavid Gibsonvoid subpage_prot_free(struct mm_struct *mm) 27fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras{ 28d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson struct subpage_prot_table *spt = &mm->context.spt; 29fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras unsigned long i, j, addr; 30fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras u32 **p; 31fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 32fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras for (i = 0; i < 4; ++i) { 33fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (spt->low_prot[i]) { 34fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras free_page((unsigned long)spt->low_prot[i]); 35fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spt->low_prot[i] = NULL; 36fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 37fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 38fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras addr = 0; 39fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras for (i = 0; i < 2; ++i) { 40fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras p = spt->protptrs[i]; 41fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!p) 42fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras continue; 43fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spt->protptrs[i] = NULL; 44fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras for (j = 0; j < SBP_L2_COUNT && addr < spt->maxaddr; 45fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras ++j, addr += PAGE_SIZE) 46fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (p[j]) 47fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras free_page((unsigned long)p[j]); 48fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras free_page((unsigned long)p); 49fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 50fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spt->maxaddr = 0; 51fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras} 52fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 53d28513bc7f675d28b479db666d572e078ecf182dDavid Gibsonvoid subpage_prot_init_new_context(struct mm_struct *mm) 54d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson{ 55d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson struct subpage_prot_table *spt = &mm->context.spt; 56d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson 57d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson memset(spt, 0, sizeof(*spt)); 58d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson} 59d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson 60fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerrasstatic void hpte_flush_range(struct mm_struct *mm, unsigned long addr, 61fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras int npages) 62fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras{ 63fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pgd_t *pgd; 64fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pud_t *pud; 65fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pmd_t *pmd; 66fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pte_t *pte; 67fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spinlock_t *ptl; 68fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 69fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pgd = pgd_offset(mm, addr); 70fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (pgd_none(*pgd)) 71fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras return; 72fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pud = pud_offset(pgd, addr); 73fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (pud_none(*pud)) 74fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras return; 75fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pmd = pmd_offset(pud, addr); 76fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (pmd_none(*pmd)) 77fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras return; 78fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 79fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras arch_enter_lazy_mmu_mode(); 80fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras for (; npages > 0; --npages) { 8188247e8d7ba6639f2c199e147ebbc91f7673150cAneesh Kumar K.V pte_update(mm, addr, pte, 0, 0, 0); 82fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras addr += PAGE_SIZE; 83fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras ++pte; 84fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 85fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras arch_leave_lazy_mmu_mode(); 86fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras pte_unmap_unlock(pte - 1, ptl); 87fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras} 88fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 89fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras/* 90fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * Clear the subpage protection map for an address range, allowing 91fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * all accesses that are allowed by the pte permissions. 92fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras */ 93fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerrasstatic void subpage_prot_clear(unsigned long addr, unsigned long len) 94fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras{ 95fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras struct mm_struct *mm = current->mm; 96d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson struct subpage_prot_table *spt = &mm->context.spt; 97fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras u32 **spm, *spp; 986b5e7229bbd59f0cfce7015fd46736fc93d8c8c3Joe MacDonald unsigned long i; 996b5e7229bbd59f0cfce7015fd46736fc93d8c8c3Joe MacDonald size_t nw; 100fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras unsigned long next, limit; 101fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 102fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras down_write(&mm->mmap_sem); 103fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras limit = addr + len; 104fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (limit > spt->maxaddr) 105fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras limit = spt->maxaddr; 106fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras for (; addr < limit; addr = next) { 107fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras next = pmd_addr_end(addr, limit); 108b0d436c739b0d4afcdfe2e97d4d1ee41ea2db62eAnton Blanchard if (addr < 0x100000000UL) { 109fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spm = spt->low_prot; 110fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } else { 111fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spm = spt->protptrs[addr >> SBP_L3_SHIFT]; 112fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!spm) 113fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras continue; 114fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 115fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spp = spm[(addr >> SBP_L2_SHIFT) & (SBP_L2_COUNT - 1)]; 116fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!spp) 117fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras continue; 118fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spp += (addr >> PAGE_SHIFT) & (SBP_L1_COUNT - 1); 119fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 120fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras i = (addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 121fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras nw = PTRS_PER_PTE - i; 122fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (addr + (nw << PAGE_SHIFT) > next) 123fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras nw = (next - addr) >> PAGE_SHIFT; 124fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 125fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras memset(spp, 0, nw * sizeof(u32)); 126fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 127fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras /* now flush any existing HPTEs for the range */ 128fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras hpte_flush_range(mm, addr, nw); 129fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 130fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras up_write(&mm->mmap_sem); 131fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras} 132fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 133d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V#ifdef CONFIG_TRANSPARENT_HUGEPAGE 134d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.Vstatic int subpage_walk_pmd_entry(pmd_t *pmd, unsigned long addr, 135d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V unsigned long end, struct mm_walk *walk) 136d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V{ 137d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V struct vm_area_struct *vma = walk->private; 138d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V split_huge_page_pmd(vma, addr, pmd); 139d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V return 0; 140d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V} 141d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V 142d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.Vstatic void subpage_mark_vma_nohuge(struct mm_struct *mm, unsigned long addr, 143d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V unsigned long len) 144d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V{ 145d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V struct vm_area_struct *vma; 146d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V struct mm_walk subpage_proto_walk = { 147d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V .mm = mm, 148d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V .pmd_entry = subpage_walk_pmd_entry, 149d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V }; 150d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V 151d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V /* 152d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V * We don't try too hard, we just mark all the vma in that range 153d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V * VM_NOHUGEPAGE and split them. 154d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V */ 155d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V vma = find_vma(mm, addr); 156d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V /* 157d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V * If the range is in unmapped range, just return 158d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V */ 159d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V if (vma && ((addr + len) <= vma->vm_start)) 160d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V return; 161d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V 162d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V while (vma) { 163d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V if (vma->vm_start >= (addr + len)) 164d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V break; 165d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V vma->vm_flags |= VM_NOHUGEPAGE; 166d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V subpage_proto_walk.private = vma; 167d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V walk_page_range(vma->vm_start, vma->vm_end, 168d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V &subpage_proto_walk); 169d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V vma = vma->vm_next; 170d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V } 171d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V} 172d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V#else 173d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.Vstatic void subpage_mark_vma_nohuge(struct mm_struct *mm, unsigned long addr, 174d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V unsigned long len) 175d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V{ 176d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V return; 177d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V} 178d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V#endif 179d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V 180fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras/* 181fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * Copy in a subpage protection map for an address range. 182fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * The map has 2 bits per 4k subpage, so 32 bits per 64k page. 183fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * Each 2-bit field is 0 to allow any access, 1 to prevent writes, 184fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * 2 or 3 to prevent all accesses. 185fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * Note that the normal page protections also apply; the subpage 186fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * protection mechanism is an additional constraint, so putting 0 187fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * in a 2-bit field won't allow writes to a page that is otherwise 188fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras * write-protected. 189fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras */ 190fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerraslong sys_subpage_prot(unsigned long addr, unsigned long len, u32 __user *map) 191fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras{ 192fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras struct mm_struct *mm = current->mm; 193d28513bc7f675d28b479db666d572e078ecf182dDavid Gibson struct subpage_prot_table *spt = &mm->context.spt; 194fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras u32 **spm, *spp; 1956b5e7229bbd59f0cfce7015fd46736fc93d8c8c3Joe MacDonald unsigned long i; 1966b5e7229bbd59f0cfce7015fd46736fc93d8c8c3Joe MacDonald size_t nw; 197fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras unsigned long next, limit; 198fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras int err; 199fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 200fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras /* Check parameters */ 201fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if ((addr & ~PAGE_MASK) || (len & ~PAGE_MASK) || 202fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras addr >= TASK_SIZE || len >= TASK_SIZE || addr + len > TASK_SIZE) 203fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras return -EINVAL; 204fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 205fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (is_hugepage_only_range(mm, addr, len)) 206fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras return -EINVAL; 207fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 208fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!map) { 209fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras /* Clear out the protection map for the address range */ 210fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras subpage_prot_clear(addr, len); 211fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras return 0; 212fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 213fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 214fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!access_ok(VERIFY_READ, map, (len >> PAGE_SHIFT) * sizeof(u32))) 215fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras return -EFAULT; 216fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 217fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras down_write(&mm->mmap_sem); 218d8e355a20f9dd45deea4c33db649dda59bdbd293Aneesh Kumar K.V subpage_mark_vma_nohuge(mm, addr, len); 219fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras for (limit = addr + len; addr < limit; addr = next) { 220fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras next = pmd_addr_end(addr, limit); 221fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras err = -ENOMEM; 222b0d436c739b0d4afcdfe2e97d4d1ee41ea2db62eAnton Blanchard if (addr < 0x100000000UL) { 223fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spm = spt->low_prot; 224fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } else { 225fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spm = spt->protptrs[addr >> SBP_L3_SHIFT]; 226fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!spm) { 227fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spm = (u32 **)get_zeroed_page(GFP_KERNEL); 228fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!spm) 229fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras goto out; 230fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spt->protptrs[addr >> SBP_L3_SHIFT] = spm; 231fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 232fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 233fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spm += (addr >> SBP_L2_SHIFT) & (SBP_L2_COUNT - 1); 234fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spp = *spm; 235fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!spp) { 236fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spp = (u32 *)get_zeroed_page(GFP_KERNEL); 237fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (!spp) 238fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras goto out; 239fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras *spm = spp; 240fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 241fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spp += (addr >> PAGE_SHIFT) & (SBP_L1_COUNT - 1); 242fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 243fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras local_irq_disable(); 244fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras demote_segment_4k(mm, addr); 245fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras local_irq_enable(); 246fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 247fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras i = (addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 248fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras nw = PTRS_PER_PTE - i; 249fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (addr + (nw << PAGE_SHIFT) > next) 250fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras nw = (next - addr) >> PAGE_SHIFT; 251fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 252fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras up_write(&mm->mmap_sem); 253fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras err = -EFAULT; 254fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (__copy_from_user(spp, map, nw * sizeof(u32))) 255fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras goto out2; 256fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras map += nw; 257fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras down_write(&mm->mmap_sem); 258fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras 259fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras /* now flush any existing HPTEs for the range */ 260fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras hpte_flush_range(mm, addr, nw); 261fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras } 262fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras if (limit > spt->maxaddr) 263fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras spt->maxaddr = limit; 264fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras err = 0; 265fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras out: 266fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras up_write(&mm->mmap_sem); 267fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras out2: 268fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras return err; 269fa28237cfcc5827553044cbd6ee52e33692b0faaPaul Mackerras} 270