1f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens/* 2f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * arch/s390/mm/vmem.c 3f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * 4f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Copyright IBM Corp. 2006 5f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> 6f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens */ 7f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 8f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <linux/bootmem.h> 9f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <linux/pfn.h> 10f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <linux/mm.h> 11f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <linux/module.h> 12f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <linux/list.h> 1353492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer#include <linux/hugetlb.h> 145a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 15f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <asm/pgalloc.h> 16f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <asm/pgtable.h> 17f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <asm/setup.h> 18f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens#include <asm/tlbflush.h> 1953492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer#include <asm/sections.h> 20f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 21f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensstatic DEFINE_MUTEX(vmem_mutex); 22f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 23f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensstruct memory_segment { 24f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens struct list_head list; 25f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens unsigned long start; 26f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens unsigned long size; 27f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens}; 28f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 29f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensstatic LIST_HEAD(mem_segs); 30f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 3167060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstensstatic void __ref *vmem_alloc_pages(unsigned int order) 3267060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens{ 3367060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens if (slab_is_available()) 3467060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens return (void *)__get_free_pages(GFP_KERNEL, order); 3567060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens return alloc_bootmem_pages((1 << order) * PAGE_SIZE); 3667060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens} 3767060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens 3867060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstensstatic inline pud_t *vmem_pud_alloc(void) 395a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky{ 405a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky pud_t *pud = NULL; 415a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky 425a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky#ifdef CONFIG_64BIT 4367060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens pud = vmem_alloc_pages(2); 445a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky if (!pud) 455a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky return NULL; 468fc63658681f32e6e29f6d1138de933d7272e0ecHeiko Carstens clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); 475a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky#endif 485a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky return pud; 495a216a20837c5f5fa1ca4b8ae8991ffd96b08e6fMartin Schwidefsky} 50190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky 5167060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstensstatic inline pmd_t *vmem_pmd_alloc(void) 52f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 533610cce87af0693603db171d5b6f6735f5e3dc5bMartin Schwidefsky pmd_t *pmd = NULL; 54f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 553610cce87af0693603db171d5b6f6735f5e3dc5bMartin Schwidefsky#ifdef CONFIG_64BIT 5667060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens pmd = vmem_alloc_pages(2); 57f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!pmd) 58f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return NULL; 598fc63658681f32e6e29f6d1138de933d7272e0ecHeiko Carstens clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); 603610cce87af0693603db171d5b6f6735f5e3dc5bMartin Schwidefsky#endif 61f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return pmd; 62f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 63f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 64e5992f2e6c3829cd43dbc4438ee13dcd6506f7f3Martin Schwidefskystatic pte_t __ref *vmem_pte_alloc(unsigned long address) 65f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 66146e4b3c8b92071b18f0b2e6f47165bad4f9e825Martin Schwidefsky pte_t *pte; 67f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 68146e4b3c8b92071b18f0b2e6f47165bad4f9e825Martin Schwidefsky if (slab_is_available()) 69e5992f2e6c3829cd43dbc4438ee13dcd6506f7f3Martin Schwidefsky pte = (pte_t *) page_table_alloc(&init_mm, address); 70146e4b3c8b92071b18f0b2e6f47165bad4f9e825Martin Schwidefsky else 71146e4b3c8b92071b18f0b2e6f47165bad4f9e825Martin Schwidefsky pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); 72f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!pte) 73f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return NULL; 746af7eea2aee57b869f34eba0a94ef122fe90fbfdChristian Borntraeger clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, 756af7eea2aee57b869f34eba0a94ef122fe90fbfdChristian Borntraeger PTRS_PER_PTE * sizeof(pte_t)); 76f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return pte; 77f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 78f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 79f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens/* 80f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Add a physical memory range to the 1:1 mapping. 81f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens */ 8217f345808563d2f425b2b15d60c4a5b00112e9ebHeiko Carstensstatic int vmem_add_mem(unsigned long start, unsigned long size, int ro) 83f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 84f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens unsigned long address; 85f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pgd_t *pg_dir; 86190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pud_t *pu_dir; 87f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pmd_t *pm_dir; 88f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pte_t *pt_dir; 89f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pte_t pte; 90f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens int ret = -ENOMEM; 91f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 92f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens for (address = start; address < start + size; address += PAGE_SIZE) { 93f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pg_dir = pgd_offset_k(address); 94f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (pgd_none(*pg_dir)) { 95190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pu_dir = vmem_pud_alloc(); 96190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky if (!pu_dir) 97190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky goto out; 98b2fa47e6bf5148aa6dbf22ec79f18141b421eebaMartin Schwidefsky pgd_populate(&init_mm, pg_dir, pu_dir); 99190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky } 100190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky 101190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pu_dir = pud_offset(pg_dir, address); 102190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky if (pud_none(*pu_dir)) { 103f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pm_dir = vmem_pmd_alloc(); 104f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!pm_dir) 105f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out; 106b2fa47e6bf5148aa6dbf22ec79f18141b421eebaMartin Schwidefsky pud_populate(&init_mm, pu_dir, pm_dir); 107f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 108f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 10953492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer pte = mk_pte_phys(address, __pgprot(ro ? _PAGE_RO : 0)); 110190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pm_dir = pmd_offset(pu_dir, address); 11153492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer 11253492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer#ifdef __s390x__ 11353492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer if (MACHINE_HAS_HPAGE && !(address & ~HPAGE_MASK) && 11453492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer (address + HPAGE_SIZE <= start + size) && 11553492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer (address >= HPAGE_SIZE)) { 1166af7eea2aee57b869f34eba0a94ef122fe90fbfdChristian Borntraeger pte_val(pte) |= _SEGMENT_ENTRY_LARGE; 11753492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer pmd_val(*pm_dir) = pte_val(pte); 11853492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer address += HPAGE_SIZE - PAGE_SIZE; 11953492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer continue; 12053492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer } 12153492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer#endif 122f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (pmd_none(*pm_dir)) { 123e5992f2e6c3829cd43dbc4438ee13dcd6506f7f3Martin Schwidefsky pt_dir = vmem_pte_alloc(address); 124f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!pt_dir) 125f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out; 126b2fa47e6bf5148aa6dbf22ec79f18141b421eebaMartin Schwidefsky pmd_populate(&init_mm, pm_dir, pt_dir); 127f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 128f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 129f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pt_dir = pte_offset_kernel(pm_dir, address); 130c1821c2e9711adc3cd298a16b7237c92a2cee78dGerald Schaefer *pt_dir = pte; 131f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 132f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens ret = 0; 133f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensout: 134f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens flush_tlb_kernel_range(start, start + size); 135f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return ret; 136f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 137f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 138f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens/* 139f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Remove a physical memory range from the 1:1 mapping. 140f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Currently only invalidates page table entries. 141f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens */ 142f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensstatic void vmem_remove_range(unsigned long start, unsigned long size) 143f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 144f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens unsigned long address; 145f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pgd_t *pg_dir; 146190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pud_t *pu_dir; 147f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pmd_t *pm_dir; 148f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pte_t *pt_dir; 149f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pte_t pte; 150f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 151f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pte_val(pte) = _PAGE_TYPE_EMPTY; 152f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens for (address = start; address < start + size; address += PAGE_SIZE) { 153f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pg_dir = pgd_offset_k(address); 154190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pu_dir = pud_offset(pg_dir, address); 155190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky if (pud_none(*pu_dir)) 156f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens continue; 157190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pm_dir = pmd_offset(pu_dir, address); 158f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (pmd_none(*pm_dir)) 159f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens continue; 16053492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer 16153492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer if (pmd_huge(*pm_dir)) { 162b2fa47e6bf5148aa6dbf22ec79f18141b421eebaMartin Schwidefsky pmd_clear(pm_dir); 16353492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer address += HPAGE_SIZE - PAGE_SIZE; 16453492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer continue; 16553492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer } 16653492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer 167f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pt_dir = pte_offset_kernel(pm_dir, address); 168c1821c2e9711adc3cd298a16b7237c92a2cee78dGerald Schaefer *pt_dir = pte; 169f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 170f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens flush_tlb_kernel_range(start, start + size); 171f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 172f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 173f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens/* 174f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Add a backed mem_map array to the virtual mem_map array. 175f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens */ 17617f345808563d2f425b2b15d60c4a5b00112e9ebHeiko Carstensint __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) 177f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 178f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens unsigned long address, start_addr, end_addr; 179f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pgd_t *pg_dir; 180190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pud_t *pu_dir; 181f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pmd_t *pm_dir; 182f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pte_t *pt_dir; 183f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pte_t pte; 184f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens int ret = -ENOMEM; 185f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 18617f345808563d2f425b2b15d60c4a5b00112e9ebHeiko Carstens start_addr = (unsigned long) start; 18717f345808563d2f425b2b15d60c4a5b00112e9ebHeiko Carstens end_addr = (unsigned long) (start + nr); 188f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 189f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens for (address = start_addr; address < end_addr; address += PAGE_SIZE) { 190f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pg_dir = pgd_offset_k(address); 191f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (pgd_none(*pg_dir)) { 192190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pu_dir = vmem_pud_alloc(); 193190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky if (!pu_dir) 194190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky goto out; 195b2fa47e6bf5148aa6dbf22ec79f18141b421eebaMartin Schwidefsky pgd_populate(&init_mm, pg_dir, pu_dir); 196190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky } 197190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky 198190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pu_dir = pud_offset(pg_dir, address); 199190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky if (pud_none(*pu_dir)) { 200f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pm_dir = vmem_pmd_alloc(); 201f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!pm_dir) 202f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out; 203b2fa47e6bf5148aa6dbf22ec79f18141b421eebaMartin Schwidefsky pud_populate(&init_mm, pu_dir, pm_dir); 204f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 205f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 206190a1d722a59725706daf832bc8a511ed62f249dMartin Schwidefsky pm_dir = pmd_offset(pu_dir, address); 207f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (pmd_none(*pm_dir)) { 208e5992f2e6c3829cd43dbc4438ee13dcd6506f7f3Martin Schwidefsky pt_dir = vmem_pte_alloc(address); 209f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!pt_dir) 210f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out; 211b2fa47e6bf5148aa6dbf22ec79f18141b421eebaMartin Schwidefsky pmd_populate(&init_mm, pm_dir, pt_dir); 212f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 213f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 214f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pt_dir = pte_offset_kernel(pm_dir, address); 215f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (pte_none(*pt_dir)) { 216f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens unsigned long new_page; 217f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 21867060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens new_page =__pa(vmem_alloc_pages(0)); 219f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!new_page) 220f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out; 221f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL); 222c1821c2e9711adc3cd298a16b7237c92a2cee78dGerald Schaefer *pt_dir = pte; 223f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 224f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 22567060d9c1f5d91c917cc51bed464cb5638eaddbcHeiko Carstens memset(start, 0, nr * sizeof(struct page)); 226f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens ret = 0; 227f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensout: 228f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens flush_tlb_kernel_range(start_addr, end_addr); 229f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return ret; 230f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 231f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 232f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens/* 233f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Add memory segment to the segment list if it doesn't overlap with 234f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * an already present segment. 235f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens */ 236f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensstatic int insert_memory_segment(struct memory_segment *seg) 237f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 238f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens struct memory_segment *tmp; 239f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 240ee0ddadd086e25503f81be551c43f66472300acdHeiko Carstens if (seg->start + seg->size > VMEM_MAX_PHYS || 241f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens seg->start + seg->size < seg->start) 242f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return -ERANGE; 243f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 244f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens list_for_each_entry(tmp, &mem_segs, list) { 245f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (seg->start >= tmp->start + tmp->size) 246f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens continue; 247f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (seg->start + seg->size <= tmp->start) 248f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens continue; 249f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return -ENOSPC; 250f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 251f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens list_add(&seg->list, &mem_segs); 252f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return 0; 253f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 254f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 255f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens/* 256f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Remove memory segment from the segment list. 257f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens */ 258f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensstatic void remove_memory_segment(struct memory_segment *seg) 259f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 260f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens list_del(&seg->list); 261f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 262f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 263f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensstatic void __remove_shared_memory(struct memory_segment *seg) 264f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 265f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens remove_memory_segment(seg); 266f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens vmem_remove_range(seg->start, seg->size); 267f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 268f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 26917f345808563d2f425b2b15d60c4a5b00112e9ebHeiko Carstensint vmem_remove_mapping(unsigned long start, unsigned long size) 270f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 271f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens struct memory_segment *seg; 272f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens int ret; 273f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 274f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens mutex_lock(&vmem_mutex); 275f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 276f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens ret = -ENOENT; 277f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens list_for_each_entry(seg, &mem_segs, list) { 278f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (seg->start == start && seg->size == size) 279f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens break; 280f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 281f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 282f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (seg->start != start || seg->size != size) 283f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out; 284f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 285f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens ret = 0; 286f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens __remove_shared_memory(seg); 287f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens kfree(seg); 288f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensout: 289f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens mutex_unlock(&vmem_mutex); 290f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return ret; 291f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 292f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 29317f345808563d2f425b2b15d60c4a5b00112e9ebHeiko Carstensint vmem_add_mapping(unsigned long start, unsigned long size) 294f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 295f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens struct memory_segment *seg; 296f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens int ret; 297f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 298f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens mutex_lock(&vmem_mutex); 299f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens ret = -ENOMEM; 300f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens seg = kzalloc(sizeof(*seg), GFP_KERNEL); 301f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!seg) 302f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out; 303f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens seg->start = start; 304f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens seg->size = size; 305f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 306f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens ret = insert_memory_segment(seg); 307f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (ret) 308f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out_free; 309f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 31053492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer ret = vmem_add_mem(start, size, 0); 311f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (ret) 312f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out_remove; 313f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens goto out; 314f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 315f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensout_remove: 316f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens __remove_shared_memory(seg); 317f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensout_free: 318f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens kfree(seg); 319f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensout: 320f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens mutex_unlock(&vmem_mutex); 321f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return ret; 322f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 323f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 324f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens/* 325f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * map whole physical memory to virtual memory (identity mapping) 3265fd9c6e214547a32d3da6ee4284c79004d667bc8Christian Borntraeger * we reserve enough space in the vmalloc area for vmemmap to hotplug 3275fd9c6e214547a32d3da6ee4284c79004d667bc8Christian Borntraeger * additional memory segments. 328f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens */ 329f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensvoid __init vmem_map_init(void) 330f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 33153492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer unsigned long ro_start, ro_end; 33253492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer unsigned long start, end; 333f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens int i; 334f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 33553492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer ro_start = ((unsigned long)&_stext) & PAGE_MASK; 33653492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer ro_end = PFN_ALIGN((unsigned long)&_eshared); 33753492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) { 33860a0c68df2632feaa4a986af084650d1165d89c5Michael Holzheu if (memory_chunk[i].type == CHUNK_CRASHK || 33960a0c68df2632feaa4a986af084650d1165d89c5Michael Holzheu memory_chunk[i].type == CHUNK_OLDMEM) 34060a0c68df2632feaa4a986af084650d1165d89c5Michael Holzheu continue; 34153492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer start = memory_chunk[i].addr; 34253492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer end = memory_chunk[i].addr + memory_chunk[i].size; 34353492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer if (start >= ro_end || end <= ro_start) 34453492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(start, end - start, 0); 34553492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer else if (start >= ro_start && end <= ro_end) 34653492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(start, end - start, 1); 34753492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer else if (start >= ro_start) { 34853492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(start, ro_end - start, 1); 34953492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(ro_end, end - ro_end, 0); 35053492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer } else if (end < ro_end) { 35153492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(start, ro_start - start, 0); 35253492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(ro_start, end - ro_start, 1); 35353492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer } else { 35453492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(start, ro_start - start, 0); 35553492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(ro_start, ro_end - ro_start, 1); 35653492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer vmem_add_mem(ro_end, end - ro_end, 0); 35753492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer } 35853492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer } 359f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 360f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 361f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens/* 362f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * Convert memory chunk array to a memory segment list so there is a single 363f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens * list that contains both r/w memory and shared memory segments. 364f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens */ 365f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstensstatic int __init vmem_convert_memory_chunk(void) 366f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens{ 367f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens struct memory_segment *seg; 368f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens int i; 369f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 370f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens mutex_lock(&vmem_mutex); 3719f4b0ba81f158df459fa2cfc98ab1475c090f29cHeiko Carstens for (i = 0; i < MEMORY_CHUNKS; i++) { 372f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!memory_chunk[i].size) 373f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens continue; 37460a0c68df2632feaa4a986af084650d1165d89c5Michael Holzheu if (memory_chunk[i].type == CHUNK_CRASHK || 37560a0c68df2632feaa4a986af084650d1165d89c5Michael Holzheu memory_chunk[i].type == CHUNK_OLDMEM) 37660a0c68df2632feaa4a986af084650d1165d89c5Michael Holzheu continue; 377f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens seg = kzalloc(sizeof(*seg), GFP_KERNEL); 378f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens if (!seg) 379f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens panic("Out of memory...\n"); 380f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens seg->start = memory_chunk[i].addr; 381f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens seg->size = memory_chunk[i].size; 382f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens insert_memory_segment(seg); 383f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens } 384f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens mutex_unlock(&vmem_mutex); 385f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens return 0; 386f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens} 387f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstens 388f4eb07c17df2e6cf9bd58bfcd9cc9e05e9489d07Heiko Carstenscore_initcall(vmem_convert_memory_chunk); 389