114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras/* 214cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * PowerPC version 314cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 414cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * 514cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 614cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * and Cort Dougan (PReP) (cort@cs.nmt.edu) 714cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * Copyright (C) 1996 Paul Mackerras 814cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * 914cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * Derived from "arch/i386/mm/init.c" 1014cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 1114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * 1214cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * Dave Engebretsen <engebret@us.ibm.com> 1314cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * Rework for PPC64 port. 1414cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * 1514cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * This program is free software; you can redistribute it and/or 1614cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * modify it under the terms of the GNU General Public License 1714cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * as published by the Free Software Foundation; either version 1814cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * 2 of the License, or (at your option) any later version. 1914cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras * 2014cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras */ 2114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras 22cec08e7a948326b01555be6311480aa08e637de2Benjamin Herrenschmidt#undef DEBUG 23cec08e7a948326b01555be6311480aa08e637de2Benjamin Herrenschmidt 2414cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/signal.h> 2514cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/sched.h> 2614cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/kernel.h> 2714cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/errno.h> 2814cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/string.h> 2914cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/types.h> 3014cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/mman.h> 3114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/mm.h> 3214cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/swap.h> 3314cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/stddef.h> 3414cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/vmalloc.h> 3514cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/init.h> 3614cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/delay.h> 3714cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/bootmem.h> 3814cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/highmem.h> 3914cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/idr.h> 4014cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/nodemask.h> 4114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <linux/module.h> 42c9cf55285e87ac423c45d9efca750d3f50234d10Randy Dunlap#include <linux/poison.h> 4395f72d1ed41a66f1c1c29c24d479de81a0bea36fYinghai Lu#include <linux/memblock.h> 44a4fe3ce7699bfe1bd88f816b55d42d8fe1dac655David Gibson#include <linux/hugetlb.h> 455a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 4614cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras 4714cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/pgalloc.h> 4814cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/page.h> 4914cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/prom.h> 5014cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/rtas.h> 5114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/io.h> 5214cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/mmu_context.h> 5314cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/pgtable.h> 5414cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/mmu.h> 5514cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/uaccess.h> 5614cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/smp.h> 5714cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/machdep.h> 5814cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/tlb.h> 5914cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/eeh.h> 6014cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/processor.h> 6114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/mmzone.h> 6214cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/cputable.h> 6314cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/sections.h> 6414cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/iommu.h> 6514cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#include <asm/vdso.h> 66800fc3eeb0eed3bf98d621c0da24d68cabcf6526David Gibson 67800fc3eeb0eed3bf98d621c0da24d68cabcf6526David Gibson#include "mmu_decl.h" 6814cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras 69944916858a430a0627e483657d4cfa2cd2dfb4f7Benjamin Herrenschmidt#ifdef CONFIG_PPC_STD_MMU_64 7014cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#if PGTABLE_RANGE > USER_VSID_RANGE 7114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#warning Limited user VSID range means pagetable space is wasted 7214cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#endif 7314cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras 7414cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE) 7514cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#warning TASK_SIZE is smaller than it needs to be. 7614cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras#endif 77944916858a430a0627e483657d4cfa2cd2dfb4f7Benjamin Herrenschmidt#endif /* CONFIG_PPC_STD_MMU_64 */ 7814cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras 7937dd2badcfcec35f5e21a0926968d77a404f03c3Kumar Galaphys_addr_t memstart_addr = ~0; 8079c3095fb39964d0d44368cbbb4eff5b52c43d2cSonny RaoEXPORT_SYMBOL_GPL(memstart_addr); 8137dd2badcfcec35f5e21a0926968d77a404f03c3Kumar Galaphys_addr_t kernstart_addr; 8279c3095fb39964d0d44368cbbb4eff5b52c43d2cSonny RaoEXPORT_SYMBOL_GPL(kernstart_addr); 83d7917ba7051e3fd12ebe2d5a09b29fb3a2b38190Kumar Gala 8451cc50685a4275c6a02653670af9f108a64e01cfAlexey Dobriyanstatic void pgd_ctor(void *addr) 8514cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras{ 8651cc50685a4275c6a02653670af9f108a64e01cfAlexey Dobriyan memset(addr, 0, PGD_TABLE_SIZE); 8751cc50685a4275c6a02653670af9f108a64e01cfAlexey Dobriyan} 8851cc50685a4275c6a02653670af9f108a64e01cfAlexey Dobriyan 8951cc50685a4275c6a02653670af9f108a64e01cfAlexey Dobriyanstatic void pmd_ctor(void *addr) 9051cc50685a4275c6a02653670af9f108a64e01cfAlexey Dobriyan{ 91f940f5289873af2ad2c4e73f88c24ad2b8fe3f87Aneesh Kumar K.V#ifdef CONFIG_TRANSPARENT_HUGEPAGE 92f940f5289873af2ad2c4e73f88c24ad2b8fe3f87Aneesh Kumar K.V memset(addr, 0, PMD_TABLE_SIZE * 2); 93f940f5289873af2ad2c4e73f88c24ad2b8fe3f87Aneesh Kumar K.V#else 9451cc50685a4275c6a02653670af9f108a64e01cfAlexey Dobriyan memset(addr, 0, PMD_TABLE_SIZE); 95f940f5289873af2ad2c4e73f88c24ad2b8fe3f87Aneesh Kumar K.V#endif 9614cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras} 9714cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras 98a0668cdc154e54bf0c85182e0535eea237d53146David Gibsonstruct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE]; 99a0668cdc154e54bf0c85182e0535eea237d53146David Gibson 100a0668cdc154e54bf0c85182e0535eea237d53146David Gibson/* 101a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * Create a kmem_cache() for pagetables. This is not used for PTE 102a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * pages - they're linked to struct page, come from the normal free 103a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * pages pool and have a different entry size (see real_pte_t) to 104a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * everything else. Caches created by this function are used for all 105a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * the higher level pagetables, and for hugepage pagetables. 106a0668cdc154e54bf0c85182e0535eea237d53146David Gibson */ 107a0668cdc154e54bf0c85182e0535eea237d53146David Gibsonvoid pgtable_cache_add(unsigned shift, void (*ctor)(void *)) 108a0668cdc154e54bf0c85182e0535eea237d53146David Gibson{ 109a0668cdc154e54bf0c85182e0535eea237d53146David Gibson char *name; 110a0668cdc154e54bf0c85182e0535eea237d53146David Gibson unsigned long table_size = sizeof(void *) << shift; 111a0668cdc154e54bf0c85182e0535eea237d53146David Gibson unsigned long align = table_size; 112a0668cdc154e54bf0c85182e0535eea237d53146David Gibson 113a0668cdc154e54bf0c85182e0535eea237d53146David Gibson /* When batching pgtable pointers for RCU freeing, we store 114a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * the index size in the low bits. Table alignment must be 115a4fe3ce7699bfe1bd88f816b55d42d8fe1dac655David Gibson * big enough to fit it. 116a4fe3ce7699bfe1bd88f816b55d42d8fe1dac655David Gibson * 117a4fe3ce7699bfe1bd88f816b55d42d8fe1dac655David Gibson * Likewise, hugeapge pagetable pointers contain a (different) 118a4fe3ce7699bfe1bd88f816b55d42d8fe1dac655David Gibson * shift value in the low bits. All tables must be aligned so 119a4fe3ce7699bfe1bd88f816b55d42d8fe1dac655David Gibson * as to leave enough 0 bits in the address to contain it. */ 120a4fe3ce7699bfe1bd88f816b55d42d8fe1dac655David Gibson unsigned long minalign = max(MAX_PGTABLE_INDEX_SIZE + 1, 121a4fe3ce7699bfe1bd88f816b55d42d8fe1dac655David Gibson HUGEPD_SHIFT_MASK + 1); 122a0668cdc154e54bf0c85182e0535eea237d53146David Gibson struct kmem_cache *new; 123a0668cdc154e54bf0c85182e0535eea237d53146David Gibson 124a0668cdc154e54bf0c85182e0535eea237d53146David Gibson /* It would be nice if this was a BUILD_BUG_ON(), but at the 125a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * moment, gcc doesn't seem to recognize is_power_of_2 as a 126a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * constant expression, so so much for that. */ 127a0668cdc154e54bf0c85182e0535eea237d53146David Gibson BUG_ON(!is_power_of_2(minalign)); 128a0668cdc154e54bf0c85182e0535eea237d53146David Gibson BUG_ON((shift < 1) || (shift > MAX_PGTABLE_INDEX_SIZE)); 129a0668cdc154e54bf0c85182e0535eea237d53146David Gibson 130a0668cdc154e54bf0c85182e0535eea237d53146David Gibson if (PGT_CACHE(shift)) 131a0668cdc154e54bf0c85182e0535eea237d53146David Gibson return; /* Already have a cache of this size */ 132a0668cdc154e54bf0c85182e0535eea237d53146David Gibson 133a0668cdc154e54bf0c85182e0535eea237d53146David Gibson align = max_t(unsigned long, align, minalign); 134a0668cdc154e54bf0c85182e0535eea237d53146David Gibson name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift); 135a0668cdc154e54bf0c85182e0535eea237d53146David Gibson new = kmem_cache_create(name, table_size, align, 0, ctor); 136cf9427b85e90bb1ff90e2397ff419691d983c68bAneesh Kumar K.V pgtable_cache[shift - 1] = new; 137a0668cdc154e54bf0c85182e0535eea237d53146David Gibson pr_debug("Allocated pgtable cache for order %d\n", shift); 138a0668cdc154e54bf0c85182e0535eea237d53146David Gibson} 139a0668cdc154e54bf0c85182e0535eea237d53146David Gibson 14014cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras 14114cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerrasvoid pgtable_cache_init(void) 14214cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras{ 143a0668cdc154e54bf0c85182e0535eea237d53146David Gibson pgtable_cache_add(PGD_INDEX_SIZE, pgd_ctor); 144f940f5289873af2ad2c4e73f88c24ad2b8fe3f87Aneesh Kumar K.V pgtable_cache_add(PMD_CACHE_INDEX, pmd_ctor); 145f940f5289873af2ad2c4e73f88c24ad2b8fe3f87Aneesh Kumar K.V if (!PGT_CACHE(PGD_INDEX_SIZE) || !PGT_CACHE(PMD_CACHE_INDEX)) 146a0668cdc154e54bf0c85182e0535eea237d53146David Gibson panic("Couldn't allocate pgtable caches"); 147a0668cdc154e54bf0c85182e0535eea237d53146David Gibson /* In all current configs, when the PUD index exists it's the 148a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * same size as either the pgd or pmd index. Verify that the 149a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * initialization above has also created a PUD cache. This 150a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * will need re-examiniation if we add new possibilities for 151a0668cdc154e54bf0c85182e0535eea237d53146David Gibson * the pagetable layout. */ 152a0668cdc154e54bf0c85182e0535eea237d53146David Gibson BUG_ON(PUD_INDEX_SIZE && !PGT_CACHE(PUD_INDEX_SIZE)); 15314cf11af6cf608eb8c23e989ddb17a715ddce109Paul Mackerras} 154d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 155d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft#ifdef CONFIG_SPARSEMEM_VMEMMAP 156d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft/* 157d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft * Given an address within the vmemmap, determine the pfn of the page that 158d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft * represents the start of the section it is within. Note that we have to 159d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft * do this by hand as the proffered address may not be correctly aligned. 160d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft * Subtraction of non-aligned pointers produces undefined results. 161d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft */ 16209de9ff872bca25ba717a7075c9ee49b1bdb56d2Michael Ellermanstatic unsigned long __meminit vmemmap_section_start(unsigned long page) 163d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft{ 164d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft unsigned long offset = page - ((unsigned long)(vmemmap)); 165d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 166d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft /* Return the pfn of the start of the section. */ 167d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft return (offset / sizeof(struct page)) & PAGE_SECTION_MASK; 168d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft} 169d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 170d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft/* 171d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft * Check if this vmemmap page is already initialised. If any section 172d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft * which overlaps this vmemmap page is initialised then this page is 173d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft * initialised already. 174d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft */ 17509de9ff872bca25ba717a7075c9ee49b1bdb56d2Michael Ellermanstatic int __meminit vmemmap_populated(unsigned long start, int page_size) 176d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft{ 177d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft unsigned long end = start + page_size; 17816a05bff128de196fc17edd2beaa40d0f07ae04aLi Zhong start = (unsigned long)(pfn_to_page(vmemmap_section_start(start))); 179d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 180d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page))) 18116a05bff128de196fc17edd2beaa40d0f07ae04aLi Zhong if (pfn_valid(page_to_pfn((struct page *)start))) 182d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft return 1; 183d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 184d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft return 0; 185d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft} 186d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 18732a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt/* On hash-based CPUs, the vmemmap is bolted in the hash table. 18832a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt * 18932a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt * On Book3E CPUs, the vmemmap is currently mapped in the top half of 19032a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt * the vmalloc space using normal page tables, though the size of 19132a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt * pages encoded in the PTEs can be different 19232a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt */ 19332a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt 19432a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt#ifdef CONFIG_PPC_BOOK3E 19532a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidtstatic void __meminit vmemmap_create_mapping(unsigned long start, 19632a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt unsigned long page_size, 19732a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt unsigned long phys) 19832a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt{ 19932a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt /* Create a PTE encoding without page size */ 20032a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt unsigned long i, flags = _PAGE_PRESENT | _PAGE_ACCESSED | 20132a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt _PAGE_KERNEL_RW; 20232a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt 20332a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt /* PTEs only contain page size encodings up to 32M */ 20432a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt BUG_ON(mmu_psize_defs[mmu_vmemmap_psize].enc > 0xf); 20532a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt 20632a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt /* Encode the size in the PTE */ 20732a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt flags |= mmu_psize_defs[mmu_vmemmap_psize].enc << 8; 20832a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt 20932a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt /* For each PTE for that area, map things. Note that we don't 21032a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt * increment phys because all PTEs are of the large size and 21132a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt * thus must have the low bits clear 21232a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt */ 21332a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt for (i = 0; i < page_size; i += PAGE_SIZE) 21432a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt BUG_ON(map_kernel_page(start + i, phys, flags)); 21532a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt} 216ed5694a8464a133582c632f8081324408bcc486dLi Zhong 217ed5694a8464a133582c632f8081324408bcc486dLi Zhong#ifdef CONFIG_MEMORY_HOTPLUG 218ed5694a8464a133582c632f8081324408bcc486dLi Zhongstatic void vmemmap_remove_mapping(unsigned long start, 219ed5694a8464a133582c632f8081324408bcc486dLi Zhong unsigned long page_size) 220ed5694a8464a133582c632f8081324408bcc486dLi Zhong{ 221ed5694a8464a133582c632f8081324408bcc486dLi Zhong} 222ed5694a8464a133582c632f8081324408bcc486dLi Zhong#endif 22332a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt#else /* CONFIG_PPC_BOOK3E */ 22432a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidtstatic void __meminit vmemmap_create_mapping(unsigned long start, 22532a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt unsigned long page_size, 22632a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt unsigned long phys) 22732a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt{ 22832a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt int mapped = htab_bolt_mapping(start, start + page_size, phys, 22983d5e64b7efa7f39b10ff5e92792e807a720289cAneesh Kumar K.V pgprot_val(PAGE_KERNEL), 23083d5e64b7efa7f39b10ff5e92792e807a720289cAneesh Kumar K.V mmu_vmemmap_psize, 23132a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt mmu_kernel_ssize); 23232a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt BUG_ON(mapped < 0); 23332a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt} 234ed5694a8464a133582c632f8081324408bcc486dLi Zhong 235ed5694a8464a133582c632f8081324408bcc486dLi Zhong#ifdef CONFIG_MEMORY_HOTPLUG 236ed5694a8464a133582c632f8081324408bcc486dLi Zhongstatic void vmemmap_remove_mapping(unsigned long start, 237ed5694a8464a133582c632f8081324408bcc486dLi Zhong unsigned long page_size) 238ed5694a8464a133582c632f8081324408bcc486dLi Zhong{ 239ed5694a8464a133582c632f8081324408bcc486dLi Zhong int mapped = htab_remove_mapping(start, start + page_size, 240ed5694a8464a133582c632f8081324408bcc486dLi Zhong mmu_vmemmap_psize, 241ed5694a8464a133582c632f8081324408bcc486dLi Zhong mmu_kernel_ssize); 242ed5694a8464a133582c632f8081324408bcc486dLi Zhong BUG_ON(mapped < 0); 243ed5694a8464a133582c632f8081324408bcc486dLi Zhong} 244ed5694a8464a133582c632f8081324408bcc486dLi Zhong#endif 245ed5694a8464a133582c632f8081324408bcc486dLi Zhong 24632a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt#endif /* CONFIG_PPC_BOOK3E */ 24732a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt 24891eea67c6d8704396a98226508c56a8501e141e3Mark Nelsonstruct vmemmap_backing *vmemmap_list; 249bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhongstatic struct vmemmap_backing *next; 250bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhongstatic int num_left; 251bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhongstatic int num_freed; 25291eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 25391eea67c6d8704396a98226508c56a8501e141e3Mark Nelsonstatic __meminit struct vmemmap_backing * vmemmap_list_alloc(int node) 25491eea67c6d8704396a98226508c56a8501e141e3Mark Nelson{ 255bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong struct vmemmap_backing *vmem_back; 256bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong /* get from freed entries first */ 257bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong if (num_freed) { 258bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong num_freed--; 259bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong vmem_back = next; 260bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong next = next->list; 261bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong 262bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong return vmem_back; 263bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong } 26491eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 26591eea67c6d8704396a98226508c56a8501e141e3Mark Nelson /* allocate a page when required and hand out chunks */ 266bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong if (!num_left) { 26791eea67c6d8704396a98226508c56a8501e141e3Mark Nelson next = vmemmap_alloc_block(PAGE_SIZE, node); 26891eea67c6d8704396a98226508c56a8501e141e3Mark Nelson if (unlikely(!next)) { 26991eea67c6d8704396a98226508c56a8501e141e3Mark Nelson WARN_ON(1); 27091eea67c6d8704396a98226508c56a8501e141e3Mark Nelson return NULL; 27191eea67c6d8704396a98226508c56a8501e141e3Mark Nelson } 27291eea67c6d8704396a98226508c56a8501e141e3Mark Nelson num_left = PAGE_SIZE / sizeof(struct vmemmap_backing); 27391eea67c6d8704396a98226508c56a8501e141e3Mark Nelson } 27491eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 27591eea67c6d8704396a98226508c56a8501e141e3Mark Nelson num_left--; 27691eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 27791eea67c6d8704396a98226508c56a8501e141e3Mark Nelson return next++; 27891eea67c6d8704396a98226508c56a8501e141e3Mark Nelson} 27991eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 28091eea67c6d8704396a98226508c56a8501e141e3Mark Nelsonstatic __meminit void vmemmap_list_populate(unsigned long phys, 28191eea67c6d8704396a98226508c56a8501e141e3Mark Nelson unsigned long start, 28291eea67c6d8704396a98226508c56a8501e141e3Mark Nelson int node) 28391eea67c6d8704396a98226508c56a8501e141e3Mark Nelson{ 28491eea67c6d8704396a98226508c56a8501e141e3Mark Nelson struct vmemmap_backing *vmem_back; 28591eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 28691eea67c6d8704396a98226508c56a8501e141e3Mark Nelson vmem_back = vmemmap_list_alloc(node); 28791eea67c6d8704396a98226508c56a8501e141e3Mark Nelson if (unlikely(!vmem_back)) { 28891eea67c6d8704396a98226508c56a8501e141e3Mark Nelson WARN_ON(1); 28991eea67c6d8704396a98226508c56a8501e141e3Mark Nelson return; 29091eea67c6d8704396a98226508c56a8501e141e3Mark Nelson } 29191eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 29291eea67c6d8704396a98226508c56a8501e141e3Mark Nelson vmem_back->phys = phys; 29391eea67c6d8704396a98226508c56a8501e141e3Mark Nelson vmem_back->virt_addr = start; 29491eea67c6d8704396a98226508c56a8501e141e3Mark Nelson vmem_back->list = vmemmap_list; 29591eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 29691eea67c6d8704396a98226508c56a8501e141e3Mark Nelson vmemmap_list = vmem_back; 29791eea67c6d8704396a98226508c56a8501e141e3Mark Nelson} 29891eea67c6d8704396a98226508c56a8501e141e3Mark Nelson 29971b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhongint __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 30071b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong{ 30171b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift; 30271b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 30371b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong /* Align to the page size of the linear mapping. */ 30471b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong start = _ALIGN_DOWN(start, page_size); 30571b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 30671b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong pr_debug("vmemmap_populate %lx..%lx, node %d\n", start, end, node); 30771b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 30871b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong for (; start < end; start += page_size) { 30971b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong void *p; 31071b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 31171b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong if (vmemmap_populated(start, page_size)) 31271b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong continue; 31371b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 31471b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong p = vmemmap_alloc_block(page_size, node); 31571b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong if (!p) 31671b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong return -ENOMEM; 31771b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 31871b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong vmemmap_list_populate(__pa(p), start, node); 31971b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 32071b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong pr_debug(" * %016lx..%016lx allocated at %p\n", 32171b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong start, start + page_size, p); 32271b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 32371b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong vmemmap_create_mapping(start, page_size, __pa(p)); 32471b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong } 32571b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 32671b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong return 0; 32771b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong} 32871b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 32971b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong#ifdef CONFIG_MEMORY_HOTPLUG 330bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhongstatic unsigned long vmemmap_list_free(unsigned long start) 331bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong{ 332bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong struct vmemmap_backing *vmem_back, *vmem_back_prev; 333bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong 334bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong vmem_back_prev = vmem_back = vmemmap_list; 335bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong 336bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong /* look for it with prev pointer recorded */ 337bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong for (; vmem_back; vmem_back = vmem_back->list) { 338bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong if (vmem_back->virt_addr == start) 339bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong break; 340bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong vmem_back_prev = vmem_back; 341bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong } 342bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong 343bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong if (unlikely(!vmem_back)) { 344bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong WARN_ON(1); 345bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong return 0; 346bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong } 347bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong 348bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong /* remove it from vmemmap_list */ 349bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong if (vmem_back == vmemmap_list) /* remove head */ 350bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong vmemmap_list = vmem_back->list; 351bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong else 352bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong vmem_back_prev->list = vmem_back->list; 353bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong 354bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong /* next point to this freed entry */ 355bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong vmem_back->list = next; 356bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong next = vmem_back; 357bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong num_freed++; 358bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong 359bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong return vmem_back->phys; 360bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong} 361bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong 36271b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhongvoid __ref vmemmap_free(unsigned long start, unsigned long end) 363d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft{ 364cec08e7a948326b01555be6311480aa08e637de2Benjamin Herrenschmidt unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift; 365d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 366d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft start = _ALIGN_DOWN(start, page_size); 367d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 36871b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong pr_debug("vmemmap_free %lx...%lx\n", start, end); 36932a74949b7337726e76d69f51c48715431126c6cBenjamin Herrenschmidt 370d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft for (; start < end; start += page_size) { 37171b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong unsigned long addr; 372d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 37371b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong /* 37471b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong * the section has already be marked as invalid, so 37571b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong * vmemmap_populated() true means some other sections still 37671b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong * in this page, so skip it. 37771b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong */ 378d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft if (vmemmap_populated(start, page_size)) 379d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft continue; 380d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft 38171b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong addr = vmemmap_list_free(start); 38271b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong if (addr) { 38371b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong struct page *page = pfn_to_page(addr >> PAGE_SHIFT); 38471b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 38571b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong if (PageReserved(page)) { 38671b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong /* allocated from bootmem */ 38771b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong if (page_size < PAGE_SIZE) { 38871b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong /* 38971b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong * this shouldn't happen, but if it is 39071b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong * the case, leave the memory there 39171b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong */ 39271b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong WARN_ON_ONCE(1); 39371b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong } else { 39471b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong unsigned int nr_pages = 39571b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 1 << get_order(page_size); 39671b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong while (nr_pages--) 39771b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong free_reserved_page(page++); 39871b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong } 39971b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong } else 40071b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong free_pages((unsigned long)(__va(addr)), 40171b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong get_order(page_size)); 40271b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong 40371b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong vmemmap_remove_mapping(start, page_size); 40471b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong } 405d29eff7bca60c9ee401d691d4562a4abca8de543Andy Whitcroft } 4060197518cd3672029618a16a57597946a094ac7a8Tang Chen} 40771b0bfe4f1608dbabb54a1e964046267a2c7f7b3Li Zhong#endif 408f7e3334a6bcb42e7295a9bd9cb36ca4e6e4e66b4Nathan Fontenotvoid register_page_bootmem_memmap(unsigned long section_nr, 409f7e3334a6bcb42e7295a9bd9cb36ca4e6e4e66b4Nathan Fontenot struct page *start_page, unsigned long size) 410f7e3334a6bcb42e7295a9bd9cb36ca4e6e4e66b4Nathan Fontenot{ 411f7e3334a6bcb42e7295a9bd9cb36ca4e6e4e66b4Nathan Fontenot} 412cd3db0c4ca3d237e7ad20f7107216e575705d2b0Benjamin Herrenschmidt 4138e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy/* 4148e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * We do not have access to the sparsemem vmemmap, so we fallback to 4158e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * walking the list of sparsemem blocks which we already maintain for 4168e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * the sake of crashdump. In the long run, we might want to maintain 4178e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * a tree if performance of that linear walk becomes a problem. 4188e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * 4198e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * realmode_pfn_to_page functions can fail due to: 4208e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * 1) As real sparsemem blocks do not lay in RAM continously (they 4218e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * are in virtual address space which is not available in the real mode), 4228e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * the requested page struct can be split between blocks so get_page/put_page 4238e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * may fail. 4248e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * 2) When huge pages are used, the get_page/put_page API will fail 4258e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * in real mode as the linked addresses in the page struct are virtual 4268e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy * too. 4278e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy */ 4288e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiystruct page *realmode_pfn_to_page(unsigned long pfn) 4298e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy{ 4308e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy struct vmemmap_backing *vmem_back; 4318e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy struct page *page; 4328e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift; 4338e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy unsigned long pg_va = (unsigned long) pfn_to_page(pfn); 4348e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy 4358e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy for (vmem_back = vmemmap_list; vmem_back; vmem_back = vmem_back->list) { 4368e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy if (pg_va < vmem_back->virt_addr) 4378e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy continue; 4388e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy 439bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong /* After vmemmap_list entry free is possible, need check all */ 440bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong if ((pg_va + sizeof(struct page)) <= 441bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong (vmem_back->virt_addr + page_size)) { 442bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong page = (struct page *) (vmem_back->phys + pg_va - 4438e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy vmem_back->virt_addr); 444bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong return page; 445bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong } 4468e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy } 4478e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy 448bd8cb03dbe77a529945aa270a18c1ba074f729c6Li Zhong /* Probably that page struct is split between real pages */ 4498e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy return NULL; 4508e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy} 4518e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey KardashevskiyEXPORT_SYMBOL_GPL(realmode_pfn_to_page); 4528e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy 4538e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy#elif defined(CONFIG_FLATMEM) 4548e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy 4558e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiystruct page *realmode_pfn_to_page(unsigned long pfn) 4568e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy{ 4578e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy struct page *page = pfn_to_page(pfn); 4588e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy return page; 4598e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy} 4608e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey KardashevskiyEXPORT_SYMBOL_GPL(realmode_pfn_to_page); 4618e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy 4628e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23Alexey Kardashevskiy#endif /* CONFIG_SPARSEMEM_VMEMMAP/CONFIG_FLATMEM */ 463