init.c revision fa62aafea9e415cd1efd8c4054106112fe809f19
15a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/gfp.h> 22c1b284e4fa260fd922b9a65c99169e2630c6862Jaswinder Singh Rajput#include <linux/initrd.h> 3540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <linux/ioport.h> 4e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <linux/swap.h> 5a9ce6bc15100023b411f8117e53a016d61889800Yinghai Lu#include <linux/memblock.h> 6176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <linux/bootmem.h> /* for max_low_pfn */ 7540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 8e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/cacheflush.h> 9f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/e820.h> 104fcb208391be5cf82c6fe2779c5eb9245ac97e91Pekka Enberg#include <asm/init.h> 11e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/page.h> 12540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <asm/page_types.h> 13e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/sections.h> 14498343967613183611ac37dccb2846496d954c06Jan Beulich#include <asm/setup.h> 15f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/tlbflush.h> 169518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg#include <asm/tlb.h> 1776c06927f2a78143763dcff9b4c362d15eb29cc2Jaswinder Singh Rajput#include <asm/proto.h> 18176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <asm/dma.h> /* for MAX_DMA_PFN */ 199518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg 20d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __initdata pgt_buf_start; 21d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __meminitdata pgt_buf_end; 22d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __meminitdata pgt_buf_top; 23f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 24f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint after_bootmem; 25f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 26f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint direct_gbpages 27f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_DIRECT_GBPAGES 28f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg = 1 29f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 30f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg; 31f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 32844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shinstruct map_range { 33844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long start; 34844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long end; 35844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned page_size_mask; 36844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin}; 37844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin 38fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lustatic int page_size_mask; 39844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin/* 40844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin * First calculate space needed for kernel direct mapping page tables to cover 41844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin * mr[0].start to mr[nr_range - 1].end, while accounting for possible 2M and 1GB 42844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin * pages. Then find enough contiguous space for those page tables. 43844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin */ 44844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shinstatic void __init find_early_table_space(struct map_range *mr, int nr_range) 45f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 46844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin int i; 47844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long puds = 0, pmds = 0, ptes = 0, tables; 48844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long start = 0, good_end; 49a9ce6bc15100023b411f8117e53a016d61889800Yinghai Lu phys_addr_t base; 50f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 51844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin for (i = 0; i < nr_range; i++) { 52844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long range, extra; 53f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 54844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin range = mr[i].end - mr[i].start; 55844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin puds += (range + PUD_SIZE - 1) >> PUD_SHIFT; 56f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 57844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin if (mr[i].page_size_mask & (1 << PG_LEVEL_1G)) { 58844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin extra = range - ((range >> PUD_SHIFT) << PUD_SHIFT); 59844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin pmds += (extra + PMD_SIZE - 1) >> PMD_SHIFT; 60844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin } else { 61844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin pmds += (range + PMD_SIZE - 1) >> PMD_SHIFT; 62844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin } 63f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 64844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin if (mr[i].page_size_mask & (1 << PG_LEVEL_2M)) { 65844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin extra = range - ((range >> PMD_SHIFT) << PMD_SHIFT); 66f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 67844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin extra += PMD_SIZE; 68f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 69844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin ptes += (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; 70844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin } else { 71844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin ptes += (range + PAGE_SIZE - 1) >> PAGE_SHIFT; 72844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin } 73844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin } 74f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 75844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin tables = roundup(puds * sizeof(pud_t), PAGE_SIZE); 76844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE); 77f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE); 78f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 79f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 80f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* for fixmap */ 81f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE); 82f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 838548c84da2f47e71bbbe300f55edb768492575f7Takashi Iwai good_end = max_pfn_mapped << PAGE_SHIFT; 841411e0ec3123ae4c4ead6bfc9fe3ee5a3ae5c327Yinghai Lu 854b239f458c229de044d6905c2b0f9fe16ed9e01eYinghai Lu base = memblock_find_in_range(start, good_end, tables, PAGE_SIZE); 861f5026a7e21e409c2b9dd54f6dfb9446511fb7c5Tejun Heo if (!base) 87f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("Cannot find space for the kernel page tables"); 88f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 89d1b19426b04787e48f2689923e28d37b488969b0Yinghai Lu pgt_buf_start = base >> PAGE_SHIFT; 90d1b19426b04787e48f2689923e28d37b488969b0Yinghai Lu pgt_buf_end = pgt_buf_start; 91d1b19426b04787e48f2689923e28d37b488969b0Yinghai Lu pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); 92f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 93365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_DEBUG "kernel direct mapping tables up to %#lx @ [mem %#010lx-%#010lx]\n", 94844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin mr[nr_range - 1].end - 1, pgt_buf_start << PAGE_SHIFT, 95365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas (pgt_buf_top << PAGE_SHIFT) - 1); 96f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 97f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 98fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Luvoid probe_page_size_mask(void) 99fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu{ 100fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK) 101fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* 102fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 103fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * This will simplify cpa(), which otherwise needs to support splitting 104fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * large pages into small in interrupt context, etc. 105fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu */ 106fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (direct_gbpages) 107fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu page_size_mask |= 1 << PG_LEVEL_1G; 108fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pse) 109fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu page_size_mask |= 1 << PG_LEVEL_2M; 110fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu#endif 111fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu 112fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* Enable PSE if available */ 113fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pse) 114fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu set_in_cr4(X86_CR4_PSE); 115fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu 116fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* Enable PGE if available */ 117fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pge) { 118fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu set_in_cr4(X86_CR4_PGE); 119fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu __supported_pte_mask |= _PAGE_GLOBAL; 120fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu } 121fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu} 12253f8023febf9b3e18d8fb0d99c55010e473ce53dSedat Dilekvoid __init native_pagetable_reserve(u64 start, u64 end) 123279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini{ 12424aa07882b672fff2da2f5c955759f0bd13d32d5Tejun Heo memblock_reserve(start, end - start); 125279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini} 126279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini 127f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 128f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 3 129f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 130f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 5 131f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 132f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 133dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulichstatic int __meminit save_mr(struct map_range *mr, int nr_range, 134dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long start_pfn, unsigned long end_pfn, 135dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long page_size_mask) 136f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 137f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 138f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (nr_range >= NR_RANGE_MR) 139f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("run out of range for init_memory_mapping\n"); 140f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].start = start_pfn<<PAGE_SHIFT; 141f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].end = end_pfn<<PAGE_SHIFT; 142f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].page_size_mask = page_size_mask; 143f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range++; 144f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 145f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 146f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return nr_range; 147f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 148f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 149f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg/* 150f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Setup the direct mapping of the physical memory at PAGE_OFFSET. 151f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * This runs before bootmem is initialized and gets pages directly from 152f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * the physical memory. To access them they are temporarily mapped. 153f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 154f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergunsigned long __init_refok init_memory_mapping(unsigned long start, 155f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long end) 156f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 157f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long start_pfn, end_pfn; 158c77a3b59c624454c501cbfa1a3611d5a00bf9532Pekka Enberg unsigned long ret = 0; 159f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long pos; 160f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg struct map_range mr[NR_RANGE_MR]; 161f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg int nr_range, i; 162f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 163365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_INFO "init_memory_mapping: [mem %#010lx-%#010lx]\n", 164365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas start, end - 1); 165f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 166f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memset(mr, 0, sizeof(mr)); 167f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = 0; 168f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 169f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* head if not big page alignment ? */ 170f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = start >> PAGE_SHIFT; 171f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = start_pfn << PAGE_SHIFT; 172f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 173f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 174f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Don't use a large page for the first 2/4MB of memory 175f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * because there are often fixed size MTRRs in there 176f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * and overlapping MTRRs into large pages can cause 177f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * slowdowns. 178f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 179f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (pos == 0) 180f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT); 181f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg else 182f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 183f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 184f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 185f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT) 186f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 187f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 188f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (end_pfn > (end >> PAGE_SHIFT)) 189f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = end >> PAGE_SHIFT; 190f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 191f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 192f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 193f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 194f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 195f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (2M) range */ 196f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 197f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 198f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 199f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 200f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 201f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 202f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PUD_SHIFT - PAGE_SHIFT); 203f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT))) 204f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)); 205f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 206f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 207f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 208f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 209f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 210f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 211f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 212f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 213f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_64 214f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (1G) range */ 215f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 216f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PUD_SHIFT - PAGE_SHIFT); 217f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT); 218f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 219f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 220f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & 221f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 222f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 223f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 224f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 225f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (1G) alignment */ 226f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 227f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 228f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 229f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 230f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 231f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 232f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 233f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 234f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 235f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 236f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (2M) alignment */ 237f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = pos>>PAGE_SHIFT; 238f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = end>>PAGE_SHIFT; 239f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 240f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 241f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* try to merge same page size and continuous */ 242f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 243f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long old_start; 244f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (mr[i].end != mr[i+1].start || 245f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask != mr[i+1].page_size_mask) 246f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg continue; 247f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* move it */ 248f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg old_start = mr[i].start; 249f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memmove(&mr[i], &mr[i+1], 250f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (nr_range - 1 - i) * sizeof(struct map_range)); 251f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i--].start = old_start; 252f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range--; 253f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 254f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 255f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 256365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n", 257365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas mr[i].start, mr[i].end - 1, 258f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 259f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 260f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 261f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 262f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Find space for the kernel direct mapping tables. 263f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * 264f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Later we should allocate these tables in the local node of the 265f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * memory mapped. Unfortunately this is done currently before the 266f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * nodes are discovered. 267f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 268f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (!after_bootmem) 269844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin find_early_table_space(mr, nr_range); 270f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 271f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 272f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 273f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask); 274f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 275f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 276f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg early_ioremap_page_table_range_init(); 277f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 278f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg load_cr3(swapper_pg_dir); 279f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 280f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 281f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg __flush_tlb_all(); 282f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 283279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini /* 284279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * Reserve the kernel pagetable pages we used (pgt_buf_start - 285279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * pgt_buf_end) and free the other ones (pgt_buf_end - pgt_buf_top) 286279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * so that they can be reused for other purposes. 287279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * 28824aa07882b672fff2da2f5c955759f0bd13d32d5Tejun Heo * On native it just means calling memblock_reserve, on Xen it also 28924aa07882b672fff2da2f5c955759f0bd13d32d5Tejun Heo * means marking RW the pagetable pages that we allocated before 290279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * but that haven't been used. 291279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * 292279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * In fact on xen we mark RO the whole range pgt_buf_start - 293279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * pgt_buf_top, because we have to make sure that when 294279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * init_memory_mapping reaches the pagetable pages area, it maps 295279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * RO all the pagetable pages, including the ones that are beyond 296279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * pgt_buf_end at that time. 297279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini */ 298d1b19426b04787e48f2689923e28d37b488969b0Yinghai Lu if (!after_bootmem && pgt_buf_end > pgt_buf_start) 299279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini x86_init.mapping.pagetable_reserve(PFN_PHYS(pgt_buf_start), 300279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini PFN_PHYS(pgt_buf_end)); 301f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 302f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (!after_bootmem) 303f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg early_memtest(start, end); 304f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 305f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return ret >> PAGE_SHIFT; 306f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 307f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 308e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 309540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg/* 310540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * devmem_is_allowed() checks to see if /dev/mem access to a certain address 311540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * is valid. The argument is a physical page number. 312540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 313540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 314540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * On x86, access has to be given to the first megabyte of ram because that area 315540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * contains bios code and data regions used by X and dosemu and similar apps. 316540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * Access has to be given to non-kernel-ram areas as well, these contain the PCI 317540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * mmio resources as well as potential bios/acpi data regions. 318540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg */ 319540aca06b737cc38965b52eeceefba3d24376461Pekka Enbergint devmem_is_allowed(unsigned long pagenr) 320540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg{ 32173e8f3d7e2cb23614d5115703d76d8e54764b641T Makphaibulchoke if (pagenr < 256) 322540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 323540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 324540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 325540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (!page_is_ram(pagenr)) 326540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 327540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 328540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg} 329540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 330e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_init_pages(char *what, unsigned long begin, unsigned long end) 331e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 332c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long addr; 333c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long begin_aligned, end_aligned; 334e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 335c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* Make sure boundaries are page aligned */ 336c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin_aligned = PAGE_ALIGN(begin); 337c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end_aligned = end & PAGE_MASK; 338c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 339c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 340c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin = begin_aligned; 341c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end = end_aligned; 342c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu } 343c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 344c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (begin >= end) 345e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg return; 346e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 347c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu addr = begin; 348c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 349e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 350e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * If debugging page accesses then do not free this memory but 351e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * mark them not present - any buggy init-section access will 352e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * create a kernel page fault: 353e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 354e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#ifdef CONFIG_DEBUG_PAGEALLOC 355365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n", 356365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas begin, end - 1); 357e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 358e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#else 359e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 360e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * We just marked the kernel text read only above, now that 361e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * we are going to free part of that, we need to make that 3625bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET * writeable and non-executable first. 363e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 3645bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); 365e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 366e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 367e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 368e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 369e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg for (; addr < end; addr += PAGE_SIZE) { 370e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg ClearPageReserved(virt_to_page(addr)); 371e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg init_page_count(virt_to_page(addr)); 372c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 373e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_page(addr); 374e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg totalram_pages++; 375e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg } 376e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#endif 377e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 378e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 379e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_initmem(void) 380e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 381e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_init_pages("unused kernel memory", 382e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_begin), 383e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_end)); 384e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 385731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg 386731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#ifdef CONFIG_BLK_DEV_INITRD 3870d26d1d873a302828e064737746c53a2689e6c0fJan Beulichvoid __init free_initrd_mem(unsigned long start, unsigned long end) 388731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg{ 389c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* 390c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * end could be not aligned, and We can not align that, 391c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * decompresser could be confused by aligned initrd_end 392c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * We already reserve the end partial page before in 393c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - i386_start_kernel() 394c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - x86_64_start_kernel() 395c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - relocate_initrd() 396c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * So here We can do PAGE_ALIGN() safely to get partial page to be freed 397c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu */ 398c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu free_init_pages("initrd memory", start, PAGE_ALIGN(end)); 399731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg} 400731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#endif 401176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 402176239153049a023d060ce95b05f7ef31667e362Pekka Enbergvoid __init zone_sizes_init(void) 403176239153049a023d060ce95b05f7ef31667e362Pekka Enberg{ 404176239153049a023d060ce95b05f7ef31667e362Pekka Enberg unsigned long max_zone_pfns[MAX_NR_ZONES]; 405176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 406176239153049a023d060ce95b05f7ef31667e362Pekka Enberg memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 407176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 408176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA 409176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 410176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 411176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA32 412176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 413176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 414176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 415176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_HIGHMEM 416176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_HIGHMEM] = max_pfn; 417176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 418176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 419176239153049a023d060ce95b05f7ef31667e362Pekka Enberg free_area_init_nodes(max_zone_pfns); 420176239153049a023d060ce95b05f7ef31667e362Pekka Enberg} 421176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 422