init.c revision c967da6a0ba837f762042e931d4afcf72045547c
12c1b284e4fa260fd922b9a65c99169e2630c6862Jaswinder Singh Rajput#include <linux/initrd.h> 2540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <linux/ioport.h> 3e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <linux/swap.h> 4540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 5e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/cacheflush.h> 6f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/e820.h> 74fcb208391be5cf82c6fe2779c5eb9245ac97e91Pekka Enberg#include <asm/init.h> 8e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/page.h> 9540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <asm/page_types.h> 10e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/sections.h> 11498343967613183611ac37dccb2846496d954c06Jan Beulich#include <asm/setup.h> 12e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/system.h> 13f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/tlbflush.h> 149518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg#include <asm/tlb.h> 1576c06927f2a78143763dcff9b4c362d15eb29cc2Jaswinder Singh Rajput#include <asm/proto.h> 169518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg 179518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka EnbergDEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 18f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 19298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enbergunsigned long __initdata e820_table_start; 20298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enbergunsigned long __meminitdata e820_table_end; 21298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enbergunsigned long __meminitdata e820_table_top; 22f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 23f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint after_bootmem; 24f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 25f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint direct_gbpages 26f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_DIRECT_GBPAGES 27f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg = 1 28f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 29f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg; 30f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 31f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergstatic void __init find_early_table_space(unsigned long end, int use_pse, 32f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg int use_gbpages) 33f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 34f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long puds, pmds, ptes, tables, start; 35f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 36f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; 37f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables = roundup(puds * sizeof(pud_t), PAGE_SIZE); 38f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 39f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (use_gbpages) { 40f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long extra; 41f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 42f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT); 43f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT; 44f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } else 45f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT; 46f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 47f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE); 48f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 49f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (use_pse) { 50f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long extra; 51f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 52f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT); 53f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 54f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg extra += PMD_SIZE; 55f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 56f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; 57f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } else 58f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; 59f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 60f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE); 61f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 62f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 63f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* for fixmap */ 64f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE); 65f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 66f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 67f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 68f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * RED-PEN putting page tables only on node 0 could 69f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * cause a hotspot and fill up ZONE_DMA. The page tables 70f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * need roughly 0.5KB per GB. 71f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 72f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 73f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start = 0x7000; 7480989ce0643c1034822f3e339ed8d790b649abe1Yinghai Lu#else 75f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start = 0x8000; 76f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 7780989ce0643c1034822f3e339ed8d790b649abe1Yinghai Lu e820_table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT, 7880989ce0643c1034822f3e339ed8d790b649abe1Yinghai Lu tables, PAGE_SIZE); 79298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enberg if (e820_table_start == -1UL) 80f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("Cannot find space for the kernel page tables"); 81f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 82298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enberg e820_table_start >>= PAGE_SHIFT; 83298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enberg e820_table_end = e820_table_start; 84298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enberg e820_table_top = e820_table_start + (tables >> PAGE_SHIFT); 85f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 86f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n", 87298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enberg end, e820_table_start << PAGE_SHIFT, e820_table_top << PAGE_SHIFT); 88f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 89f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 90f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergstruct map_range { 91f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long start; 92f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long end; 93f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned page_size_mask; 94f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg}; 95f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 96f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 97f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 3 98f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 99f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 5 100f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 101f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 102dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulichstatic int __meminit save_mr(struct map_range *mr, int nr_range, 103dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long start_pfn, unsigned long end_pfn, 104dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long page_size_mask) 105f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 106f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 107f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (nr_range >= NR_RANGE_MR) 108f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("run out of range for init_memory_mapping\n"); 109f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].start = start_pfn<<PAGE_SHIFT; 110f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].end = end_pfn<<PAGE_SHIFT; 111f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].page_size_mask = page_size_mask; 112f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range++; 113f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 114f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 115f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return nr_range; 116f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 117f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 118f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg/* 119f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Setup the direct mapping of the physical memory at PAGE_OFFSET. 120f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * This runs before bootmem is initialized and gets pages directly from 121f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * the physical memory. To access them they are temporarily mapped. 122f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 123f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergunsigned long __init_refok init_memory_mapping(unsigned long start, 124f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long end) 125f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 126f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long page_size_mask = 0; 127f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long start_pfn, end_pfn; 128c77a3b59c624454c501cbfa1a3611d5a00bf9532Pekka Enberg unsigned long ret = 0; 129f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long pos; 130f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 131f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg struct map_range mr[NR_RANGE_MR]; 132f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg int nr_range, i; 133f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg int use_pse, use_gbpages; 134f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 135f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg printk(KERN_INFO "init_memory_mapping: %016lx-%016lx\n", start, end); 136f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 137f85612967c93b67b10dd240e3e8bf8a0eee9def7Vegard Nossum#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK) 138f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 139f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 140f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * This will simplify cpa(), which otherwise needs to support splitting 141f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * large pages into small in interrupt context, etc. 142f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 143f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg use_pse = use_gbpages = 0; 144f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else 145f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg use_pse = cpu_has_pse; 146f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg use_gbpages = direct_gbpages; 147f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 148f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 149f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* Enable PSE if available */ 150f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (cpu_has_pse) 151f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg set_in_cr4(X86_CR4_PSE); 152f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 153f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* Enable PGE if available */ 154f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (cpu_has_pge) { 155f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg set_in_cr4(X86_CR4_PGE); 156f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg __supported_pte_mask |= _PAGE_GLOBAL; 157f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 158f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 159f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (use_gbpages) 160f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask |= 1 << PG_LEVEL_1G; 161f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (use_pse) 162f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask |= 1 << PG_LEVEL_2M; 163f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 164f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memset(mr, 0, sizeof(mr)); 165f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = 0; 166f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 167f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* head if not big page alignment ? */ 168f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = start >> PAGE_SHIFT; 169f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = start_pfn << PAGE_SHIFT; 170f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 171f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 172f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Don't use a large page for the first 2/4MB of memory 173f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * because there are often fixed size MTRRs in there 174f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * and overlapping MTRRs into large pages can cause 175f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * slowdowns. 176f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 177f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (pos == 0) 178f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT); 179f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg else 180f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 181f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 182f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 183f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT) 184f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 185f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 186f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (end_pfn > (end >> PAGE_SHIFT)) 187f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = end >> PAGE_SHIFT; 188f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 189f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 190f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 191f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 192f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 193f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (2M) range */ 194f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 195f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 196f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 197f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 198f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 199f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 200f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PUD_SHIFT - PAGE_SHIFT); 201f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT))) 202f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)); 203f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 204f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 205f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 206f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 207f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 208f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 209f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 210f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 211f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_64 212f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (1G) range */ 213f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 214f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PUD_SHIFT - PAGE_SHIFT); 215f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT); 216f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 217f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 218f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & 219f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 220f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 221f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 222f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 223f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (1G) alignment */ 224f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 225f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 226f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 227f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 228f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 229f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 230f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 231f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 232f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 233f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 234f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (2M) alignment */ 235f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = pos>>PAGE_SHIFT; 236f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = end>>PAGE_SHIFT; 237f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 238f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 239f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* try to merge same page size and continuous */ 240f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 241f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long old_start; 242f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (mr[i].end != mr[i+1].start || 243f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask != mr[i+1].page_size_mask) 244f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg continue; 245f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* move it */ 246f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg old_start = mr[i].start; 247f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memmove(&mr[i], &mr[i+1], 248f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (nr_range - 1 - i) * sizeof(struct map_range)); 249f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i--].start = old_start; 250f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range--; 251f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 252f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 253f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 254f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg printk(KERN_DEBUG " %010lx - %010lx page %s\n", 255f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].start, mr[i].end, 256f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 257f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 258f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 259f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 260f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Find space for the kernel direct mapping tables. 261f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * 262f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Later we should allocate these tables in the local node of the 263f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * memory mapped. Unfortunately this is done currently before the 264f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * nodes are discovered. 265f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 266f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (!after_bootmem) 267f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg find_early_table_space(end, use_pse, use_gbpages); 268f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 269f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 270f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 271f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask); 272f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 273f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 274f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg early_ioremap_page_table_range_init(); 275f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 276f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg load_cr3(swapper_pg_dir); 277f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 278f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 279f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_64 280498343967613183611ac37dccb2846496d954c06Jan Beulich if (!after_bootmem && !start) { 281498343967613183611ac37dccb2846496d954c06Jan Beulich pud_t *pud; 282498343967613183611ac37dccb2846496d954c06Jan Beulich pmd_t *pmd; 283498343967613183611ac37dccb2846496d954c06Jan Beulich 284f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mmu_cr4_features = read_cr4(); 285498343967613183611ac37dccb2846496d954c06Jan Beulich 286498343967613183611ac37dccb2846496d954c06Jan Beulich /* 287498343967613183611ac37dccb2846496d954c06Jan Beulich * _brk_end cannot change anymore, but it and _end may be 288498343967613183611ac37dccb2846496d954c06Jan Beulich * located on different 2M pages. cleanup_highmap(), however, 289498343967613183611ac37dccb2846496d954c06Jan Beulich * can only consider _end when it runs, so destroy any 290498343967613183611ac37dccb2846496d954c06Jan Beulich * mappings beyond _brk_end here. 291498343967613183611ac37dccb2846496d954c06Jan Beulich */ 292498343967613183611ac37dccb2846496d954c06Jan Beulich pud = pud_offset(pgd_offset_k(_brk_end), _brk_end); 293498343967613183611ac37dccb2846496d954c06Jan Beulich pmd = pmd_offset(pud, _brk_end - 1); 294498343967613183611ac37dccb2846496d954c06Jan Beulich while (++pmd <= pmd_offset(pud, (unsigned long)_end - 1)) 295498343967613183611ac37dccb2846496d954c06Jan Beulich pmd_clear(pmd); 296498343967613183611ac37dccb2846496d954c06Jan Beulich } 297f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 298f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg __flush_tlb_all(); 299f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 300298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enberg if (!after_bootmem && e820_table_end > e820_table_start) 301298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enberg reserve_early(e820_table_start << PAGE_SHIFT, 302298af9d89f3f5292e81a0a00f729c415adc4d8fbPekka Enberg e820_table_end << PAGE_SHIFT, "PGTABLE"); 303f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 304f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (!after_bootmem) 305f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg early_memtest(start, end); 306f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 307f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return ret >> PAGE_SHIFT; 308f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 309f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 310e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 311540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg/* 312540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * devmem_is_allowed() checks to see if /dev/mem access to a certain address 313540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * is valid. The argument is a physical page number. 314540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 315540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 316540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * On x86, access has to be given to the first megabyte of ram because that area 317540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * contains bios code and data regions used by X and dosemu and similar apps. 318540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * Access has to be given to non-kernel-ram areas as well, these contain the PCI 319540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * mmio resources as well as potential bios/acpi data regions. 320540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg */ 321540aca06b737cc38965b52eeceefba3d24376461Pekka Enbergint devmem_is_allowed(unsigned long pagenr) 322540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg{ 323540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (pagenr <= 256) 324540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 325540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 326540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 327540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (!page_is_ram(pagenr)) 328540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 329540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 330540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg} 331540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 332e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_init_pages(char *what, unsigned long begin, unsigned long end) 333e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 334c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long addr; 335c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long begin_aligned, end_aligned; 336e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 337c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* Make sure boundaries are page aligned */ 338c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin_aligned = PAGE_ALIGN(begin); 339c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end_aligned = end & PAGE_MASK; 340c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 341c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 342c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin = begin_aligned; 343c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end = end_aligned; 344c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu } 345c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 346c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (begin >= end) 347e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg return; 348e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 349c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu addr = begin; 350c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 351e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 352e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * If debugging page accesses then do not free this memory but 353e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * mark them not present - any buggy init-section access will 354e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * create a kernel page fault: 355e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 356e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#ifdef CONFIG_DEBUG_PAGEALLOC 357e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg printk(KERN_INFO "debug: unmapping init memory %08lx..%08lx\n", 358c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin, end); 359e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 360e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#else 361e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 362e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * We just marked the kernel text read only above, now that 363e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * we are going to free part of that, we need to make that 364e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * writeable first. 365e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 366e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 367e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 368e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 369e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 370e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg for (; addr < end; addr += PAGE_SIZE) { 371e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg ClearPageReserved(virt_to_page(addr)); 372e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg init_page_count(virt_to_page(addr)); 373c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 374e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_page(addr); 375e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg totalram_pages++; 376e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg } 377e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#endif 378e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 379e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 380e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_initmem(void) 381e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 382e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_init_pages("unused kernel memory", 383e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_begin), 384e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_end)); 385e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 386731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg 387731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#ifdef CONFIG_BLK_DEV_INITRD 388731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enbergvoid free_initrd_mem(unsigned long start, unsigned long end) 389731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg{ 390c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* 391c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * end could be not aligned, and We can not align that, 392c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * decompresser could be confused by aligned initrd_end 393c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * We already reserve the end partial page before in 394c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - i386_start_kernel() 395c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - x86_64_start_kernel() 396c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - relocate_initrd() 397c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * So here We can do PAGE_ALIGN() safely to get partial page to be freed 398c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu */ 399c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu free_init_pages("initrd memory", start, PAGE_ALIGN(end)); 400731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg} 401731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#endif 402