init.c revision 0d26d1d873a302828e064737746c53a2689e6c0f
15a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/gfp.h> 22c1b284e4fa260fd922b9a65c99169e2630c6862Jaswinder Singh Rajput#include <linux/initrd.h> 3540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <linux/ioport.h> 4e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <linux/swap.h> 5a9ce6bc15100023b411f8117e53a016d61889800Yinghai Lu#include <linux/memblock.h> 6176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <linux/bootmem.h> /* for max_low_pfn */ 7540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 8e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/cacheflush.h> 9f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/e820.h> 104fcb208391be5cf82c6fe2779c5eb9245ac97e91Pekka Enberg#include <asm/init.h> 11e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/page.h> 12540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <asm/page_types.h> 13e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/sections.h> 14498343967613183611ac37dccb2846496d954c06Jan Beulich#include <asm/setup.h> 15f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/tlbflush.h> 169518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg#include <asm/tlb.h> 1776c06927f2a78143763dcff9b4c362d15eb29cc2Jaswinder Singh Rajput#include <asm/proto.h> 18176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <asm/dma.h> /* for MAX_DMA_PFN */ 199518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg 20d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __initdata pgt_buf_start; 21d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __meminitdata pgt_buf_end; 22d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __meminitdata pgt_buf_top; 23f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 24f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint after_bootmem; 25f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 26f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint direct_gbpages 27f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_DIRECT_GBPAGES 28f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg = 1 29f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 30f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg; 31f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 32722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Congstruct map_range { 33722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong unsigned long start; 34722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong unsigned long end; 35722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong unsigned page_size_mask; 36722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong}; 37722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong 38722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Congstatic void __init find_early_table_space(struct map_range *mr, unsigned long end, 39722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong int use_pse, int use_gbpages) 40f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 414b239f458c229de044d6905c2b0f9fe16ed9e01eYinghai Lu unsigned long puds, pmds, ptes, tables, start = 0, good_end = end; 42a9ce6bc15100023b411f8117e53a016d61889800Yinghai Lu phys_addr_t base; 43f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 44f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; 45f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables = roundup(puds * sizeof(pud_t), PAGE_SIZE); 46f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 47f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (use_gbpages) { 48f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long extra; 49f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 50f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT); 51f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT; 52f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } else 53f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT; 54f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 55f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE); 56f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 57f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (use_pse) { 58f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long extra; 59f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 60f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT); 61f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 62f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg extra += PMD_SIZE; 63f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 64722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong /* The first 2/4M doesn't use large pages. */ 65bd2753b2dda7bb43c7468826de75f49c6a7e8965Yinghai Lu if (mr->start < PMD_SIZE) 66bd2753b2dda7bb43c7468826de75f49c6a7e8965Yinghai Lu extra += mr->end - mr->start; 67722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong 68f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; 69f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } else 70f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; 71f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 72f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE); 73f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 74f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 75f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* for fixmap */ 76f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE); 77f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 788548c84da2f47e71bbbe300f55edb768492575f7Takashi Iwai good_end = max_pfn_mapped << PAGE_SHIFT; 791411e0ec3123ae4c4ead6bfc9fe3ee5a3ae5c327Yinghai Lu 804b239f458c229de044d6905c2b0f9fe16ed9e01eYinghai Lu base = memblock_find_in_range(start, good_end, tables, PAGE_SIZE); 811f5026a7e21e409c2b9dd54f6dfb9446511fb7c5Tejun Heo if (!base) 82f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("Cannot find space for the kernel page tables"); 83f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 84d1b19426b04787e48f2689923e28d37b488969b0Yinghai Lu pgt_buf_start = base >> PAGE_SHIFT; 85d1b19426b04787e48f2689923e28d37b488969b0Yinghai Lu pgt_buf_end = pgt_buf_start; 86d1b19426b04787e48f2689923e28d37b488969b0Yinghai Lu pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); 87f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 88365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_DEBUG "kernel direct mapping tables up to %#lx @ [mem %#010lx-%#010lx]\n", 89365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas end - 1, pgt_buf_start << PAGE_SHIFT, 90365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas (pgt_buf_top << PAGE_SHIFT) - 1); 91f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 92f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 9353f8023febf9b3e18d8fb0d99c55010e473ce53dSedat Dilekvoid __init native_pagetable_reserve(u64 start, u64 end) 94279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini{ 9524aa07882b672fff2da2f5c955759f0bd13d32d5Tejun Heo memblock_reserve(start, end - start); 96279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini} 97279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini 98f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 99f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 3 100f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 101f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 5 102f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 103f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 104dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulichstatic int __meminit save_mr(struct map_range *mr, int nr_range, 105dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long start_pfn, unsigned long end_pfn, 106dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long page_size_mask) 107f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 108f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 109f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (nr_range >= NR_RANGE_MR) 110f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("run out of range for init_memory_mapping\n"); 111f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].start = start_pfn<<PAGE_SHIFT; 112f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].end = end_pfn<<PAGE_SHIFT; 113f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].page_size_mask = page_size_mask; 114f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range++; 115f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 116f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 117f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return nr_range; 118f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 119f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 120f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg/* 121f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Setup the direct mapping of the physical memory at PAGE_OFFSET. 122f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * This runs before bootmem is initialized and gets pages directly from 123f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * the physical memory. To access them they are temporarily mapped. 124f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 125f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergunsigned long __init_refok init_memory_mapping(unsigned long start, 126f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long end) 127f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 128f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long page_size_mask = 0; 129f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long start_pfn, end_pfn; 130c77a3b59c624454c501cbfa1a3611d5a00bf9532Pekka Enberg unsigned long ret = 0; 131f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long pos; 132f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 133f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg struct map_range mr[NR_RANGE_MR]; 134f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg int nr_range, i; 135f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg int use_pse, use_gbpages; 136f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 137365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_INFO "init_memory_mapping: [mem %#010lx-%#010lx]\n", 138365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas start, end - 1); 139f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 140f85612967c93b67b10dd240e3e8bf8a0eee9def7Vegard Nossum#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK) 141f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 142f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 143f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * This will simplify cpa(), which otherwise needs to support splitting 144f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * large pages into small in interrupt context, etc. 145f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 146f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg use_pse = use_gbpages = 0; 147f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else 148f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg use_pse = cpu_has_pse; 149f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg use_gbpages = direct_gbpages; 150f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 151f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 152f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* Enable PSE if available */ 153f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (cpu_has_pse) 154f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg set_in_cr4(X86_CR4_PSE); 155f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 156f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* Enable PGE if available */ 157f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (cpu_has_pge) { 158f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg set_in_cr4(X86_CR4_PGE); 159f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg __supported_pte_mask |= _PAGE_GLOBAL; 160f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 161f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 162f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (use_gbpages) 163f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask |= 1 << PG_LEVEL_1G; 164f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (use_pse) 165f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask |= 1 << PG_LEVEL_2M; 166f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 167f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memset(mr, 0, sizeof(mr)); 168f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = 0; 169f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 170f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* head if not big page alignment ? */ 171f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = start >> PAGE_SHIFT; 172f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = start_pfn << PAGE_SHIFT; 173f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 174f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 175f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Don't use a large page for the first 2/4MB of memory 176f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * because there are often fixed size MTRRs in there 177f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * and overlapping MTRRs into large pages can cause 178f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * slowdowns. 179f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 180f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (pos == 0) 181f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT); 182f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg else 183f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 184f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 185f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 186f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT) 187f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 188f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 189f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (end_pfn > (end >> PAGE_SHIFT)) 190f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = end >> PAGE_SHIFT; 191f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 192f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 193f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 194f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 195f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 196f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (2M) range */ 197f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 198f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 199f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 200f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 201f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 202f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 203f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PUD_SHIFT - PAGE_SHIFT); 204f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT))) 205f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)); 206f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 207f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 208f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 209f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 210f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 211f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 212f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 213f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 214f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_64 215f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (1G) range */ 216f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 217f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PUD_SHIFT - PAGE_SHIFT); 218f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT); 219f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 220f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 221f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & 222f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 223f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 224f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 225f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 226f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (1G) alignment */ 227f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 228f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 229f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 230f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 231f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 232f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 233f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 234f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 235f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 236f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 237f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (2M) alignment */ 238f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = pos>>PAGE_SHIFT; 239f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = end>>PAGE_SHIFT; 240f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 241f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 242f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* try to merge same page size and continuous */ 243f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 244f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long old_start; 245f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (mr[i].end != mr[i+1].start || 246f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask != mr[i+1].page_size_mask) 247f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg continue; 248f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* move it */ 249f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg old_start = mr[i].start; 250f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memmove(&mr[i], &mr[i+1], 251f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (nr_range - 1 - i) * sizeof(struct map_range)); 252f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i--].start = old_start; 253f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range--; 254f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 255f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 256f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 257365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n", 258365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas mr[i].start, mr[i].end - 1, 259f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 260f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 261f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 262f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 263f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Find space for the kernel direct mapping tables. 264f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * 265f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Later we should allocate these tables in the local node of the 266f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * memory mapped. Unfortunately this is done currently before the 267f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * nodes are discovered. 268f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 269f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (!after_bootmem) 270722bc6b16771ed80871e1fd81c86d3627dda2ac8WANG Cong find_early_table_space(&mr[0], end, use_pse, use_gbpages); 271f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 272f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 273f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 274f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask); 275f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 276f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 277f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg early_ioremap_page_table_range_init(); 278f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 279f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg load_cr3(swapper_pg_dir); 280f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 281f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 282f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg __flush_tlb_all(); 283f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 284279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini /* 285279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * Reserve the kernel pagetable pages we used (pgt_buf_start - 286279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * pgt_buf_end) and free the other ones (pgt_buf_end - pgt_buf_top) 287279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * so that they can be reused for other purposes. 288279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * 28924aa07882b672fff2da2f5c955759f0bd13d32d5Tejun Heo * On native it just means calling memblock_reserve, on Xen it also 29024aa07882b672fff2da2f5c955759f0bd13d32d5Tejun Heo * means marking RW the pagetable pages that we allocated before 291279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * but that haven't been used. 292279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * 293279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * In fact on xen we mark RO the whole range pgt_buf_start - 294279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * pgt_buf_top, because we have to make sure that when 295279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * init_memory_mapping reaches the pagetable pages area, it maps 296279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * RO all the pagetable pages, including the ones that are beyond 297279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini * pgt_buf_end at that time. 298279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini */ 299d1b19426b04787e48f2689923e28d37b488969b0Yinghai Lu if (!after_bootmem && pgt_buf_end > pgt_buf_start) 300279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini x86_init.mapping.pagetable_reserve(PFN_PHYS(pgt_buf_start), 301279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini PFN_PHYS(pgt_buf_end)); 302f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 303f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (!after_bootmem) 304f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg early_memtest(start, end); 305f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 306f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return ret >> PAGE_SHIFT; 307f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 308f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 309e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 310540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg/* 311540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * devmem_is_allowed() checks to see if /dev/mem access to a certain address 312540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * is valid. The argument is a physical page number. 313540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 314540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 315540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * On x86, access has to be given to the first megabyte of ram because that area 316540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * contains bios code and data regions used by X and dosemu and similar apps. 317540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * Access has to be given to non-kernel-ram areas as well, these contain the PCI 318540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * mmio resources as well as potential bios/acpi data regions. 319540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg */ 320540aca06b737cc38965b52eeceefba3d24376461Pekka Enbergint devmem_is_allowed(unsigned long pagenr) 321540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg{ 322540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (pagenr <= 256) 323540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 324540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 325540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 326540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (!page_is_ram(pagenr)) 327540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 328540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 329540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg} 330540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 331e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_init_pages(char *what, unsigned long begin, unsigned long end) 332e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 333c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long addr; 334c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long begin_aligned, end_aligned; 335e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 336c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* Make sure boundaries are page aligned */ 337c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin_aligned = PAGE_ALIGN(begin); 338c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end_aligned = end & PAGE_MASK; 339c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 340c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 341c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin = begin_aligned; 342c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end = end_aligned; 343c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu } 344c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 345c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (begin >= end) 346e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg return; 347e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 348c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu addr = begin; 349c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 350e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 351e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * If debugging page accesses then do not free this memory but 352e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * mark them not present - any buggy init-section access will 353e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * create a kernel page fault: 354e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 355e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#ifdef CONFIG_DEBUG_PAGEALLOC 356365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n", 357365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas begin, end - 1); 358e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 359e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#else 360e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 361e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * We just marked the kernel text read only above, now that 362e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * we are going to free part of that, we need to make that 3635bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET * writeable and non-executable first. 364e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 3655bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); 366e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 367e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 368e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 369e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 370e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg for (; addr < end; addr += PAGE_SIZE) { 371e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg ClearPageReserved(virt_to_page(addr)); 372e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg init_page_count(virt_to_page(addr)); 373c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 374e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_page(addr); 375e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg totalram_pages++; 376e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg } 377e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#endif 378e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 379e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 380e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_initmem(void) 381e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 382e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_init_pages("unused kernel memory", 383e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_begin), 384e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_end)); 385e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 386731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg 387731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#ifdef CONFIG_BLK_DEV_INITRD 3880d26d1d873a302828e064737746c53a2689e6c0fJan Beulichvoid __init free_initrd_mem(unsigned long start, unsigned long end) 389731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg{ 390c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* 391c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * end could be not aligned, and We can not align that, 392c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * decompresser could be confused by aligned initrd_end 393c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * We already reserve the end partial page before in 394c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - i386_start_kernel() 395c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - x86_64_start_kernel() 396c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - relocate_initrd() 397c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * So here We can do PAGE_ALIGN() safely to get partial page to be freed 398c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu */ 399c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu free_init_pages("initrd memory", start, PAGE_ALIGN(end)); 400731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg} 401731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#endif 402176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 403176239153049a023d060ce95b05f7ef31667e362Pekka Enbergvoid __init zone_sizes_init(void) 404176239153049a023d060ce95b05f7ef31667e362Pekka Enberg{ 405176239153049a023d060ce95b05f7ef31667e362Pekka Enberg unsigned long max_zone_pfns[MAX_NR_ZONES]; 406176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 407176239153049a023d060ce95b05f7ef31667e362Pekka Enberg memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 408176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 409176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA 410176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 411176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 412176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA32 413176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 414176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 415176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 416176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_HIGHMEM 417176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_HIGHMEM] = max_pfn; 418176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 419176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 420176239153049a023d060ce95b05f7ef31667e362Pekka Enberg free_area_init_nodes(max_zone_pfns); 421176239153049a023d060ce95b05f7ef31667e362Pekka Enberg} 422176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 423