init.c revision 9985b4c6fa7d660f685918a58282275e9e35d8e0
15a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/gfp.h> 22c1b284e4fa260fd922b9a65c99169e2630c6862Jaswinder Singh Rajput#include <linux/initrd.h> 3540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <linux/ioport.h> 4e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <linux/swap.h> 5a9ce6bc15100023b411f8117e53a016d61889800Yinghai Lu#include <linux/memblock.h> 6176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <linux/bootmem.h> /* for max_low_pfn */ 7540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 8e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/cacheflush.h> 9f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/e820.h> 104fcb208391be5cf82c6fe2779c5eb9245ac97e91Pekka Enberg#include <asm/init.h> 11e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/page.h> 12540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <asm/page_types.h> 13e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/sections.h> 14498343967613183611ac37dccb2846496d954c06Jan Beulich#include <asm/setup.h> 15f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/tlbflush.h> 169518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg#include <asm/tlb.h> 1776c06927f2a78143763dcff9b4c362d15eb29cc2Jaswinder Singh Rajput#include <asm/proto.h> 18176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <asm/dma.h> /* for MAX_DMA_PFN */ 199518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg 205c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu#include "mm_internal.h" 215c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 22d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __initdata pgt_buf_start; 23d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __meminitdata pgt_buf_end; 24d1b19426b04787e48f2689923e28d37b488969b0Yinghai Luunsigned long __meminitdata pgt_buf_top; 25f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 269985b4c6fa7d660f685918a58282275e9e35d8e0Yinghai Lustatic unsigned long min_pfn_mapped; 279985b4c6fa7d660f685918a58282275e9e35d8e0Yinghai Lu 285c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu__ref void *alloc_low_page(void) 295c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu{ 305c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu unsigned long pfn; 315c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu void *adr; 325c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 335c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu#ifdef CONFIG_X86_64 345c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (after_bootmem) { 355c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu adr = (void *)get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK); 365c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 375c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu return adr; 385c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu } 395c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu#endif 405c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 415c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if ((pgt_buf_end + 1) >= pgt_buf_top) { 425c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu unsigned long ret; 435c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (min_pfn_mapped >= max_pfn_mapped) 445c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu panic("alloc_low_page: ran out of memory"); 455c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT, 465c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu max_pfn_mapped << PAGE_SHIFT, 475c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu PAGE_SIZE, PAGE_SIZE); 485c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (!ret) 495c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu panic("alloc_low_page: can not alloc memory"); 505c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu memblock_reserve(ret, PAGE_SIZE); 515c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu pfn = ret >> PAGE_SHIFT; 525c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu } else 535c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu pfn = pgt_buf_end++; 545c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 555c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu adr = __va(pfn * PAGE_SIZE); 565c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu clear_page(adr); 575c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu return adr; 585c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu} 595c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 608d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu/* need 4 4k for initial PMD_SIZE, 4k for 0-ISA_END_ADDRESS */ 618d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu#define INIT_PGT_BUF_SIZE (5 * PAGE_SIZE) 628d57470d8f859635deffe3919d7d4867b488b85aYinghai LuRESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE); 638d57470d8f859635deffe3919d7d4867b488b85aYinghai Luvoid __init early_alloc_pgt_buf(void) 648d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu{ 658d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long tables = INIT_PGT_BUF_SIZE; 668d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu phys_addr_t base; 678d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 688d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu base = __pa(extend_brk(tables, PAGE_SIZE)); 698d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 708d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_start = base >> PAGE_SHIFT; 718d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_end = pgt_buf_start; 728d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); 738d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu} 748d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 75f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint after_bootmem; 76f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 77f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint direct_gbpages 78f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_DIRECT_GBPAGES 79f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg = 1 80f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 81f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg; 82f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 83844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shinstruct map_range { 84844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long start; 85844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long end; 86844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned page_size_mask; 87844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin}; 88844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin 89fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lustatic int page_size_mask; 90f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 9122ddfcaa0dbae992332381d41b8a1fbc72269a13Yinghai Lustatic void __init probe_page_size_mask(void) 92fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu{ 93fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK) 94fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* 95fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 96fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * This will simplify cpa(), which otherwise needs to support splitting 97fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * large pages into small in interrupt context, etc. 98fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu */ 99fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (direct_gbpages) 100fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu page_size_mask |= 1 << PG_LEVEL_1G; 101fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pse) 102fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu page_size_mask |= 1 << PG_LEVEL_2M; 103fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu#endif 104fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu 105fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* Enable PSE if available */ 106fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pse) 107fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu set_in_cr4(X86_CR4_PSE); 108fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu 109fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* Enable PGE if available */ 110fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pge) { 111fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu set_in_cr4(X86_CR4_PGE); 112fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu __supported_pte_mask |= _PAGE_GLOBAL; 113fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu } 114fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu} 11553f8023febf9b3e18d8fb0d99c55010e473ce53dSedat Dilekvoid __init native_pagetable_reserve(u64 start, u64 end) 116279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini{ 11724aa07882b672fff2da2f5c955759f0bd13d32d5Tejun Heo memblock_reserve(start, end - start); 118279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini} 119279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini 120f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 121f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 3 122f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 123f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 5 124f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 125f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 126dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulichstatic int __meminit save_mr(struct map_range *mr, int nr_range, 127dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long start_pfn, unsigned long end_pfn, 128dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long page_size_mask) 129f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 130f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 131f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (nr_range >= NR_RANGE_MR) 132f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("run out of range for init_memory_mapping\n"); 133f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].start = start_pfn<<PAGE_SHIFT; 134f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].end = end_pfn<<PAGE_SHIFT; 135f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].page_size_mask = page_size_mask; 136f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range++; 137f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 138f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 139f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return nr_range; 140f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 141f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 142aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu/* 143aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu * adjust the page_size_mask for small range to go with 144aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu * big page size instead small one if nearby are ram too. 145aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu */ 146aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lustatic void __init_refok adjust_range_page_size_mask(struct map_range *mr, 147aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu int nr_range) 148aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu{ 149aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu int i; 150aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 151aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu for (i = 0; i < nr_range; i++) { 152aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((page_size_mask & (1<<PG_LEVEL_2M)) && 153aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) { 154aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long start = round_down(mr[i].start, PMD_SIZE); 155aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long end = round_up(mr[i].end, PMD_SIZE); 156aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 157aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu#ifdef CONFIG_X86_32 158aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((end >> PAGE_SHIFT) > max_low_pfn) 159aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu continue; 160aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu#endif 161aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 162aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if (memblock_is_region_memory(start, end - start)) 163aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu mr[i].page_size_mask |= 1<<PG_LEVEL_2M; 164aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 165aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((page_size_mask & (1<<PG_LEVEL_1G)) && 166aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) { 167aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long start = round_down(mr[i].start, PUD_SIZE); 168aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long end = round_up(mr[i].end, PUD_SIZE); 169aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 170aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if (memblock_is_region_memory(start, end - start)) 171aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu mr[i].page_size_mask |= 1<<PG_LEVEL_1G; 172aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 173aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 174aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu} 175aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 1764e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lustatic int __meminit split_mem_range(struct map_range *mr, int nr_range, 1774e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long start, 1784e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long end) 179f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 180f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long start_pfn, end_pfn; 181f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long pos; 1824e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu int i; 183f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 184f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* head if not big page alignment ? */ 185f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = start >> PAGE_SHIFT; 186f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = start_pfn << PAGE_SHIFT; 187f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 188f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 189f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Don't use a large page for the first 2/4MB of memory 190f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * because there are often fixed size MTRRs in there 191f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * and overlapping MTRRs into large pages can cause 192f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * slowdowns. 193f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 194f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (pos == 0) 195f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT); 196f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg else 197f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 198f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 199f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 200f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT) 201f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 202f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 203f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (end_pfn > (end >> PAGE_SHIFT)) 204f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = end >> PAGE_SHIFT; 205f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 206f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 207f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 208f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 209f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 210f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (2M) range */ 211f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 212f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 213f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 214f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 215f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 216f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 217f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PUD_SHIFT - PAGE_SHIFT); 218f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT))) 219f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)); 220f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 221f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 222f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 223f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 224f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 225f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 226f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 227f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 228f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_64 229f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (1G) range */ 230f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 231f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PUD_SHIFT - PAGE_SHIFT); 232f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT); 233f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 234f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 235f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & 236f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 237f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 238f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 239f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 240f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (1G) alignment */ 241f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 242f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg << (PMD_SHIFT - PAGE_SHIFT); 243f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 244f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 245f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 246f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 247f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg pos = end_pfn << PAGE_SHIFT; 248f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 249f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 250f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 251f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (2M) alignment */ 252f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg start_pfn = pos>>PAGE_SHIFT; 253f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg end_pfn = end>>PAGE_SHIFT; 254f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 255f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 256f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* try to merge same page size and continuous */ 257f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 258f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long old_start; 259f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (mr[i].end != mr[i+1].start || 260f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask != mr[i+1].page_size_mask) 261f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg continue; 262f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* move it */ 263f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg old_start = mr[i].start; 264f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memmove(&mr[i], &mr[i+1], 265f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (nr_range - 1 - i) * sizeof(struct map_range)); 266f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i--].start = old_start; 267f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range--; 268f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 269f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 270aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if (!after_bootmem) 271aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu adjust_range_page_size_mask(mr, nr_range); 272aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 273f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 274365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n", 275365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas mr[i].start, mr[i].end - 1, 276f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 277f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 278f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 2794e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu return nr_range; 2804e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu} 2814e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 28266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shinstatic struct range pfn_mapped[E820_X_MAX]; 28366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shinstatic int nr_pfn_mapped; 28466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 28566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shinstatic void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn) 28666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 28766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX, 28866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped, start_pfn, end_pfn); 28966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX); 29066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 29166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin max_pfn_mapped = max(max_pfn_mapped, end_pfn); 29266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 29366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin if (start_pfn < (1UL<<(32-PAGE_SHIFT))) 29466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin max_low_pfn_mapped = max(max_low_pfn_mapped, 29566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin min(end_pfn, 1UL<<(32-PAGE_SHIFT))); 29666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 29766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 29866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shinbool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn) 29966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 30066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin int i; 30166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 30266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin for (i = 0; i < nr_pfn_mapped; i++) 30366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin if ((start_pfn >= pfn_mapped[i].start) && 30466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin (end_pfn <= pfn_mapped[i].end)) 30566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin return true; 30666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 30766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin return false; 30866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 30966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 3102086fe1159a9a75233b533986ccfcbd192bd9372Yinghai Lu/* 3114e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * Setup the direct mapping of the physical memory at PAGE_OFFSET. 3124e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * This runs before bootmem is initialized and gets pages directly from 3134e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * the physical memory. To access them they are temporarily mapped. 3144e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu */ 3154e33e06555329e93523b3d2590b9210bf84120a3Yinghai Luunsigned long __init_refok init_memory_mapping(unsigned long start, 3164e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long end) 3174e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu{ 3184e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu struct map_range mr[NR_RANGE_MR]; 3194e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long ret = 0; 3204e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu int nr_range, i; 3214e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 3224e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n", 3234e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu start, end - 1); 3244e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 3254e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu memset(mr, 0, sizeof(mr)); 3264e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu nr_range = split_mem_range(mr, 0, start, end); 3274e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 328f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 329f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 330f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask); 331f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 332f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 333f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg early_ioremap_page_table_range_init(); 334f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 335f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg load_cr3(swapper_pg_dir); 336f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 337f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 338f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg __flush_tlb_all(); 339f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 34066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT); 34166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 342c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu return ret >> PAGE_SHIFT; 343c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu} 344c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu 34566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin/* 3468d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu * would have hole in the middle or ends, and only ram parts will be mapped. 34766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin */ 3488d57470d8f859635deffe3919d7d4867b488b85aYinghai Lustatic unsigned long __init init_range_memory_mapping( 3498d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long range_start, 350f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu unsigned long range_end) 35166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 35266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin unsigned long start_pfn, end_pfn; 3538d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long mapped_ram_size = 0; 35466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin int i; 35566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 35666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { 35766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin u64 start = (u64)start_pfn << PAGE_SHIFT; 35866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin u64 end = (u64)end_pfn << PAGE_SHIFT; 35966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 360f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu if (end <= range_start) 36166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin continue; 36266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 363f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu if (start < range_start) 364f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu start = range_start; 365f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu 366f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu if (start >= range_end) 36766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin continue; 36866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 369f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu if (end > range_end) 370f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu end = range_end; 37166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 372f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu init_memory_mapping(start, end); 3738d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 3748d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu mapped_ram_size += end - start; 37566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin } 3768d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 3778d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu return mapped_ram_size; 37866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 37966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 3808d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu/* (PUD_SHIFT-PMD_SHIFT)/2 */ 3818d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu#define STEP_SIZE_SHIFT 5 382c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Luvoid __init init_mem_mapping(void) 383c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu{ 3848d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long end, real_end, start, last_start; 3858d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long step_size; 3868d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long addr; 3878d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long mapped_ram_size = 0; 3888d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long new_mapped_ram_size; 389ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu 390c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu probe_page_size_mask(); 391c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu 392c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu#ifdef CONFIG_X86_64 393ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu end = max_pfn << PAGE_SHIFT; 394c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu#else 395ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu end = max_low_pfn << PAGE_SHIFT; 396c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu#endif 397ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu 398f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu /* the ISA range is always mapped regardless of memory holes */ 399f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu init_memory_mapping(0, ISA_END_ADDRESS); 4008d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4018d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu /* xen has big range in reserved near end of ram, skip it at first */ 4028d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE, 4038d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu PAGE_SIZE); 4048d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu real_end = addr + PMD_SIZE; 4058d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4068d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu /* step_size need to be small so pgt_buf from BRK could cover it */ 4078d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu step_size = PMD_SIZE; 4088d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu max_pfn_mapped = 0; /* will get exact value next */ 4098d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu min_pfn_mapped = real_end >> PAGE_SHIFT; 4108d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start = start = real_end; 4118d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu while (last_start > ISA_END_ADDRESS) { 4128d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (last_start > step_size) { 4138d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu start = round_down(last_start - 1, step_size); 4148d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (start < ISA_END_ADDRESS) 4158d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu start = ISA_END_ADDRESS; 4168d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu } else 4178d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu start = ISA_END_ADDRESS; 4188d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu new_mapped_ram_size = init_range_memory_mapping(start, 4198d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start); 4208d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start = start; 4218d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu min_pfn_mapped = last_start >> PAGE_SHIFT; 4228d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu /* only increase step_size after big range get mapped */ 4238d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (new_mapped_ram_size > mapped_ram_size) 4248d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu step_size <<= STEP_SIZE_SHIFT; 4258d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu mapped_ram_size += new_mapped_ram_size; 4268d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu } 4278d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4288d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (real_end < end) 4298d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu init_range_memory_mapping(real_end, end); 4308d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 431f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu#ifdef CONFIG_X86_64 432f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu if (max_pfn > max_low_pfn) { 433f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu /* can we preseve max_low_pfn ?*/ 434f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu max_low_pfn = max_pfn; 435f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu } 436f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu#endif 437c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu early_memtest(0, max_pfn_mapped << PAGE_SHIFT); 43822ddfcaa0dbae992332381d41b8a1fbc72269a13Yinghai Lu} 439e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 440540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg/* 441540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * devmem_is_allowed() checks to see if /dev/mem access to a certain address 442540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * is valid. The argument is a physical page number. 443540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 444540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 445540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * On x86, access has to be given to the first megabyte of ram because that area 446540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * contains bios code and data regions used by X and dosemu and similar apps. 447540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * Access has to be given to non-kernel-ram areas as well, these contain the PCI 448540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * mmio resources as well as potential bios/acpi data regions. 449540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg */ 450540aca06b737cc38965b52eeceefba3d24376461Pekka Enbergint devmem_is_allowed(unsigned long pagenr) 451540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg{ 45273e8f3d7e2cb23614d5115703d76d8e54764b641T Makphaibulchoke if (pagenr < 256) 453540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 454540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 455540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 456540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (!page_is_ram(pagenr)) 457540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 458540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 459540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg} 460540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 461e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_init_pages(char *what, unsigned long begin, unsigned long end) 462e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 463c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long addr; 464c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long begin_aligned, end_aligned; 465e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 466c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* Make sure boundaries are page aligned */ 467c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin_aligned = PAGE_ALIGN(begin); 468c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end_aligned = end & PAGE_MASK; 469c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 470c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 471c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin = begin_aligned; 472c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end = end_aligned; 473c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu } 474c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 475c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (begin >= end) 476e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg return; 477e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 478c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu addr = begin; 479c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 480e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 481e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * If debugging page accesses then do not free this memory but 482e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * mark them not present - any buggy init-section access will 483e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * create a kernel page fault: 484e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 485e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#ifdef CONFIG_DEBUG_PAGEALLOC 486365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n", 487365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas begin, end - 1); 488e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 489e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#else 490e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 491e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * We just marked the kernel text read only above, now that 492e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * we are going to free part of that, we need to make that 4935bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET * writeable and non-executable first. 494e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 4955bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); 496e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 497e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 498e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 499e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 500e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg for (; addr < end; addr += PAGE_SIZE) { 501e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg ClearPageReserved(virt_to_page(addr)); 502e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg init_page_count(virt_to_page(addr)); 503c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 504e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_page(addr); 505e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg totalram_pages++; 506e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg } 507e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#endif 508e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 509e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 510e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_initmem(void) 511e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 512e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_init_pages("unused kernel memory", 513e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_begin), 514e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_end)); 515e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 516731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg 517731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#ifdef CONFIG_BLK_DEV_INITRD 5180d26d1d873a302828e064737746c53a2689e6c0fJan Beulichvoid __init free_initrd_mem(unsigned long start, unsigned long end) 519731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg{ 520c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* 521c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * end could be not aligned, and We can not align that, 522c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * decompresser could be confused by aligned initrd_end 523c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * We already reserve the end partial page before in 524c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - i386_start_kernel() 525c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - x86_64_start_kernel() 526c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - relocate_initrd() 527c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * So here We can do PAGE_ALIGN() safely to get partial page to be freed 528c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu */ 529c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu free_init_pages("initrd memory", start, PAGE_ALIGN(end)); 530731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg} 531731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#endif 532176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 533176239153049a023d060ce95b05f7ef31667e362Pekka Enbergvoid __init zone_sizes_init(void) 534176239153049a023d060ce95b05f7ef31667e362Pekka Enberg{ 535176239153049a023d060ce95b05f7ef31667e362Pekka Enberg unsigned long max_zone_pfns[MAX_NR_ZONES]; 536176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 537176239153049a023d060ce95b05f7ef31667e362Pekka Enberg memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 538176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 539176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA 540176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 541176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 542176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA32 543176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 544176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 545176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 546176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_HIGHMEM 547176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_HIGHMEM] = max_pfn; 548176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 549176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 550176239153049a023d060ce95b05f7ef31667e362Pekka Enberg free_area_init_nodes(max_zone_pfns); 551176239153049a023d060ce95b05f7ef31667e362Pekka Enberg} 552176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 553