15a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/gfp.h> 22c1b284e4fa260fd922b9a65c99169e2630c6862Jaswinder Singh Rajput#include <linux/initrd.h> 3540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <linux/ioport.h> 4e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <linux/swap.h> 5a9ce6bc15100023b411f8117e53a016d61889800Yinghai Lu#include <linux/memblock.h> 6176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <linux/bootmem.h> /* for max_low_pfn */ 7540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 8e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/cacheflush.h> 9f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/e820.h> 104fcb208391be5cf82c6fe2779c5eb9245ac97e91Pekka Enberg#include <asm/init.h> 11e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/page.h> 12540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <asm/page_types.h> 13e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/sections.h> 14498343967613183611ac37dccb2846496d954c06Jan Beulich#include <asm/setup.h> 15f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/tlbflush.h> 169518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg#include <asm/tlb.h> 1776c06927f2a78143763dcff9b4c362d15eb29cc2Jaswinder Singh Rajput#include <asm/proto.h> 18176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <asm/dma.h> /* for MAX_DMA_PFN */ 19cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu#include <asm/microcode.h> 209518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg 21d17d8f9dedb9dd76fd540a5c497101529d9eb25aDave Hansen/* 22d17d8f9dedb9dd76fd540a5c497101529d9eb25aDave Hansen * We need to define the tracepoints somewhere, and tlb.c 23d17d8f9dedb9dd76fd540a5c497101529d9eb25aDave Hansen * is only compied when SMP=y. 24d17d8f9dedb9dd76fd540a5c497101529d9eb25aDave Hansen */ 25d17d8f9dedb9dd76fd540a5c497101529d9eb25aDave Hansen#define CREATE_TRACE_POINTS 26d17d8f9dedb9dd76fd540a5c497101529d9eb25aDave Hansen#include <trace/events/tlb.h> 27d17d8f9dedb9dd76fd540a5c497101529d9eb25aDave Hansen 285c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu#include "mm_internal.h" 295c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 30cf47065961b48727b4e47bc3e2e67f4996878437Yinghai Lustatic unsigned long __initdata pgt_buf_start; 31cf47065961b48727b4e47bc3e2e67f4996878437Yinghai Lustatic unsigned long __initdata pgt_buf_end; 32cf47065961b48727b4e47bc3e2e67f4996878437Yinghai Lustatic unsigned long __initdata pgt_buf_top; 33f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 349985b4c6fa7d660f685918a58282275e9e35d8e0Yinghai Lustatic unsigned long min_pfn_mapped; 359985b4c6fa7d660f685918a58282275e9e35d8e0Yinghai Lu 36c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lustatic bool __initdata can_use_brk_pgt = true; 37c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu 38ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini/* 39ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * Pages returned are already directly mapped. 40ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * 41ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * Changing that is likely to break Xen, see commit: 42ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * 43ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * 279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve 44ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * 45ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * for detailed information. 46ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini */ 4722c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu__ref void *alloc_low_pages(unsigned int num) 485c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu{ 495c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu unsigned long pfn; 5022c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu int i; 515c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 525c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (after_bootmem) { 5322c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu unsigned int order; 545c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 5522c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu order = get_order((unsigned long)num << PAGE_SHIFT); 5622c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK | 5722c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu __GFP_ZERO, order); 585c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu } 595c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 60c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) { 615c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu unsigned long ret; 625c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (min_pfn_mapped >= max_pfn_mapped) 63d4dd100f2ebb2bc9aca5378ad3cb333b6117069cZhi Yong Wu panic("alloc_low_pages: ran out of memory"); 645c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT, 655c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu max_pfn_mapped << PAGE_SHIFT, 6622c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu PAGE_SIZE * num , PAGE_SIZE); 675c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (!ret) 68d4dd100f2ebb2bc9aca5378ad3cb333b6117069cZhi Yong Wu panic("alloc_low_pages: can not alloc memory"); 6922c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu memblock_reserve(ret, PAGE_SIZE * num); 705c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu pfn = ret >> PAGE_SHIFT; 7122c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu } else { 7222c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu pfn = pgt_buf_end; 7322c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu pgt_buf_end += num; 74c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n", 75c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1); 7622c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu } 7722c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu 7822c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu for (i = 0; i < num; i++) { 7922c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu void *adr; 8022c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu 8122c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu adr = __va((pfn + i) << PAGE_SHIFT); 8222c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu clear_page(adr); 8322c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu } 845c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 8522c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu return __va(pfn << PAGE_SHIFT); 865c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu} 875c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 88527bf129f9a780e11b251cf2467dc30118a57d16Yinghai Lu/* need 3 4k for initial PMD_SIZE, 3 4k for 0-ISA_END_ADDRESS */ 89527bf129f9a780e11b251cf2467dc30118a57d16Yinghai Lu#define INIT_PGT_BUF_SIZE (6 * PAGE_SIZE) 908d57470d8f859635deffe3919d7d4867b488b85aYinghai LuRESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE); 918d57470d8f859635deffe3919d7d4867b488b85aYinghai Luvoid __init early_alloc_pgt_buf(void) 928d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu{ 938d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long tables = INIT_PGT_BUF_SIZE; 948d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu phys_addr_t base; 958d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 968d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu base = __pa(extend_brk(tables, PAGE_SIZE)); 978d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 988d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_start = base >> PAGE_SHIFT; 998d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_end = pgt_buf_start; 1008d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); 1018d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu} 1028d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 103f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint after_bootmem; 104f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 105f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint direct_gbpages 106f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_DIRECT_GBPAGES 107f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg = 1 108f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 109f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg; 110f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 111148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lustatic void __init init_gbpages(void) 112148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu{ 113148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu#ifdef CONFIG_X86_64 114148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu if (direct_gbpages && cpu_has_gbpages) 115148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu printk(KERN_INFO "Using GB pages for direct mapping\n"); 116148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu else 117148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu direct_gbpages = 0; 118148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu#endif 119148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu} 120148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu 121844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shinstruct map_range { 122844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long start; 123844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long end; 124844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned page_size_mask; 125844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin}; 126844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin 127fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lustatic int page_size_mask; 128f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 12922ddfcaa0dbae992332381d41b8a1fbc72269a13Yinghai Lustatic void __init probe_page_size_mask(void) 130fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu{ 131148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu init_gbpages(); 132148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu 133fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK) 134fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* 135fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 136fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * This will simplify cpa(), which otherwise needs to support splitting 137fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * large pages into small in interrupt context, etc. 138fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu */ 139fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (direct_gbpages) 140fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu page_size_mask |= 1 << PG_LEVEL_1G; 141fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pse) 142fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu page_size_mask |= 1 << PG_LEVEL_2M; 143fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu#endif 144fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu 145fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* Enable PSE if available */ 146fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pse) 147fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu set_in_cr4(X86_CR4_PSE); 148fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu 149fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* Enable PGE if available */ 150fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pge) { 151fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu set_in_cr4(X86_CR4_PGE); 152fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu __supported_pte_mask |= _PAGE_GLOBAL; 153fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu } 154fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu} 155279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini 156f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 157f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 3 158f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 159f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 5 160f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 161f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 162dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulichstatic int __meminit save_mr(struct map_range *mr, int nr_range, 163dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long start_pfn, unsigned long end_pfn, 164dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long page_size_mask) 165f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 166f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 167f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (nr_range >= NR_RANGE_MR) 168f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("run out of range for init_memory_mapping\n"); 169f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].start = start_pfn<<PAGE_SHIFT; 170f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].end = end_pfn<<PAGE_SHIFT; 171f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].page_size_mask = page_size_mask; 172f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range++; 173f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 174f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 175f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return nr_range; 176f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 177f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 178aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu/* 179aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu * adjust the page_size_mask for small range to go with 180aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu * big page size instead small one if nearby are ram too. 181aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu */ 182aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lustatic void __init_refok adjust_range_page_size_mask(struct map_range *mr, 183aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu int nr_range) 184aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu{ 185aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu int i; 186aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 187aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu for (i = 0; i < nr_range; i++) { 188aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((page_size_mask & (1<<PG_LEVEL_2M)) && 189aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) { 190aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long start = round_down(mr[i].start, PMD_SIZE); 191aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long end = round_up(mr[i].end, PMD_SIZE); 192aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 193aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu#ifdef CONFIG_X86_32 194aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((end >> PAGE_SHIFT) > max_low_pfn) 195aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu continue; 196aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu#endif 197aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 198aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if (memblock_is_region_memory(start, end - start)) 199aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu mr[i].page_size_mask |= 1<<PG_LEVEL_2M; 200aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 201aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((page_size_mask & (1<<PG_LEVEL_1G)) && 202aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) { 203aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long start = round_down(mr[i].start, PUD_SIZE); 204aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long end = round_up(mr[i].end, PUD_SIZE); 205aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 206aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if (memblock_is_region_memory(start, end - start)) 207aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu mr[i].page_size_mask |= 1<<PG_LEVEL_1G; 208aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 209aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 210aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu} 211aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 2124e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lustatic int __meminit split_mem_range(struct map_range *mr, int nr_range, 2134e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long start, 2144e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long end) 215f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 2162e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu unsigned long start_pfn, end_pfn, limit_pfn; 2171829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu unsigned long pfn; 2184e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu int i; 219f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 2202e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu limit_pfn = PFN_DOWN(end); 2212e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu 222f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* head if not big page alignment ? */ 2231829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = start_pfn = PFN_DOWN(start); 224f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 225f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 226f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Don't use a large page for the first 2/4MB of memory 227f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * because there are often fixed size MTRRs in there 228f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * and overlapping MTRRs into large pages can cause 229f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * slowdowns. 230f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 2311829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu if (pfn == 0) 23284d770019bb990dcd8013d9d08174d0e1516b517Yinghai Lu end_pfn = PFN_DOWN(PMD_SIZE); 233f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg else 2341829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 235f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 2361829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 237f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 2382e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu if (end_pfn > limit_pfn) 2392e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = limit_pfn; 240f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 241f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 2421829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = end_pfn; 243f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 244f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 245f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (2M) range */ 2461829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 247f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 2482e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 249f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 2501829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); 2512e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE))) 2522e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 253f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 254f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 255f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 256f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 257f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 2581829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = end_pfn; 259f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 260f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 261f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_64 262f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (1G) range */ 2631829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); 2642e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE)); 265f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 266f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 267f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & 268f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 2691829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = end_pfn; 270f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 271f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 272f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (1G) alignment */ 2731829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 2742e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 275f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 276f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 277f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 2781829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = end_pfn; 279f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 280f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 281f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 282f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (2M) alignment */ 2831829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu start_pfn = pfn; 2842e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = limit_pfn; 285f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 286f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 2877de3d66b1387ddf5a37d9689e5eb8510fb75c765Yinghai Lu if (!after_bootmem) 2887de3d66b1387ddf5a37d9689e5eb8510fb75c765Yinghai Lu adjust_range_page_size_mask(mr, nr_range); 2897de3d66b1387ddf5a37d9689e5eb8510fb75c765Yinghai Lu 290f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* try to merge same page size and continuous */ 291f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 292f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long old_start; 293f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (mr[i].end != mr[i+1].start || 294f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask != mr[i+1].page_size_mask) 295f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg continue; 296f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* move it */ 297f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg old_start = mr[i].start; 298f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memmove(&mr[i], &mr[i+1], 299f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (nr_range - 1 - i) * sizeof(struct map_range)); 300f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i--].start = old_start; 301f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range--; 302f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 303f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 304f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 305365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n", 306365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas mr[i].start, mr[i].end - 1, 307f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 308f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 309f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 3104e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu return nr_range; 3114e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu} 3124e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 3130e691cf824f76adefb4498fe39c300aba2c2575aYinghai Lustruct range pfn_mapped[E820_X_MAX]; 3140e691cf824f76adefb4498fe39c300aba2c2575aYinghai Luint nr_pfn_mapped; 31566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 31666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shinstatic void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn) 31766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 31866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX, 31966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped, start_pfn, end_pfn); 32066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX); 32166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 32266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin max_pfn_mapped = max(max_pfn_mapped, end_pfn); 32366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 32466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin if (start_pfn < (1UL<<(32-PAGE_SHIFT))) 32566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin max_low_pfn_mapped = max(max_low_pfn_mapped, 32666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin min(end_pfn, 1UL<<(32-PAGE_SHIFT))); 32766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 32866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 32966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shinbool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn) 33066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 33166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin int i; 33266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 33366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin for (i = 0; i < nr_pfn_mapped; i++) 33466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin if ((start_pfn >= pfn_mapped[i].start) && 33566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin (end_pfn <= pfn_mapped[i].end)) 33666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin return true; 33766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 33866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin return false; 33966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 34066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 3412086fe1159a9a75233b533986ccfcbd192bd9372Yinghai Lu/* 3424e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * Setup the direct mapping of the physical memory at PAGE_OFFSET. 3434e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * This runs before bootmem is initialized and gets pages directly from 3444e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * the physical memory. To access them they are temporarily mapped. 3454e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu */ 3464e33e06555329e93523b3d2590b9210bf84120a3Yinghai Luunsigned long __init_refok init_memory_mapping(unsigned long start, 3474e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long end) 3484e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu{ 3494e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu struct map_range mr[NR_RANGE_MR]; 3504e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long ret = 0; 3514e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu int nr_range, i; 3524e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 3534e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n", 3544e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu start, end - 1); 3554e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 3564e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu memset(mr, 0, sizeof(mr)); 3574e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu nr_range = split_mem_range(mr, 0, start, end); 3584e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 359f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 360f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 361f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask); 362f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 36366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT); 36466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 365c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu return ret >> PAGE_SHIFT; 366c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu} 367c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu 36866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin/* 369cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * We need to iterate through the E820 memory map and create direct mappings 370cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * for only E820_RAM and E820_KERN_RESERVED regions. We cannot simply 371cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * create direct mappings for all pfns from [0 to max_low_pfn) and 372cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * [4GB to max_pfn) because of possible memory holes in high addresses 373cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * that cannot be marked as UC by fixed/variable range MTRRs. 374cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * Depending on the alignment of E820 ranges, this may possibly result 375cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * in using smaller size (i.e. 4K instead of 2M or 1G) page tables. 376cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * 377cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * init_mem_mapping() calls init_range_memory_mapping() with big range. 378cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * That range would have hole in the middle or ends, and only ram parts 379cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * will be mapped in init_range_memory_mapping(). 38066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin */ 3818d57470d8f859635deffe3919d7d4867b488b85aYinghai Lustatic unsigned long __init init_range_memory_mapping( 382b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu unsigned long r_start, 383b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu unsigned long r_end) 38466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 38566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin unsigned long start_pfn, end_pfn; 3868d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long mapped_ram_size = 0; 38766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin int i; 38866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 38966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { 390b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end); 391b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end); 392b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu if (start >= end) 39366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin continue; 39466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 395c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu /* 396c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu * if it is overlapping with brk pgt, we need to 397c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu * alloc pgt buf from memblock instead. 398c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu */ 399c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >= 400c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu min(end, (u64)pgt_buf_top<<PAGE_SHIFT); 401f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu init_memory_mapping(start, end); 4028d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu mapped_ram_size += end - start; 403c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu can_use_brk_pgt = true; 40466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin } 4058d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4068d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu return mapped_ram_size; 40766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 40866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 4096979287a7df66a92d6f308338e972a406f9ef842Yinghai Lustatic unsigned long __init get_new_step_size(unsigned long step_size) 4106979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu{ 4116979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu /* 4126979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * Explain why we shift by 5 and why we don't have to worry about 4136979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * 'step_size << 5' overflowing: 4146979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * 4156979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * initial mapped size is PMD_SIZE (2M). 4166979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * We can not set step_size to be PUD_SIZE (1G) yet. 4176979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * In worse case, when we cross the 1G boundary, and 4186979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k) 4196979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * to map 1G range with PTE. Use 5 as shift for now. 4206979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * 4216979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * Don't need to worry about overflow, on 32bit, when step_size 4226979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * is 0, round_down() returns 0 for start, and that turns it 4236979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu * into 0x100000000ULL. 4246979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu */ 4256979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu return step_size << 5; 4266979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu} 4276979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu 4280167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen/** 4290167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen * memory_map_top_down - Map [map_start, map_end) top down 4300167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen * @map_start: start address of the target memory range 4310167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen * @map_end: end address of the target memory range 4320167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen * 4330167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen * This function will setup direct mapping for memory range 4340167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen * [map_start, map_end) in top-down. That said, the page tables 4350167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen * will be allocated at the end of the memory, and we map the 4360167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen * memory in top-down. 4370167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen */ 4380167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chenstatic void __init memory_map_top_down(unsigned long map_start, 4390167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen unsigned long map_end) 440c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu{ 4410167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen unsigned long real_end, start, last_start; 4428d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long step_size; 4438d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long addr; 4448d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long mapped_ram_size = 0; 4458d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long new_mapped_ram_size; 446ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu 44798e7a989979b185f49e86ddaed2ad6890299d9f0Yinghai Lu /* xen has big range in reserved near end of ram, skip it at first.*/ 4480167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen addr = memblock_find_in_range(map_start, map_end, PMD_SIZE, PMD_SIZE); 4498d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu real_end = addr + PMD_SIZE; 4508d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4518d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu /* step_size need to be small so pgt_buf from BRK could cover it */ 4528d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu step_size = PMD_SIZE; 4538d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu max_pfn_mapped = 0; /* will get exact value next */ 4548d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu min_pfn_mapped = real_end >> PAGE_SHIFT; 4558d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start = start = real_end; 456cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei 457cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei /* 458cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * We start from the top (end of memory) and go to the bottom. 459cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * The memblock_find_in_range() gets us a block of RAM from the 460cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages 461cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei * for page table. 462cf8b166d5c1c89aad6c436a954fa40fd18a75bfbZhang Yanfei */ 4630167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen while (last_start > map_start) { 4648d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (last_start > step_size) { 4658d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu start = round_down(last_start - 1, step_size); 4660167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen if (start < map_start) 4670167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen start = map_start; 4688d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu } else 4690167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen start = map_start; 4708d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu new_mapped_ram_size = init_range_memory_mapping(start, 4718d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start); 4728d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start = start; 4738d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu min_pfn_mapped = last_start >> PAGE_SHIFT; 4748d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu /* only increase step_size after big range get mapped */ 4758d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (new_mapped_ram_size > mapped_ram_size) 4766979287a7df66a92d6f308338e972a406f9ef842Yinghai Lu step_size = get_new_step_size(step_size); 4778d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu mapped_ram_size += new_mapped_ram_size; 4788d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu } 4798d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4800167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen if (real_end < map_end) 4810167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen init_range_memory_mapping(real_end, map_end); 4820167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen} 4830167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen 484b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen/** 485b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * memory_map_bottom_up - Map [map_start, map_end) bottom up 486b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * @map_start: start address of the target memory range 487b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * @map_end: end address of the target memory range 488b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * 489b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * This function will setup direct mapping for memory range 490b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * [map_start, map_end) in bottom-up. Since we have limited the 491b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * bottom-up allocation above the kernel, the page tables will 492b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * be allocated just above the kernel and we map the memory 493b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * in [map_start, map_end) in bottom-up. 494b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen */ 495b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chenstatic void __init memory_map_bottom_up(unsigned long map_start, 496b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen unsigned long map_end) 497b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen{ 498b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen unsigned long next, new_mapped_ram_size, start; 499b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen unsigned long mapped_ram_size = 0; 500b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen /* step_size need to be small so pgt_buf from BRK could cover it */ 501b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen unsigned long step_size = PMD_SIZE; 502b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen 503b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen start = map_start; 504b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen min_pfn_mapped = start >> PAGE_SHIFT; 505b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen 506b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen /* 507b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * We start from the bottom (@map_start) and go to the top (@map_end). 508b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * The memblock_find_in_range() gets us a block of RAM from the 509b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages 510b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * for page table. 511b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen */ 512b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen while (start < map_end) { 513b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen if (map_end - start > step_size) { 514b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen next = round_up(start + 1, step_size); 515b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen if (next > map_end) 516b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen next = map_end; 517b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen } else 518b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen next = map_end; 519b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen 520b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen new_mapped_ram_size = init_range_memory_mapping(start, next); 521b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen start = next; 522b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen 523b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen if (new_mapped_ram_size > mapped_ram_size) 524b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen step_size = get_new_step_size(step_size); 525b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen mapped_ram_size += new_mapped_ram_size; 526b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen } 527b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen} 528b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen 5290167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chenvoid __init init_mem_mapping(void) 5300167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen{ 5310167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen unsigned long end; 5320167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen 5330167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen probe_page_size_mask(); 5340167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen 5350167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen#ifdef CONFIG_X86_64 5360167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen end = max_pfn << PAGE_SHIFT; 5370167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen#else 5380167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen end = max_low_pfn << PAGE_SHIFT; 5390167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen#endif 5400167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen 5410167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen /* the ISA range is always mapped regardless of memory holes */ 5420167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen init_memory_mapping(0, ISA_END_ADDRESS); 5430167d7d8b0beb4cf12076b47e4dc73897ae5acb0Tang Chen 544b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen /* 545b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * If the allocation is in bottom-up direction, we setup direct mapping 546b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * in bottom-up, otherwise we setup direct mapping in top-down. 547b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen */ 548b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen if (memblock_bottom_up()) { 549b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen unsigned long kernel_end = __pa_symbol(_end); 550b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen 551b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen /* 552b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * we need two separate calls here. This is because we want to 553b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * allocate page tables above the kernel. So we first map 554b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * [kernel_end, end) to make memory above the kernel be mapped 555b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * as soon as possible. And then use page tables allocated above 556b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen * the kernel to map [ISA_END_ADDRESS, kernel_end). 557b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen */ 558b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen memory_map_bottom_up(kernel_end, end); 559b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen memory_map_bottom_up(ISA_END_ADDRESS, kernel_end); 560b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen } else { 561b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen memory_map_top_down(ISA_END_ADDRESS, end); 562b959ed6c73845aebf51afb8f76bb74b9388344d2Tang Chen } 5638d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 564f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu#ifdef CONFIG_X86_64 565f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu if (max_pfn > max_low_pfn) { 566f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu /* can we preseve max_low_pfn ?*/ 567f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu max_low_pfn = max_pfn; 568f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu } 569719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu#else 570719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu early_ioremap_page_table_range_init(); 5718170e6bed465b4b0c7687f93e9948aca4358a33bH. Peter Anvin#endif 5728170e6bed465b4b0c7687f93e9948aca4358a33bH. Peter Anvin 573719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu load_cr3(swapper_pg_dir); 574719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu __flush_tlb_all(); 575719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu 576c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu early_memtest(0, max_pfn_mapped << PAGE_SHIFT); 57722ddfcaa0dbae992332381d41b8a1fbc72269a13Yinghai Lu} 578e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 579540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg/* 580540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * devmem_is_allowed() checks to see if /dev/mem access to a certain address 581540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * is valid. The argument is a physical page number. 582540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 583540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 584540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * On x86, access has to be given to the first megabyte of ram because that area 585540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * contains bios code and data regions used by X and dosemu and similar apps. 586540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * Access has to be given to non-kernel-ram areas as well, these contain the PCI 587540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * mmio resources as well as potential bios/acpi data regions. 588540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg */ 589540aca06b737cc38965b52eeceefba3d24376461Pekka Enbergint devmem_is_allowed(unsigned long pagenr) 590540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg{ 59173e8f3d7e2cb23614d5115703d76d8e54764b641T Makphaibulchoke if (pagenr < 256) 592540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 593540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 594540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 595540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (!page_is_ram(pagenr)) 596540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 597540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 598540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg} 599540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 600e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_init_pages(char *what, unsigned long begin, unsigned long end) 601e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 602c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long begin_aligned, end_aligned; 603e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 604c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* Make sure boundaries are page aligned */ 605c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin_aligned = PAGE_ALIGN(begin); 606c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end_aligned = end & PAGE_MASK; 607c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 608c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 609c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin = begin_aligned; 610c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end = end_aligned; 611c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu } 612c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 613c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (begin >= end) 614e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg return; 615e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 616e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 617e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * If debugging page accesses then do not free this memory but 618e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * mark them not present - any buggy init-section access will 619e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * create a kernel page fault: 620e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 621e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#ifdef CONFIG_DEBUG_PAGEALLOC 622365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n", 623365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas begin, end - 1); 624e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 625e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#else 626e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 627e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * We just marked the kernel text read only above, now that 628e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * we are going to free part of that, we need to make that 6295bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET * writeable and non-executable first. 630e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 6315bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); 632e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 633e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 634c88442ec45f30d587b38b935a14acde4e217a926Jiang Liu free_reserved_area((void *)begin, (void *)end, POISON_FREE_INITMEM, what); 635e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#endif 636e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 637e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 638e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_initmem(void) 639e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 640c88442ec45f30d587b38b935a14acde4e217a926Jiang Liu free_init_pages("unused kernel", 641e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_begin), 642e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_end)); 643e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 644731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg 645731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#ifdef CONFIG_BLK_DEV_INITRD 6460d26d1d873a302828e064737746c53a2689e6c0fJan Beulichvoid __init free_initrd_mem(unsigned long start, unsigned long end) 647731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg{ 648cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu#ifdef CONFIG_MICROCODE_EARLY 649cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu /* 650cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu * Remember, initrd memory may contain microcode or other useful things. 651cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu * Before we lose initrd mem, we need to find a place to hold them 652cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu * now that normal virtual memory is enabled. 653cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu */ 654cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu save_microcode_in_initrd(); 655cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu#endif 656cd745be89e1580e8a1b47454a39f97f9c5c4b1e0Fenghua Yu 657c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* 658c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * end could be not aligned, and We can not align that, 659c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * decompresser could be confused by aligned initrd_end 660c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * We already reserve the end partial page before in 661c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - i386_start_kernel() 662c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - x86_64_start_kernel() 663c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - relocate_initrd() 664c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * So here We can do PAGE_ALIGN() safely to get partial page to be freed 665c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu */ 666c88442ec45f30d587b38b935a14acde4e217a926Jiang Liu free_init_pages("initrd", start, PAGE_ALIGN(end)); 667731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg} 668731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#endif 669176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 670176239153049a023d060ce95b05f7ef31667e362Pekka Enbergvoid __init zone_sizes_init(void) 671176239153049a023d060ce95b05f7ef31667e362Pekka Enberg{ 672176239153049a023d060ce95b05f7ef31667e362Pekka Enberg unsigned long max_zone_pfns[MAX_NR_ZONES]; 673176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 674176239153049a023d060ce95b05f7ef31667e362Pekka Enberg memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 675176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 676176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA 677176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 678176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 679176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA32 680176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 681176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 682176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 683176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_HIGHMEM 684176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_HIGHMEM] = max_pfn; 685176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 686176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 687176239153049a023d060ce95b05f7ef31667e362Pekka Enberg free_area_init_nodes(max_zone_pfns); 688176239153049a023d060ce95b05f7ef31667e362Pekka Enberg} 689176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 690