init.c revision 0e691cf824f76adefb4498fe39c300aba2c2575a
15a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/gfp.h> 22c1b284e4fa260fd922b9a65c99169e2630c6862Jaswinder Singh Rajput#include <linux/initrd.h> 3540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <linux/ioport.h> 4e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <linux/swap.h> 5a9ce6bc15100023b411f8117e53a016d61889800Yinghai Lu#include <linux/memblock.h> 6176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <linux/bootmem.h> /* for max_low_pfn */ 7540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 8e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/cacheflush.h> 9f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/e820.h> 104fcb208391be5cf82c6fe2779c5eb9245ac97e91Pekka Enberg#include <asm/init.h> 11e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/page.h> 12540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg#include <asm/page_types.h> 13e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#include <asm/sections.h> 14498343967613183611ac37dccb2846496d954c06Jan Beulich#include <asm/setup.h> 15f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#include <asm/tlbflush.h> 169518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg#include <asm/tlb.h> 1776c06927f2a78143763dcff9b4c362d15eb29cc2Jaswinder Singh Rajput#include <asm/proto.h> 18176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#include <asm/dma.h> /* for MAX_DMA_PFN */ 199518e0e4350a5ea8ca200ce320b28d6284a7b0cePekka Enberg 205c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu#include "mm_internal.h" 215c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 22cf47065961b48727b4e47bc3e2e67f4996878437Yinghai Lustatic unsigned long __initdata pgt_buf_start; 23cf47065961b48727b4e47bc3e2e67f4996878437Yinghai Lustatic unsigned long __initdata pgt_buf_end; 24cf47065961b48727b4e47bc3e2e67f4996878437Yinghai Lustatic unsigned long __initdata pgt_buf_top; 25f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 269985b4c6fa7d660f685918a58282275e9e35d8e0Yinghai Lustatic unsigned long min_pfn_mapped; 279985b4c6fa7d660f685918a58282275e9e35d8e0Yinghai Lu 28c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lustatic bool __initdata can_use_brk_pgt = true; 29c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu 30ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini/* 31ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * Pages returned are already directly mapped. 32ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * 33ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * Changing that is likely to break Xen, see commit: 34ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * 35ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * 279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve 36ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * 37ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini * for detailed information. 38ddd3509df8f8d4f1cf4784f559d702ce00dc8846Stefano Stabellini */ 3922c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu__ref void *alloc_low_pages(unsigned int num) 405c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu{ 415c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu unsigned long pfn; 4222c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu int i; 435c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 445c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (after_bootmem) { 4522c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu unsigned int order; 465c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 4722c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu order = get_order((unsigned long)num << PAGE_SHIFT); 4822c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK | 4922c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu __GFP_ZERO, order); 505c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu } 515c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 52c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) { 535c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu unsigned long ret; 545c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (min_pfn_mapped >= max_pfn_mapped) 555c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu panic("alloc_low_page: ran out of memory"); 565c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT, 575c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu max_pfn_mapped << PAGE_SHIFT, 5822c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu PAGE_SIZE * num , PAGE_SIZE); 595c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu if (!ret) 605c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu panic("alloc_low_page: can not alloc memory"); 6122c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu memblock_reserve(ret, PAGE_SIZE * num); 625c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu pfn = ret >> PAGE_SHIFT; 6322c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu } else { 6422c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu pfn = pgt_buf_end; 6522c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu pgt_buf_end += num; 66c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n", 67c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1); 6822c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu } 6922c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu 7022c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu for (i = 0; i < num; i++) { 7122c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu void *adr; 7222c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu 7322c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu adr = __va((pfn + i) << PAGE_SHIFT); 7422c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu clear_page(adr); 7522c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu } 765c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 7722c8ca2ac256bb681be791858b35502b5d37e73bYinghai Lu return __va(pfn << PAGE_SHIFT); 785c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu} 795c51bdbe4c74dce7996d0bbfa39974775cc3f13cYinghai Lu 808d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu/* need 4 4k for initial PMD_SIZE, 4k for 0-ISA_END_ADDRESS */ 818d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu#define INIT_PGT_BUF_SIZE (5 * PAGE_SIZE) 828d57470d8f859635deffe3919d7d4867b488b85aYinghai LuRESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE); 838d57470d8f859635deffe3919d7d4867b488b85aYinghai Luvoid __init early_alloc_pgt_buf(void) 848d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu{ 858d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long tables = INIT_PGT_BUF_SIZE; 868d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu phys_addr_t base; 878d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 888d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu base = __pa(extend_brk(tables, PAGE_SIZE)); 898d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 908d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_start = base >> PAGE_SHIFT; 918d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_end = pgt_buf_start; 928d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); 938d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu} 948d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 95f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint after_bootmem; 96f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 97f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enbergint direct_gbpages 98f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_DIRECT_GBPAGES 99f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg = 1 100f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 101f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg; 102f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 103148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lustatic void __init init_gbpages(void) 104148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu{ 105148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu#ifdef CONFIG_X86_64 106148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu if (direct_gbpages && cpu_has_gbpages) 107148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu printk(KERN_INFO "Using GB pages for direct mapping\n"); 108148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu else 109148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu direct_gbpages = 0; 110148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu#endif 111148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu} 112148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu 113844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shinstruct map_range { 114844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long start; 115844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned long end; 116844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin unsigned page_size_mask; 117844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin}; 118844ab6f993b1d32eb40512503d35ff6ad0c57030Jacob Shin 119fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lustatic int page_size_mask; 120f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 12122ddfcaa0dbae992332381d41b8a1fbc72269a13Yinghai Lustatic void __init probe_page_size_mask(void) 122fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu{ 123148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu init_gbpages(); 124148b20989e0b83cb301e1fcd9e987c7abde05333Yinghai Lu 125fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK) 126fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* 127fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 128fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * This will simplify cpa(), which otherwise needs to support splitting 129fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu * large pages into small in interrupt context, etc. 130fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu */ 131fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (direct_gbpages) 132fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu page_size_mask |= 1 << PG_LEVEL_1G; 133fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pse) 134fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu page_size_mask |= 1 << PG_LEVEL_2M; 135fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu#endif 136fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu 137fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* Enable PSE if available */ 138fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pse) 139fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu set_in_cr4(X86_CR4_PSE); 140fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu 141fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu /* Enable PGE if available */ 142fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu if (cpu_has_pge) { 143fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu set_in_cr4(X86_CR4_PGE); 144fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu __supported_pte_mask |= _PAGE_GLOBAL; 145fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu } 146fa62aafea9e415cd1efd8c4054106112fe809f19Yinghai Lu} 147279b706bf800b5967037f492dbe4fc5081ad5d0fStefano Stabellini 148f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 149f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 3 150f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 151f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#define NR_RANGE_MR 5 152f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 153f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 154dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulichstatic int __meminit save_mr(struct map_range *mr, int nr_range, 155dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long start_pfn, unsigned long end_pfn, 156dc9dd5cc854cde110d2421f3a11fec7597e059c1Jan Beulich unsigned long page_size_mask) 157f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 158f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 159f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (nr_range >= NR_RANGE_MR) 160f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg panic("run out of range for init_memory_mapping\n"); 161f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].start = start_pfn<<PAGE_SHIFT; 162f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].end = end_pfn<<PAGE_SHIFT; 163f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[nr_range].page_size_mask = page_size_mask; 164f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range++; 165f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 166f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 167f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg return nr_range; 168f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg} 169f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 170aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu/* 171aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu * adjust the page_size_mask for small range to go with 172aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu * big page size instead small one if nearby are ram too. 173aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu */ 174aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lustatic void __init_refok adjust_range_page_size_mask(struct map_range *mr, 175aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu int nr_range) 176aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu{ 177aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu int i; 178aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 179aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu for (i = 0; i < nr_range; i++) { 180aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((page_size_mask & (1<<PG_LEVEL_2M)) && 181aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) { 182aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long start = round_down(mr[i].start, PMD_SIZE); 183aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long end = round_up(mr[i].end, PMD_SIZE); 184aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 185aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu#ifdef CONFIG_X86_32 186aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((end >> PAGE_SHIFT) > max_low_pfn) 187aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu continue; 188aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu#endif 189aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 190aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if (memblock_is_region_memory(start, end - start)) 191aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu mr[i].page_size_mask |= 1<<PG_LEVEL_2M; 192aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 193aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if ((page_size_mask & (1<<PG_LEVEL_1G)) && 194aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) { 195aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long start = round_down(mr[i].start, PUD_SIZE); 196aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu unsigned long end = round_up(mr[i].end, PUD_SIZE); 197aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 198aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if (memblock_is_region_memory(start, end - start)) 199aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu mr[i].page_size_mask |= 1<<PG_LEVEL_1G; 200aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 201aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu } 202aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu} 203aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 2044e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lustatic int __meminit split_mem_range(struct map_range *mr, int nr_range, 2054e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long start, 2064e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long end) 207f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg{ 2082e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu unsigned long start_pfn, end_pfn, limit_pfn; 2091829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu unsigned long pfn; 2104e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu int i; 211f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 2122e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu limit_pfn = PFN_DOWN(end); 2132e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu 214f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* head if not big page alignment ? */ 2151829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = start_pfn = PFN_DOWN(start); 216f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 217f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* 218f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * Don't use a large page for the first 2/4MB of memory 219f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * because there are often fixed size MTRRs in there 220f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * and overlapping MTRRs into large pages can cause 221f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg * slowdowns. 222f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg */ 2231829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu if (pfn == 0) 22484d770019bb990dcd8013d9d08174d0e1516b517Yinghai Lu end_pfn = PFN_DOWN(PMD_SIZE); 225f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg else 2261829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 227f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 2281829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 229f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 2302e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu if (end_pfn > limit_pfn) 2312e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = limit_pfn; 232f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 233f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 2341829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = end_pfn; 235f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 236f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 237f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (2M) range */ 2381829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 239f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_32 2402e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 241f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#else /* CONFIG_X86_64 */ 2421829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); 2432e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE))) 2442e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 245f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 246f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 247f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 248f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 249f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 2501829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = end_pfn; 251f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 252f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 253f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#ifdef CONFIG_X86_64 254f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* big page (1G) range */ 2551829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); 2562e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE)); 257f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 258f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 259f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & 260f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 2611829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = end_pfn; 262f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 263f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 264f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (1G) alignment */ 2651829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 2662e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 267f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (start_pfn < end_pfn) { 268f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 269f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg page_size_mask & (1<<PG_LEVEL_2M)); 2701829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu pfn = end_pfn; 271f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 272f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg#endif 273f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 274f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* tail is not big page (2M) alignment */ 2751829ae9ad7380bf17333ab9ad1610631d9cb8664Yinghai Lu start_pfn = pfn; 2762e8059edb6fc5887e8e022d9e04fba26c9e0abcbYinghai Lu end_pfn = limit_pfn; 277f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 278f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 279f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* try to merge same page size and continuous */ 280f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 281f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg unsigned long old_start; 282f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg if (mr[i].end != mr[i+1].start || 283f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask != mr[i+1].page_size_mask) 284f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg continue; 285f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg /* move it */ 286f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg old_start = mr[i].start; 287f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg memmove(&mr[i], &mr[i+1], 288f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (nr_range - 1 - i) * sizeof(struct map_range)); 289f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i--].start = old_start; 290f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg nr_range--; 291f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg } 292f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 293aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu if (!after_bootmem) 294aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu adjust_range_page_size_mask(mr, nr_range); 295aeebe84cc96cde4181807bc67c300c550d0ef123Yinghai Lu 296f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 297365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n", 298365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas mr[i].start, mr[i].end - 1, 299f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 300f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 301f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 3024e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu return nr_range; 3034e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu} 3044e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 3050e691cf824f76adefb4498fe39c300aba2c2575aYinghai Lustruct range pfn_mapped[E820_X_MAX]; 3060e691cf824f76adefb4498fe39c300aba2c2575aYinghai Luint nr_pfn_mapped; 30766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 30866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shinstatic void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn) 30966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 31066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX, 31166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped, start_pfn, end_pfn); 31266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX); 31366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 31466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin max_pfn_mapped = max(max_pfn_mapped, end_pfn); 31566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 31666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin if (start_pfn < (1UL<<(32-PAGE_SHIFT))) 31766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin max_low_pfn_mapped = max(max_low_pfn_mapped, 31866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin min(end_pfn, 1UL<<(32-PAGE_SHIFT))); 31966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 32066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 32166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shinbool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn) 32266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 32366520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin int i; 32466520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 32566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin for (i = 0; i < nr_pfn_mapped; i++) 32666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin if ((start_pfn >= pfn_mapped[i].start) && 32766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin (end_pfn <= pfn_mapped[i].end)) 32866520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin return true; 32966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 33066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin return false; 33166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 33266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 3332086fe1159a9a75233b533986ccfcbd192bd9372Yinghai Lu/* 3344e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * Setup the direct mapping of the physical memory at PAGE_OFFSET. 3354e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * This runs before bootmem is initialized and gets pages directly from 3364e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu * the physical memory. To access them they are temporarily mapped. 3374e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu */ 3384e33e06555329e93523b3d2590b9210bf84120a3Yinghai Luunsigned long __init_refok init_memory_mapping(unsigned long start, 3394e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long end) 3404e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu{ 3414e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu struct map_range mr[NR_RANGE_MR]; 3424e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu unsigned long ret = 0; 3434e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu int nr_range, i; 3444e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 3454e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n", 3464e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu start, end - 1); 3474e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 3484e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu memset(mr, 0, sizeof(mr)); 3494e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu nr_range = split_mem_range(mr, 0, start, end); 3504e33e06555329e93523b3d2590b9210bf84120a3Yinghai Lu 351f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg for (i = 0; i < nr_range; i++) 352f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 353f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg mr[i].page_size_mask); 354f765090a2617b8d9cb73b71e0aa850c29460d8bePekka Enberg 35566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT); 35666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 357c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu return ret >> PAGE_SHIFT; 358c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu} 359c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu 36066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin/* 3618d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu * would have hole in the middle or ends, and only ram parts will be mapped. 36266520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin */ 3638d57470d8f859635deffe3919d7d4867b488b85aYinghai Lustatic unsigned long __init init_range_memory_mapping( 364b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu unsigned long r_start, 365b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu unsigned long r_end) 36666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin{ 36766520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin unsigned long start_pfn, end_pfn; 3688d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long mapped_ram_size = 0; 36966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin int i; 37066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 37166520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { 372b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end); 373b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end); 374b8fd39c036ab982aa087b7ee671f86e2574d31f2Yinghai Lu if (start >= end) 37566520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin continue; 37666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 377c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu /* 378c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu * if it is overlapping with brk pgt, we need to 379c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu * alloc pgt buf from memblock instead. 380c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu */ 381c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >= 382c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu min(end, (u64)pgt_buf_top<<PAGE_SHIFT); 383f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu init_memory_mapping(start, end); 3848d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu mapped_ram_size += end - start; 385c9b3234a6abadaa12684083d39552939baaed1f4Yinghai Lu can_use_brk_pgt = true; 38666520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin } 3878d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 3888d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu return mapped_ram_size; 38966520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin} 39066520ebc2df3fe52eb4792f8101fac573b766bafJacob Shin 3918d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu/* (PUD_SHIFT-PMD_SHIFT)/2 */ 3928d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu#define STEP_SIZE_SHIFT 5 393c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Luvoid __init init_mem_mapping(void) 394c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu{ 3958d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long end, real_end, start, last_start; 3968d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long step_size; 3978d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long addr; 3988d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long mapped_ram_size = 0; 3998d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu unsigned long new_mapped_ram_size; 400ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu 401c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu probe_page_size_mask(); 402c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu 403c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu#ifdef CONFIG_X86_64 404ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu end = max_pfn << PAGE_SHIFT; 405c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu#else 406ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu end = max_low_pfn << PAGE_SHIFT; 407c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu#endif 408ab9519376e86fbbf3c64e5a2b8b005958ea3e9ccYinghai Lu 409f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu /* the ISA range is always mapped regardless of memory holes */ 410f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu init_memory_mapping(0, ISA_END_ADDRESS); 4118d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4128d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu /* xen has big range in reserved near end of ram, skip it at first */ 4138d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE, 4148d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu PAGE_SIZE); 4158d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu real_end = addr + PMD_SIZE; 4168d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4178d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu /* step_size need to be small so pgt_buf from BRK could cover it */ 4188d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu step_size = PMD_SIZE; 4198d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu max_pfn_mapped = 0; /* will get exact value next */ 4208d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu min_pfn_mapped = real_end >> PAGE_SHIFT; 4218d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start = start = real_end; 4228d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu while (last_start > ISA_END_ADDRESS) { 4238d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (last_start > step_size) { 4248d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu start = round_down(last_start - 1, step_size); 4258d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (start < ISA_END_ADDRESS) 4268d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu start = ISA_END_ADDRESS; 4278d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu } else 4288d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu start = ISA_END_ADDRESS; 4298d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu new_mapped_ram_size = init_range_memory_mapping(start, 4308d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start); 4318d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu last_start = start; 4328d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu min_pfn_mapped = last_start >> PAGE_SHIFT; 4338d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu /* only increase step_size after big range get mapped */ 4348d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (new_mapped_ram_size > mapped_ram_size) 4358d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu step_size <<= STEP_SIZE_SHIFT; 4368d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu mapped_ram_size += new_mapped_ram_size; 4378d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu } 4388d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 4398d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu if (real_end < end) 4408d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu init_range_memory_mapping(real_end, end); 4418d57470d8f859635deffe3919d7d4867b488b85aYinghai Lu 442f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu#ifdef CONFIG_X86_64 443f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu if (max_pfn > max_low_pfn) { 444f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu /* can we preseve max_low_pfn ?*/ 445f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu max_low_pfn = max_pfn; 446f763ad1d3870abb811ec7520b4c1adc56471a3a4Yinghai Lu } 447719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu#else 448719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu early_ioremap_page_table_range_init(); 4498170e6bed465b4b0c7687f93e9948aca4358a33bH. Peter Anvin#endif 4508170e6bed465b4b0c7687f93e9948aca4358a33bH. Peter Anvin 451719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu load_cr3(swapper_pg_dir); 452719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu __flush_tlb_all(); 453719272c45b821d38608fc333700bde1a89c56c59Yinghai Lu 454c14fa0b63b5b4234667c03fdc3314c0881caa514Yinghai Lu early_memtest(0, max_pfn_mapped << PAGE_SHIFT); 45522ddfcaa0dbae992332381d41b8a1fbc72269a13Yinghai Lu} 456e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 457540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg/* 458540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * devmem_is_allowed() checks to see if /dev/mem access to a certain address 459540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * is valid. The argument is a physical page number. 460540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 461540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * 462540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * On x86, access has to be given to the first megabyte of ram because that area 463540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * contains bios code and data regions used by X and dosemu and similar apps. 464540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * Access has to be given to non-kernel-ram areas as well, these contain the PCI 465540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg * mmio resources as well as potential bios/acpi data regions. 466540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg */ 467540aca06b737cc38965b52eeceefba3d24376461Pekka Enbergint devmem_is_allowed(unsigned long pagenr) 468540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg{ 46973e8f3d7e2cb23614d5115703d76d8e54764b641T Makphaibulchoke if (pagenr < 256) 470540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 471540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 472540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 473540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg if (!page_is_ram(pagenr)) 474540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 1; 475540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg return 0; 476540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg} 477540aca06b737cc38965b52eeceefba3d24376461Pekka Enberg 478e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_init_pages(char *what, unsigned long begin, unsigned long end) 479e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 480c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long addr; 481c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu unsigned long begin_aligned, end_aligned; 482e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 483c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* Make sure boundaries are page aligned */ 484c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin_aligned = PAGE_ALIGN(begin); 485c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end_aligned = end & PAGE_MASK; 486c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 487c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 488c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu begin = begin_aligned; 489c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu end = end_aligned; 490c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu } 491c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 492c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu if (begin >= end) 493e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg return; 494e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 495c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu addr = begin; 496c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu 497e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 498e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * If debugging page accesses then do not free this memory but 499e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * mark them not present - any buggy init-section access will 500e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * create a kernel page fault: 501e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 502e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#ifdef CONFIG_DEBUG_PAGEALLOC 503365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n", 504365811d6f9bd98543bedc02b72d94f0f0faf3670Bjorn Helgaas begin, end - 1); 505e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 506e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#else 507e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg /* 508e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * We just marked the kernel text read only above, now that 509e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg * we are going to free part of that, we need to make that 5105bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET * writeable and non-executable first. 511e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg */ 5125bd5a452662bc37c54fb6828db1a3faf87e6511cMatthieu CASTET set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); 513e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 514e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 515e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 516e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 517e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg for (; addr < end; addr += PAGE_SIZE) { 518e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg ClearPageReserved(virt_to_page(addr)); 519e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg init_page_count(virt_to_page(addr)); 520c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 521e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_page(addr); 522e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg totalram_pages++; 523e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg } 524e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg#endif 525e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 526e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg 527e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enbergvoid free_initmem(void) 528e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg{ 529e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg free_init_pages("unused kernel memory", 530e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_begin), 531e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg (unsigned long)(&__init_end)); 532e5b2bb552706ca0e30795ee84caacbb37cec5705Pekka Enberg} 533731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg 534731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#ifdef CONFIG_BLK_DEV_INITRD 5350d26d1d873a302828e064737746c53a2689e6c0fJan Beulichvoid __init free_initrd_mem(unsigned long start, unsigned long end) 536731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg{ 537c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu /* 538c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * end could be not aligned, and We can not align that, 539c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * decompresser could be confused by aligned initrd_end 540c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * We already reserve the end partial page before in 541c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - i386_start_kernel() 542c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - x86_64_start_kernel() 543c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * - relocate_initrd() 544c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu * So here We can do PAGE_ALIGN() safely to get partial page to be freed 545c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu */ 546c967da6a0ba837f762042e931d4afcf72045547cYinghai Lu free_init_pages("initrd memory", start, PAGE_ALIGN(end)); 547731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg} 548731ddea63600c24ff01e6e5144cea88bf7266ac5Pekka Enberg#endif 549176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 550176239153049a023d060ce95b05f7ef31667e362Pekka Enbergvoid __init zone_sizes_init(void) 551176239153049a023d060ce95b05f7ef31667e362Pekka Enberg{ 552176239153049a023d060ce95b05f7ef31667e362Pekka Enberg unsigned long max_zone_pfns[MAX_NR_ZONES]; 553176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 554176239153049a023d060ce95b05f7ef31667e362Pekka Enberg memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 555176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 556176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA 557176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 558176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 559176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_ZONE_DMA32 560176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 561176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 562176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 563176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#ifdef CONFIG_HIGHMEM 564176239153049a023d060ce95b05f7ef31667e362Pekka Enberg max_zone_pfns[ZONE_HIGHMEM] = max_pfn; 565176239153049a023d060ce95b05f7ef31667e362Pekka Enberg#endif 566176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 567176239153049a023d060ce95b05f7ef31667e362Pekka Enberg free_area_init_nodes(max_zone_pfns); 568176239153049a023d060ce95b05f7ef31667e362Pekka Enberg} 569176239153049a023d060ce95b05f7ef31667e362Pekka Enberg 570