sparse.c revision 98f3cfc1dc7a53b629d43b7844a9b3f786213048
1d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 2d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * sparse memory mappings. 3d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 4d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/mm.h> 5d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/mmzone.h> 6d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/bootmem.h> 70b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen#include <linux/highmem.h> 8d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/module.h> 928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#include <linux/spinlock.h> 100b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen#include <linux/vmalloc.h> 11d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <asm/dma.h> 128f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#include <asm/pgalloc.h> 138f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#include <asm/pgtable.h> 14d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 15d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 16d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * Permanent SPARSEMEM data: 17d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * 18d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * 1) mem_section - memory sections, mem_map's for valid memory 19d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 203e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#ifdef CONFIG_SPARSEMEM_EXTREME 21802f192e4a600f7ef84ca25c8b818c8830acef5aBob Piccostruct mem_section *mem_section[NR_SECTION_ROOTS] 2222fc6eccbf4ce4eb6265e6ada7b50a7b9cc57d05Ravikiran G Thirumalai ____cacheline_internodealigned_in_smp; 233e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#else 243e347261a80b57df792ab9464b5f0ed59add53a8Bob Piccostruct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 2522fc6eccbf4ce4eb6265e6ada7b50a7b9cc57d05Ravikiran G Thirumalai ____cacheline_internodealigned_in_smp; 263e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#endif 273e347261a80b57df792ab9464b5f0ed59add53a8Bob PiccoEXPORT_SYMBOL(mem_section); 283e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 2989689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#ifdef NODE_NOT_IN_PAGE_FLAGS 3089689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter/* 3189689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * If we did not store the node number in the page then we have to 3289689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * do a lookup in the section_to_node_table in order to find which 3389689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * node the page belongs to. 3489689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter */ 3589689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#if MAX_NUMNODES <= 256 3689689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameterstatic u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 3789689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#else 3889689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameterstatic u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 3989689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#endif 4089689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter 4125ba77c141dbcd2602dd0171824d0d72aa023a01Andy Whitcroftint page_to_nid(struct page *page) 4289689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter{ 4389689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter return section_to_node_table[page_to_section(page)]; 4489689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter} 4589689ae7f95995723fbcd5c116c47933a3bb8b13Christoph LameterEXPORT_SYMBOL(page_to_nid); 4685770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft 4785770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroftstatic void set_section_nid(unsigned long section_nr, int nid) 4885770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft{ 4985770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft section_to_node_table[section_nr] = nid; 5085770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft} 5185770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft#else /* !NODE_NOT_IN_PAGE_FLAGS */ 5285770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroftstatic inline void set_section_nid(unsigned long section_nr, int nid) 5385770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft{ 5485770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft} 5589689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#endif 5689689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter 573e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#ifdef CONFIG_SPARSEMEM_EXTREME 58577a32f620271416d05f852477151fb51c790bc6Sam Ravnborgstatic struct mem_section noinline __init_refok *sparse_index_alloc(int nid) 5928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen{ 6028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen struct mem_section *section = NULL; 6128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen unsigned long array_size = SECTIONS_PER_ROOT * 6228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen sizeof(struct mem_section); 6328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 6439d24e64263cd3211705d3b61ea4171c65030921Mike Kravetz if (slab_is_available()) 6546a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz section = kmalloc_node(array_size, GFP_KERNEL, nid); 6646a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz else 6746a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz section = alloc_bootmem_node(NODE_DATA(nid), array_size); 6828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 6928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen if (section) 7028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen memset(section, 0, array_size); 7128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 7228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return section; 733e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco} 74802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 75a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Gotostatic int __meminit sparse_index_init(unsigned long section_nr, int nid) 76802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco{ 7734af946a22724c4e2b204957f2b24b22a0fb121cIngo Molnar static DEFINE_SPINLOCK(index_init_lock); 7828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen unsigned long root = SECTION_NR_TO_ROOT(section_nr); 7928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen struct mem_section *section; 8028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen int ret = 0; 81802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 82802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco if (mem_section[root]) 8328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return -EEXIST; 843e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 8528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen section = sparse_index_alloc(nid); 8628ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen /* 8728ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen * This lock keeps two different sections from 8828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen * reallocating for the same index 8928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen */ 9028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen spin_lock(&index_init_lock); 913e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 9228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen if (mem_section[root]) { 9328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen ret = -EEXIST; 9428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen goto out; 9528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen } 9628ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 9728ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen mem_section[root] = section; 9828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansenout: 9928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen spin_unlock(&index_init_lock); 10028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return ret; 10128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen} 10228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#else /* !SPARSEMEM_EXTREME */ 10328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansenstatic inline int sparse_index_init(unsigned long section_nr, int nid) 10428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen{ 10528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return 0; 106802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco} 10728ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#endif 10828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 1094ca644d970bf2542623228a4624af356d20ca267Dave Hansen/* 1104ca644d970bf2542623228a4624af356d20ca267Dave Hansen * Although written for the SPARSEMEM_EXTREME case, this happens 111cd881a6b22902b356cacf8fd2e4e895871068eecAndy Whitcroft * to also work for the flat array case because 1124ca644d970bf2542623228a4624af356d20ca267Dave Hansen * NR_SECTION_ROOTS==NR_MEM_SECTIONS. 1134ca644d970bf2542623228a4624af356d20ca267Dave Hansen */ 1144ca644d970bf2542623228a4624af356d20ca267Dave Hansenint __section_nr(struct mem_section* ms) 1154ca644d970bf2542623228a4624af356d20ca267Dave Hansen{ 1164ca644d970bf2542623228a4624af356d20ca267Dave Hansen unsigned long root_nr; 1174ca644d970bf2542623228a4624af356d20ca267Dave Hansen struct mem_section* root; 1184ca644d970bf2542623228a4624af356d20ca267Dave Hansen 11912783b002db1f02c29353c8f698a85514420b9f4Mike Kravetz for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { 12012783b002db1f02c29353c8f698a85514420b9f4Mike Kravetz root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); 1214ca644d970bf2542623228a4624af356d20ca267Dave Hansen if (!root) 1224ca644d970bf2542623228a4624af356d20ca267Dave Hansen continue; 1234ca644d970bf2542623228a4624af356d20ca267Dave Hansen 1244ca644d970bf2542623228a4624af356d20ca267Dave Hansen if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) 1254ca644d970bf2542623228a4624af356d20ca267Dave Hansen break; 1264ca644d970bf2542623228a4624af356d20ca267Dave Hansen } 1274ca644d970bf2542623228a4624af356d20ca267Dave Hansen 1284ca644d970bf2542623228a4624af356d20ca267Dave Hansen return (root_nr * SECTIONS_PER_ROOT) + (ms - root); 1294ca644d970bf2542623228a4624af356d20ca267Dave Hansen} 1304ca644d970bf2542623228a4624af356d20ca267Dave Hansen 13130c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft/* 13230c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * During early boot, before section_mem_map is used for an actual 13330c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * mem_map, we use section_mem_map to store the section's NUMA 13430c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * node. This keeps us from having to use another data structure. The 13530c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * node information is cleared just before we store the real mem_map. 13630c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft */ 13730c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroftstatic inline unsigned long sparse_encode_early_nid(int nid) 13830c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft{ 13930c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft return (nid << SECTION_NID_SHIFT); 14030c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft} 14130c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft 14230c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroftstatic inline int sparse_early_nid(struct mem_section *section) 14330c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft{ 14430c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft return (section->section_mem_map >> SECTION_NID_SHIFT); 14530c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft} 14630c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft 147d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* Record a memory area against a node. */ 148a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Gotovoid __init memory_present(int nid, unsigned long start, unsigned long end) 149d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft{ 150d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long pfn; 151d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 152d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft start &= PAGE_SECTION_MASK; 153d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 154d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long section = pfn_to_section_nr(pfn); 155802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco struct mem_section *ms; 156802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 157802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco sparse_index_init(section, nid); 15885770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft set_section_nid(section, nid); 159802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 160802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco ms = __nr_to_section(section); 161802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco if (!ms->section_mem_map) 16230c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft ms->section_mem_map = sparse_encode_early_nid(nid) | 16330c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft SECTION_MARKED_PRESENT; 164d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft } 165d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft} 166d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 167d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 168d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * Only used by the i386 NUMA architecures, but relatively 169d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * generic code. 170d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 171d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroftunsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, 172d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long end_pfn) 173d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft{ 174d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long pfn; 175d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long nr_pages = 0; 176d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 177d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 178d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft if (nid != early_pfn_to_nid(pfn)) 179d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft continue; 180d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 181540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (pfn_present(pfn)) 182d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft nr_pages += PAGES_PER_SECTION; 183d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft } 184d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 185d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft return nr_pages * sizeof(struct page); 186d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft} 187d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 188d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 18929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * Subtle, we encode the real pfn into the mem_map such that 19029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * the identity pfn - section_mem_map will return the actual 19129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * physical page frame number. 19229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 19329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroftstatic unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 19429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 19529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 19629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 19729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 19829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft/* 19929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * We need this if we ever free the mem_maps. While not implemented yet, 20029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * this function is included for parity with its sibling. 20129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 20229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroftstatic __attribute((unused)) 20329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroftstruct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 20429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 20529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 20629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 20729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 208a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Gotostatic int __meminit sparse_init_one_section(struct mem_section *ms, 2095c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long pnum, struct page *mem_map, 2105c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *pageblock_bitmap) 21129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 212540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (!present_section(ms)) 21329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return -EINVAL; 21429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 21530c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft ms->section_mem_map &= ~SECTION_MAP_MASK; 216540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | 217540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft SECTION_HAS_MEM_MAP; 2185c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman ms->pageblock_flags = pageblock_bitmap; 21929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 22029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return 1; 22129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 22229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 223dec2e6b7aa5d45bc3508e19907a7716b0c5307e5Sam Ravnborg__attribute__((weak)) __init 2242e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan haivoid *alloc_bootmem_high_node(pg_data_t *pgdat, unsigned long size) 2252e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai{ 2262e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai return NULL; 2272e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai} 2282e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai 2295c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gormanstatic unsigned long usemap_size(void) 2305c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 2315c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long size_bytes; 2325c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8; 2335c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman size_bytes = roundup(size_bytes, sizeof(unsigned long)); 2345c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return size_bytes; 2355c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 2365c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2375c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman#ifdef CONFIG_MEMORY_HOTPLUG 2385c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gormanstatic unsigned long *__kmalloc_section_usemap(void) 2395c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 2405c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return kmalloc(usemap_size(), GFP_KERNEL); 2415c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 2425c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman#endif /* CONFIG_MEMORY_HOTPLUG */ 2435c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2445c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gormanstatic unsigned long *sparse_early_usemap_alloc(unsigned long pnum) 2455c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 2465c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *usemap; 2475c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman struct mem_section *ms = __nr_to_section(pnum); 2485c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman int nid = sparse_early_nid(ms); 2495c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2505c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap = alloc_bootmem_node(NODE_DATA(nid), usemap_size()); 2515c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman if (usemap) 2525c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return usemap; 2535c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2545c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman /* Stupid: suppress gcc warning for SPARSEMEM && !NUMA */ 2555c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman nid = 0; 2565c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2575c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman printk(KERN_WARNING "%s: allocation failed\n", __FUNCTION__); 2585c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return NULL; 2595c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 2605c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2618f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#ifndef CONFIG_SPARSEMEM_VMEMMAP 26298f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostruct page __init *sparse_mem_map_populate(unsigned long pnum, int nid) 26329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 26429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft struct page *map; 26529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 26629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); 26729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft if (map) 26829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return map; 26929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 2702e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai map = alloc_bootmem_high_node(NODE_DATA(nid), 2712e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai sizeof(struct page) * PAGES_PER_SECTION); 2722e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai if (map) 2732e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai return map; 2742e1c49db4c640b35df13889b86b9d62215ade4b6Zou Nan hai 27529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft map = alloc_bootmem_node(NODE_DATA(nid), 27629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft sizeof(struct page) * PAGES_PER_SECTION); 2778f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter return map; 2788f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter} 2798f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 2808f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter 2818f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameterstruct page __init *sparse_early_mem_map_alloc(unsigned long pnum) 2828f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter{ 2838f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter struct page *map; 2848f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter struct mem_section *ms = __nr_to_section(pnum); 2858f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter int nid = sparse_early_nid(ms); 2868f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter 28798f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto map = sparse_mem_map_populate(pnum, nid); 28829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft if (map) 28929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return map; 29029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 2918f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter printk(KERN_ERR "%s: sparsemem memory map backing failed " 2928f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter "some memory will not be available.\n", __FUNCTION__); 293802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco ms->section_mem_map = 0; 29429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return NULL; 29529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 29629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 297193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell/* 298193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell * Allocate the accumulated non-linear sections, allocate a mem_map 299193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell * for each and record the physical to section mapping. 300193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell */ 301193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwellvoid __init sparse_init(void) 302193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell{ 303193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell unsigned long pnum; 304193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell struct page *map; 3055c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *usemap; 306193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 307193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 308540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (!present_section_nr(pnum)) 309193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell continue; 310193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 311193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell map = sparse_early_mem_map_alloc(pnum); 312193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell if (!map) 313193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell continue; 3145c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 3155c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap = sparse_early_usemap_alloc(pnum); 3165c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman if (!usemap) 3175c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman continue; 3185c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 3195c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman sparse_init_one_section(__nr_to_section(pnum), pnum, map, 3205c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap); 321193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell } 322193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell} 323193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 324193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell#ifdef CONFIG_MEMORY_HOTPLUG 32598f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#ifdef CONFIG_SPARSEMEM_VMEMMAP 32698f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 32798f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto unsigned long nr_pages) 32898f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 32998f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto /* This will make the necessary allocations eventually. */ 33098f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return sparse_mem_map_populate(pnum, nid); 33198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 33298f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 33398f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 33498f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return; /* XXX: Not implemented yet */ 33598f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 33698f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#else 3370b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenstatic struct page *__kmalloc_section_memmap(unsigned long nr_pages) 3380b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen{ 3390b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct page *page, *ret; 3400b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long memmap_size = sizeof(struct page) * nr_pages; 3410b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 342f2d0aa5bf8d4f7ae4cb1a7feebf5b1afddd0b9b0Yasunori Goto page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); 3430b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (page) 3440b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto got_map_page; 3450b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 3460b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = vmalloc(memmap_size); 3470b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (ret) 3480b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto got_map_ptr; 3490b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 3500b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return NULL; 3510b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansengot_map_page: 3520b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); 3530b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansengot_map_ptr: 3540b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen memset(ret, 0, memmap_size); 3550b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 3560b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return ret; 3570b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen} 3580b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 35998f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 36098f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto unsigned long nr_pages) 36198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 36298f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return __kmalloc_section_memmap(nr_pages); 36398f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 36498f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto 3650b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenstatic int vaddr_in_vmalloc_area(void *addr) 3660b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen{ 3670b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (addr >= (void *)VMALLOC_START && 3680b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen addr < (void *)VMALLOC_END) 3690b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return 1; 3700b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return 0; 3710b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen} 3720b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 3730b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenstatic void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 3740b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen{ 3750b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (vaddr_in_vmalloc_area(memmap)) 3760b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen vfree(memmap); 3770b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen else 3780b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen free_pages((unsigned long)memmap, 3790b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen get_order(sizeof(struct page) * nr_pages)); 3800b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen} 38198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 3820b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 38329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft/* 38429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * returns the number of sections whose mem_maps were properly 38529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * set. If this is <=0, then that means that the passed-in 38629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * map was not consumed and must be freed. 38729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 3880b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenint sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 3890b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen int nr_pages) 39029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 3910b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long section_nr = pfn_to_section_nr(start_pfn); 3920b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct pglist_data *pgdat = zone->zone_pgdat; 3930b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct mem_section *ms; 3940b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct page *memmap; 3955c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *usemap; 3960b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long flags; 3970b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen int ret; 39829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 3990b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen /* 4000b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen * no locking for this, because it does its own 4010b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen * plus, it does a kmalloc 4020b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen */ 4030b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen sparse_index_init(section_nr, pgdat->node_id); 40498f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages); 4055c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap = __kmalloc_section_usemap(); 4060b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 4070b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen pgdat_resize_lock(pgdat, &flags); 40829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 4090b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ms = __pfn_to_section(start_pfn); 4100b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (ms->section_mem_map & SECTION_MARKED_PRESENT) { 4110b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = -EEXIST; 4120b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto out; 4130b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen } 4145c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 4155c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman if (!usemap) { 4165c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman ret = -ENOMEM; 4175c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman goto out; 4185c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman } 41929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft ms->section_mem_map |= SECTION_MARKED_PRESENT; 42029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 4215c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman ret = sparse_init_one_section(ms, section_nr, memmap, usemap); 4220b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 4230b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenout: 4240b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen pgdat_resize_unlock(pgdat, &flags); 42546a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz if (ret <= 0) 42646a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz __kfree_section_memmap(memmap, nr_pages); 4270b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return ret; 42829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 429a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Goto#endif 430