sparse.c revision 9e2779fa281cfda13ac060753d674bbcaa23367e
1d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 2d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * sparse memory mappings. 3d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 4d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/mm.h> 5d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/mmzone.h> 6d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/bootmem.h> 70b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen#include <linux/highmem.h> 8d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/module.h> 928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#include <linux/spinlock.h> 100b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen#include <linux/vmalloc.h> 11d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <asm/dma.h> 128f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#include <asm/pgalloc.h> 138f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#include <asm/pgtable.h> 14d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 15d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 16d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * Permanent SPARSEMEM data: 17d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * 18d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * 1) mem_section - memory sections, mem_map's for valid memory 19d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 203e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#ifdef CONFIG_SPARSEMEM_EXTREME 21802f192e4a600f7ef84ca25c8b818c8830acef5aBob Piccostruct mem_section *mem_section[NR_SECTION_ROOTS] 2222fc6eccbf4ce4eb6265e6ada7b50a7b9cc57d05Ravikiran G Thirumalai ____cacheline_internodealigned_in_smp; 233e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#else 243e347261a80b57df792ab9464b5f0ed59add53a8Bob Piccostruct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 2522fc6eccbf4ce4eb6265e6ada7b50a7b9cc57d05Ravikiran G Thirumalai ____cacheline_internodealigned_in_smp; 263e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#endif 273e347261a80b57df792ab9464b5f0ed59add53a8Bob PiccoEXPORT_SYMBOL(mem_section); 283e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 2989689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#ifdef NODE_NOT_IN_PAGE_FLAGS 3089689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter/* 3189689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * If we did not store the node number in the page then we have to 3289689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * do a lookup in the section_to_node_table in order to find which 3389689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * node the page belongs to. 3489689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter */ 3589689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#if MAX_NUMNODES <= 256 3689689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameterstatic u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 3789689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#else 3889689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameterstatic u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 3989689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#endif 4089689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter 4125ba77c141dbcd2602dd0171824d0d72aa023a01Andy Whitcroftint page_to_nid(struct page *page) 4289689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter{ 4389689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter return section_to_node_table[page_to_section(page)]; 4489689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter} 4589689ae7f95995723fbcd5c116c47933a3bb8b13Christoph LameterEXPORT_SYMBOL(page_to_nid); 4685770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft 4785770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroftstatic void set_section_nid(unsigned long section_nr, int nid) 4885770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft{ 4985770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft section_to_node_table[section_nr] = nid; 5085770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft} 5185770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft#else /* !NODE_NOT_IN_PAGE_FLAGS */ 5285770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroftstatic inline void set_section_nid(unsigned long section_nr, int nid) 5385770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft{ 5485770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft} 5589689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#endif 5689689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter 573e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#ifdef CONFIG_SPARSEMEM_EXTREME 58577a32f620271416d05f852477151fb51c790bc6Sam Ravnborgstatic struct mem_section noinline __init_refok *sparse_index_alloc(int nid) 5928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen{ 6028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen struct mem_section *section = NULL; 6128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen unsigned long array_size = SECTIONS_PER_ROOT * 6228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen sizeof(struct mem_section); 6328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 6439d24e64263cd3211705d3b61ea4171c65030921Mike Kravetz if (slab_is_available()) 6546a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz section = kmalloc_node(array_size, GFP_KERNEL, nid); 6646a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz else 6746a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz section = alloc_bootmem_node(NODE_DATA(nid), array_size); 6828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 6928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen if (section) 7028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen memset(section, 0, array_size); 7128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 7228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return section; 733e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco} 74802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 75a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Gotostatic int __meminit sparse_index_init(unsigned long section_nr, int nid) 76802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco{ 7734af946a22724c4e2b204957f2b24b22a0fb121cIngo Molnar static DEFINE_SPINLOCK(index_init_lock); 7828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen unsigned long root = SECTION_NR_TO_ROOT(section_nr); 7928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen struct mem_section *section; 8028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen int ret = 0; 81802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 82802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco if (mem_section[root]) 8328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return -EEXIST; 843e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 8528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen section = sparse_index_alloc(nid); 86af0cd5a7c3cded50c25e98acd94912d17a0eb914WANG Cong if (!section) 87af0cd5a7c3cded50c25e98acd94912d17a0eb914WANG Cong return -ENOMEM; 8828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen /* 8928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen * This lock keeps two different sections from 9028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen * reallocating for the same index 9128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen */ 9228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen spin_lock(&index_init_lock); 933e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 9428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen if (mem_section[root]) { 9528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen ret = -EEXIST; 9628ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen goto out; 9728ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen } 9828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 9928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen mem_section[root] = section; 10028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansenout: 10128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen spin_unlock(&index_init_lock); 10228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return ret; 10328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen} 10428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#else /* !SPARSEMEM_EXTREME */ 10528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansenstatic inline int sparse_index_init(unsigned long section_nr, int nid) 10628ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen{ 10728ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return 0; 108802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco} 10928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#endif 11028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 1114ca644d970bf2542623228a4624af356d20ca267Dave Hansen/* 1124ca644d970bf2542623228a4624af356d20ca267Dave Hansen * Although written for the SPARSEMEM_EXTREME case, this happens 113cd881a6b22902b356cacf8fd2e4e895871068eecAndy Whitcroft * to also work for the flat array case because 1144ca644d970bf2542623228a4624af356d20ca267Dave Hansen * NR_SECTION_ROOTS==NR_MEM_SECTIONS. 1154ca644d970bf2542623228a4624af356d20ca267Dave Hansen */ 1164ca644d970bf2542623228a4624af356d20ca267Dave Hansenint __section_nr(struct mem_section* ms) 1174ca644d970bf2542623228a4624af356d20ca267Dave Hansen{ 1184ca644d970bf2542623228a4624af356d20ca267Dave Hansen unsigned long root_nr; 1194ca644d970bf2542623228a4624af356d20ca267Dave Hansen struct mem_section* root; 1204ca644d970bf2542623228a4624af356d20ca267Dave Hansen 12112783b002db1f02c29353c8f698a85514420b9f4Mike Kravetz for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { 12212783b002db1f02c29353c8f698a85514420b9f4Mike Kravetz root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); 1234ca644d970bf2542623228a4624af356d20ca267Dave Hansen if (!root) 1244ca644d970bf2542623228a4624af356d20ca267Dave Hansen continue; 1254ca644d970bf2542623228a4624af356d20ca267Dave Hansen 1264ca644d970bf2542623228a4624af356d20ca267Dave Hansen if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) 1274ca644d970bf2542623228a4624af356d20ca267Dave Hansen break; 1284ca644d970bf2542623228a4624af356d20ca267Dave Hansen } 1294ca644d970bf2542623228a4624af356d20ca267Dave Hansen 1304ca644d970bf2542623228a4624af356d20ca267Dave Hansen return (root_nr * SECTIONS_PER_ROOT) + (ms - root); 1314ca644d970bf2542623228a4624af356d20ca267Dave Hansen} 1324ca644d970bf2542623228a4624af356d20ca267Dave Hansen 13330c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft/* 13430c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * During early boot, before section_mem_map is used for an actual 13530c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * mem_map, we use section_mem_map to store the section's NUMA 13630c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * node. This keeps us from having to use another data structure. The 13730c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * node information is cleared just before we store the real mem_map. 13830c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft */ 13930c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroftstatic inline unsigned long sparse_encode_early_nid(int nid) 14030c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft{ 14130c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft return (nid << SECTION_NID_SHIFT); 14230c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft} 14330c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft 14430c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroftstatic inline int sparse_early_nid(struct mem_section *section) 14530c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft{ 14630c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft return (section->section_mem_map >> SECTION_NID_SHIFT); 14730c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft} 14830c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft 149d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* Record a memory area against a node. */ 150a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Gotovoid __init memory_present(int nid, unsigned long start, unsigned long end) 151d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft{ 152d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long pfn; 153d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 154d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft start &= PAGE_SECTION_MASK; 155d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 156d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long section = pfn_to_section_nr(pfn); 157802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco struct mem_section *ms; 158802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 159802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco sparse_index_init(section, nid); 16085770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft set_section_nid(section, nid); 161802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 162802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco ms = __nr_to_section(section); 163802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco if (!ms->section_mem_map) 16430c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft ms->section_mem_map = sparse_encode_early_nid(nid) | 16530c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft SECTION_MARKED_PRESENT; 166d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft } 167d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft} 168d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 169d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 170d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * Only used by the i386 NUMA architecures, but relatively 171d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * generic code. 172d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 173d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroftunsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, 174d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long end_pfn) 175d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft{ 176d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long pfn; 177d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long nr_pages = 0; 178d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 179d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 180d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft if (nid != early_pfn_to_nid(pfn)) 181d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft continue; 182d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 183540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (pfn_present(pfn)) 184d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft nr_pages += PAGES_PER_SECTION; 185d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft } 186d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 187d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft return nr_pages * sizeof(struct page); 188d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft} 189d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 190d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 19129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * Subtle, we encode the real pfn into the mem_map such that 19229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * the identity pfn - section_mem_map will return the actual 19329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * physical page frame number. 19429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 19529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroftstatic unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 19629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 19729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 19829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 19929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 20029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft/* 20129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * We need this if we ever free the mem_maps. While not implemented yet, 20229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * this function is included for parity with its sibling. 20329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 20429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroftstatic __attribute((unused)) 20529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroftstruct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 20629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 20729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 20829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 20929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 210a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Gotostatic int __meminit sparse_init_one_section(struct mem_section *ms, 2115c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long pnum, struct page *mem_map, 2125c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *pageblock_bitmap) 21329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 214540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (!present_section(ms)) 21529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return -EINVAL; 21629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 21730c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft ms->section_mem_map &= ~SECTION_MAP_MASK; 218540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | 219540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft SECTION_HAS_MEM_MAP; 2205c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman ms->pageblock_flags = pageblock_bitmap; 22129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 22229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return 1; 22329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 22429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 2255c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gormanstatic unsigned long usemap_size(void) 2265c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 2275c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long size_bytes; 2285c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8; 2295c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman size_bytes = roundup(size_bytes, sizeof(unsigned long)); 2305c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return size_bytes; 2315c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 2325c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2335c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman#ifdef CONFIG_MEMORY_HOTPLUG 2345c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gormanstatic unsigned long *__kmalloc_section_usemap(void) 2355c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 2365c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return kmalloc(usemap_size(), GFP_KERNEL); 2375c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 2385c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman#endif /* CONFIG_MEMORY_HOTPLUG */ 2395c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2405c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gormanstatic unsigned long *sparse_early_usemap_alloc(unsigned long pnum) 2415c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 2425c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *usemap; 2435c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman struct mem_section *ms = __nr_to_section(pnum); 2445c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman int nid = sparse_early_nid(ms); 2455c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2465c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap = alloc_bootmem_node(NODE_DATA(nid), usemap_size()); 2475c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman if (usemap) 2485c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return usemap; 2495c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2505c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman /* Stupid: suppress gcc warning for SPARSEMEM && !NUMA */ 2515c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman nid = 0; 2525c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2535c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman printk(KERN_WARNING "%s: allocation failed\n", __FUNCTION__); 2545c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return NULL; 2555c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 2565c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2578f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#ifndef CONFIG_SPARSEMEM_VMEMMAP 25898f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostruct page __init *sparse_mem_map_populate(unsigned long pnum, int nid) 25929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 26029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft struct page *map; 26129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 26229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); 26329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft if (map) 26429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return map; 26529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 26629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft map = alloc_bootmem_node(NODE_DATA(nid), 26729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft sizeof(struct page) * PAGES_PER_SECTION); 2688f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter return map; 2698f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter} 2708f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 2718f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter 2728f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameterstruct page __init *sparse_early_mem_map_alloc(unsigned long pnum) 2738f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter{ 2748f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter struct page *map; 2758f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter struct mem_section *ms = __nr_to_section(pnum); 2768f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter int nid = sparse_early_nid(ms); 2778f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter 27898f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto map = sparse_mem_map_populate(pnum, nid); 27929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft if (map) 28029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return map; 28129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 2828f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter printk(KERN_ERR "%s: sparsemem memory map backing failed " 2838f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter "some memory will not be available.\n", __FUNCTION__); 284802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco ms->section_mem_map = 0; 28529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return NULL; 28629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 28729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 288193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell/* 289193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell * Allocate the accumulated non-linear sections, allocate a mem_map 290193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell * for each and record the physical to section mapping. 291193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell */ 292193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwellvoid __init sparse_init(void) 293193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell{ 294193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell unsigned long pnum; 295193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell struct page *map; 2965c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *usemap; 297193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 298193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 299540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (!present_section_nr(pnum)) 300193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell continue; 301193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 302193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell map = sparse_early_mem_map_alloc(pnum); 303193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell if (!map) 304193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell continue; 3055c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 3065c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap = sparse_early_usemap_alloc(pnum); 3075c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman if (!usemap) 3085c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman continue; 3095c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 3105c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman sparse_init_one_section(__nr_to_section(pnum), pnum, map, 3115c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap); 312193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell } 313193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell} 314193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 315193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell#ifdef CONFIG_MEMORY_HOTPLUG 31698f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#ifdef CONFIG_SPARSEMEM_VMEMMAP 31798f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 31898f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto unsigned long nr_pages) 31998f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 32098f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto /* This will make the necessary allocations eventually. */ 32198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return sparse_mem_map_populate(pnum, nid); 32298f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 32398f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 32498f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 32598f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return; /* XXX: Not implemented yet */ 32698f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 32798f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#else 3280b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenstatic struct page *__kmalloc_section_memmap(unsigned long nr_pages) 3290b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen{ 3300b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct page *page, *ret; 3310b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long memmap_size = sizeof(struct page) * nr_pages; 3320b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 333f2d0aa5bf8d4f7ae4cb1a7feebf5b1afddd0b9b0Yasunori Goto page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); 3340b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (page) 3350b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto got_map_page; 3360b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 3370b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = vmalloc(memmap_size); 3380b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (ret) 3390b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto got_map_ptr; 3400b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 3410b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return NULL; 3420b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansengot_map_page: 3430b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); 3440b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansengot_map_ptr: 3450b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen memset(ret, 0, memmap_size); 3460b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 3470b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return ret; 3480b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen} 3490b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 35098f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 35198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto unsigned long nr_pages) 35298f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 35398f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return __kmalloc_section_memmap(nr_pages); 35498f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 35598f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto 3560b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenstatic void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 3570b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen{ 3589e2779fa281cfda13ac060753d674bbcaa23367eChristoph Lameter if (is_vmalloc_addr(memmap)) 3590b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen vfree(memmap); 3600b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen else 3610b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen free_pages((unsigned long)memmap, 3620b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen get_order(sizeof(struct page) * nr_pages)); 3630b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen} 36498f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 3650b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 36629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft/* 36729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * returns the number of sections whose mem_maps were properly 36829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * set. If this is <=0, then that means that the passed-in 36929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * map was not consumed and must be freed. 37029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 3710b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenint sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 3720b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen int nr_pages) 37329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 3740b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long section_nr = pfn_to_section_nr(start_pfn); 3750b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct pglist_data *pgdat = zone->zone_pgdat; 3760b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct mem_section *ms; 3770b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct page *memmap; 3785c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *usemap; 3790b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long flags; 3800b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen int ret; 38129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 3820b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen /* 3830b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen * no locking for this, because it does its own 3840b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen * plus, it does a kmalloc 3850b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen */ 386bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong ret = sparse_index_init(section_nr, pgdat->node_id); 387bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong if (ret < 0 && ret != -EEXIST) 388bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong return ret; 38998f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages); 390bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong if (!memmap) 391bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong return -ENOMEM; 3925c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap = __kmalloc_section_usemap(); 393bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong if (!usemap) { 394bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong __kfree_section_memmap(memmap, nr_pages); 395bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong return -ENOMEM; 396bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong } 3970b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 3980b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen pgdat_resize_lock(pgdat, &flags); 39929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 4000b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ms = __pfn_to_section(start_pfn); 4010b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (ms->section_mem_map & SECTION_MARKED_PRESENT) { 4020b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = -EEXIST; 4030b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto out; 4040b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen } 4055c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 40629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft ms->section_mem_map |= SECTION_MARKED_PRESENT; 40729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 4085c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman ret = sparse_init_one_section(ms, section_nr, memmap, usemap); 4090b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 4100b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenout: 4110b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen pgdat_resize_unlock(pgdat, &flags); 412bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong if (ret <= 0) { 413bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong kfree(usemap); 41446a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz __kfree_section_memmap(memmap, nr_pages); 415bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong } 4160b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return ret; 41729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 418a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Goto#endif 419