sparse.c revision a4322e1bad91fbca27056fc38d2cbca3f1eae0cf
1d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 2d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * sparse memory mappings. 3d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 4d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/mm.h> 5d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/mmzone.h> 6d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/bootmem.h> 70b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen#include <linux/highmem.h> 8d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <linux/module.h> 928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#include <linux/spinlock.h> 100b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen#include <linux/vmalloc.h> 110c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto#include "internal.h" 12d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft#include <asm/dma.h> 138f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#include <asm/pgalloc.h> 148f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#include <asm/pgtable.h> 15d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 16d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 17d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * Permanent SPARSEMEM data: 18d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * 19d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * 1) mem_section - memory sections, mem_map's for valid memory 20d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 213e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#ifdef CONFIG_SPARSEMEM_EXTREME 22802f192e4a600f7ef84ca25c8b818c8830acef5aBob Piccostruct mem_section *mem_section[NR_SECTION_ROOTS] 2322fc6eccbf4ce4eb6265e6ada7b50a7b9cc57d05Ravikiran G Thirumalai ____cacheline_internodealigned_in_smp; 243e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#else 253e347261a80b57df792ab9464b5f0ed59add53a8Bob Piccostruct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 2622fc6eccbf4ce4eb6265e6ada7b50a7b9cc57d05Ravikiran G Thirumalai ____cacheline_internodealigned_in_smp; 273e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#endif 283e347261a80b57df792ab9464b5f0ed59add53a8Bob PiccoEXPORT_SYMBOL(mem_section); 293e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 3089689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#ifdef NODE_NOT_IN_PAGE_FLAGS 3189689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter/* 3289689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * If we did not store the node number in the page then we have to 3389689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * do a lookup in the section_to_node_table in order to find which 3489689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter * node the page belongs to. 3589689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter */ 3689689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#if MAX_NUMNODES <= 256 3789689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameterstatic u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 3889689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#else 3989689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameterstatic u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 4089689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#endif 4189689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter 4225ba77c141dbcd2602dd0171824d0d72aa023a01Andy Whitcroftint page_to_nid(struct page *page) 4389689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter{ 4489689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter return section_to_node_table[page_to_section(page)]; 4589689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter} 4689689ae7f95995723fbcd5c116c47933a3bb8b13Christoph LameterEXPORT_SYMBOL(page_to_nid); 4785770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft 4885770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroftstatic void set_section_nid(unsigned long section_nr, int nid) 4985770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft{ 5085770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft section_to_node_table[section_nr] = nid; 5185770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft} 5285770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft#else /* !NODE_NOT_IN_PAGE_FLAGS */ 5385770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroftstatic inline void set_section_nid(unsigned long section_nr, int nid) 5485770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft{ 5585770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft} 5689689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter#endif 5789689ae7f95995723fbcd5c116c47933a3bb8b13Christoph Lameter 583e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco#ifdef CONFIG_SPARSEMEM_EXTREME 59577a32f620271416d05f852477151fb51c790bc6Sam Ravnborgstatic struct mem_section noinline __init_refok *sparse_index_alloc(int nid) 6028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen{ 6128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen struct mem_section *section = NULL; 6228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen unsigned long array_size = SECTIONS_PER_ROOT * 6328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen sizeof(struct mem_section); 6428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 65f52407ce2deac76c87abc8211a63ea152ba72d54Shaohua Li if (slab_is_available()) { 66f52407ce2deac76c87abc8211a63ea152ba72d54Shaohua Li if (node_state(nid, N_HIGH_MEMORY)) 67f52407ce2deac76c87abc8211a63ea152ba72d54Shaohua Li section = kmalloc_node(array_size, GFP_KERNEL, nid); 68f52407ce2deac76c87abc8211a63ea152ba72d54Shaohua Li else 69f52407ce2deac76c87abc8211a63ea152ba72d54Shaohua Li section = kmalloc(array_size, GFP_KERNEL); 70f52407ce2deac76c87abc8211a63ea152ba72d54Shaohua Li } else 7146a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz section = alloc_bootmem_node(NODE_DATA(nid), array_size); 7228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 7328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen if (section) 7428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen memset(section, 0, array_size); 7528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 7628ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return section; 773e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco} 78802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 79a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Gotostatic int __meminit sparse_index_init(unsigned long section_nr, int nid) 80802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco{ 8134af946a22724c4e2b204957f2b24b22a0fb121cIngo Molnar static DEFINE_SPINLOCK(index_init_lock); 8228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen unsigned long root = SECTION_NR_TO_ROOT(section_nr); 8328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen struct mem_section *section; 8428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen int ret = 0; 85802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 86802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco if (mem_section[root]) 8728ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return -EEXIST; 883e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 8928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen section = sparse_index_alloc(nid); 90af0cd5a7c3cded50c25e98acd94912d17a0eb914WANG Cong if (!section) 91af0cd5a7c3cded50c25e98acd94912d17a0eb914WANG Cong return -ENOMEM; 9228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen /* 9328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen * This lock keeps two different sections from 9428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen * reallocating for the same index 9528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen */ 9628ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen spin_lock(&index_init_lock); 973e347261a80b57df792ab9464b5f0ed59add53a8Bob Picco 9828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen if (mem_section[root]) { 9928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen ret = -EEXIST; 10028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen goto out; 10128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen } 10228ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 10328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen mem_section[root] = section; 10428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansenout: 10528ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen spin_unlock(&index_init_lock); 10628ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return ret; 10728ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen} 10828ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#else /* !SPARSEMEM_EXTREME */ 10928ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansenstatic inline int sparse_index_init(unsigned long section_nr, int nid) 11028ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen{ 11128ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen return 0; 112802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco} 11328ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen#endif 11428ae55c98e4d16eac9a05a8a259d7763ef3aeb18Dave Hansen 1154ca644d970bf2542623228a4624af356d20ca267Dave Hansen/* 1164ca644d970bf2542623228a4624af356d20ca267Dave Hansen * Although written for the SPARSEMEM_EXTREME case, this happens 117cd881a6b22902b356cacf8fd2e4e895871068eecAndy Whitcroft * to also work for the flat array case because 1184ca644d970bf2542623228a4624af356d20ca267Dave Hansen * NR_SECTION_ROOTS==NR_MEM_SECTIONS. 1194ca644d970bf2542623228a4624af356d20ca267Dave Hansen */ 1204ca644d970bf2542623228a4624af356d20ca267Dave Hansenint __section_nr(struct mem_section* ms) 1214ca644d970bf2542623228a4624af356d20ca267Dave Hansen{ 1224ca644d970bf2542623228a4624af356d20ca267Dave Hansen unsigned long root_nr; 1234ca644d970bf2542623228a4624af356d20ca267Dave Hansen struct mem_section* root; 1244ca644d970bf2542623228a4624af356d20ca267Dave Hansen 12512783b002db1f02c29353c8f698a85514420b9f4Mike Kravetz for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { 12612783b002db1f02c29353c8f698a85514420b9f4Mike Kravetz root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); 1274ca644d970bf2542623228a4624af356d20ca267Dave Hansen if (!root) 1284ca644d970bf2542623228a4624af356d20ca267Dave Hansen continue; 1294ca644d970bf2542623228a4624af356d20ca267Dave Hansen 1304ca644d970bf2542623228a4624af356d20ca267Dave Hansen if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) 1314ca644d970bf2542623228a4624af356d20ca267Dave Hansen break; 1324ca644d970bf2542623228a4624af356d20ca267Dave Hansen } 1334ca644d970bf2542623228a4624af356d20ca267Dave Hansen 1344ca644d970bf2542623228a4624af356d20ca267Dave Hansen return (root_nr * SECTIONS_PER_ROOT) + (ms - root); 1354ca644d970bf2542623228a4624af356d20ca267Dave Hansen} 1364ca644d970bf2542623228a4624af356d20ca267Dave Hansen 13730c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft/* 13830c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * During early boot, before section_mem_map is used for an actual 13930c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * mem_map, we use section_mem_map to store the section's NUMA 14030c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * node. This keeps us from having to use another data structure. The 14130c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft * node information is cleared just before we store the real mem_map. 14230c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft */ 14330c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroftstatic inline unsigned long sparse_encode_early_nid(int nid) 14430c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft{ 14530c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft return (nid << SECTION_NID_SHIFT); 14630c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft} 14730c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft 14830c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroftstatic inline int sparse_early_nid(struct mem_section *section) 14930c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft{ 15030c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft return (section->section_mem_map >> SECTION_NID_SHIFT); 15130c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft} 15230c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft 1532dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman/* Validate the physical addressing limitations of the model */ 1542dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gormanvoid __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 1552dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman unsigned long *end_pfn) 156d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft{ 1572dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 158d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 159bead9a3abd15710b0bdfd418daef606722d86282Ingo Molnar /* 160bead9a3abd15710b0bdfd418daef606722d86282Ingo Molnar * Sanity checks - do not allow an architecture to pass 161bead9a3abd15710b0bdfd418daef606722d86282Ingo Molnar * in larger pfns than the maximum scope of sparsemem: 162bead9a3abd15710b0bdfd418daef606722d86282Ingo Molnar */ 1632dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman if (*start_pfn > max_sparsemem_pfn) { 1642dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 1652dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 1662dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman *start_pfn, *end_pfn, max_sparsemem_pfn); 1672dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman WARN_ON_ONCE(1); 1682dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman *start_pfn = max_sparsemem_pfn; 1692dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman *end_pfn = max_sparsemem_pfn; 170ef161a9863b045909142daea9490b067997f3dc5Cyrill Gorcunov } else if (*end_pfn > max_sparsemem_pfn) { 1712dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 1722dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 1732dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman *start_pfn, *end_pfn, max_sparsemem_pfn); 1742dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman WARN_ON_ONCE(1); 1752dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman *end_pfn = max_sparsemem_pfn; 1762dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman } 1772dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman} 1782dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman 1792dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman/* Record a memory area against a node. */ 1802dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gormanvoid __init memory_present(int nid, unsigned long start, unsigned long end) 1812dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman{ 1822dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman unsigned long pfn; 183bead9a3abd15710b0bdfd418daef606722d86282Ingo Molnar 184d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft start &= PAGE_SECTION_MASK; 1852dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman mminit_validate_memmodel_limits(&start, &end); 186d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 187d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long section = pfn_to_section_nr(pfn); 188802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco struct mem_section *ms; 189802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 190802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco sparse_index_init(section, nid); 19185770ffe4f0cdd4396b17f14762adc25a571a348Andy Whitcroft set_section_nid(section, nid); 192802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco 193802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco ms = __nr_to_section(section); 194802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco if (!ms->section_mem_map) 19530c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft ms->section_mem_map = sparse_encode_early_nid(nid) | 19630c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft SECTION_MARKED_PRESENT; 197d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft } 198d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft} 199d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 200d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 201d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * Only used by the i386 NUMA architecures, but relatively 202d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft * generic code. 203d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft */ 204d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroftunsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, 205d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long end_pfn) 206d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft{ 207d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long pfn; 208d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft unsigned long nr_pages = 0; 209d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 2102dbb51c49f4fecb8330e43247a0edfbc4b2b8974Mel Gorman mminit_validate_memmodel_limits(&start_pfn, &end_pfn); 211d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 212d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft if (nid != early_pfn_to_nid(pfn)) 213d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft continue; 214d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 215540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (pfn_present(pfn)) 216d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft nr_pages += PAGES_PER_SECTION; 217d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft } 218d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 219d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft return nr_pages * sizeof(struct page); 220d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft} 221d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft 222d41dee369bff3b9dcb6328d4d822926c28cc2594Andy Whitcroft/* 22329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * Subtle, we encode the real pfn into the mem_map such that 22429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * the identity pfn - section_mem_map will return the actual 22529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * physical page frame number. 22629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 22729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroftstatic unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 22829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 22929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 23029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 23129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 23229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft/* 233ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty * Decode mem_map from the coded memmap 23429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 23529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroftstruct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 23629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 237ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty /* mask off the extra low bits of information */ 238ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty coded_mem_map &= SECTION_MAP_MASK; 23929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 24029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 24129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 242a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Gotostatic int __meminit sparse_init_one_section(struct mem_section *ms, 2435c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long pnum, struct page *mem_map, 2445c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *pageblock_bitmap) 24529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 246540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (!present_section(ms)) 24729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return -EINVAL; 24829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 24930c253e6da655d73eb8bfe2adca9b8f4d82fb81eAndy Whitcroft ms->section_mem_map &= ~SECTION_MAP_MASK; 250540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | 251540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft SECTION_HAS_MEM_MAP; 2525c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman ms->pageblock_flags = pageblock_bitmap; 25329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 25429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return 1; 25529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 25629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 25704753278769f3b6c3b79a080edb52f21d83bf6e2Yasunori Gotounsigned long usemap_size(void) 2585c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 2595c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long size_bytes; 2605c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8; 2615c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman size_bytes = roundup(size_bytes, sizeof(unsigned long)); 2625c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return size_bytes; 2635c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 2645c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 2655c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman#ifdef CONFIG_MEMORY_HOTPLUG 2665c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gormanstatic unsigned long *__kmalloc_section_usemap(void) 2675c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 2685c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman return kmalloc(usemap_size(), GFP_KERNEL); 2695c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 2705c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman#endif /* CONFIG_MEMORY_HOTPLUG */ 2715c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 27248c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto#ifdef CONFIG_MEMORY_HOTREMOVE 27348c906823f3927b981db9f0b03c2e2499977ee93Yasunori Gotostatic unsigned long * __init 274a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lusparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 275a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu unsigned long count) 27648c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto{ 27748c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto unsigned long section_nr; 27848c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 27948c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto /* 28048c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * A page may contain usemaps for other sections preventing the 28148c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * page being freed and making a section unremovable while 28248c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * other sections referencing the usemap retmain active. Similarly, 28348c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * a pgdat can prevent a section being removed. If section A 28448c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * contains a pgdat and section B contains the usemap, both 28548c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * sections become inter-dependent. This allocates usemaps 28648c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * from the same section as the pgdat where possible to avoid 28748c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * this problem. 28848c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto */ 28948c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto section_nr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT); 290a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu return alloc_bootmem_section(usemap_size() * count, section_nr); 29148c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto} 29248c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 29348c906823f3927b981db9f0b03c2e2499977ee93Yasunori Gotostatic void __init check_usemap_section_nr(int nid, unsigned long *usemap) 29448c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto{ 29548c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto unsigned long usemap_snr, pgdat_snr; 29648c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto static unsigned long old_usemap_snr = NR_MEM_SECTIONS; 29748c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto static unsigned long old_pgdat_snr = NR_MEM_SECTIONS; 29848c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto struct pglist_data *pgdat = NODE_DATA(nid); 29948c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto int usemap_nid; 30048c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 30148c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT); 30248c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT); 30348c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto if (usemap_snr == pgdat_snr) 30448c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto return; 30548c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 30648c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr) 30748c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto /* skip redundant message */ 30848c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto return; 30948c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 31048c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto old_usemap_snr = usemap_snr; 31148c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto old_pgdat_snr = pgdat_snr; 31248c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 31348c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr)); 31448c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto if (usemap_nid != nid) { 31548c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto printk(KERN_INFO 31648c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto "node %d must be removed before remove section %ld\n", 31748c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto nid, usemap_snr); 31848c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto return; 31948c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto } 32048c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto /* 32148c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * There is a circular dependency. 32248c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * Some platforms allow un-removable section because they will just 32348c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * gather other removable sections for dynamic partitioning. 32448c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto * Just notify un-removable section's number here. 32548c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto */ 32648c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr, 32748c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto pgdat_snr, nid); 32848c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto printk(KERN_CONT 32948c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto " have a circular dependency on usemap and pgdat allocations\n"); 33048c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto} 33148c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto#else 33248c906823f3927b981db9f0b03c2e2499977ee93Yasunori Gotostatic unsigned long * __init 333a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lusparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 334a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu unsigned long count) 33548c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto{ 33648c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto return NULL; 33748c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto} 33848c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 33948c906823f3927b981db9f0b03c2e2499977ee93Yasunori Gotostatic void __init check_usemap_section_nr(int nid, unsigned long *usemap) 34048c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto{ 34148c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto} 34248c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto#endif /* CONFIG_MEMORY_HOTREMOVE */ 34348c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 344a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lustatic void __init sparse_early_usemaps_alloc_node(unsigned long**usemap_map, 345a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu unsigned long pnum_begin, 346a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu unsigned long pnum_end, 347a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu unsigned long usemap_count, int nodeid) 3485c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman{ 349a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu void *usemap; 350a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu unsigned long pnum; 351a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu int size = usemap_size(); 3525c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 353a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid), 354a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap_count); 35548c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto if (usemap) { 356a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 357a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu if (!present_section_nr(pnum)) 358a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu continue; 359a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap_map[pnum] = usemap; 360a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap += size; 361a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu } 362a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu return; 36348c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto } 36448c906823f3927b981db9f0b03c2e2499977ee93Yasunori Goto 365a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap = alloc_bootmem_node(NODE_DATA(nodeid), size * usemap_count); 366a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu if (usemap) { 367a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 368a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu if (!present_section_nr(pnum)) 369a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu continue; 370a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap_map[pnum] = usemap; 371a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap += size; 372a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu check_usemap_section_nr(nodeid, usemap_map[pnum]); 373a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu } 374a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu return; 375a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu } 3765c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 377d40cee245ff6ad05d3448401d7320be82c1c5af1Harvey Harrison printk(KERN_WARNING "%s: allocation failed\n", __func__); 3785c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman} 3795c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 3808f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#ifndef CONFIG_SPARSEMEM_VMEMMAP 38198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostruct page __init *sparse_mem_map_populate(unsigned long pnum, int nid) 38229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 38329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft struct page *map; 38429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 38529751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); 38629751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft if (map) 38729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return map; 38829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 3899d99217a02a06a7cc83f065b73e976970970c58cYasunori Goto map = alloc_bootmem_pages_node(NODE_DATA(nid), 3909d99217a02a06a7cc83f065b73e976970970c58cYasunori Goto PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION)); 3918f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter return map; 3928f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter} 3938f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 3948f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter 3959e5c6da71e89fa25ced6e88182225a99941bec90Adrian Bunkstatic struct page __init *sparse_early_mem_map_alloc(unsigned long pnum) 3968f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter{ 3978f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter struct page *map; 3988f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter struct mem_section *ms = __nr_to_section(pnum); 3998f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter int nid = sparse_early_nid(ms); 4008f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter 40198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto map = sparse_mem_map_populate(pnum, nid); 40229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft if (map) 40329751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return map; 40429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 4058f6aac419bd590f535fb110875a51f7db2b62b5bChristoph Lameter printk(KERN_ERR "%s: sparsemem memory map backing failed " 406d40cee245ff6ad05d3448401d7320be82c1c5af1Harvey Harrison "some memory will not be available.\n", __func__); 407802f192e4a600f7ef84ca25c8b818c8830acef5aBob Picco ms->section_mem_map = 0; 40829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft return NULL; 40929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 41029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 411c2b91e2eec9678dbda274e906cc32ea8f711da3bYinghai Luvoid __attribute__((weak)) __meminit vmemmap_populate_print_last(void) 412c2b91e2eec9678dbda274e906cc32ea8f711da3bYinghai Lu{ 413c2b91e2eec9678dbda274e906cc32ea8f711da3bYinghai Lu} 414a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu 415193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell/* 416193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell * Allocate the accumulated non-linear sections, allocate a mem_map 417193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell * for each and record the physical to section mapping. 418193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell */ 419193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwellvoid __init sparse_init(void) 420193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell{ 421193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell unsigned long pnum; 422193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell struct page *map; 4235c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *usemap; 424e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu unsigned long **usemap_map; 425e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu int size; 426a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu int nodeid_begin = 0; 427a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu unsigned long pnum_begin = 0; 428a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu unsigned long usemap_count; 429e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu 430e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu /* 431e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * map is using big page (aka 2M in x86 64 bit) 432e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * usemap is less one page (aka 24 bytes) 433e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * so alloc 2M (with 2M align) and 24 bytes in turn will 434e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * make next 2M slip to one more 2M later. 435e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * then in big system, the memory will have a lot of holes... 436e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * here try to allocate 2M pages continously. 437e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * 438e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * powerpc need to call sparse_init_one_section right after each 439e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu * sparse_early_mem_map_alloc, so allocate usemap_map at first. 440e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu */ 441e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu size = sizeof(unsigned long *) * NR_MEM_SECTIONS; 442e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu usemap_map = alloc_bootmem(size); 443e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu if (!usemap_map) 444e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu panic("can not allocate usemap_map\n"); 445193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 446193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 447a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu struct mem_section *ms; 448a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu 449540557b9439ec19668553830c90222f9fb0c2e95Andy Whitcroft if (!present_section_nr(pnum)) 450193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell continue; 451a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu ms = __nr_to_section(pnum); 452a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu nodeid_begin = sparse_early_nid(ms); 453a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu pnum_begin = pnum; 454a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu break; 455a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu } 456a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap_count = 1; 457a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) { 458a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu struct mem_section *ms; 459a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu int nodeid; 460a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu 461a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu if (!present_section_nr(pnum)) 462a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu continue; 463a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu ms = __nr_to_section(pnum); 464a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu nodeid = sparse_early_nid(ms); 465a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu if (nodeid == nodeid_begin) { 466a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap_count++; 467a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu continue; 468a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu } 469a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu /* ok, we need to take cake of from pnum_begin to pnum - 1*/ 470a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, pnum, 471a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap_count, nodeid_begin); 472a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu /* new start, update count etc*/ 473a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu nodeid_begin = nodeid; 474a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu pnum_begin = pnum; 475a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap_count = 1; 476e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu } 477a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu /* ok, last chunk */ 478a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, NR_MEM_SECTIONS, 479a4322e1bad91fbca27056fc38d2cbca3f1eae0cfYinghai Lu usemap_count, nodeid_begin); 480193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 481e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 482e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu if (!present_section_nr(pnum)) 483193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell continue; 4845c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 485e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu usemap = usemap_map[pnum]; 4865c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman if (!usemap) 4875c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman continue; 4885c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 489e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu map = sparse_early_mem_map_alloc(pnum); 490e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu if (!map) 491e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu continue; 492e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu 4935c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman sparse_init_one_section(__nr_to_section(pnum), pnum, map, 4945c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap); 495193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell } 496e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu 497c2b91e2eec9678dbda274e906cc32ea8f711da3bYinghai Lu vmemmap_populate_print_last(); 498c2b91e2eec9678dbda274e906cc32ea8f711da3bYinghai Lu 499e123dd3f0ec1664576456ea1ea045591a0a95f0cYinghai Lu free_bootmem(__pa(usemap_map), size); 500193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell} 501193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell 502193faea9280a809cc30e81d7e503e01b1d7b7042Stephen Rothwell#ifdef CONFIG_MEMORY_HOTPLUG 50398f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#ifdef CONFIG_SPARSEMEM_VMEMMAP 50498f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 50598f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto unsigned long nr_pages) 50698f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 50798f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto /* This will make the necessary allocations eventually. */ 50898f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return sparse_mem_map_populate(pnum, nid); 50998f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 51098f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 51198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 51298f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return; /* XXX: Not implemented yet */ 51398f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 5140c0a4a517a31e05efb38304668198a873bfec6caYasunori Gotostatic void free_map_bootmem(struct page *page, unsigned long nr_pages) 5150c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto{ 5160c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto} 51798f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#else 5180b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenstatic struct page *__kmalloc_section_memmap(unsigned long nr_pages) 5190b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen{ 5200b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct page *page, *ret; 5210b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long memmap_size = sizeof(struct page) * nr_pages; 5220b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 523f2d0aa5bf8d4f7ae4cb1a7feebf5b1afddd0b9b0Yasunori Goto page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); 5240b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (page) 5250b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto got_map_page; 5260b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 5270b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = vmalloc(memmap_size); 5280b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (ret) 5290b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto got_map_ptr; 5300b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 5310b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return NULL; 5320b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansengot_map_page: 5330b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); 5340b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansengot_map_ptr: 5350b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen memset(ret, 0, memmap_size); 5360b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 5370b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return ret; 5380b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen} 5390b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 54098f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Gotostatic inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 54198f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto unsigned long nr_pages) 54298f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto{ 54398f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto return __kmalloc_section_memmap(nr_pages); 54498f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto} 54598f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto 5460b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenstatic void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 5470b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen{ 5489e2779fa281cfda13ac060753d674bbcaa23367eChristoph Lameter if (is_vmalloc_addr(memmap)) 5490b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen vfree(memmap); 5500b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen else 5510b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen free_pages((unsigned long)memmap, 5520b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen get_order(sizeof(struct page) * nr_pages)); 5530b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen} 5540c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 5550c0a4a517a31e05efb38304668198a873bfec6caYasunori Gotostatic void free_map_bootmem(struct page *page, unsigned long nr_pages) 5560c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto{ 5570c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto unsigned long maps_section_nr, removing_section_nr, i; 5580c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto int magic; 5590c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 5600c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto for (i = 0; i < nr_pages; i++, page++) { 5610c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto magic = atomic_read(&page->_mapcount); 5620c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 5630c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto BUG_ON(magic == NODE_INFO); 5640c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 5650c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 5660c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto removing_section_nr = page->private; 5670c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 5680c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto /* 5690c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto * When this function is called, the removing section is 5700c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto * logical offlined state. This means all pages are isolated 5710c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto * from page allocator. If removing section's memmap is placed 5720c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto * on the same section, it must not be freed. 5730c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto * If it is freed, page allocator may allocate it which will 5740c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto * be removed physically soon. 5750c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto */ 5760c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto if (maps_section_nr != removing_section_nr) 5770c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto put_page_bootmem(page); 5780c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto } 5790c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto} 58098f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 5810b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 582ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavartystatic void free_section_usemap(struct page *memmap, unsigned long *usemap) 583ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty{ 5840c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto struct page *usemap_page; 5850c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto unsigned long nr_pages; 5860c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 587ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty if (!usemap) 588ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty return; 589ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty 5900c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto usemap_page = virt_to_page(usemap); 591ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty /* 592ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty * Check to see if allocation came from hot-plug-add 593ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty */ 5940c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto if (PageSlab(usemap_page)) { 595ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty kfree(usemap); 596ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty if (memmap) 597ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty __kfree_section_memmap(memmap, PAGES_PER_SECTION); 598ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty return; 599ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty } 600ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty 601ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty /* 6020c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto * The usemap came from bootmem. This is packed with other usemaps 6030c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto * on the section which has pgdat at boot time. Just keep it as is now. 604ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty */ 6050c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 6060c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto if (memmap) { 6070c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto struct page *memmap_page; 6080c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto memmap_page = virt_to_page(memmap); 6090c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 6100c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 6110c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto >> PAGE_SHIFT; 6120c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto 6130c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto free_map_bootmem(memmap_page, nr_pages); 6140c0a4a517a31e05efb38304668198a873bfec6caYasunori Goto } 615ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty} 616ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty 61729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft/* 61829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * returns the number of sections whose mem_maps were properly 61929751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * set. If this is <=0, then that means that the passed-in 62029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft * map was not consumed and must be freed. 62129751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft */ 62231168481c32c8a485e1003af9433124dede57f8dAl Viroint __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 6230b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen int nr_pages) 62429751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft{ 6250b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long section_nr = pfn_to_section_nr(start_pfn); 6260b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct pglist_data *pgdat = zone->zone_pgdat; 6270b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct mem_section *ms; 6280b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen struct page *memmap; 6295c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman unsigned long *usemap; 6300b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen unsigned long flags; 6310b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen int ret; 63229751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 6330b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen /* 6340b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen * no locking for this, because it does its own 6350b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen * plus, it does a kmalloc 6360b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen */ 637bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong ret = sparse_index_init(section_nr, pgdat->node_id); 638bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong if (ret < 0 && ret != -EEXIST) 639bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong return ret; 64098f3cfc1dc7a53b629d43b7844a9b3f786213048Yasunori Goto memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages); 641bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong if (!memmap) 642bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong return -ENOMEM; 6435c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman usemap = __kmalloc_section_usemap(); 644bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong if (!usemap) { 645bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong __kfree_section_memmap(memmap, nr_pages); 646bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong return -ENOMEM; 647bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong } 6480b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 6490b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen pgdat_resize_lock(pgdat, &flags); 65029751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 6510b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ms = __pfn_to_section(start_pfn); 6520b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen if (ms->section_mem_map & SECTION_MARKED_PRESENT) { 6530b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen ret = -EEXIST; 6540b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen goto out; 6550b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen } 6565c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman 65729751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft ms->section_mem_map |= SECTION_MARKED_PRESENT; 65829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft 6595c0e3066474b57c56ff0d88ca31d95bd14232feeMel Gorman ret = sparse_init_one_section(ms, section_nr, memmap, usemap); 6600b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen 6610b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansenout: 6620b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen pgdat_resize_unlock(pgdat, &flags); 663bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong if (ret <= 0) { 664bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong kfree(usemap); 66546a66eecdf7bc12562ecb492297447ed0e1ecf59Mike Kravetz __kfree_section_memmap(memmap, nr_pages); 666bbd0682596f7a434467ee551fee18d5f0b818539WANG Cong } 6670b0acbec1bed75ec1e1daa7f7006323a2a2b2844Dave Hansen return ret; 66829751f6991e845f7d002a6ae520bf996b38c8dcdAndy Whitcroft} 669ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty 670ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavartyvoid sparse_remove_one_section(struct zone *zone, struct mem_section *ms) 671ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty{ 672ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty struct page *memmap = NULL; 673ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty unsigned long *usemap = NULL; 674ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty 675ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty if (ms->section_mem_map) { 676ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty usemap = ms->pageblock_flags; 677ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty memmap = sparse_decode_mem_map(ms->section_mem_map, 678ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty __section_nr(ms)); 679ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty ms->section_mem_map = 0; 680ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty ms->pageblock_flags = NULL; 681ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty } 682ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty 683ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty free_section_usemap(memmap, usemap); 684ea01ea937dcae2caa146dea1918cccf2f16ed3c4Badari Pulavarty} 685a3142c8e1dd57ff48040bdb3478cff9312543dc3Yasunori Goto#endif 686