sparse.c revision 28ae55c98e4d16eac9a05a8a259d7763ef3aeb18 
1/*
2 * sparse memory mappings.
3 */
4#include <linux/config.h>
5#include <linux/mm.h>
6#include <linux/mmzone.h>
7#include <linux/bootmem.h>
8#include <linux/module.h>
9#include <linux/spinlock.h>
10#include <asm/dma.h>
11
12/*
13 * Permanent SPARSEMEM data:
14 *
15 * 1) mem_section	- memory sections, mem_map's for valid memory
16 */
17#ifdef CONFIG_SPARSEMEM_EXTREME
18struct mem_section *mem_section[NR_SECTION_ROOTS]
19	____cacheline_maxaligned_in_smp;
20#else
21struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
22	____cacheline_maxaligned_in_smp;
23#endif
24EXPORT_SYMBOL(mem_section);
25
26#ifdef CONFIG_SPARSEMEM_EXTREME
27static struct mem_section *sparse_index_alloc(int nid)
28{
29	struct mem_section *section = NULL;
30	unsigned long array_size = SECTIONS_PER_ROOT *
31				   sizeof(struct mem_section);
32
33	section = alloc_bootmem_node(NODE_DATA(nid), array_size);
34
35	if (section)
36		memset(section, 0, array_size);
37
38	return section;
39}
40
41static int sparse_index_init(unsigned long section_nr, int nid)
42{
43	static spinlock_t index_init_lock = SPIN_LOCK_UNLOCKED;
44	unsigned long root = SECTION_NR_TO_ROOT(section_nr);
45	struct mem_section *section;
46	int ret = 0;
47
48	if (mem_section[root])
49		return -EEXIST;
50
51	section = sparse_index_alloc(nid);
52	/*
53	 * This lock keeps two different sections from
54	 * reallocating for the same index
55	 */
56	spin_lock(&index_init_lock);
57
58	if (mem_section[root]) {
59		ret = -EEXIST;
60		goto out;
61	}
62
63	mem_section[root] = section;
64out:
65	spin_unlock(&index_init_lock);
66	return ret;
67}
68#else /* !SPARSEMEM_EXTREME */
69static inline int sparse_index_init(unsigned long section_nr, int nid)
70{
71	return 0;
72}
73#endif
74
75/* Record a memory area against a node. */
76void memory_present(int nid, unsigned long start, unsigned long end)
77{
78	unsigned long pfn;
79
80	start &= PAGE_SECTION_MASK;
81	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
82		unsigned long section = pfn_to_section_nr(pfn);
83		struct mem_section *ms;
84
85		sparse_index_init(section, nid);
86
87		ms = __nr_to_section(section);
88		if (!ms->section_mem_map)
89			ms->section_mem_map = SECTION_MARKED_PRESENT;
90	}
91}
92
93/*
94 * Only used by the i386 NUMA architecures, but relatively
95 * generic code.
96 */
97unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn,
98						     unsigned long end_pfn)
99{
100	unsigned long pfn;
101	unsigned long nr_pages = 0;
102
103	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
104		if (nid != early_pfn_to_nid(pfn))
105			continue;
106
107		if (pfn_valid(pfn))
108			nr_pages += PAGES_PER_SECTION;
109	}
110
111	return nr_pages * sizeof(struct page);
112}
113
114/*
115 * Subtle, we encode the real pfn into the mem_map such that
116 * the identity pfn - section_mem_map will return the actual
117 * physical page frame number.
118 */
119static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
120{
121	return (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
122}
123
124/*
125 * We need this if we ever free the mem_maps.  While not implemented yet,
126 * this function is included for parity with its sibling.
127 */
128static __attribute((unused))
129struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
130{
131	return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
132}
133
134static int sparse_init_one_section(struct mem_section *ms,
135		unsigned long pnum, struct page *mem_map)
136{
137	if (!valid_section(ms))
138		return -EINVAL;
139
140	ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum);
141
142	return 1;
143}
144
145static struct page *sparse_early_mem_map_alloc(unsigned long pnum)
146{
147	struct page *map;
148	int nid = early_pfn_to_nid(section_nr_to_pfn(pnum));
149	struct mem_section *ms = __nr_to_section(pnum);
150
151	map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION);
152	if (map)
153		return map;
154
155	map = alloc_bootmem_node(NODE_DATA(nid),
156			sizeof(struct page) * PAGES_PER_SECTION);
157	if (map)
158		return map;
159
160	printk(KERN_WARNING "%s: allocation failed\n", __FUNCTION__);
161	ms->section_mem_map = 0;
162	return NULL;
163}
164
165/*
166 * Allocate the accumulated non-linear sections, allocate a mem_map
167 * for each and record the physical to section mapping.
168 */
169void sparse_init(void)
170{
171	unsigned long pnum;
172	struct page *map;
173
174	for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
175		if (!valid_section_nr(pnum))
176			continue;
177
178		map = sparse_early_mem_map_alloc(pnum);
179		if (!map)
180			continue;
181		sparse_init_one_section(__nr_to_section(pnum), pnum, map);
182	}
183}
184
185/*
186 * returns the number of sections whose mem_maps were properly
187 * set.  If this is <=0, then that means that the passed-in
188 * map was not consumed and must be freed.
189 */
190int sparse_add_one_section(unsigned long start_pfn, int nr_pages, struct page *map)
191{
192	struct mem_section *ms = __pfn_to_section(start_pfn);
193
194	if (ms->section_mem_map & SECTION_MARKED_PRESENT)
195		return -EEXIST;
196
197	ms->section_mem_map |= SECTION_MARKED_PRESENT;
198
199	return sparse_init_one_section(ms, pfn_to_section_nr(start_pfn), map);
200}
201