page_cgroup.c revision dc19f9db38295f811d9041bd89b113beccbd763a
1#include <linux/mm.h> 2#include <linux/mmzone.h> 3#include <linux/bootmem.h> 4#include <linux/bit_spinlock.h> 5#include <linux/page_cgroup.h> 6#include <linux/hash.h> 7#include <linux/slab.h> 8#include <linux/memory.h> 9#include <linux/vmalloc.h> 10#include <linux/cgroup.h> 11 12static void __meminit 13__init_page_cgroup(struct page_cgroup *pc, unsigned long pfn) 14{ 15 pc->flags = 0; 16 pc->mem_cgroup = NULL; 17 pc->page = pfn_to_page(pfn); 18} 19static unsigned long total_usage; 20 21#if !defined(CONFIG_SPARSEMEM) 22 23 24void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat) 25{ 26 pgdat->node_page_cgroup = NULL; 27} 28 29struct page_cgroup *lookup_page_cgroup(struct page *page) 30{ 31 unsigned long pfn = page_to_pfn(page); 32 unsigned long offset; 33 struct page_cgroup *base; 34 35 base = NODE_DATA(page_to_nid(page))->node_page_cgroup; 36 if (unlikely(!base)) 37 return NULL; 38 39 offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn; 40 return base + offset; 41} 42 43static int __init alloc_node_page_cgroup(int nid) 44{ 45 struct page_cgroup *base, *pc; 46 unsigned long table_size; 47 unsigned long start_pfn, nr_pages, index; 48 49 start_pfn = NODE_DATA(nid)->node_start_pfn; 50 nr_pages = NODE_DATA(nid)->node_spanned_pages; 51 52 table_size = sizeof(struct page_cgroup) * nr_pages; 53 54 base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), 55 table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); 56 if (!base) 57 return -ENOMEM; 58 for (index = 0; index < nr_pages; index++) { 59 pc = base + index; 60 __init_page_cgroup(pc, start_pfn + index); 61 } 62 NODE_DATA(nid)->node_page_cgroup = base; 63 total_usage += table_size; 64 return 0; 65} 66 67void __init page_cgroup_init(void) 68{ 69 70 int nid, fail; 71 72 if (mem_cgroup_subsys.disabled) 73 return; 74 75 for_each_online_node(nid) { 76 fail = alloc_node_page_cgroup(nid); 77 if (fail) 78 goto fail; 79 } 80 printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage); 81 printk(KERN_INFO "please try cgroup_disable=memory option if you" 82 " don't want\n"); 83 return; 84fail: 85 printk(KERN_CRIT "allocation of page_cgroup was failed.\n"); 86 printk(KERN_CRIT "please try cgroup_disable=memory boot option\n"); 87 panic("Out of memory"); 88} 89 90#else /* CONFIG_FLAT_NODE_MEM_MAP */ 91 92struct page_cgroup *lookup_page_cgroup(struct page *page) 93{ 94 unsigned long pfn = page_to_pfn(page); 95 struct mem_section *section = __pfn_to_section(pfn); 96 97 return section->page_cgroup + pfn; 98} 99 100/* __alloc_bootmem...() is protected by !slab_available() */ 101int __init_refok init_section_page_cgroup(unsigned long pfn) 102{ 103 struct mem_section *section; 104 struct page_cgroup *base, *pc; 105 unsigned long table_size; 106 int nid, index; 107 108 section = __pfn_to_section(pfn); 109 110 if (!section->page_cgroup) { 111 nid = page_to_nid(pfn_to_page(pfn)); 112 table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; 113 if (slab_is_available()) { 114 base = kmalloc_node(table_size, GFP_KERNEL, nid); 115 if (!base) 116 base = vmalloc_node(table_size, nid); 117 } else { 118 base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), 119 table_size, 120 PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); 121 } 122 } else { 123 /* 124 * We don't have to allocate page_cgroup again, but 125 * address of memmap may be changed. So, we have to initialize 126 * again. 127 */ 128 base = section->page_cgroup + pfn; 129 table_size = 0; 130 /* check address of memmap is changed or not. */ 131 if (base->page == pfn_to_page(pfn)) 132 return 0; 133 } 134 135 if (!base) { 136 printk(KERN_ERR "page cgroup allocation failure\n"); 137 return -ENOMEM; 138 } 139 140 for (index = 0; index < PAGES_PER_SECTION; index++) { 141 pc = base + index; 142 __init_page_cgroup(pc, pfn + index); 143 } 144 145 section = __pfn_to_section(pfn); 146 section->page_cgroup = base - pfn; 147 total_usage += table_size; 148 return 0; 149} 150#ifdef CONFIG_MEMORY_HOTPLUG 151void __free_page_cgroup(unsigned long pfn) 152{ 153 struct mem_section *ms; 154 struct page_cgroup *base; 155 156 ms = __pfn_to_section(pfn); 157 if (!ms || !ms->page_cgroup) 158 return; 159 base = ms->page_cgroup + pfn; 160 if (is_vmalloc_addr(base)) { 161 vfree(base); 162 ms->page_cgroup = NULL; 163 } else { 164 struct page *page = virt_to_page(base); 165 if (!PageReserved(page)) { /* Is bootmem ? */ 166 kfree(base); 167 ms->page_cgroup = NULL; 168 } 169 } 170} 171 172int __meminit online_page_cgroup(unsigned long start_pfn, 173 unsigned long nr_pages, 174 int nid) 175{ 176 unsigned long start, end, pfn; 177 int fail = 0; 178 179 start = start_pfn & ~(PAGES_PER_SECTION - 1); 180 end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); 181 182 for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) { 183 if (!pfn_present(pfn)) 184 continue; 185 fail = init_section_page_cgroup(pfn); 186 } 187 if (!fail) 188 return 0; 189 190 /* rollback */ 191 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) 192 __free_page_cgroup(pfn); 193 194 return -ENOMEM; 195} 196 197int __meminit offline_page_cgroup(unsigned long start_pfn, 198 unsigned long nr_pages, int nid) 199{ 200 unsigned long start, end, pfn; 201 202 start = start_pfn & ~(PAGES_PER_SECTION - 1); 203 end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); 204 205 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) 206 __free_page_cgroup(pfn); 207 return 0; 208 209} 210 211static int __meminit page_cgroup_callback(struct notifier_block *self, 212 unsigned long action, void *arg) 213{ 214 struct memory_notify *mn = arg; 215 int ret = 0; 216 switch (action) { 217 case MEM_GOING_ONLINE: 218 ret = online_page_cgroup(mn->start_pfn, 219 mn->nr_pages, mn->status_change_nid); 220 break; 221 case MEM_OFFLINE: 222 offline_page_cgroup(mn->start_pfn, 223 mn->nr_pages, mn->status_change_nid); 224 break; 225 case MEM_CANCEL_ONLINE: 226 case MEM_GOING_OFFLINE: 227 break; 228 case MEM_ONLINE: 229 case MEM_CANCEL_OFFLINE: 230 break; 231 } 232 233 if (ret) 234 ret = notifier_from_errno(ret); 235 else 236 ret = NOTIFY_OK; 237 238 return ret; 239} 240 241#endif 242 243void __init page_cgroup_init(void) 244{ 245 unsigned long pfn; 246 int fail = 0; 247 248 if (mem_cgroup_subsys.disabled) 249 return; 250 251 for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) { 252 if (!pfn_present(pfn)) 253 continue; 254 fail = init_section_page_cgroup(pfn); 255 } 256 if (fail) { 257 printk(KERN_CRIT "try cgroup_disable=memory boot option\n"); 258 panic("Out of memory"); 259 } else { 260 hotplug_memory_notifier(page_cgroup_callback, 0); 261 } 262 printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage); 263 printk(KERN_INFO "please try cgroup_disable=memory option if you don't" 264 " want\n"); 265} 266 267void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat) 268{ 269 return; 270} 271 272#endif 273