vmstat.c revision d23ad42324cc4378132e51f2fc5c9ba6cbe75182 
1/*
2 *  linux/mm/vmstat.c
3 *
4 *  Manages VM statistics
5 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
6 *
7 *  zoned VM statistics
8 *  Copyright (C) 2006 Silicon Graphics, Inc.,
9 *		Christoph Lameter <christoph@lameter.com>
10 */
11
12#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/cpu.h>
15
16void __get_zone_counts(unsigned long *active, unsigned long *inactive,
17			unsigned long *free, struct pglist_data *pgdat)
18{
19	*active = node_page_state(pgdat->node_id, NR_ACTIVE);
20	*inactive = node_page_state(pgdat->node_id, NR_INACTIVE);
21	*free = node_page_state(pgdat->node_id, NR_FREE_PAGES);
22}
23
24void get_zone_counts(unsigned long *active,
25		unsigned long *inactive, unsigned long *free)
26{
27	*active = global_page_state(NR_ACTIVE);
28	*inactive = global_page_state(NR_INACTIVE);
29	*free = global_page_state(NR_FREE_PAGES);
30}
31
32#ifdef CONFIG_VM_EVENT_COUNTERS
33DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
34EXPORT_PER_CPU_SYMBOL(vm_event_states);
35
36static void sum_vm_events(unsigned long *ret, cpumask_t *cpumask)
37{
38	int cpu = 0;
39	int i;
40
41	memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
42
43	cpu = first_cpu(*cpumask);
44	while (cpu < NR_CPUS) {
45		struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
46
47		cpu = next_cpu(cpu, *cpumask);
48
49		if (cpu < NR_CPUS)
50			prefetch(&per_cpu(vm_event_states, cpu));
51
52
53		for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
54			ret[i] += this->event[i];
55	}
56}
57
58/*
59 * Accumulate the vm event counters across all CPUs.
60 * The result is unavoidably approximate - it can change
61 * during and after execution of this function.
62*/
63void all_vm_events(unsigned long *ret)
64{
65	sum_vm_events(ret, &cpu_online_map);
66}
67EXPORT_SYMBOL_GPL(all_vm_events);
68
69#ifdef CONFIG_HOTPLUG
70/*
71 * Fold the foreign cpu events into our own.
72 *
73 * This is adding to the events on one processor
74 * but keeps the global counts constant.
75 */
76void vm_events_fold_cpu(int cpu)
77{
78	struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
79	int i;
80
81	for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
82		count_vm_events(i, fold_state->event[i]);
83		fold_state->event[i] = 0;
84	}
85}
86#endif /* CONFIG_HOTPLUG */
87
88#endif /* CONFIG_VM_EVENT_COUNTERS */
89
90/*
91 * Manage combined zone based / global counters
92 *
93 * vm_stat contains the global counters
94 */
95atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
96EXPORT_SYMBOL(vm_stat);
97
98#ifdef CONFIG_SMP
99
100static int calculate_threshold(struct zone *zone)
101{
102	int threshold;
103	int mem;	/* memory in 128 MB units */
104
105	/*
106	 * The threshold scales with the number of processors and the amount
107	 * of memory per zone. More memory means that we can defer updates for
108	 * longer, more processors could lead to more contention.
109 	 * fls() is used to have a cheap way of logarithmic scaling.
110	 *
111	 * Some sample thresholds:
112	 *
113	 * Threshold	Processors	(fls)	Zonesize	fls(mem+1)
114	 * ------------------------------------------------------------------
115	 * 8		1		1	0.9-1 GB	4
116	 * 16		2		2	0.9-1 GB	4
117	 * 20 		2		2	1-2 GB		5
118	 * 24		2		2	2-4 GB		6
119	 * 28		2		2	4-8 GB		7
120	 * 32		2		2	8-16 GB		8
121	 * 4		2		2	<128M		1
122	 * 30		4		3	2-4 GB		5
123	 * 48		4		3	8-16 GB		8
124	 * 32		8		4	1-2 GB		4
125	 * 32		8		4	0.9-1GB		4
126	 * 10		16		5	<128M		1
127	 * 40		16		5	900M		4
128	 * 70		64		7	2-4 GB		5
129	 * 84		64		7	4-8 GB		6
130	 * 108		512		9	4-8 GB		6
131	 * 125		1024		10	8-16 GB		8
132	 * 125		1024		10	16-32 GB	9
133	 */
134
135	mem = zone->present_pages >> (27 - PAGE_SHIFT);
136
137	threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
138
139	/*
140	 * Maximum threshold is 125
141	 */
142	threshold = min(125, threshold);
143
144	return threshold;
145}
146
147/*
148 * Refresh the thresholds for each zone.
149 */
150static void refresh_zone_stat_thresholds(void)
151{
152	struct zone *zone;
153	int cpu;
154	int threshold;
155
156	for_each_zone(zone) {
157
158		if (!zone->present_pages)
159			continue;
160
161		threshold = calculate_threshold(zone);
162
163		for_each_online_cpu(cpu)
164			zone_pcp(zone, cpu)->stat_threshold = threshold;
165	}
166}
167
168/*
169 * For use when we know that interrupts are disabled.
170 */
171void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
172				int delta)
173{
174	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
175	s8 *p = pcp->vm_stat_diff + item;
176	long x;
177
178	x = delta + *p;
179
180	if (unlikely(x > pcp->stat_threshold || x < -pcp->stat_threshold)) {
181		zone_page_state_add(x, zone, item);
182		x = 0;
183	}
184	*p = x;
185}
186EXPORT_SYMBOL(__mod_zone_page_state);
187
188/*
189 * For an unknown interrupt state
190 */
191void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
192					int delta)
193{
194	unsigned long flags;
195
196	local_irq_save(flags);
197	__mod_zone_page_state(zone, item, delta);
198	local_irq_restore(flags);
199}
200EXPORT_SYMBOL(mod_zone_page_state);
201
202/*
203 * Optimized increment and decrement functions.
204 *
205 * These are only for a single page and therefore can take a struct page *
206 * argument instead of struct zone *. This allows the inclusion of the code
207 * generated for page_zone(page) into the optimized functions.
208 *
209 * No overflow check is necessary and therefore the differential can be
210 * incremented or decremented in place which may allow the compilers to
211 * generate better code.
212 * The increment or decrement is known and therefore one boundary check can
213 * be omitted.
214 *
215 * NOTE: These functions are very performance sensitive. Change only
216 * with care.
217 *
218 * Some processors have inc/dec instructions that are atomic vs an interrupt.
219 * However, the code must first determine the differential location in a zone
220 * based on the processor number and then inc/dec the counter. There is no
221 * guarantee without disabling preemption that the processor will not change
222 * in between and therefore the atomicity vs. interrupt cannot be exploited
223 * in a useful way here.
224 */
225void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
226{
227	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
228	s8 *p = pcp->vm_stat_diff + item;
229
230	(*p)++;
231
232	if (unlikely(*p > pcp->stat_threshold)) {
233		int overstep = pcp->stat_threshold / 2;
234
235		zone_page_state_add(*p + overstep, zone, item);
236		*p = -overstep;
237	}
238}
239
240void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
241{
242	__inc_zone_state(page_zone(page), item);
243}
244EXPORT_SYMBOL(__inc_zone_page_state);
245
246void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
247{
248	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
249	s8 *p = pcp->vm_stat_diff + item;
250
251	(*p)--;
252
253	if (unlikely(*p < - pcp->stat_threshold)) {
254		int overstep = pcp->stat_threshold / 2;
255
256		zone_page_state_add(*p - overstep, zone, item);
257		*p = overstep;
258	}
259}
260
261void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
262{
263	__dec_zone_state(page_zone(page), item);
264}
265EXPORT_SYMBOL(__dec_zone_page_state);
266
267void inc_zone_state(struct zone *zone, enum zone_stat_item item)
268{
269	unsigned long flags;
270
271	local_irq_save(flags);
272	__inc_zone_state(zone, item);
273	local_irq_restore(flags);
274}
275
276void inc_zone_page_state(struct page *page, enum zone_stat_item item)
277{
278	unsigned long flags;
279	struct zone *zone;
280
281	zone = page_zone(page);
282	local_irq_save(flags);
283	__inc_zone_state(zone, item);
284	local_irq_restore(flags);
285}
286EXPORT_SYMBOL(inc_zone_page_state);
287
288void dec_zone_page_state(struct page *page, enum zone_stat_item item)
289{
290	unsigned long flags;
291
292	local_irq_save(flags);
293	__dec_zone_page_state(page, item);
294	local_irq_restore(flags);
295}
296EXPORT_SYMBOL(dec_zone_page_state);
297
298/*
299 * Update the zone counters for one cpu.
300 */
301void refresh_cpu_vm_stats(int cpu)
302{
303	struct zone *zone;
304	int i;
305	unsigned long flags;
306
307	for_each_zone(zone) {
308		struct per_cpu_pageset *pcp;
309
310		if (!populated_zone(zone))
311			continue;
312
313		pcp = zone_pcp(zone, cpu);
314
315		for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
316			if (pcp->vm_stat_diff[i]) {
317				local_irq_save(flags);
318				zone_page_state_add(pcp->vm_stat_diff[i],
319					zone, i);
320				pcp->vm_stat_diff[i] = 0;
321				local_irq_restore(flags);
322			}
323	}
324}
325
326static void __refresh_cpu_vm_stats(void *dummy)
327{
328	refresh_cpu_vm_stats(smp_processor_id());
329}
330
331/*
332 * Consolidate all counters.
333 *
334 * Note that the result is less inaccurate but still inaccurate
335 * if concurrent processes are allowed to run.
336 */
337void refresh_vm_stats(void)
338{
339	on_each_cpu(__refresh_cpu_vm_stats, NULL, 0, 1);
340}
341EXPORT_SYMBOL(refresh_vm_stats);
342
343#endif
344
345#ifdef CONFIG_NUMA
346/*
347 * zonelist = the list of zones passed to the allocator
348 * z 	    = the zone from which the allocation occurred.
349 *
350 * Must be called with interrupts disabled.
351 */
352void zone_statistics(struct zonelist *zonelist, struct zone *z)
353{
354	if (z->zone_pgdat == zonelist->zones[0]->zone_pgdat) {
355		__inc_zone_state(z, NUMA_HIT);
356	} else {
357		__inc_zone_state(z, NUMA_MISS);
358		__inc_zone_state(zonelist->zones[0], NUMA_FOREIGN);
359	}
360	if (z->node == numa_node_id())
361		__inc_zone_state(z, NUMA_LOCAL);
362	else
363		__inc_zone_state(z, NUMA_OTHER);
364}
365#endif
366
367#ifdef CONFIG_PROC_FS
368
369#include <linux/seq_file.h>
370
371static void *frag_start(struct seq_file *m, loff_t *pos)
372{
373	pg_data_t *pgdat;
374	loff_t node = *pos;
375	for (pgdat = first_online_pgdat();
376	     pgdat && node;
377	     pgdat = next_online_pgdat(pgdat))
378		--node;
379
380	return pgdat;
381}
382
383static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
384{
385	pg_data_t *pgdat = (pg_data_t *)arg;
386
387	(*pos)++;
388	return next_online_pgdat(pgdat);
389}
390
391static void frag_stop(struct seq_file *m, void *arg)
392{
393}
394
395/*
396 * This walks the free areas for each zone.
397 */
398static int frag_show(struct seq_file *m, void *arg)
399{
400	pg_data_t *pgdat = (pg_data_t *)arg;
401	struct zone *zone;
402	struct zone *node_zones = pgdat->node_zones;
403	unsigned long flags;
404	int order;
405
406	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
407		if (!populated_zone(zone))
408			continue;
409
410		spin_lock_irqsave(&zone->lock, flags);
411		seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
412		for (order = 0; order < MAX_ORDER; ++order)
413			seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
414		spin_unlock_irqrestore(&zone->lock, flags);
415		seq_putc(m, '\n');
416	}
417	return 0;
418}
419
420const struct seq_operations fragmentation_op = {
421	.start	= frag_start,
422	.next	= frag_next,
423	.stop	= frag_stop,
424	.show	= frag_show,
425};
426
427#ifdef CONFIG_ZONE_DMA32
428#define TEXT_FOR_DMA32(xx) xx "_dma32",
429#else
430#define TEXT_FOR_DMA32(xx)
431#endif
432
433#ifdef CONFIG_HIGHMEM
434#define TEXT_FOR_HIGHMEM(xx) xx "_high",
435#else
436#define TEXT_FOR_HIGHMEM(xx)
437#endif
438
439#define TEXTS_FOR_ZONES(xx) xx "_dma", TEXT_FOR_DMA32(xx) xx "_normal", \
440					TEXT_FOR_HIGHMEM(xx)
441
442static const char * const vmstat_text[] = {
443	/* Zoned VM counters */
444	"nr_free_pages",
445	"nr_active",
446	"nr_inactive",
447	"nr_anon_pages",
448	"nr_mapped",
449	"nr_file_pages",
450	"nr_slab_reclaimable",
451	"nr_slab_unreclaimable",
452	"nr_page_table_pages",
453	"nr_dirty",
454	"nr_writeback",
455	"nr_unstable",
456	"nr_bounce",
457	"nr_vmscan_write",
458
459#ifdef CONFIG_NUMA
460	"numa_hit",
461	"numa_miss",
462	"numa_foreign",
463	"numa_interleave",
464	"numa_local",
465	"numa_other",
466#endif
467
468#ifdef CONFIG_VM_EVENT_COUNTERS
469	"pgpgin",
470	"pgpgout",
471	"pswpin",
472	"pswpout",
473
474	TEXTS_FOR_ZONES("pgalloc")
475
476	"pgfree",
477	"pgactivate",
478	"pgdeactivate",
479
480	"pgfault",
481	"pgmajfault",
482
483	TEXTS_FOR_ZONES("pgrefill")
484	TEXTS_FOR_ZONES("pgsteal")
485	TEXTS_FOR_ZONES("pgscan_kswapd")
486	TEXTS_FOR_ZONES("pgscan_direct")
487
488	"pginodesteal",
489	"slabs_scanned",
490	"kswapd_steal",
491	"kswapd_inodesteal",
492	"pageoutrun",
493	"allocstall",
494
495	"pgrotated",
496#endif
497};
498
499/*
500 * Output information about zones in @pgdat.
501 */
502static int zoneinfo_show(struct seq_file *m, void *arg)
503{
504	pg_data_t *pgdat = arg;
505	struct zone *zone;
506	struct zone *node_zones = pgdat->node_zones;
507	unsigned long flags;
508
509	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; zone++) {
510		int i;
511
512		if (!populated_zone(zone))
513			continue;
514
515		spin_lock_irqsave(&zone->lock, flags);
516		seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
517		seq_printf(m,
518			   "\n  pages free     %lu"
519			   "\n        min      %lu"
520			   "\n        low      %lu"
521			   "\n        high     %lu"
522			   "\n        scanned  %lu (a: %lu i: %lu)"
523			   "\n        spanned  %lu"
524			   "\n        present  %lu",
525			   zone_page_state(zone, NR_FREE_PAGES),
526			   zone->pages_min,
527			   zone->pages_low,
528			   zone->pages_high,
529			   zone->pages_scanned,
530			   zone->nr_scan_active, zone->nr_scan_inactive,
531			   zone->spanned_pages,
532			   zone->present_pages);
533
534		for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
535			seq_printf(m, "\n    %-12s %lu", vmstat_text[i],
536					zone_page_state(zone, i));
537
538		seq_printf(m,
539			   "\n        protection: (%lu",
540			   zone->lowmem_reserve[0]);
541		for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
542			seq_printf(m, ", %lu", zone->lowmem_reserve[i]);
543		seq_printf(m,
544			   ")"
545			   "\n  pagesets");
546		for_each_online_cpu(i) {
547			struct per_cpu_pageset *pageset;
548			int j;
549
550			pageset = zone_pcp(zone, i);
551			for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) {
552				seq_printf(m,
553					   "\n    cpu: %i pcp: %i"
554					   "\n              count: %i"
555					   "\n              high:  %i"
556					   "\n              batch: %i",
557					   i, j,
558					   pageset->pcp[j].count,
559					   pageset->pcp[j].high,
560					   pageset->pcp[j].batch);
561			}
562#ifdef CONFIG_SMP
563			seq_printf(m, "\n  vm stats threshold: %d",
564					pageset->stat_threshold);
565#endif
566		}
567		seq_printf(m,
568			   "\n  all_unreclaimable: %u"
569			   "\n  prev_priority:     %i"
570			   "\n  start_pfn:         %lu",
571			   zone->all_unreclaimable,
572			   zone->prev_priority,
573			   zone->zone_start_pfn);
574		spin_unlock_irqrestore(&zone->lock, flags);
575		seq_putc(m, '\n');
576	}
577	return 0;
578}
579
580const struct seq_operations zoneinfo_op = {
581	.start	= frag_start, /* iterate over all zones. The same as in
582			       * fragmentation. */
583	.next	= frag_next,
584	.stop	= frag_stop,
585	.show	= zoneinfo_show,
586};
587
588static void *vmstat_start(struct seq_file *m, loff_t *pos)
589{
590	unsigned long *v;
591#ifdef CONFIG_VM_EVENT_COUNTERS
592	unsigned long *e;
593#endif
594	int i;
595
596	if (*pos >= ARRAY_SIZE(vmstat_text))
597		return NULL;
598
599#ifdef CONFIG_VM_EVENT_COUNTERS
600	v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long)
601			+ sizeof(struct vm_event_state), GFP_KERNEL);
602#else
603	v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long),
604			GFP_KERNEL);
605#endif
606	m->private = v;
607	if (!v)
608		return ERR_PTR(-ENOMEM);
609	for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
610		v[i] = global_page_state(i);
611#ifdef CONFIG_VM_EVENT_COUNTERS
612	e = v + NR_VM_ZONE_STAT_ITEMS;
613	all_vm_events(e);
614	e[PGPGIN] /= 2;		/* sectors -> kbytes */
615	e[PGPGOUT] /= 2;
616#endif
617	return v + *pos;
618}
619
620static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
621{
622	(*pos)++;
623	if (*pos >= ARRAY_SIZE(vmstat_text))
624		return NULL;
625	return (unsigned long *)m->private + *pos;
626}
627
628static int vmstat_show(struct seq_file *m, void *arg)
629{
630	unsigned long *l = arg;
631	unsigned long off = l - (unsigned long *)m->private;
632
633	seq_printf(m, "%s %lu\n", vmstat_text[off], *l);
634	return 0;
635}
636
637static void vmstat_stop(struct seq_file *m, void *arg)
638{
639	kfree(m->private);
640	m->private = NULL;
641}
642
643const struct seq_operations vmstat_op = {
644	.start	= vmstat_start,
645	.next	= vmstat_next,
646	.stop	= vmstat_stop,
647	.show	= vmstat_show,
648};
649
650#endif /* CONFIG_PROC_FS */
651
652#ifdef CONFIG_SMP
653/*
654 * Use the cpu notifier to insure that the thresholds are recalculated
655 * when necessary.
656 */
657static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb,
658		unsigned long action,
659		void *hcpu)
660{
661	switch (action) {
662	case CPU_UP_PREPARE:
663	case CPU_UP_CANCELED:
664	case CPU_DEAD:
665		refresh_zone_stat_thresholds();
666		break;
667	default:
668		break;
669	}
670	return NOTIFY_OK;
671}
672
673static struct notifier_block __cpuinitdata vmstat_notifier =
674	{ &vmstat_cpuup_callback, NULL, 0 };
675
676int __init setup_vmstat(void)
677{
678	refresh_zone_stat_thresholds();
679	register_cpu_notifier(&vmstat_notifier);
680	return 0;
681}
682module_init(setup_vmstat)
683#endif
684