vmstat.c revision 27bf71c2a7e596ed34e9bf2d4a5030321a09a1ad 
1/*
2 *  linux/mm/vmstat.c
3 *
4 *  Manages VM statistics
5 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
6 *
7 *  zoned VM statistics
8 *  Copyright (C) 2006 Silicon Graphics, Inc.,
9 *		Christoph Lameter <christoph@lameter.com>
10 */
11
12#include <linux/config.h>
13#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/cpu.h>
16
17void __get_zone_counts(unsigned long *active, unsigned long *inactive,
18			unsigned long *free, struct pglist_data *pgdat)
19{
20	struct zone *zones = pgdat->node_zones;
21	int i;
22
23	*active = 0;
24	*inactive = 0;
25	*free = 0;
26	for (i = 0; i < MAX_NR_ZONES; i++) {
27		*active += zones[i].nr_active;
28		*inactive += zones[i].nr_inactive;
29		*free += zones[i].free_pages;
30	}
31}
32
33void get_zone_counts(unsigned long *active,
34		unsigned long *inactive, unsigned long *free)
35{
36	struct pglist_data *pgdat;
37
38	*active = 0;
39	*inactive = 0;
40	*free = 0;
41	for_each_online_pgdat(pgdat) {
42		unsigned long l, m, n;
43		__get_zone_counts(&l, &m, &n, pgdat);
44		*active += l;
45		*inactive += m;
46		*free += n;
47	}
48}
49
50#ifdef CONFIG_VM_EVENT_COUNTERS
51DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
52EXPORT_PER_CPU_SYMBOL(vm_event_states);
53
54static void sum_vm_events(unsigned long *ret, cpumask_t *cpumask)
55{
56	int cpu = 0;
57	int i;
58
59	memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
60
61	cpu = first_cpu(*cpumask);
62	while (cpu < NR_CPUS) {
63		struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
64
65		cpu = next_cpu(cpu, *cpumask);
66
67		if (cpu < NR_CPUS)
68			prefetch(&per_cpu(vm_event_states, cpu));
69
70
71		for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
72			ret[i] += this->event[i];
73	}
74}
75
76/*
77 * Accumulate the vm event counters across all CPUs.
78 * The result is unavoidably approximate - it can change
79 * during and after execution of this function.
80*/
81void all_vm_events(unsigned long *ret)
82{
83	sum_vm_events(ret, &cpu_online_map);
84}
85EXPORT_SYMBOL_GPL(all_vm_events);
86
87#ifdef CONFIG_HOTPLUG
88/*
89 * Fold the foreign cpu events into our own.
90 *
91 * This is adding to the events on one processor
92 * but keeps the global counts constant.
93 */
94void vm_events_fold_cpu(int cpu)
95{
96	struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
97	int i;
98
99	for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
100		count_vm_events(i, fold_state->event[i]);
101		fold_state->event[i] = 0;
102	}
103}
104#endif /* CONFIG_HOTPLUG */
105
106#endif /* CONFIG_VM_EVENT_COUNTERS */
107
108/*
109 * Manage combined zone based / global counters
110 *
111 * vm_stat contains the global counters
112 */
113atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
114EXPORT_SYMBOL(vm_stat);
115
116#ifdef CONFIG_SMP
117
118static int calculate_threshold(struct zone *zone)
119{
120	int threshold;
121	int mem;	/* memory in 128 MB units */
122
123	/*
124	 * The threshold scales with the number of processors and the amount
125	 * of memory per zone. More memory means that we can defer updates for
126	 * longer, more processors could lead to more contention.
127 	 * fls() is used to have a cheap way of logarithmic scaling.
128	 *
129	 * Some sample thresholds:
130	 *
131	 * Threshold	Processors	(fls)	Zonesize	fls(mem+1)
132	 * ------------------------------------------------------------------
133	 * 8		1		1	0.9-1 GB	4
134	 * 16		2		2	0.9-1 GB	4
135	 * 20 		2		2	1-2 GB		5
136	 * 24		2		2	2-4 GB		6
137	 * 28		2		2	4-8 GB		7
138	 * 32		2		2	8-16 GB		8
139	 * 4		2		2	<128M		1
140	 * 30		4		3	2-4 GB		5
141	 * 48		4		3	8-16 GB		8
142	 * 32		8		4	1-2 GB		4
143	 * 32		8		4	0.9-1GB		4
144	 * 10		16		5	<128M		1
145	 * 40		16		5	900M		4
146	 * 70		64		7	2-4 GB		5
147	 * 84		64		7	4-8 GB		6
148	 * 108		512		9	4-8 GB		6
149	 * 125		1024		10	8-16 GB		8
150	 * 125		1024		10	16-32 GB	9
151	 */
152
153	mem = zone->present_pages >> (27 - PAGE_SHIFT);
154
155	threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
156
157	/*
158	 * Maximum threshold is 125
159	 */
160	threshold = min(125, threshold);
161
162	return threshold;
163}
164
165/*
166 * Refresh the thresholds for each zone.
167 */
168static void refresh_zone_stat_thresholds(void)
169{
170	struct zone *zone;
171	int cpu;
172	int threshold;
173
174	for_each_zone(zone) {
175
176		if (!zone->present_pages)
177			continue;
178
179		threshold = calculate_threshold(zone);
180
181		for_each_online_cpu(cpu)
182			zone_pcp(zone, cpu)->stat_threshold = threshold;
183	}
184}
185
186/*
187 * For use when we know that interrupts are disabled.
188 */
189void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
190				int delta)
191{
192	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
193	s8 *p = pcp->vm_stat_diff + item;
194	long x;
195
196	x = delta + *p;
197
198	if (unlikely(x > pcp->stat_threshold || x < -pcp->stat_threshold)) {
199		zone_page_state_add(x, zone, item);
200		x = 0;
201	}
202	*p = x;
203}
204EXPORT_SYMBOL(__mod_zone_page_state);
205
206/*
207 * For an unknown interrupt state
208 */
209void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
210					int delta)
211{
212	unsigned long flags;
213
214	local_irq_save(flags);
215	__mod_zone_page_state(zone, item, delta);
216	local_irq_restore(flags);
217}
218EXPORT_SYMBOL(mod_zone_page_state);
219
220/*
221 * Optimized increment and decrement functions.
222 *
223 * These are only for a single page and therefore can take a struct page *
224 * argument instead of struct zone *. This allows the inclusion of the code
225 * generated for page_zone(page) into the optimized functions.
226 *
227 * No overflow check is necessary and therefore the differential can be
228 * incremented or decremented in place which may allow the compilers to
229 * generate better code.
230 * The increment or decrement is known and therefore one boundary check can
231 * be omitted.
232 *
233 * NOTE: These functions are very performance sensitive. Change only
234 * with care.
235 *
236 * Some processors have inc/dec instructions that are atomic vs an interrupt.
237 * However, the code must first determine the differential location in a zone
238 * based on the processor number and then inc/dec the counter. There is no
239 * guarantee without disabling preemption that the processor will not change
240 * in between and therefore the atomicity vs. interrupt cannot be exploited
241 * in a useful way here.
242 */
243static void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
244{
245	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
246	s8 *p = pcp->vm_stat_diff + item;
247
248	(*p)++;
249
250	if (unlikely(*p > pcp->stat_threshold)) {
251		int overstep = pcp->stat_threshold / 2;
252
253		zone_page_state_add(*p + overstep, zone, item);
254		*p = -overstep;
255	}
256}
257
258void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
259{
260	__inc_zone_state(page_zone(page), item);
261}
262EXPORT_SYMBOL(__inc_zone_page_state);
263
264void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
265{
266	struct zone *zone = page_zone(page);
267	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
268	s8 *p = pcp->vm_stat_diff + item;
269
270	(*p)--;
271
272	if (unlikely(*p < - pcp->stat_threshold)) {
273		int overstep = pcp->stat_threshold / 2;
274
275		zone_page_state_add(*p - overstep, zone, item);
276		*p = overstep;
277	}
278}
279EXPORT_SYMBOL(__dec_zone_page_state);
280
281void inc_zone_state(struct zone *zone, enum zone_stat_item item)
282{
283	unsigned long flags;
284
285	local_irq_save(flags);
286	__inc_zone_state(zone, item);
287	local_irq_restore(flags);
288}
289
290void inc_zone_page_state(struct page *page, enum zone_stat_item item)
291{
292	unsigned long flags;
293	struct zone *zone;
294
295	zone = page_zone(page);
296	local_irq_save(flags);
297	__inc_zone_state(zone, item);
298	local_irq_restore(flags);
299}
300EXPORT_SYMBOL(inc_zone_page_state);
301
302void dec_zone_page_state(struct page *page, enum zone_stat_item item)
303{
304	unsigned long flags;
305
306	local_irq_save(flags);
307	__dec_zone_page_state(page, item);
308	local_irq_restore(flags);
309}
310EXPORT_SYMBOL(dec_zone_page_state);
311
312/*
313 * Update the zone counters for one cpu.
314 */
315void refresh_cpu_vm_stats(int cpu)
316{
317	struct zone *zone;
318	int i;
319	unsigned long flags;
320
321	for_each_zone(zone) {
322		struct per_cpu_pageset *pcp;
323
324		pcp = zone_pcp(zone, cpu);
325
326		for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
327			if (pcp->vm_stat_diff[i]) {
328				local_irq_save(flags);
329				zone_page_state_add(pcp->vm_stat_diff[i],
330					zone, i);
331				pcp->vm_stat_diff[i] = 0;
332				local_irq_restore(flags);
333			}
334	}
335}
336
337static void __refresh_cpu_vm_stats(void *dummy)
338{
339	refresh_cpu_vm_stats(smp_processor_id());
340}
341
342/*
343 * Consolidate all counters.
344 *
345 * Note that the result is less inaccurate but still inaccurate
346 * if concurrent processes are allowed to run.
347 */
348void refresh_vm_stats(void)
349{
350	on_each_cpu(__refresh_cpu_vm_stats, NULL, 0, 1);
351}
352EXPORT_SYMBOL(refresh_vm_stats);
353
354#endif
355
356#ifdef CONFIG_NUMA
357/*
358 * zonelist = the list of zones passed to the allocator
359 * z 	    = the zone from which the allocation occurred.
360 *
361 * Must be called with interrupts disabled.
362 */
363void zone_statistics(struct zonelist *zonelist, struct zone *z)
364{
365	if (z->zone_pgdat == zonelist->zones[0]->zone_pgdat) {
366		__inc_zone_state(z, NUMA_HIT);
367	} else {
368		__inc_zone_state(z, NUMA_MISS);
369		__inc_zone_state(zonelist->zones[0], NUMA_FOREIGN);
370	}
371	if (z->zone_pgdat == NODE_DATA(numa_node_id()))
372		__inc_zone_state(z, NUMA_LOCAL);
373	else
374		__inc_zone_state(z, NUMA_OTHER);
375}
376#endif
377
378#ifdef CONFIG_PROC_FS
379
380#include <linux/seq_file.h>
381
382static void *frag_start(struct seq_file *m, loff_t *pos)
383{
384	pg_data_t *pgdat;
385	loff_t node = *pos;
386	for (pgdat = first_online_pgdat();
387	     pgdat && node;
388	     pgdat = next_online_pgdat(pgdat))
389		--node;
390
391	return pgdat;
392}
393
394static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
395{
396	pg_data_t *pgdat = (pg_data_t *)arg;
397
398	(*pos)++;
399	return next_online_pgdat(pgdat);
400}
401
402static void frag_stop(struct seq_file *m, void *arg)
403{
404}
405
406/*
407 * This walks the free areas for each zone.
408 */
409static int frag_show(struct seq_file *m, void *arg)
410{
411	pg_data_t *pgdat = (pg_data_t *)arg;
412	struct zone *zone;
413	struct zone *node_zones = pgdat->node_zones;
414	unsigned long flags;
415	int order;
416
417	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
418		if (!populated_zone(zone))
419			continue;
420
421		spin_lock_irqsave(&zone->lock, flags);
422		seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
423		for (order = 0; order < MAX_ORDER; ++order)
424			seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
425		spin_unlock_irqrestore(&zone->lock, flags);
426		seq_putc(m, '\n');
427	}
428	return 0;
429}
430
431struct seq_operations fragmentation_op = {
432	.start	= frag_start,
433	.next	= frag_next,
434	.stop	= frag_stop,
435	.show	= frag_show,
436};
437
438#ifdef CONFIG_ZONE_DMA32
439#define TEXT_FOR_DMA32(xx) xx "_dma32",
440#else
441#define TEXT_FOR_DMA32(xx)
442#endif
443
444#ifdef CONFIG_HIGHMEM
445#define TEXT_FOR_HIGHMEM(xx) xx "_high",
446#else
447#define TEXT_FOR_HIGHMEM(xx)
448#endif
449
450#define TEXTS_FOR_ZONES(xx) xx "_dma", TEXT_FOR_DMA32(xx) xx "_normal", \
451					TEXT_FOR_HIGHMEM(xx)
452
453static char *vmstat_text[] = {
454	/* Zoned VM counters */
455	"nr_anon_pages",
456	"nr_mapped",
457	"nr_file_pages",
458	"nr_slab",
459	"nr_page_table_pages",
460	"nr_dirty",
461	"nr_writeback",
462	"nr_unstable",
463	"nr_bounce",
464
465#ifdef CONFIG_NUMA
466	"numa_hit",
467	"numa_miss",
468	"numa_foreign",
469	"numa_interleave",
470	"numa_local",
471	"numa_other",
472#endif
473
474#ifdef CONFIG_VM_EVENT_COUNTERS
475	"pgpgin",
476	"pgpgout",
477	"pswpin",
478	"pswpout",
479
480	TEXTS_FOR_ZONES("pgalloc")
481
482	"pgfree",
483	"pgactivate",
484	"pgdeactivate",
485
486	"pgfault",
487	"pgmajfault",
488
489	TEXTS_FOR_ZONES("pgrefill")
490	TEXTS_FOR_ZONES("pgsteal")
491	TEXTS_FOR_ZONES("pgscan_kswapd")
492	TEXTS_FOR_ZONES("pgscan_direct")
493
494	"pginodesteal",
495	"slabs_scanned",
496	"kswapd_steal",
497	"kswapd_inodesteal",
498	"pageoutrun",
499	"allocstall",
500
501	"pgrotated",
502#endif
503};
504
505/*
506 * Output information about zones in @pgdat.
507 */
508static int zoneinfo_show(struct seq_file *m, void *arg)
509{
510	pg_data_t *pgdat = arg;
511	struct zone *zone;
512	struct zone *node_zones = pgdat->node_zones;
513	unsigned long flags;
514
515	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; zone++) {
516		int i;
517
518		if (!populated_zone(zone))
519			continue;
520
521		spin_lock_irqsave(&zone->lock, flags);
522		seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
523		seq_printf(m,
524			   "\n  pages free     %lu"
525			   "\n        min      %lu"
526			   "\n        low      %lu"
527			   "\n        high     %lu"
528			   "\n        active   %lu"
529			   "\n        inactive %lu"
530			   "\n        scanned  %lu (a: %lu i: %lu)"
531			   "\n        spanned  %lu"
532			   "\n        present  %lu",
533			   zone->free_pages,
534			   zone->pages_min,
535			   zone->pages_low,
536			   zone->pages_high,
537			   zone->nr_active,
538			   zone->nr_inactive,
539			   zone->pages_scanned,
540			   zone->nr_scan_active, zone->nr_scan_inactive,
541			   zone->spanned_pages,
542			   zone->present_pages);
543
544		for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
545			seq_printf(m, "\n    %-12s %lu", vmstat_text[i],
546					zone_page_state(zone, i));
547
548		seq_printf(m,
549			   "\n        protection: (%lu",
550			   zone->lowmem_reserve[0]);
551		for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
552			seq_printf(m, ", %lu", zone->lowmem_reserve[i]);
553		seq_printf(m,
554			   ")"
555			   "\n  pagesets");
556		for_each_online_cpu(i) {
557			struct per_cpu_pageset *pageset;
558			int j;
559
560			pageset = zone_pcp(zone, i);
561			for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) {
562				if (pageset->pcp[j].count)
563					break;
564			}
565			if (j == ARRAY_SIZE(pageset->pcp))
566				continue;
567			for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) {
568				seq_printf(m,
569					   "\n    cpu: %i pcp: %i"
570					   "\n              count: %i"
571					   "\n              high:  %i"
572					   "\n              batch: %i",
573					   i, j,
574					   pageset->pcp[j].count,
575					   pageset->pcp[j].high,
576					   pageset->pcp[j].batch);
577			}
578#ifdef CONFIG_SMP
579			seq_printf(m, "\n  vm stats threshold: %d",
580					pageset->stat_threshold);
581#endif
582		}
583		seq_printf(m,
584			   "\n  all_unreclaimable: %u"
585			   "\n  prev_priority:     %i"
586			   "\n  temp_priority:     %i"
587			   "\n  start_pfn:         %lu",
588			   zone->all_unreclaimable,
589			   zone->prev_priority,
590			   zone->temp_priority,
591			   zone->zone_start_pfn);
592		spin_unlock_irqrestore(&zone->lock, flags);
593		seq_putc(m, '\n');
594	}
595	return 0;
596}
597
598struct seq_operations zoneinfo_op = {
599	.start	= frag_start, /* iterate over all zones. The same as in
600			       * fragmentation. */
601	.next	= frag_next,
602	.stop	= frag_stop,
603	.show	= zoneinfo_show,
604};
605
606static void *vmstat_start(struct seq_file *m, loff_t *pos)
607{
608	unsigned long *v;
609#ifdef CONFIG_VM_EVENT_COUNTERS
610	unsigned long *e;
611#endif
612	int i;
613
614	if (*pos >= ARRAY_SIZE(vmstat_text))
615		return NULL;
616
617#ifdef CONFIG_VM_EVENT_COUNTERS
618	v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long)
619			+ sizeof(struct vm_event_state), GFP_KERNEL);
620#else
621	v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long),
622			GFP_KERNEL);
623#endif
624	m->private = v;
625	if (!v)
626		return ERR_PTR(-ENOMEM);
627	for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
628		v[i] = global_page_state(i);
629#ifdef CONFIG_VM_EVENT_COUNTERS
630	e = v + NR_VM_ZONE_STAT_ITEMS;
631	all_vm_events(e);
632	e[PGPGIN] /= 2;		/* sectors -> kbytes */
633	e[PGPGOUT] /= 2;
634#endif
635	return v + *pos;
636}
637
638static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
639{
640	(*pos)++;
641	if (*pos >= ARRAY_SIZE(vmstat_text))
642		return NULL;
643	return (unsigned long *)m->private + *pos;
644}
645
646static int vmstat_show(struct seq_file *m, void *arg)
647{
648	unsigned long *l = arg;
649	unsigned long off = l - (unsigned long *)m->private;
650
651	seq_printf(m, "%s %lu\n", vmstat_text[off], *l);
652	return 0;
653}
654
655static void vmstat_stop(struct seq_file *m, void *arg)
656{
657	kfree(m->private);
658	m->private = NULL;
659}
660
661struct seq_operations vmstat_op = {
662	.start	= vmstat_start,
663	.next	= vmstat_next,
664	.stop	= vmstat_stop,
665	.show	= vmstat_show,
666};
667
668#endif /* CONFIG_PROC_FS */
669
670#ifdef CONFIG_SMP
671/*
672 * Use the cpu notifier to insure that the thresholds are recalculated
673 * when necessary.
674 */
675static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb,
676		unsigned long action,
677		void *hcpu)
678{
679	switch (action) {
680		case CPU_UP_PREPARE:
681		case CPU_UP_CANCELED:
682		case CPU_DEAD:
683			refresh_zone_stat_thresholds();
684			break;
685		default:
686			break;
687	}
688	return NOTIFY_OK;
689}
690
691static struct notifier_block __cpuinitdata vmstat_notifier =
692	{ &vmstat_cpuup_callback, NULL, 0 };
693
694int __init setup_vmstat(void)
695{
696	refresh_zone_stat_thresholds();
697	register_cpu_notifier(&vmstat_notifier);
698	return 0;
699}
700module_init(setup_vmstat)
701#endif
702