pagewalk.c revision 5dc37642cbce34619e4588a9f0bdad1d2f870956 
1#include <linux/mm.h>
2#include <linux/highmem.h>
3#include <linux/sched.h>
4#include <linux/hugetlb.h>
5
6static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
7			  struct mm_walk *walk)
8{
9	pte_t *pte;
10	int err = 0;
11
12	pte = pte_offset_map(pmd, addr);
13	for (;;) {
14		err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
15		if (err)
16		       break;
17		addr += PAGE_SIZE;
18		if (addr == end)
19			break;
20		pte++;
21	}
22
23	pte_unmap(pte);
24	return err;
25}
26
27static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
28			  struct mm_walk *walk)
29{
30	pmd_t *pmd;
31	unsigned long next;
32	int err = 0;
33
34	pmd = pmd_offset(pud, addr);
35	do {
36		next = pmd_addr_end(addr, end);
37		if (pmd_none_or_clear_bad(pmd)) {
38			if (walk->pte_hole)
39				err = walk->pte_hole(addr, next, walk);
40			if (err)
41				break;
42			continue;
43		}
44		if (walk->pmd_entry)
45			err = walk->pmd_entry(pmd, addr, next, walk);
46		if (!err && walk->pte_entry)
47			err = walk_pte_range(pmd, addr, next, walk);
48		if (err)
49			break;
50	} while (pmd++, addr = next, addr != end);
51
52	return err;
53}
54
55static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
56			  struct mm_walk *walk)
57{
58	pud_t *pud;
59	unsigned long next;
60	int err = 0;
61
62	pud = pud_offset(pgd, addr);
63	do {
64		next = pud_addr_end(addr, end);
65		if (pud_none_or_clear_bad(pud)) {
66			if (walk->pte_hole)
67				err = walk->pte_hole(addr, next, walk);
68			if (err)
69				break;
70			continue;
71		}
72		if (walk->pud_entry)
73			err = walk->pud_entry(pud, addr, next, walk);
74		if (!err && (walk->pmd_entry || walk->pte_entry))
75			err = walk_pmd_range(pud, addr, next, walk);
76		if (err)
77			break;
78	} while (pud++, addr = next, addr != end);
79
80	return err;
81}
82
83/**
84 * walk_page_range - walk a memory map's page tables with a callback
85 * @mm: memory map to walk
86 * @addr: starting address
87 * @end: ending address
88 * @walk: set of callbacks to invoke for each level of the tree
89 *
90 * Recursively walk the page table for the memory area in a VMA,
91 * calling supplied callbacks. Callbacks are called in-order (first
92 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
93 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
94 *
95 * Each callback receives an entry pointer and the start and end of the
96 * associated range, and a copy of the original mm_walk for access to
97 * the ->private or ->mm fields.
98 *
99 * No locks are taken, but the bottom level iterator will map PTE
100 * directories from highmem if necessary.
101 *
102 * If any callback returns a non-zero value, the walk is aborted and
103 * the return value is propagated back to the caller. Otherwise 0 is returned.
104 */
105int walk_page_range(unsigned long addr, unsigned long end,
106		    struct mm_walk *walk)
107{
108	pgd_t *pgd;
109	unsigned long next;
110	int err = 0;
111	struct vm_area_struct *vma;
112
113	if (addr >= end)
114		return err;
115
116	if (!walk->mm)
117		return -EINVAL;
118
119	pgd = pgd_offset(walk->mm, addr);
120	do {
121		next = pgd_addr_end(addr, end);
122
123		/*
124		 * handle hugetlb vma individually because pagetable walk for
125		 * the hugetlb page is dependent on the architecture and
126		 * we can't handled it in the same manner as non-huge pages.
127		 */
128		vma = find_vma(walk->mm, addr);
129#ifdef CONFIG_HUGETLB_PAGE
130		if (vma && is_vm_hugetlb_page(vma)) {
131			pte_t *pte;
132			struct hstate *hs;
133
134			if (vma->vm_end < next)
135				next = vma->vm_end;
136			hs = hstate_vma(vma);
137			pte = huge_pte_offset(walk->mm,
138					      addr & huge_page_mask(hs));
139			if (pte && !huge_pte_none(huge_ptep_get(pte))
140			    && walk->hugetlb_entry)
141				err = walk->hugetlb_entry(pte, addr,
142							  next, walk);
143			if (err)
144				break;
145			continue;
146		}
147#endif
148		if (pgd_none_or_clear_bad(pgd)) {
149			if (walk->pte_hole)
150				err = walk->pte_hole(addr, next, walk);
151			if (err)
152				break;
153			pgd++;
154			continue;
155		}
156		if (walk->pgd_entry)
157			err = walk->pgd_entry(pgd, addr, next, walk);
158		if (!err &&
159		    (walk->pud_entry || walk->pmd_entry || walk->pte_entry))
160			err = walk_pud_range(pgd, addr, next, walk);
161		if (err)
162			break;
163		pgd++;
164	} while (addr = next, addr != end);
165
166	return err;
167}
168