pagewalk.c revision 116354d177ba2da37e91cf884e3d11e67f825efd 
1#include <linux/mm.h>
2#include <linux/highmem.h>
3#include <linux/sched.h>
4#include <linux/hugetlb.h>
5
6static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
7			  struct mm_walk *walk)
8{
9	pte_t *pte;
10	int err = 0;
11
12	pte = pte_offset_map(pmd, addr);
13	for (;;) {
14		err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
15		if (err)
16		       break;
17		addr += PAGE_SIZE;
18		if (addr == end)
19			break;
20		pte++;
21	}
22
23	pte_unmap(pte);
24	return err;
25}
26
27static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
28			  struct mm_walk *walk)
29{
30	pmd_t *pmd;
31	unsigned long next;
32	int err = 0;
33
34	pmd = pmd_offset(pud, addr);
35	do {
36		next = pmd_addr_end(addr, end);
37		if (pmd_none_or_clear_bad(pmd)) {
38			if (walk->pte_hole)
39				err = walk->pte_hole(addr, next, walk);
40			if (err)
41				break;
42			continue;
43		}
44		if (walk->pmd_entry)
45			err = walk->pmd_entry(pmd, addr, next, walk);
46		if (!err && walk->pte_entry)
47			err = walk_pte_range(pmd, addr, next, walk);
48		if (err)
49			break;
50	} while (pmd++, addr = next, addr != end);
51
52	return err;
53}
54
55static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
56			  struct mm_walk *walk)
57{
58	pud_t *pud;
59	unsigned long next;
60	int err = 0;
61
62	pud = pud_offset(pgd, addr);
63	do {
64		next = pud_addr_end(addr, end);
65		if (pud_none_or_clear_bad(pud)) {
66			if (walk->pte_hole)
67				err = walk->pte_hole(addr, next, walk);
68			if (err)
69				break;
70			continue;
71		}
72		if (walk->pud_entry)
73			err = walk->pud_entry(pud, addr, next, walk);
74		if (!err && (walk->pmd_entry || walk->pte_entry))
75			err = walk_pmd_range(pud, addr, next, walk);
76		if (err)
77			break;
78	} while (pud++, addr = next, addr != end);
79
80	return err;
81}
82
83#ifdef CONFIG_HUGETLB_PAGE
84static unsigned long hugetlb_entry_end(struct hstate *h, unsigned long addr,
85				       unsigned long end)
86{
87	unsigned long boundary = (addr & huge_page_mask(h)) + huge_page_size(h);
88	return boundary < end ? boundary : end;
89}
90
91static int walk_hugetlb_range(struct vm_area_struct *vma,
92			      unsigned long addr, unsigned long end,
93			      struct mm_walk *walk)
94{
95	struct hstate *h = hstate_vma(vma);
96	unsigned long next;
97	unsigned long hmask = huge_page_mask(h);
98	pte_t *pte;
99	int err = 0;
100
101	do {
102		next = hugetlb_entry_end(h, addr, end);
103		pte = huge_pte_offset(walk->mm, addr & hmask);
104		if (pte && walk->hugetlb_entry)
105			err = walk->hugetlb_entry(pte, hmask, addr, next, walk);
106		if (err)
107			return err;
108	} while (addr = next, addr != end);
109
110	return 0;
111}
112#endif
113
114/**
115 * walk_page_range - walk a memory map's page tables with a callback
116 * @mm: memory map to walk
117 * @addr: starting address
118 * @end: ending address
119 * @walk: set of callbacks to invoke for each level of the tree
120 *
121 * Recursively walk the page table for the memory area in a VMA,
122 * calling supplied callbacks. Callbacks are called in-order (first
123 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
124 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
125 *
126 * Each callback receives an entry pointer and the start and end of the
127 * associated range, and a copy of the original mm_walk for access to
128 * the ->private or ->mm fields.
129 *
130 * No locks are taken, but the bottom level iterator will map PTE
131 * directories from highmem if necessary.
132 *
133 * If any callback returns a non-zero value, the walk is aborted and
134 * the return value is propagated back to the caller. Otherwise 0 is returned.
135 */
136int walk_page_range(unsigned long addr, unsigned long end,
137		    struct mm_walk *walk)
138{
139	pgd_t *pgd;
140	unsigned long next;
141	int err = 0;
142	struct vm_area_struct *vma;
143
144	if (addr >= end)
145		return err;
146
147	if (!walk->mm)
148		return -EINVAL;
149
150	pgd = pgd_offset(walk->mm, addr);
151	do {
152		next = pgd_addr_end(addr, end);
153
154		/*
155		 * handle hugetlb vma individually because pagetable walk for
156		 * the hugetlb page is dependent on the architecture and
157		 * we can't handled it in the same manner as non-huge pages.
158		 */
159		vma = find_vma(walk->mm, addr);
160#ifdef CONFIG_HUGETLB_PAGE
161		if (vma && is_vm_hugetlb_page(vma)) {
162			if (vma->vm_end < next)
163				next = vma->vm_end;
164			/*
165			 * Hugepage is very tightly coupled with vma, so
166			 * walk through hugetlb entries within a given vma.
167			 */
168			err = walk_hugetlb_range(vma, addr, next, walk);
169			if (err)
170				break;
171			pgd = pgd_offset(walk->mm, next);
172			continue;
173		}
174#endif
175		if (pgd_none_or_clear_bad(pgd)) {
176			if (walk->pte_hole)
177				err = walk->pte_hole(addr, next, walk);
178			if (err)
179				break;
180			pgd++;
181			continue;
182		}
183		if (walk->pgd_entry)
184			err = walk->pgd_entry(pgd, addr, next, walk);
185		if (!err &&
186		    (walk->pud_entry || walk->pmd_entry || walk->pte_entry))
187			err = walk_pud_range(pgd, addr, next, walk);
188		if (err)
189			break;
190		pgd++;
191	} while (addr = next, addr != end);
192
193	return err;
194}
195