ion_page_pool.c revision 797a95c48c48d5b83bf1a7893602161d2e2a4654 
1/*
2 * drivers/staging/android/ion/ion_mem_pool.c
3 *
4 * Copyright (C) 2011 Google, Inc.
5 *
6 * This software is licensed under the terms of the GNU General Public
7 * License version 2, as published by the Free Software Foundation, and
8 * may be copied, distributed, and modified under those terms.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 * GNU General Public License for more details.
14 *
15 */
16
17#include <linux/debugfs.h>
18#include <linux/dma-mapping.h>
19#include <linux/err.h>
20#include <linux/fs.h>
21#include <linux/list.h>
22#include <linux/module.h>
23#include <linux/slab.h>
24#include <linux/shrinker.h>
25#include "ion_priv.h"
26
27/* #define DEBUG_PAGE_POOL_SHRINKER */
28
29static struct plist_head pools = PLIST_HEAD_INIT(pools);
30static struct shrinker shrinker;
31
32struct ion_page_pool_item {
33	struct page *page;
34	struct list_head list;
35};
36
37static void *ion_page_pool_alloc_pages(struct ion_page_pool *pool)
38{
39	struct page *page = alloc_pages(pool->gfp_mask, pool->order);
40
41	if (!page)
42		return NULL;
43	/* this is only being used to flush the page for dma,
44	   this api is not really suitable for calling from a driver
45	   but no better way to flush a page for dma exist at this time */
46	__dma_page_cpu_to_dev(page, 0, PAGE_SIZE << pool->order,
47			      DMA_BIDIRECTIONAL);
48	return page;
49}
50
51static void ion_page_pool_free_pages(struct ion_page_pool *pool,
52				     struct page *page)
53{
54	__free_pages(page, pool->order);
55}
56
57static int ion_page_pool_add(struct ion_page_pool *pool, struct page *page)
58{
59	struct ion_page_pool_item *item;
60
61	item = kmalloc(sizeof(struct ion_page_pool_item), GFP_KERNEL);
62	if (!item)
63		return -ENOMEM;
64
65	mutex_lock(&pool->mutex);
66	item->page = page;
67	if (PageHighMem(page)) {
68		list_add_tail(&item->list, &pool->high_items);
69		pool->high_count++;
70	} else {
71		list_add_tail(&item->list, &pool->low_items);
72		pool->low_count++;
73	}
74	mutex_unlock(&pool->mutex);
75	return 0;
76}
77
78static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high)
79{
80	struct ion_page_pool_item *item;
81	struct page *page;
82
83	if (high) {
84		BUG_ON(!pool->high_count);
85		item = list_first_entry(&pool->high_items,
86					struct ion_page_pool_item, list);
87		pool->high_count--;
88	} else {
89		BUG_ON(!pool->low_count);
90		item = list_first_entry(&pool->low_items,
91					struct ion_page_pool_item, list);
92		pool->low_count--;
93	}
94
95	list_del(&item->list);
96	page = item->page;
97	kfree(item);
98	return page;
99}
100
101void *ion_page_pool_alloc(struct ion_page_pool *pool)
102{
103	struct page *page = NULL;
104
105	BUG_ON(!pool);
106
107	mutex_lock(&pool->mutex);
108	if (pool->high_count)
109		page = ion_page_pool_remove(pool, true);
110	else if (pool->low_count)
111		page = ion_page_pool_remove(pool, false);
112	mutex_unlock(&pool->mutex);
113
114	if (!page)
115		page = ion_page_pool_alloc_pages(pool);
116
117	return page;
118}
119
120void ion_page_pool_free(struct ion_page_pool *pool, struct page* page)
121{
122	int ret;
123
124	ret = ion_page_pool_add(pool, page);
125	if (ret)
126		ion_page_pool_free_pages(pool, page);
127}
128
129#ifdef DEBUG_PAGE_POOL_SHRINKER
130static int debug_drop_pools_set(void *data, u64 val)
131{
132	struct shrink_control sc;
133	int objs;
134
135	sc.gfp_mask = -1;
136	sc.nr_to_scan = 0;
137
138	if (!val)
139		return 0;
140
141	objs = shrinker.shrink(&shrinker, &sc);
142	sc.nr_to_scan = objs;
143
144	shrinker.shrink(&shrinker, &sc);
145	return 0;
146}
147
148static int debug_drop_pools_get(void *data, u64 *val)
149{
150	struct shrink_control sc;
151	int objs;
152
153	sc.gfp_mask = -1;
154	sc.nr_to_scan = 0;
155
156	objs = shrinker.shrink(&shrinker, &sc);
157	*val = objs;
158	return 0;
159}
160
161DEFINE_SIMPLE_ATTRIBUTE(debug_drop_pools_fops, debug_drop_pools_get,
162                        debug_drop_pools_set, "%llu\n");
163
164static int debug_grow_pools_set(void *data, u64 val)
165{
166	struct ion_page_pool *pool;
167	struct page *page;
168
169	plist_for_each_entry(pool, &pools, list) {
170		if (val != pool->list.prio)
171			continue;
172		page = ion_page_pool_alloc_pages(pool);
173		if (page)
174			ion_page_pool_add(pool, page);
175	}
176
177	return 0;
178}
179
180DEFINE_SIMPLE_ATTRIBUTE(debug_grow_pools_fops, debug_drop_pools_get,
181			debug_grow_pools_set, "%llu\n");
182#endif
183
184static int ion_page_pool_total(bool high)
185{
186	struct ion_page_pool *pool;
187	int total = 0;
188
189	plist_for_each_entry(pool, &pools, list) {
190		total += high ? (pool->high_count + pool->low_count) *
191			(1 << pool->order) :
192			pool->low_count * (1 << pool->order);
193	}
194	return total;
195}
196
197static int ion_page_pool_shrink(struct shrinker *shrinker,
198				 struct shrink_control *sc)
199{
200	struct ion_page_pool *pool;
201	int nr_freed = 0;
202	int i;
203	bool high;
204	int nr_to_scan = sc->nr_to_scan;
205
206	if (sc->gfp_mask & __GFP_HIGHMEM)
207		high = true;
208
209	if (nr_to_scan == 0)
210		return ion_page_pool_total(high);
211
212	plist_for_each_entry(pool, &pools, list) {
213		for (i = 0; i < nr_to_scan; i++) {
214			struct page *page;
215
216			mutex_lock(&pool->mutex);
217			if (high && pool->high_count) {
218				page = ion_page_pool_remove(pool, true);
219			} else if (pool->low_count) {
220				page = ion_page_pool_remove(pool, false);
221			} else {
222				mutex_unlock(&pool->mutex);
223				break;
224			}
225			mutex_unlock(&pool->mutex);
226			ion_page_pool_free_pages(pool, page);
227			nr_freed += (1 << pool->order);
228		}
229		nr_to_scan -= i;
230	}
231
232	return ion_page_pool_total(high);
233}
234
235struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order)
236{
237	struct ion_page_pool *pool = kmalloc(sizeof(struct ion_page_pool),
238					     GFP_KERNEL);
239	if (!pool)
240		return NULL;
241	pool->high_count = 0;
242	pool->low_count = 0;
243	INIT_LIST_HEAD(&pool->low_items);
244	INIT_LIST_HEAD(&pool->high_items);
245	pool->gfp_mask = gfp_mask;
246	pool->order = order;
247	mutex_init(&pool->mutex);
248	plist_node_init(&pool->list, order);
249	plist_add(&pool->list, &pools);
250
251	return pool;
252}
253
254void ion_page_pool_destroy(struct ion_page_pool *pool)
255{
256	plist_del(&pool->list, &pools);
257	kfree(pool);
258}
259
260static int __init ion_page_pool_init(void)
261{
262	shrinker.shrink = ion_page_pool_shrink;
263	shrinker.seeks = DEFAULT_SEEKS;
264	shrinker.batch = 0;
265	register_shrinker(&shrinker);
266#ifdef DEBUG_PAGE_POOL_SHRINKER
267	debugfs_create_file("ion_pools_shrink", 0644, NULL, NULL,
268			    &debug_drop_pools_fops);
269	debugfs_create_file("ion_pools_grow", 0644, NULL, NULL,
270			    &debug_grow_pools_fops);
271#endif
272	return 0;
273}
274
275static void __exit ion_page_pool_exit(void)
276{
277	unregister_shrinker(&shrinker);
278}
279
280module_init(ion_page_pool_init);
281module_exit(ion_page_pool_exit);
282