ttm_memory.c revision ec42a6e7dcfc2e9a92fad1c132bc9e110fafeb3f 
1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28#include "ttm/ttm_memory.h"
29#include "ttm/ttm_module.h"
30#include <linux/spinlock.h>
31#include <linux/sched.h>
32#include <linux/wait.h>
33#include <linux/mm.h>
34#include <linux/module.h>
35
36#define TTM_MEMORY_ALLOC_RETRIES 4
37
38struct ttm_mem_zone {
39	struct kobject kobj;
40	struct ttm_mem_global *glob;
41	const char *name;
42	uint64_t zone_mem;
43	uint64_t emer_mem;
44	uint64_t max_mem;
45	uint64_t swap_limit;
46	uint64_t used_mem;
47};
48
49static struct attribute ttm_mem_sys = {
50	.name = "zone_memory",
51	.mode = S_IRUGO
52};
53static struct attribute ttm_mem_emer = {
54	.name = "emergency_memory",
55	.mode = S_IRUGO | S_IWUSR
56};
57static struct attribute ttm_mem_max = {
58	.name = "available_memory",
59	.mode = S_IRUGO | S_IWUSR
60};
61static struct attribute ttm_mem_swap = {
62	.name = "swap_limit",
63	.mode = S_IRUGO | S_IWUSR
64};
65static struct attribute ttm_mem_used = {
66	.name = "used_memory",
67	.mode = S_IRUGO
68};
69
70static void ttm_mem_zone_kobj_release(struct kobject *kobj)
71{
72	struct ttm_mem_zone *zone =
73		container_of(kobj, struct ttm_mem_zone, kobj);
74
75	printk(KERN_INFO TTM_PFX
76	       "Zone %7s: Used memory at exit: %llu kiB.\n",
77	       zone->name, (unsigned long long) zone->used_mem >> 10);
78	kfree(zone);
79}
80
81static ssize_t ttm_mem_zone_show(struct kobject *kobj,
82				 struct attribute *attr,
83				 char *buffer)
84{
85	struct ttm_mem_zone *zone =
86		container_of(kobj, struct ttm_mem_zone, kobj);
87	uint64_t val = 0;
88
89	spin_lock(&zone->glob->lock);
90	if (attr == &ttm_mem_sys)
91		val = zone->zone_mem;
92	else if (attr == &ttm_mem_emer)
93		val = zone->emer_mem;
94	else if (attr == &ttm_mem_max)
95		val = zone->max_mem;
96	else if (attr == &ttm_mem_swap)
97		val = zone->swap_limit;
98	else if (attr == &ttm_mem_used)
99		val = zone->used_mem;
100	spin_unlock(&zone->glob->lock);
101
102	return snprintf(buffer, PAGE_SIZE, "%llu\n",
103			(unsigned long long) val >> 10);
104}
105
106static void ttm_check_swapping(struct ttm_mem_global *glob);
107
108static ssize_t ttm_mem_zone_store(struct kobject *kobj,
109				  struct attribute *attr,
110				  const char *buffer,
111				  size_t size)
112{
113	struct ttm_mem_zone *zone =
114		container_of(kobj, struct ttm_mem_zone, kobj);
115	int chars;
116	unsigned long val;
117	uint64_t val64;
118
119	chars = sscanf(buffer, "%lu", &val);
120	if (chars == 0)
121		return size;
122
123	val64 = val;
124	val64 <<= 10;
125
126	spin_lock(&zone->glob->lock);
127	if (val64 > zone->zone_mem)
128		val64 = zone->zone_mem;
129	if (attr == &ttm_mem_emer) {
130		zone->emer_mem = val64;
131		if (zone->max_mem > val64)
132			zone->max_mem = val64;
133	} else if (attr == &ttm_mem_max) {
134		zone->max_mem = val64;
135		if (zone->emer_mem < val64)
136			zone->emer_mem = val64;
137	} else if (attr == &ttm_mem_swap)
138		zone->swap_limit = val64;
139	spin_unlock(&zone->glob->lock);
140
141	ttm_check_swapping(zone->glob);
142
143	return size;
144}
145
146static struct attribute *ttm_mem_zone_attrs[] = {
147	&ttm_mem_sys,
148	&ttm_mem_emer,
149	&ttm_mem_max,
150	&ttm_mem_swap,
151	&ttm_mem_used,
152	NULL
153};
154
155static struct sysfs_ops ttm_mem_zone_ops = {
156	.show = &ttm_mem_zone_show,
157	.store = &ttm_mem_zone_store
158};
159
160static struct kobj_type ttm_mem_zone_kobj_type = {
161	.release = &ttm_mem_zone_kobj_release,
162	.sysfs_ops = &ttm_mem_zone_ops,
163	.default_attrs = ttm_mem_zone_attrs,
164};
165
166static void ttm_mem_global_kobj_release(struct kobject *kobj)
167{
168	struct ttm_mem_global *glob =
169		container_of(kobj, struct ttm_mem_global, kobj);
170
171	kfree(glob);
172}
173
174static struct kobj_type ttm_mem_glob_kobj_type = {
175	.release = &ttm_mem_global_kobj_release,
176};
177
178static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,
179					bool from_wq, uint64_t extra)
180{
181	unsigned int i;
182	struct ttm_mem_zone *zone;
183	uint64_t target;
184
185	for (i = 0; i < glob->num_zones; ++i) {
186		zone = glob->zones[i];
187
188		if (from_wq)
189			target = zone->swap_limit;
190		else if (capable(CAP_SYS_ADMIN))
191			target = zone->emer_mem;
192		else
193			target = zone->max_mem;
194
195		target = (extra > target) ? 0ULL : target;
196
197		if (zone->used_mem > target)
198			return true;
199	}
200	return false;
201}
202
203/**
204 * At this point we only support a single shrink callback.
205 * Extend this if needed, perhaps using a linked list of callbacks.
206 * Note that this function is reentrant:
207 * many threads may try to swap out at any given time.
208 */
209
210static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,
211		       uint64_t extra)
212{
213	int ret;
214	struct ttm_mem_shrink *shrink;
215
216	spin_lock(&glob->lock);
217	if (glob->shrink == NULL)
218		goto out;
219
220	while (ttm_zones_above_swap_target(glob, from_wq, extra)) {
221		shrink = glob->shrink;
222		spin_unlock(&glob->lock);
223		ret = shrink->do_shrink(shrink);
224		spin_lock(&glob->lock);
225		if (unlikely(ret != 0))
226			goto out;
227	}
228out:
229	spin_unlock(&glob->lock);
230}
231
232
233
234static void ttm_shrink_work(struct work_struct *work)
235{
236	struct ttm_mem_global *glob =
237	    container_of(work, struct ttm_mem_global, work);
238
239	ttm_shrink(glob, true, 0ULL);
240}
241
242static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,
243				    const struct sysinfo *si)
244{
245	struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
246	uint64_t mem;
247	int ret;
248
249	if (unlikely(!zone))
250		return -ENOMEM;
251
252	mem = si->totalram - si->totalhigh;
253	mem *= si->mem_unit;
254
255	zone->name = "kernel";
256	zone->zone_mem = mem;
257	zone->max_mem = mem >> 1;
258	zone->emer_mem = (mem >> 1) + (mem >> 2);
259	zone->swap_limit = zone->max_mem - (mem >> 3);
260	zone->used_mem = 0;
261	zone->glob = glob;
262	glob->zone_kernel = zone;
263	kobject_init(&zone->kobj, &ttm_mem_zone_kobj_type);
264	ret = kobject_add(&zone->kobj, &glob->kobj, zone->name);
265	if (unlikely(ret != 0)) {
266		kobject_put(&zone->kobj);
267		return ret;
268	}
269	glob->zones[glob->num_zones++] = zone;
270	return 0;
271}
272
273#ifdef CONFIG_HIGHMEM
274static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob,
275				     const struct sysinfo *si)
276{
277	struct ttm_mem_zone *zone;
278	uint64_t mem;
279	int ret;
280
281	if (si->totalhigh == 0)
282		return 0;
283
284	zone = kzalloc(sizeof(*zone), GFP_KERNEL);
285	if (unlikely(!zone))
286		return -ENOMEM;
287
288	mem = si->totalram;
289	mem *= si->mem_unit;
290
291	zone->name = "highmem";
292	zone->zone_mem = mem;
293	zone->max_mem = mem >> 1;
294	zone->emer_mem = (mem >> 1) + (mem >> 2);
295	zone->swap_limit = zone->max_mem - (mem >> 3);
296	zone->used_mem = 0;
297	zone->glob = glob;
298	glob->zone_highmem = zone;
299	kobject_init(&zone->kobj, &ttm_mem_zone_kobj_type);
300	ret = kobject_add(&zone->kobj, &glob->kobj, zone->name);
301	if (unlikely(ret != 0)) {
302		kobject_put(&zone->kobj);
303		return ret;
304	}
305	glob->zones[glob->num_zones++] = zone;
306	return 0;
307}
308#else
309static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob,
310				   const struct sysinfo *si)
311{
312	struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
313	uint64_t mem;
314	int ret;
315
316	if (unlikely(!zone))
317		return -ENOMEM;
318
319	mem = si->totalram;
320	mem *= si->mem_unit;
321
322	/**
323	 * No special dma32 zone needed.
324	 */
325
326	if (mem <= ((uint64_t) 1ULL << 32)) {
327		kfree(zone);
328		return 0;
329	}
330
331	/*
332	 * Limit max dma32 memory to 4GB for now
333	 * until we can figure out how big this
334	 * zone really is.
335	 */
336
337	mem = ((uint64_t) 1ULL << 32);
338	zone->name = "dma32";
339	zone->zone_mem = mem;
340	zone->max_mem = mem >> 1;
341	zone->emer_mem = (mem >> 1) + (mem >> 2);
342	zone->swap_limit = zone->max_mem - (mem >> 3);
343	zone->used_mem = 0;
344	zone->glob = glob;
345	glob->zone_dma32 = zone;
346	kobject_init(&zone->kobj, &ttm_mem_zone_kobj_type);
347	ret = kobject_add(&zone->kobj, &glob->kobj, zone->name);
348	if (unlikely(ret != 0)) {
349		kobject_put(&zone->kobj);
350		return ret;
351	}
352	glob->zones[glob->num_zones++] = zone;
353	return 0;
354}
355#endif
356
357int ttm_mem_global_init(struct ttm_mem_global *glob)
358{
359	struct sysinfo si;
360	int ret;
361	int i;
362	struct ttm_mem_zone *zone;
363
364	spin_lock_init(&glob->lock);
365	glob->swap_queue = create_singlethread_workqueue("ttm_swap");
366	INIT_WORK(&glob->work, ttm_shrink_work);
367	init_waitqueue_head(&glob->queue);
368	kobject_init(&glob->kobj, &ttm_mem_glob_kobj_type);
369	ret = kobject_add(&glob->kobj,
370			  ttm_get_kobj(),
371			  "memory_accounting");
372	if (unlikely(ret != 0)) {
373		kobject_put(&glob->kobj);
374		return ret;
375	}
376
377	si_meminfo(&si);
378
379	ret = ttm_mem_init_kernel_zone(glob, &si);
380	if (unlikely(ret != 0))
381		goto out_no_zone;
382#ifdef CONFIG_HIGHMEM
383	ret = ttm_mem_init_highmem_zone(glob, &si);
384	if (unlikely(ret != 0))
385		goto out_no_zone;
386#else
387	ret = ttm_mem_init_dma32_zone(glob, &si);
388	if (unlikely(ret != 0))
389		goto out_no_zone;
390#endif
391	for (i = 0; i < glob->num_zones; ++i) {
392		zone = glob->zones[i];
393		printk(KERN_INFO TTM_PFX
394		       "Zone %7s: Available graphics memory: %llu kiB.\n",
395		       zone->name, (unsigned long long) zone->max_mem >> 10);
396	}
397	return 0;
398out_no_zone:
399	ttm_mem_global_release(glob);
400	return ret;
401}
402EXPORT_SYMBOL(ttm_mem_global_init);
403
404void ttm_mem_global_release(struct ttm_mem_global *glob)
405{
406	unsigned int i;
407	struct ttm_mem_zone *zone;
408
409	flush_workqueue(glob->swap_queue);
410	destroy_workqueue(glob->swap_queue);
411	glob->swap_queue = NULL;
412	for (i = 0; i < glob->num_zones; ++i) {
413		zone = glob->zones[i];
414		kobject_del(&zone->kobj);
415		kobject_put(&zone->kobj);
416	}
417	kobject_del(&glob->kobj);
418	kobject_put(&glob->kobj);
419}
420EXPORT_SYMBOL(ttm_mem_global_release);
421
422static void ttm_check_swapping(struct ttm_mem_global *glob)
423{
424	bool needs_swapping = false;
425	unsigned int i;
426	struct ttm_mem_zone *zone;
427
428	spin_lock(&glob->lock);
429	for (i = 0; i < glob->num_zones; ++i) {
430		zone = glob->zones[i];
431		if (zone->used_mem > zone->swap_limit) {
432			needs_swapping = true;
433			break;
434		}
435	}
436
437	spin_unlock(&glob->lock);
438
439	if (unlikely(needs_swapping))
440		(void)queue_work(glob->swap_queue, &glob->work);
441
442}
443
444static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,
445				     struct ttm_mem_zone *single_zone,
446				     uint64_t amount)
447{
448	unsigned int i;
449	struct ttm_mem_zone *zone;
450
451	spin_lock(&glob->lock);
452	for (i = 0; i < glob->num_zones; ++i) {
453		zone = glob->zones[i];
454		if (single_zone && zone != single_zone)
455			continue;
456		zone->used_mem -= amount;
457	}
458	spin_unlock(&glob->lock);
459}
460
461void ttm_mem_global_free(struct ttm_mem_global *glob,
462			 uint64_t amount)
463{
464	return ttm_mem_global_free_zone(glob, NULL, amount);
465}
466EXPORT_SYMBOL(ttm_mem_global_free);
467
468static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
469				  struct ttm_mem_zone *single_zone,
470				  uint64_t amount, bool reserve)
471{
472	uint64_t limit;
473	int ret = -ENOMEM;
474	unsigned int i;
475	struct ttm_mem_zone *zone;
476
477	spin_lock(&glob->lock);
478	for (i = 0; i < glob->num_zones; ++i) {
479		zone = glob->zones[i];
480		if (single_zone && zone != single_zone)
481			continue;
482
483		limit = (capable(CAP_SYS_ADMIN)) ?
484			zone->emer_mem : zone->max_mem;
485
486		if (zone->used_mem > limit)
487			goto out_unlock;
488	}
489
490	if (reserve) {
491		for (i = 0; i < glob->num_zones; ++i) {
492			zone = glob->zones[i];
493			if (single_zone && zone != single_zone)
494				continue;
495			zone->used_mem += amount;
496		}
497	}
498
499	ret = 0;
500out_unlock:
501	spin_unlock(&glob->lock);
502	ttm_check_swapping(glob);
503
504	return ret;
505}
506
507
508static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,
509				     struct ttm_mem_zone *single_zone,
510				     uint64_t memory,
511				     bool no_wait, bool interruptible)
512{
513	int count = TTM_MEMORY_ALLOC_RETRIES;
514
515	while (unlikely(ttm_mem_global_reserve(glob,
516					       single_zone,
517					       memory, true)
518			!= 0)) {
519		if (no_wait)
520			return -ENOMEM;
521		if (unlikely(count-- == 0))
522			return -ENOMEM;
523		ttm_shrink(glob, false, memory + (memory >> 2) + 16);
524	}
525
526	return 0;
527}
528
529int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
530			 bool no_wait, bool interruptible)
531{
532	/**
533	 * Normal allocations of kernel memory are registered in
534	 * all zones.
535	 */
536
537	return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait,
538					 interruptible);
539}
540EXPORT_SYMBOL(ttm_mem_global_alloc);
541
542int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
543			      struct page *page,
544			      bool no_wait, bool interruptible)
545{
546
547	struct ttm_mem_zone *zone = NULL;
548
549	/**
550	 * Page allocations may be registed in a single zone
551	 * only if highmem or !dma32.
552	 */
553
554#ifdef CONFIG_HIGHMEM
555	if (PageHighMem(page) && glob->zone_highmem != NULL)
556		zone = glob->zone_highmem;
557#else
558	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
559		zone = glob->zone_kernel;
560#endif
561	return ttm_mem_global_alloc_zone(glob, zone, PAGE_SIZE, no_wait,
562					 interruptible);
563}
564
565void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page)
566{
567	struct ttm_mem_zone *zone = NULL;
568
569#ifdef CONFIG_HIGHMEM
570	if (PageHighMem(page) && glob->zone_highmem != NULL)
571		zone = glob->zone_highmem;
572#else
573	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
574		zone = glob->zone_kernel;
575#endif
576	ttm_mem_global_free_zone(glob, zone, PAGE_SIZE);
577}
578
579
580size_t ttm_round_pot(size_t size)
581{
582	if ((size & (size - 1)) == 0)
583		return size;
584	else if (size > PAGE_SIZE)
585		return PAGE_ALIGN(size);
586	else {
587		size_t tmp_size = 4;
588
589		while (tmp_size < size)
590			tmp_size <<= 1;
591
592		return tmp_size;
593	}
594	return 0;
595}
596EXPORT_SYMBOL(ttm_round_pot);
597