1/* 2 * drivers/staging/android/ion/ion_heap.c 3 * 4 * Copyright (C) 2011 Google, Inc. 5 * 6 * This software is licensed under the terms of the GNU General Public 7 * License version 2, as published by the Free Software Foundation, and 8 * may be copied, distributed, and modified under those terms. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 */ 16 17#include <linux/err.h> 18#include <linux/freezer.h> 19#include <linux/kthread.h> 20#include <linux/mm.h> 21#include <linux/rtmutex.h> 22#include <linux/sched.h> 23#include <linux/scatterlist.h> 24#include <linux/vmalloc.h> 25#include "ion.h" 26#include "ion_priv.h" 27 28void *ion_heap_map_kernel(struct ion_heap *heap, 29 struct ion_buffer *buffer) 30{ 31 struct scatterlist *sg; 32 int i, j; 33 void *vaddr; 34 pgprot_t pgprot; 35 struct sg_table *table = buffer->sg_table; 36 int npages = PAGE_ALIGN(buffer->size) / PAGE_SIZE; 37 struct page **pages = vmalloc(sizeof(struct page *) * npages); 38 struct page **tmp = pages; 39 40 if (!pages) 41 return NULL; 42 43 if (buffer->flags & ION_FLAG_CACHED) 44 pgprot = PAGE_KERNEL; 45 else 46 pgprot = pgprot_writecombine(PAGE_KERNEL); 47 48 for_each_sg(table->sgl, sg, table->nents, i) { 49 int npages_this_entry = PAGE_ALIGN(sg->length) / PAGE_SIZE; 50 struct page *page = sg_page(sg); 51 52 BUG_ON(i >= npages); 53 for (j = 0; j < npages_this_entry; j++) 54 *(tmp++) = page++; 55 } 56 vaddr = vmap(pages, npages, VM_MAP, pgprot); 57 vfree(pages); 58 59 if (vaddr == NULL) 60 return ERR_PTR(-ENOMEM); 61 62 return vaddr; 63} 64 65void ion_heap_unmap_kernel(struct ion_heap *heap, 66 struct ion_buffer *buffer) 67{ 68 vunmap(buffer->vaddr); 69} 70 71int ion_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer, 72 struct vm_area_struct *vma) 73{ 74 struct sg_table *table = buffer->sg_table; 75 unsigned long addr = vma->vm_start; 76 unsigned long offset = vma->vm_pgoff * PAGE_SIZE; 77 struct scatterlist *sg; 78 int i; 79 int ret; 80 81 for_each_sg(table->sgl, sg, table->nents, i) { 82 struct page *page = sg_page(sg); 83 unsigned long remainder = vma->vm_end - addr; 84 unsigned long len = sg->length; 85 86 if (offset >= sg->length) { 87 offset -= sg->length; 88 continue; 89 } else if (offset) { 90 page += offset / PAGE_SIZE; 91 len = sg->length - offset; 92 offset = 0; 93 } 94 len = min(len, remainder); 95 ret = remap_pfn_range(vma, addr, page_to_pfn(page), len, 96 vma->vm_page_prot); 97 if (ret) 98 return ret; 99 addr += len; 100 if (addr >= vma->vm_end) 101 return 0; 102 } 103 return 0; 104} 105 106static int ion_heap_clear_pages(struct page **pages, int num, pgprot_t pgprot) 107{ 108 void *addr = vm_map_ram(pages, num, -1, pgprot); 109 110 if (!addr) 111 return -ENOMEM; 112 memset(addr, 0, PAGE_SIZE * num); 113 vm_unmap_ram(addr, num); 114 115 return 0; 116} 117 118static int ion_heap_sglist_zero(struct scatterlist *sgl, unsigned int nents, 119 pgprot_t pgprot) 120{ 121 int p = 0; 122 int ret = 0; 123 struct sg_page_iter piter; 124 struct page *pages[32]; 125 126 for_each_sg_page(sgl, &piter, nents, 0) { 127 pages[p++] = sg_page_iter_page(&piter); 128 if (p == ARRAY_SIZE(pages)) { 129 ret = ion_heap_clear_pages(pages, p, pgprot); 130 if (ret) 131 return ret; 132 p = 0; 133 } 134 } 135 if (p) 136 ret = ion_heap_clear_pages(pages, p, pgprot); 137 138 return ret; 139} 140 141int ion_heap_buffer_zero(struct ion_buffer *buffer) 142{ 143 struct sg_table *table = buffer->sg_table; 144 pgprot_t pgprot; 145 146 if (buffer->flags & ION_FLAG_CACHED) 147 pgprot = PAGE_KERNEL; 148 else 149 pgprot = pgprot_writecombine(PAGE_KERNEL); 150 151 return ion_heap_sglist_zero(table->sgl, table->nents, pgprot); 152} 153 154int ion_heap_pages_zero(struct page *page, size_t size, pgprot_t pgprot) 155{ 156 struct scatterlist sg; 157 158 sg_init_table(&sg, 1); 159 sg_set_page(&sg, page, size, 0); 160 return ion_heap_sglist_zero(&sg, 1, pgprot); 161} 162 163void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer) 164{ 165 spin_lock(&heap->free_lock); 166 list_add(&buffer->list, &heap->free_list); 167 heap->free_list_size += buffer->size; 168 spin_unlock(&heap->free_lock); 169 wake_up(&heap->waitqueue); 170} 171 172size_t ion_heap_freelist_size(struct ion_heap *heap) 173{ 174 size_t size; 175 176 spin_lock(&heap->free_lock); 177 size = heap->free_list_size; 178 spin_unlock(&heap->free_lock); 179 180 return size; 181} 182 183static size_t _ion_heap_freelist_drain(struct ion_heap *heap, size_t size, 184 bool skip_pools) 185{ 186 struct ion_buffer *buffer; 187 size_t total_drained = 0; 188 189 if (ion_heap_freelist_size(heap) == 0) 190 return 0; 191 192 spin_lock(&heap->free_lock); 193 if (size == 0) 194 size = heap->free_list_size; 195 196 while (!list_empty(&heap->free_list)) { 197 if (total_drained >= size) 198 break; 199 buffer = list_first_entry(&heap->free_list, struct ion_buffer, 200 list); 201 list_del(&buffer->list); 202 heap->free_list_size -= buffer->size; 203 if (skip_pools) 204 buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE; 205 total_drained += buffer->size; 206 spin_unlock(&heap->free_lock); 207 ion_buffer_destroy(buffer); 208 spin_lock(&heap->free_lock); 209 } 210 spin_unlock(&heap->free_lock); 211 212 return total_drained; 213} 214 215size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size) 216{ 217 return _ion_heap_freelist_drain(heap, size, false); 218} 219 220size_t ion_heap_freelist_shrink(struct ion_heap *heap, size_t size) 221{ 222 return _ion_heap_freelist_drain(heap, size, true); 223} 224 225static int ion_heap_deferred_free(void *data) 226{ 227 struct ion_heap *heap = data; 228 229 while (true) { 230 struct ion_buffer *buffer; 231 232 wait_event_freezable(heap->waitqueue, 233 ion_heap_freelist_size(heap) > 0); 234 235 spin_lock(&heap->free_lock); 236 if (list_empty(&heap->free_list)) { 237 spin_unlock(&heap->free_lock); 238 continue; 239 } 240 buffer = list_first_entry(&heap->free_list, struct ion_buffer, 241 list); 242 list_del(&buffer->list); 243 heap->free_list_size -= buffer->size; 244 spin_unlock(&heap->free_lock); 245 ion_buffer_destroy(buffer); 246 } 247 248 return 0; 249} 250 251int ion_heap_init_deferred_free(struct ion_heap *heap) 252{ 253 struct sched_param param = { .sched_priority = 0 }; 254 255 INIT_LIST_HEAD(&heap->free_list); 256 heap->free_list_size = 0; 257 spin_lock_init(&heap->free_lock); 258 init_waitqueue_head(&heap->waitqueue); 259 heap->task = kthread_run(ion_heap_deferred_free, heap, 260 "%s", heap->name); 261 if (IS_ERR(heap->task)) { 262 pr_err("%s: creating thread for deferred free failed\n", 263 __func__); 264 return PTR_ERR_OR_ZERO(heap->task); 265 } 266 sched_setscheduler(heap->task, SCHED_IDLE, ¶m); 267 return 0; 268} 269 270static unsigned long ion_heap_shrink_count(struct shrinker *shrinker, 271 struct shrink_control *sc) 272{ 273 struct ion_heap *heap = container_of(shrinker, struct ion_heap, 274 shrinker); 275 int total = 0; 276 277 total = ion_heap_freelist_size(heap) / PAGE_SIZE; 278 if (heap->ops->shrink) 279 total += heap->ops->shrink(heap, sc->gfp_mask, 0); 280 return total; 281} 282 283static unsigned long ion_heap_shrink_scan(struct shrinker *shrinker, 284 struct shrink_control *sc) 285{ 286 struct ion_heap *heap = container_of(shrinker, struct ion_heap, 287 shrinker); 288 int freed = 0; 289 int to_scan = sc->nr_to_scan; 290 291 if (to_scan == 0) 292 return 0; 293 294 /* 295 * shrink the free list first, no point in zeroing the memory if we're 296 * just going to reclaim it. Also, skip any possible page pooling. 297 */ 298 if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) 299 freed = ion_heap_freelist_shrink(heap, to_scan * PAGE_SIZE) / 300 PAGE_SIZE; 301 302 to_scan -= freed; 303 if (to_scan <= 0) 304 return freed; 305 306 if (heap->ops->shrink) 307 freed += heap->ops->shrink(heap, sc->gfp_mask, to_scan); 308 return freed; 309} 310 311void ion_heap_init_shrinker(struct ion_heap *heap) 312{ 313 heap->shrinker.count_objects = ion_heap_shrink_count; 314 heap->shrinker.scan_objects = ion_heap_shrink_scan; 315 heap->shrinker.seeks = DEFAULT_SEEKS; 316 heap->shrinker.batch = 0; 317 register_shrinker(&heap->shrinker); 318} 319 320struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data) 321{ 322 struct ion_heap *heap = NULL; 323 324 switch (heap_data->type) { 325 case ION_HEAP_TYPE_SYSTEM_CONTIG: 326 heap = ion_system_contig_heap_create(heap_data); 327 break; 328 case ION_HEAP_TYPE_SYSTEM: 329 heap = ion_system_heap_create(heap_data); 330 break; 331 case ION_HEAP_TYPE_CARVEOUT: 332 heap = ion_carveout_heap_create(heap_data); 333 break; 334 case ION_HEAP_TYPE_CHUNK: 335 heap = ion_chunk_heap_create(heap_data); 336 break; 337 case ION_HEAP_TYPE_DMA: 338 heap = ion_cma_heap_create(heap_data); 339 break; 340 default: 341 pr_err("%s: Invalid heap type %d\n", __func__, 342 heap_data->type); 343 return ERR_PTR(-EINVAL); 344 } 345 346 if (IS_ERR_OR_NULL(heap)) { 347 pr_err("%s: error creating heap %s type %d base %lu size %zu\n", 348 __func__, heap_data->name, heap_data->type, 349 heap_data->base, heap_data->size); 350 return ERR_PTR(-EINVAL); 351 } 352 353 heap->name = heap_data->name; 354 heap->id = heap_data->id; 355 return heap; 356} 357 358void ion_heap_destroy(struct ion_heap *heap) 359{ 360 if (!heap) 361 return; 362 363 switch (heap->type) { 364 case ION_HEAP_TYPE_SYSTEM_CONTIG: 365 ion_system_contig_heap_destroy(heap); 366 break; 367 case ION_HEAP_TYPE_SYSTEM: 368 ion_system_heap_destroy(heap); 369 break; 370 case ION_HEAP_TYPE_CARVEOUT: 371 ion_carveout_heap_destroy(heap); 372 break; 373 case ION_HEAP_TYPE_CHUNK: 374 ion_chunk_heap_destroy(heap); 375 break; 376 case ION_HEAP_TYPE_DMA: 377 ion_cma_heap_destroy(heap); 378 break; 379 default: 380 pr_err("%s: Invalid heap type %d\n", __func__, 381 heap->type); 382 } 383} 384