1/************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27/* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31#include "ttm/ttm_module.h" 32#include "ttm/ttm_bo_driver.h" 33#include "ttm/ttm_placement.h" 34#include <linux/jiffies.h> 35#include <linux/slab.h> 36#include <linux/sched.h> 37#include <linux/mm.h> 38#include <linux/file.h> 39#include <linux/module.h> 40#include <linux/atomic.h> 41 42#define TTM_ASSERT_LOCKED(param) 43#define TTM_DEBUG(fmt, arg...) 44#define TTM_BO_HASH_ORDER 13 45 46static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); 47static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 48static void ttm_bo_global_kobj_release(struct kobject *kobj); 49 50static struct attribute ttm_bo_count = { 51 .name = "bo_count", 52 .mode = S_IRUGO 53}; 54 55static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) 56{ 57 int i; 58 59 for (i = 0; i <= TTM_PL_PRIV5; i++) 60 if (flags & (1 << i)) { 61 *mem_type = i; 62 return 0; 63 } 64 return -EINVAL; 65} 66 67static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 68{ 69 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 70 71 printk(KERN_ERR TTM_PFX " has_type: %d\n", man->has_type); 72 printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type); 73 printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags); 74 printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset); 75 printk(KERN_ERR TTM_PFX " size: %llu\n", man->size); 76 printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n", 77 man->available_caching); 78 printk(KERN_ERR TTM_PFX " default_caching: 0x%08X\n", 79 man->default_caching); 80 if (mem_type != TTM_PL_SYSTEM) 81 (*man->func->debug)(man, TTM_PFX); 82} 83 84static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 85 struct ttm_placement *placement) 86{ 87 int i, ret, mem_type; 88 89 printk(KERN_ERR TTM_PFX "No space for %p (%lu pages, %luK, %luM)\n", 90 bo, bo->mem.num_pages, bo->mem.size >> 10, 91 bo->mem.size >> 20); 92 for (i = 0; i < placement->num_placement; i++) { 93 ret = ttm_mem_type_from_flags(placement->placement[i], 94 &mem_type); 95 if (ret) 96 return; 97 printk(KERN_ERR TTM_PFX " placement[%d]=0x%08X (%d)\n", 98 i, placement->placement[i], mem_type); 99 ttm_mem_type_debug(bo->bdev, mem_type); 100 } 101} 102 103static ssize_t ttm_bo_global_show(struct kobject *kobj, 104 struct attribute *attr, 105 char *buffer) 106{ 107 struct ttm_bo_global *glob = 108 container_of(kobj, struct ttm_bo_global, kobj); 109 110 return snprintf(buffer, PAGE_SIZE, "%lu\n", 111 (unsigned long) atomic_read(&glob->bo_count)); 112} 113 114static struct attribute *ttm_bo_global_attrs[] = { 115 &ttm_bo_count, 116 NULL 117}; 118 119static const struct sysfs_ops ttm_bo_global_ops = { 120 .show = &ttm_bo_global_show 121}; 122 123static struct kobj_type ttm_bo_glob_kobj_type = { 124 .release = &ttm_bo_global_kobj_release, 125 .sysfs_ops = &ttm_bo_global_ops, 126 .default_attrs = ttm_bo_global_attrs 127}; 128 129 130static inline uint32_t ttm_bo_type_flags(unsigned type) 131{ 132 return 1 << (type); 133} 134 135static void ttm_bo_release_list(struct kref *list_kref) 136{ 137 struct ttm_buffer_object *bo = 138 container_of(list_kref, struct ttm_buffer_object, list_kref); 139 struct ttm_bo_device *bdev = bo->bdev; 140 size_t acc_size = bo->acc_size; 141 142 BUG_ON(atomic_read(&bo->list_kref.refcount)); 143 BUG_ON(atomic_read(&bo->kref.refcount)); 144 BUG_ON(atomic_read(&bo->cpu_writers)); 145 BUG_ON(bo->sync_obj != NULL); 146 BUG_ON(bo->mem.mm_node != NULL); 147 BUG_ON(!list_empty(&bo->lru)); 148 BUG_ON(!list_empty(&bo->ddestroy)); 149 150 if (bo->ttm) 151 ttm_tt_destroy(bo->ttm); 152 atomic_dec(&bo->glob->bo_count); 153 if (bo->destroy) 154 bo->destroy(bo); 155 else { 156 kfree(bo); 157 } 158 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 159} 160 161int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) 162{ 163 if (interruptible) { 164 return wait_event_interruptible(bo->event_queue, 165 atomic_read(&bo->reserved) == 0); 166 } else { 167 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0); 168 return 0; 169 } 170} 171EXPORT_SYMBOL(ttm_bo_wait_unreserved); 172 173void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 174{ 175 struct ttm_bo_device *bdev = bo->bdev; 176 struct ttm_mem_type_manager *man; 177 178 BUG_ON(!atomic_read(&bo->reserved)); 179 180 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 181 182 BUG_ON(!list_empty(&bo->lru)); 183 184 man = &bdev->man[bo->mem.mem_type]; 185 list_add_tail(&bo->lru, &man->lru); 186 kref_get(&bo->list_kref); 187 188 if (bo->ttm != NULL) { 189 list_add_tail(&bo->swap, &bo->glob->swap_lru); 190 kref_get(&bo->list_kref); 191 } 192 } 193} 194 195int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 196{ 197 int put_count = 0; 198 199 if (!list_empty(&bo->swap)) { 200 list_del_init(&bo->swap); 201 ++put_count; 202 } 203 if (!list_empty(&bo->lru)) { 204 list_del_init(&bo->lru); 205 ++put_count; 206 } 207 208 /* 209 * TODO: Add a driver hook to delete from 210 * driver-specific LRU's here. 211 */ 212 213 return put_count; 214} 215 216int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, 217 bool interruptible, 218 bool no_wait, bool use_sequence, uint32_t sequence) 219{ 220 struct ttm_bo_global *glob = bo->glob; 221 int ret; 222 223 while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) { 224 /** 225 * Deadlock avoidance for multi-bo reserving. 226 */ 227 if (use_sequence && bo->seq_valid) { 228 /** 229 * We've already reserved this one. 230 */ 231 if (unlikely(sequence == bo->val_seq)) 232 return -EDEADLK; 233 /** 234 * Already reserved by a thread that will not back 235 * off for us. We need to back off. 236 */ 237 if (unlikely(sequence - bo->val_seq < (1 << 31))) 238 return -EAGAIN; 239 } 240 241 if (no_wait) 242 return -EBUSY; 243 244 spin_unlock(&glob->lru_lock); 245 ret = ttm_bo_wait_unreserved(bo, interruptible); 246 spin_lock(&glob->lru_lock); 247 248 if (unlikely(ret)) 249 return ret; 250 } 251 252 if (use_sequence) { 253 /** 254 * Wake up waiters that may need to recheck for deadlock, 255 * if we decreased the sequence number. 256 */ 257 if (unlikely((bo->val_seq - sequence < (1 << 31)) 258 || !bo->seq_valid)) 259 wake_up_all(&bo->event_queue); 260 261 bo->val_seq = sequence; 262 bo->seq_valid = true; 263 } else { 264 bo->seq_valid = false; 265 } 266 267 return 0; 268} 269EXPORT_SYMBOL(ttm_bo_reserve); 270 271static void ttm_bo_ref_bug(struct kref *list_kref) 272{ 273 BUG(); 274} 275 276void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, 277 bool never_free) 278{ 279 kref_sub(&bo->list_kref, count, 280 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); 281} 282 283int ttm_bo_reserve(struct ttm_buffer_object *bo, 284 bool interruptible, 285 bool no_wait, bool use_sequence, uint32_t sequence) 286{ 287 struct ttm_bo_global *glob = bo->glob; 288 int put_count = 0; 289 int ret; 290 291 spin_lock(&glob->lru_lock); 292 ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, 293 sequence); 294 if (likely(ret == 0)) 295 put_count = ttm_bo_del_from_lru(bo); 296 spin_unlock(&glob->lru_lock); 297 298 ttm_bo_list_ref_sub(bo, put_count, true); 299 300 return ret; 301} 302 303void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) 304{ 305 ttm_bo_add_to_lru(bo); 306 atomic_set(&bo->reserved, 0); 307 wake_up_all(&bo->event_queue); 308} 309 310void ttm_bo_unreserve(struct ttm_buffer_object *bo) 311{ 312 struct ttm_bo_global *glob = bo->glob; 313 314 spin_lock(&glob->lru_lock); 315 ttm_bo_unreserve_locked(bo); 316 spin_unlock(&glob->lru_lock); 317} 318EXPORT_SYMBOL(ttm_bo_unreserve); 319 320/* 321 * Call bo->mutex locked. 322 */ 323static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 324{ 325 struct ttm_bo_device *bdev = bo->bdev; 326 struct ttm_bo_global *glob = bo->glob; 327 int ret = 0; 328 uint32_t page_flags = 0; 329 330 TTM_ASSERT_LOCKED(&bo->mutex); 331 bo->ttm = NULL; 332 333 if (bdev->need_dma32) 334 page_flags |= TTM_PAGE_FLAG_DMA32; 335 336 switch (bo->type) { 337 case ttm_bo_type_device: 338 if (zero_alloc) 339 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 340 case ttm_bo_type_kernel: 341 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 342 page_flags, glob->dummy_read_page); 343 if (unlikely(bo->ttm == NULL)) 344 ret = -ENOMEM; 345 break; 346 default: 347 printk(KERN_ERR TTM_PFX "Illegal buffer object type\n"); 348 ret = -EINVAL; 349 break; 350 } 351 352 return ret; 353} 354 355static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 356 struct ttm_mem_reg *mem, 357 bool evict, bool interruptible, 358 bool no_wait_reserve, bool no_wait_gpu) 359{ 360 struct ttm_bo_device *bdev = bo->bdev; 361 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 362 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 363 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 364 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 365 int ret = 0; 366 367 if (old_is_pci || new_is_pci || 368 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 369 ret = ttm_mem_io_lock(old_man, true); 370 if (unlikely(ret != 0)) 371 goto out_err; 372 ttm_bo_unmap_virtual_locked(bo); 373 ttm_mem_io_unlock(old_man); 374 } 375 376 /* 377 * Create and bind a ttm if required. 378 */ 379 380 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 381 if (bo->ttm == NULL) { 382 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 383 ret = ttm_bo_add_ttm(bo, zero); 384 if (ret) 385 goto out_err; 386 } 387 388 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 389 if (ret) 390 goto out_err; 391 392 if (mem->mem_type != TTM_PL_SYSTEM) { 393 ret = ttm_tt_bind(bo->ttm, mem); 394 if (ret) 395 goto out_err; 396 } 397 398 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 399 if (bdev->driver->move_notify) 400 bdev->driver->move_notify(bo, mem); 401 bo->mem = *mem; 402 mem->mm_node = NULL; 403 goto moved; 404 } 405 } 406 407 if (bdev->driver->move_notify) 408 bdev->driver->move_notify(bo, mem); 409 410 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 411 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 412 ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem); 413 else if (bdev->driver->move) 414 ret = bdev->driver->move(bo, evict, interruptible, 415 no_wait_reserve, no_wait_gpu, mem); 416 else 417 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem); 418 419 if (ret) { 420 if (bdev->driver->move_notify) { 421 struct ttm_mem_reg tmp_mem = *mem; 422 *mem = bo->mem; 423 bo->mem = tmp_mem; 424 bdev->driver->move_notify(bo, mem); 425 bo->mem = *mem; 426 } 427 428 goto out_err; 429 } 430 431moved: 432 if (bo->evicted) { 433 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 434 if (ret) 435 printk(KERN_ERR TTM_PFX "Can not flush read caches\n"); 436 bo->evicted = false; 437 } 438 439 if (bo->mem.mm_node) { 440 bo->offset = (bo->mem.start << PAGE_SHIFT) + 441 bdev->man[bo->mem.mem_type].gpu_offset; 442 bo->cur_placement = bo->mem.placement; 443 } else 444 bo->offset = 0; 445 446 return 0; 447 448out_err: 449 new_man = &bdev->man[bo->mem.mem_type]; 450 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 451 ttm_tt_unbind(bo->ttm); 452 ttm_tt_destroy(bo->ttm); 453 bo->ttm = NULL; 454 } 455 456 return ret; 457} 458 459/** 460 * Call bo::reserved. 461 * Will release GPU memory type usage on destruction. 462 * This is the place to put in driver specific hooks to release 463 * driver private resources. 464 * Will release the bo::reserved lock. 465 */ 466 467static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 468{ 469 if (bo->bdev->driver->move_notify) 470 bo->bdev->driver->move_notify(bo, NULL); 471 472 if (bo->ttm) { 473 ttm_tt_unbind(bo->ttm); 474 ttm_tt_destroy(bo->ttm); 475 bo->ttm = NULL; 476 } 477 ttm_bo_mem_put(bo, &bo->mem); 478 479 atomic_set(&bo->reserved, 0); 480 481 /* 482 * Make processes trying to reserve really pick it up. 483 */ 484 smp_mb__after_atomic_dec(); 485 wake_up_all(&bo->event_queue); 486} 487 488static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 489{ 490 struct ttm_bo_device *bdev = bo->bdev; 491 struct ttm_bo_global *glob = bo->glob; 492 struct ttm_bo_driver *driver; 493 void *sync_obj = NULL; 494 void *sync_obj_arg; 495 int put_count; 496 int ret; 497 498 spin_lock(&bdev->fence_lock); 499 (void) ttm_bo_wait(bo, false, false, true); 500 if (!bo->sync_obj) { 501 502 spin_lock(&glob->lru_lock); 503 504 /** 505 * Lock inversion between bo:reserve and bdev::fence_lock here, 506 * but that's OK, since we're only trylocking. 507 */ 508 509 ret = ttm_bo_reserve_locked(bo, false, true, false, 0); 510 511 if (unlikely(ret == -EBUSY)) 512 goto queue; 513 514 spin_unlock(&bdev->fence_lock); 515 put_count = ttm_bo_del_from_lru(bo); 516 517 spin_unlock(&glob->lru_lock); 518 ttm_bo_cleanup_memtype_use(bo); 519 520 ttm_bo_list_ref_sub(bo, put_count, true); 521 522 return; 523 } else { 524 spin_lock(&glob->lru_lock); 525 } 526queue: 527 driver = bdev->driver; 528 if (bo->sync_obj) 529 sync_obj = driver->sync_obj_ref(bo->sync_obj); 530 sync_obj_arg = bo->sync_obj_arg; 531 532 kref_get(&bo->list_kref); 533 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 534 spin_unlock(&glob->lru_lock); 535 spin_unlock(&bdev->fence_lock); 536 537 if (sync_obj) { 538 driver->sync_obj_flush(sync_obj, sync_obj_arg); 539 driver->sync_obj_unref(&sync_obj); 540 } 541 schedule_delayed_work(&bdev->wq, 542 ((HZ / 100) < 1) ? 1 : HZ / 100); 543} 544 545/** 546 * function ttm_bo_cleanup_refs 547 * If bo idle, remove from delayed- and lru lists, and unref. 548 * If not idle, do nothing. 549 * 550 * @interruptible Any sleeps should occur interruptibly. 551 * @no_wait_reserve Never wait for reserve. Return -EBUSY instead. 552 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 553 */ 554 555static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 556 bool interruptible, 557 bool no_wait_reserve, 558 bool no_wait_gpu) 559{ 560 struct ttm_bo_device *bdev = bo->bdev; 561 struct ttm_bo_global *glob = bo->glob; 562 int put_count; 563 int ret = 0; 564 565retry: 566 spin_lock(&bdev->fence_lock); 567 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 568 spin_unlock(&bdev->fence_lock); 569 570 if (unlikely(ret != 0)) 571 return ret; 572 573 spin_lock(&glob->lru_lock); 574 575 if (unlikely(list_empty(&bo->ddestroy))) { 576 spin_unlock(&glob->lru_lock); 577 return 0; 578 } 579 580 ret = ttm_bo_reserve_locked(bo, interruptible, 581 no_wait_reserve, false, 0); 582 583 if (unlikely(ret != 0)) { 584 spin_unlock(&glob->lru_lock); 585 return ret; 586 } 587 588 /** 589 * We can re-check for sync object without taking 590 * the bo::lock since setting the sync object requires 591 * also bo::reserved. A busy object at this point may 592 * be caused by another thread recently starting an accelerated 593 * eviction. 594 */ 595 596 if (unlikely(bo->sync_obj)) { 597 atomic_set(&bo->reserved, 0); 598 wake_up_all(&bo->event_queue); 599 spin_unlock(&glob->lru_lock); 600 goto retry; 601 } 602 603 put_count = ttm_bo_del_from_lru(bo); 604 list_del_init(&bo->ddestroy); 605 ++put_count; 606 607 spin_unlock(&glob->lru_lock); 608 ttm_bo_cleanup_memtype_use(bo); 609 610 ttm_bo_list_ref_sub(bo, put_count, true); 611 612 return 0; 613} 614 615/** 616 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 617 * encountered buffers. 618 */ 619 620static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 621{ 622 struct ttm_bo_global *glob = bdev->glob; 623 struct ttm_buffer_object *entry = NULL; 624 int ret = 0; 625 626 spin_lock(&glob->lru_lock); 627 if (list_empty(&bdev->ddestroy)) 628 goto out_unlock; 629 630 entry = list_first_entry(&bdev->ddestroy, 631 struct ttm_buffer_object, ddestroy); 632 kref_get(&entry->list_kref); 633 634 for (;;) { 635 struct ttm_buffer_object *nentry = NULL; 636 637 if (entry->ddestroy.next != &bdev->ddestroy) { 638 nentry = list_first_entry(&entry->ddestroy, 639 struct ttm_buffer_object, ddestroy); 640 kref_get(&nentry->list_kref); 641 } 642 643 spin_unlock(&glob->lru_lock); 644 ret = ttm_bo_cleanup_refs(entry, false, !remove_all, 645 !remove_all); 646 kref_put(&entry->list_kref, ttm_bo_release_list); 647 entry = nentry; 648 649 if (ret || !entry) 650 goto out; 651 652 spin_lock(&glob->lru_lock); 653 if (list_empty(&entry->ddestroy)) 654 break; 655 } 656 657out_unlock: 658 spin_unlock(&glob->lru_lock); 659out: 660 if (entry) 661 kref_put(&entry->list_kref, ttm_bo_release_list); 662 return ret; 663} 664 665static void ttm_bo_delayed_workqueue(struct work_struct *work) 666{ 667 struct ttm_bo_device *bdev = 668 container_of(work, struct ttm_bo_device, wq.work); 669 670 if (ttm_bo_delayed_delete(bdev, false)) { 671 schedule_delayed_work(&bdev->wq, 672 ((HZ / 100) < 1) ? 1 : HZ / 100); 673 } 674} 675 676static void ttm_bo_release(struct kref *kref) 677{ 678 struct ttm_buffer_object *bo = 679 container_of(kref, struct ttm_buffer_object, kref); 680 struct ttm_bo_device *bdev = bo->bdev; 681 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 682 683 if (likely(bo->vm_node != NULL)) { 684 rb_erase(&bo->vm_rb, &bdev->addr_space_rb); 685 drm_mm_put_block(bo->vm_node); 686 bo->vm_node = NULL; 687 } 688 write_unlock(&bdev->vm_lock); 689 ttm_mem_io_lock(man, false); 690 ttm_mem_io_free_vm(bo); 691 ttm_mem_io_unlock(man); 692 ttm_bo_cleanup_refs_or_queue(bo); 693 kref_put(&bo->list_kref, ttm_bo_release_list); 694 write_lock(&bdev->vm_lock); 695} 696 697void ttm_bo_unref(struct ttm_buffer_object **p_bo) 698{ 699 struct ttm_buffer_object *bo = *p_bo; 700 struct ttm_bo_device *bdev = bo->bdev; 701 702 *p_bo = NULL; 703 write_lock(&bdev->vm_lock); 704 kref_put(&bo->kref, ttm_bo_release); 705 write_unlock(&bdev->vm_lock); 706} 707EXPORT_SYMBOL(ttm_bo_unref); 708 709int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 710{ 711 return cancel_delayed_work_sync(&bdev->wq); 712} 713EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 714 715void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 716{ 717 if (resched) 718 schedule_delayed_work(&bdev->wq, 719 ((HZ / 100) < 1) ? 1 : HZ / 100); 720} 721EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 722 723static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 724 bool no_wait_reserve, bool no_wait_gpu) 725{ 726 struct ttm_bo_device *bdev = bo->bdev; 727 struct ttm_mem_reg evict_mem; 728 struct ttm_placement placement; 729 int ret = 0; 730 731 spin_lock(&bdev->fence_lock); 732 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 733 spin_unlock(&bdev->fence_lock); 734 735 if (unlikely(ret != 0)) { 736 if (ret != -ERESTARTSYS) { 737 printk(KERN_ERR TTM_PFX 738 "Failed to expire sync object before " 739 "buffer eviction.\n"); 740 } 741 goto out; 742 } 743 744 BUG_ON(!atomic_read(&bo->reserved)); 745 746 evict_mem = bo->mem; 747 evict_mem.mm_node = NULL; 748 evict_mem.bus.io_reserved_vm = false; 749 evict_mem.bus.io_reserved_count = 0; 750 751 placement.fpfn = 0; 752 placement.lpfn = 0; 753 placement.num_placement = 0; 754 placement.num_busy_placement = 0; 755 bdev->driver->evict_flags(bo, &placement); 756 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 757 no_wait_reserve, no_wait_gpu); 758 if (ret) { 759 if (ret != -ERESTARTSYS) { 760 printk(KERN_ERR TTM_PFX 761 "Failed to find memory space for " 762 "buffer 0x%p eviction.\n", bo); 763 ttm_bo_mem_space_debug(bo, &placement); 764 } 765 goto out; 766 } 767 768 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 769 no_wait_reserve, no_wait_gpu); 770 if (ret) { 771 if (ret != -ERESTARTSYS) 772 printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); 773 ttm_bo_mem_put(bo, &evict_mem); 774 goto out; 775 } 776 bo->evicted = true; 777out: 778 return ret; 779} 780 781static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 782 uint32_t mem_type, 783 bool interruptible, bool no_wait_reserve, 784 bool no_wait_gpu) 785{ 786 struct ttm_bo_global *glob = bdev->glob; 787 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 788 struct ttm_buffer_object *bo; 789 int ret, put_count = 0; 790 791retry: 792 spin_lock(&glob->lru_lock); 793 if (list_empty(&man->lru)) { 794 spin_unlock(&glob->lru_lock); 795 return -EBUSY; 796 } 797 798 bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); 799 kref_get(&bo->list_kref); 800 801 if (!list_empty(&bo->ddestroy)) { 802 spin_unlock(&glob->lru_lock); 803 ret = ttm_bo_cleanup_refs(bo, interruptible, 804 no_wait_reserve, no_wait_gpu); 805 kref_put(&bo->list_kref, ttm_bo_release_list); 806 807 if (likely(ret == 0 || ret == -ERESTARTSYS)) 808 return ret; 809 810 goto retry; 811 } 812 813 ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); 814 815 if (unlikely(ret == -EBUSY)) { 816 spin_unlock(&glob->lru_lock); 817 if (likely(!no_wait_gpu)) 818 ret = ttm_bo_wait_unreserved(bo, interruptible); 819 820 kref_put(&bo->list_kref, ttm_bo_release_list); 821 822 /** 823 * We *need* to retry after releasing the lru lock. 824 */ 825 826 if (unlikely(ret != 0)) 827 return ret; 828 goto retry; 829 } 830 831 put_count = ttm_bo_del_from_lru(bo); 832 spin_unlock(&glob->lru_lock); 833 834 BUG_ON(ret != 0); 835 836 ttm_bo_list_ref_sub(bo, put_count, true); 837 838 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); 839 ttm_bo_unreserve(bo); 840 841 kref_put(&bo->list_kref, ttm_bo_release_list); 842 return ret; 843} 844 845void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 846{ 847 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 848 849 if (mem->mm_node) 850 (*man->func->put_node)(man, mem); 851} 852EXPORT_SYMBOL(ttm_bo_mem_put); 853 854/** 855 * Repeatedly evict memory from the LRU for @mem_type until we create enough 856 * space, or we've evicted everything and there isn't enough space. 857 */ 858static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 859 uint32_t mem_type, 860 struct ttm_placement *placement, 861 struct ttm_mem_reg *mem, 862 bool interruptible, 863 bool no_wait_reserve, 864 bool no_wait_gpu) 865{ 866 struct ttm_bo_device *bdev = bo->bdev; 867 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 868 int ret; 869 870 do { 871 ret = (*man->func->get_node)(man, bo, placement, mem); 872 if (unlikely(ret != 0)) 873 return ret; 874 if (mem->mm_node) 875 break; 876 ret = ttm_mem_evict_first(bdev, mem_type, interruptible, 877 no_wait_reserve, no_wait_gpu); 878 if (unlikely(ret != 0)) 879 return ret; 880 } while (1); 881 if (mem->mm_node == NULL) 882 return -ENOMEM; 883 mem->mem_type = mem_type; 884 return 0; 885} 886 887static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 888 uint32_t cur_placement, 889 uint32_t proposed_placement) 890{ 891 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 892 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 893 894 /** 895 * Keep current caching if possible. 896 */ 897 898 if ((cur_placement & caching) != 0) 899 result |= (cur_placement & caching); 900 else if ((man->default_caching & caching) != 0) 901 result |= man->default_caching; 902 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 903 result |= TTM_PL_FLAG_CACHED; 904 else if ((TTM_PL_FLAG_WC & caching) != 0) 905 result |= TTM_PL_FLAG_WC; 906 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 907 result |= TTM_PL_FLAG_UNCACHED; 908 909 return result; 910} 911 912static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 913 uint32_t mem_type, 914 uint32_t proposed_placement, 915 uint32_t *masked_placement) 916{ 917 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 918 919 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 920 return false; 921 922 if ((proposed_placement & man->available_caching) == 0) 923 return false; 924 925 cur_flags |= (proposed_placement & man->available_caching); 926 927 *masked_placement = cur_flags; 928 return true; 929} 930 931/** 932 * Creates space for memory region @mem according to its type. 933 * 934 * This function first searches for free space in compatible memory types in 935 * the priority order defined by the driver. If free space isn't found, then 936 * ttm_bo_mem_force_space is attempted in priority order to evict and find 937 * space. 938 */ 939int ttm_bo_mem_space(struct ttm_buffer_object *bo, 940 struct ttm_placement *placement, 941 struct ttm_mem_reg *mem, 942 bool interruptible, bool no_wait_reserve, 943 bool no_wait_gpu) 944{ 945 struct ttm_bo_device *bdev = bo->bdev; 946 struct ttm_mem_type_manager *man; 947 uint32_t mem_type = TTM_PL_SYSTEM; 948 uint32_t cur_flags = 0; 949 bool type_found = false; 950 bool type_ok = false; 951 bool has_erestartsys = false; 952 int i, ret; 953 954 mem->mm_node = NULL; 955 for (i = 0; i < placement->num_placement; ++i) { 956 ret = ttm_mem_type_from_flags(placement->placement[i], 957 &mem_type); 958 if (ret) 959 return ret; 960 man = &bdev->man[mem_type]; 961 962 type_ok = ttm_bo_mt_compatible(man, 963 mem_type, 964 placement->placement[i], 965 &cur_flags); 966 967 if (!type_ok) 968 continue; 969 970 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 971 cur_flags); 972 /* 973 * Use the access and other non-mapping-related flag bits from 974 * the memory placement flags to the current flags 975 */ 976 ttm_flag_masked(&cur_flags, placement->placement[i], 977 ~TTM_PL_MASK_MEMTYPE); 978 979 if (mem_type == TTM_PL_SYSTEM) 980 break; 981 982 if (man->has_type && man->use_type) { 983 type_found = true; 984 ret = (*man->func->get_node)(man, bo, placement, mem); 985 if (unlikely(ret)) 986 return ret; 987 } 988 if (mem->mm_node) 989 break; 990 } 991 992 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 993 mem->mem_type = mem_type; 994 mem->placement = cur_flags; 995 return 0; 996 } 997 998 if (!type_found) 999 return -EINVAL; 1000 1001 for (i = 0; i < placement->num_busy_placement; ++i) { 1002 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 1003 &mem_type); 1004 if (ret) 1005 return ret; 1006 man = &bdev->man[mem_type]; 1007 if (!man->has_type) 1008 continue; 1009 if (!ttm_bo_mt_compatible(man, 1010 mem_type, 1011 placement->busy_placement[i], 1012 &cur_flags)) 1013 continue; 1014 1015 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1016 cur_flags); 1017 /* 1018 * Use the access and other non-mapping-related flag bits from 1019 * the memory placement flags to the current flags 1020 */ 1021 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 1022 ~TTM_PL_MASK_MEMTYPE); 1023 1024 1025 if (mem_type == TTM_PL_SYSTEM) { 1026 mem->mem_type = mem_type; 1027 mem->placement = cur_flags; 1028 mem->mm_node = NULL; 1029 return 0; 1030 } 1031 1032 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 1033 interruptible, no_wait_reserve, no_wait_gpu); 1034 if (ret == 0 && mem->mm_node) { 1035 mem->placement = cur_flags; 1036 return 0; 1037 } 1038 if (ret == -ERESTARTSYS) 1039 has_erestartsys = true; 1040 } 1041 ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 1042 return ret; 1043} 1044EXPORT_SYMBOL(ttm_bo_mem_space); 1045 1046int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait) 1047{ 1048 if ((atomic_read(&bo->cpu_writers) > 0) && no_wait) 1049 return -EBUSY; 1050 1051 return wait_event_interruptible(bo->event_queue, 1052 atomic_read(&bo->cpu_writers) == 0); 1053} 1054EXPORT_SYMBOL(ttm_bo_wait_cpu); 1055 1056int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1057 struct ttm_placement *placement, 1058 bool interruptible, bool no_wait_reserve, 1059 bool no_wait_gpu) 1060{ 1061 int ret = 0; 1062 struct ttm_mem_reg mem; 1063 struct ttm_bo_device *bdev = bo->bdev; 1064 1065 BUG_ON(!atomic_read(&bo->reserved)); 1066 1067 /* 1068 * FIXME: It's possible to pipeline buffer moves. 1069 * Have the driver move function wait for idle when necessary, 1070 * instead of doing it here. 1071 */ 1072 spin_lock(&bdev->fence_lock); 1073 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1074 spin_unlock(&bdev->fence_lock); 1075 if (ret) 1076 return ret; 1077 mem.num_pages = bo->num_pages; 1078 mem.size = mem.num_pages << PAGE_SHIFT; 1079 mem.page_alignment = bo->mem.page_alignment; 1080 mem.bus.io_reserved_vm = false; 1081 mem.bus.io_reserved_count = 0; 1082 /* 1083 * Determine where to move the buffer. 1084 */ 1085 ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu); 1086 if (ret) 1087 goto out_unlock; 1088 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu); 1089out_unlock: 1090 if (ret && mem.mm_node) 1091 ttm_bo_mem_put(bo, &mem); 1092 return ret; 1093} 1094 1095static int ttm_bo_mem_compat(struct ttm_placement *placement, 1096 struct ttm_mem_reg *mem) 1097{ 1098 int i; 1099 1100 if (mem->mm_node && placement->lpfn != 0 && 1101 (mem->start < placement->fpfn || 1102 mem->start + mem->num_pages > placement->lpfn)) 1103 return -1; 1104 1105 for (i = 0; i < placement->num_placement; i++) { 1106 if ((placement->placement[i] & mem->placement & 1107 TTM_PL_MASK_CACHING) && 1108 (placement->placement[i] & mem->placement & 1109 TTM_PL_MASK_MEM)) 1110 return i; 1111 } 1112 return -1; 1113} 1114 1115int ttm_bo_validate(struct ttm_buffer_object *bo, 1116 struct ttm_placement *placement, 1117 bool interruptible, bool no_wait_reserve, 1118 bool no_wait_gpu) 1119{ 1120 int ret; 1121 1122 BUG_ON(!atomic_read(&bo->reserved)); 1123 /* Check that range is valid */ 1124 if (placement->lpfn || placement->fpfn) 1125 if (placement->fpfn > placement->lpfn || 1126 (placement->lpfn - placement->fpfn) < bo->num_pages) 1127 return -EINVAL; 1128 /* 1129 * Check whether we need to move buffer. 1130 */ 1131 ret = ttm_bo_mem_compat(placement, &bo->mem); 1132 if (ret < 0) { 1133 ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu); 1134 if (ret) 1135 return ret; 1136 } else { 1137 /* 1138 * Use the access and other non-mapping-related flag bits from 1139 * the compatible memory placement flags to the active flags 1140 */ 1141 ttm_flag_masked(&bo->mem.placement, placement->placement[ret], 1142 ~TTM_PL_MASK_MEMTYPE); 1143 } 1144 /* 1145 * We might need to add a TTM. 1146 */ 1147 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1148 ret = ttm_bo_add_ttm(bo, true); 1149 if (ret) 1150 return ret; 1151 } 1152 return 0; 1153} 1154EXPORT_SYMBOL(ttm_bo_validate); 1155 1156int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1157 struct ttm_placement *placement) 1158{ 1159 BUG_ON((placement->fpfn || placement->lpfn) && 1160 (bo->mem.num_pages > (placement->lpfn - placement->fpfn))); 1161 1162 return 0; 1163} 1164 1165int ttm_bo_init(struct ttm_bo_device *bdev, 1166 struct ttm_buffer_object *bo, 1167 unsigned long size, 1168 enum ttm_bo_type type, 1169 struct ttm_placement *placement, 1170 uint32_t page_alignment, 1171 unsigned long buffer_start, 1172 bool interruptible, 1173 struct file *persistent_swap_storage, 1174 size_t acc_size, 1175 void (*destroy) (struct ttm_buffer_object *)) 1176{ 1177 int ret = 0; 1178 unsigned long num_pages; 1179 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1180 1181 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1182 if (ret) { 1183 printk(KERN_ERR TTM_PFX "Out of kernel memory.\n"); 1184 if (destroy) 1185 (*destroy)(bo); 1186 else 1187 kfree(bo); 1188 return -ENOMEM; 1189 } 1190 1191 size += buffer_start & ~PAGE_MASK; 1192 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1193 if (num_pages == 0) { 1194 printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n"); 1195 if (destroy) 1196 (*destroy)(bo); 1197 else 1198 kfree(bo); 1199 return -EINVAL; 1200 } 1201 bo->destroy = destroy; 1202 1203 kref_init(&bo->kref); 1204 kref_init(&bo->list_kref); 1205 atomic_set(&bo->cpu_writers, 0); 1206 atomic_set(&bo->reserved, 1); 1207 init_waitqueue_head(&bo->event_queue); 1208 INIT_LIST_HEAD(&bo->lru); 1209 INIT_LIST_HEAD(&bo->ddestroy); 1210 INIT_LIST_HEAD(&bo->swap); 1211 INIT_LIST_HEAD(&bo->io_reserve_lru); 1212 bo->bdev = bdev; 1213 bo->glob = bdev->glob; 1214 bo->type = type; 1215 bo->num_pages = num_pages; 1216 bo->mem.size = num_pages << PAGE_SHIFT; 1217 bo->mem.mem_type = TTM_PL_SYSTEM; 1218 bo->mem.num_pages = bo->num_pages; 1219 bo->mem.mm_node = NULL; 1220 bo->mem.page_alignment = page_alignment; 1221 bo->mem.bus.io_reserved_vm = false; 1222 bo->mem.bus.io_reserved_count = 0; 1223 bo->buffer_start = buffer_start & PAGE_MASK; 1224 bo->priv_flags = 0; 1225 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1226 bo->seq_valid = false; 1227 bo->persistent_swap_storage = persistent_swap_storage; 1228 bo->acc_size = acc_size; 1229 atomic_inc(&bo->glob->bo_count); 1230 1231 ret = ttm_bo_check_placement(bo, placement); 1232 if (unlikely(ret != 0)) 1233 goto out_err; 1234 1235 /* 1236 * For ttm_bo_type_device buffers, allocate 1237 * address space from the device. 1238 */ 1239 if (bo->type == ttm_bo_type_device) { 1240 ret = ttm_bo_setup_vm(bo); 1241 if (ret) 1242 goto out_err; 1243 } 1244 1245 ret = ttm_bo_validate(bo, placement, interruptible, false, false); 1246 if (ret) 1247 goto out_err; 1248 1249 ttm_bo_unreserve(bo); 1250 return 0; 1251 1252out_err: 1253 ttm_bo_unreserve(bo); 1254 ttm_bo_unref(&bo); 1255 1256 return ret; 1257} 1258EXPORT_SYMBOL(ttm_bo_init); 1259 1260size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1261 unsigned long bo_size, 1262 unsigned struct_size) 1263{ 1264 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1265 size_t size = 0; 1266 1267 size += ttm_round_pot(struct_size); 1268 size += PAGE_ALIGN(npages * sizeof(void *)); 1269 size += ttm_round_pot(sizeof(struct ttm_tt)); 1270 return size; 1271} 1272EXPORT_SYMBOL(ttm_bo_acc_size); 1273 1274size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1275 unsigned long bo_size, 1276 unsigned struct_size) 1277{ 1278 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1279 size_t size = 0; 1280 1281 size += ttm_round_pot(struct_size); 1282 size += PAGE_ALIGN(npages * sizeof(void *)); 1283 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1284 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1285 return size; 1286} 1287EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1288 1289int ttm_bo_create(struct ttm_bo_device *bdev, 1290 unsigned long size, 1291 enum ttm_bo_type type, 1292 struct ttm_placement *placement, 1293 uint32_t page_alignment, 1294 unsigned long buffer_start, 1295 bool interruptible, 1296 struct file *persistent_swap_storage, 1297 struct ttm_buffer_object **p_bo) 1298{ 1299 struct ttm_buffer_object *bo; 1300 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1301 size_t acc_size; 1302 int ret; 1303 1304 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1305 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1306 if (unlikely(ret != 0)) 1307 return ret; 1308 1309 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1310 1311 if (unlikely(bo == NULL)) { 1312 ttm_mem_global_free(mem_glob, acc_size); 1313 return -ENOMEM; 1314 } 1315 1316 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1317 buffer_start, interruptible, 1318 persistent_swap_storage, acc_size, NULL); 1319 if (likely(ret == 0)) 1320 *p_bo = bo; 1321 1322 return ret; 1323} 1324EXPORT_SYMBOL(ttm_bo_create); 1325 1326static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1327 unsigned mem_type, bool allow_errors) 1328{ 1329 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1330 struct ttm_bo_global *glob = bdev->glob; 1331 int ret; 1332 1333 /* 1334 * Can't use standard list traversal since we're unlocking. 1335 */ 1336 1337 spin_lock(&glob->lru_lock); 1338 while (!list_empty(&man->lru)) { 1339 spin_unlock(&glob->lru_lock); 1340 ret = ttm_mem_evict_first(bdev, mem_type, false, false, false); 1341 if (ret) { 1342 if (allow_errors) { 1343 return ret; 1344 } else { 1345 printk(KERN_ERR TTM_PFX 1346 "Cleanup eviction failed\n"); 1347 } 1348 } 1349 spin_lock(&glob->lru_lock); 1350 } 1351 spin_unlock(&glob->lru_lock); 1352 return 0; 1353} 1354 1355int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1356{ 1357 struct ttm_mem_type_manager *man; 1358 int ret = -EINVAL; 1359 1360 if (mem_type >= TTM_NUM_MEM_TYPES) { 1361 printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", mem_type); 1362 return ret; 1363 } 1364 man = &bdev->man[mem_type]; 1365 1366 if (!man->has_type) { 1367 printk(KERN_ERR TTM_PFX "Trying to take down uninitialized " 1368 "memory manager type %u\n", mem_type); 1369 return ret; 1370 } 1371 1372 man->use_type = false; 1373 man->has_type = false; 1374 1375 ret = 0; 1376 if (mem_type > 0) { 1377 ttm_bo_force_list_clean(bdev, mem_type, false); 1378 1379 ret = (*man->func->takedown)(man); 1380 } 1381 1382 return ret; 1383} 1384EXPORT_SYMBOL(ttm_bo_clean_mm); 1385 1386int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1387{ 1388 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1389 1390 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1391 printk(KERN_ERR TTM_PFX 1392 "Illegal memory manager memory type %u.\n", 1393 mem_type); 1394 return -EINVAL; 1395 } 1396 1397 if (!man->has_type) { 1398 printk(KERN_ERR TTM_PFX 1399 "Memory type %u has not been initialized.\n", 1400 mem_type); 1401 return 0; 1402 } 1403 1404 return ttm_bo_force_list_clean(bdev, mem_type, true); 1405} 1406EXPORT_SYMBOL(ttm_bo_evict_mm); 1407 1408int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1409 unsigned long p_size) 1410{ 1411 int ret = -EINVAL; 1412 struct ttm_mem_type_manager *man; 1413 1414 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1415 man = &bdev->man[type]; 1416 BUG_ON(man->has_type); 1417 man->io_reserve_fastpath = true; 1418 man->use_io_reserve_lru = false; 1419 mutex_init(&man->io_reserve_mutex); 1420 INIT_LIST_HEAD(&man->io_reserve_lru); 1421 1422 ret = bdev->driver->init_mem_type(bdev, type, man); 1423 if (ret) 1424 return ret; 1425 man->bdev = bdev; 1426 1427 ret = 0; 1428 if (type != TTM_PL_SYSTEM) { 1429 ret = (*man->func->init)(man, p_size); 1430 if (ret) 1431 return ret; 1432 } 1433 man->has_type = true; 1434 man->use_type = true; 1435 man->size = p_size; 1436 1437 INIT_LIST_HEAD(&man->lru); 1438 1439 return 0; 1440} 1441EXPORT_SYMBOL(ttm_bo_init_mm); 1442 1443static void ttm_bo_global_kobj_release(struct kobject *kobj) 1444{ 1445 struct ttm_bo_global *glob = 1446 container_of(kobj, struct ttm_bo_global, kobj); 1447 1448 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1449 __free_page(glob->dummy_read_page); 1450 kfree(glob); 1451} 1452 1453void ttm_bo_global_release(struct drm_global_reference *ref) 1454{ 1455 struct ttm_bo_global *glob = ref->object; 1456 1457 kobject_del(&glob->kobj); 1458 kobject_put(&glob->kobj); 1459} 1460EXPORT_SYMBOL(ttm_bo_global_release); 1461 1462int ttm_bo_global_init(struct drm_global_reference *ref) 1463{ 1464 struct ttm_bo_global_ref *bo_ref = 1465 container_of(ref, struct ttm_bo_global_ref, ref); 1466 struct ttm_bo_global *glob = ref->object; 1467 int ret; 1468 1469 mutex_init(&glob->device_list_mutex); 1470 spin_lock_init(&glob->lru_lock); 1471 glob->mem_glob = bo_ref->mem_glob; 1472 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1473 1474 if (unlikely(glob->dummy_read_page == NULL)) { 1475 ret = -ENOMEM; 1476 goto out_no_drp; 1477 } 1478 1479 INIT_LIST_HEAD(&glob->swap_lru); 1480 INIT_LIST_HEAD(&glob->device_list); 1481 1482 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1483 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1484 if (unlikely(ret != 0)) { 1485 printk(KERN_ERR TTM_PFX 1486 "Could not register buffer object swapout.\n"); 1487 goto out_no_shrink; 1488 } 1489 1490 atomic_set(&glob->bo_count, 0); 1491 1492 ret = kobject_init_and_add( 1493 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1494 if (unlikely(ret != 0)) 1495 kobject_put(&glob->kobj); 1496 return ret; 1497out_no_shrink: 1498 __free_page(glob->dummy_read_page); 1499out_no_drp: 1500 kfree(glob); 1501 return ret; 1502} 1503EXPORT_SYMBOL(ttm_bo_global_init); 1504 1505 1506int ttm_bo_device_release(struct ttm_bo_device *bdev) 1507{ 1508 int ret = 0; 1509 unsigned i = TTM_NUM_MEM_TYPES; 1510 struct ttm_mem_type_manager *man; 1511 struct ttm_bo_global *glob = bdev->glob; 1512 1513 while (i--) { 1514 man = &bdev->man[i]; 1515 if (man->has_type) { 1516 man->use_type = false; 1517 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1518 ret = -EBUSY; 1519 printk(KERN_ERR TTM_PFX 1520 "DRM memory manager type %d " 1521 "is not clean.\n", i); 1522 } 1523 man->has_type = false; 1524 } 1525 } 1526 1527 mutex_lock(&glob->device_list_mutex); 1528 list_del(&bdev->device_list); 1529 mutex_unlock(&glob->device_list_mutex); 1530 1531 cancel_delayed_work_sync(&bdev->wq); 1532 1533 while (ttm_bo_delayed_delete(bdev, true)) 1534 ; 1535 1536 spin_lock(&glob->lru_lock); 1537 if (list_empty(&bdev->ddestroy)) 1538 TTM_DEBUG("Delayed destroy list was clean\n"); 1539 1540 if (list_empty(&bdev->man[0].lru)) 1541 TTM_DEBUG("Swap list was clean\n"); 1542 spin_unlock(&glob->lru_lock); 1543 1544 BUG_ON(!drm_mm_clean(&bdev->addr_space_mm)); 1545 write_lock(&bdev->vm_lock); 1546 drm_mm_takedown(&bdev->addr_space_mm); 1547 write_unlock(&bdev->vm_lock); 1548 1549 return ret; 1550} 1551EXPORT_SYMBOL(ttm_bo_device_release); 1552 1553int ttm_bo_device_init(struct ttm_bo_device *bdev, 1554 struct ttm_bo_global *glob, 1555 struct ttm_bo_driver *driver, 1556 uint64_t file_page_offset, 1557 bool need_dma32) 1558{ 1559 int ret = -EINVAL; 1560 1561 rwlock_init(&bdev->vm_lock); 1562 bdev->driver = driver; 1563 1564 memset(bdev->man, 0, sizeof(bdev->man)); 1565 1566 /* 1567 * Initialize the system memory buffer type. 1568 * Other types need to be driver / IOCTL initialized. 1569 */ 1570 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1571 if (unlikely(ret != 0)) 1572 goto out_no_sys; 1573 1574 bdev->addr_space_rb = RB_ROOT; 1575 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); 1576 if (unlikely(ret != 0)) 1577 goto out_no_addr_mm; 1578 1579 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1580 bdev->nice_mode = true; 1581 INIT_LIST_HEAD(&bdev->ddestroy); 1582 bdev->dev_mapping = NULL; 1583 bdev->glob = glob; 1584 bdev->need_dma32 = need_dma32; 1585 bdev->val_seq = 0; 1586 spin_lock_init(&bdev->fence_lock); 1587 mutex_lock(&glob->device_list_mutex); 1588 list_add_tail(&bdev->device_list, &glob->device_list); 1589 mutex_unlock(&glob->device_list_mutex); 1590 1591 return 0; 1592out_no_addr_mm: 1593 ttm_bo_clean_mm(bdev, 0); 1594out_no_sys: 1595 return ret; 1596} 1597EXPORT_SYMBOL(ttm_bo_device_init); 1598 1599/* 1600 * buffer object vm functions. 1601 */ 1602 1603bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1604{ 1605 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1606 1607 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1608 if (mem->mem_type == TTM_PL_SYSTEM) 1609 return false; 1610 1611 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1612 return false; 1613 1614 if (mem->placement & TTM_PL_FLAG_CACHED) 1615 return false; 1616 } 1617 return true; 1618} 1619 1620void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1621{ 1622 struct ttm_bo_device *bdev = bo->bdev; 1623 loff_t offset = (loff_t) bo->addr_space_offset; 1624 loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT; 1625 1626 if (!bdev->dev_mapping) 1627 return; 1628 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); 1629 ttm_mem_io_free_vm(bo); 1630} 1631 1632void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1633{ 1634 struct ttm_bo_device *bdev = bo->bdev; 1635 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1636 1637 ttm_mem_io_lock(man, false); 1638 ttm_bo_unmap_virtual_locked(bo); 1639 ttm_mem_io_unlock(man); 1640} 1641 1642 1643EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1644 1645static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1646{ 1647 struct ttm_bo_device *bdev = bo->bdev; 1648 struct rb_node **cur = &bdev->addr_space_rb.rb_node; 1649 struct rb_node *parent = NULL; 1650 struct ttm_buffer_object *cur_bo; 1651 unsigned long offset = bo->vm_node->start; 1652 unsigned long cur_offset; 1653 1654 while (*cur) { 1655 parent = *cur; 1656 cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb); 1657 cur_offset = cur_bo->vm_node->start; 1658 if (offset < cur_offset) 1659 cur = &parent->rb_left; 1660 else if (offset > cur_offset) 1661 cur = &parent->rb_right; 1662 else 1663 BUG(); 1664 } 1665 1666 rb_link_node(&bo->vm_rb, parent, cur); 1667 rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb); 1668} 1669 1670/** 1671 * ttm_bo_setup_vm: 1672 * 1673 * @bo: the buffer to allocate address space for 1674 * 1675 * Allocate address space in the drm device so that applications 1676 * can mmap the buffer and access the contents. This only 1677 * applies to ttm_bo_type_device objects as others are not 1678 * placed in the drm device address space. 1679 */ 1680 1681static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1682{ 1683 struct ttm_bo_device *bdev = bo->bdev; 1684 int ret; 1685 1686retry_pre_get: 1687 ret = drm_mm_pre_get(&bdev->addr_space_mm); 1688 if (unlikely(ret != 0)) 1689 return ret; 1690 1691 write_lock(&bdev->vm_lock); 1692 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, 1693 bo->mem.num_pages, 0, 0); 1694 1695 if (unlikely(bo->vm_node == NULL)) { 1696 ret = -ENOMEM; 1697 goto out_unlock; 1698 } 1699 1700 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, 1701 bo->mem.num_pages, 0); 1702 1703 if (unlikely(bo->vm_node == NULL)) { 1704 write_unlock(&bdev->vm_lock); 1705 goto retry_pre_get; 1706 } 1707 1708 ttm_bo_vm_insert_rb(bo); 1709 write_unlock(&bdev->vm_lock); 1710 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; 1711 1712 return 0; 1713out_unlock: 1714 write_unlock(&bdev->vm_lock); 1715 return ret; 1716} 1717 1718int ttm_bo_wait(struct ttm_buffer_object *bo, 1719 bool lazy, bool interruptible, bool no_wait) 1720{ 1721 struct ttm_bo_driver *driver = bo->bdev->driver; 1722 struct ttm_bo_device *bdev = bo->bdev; 1723 void *sync_obj; 1724 void *sync_obj_arg; 1725 int ret = 0; 1726 1727 if (likely(bo->sync_obj == NULL)) 1728 return 0; 1729 1730 while (bo->sync_obj) { 1731 1732 if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) { 1733 void *tmp_obj = bo->sync_obj; 1734 bo->sync_obj = NULL; 1735 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1736 spin_unlock(&bdev->fence_lock); 1737 driver->sync_obj_unref(&tmp_obj); 1738 spin_lock(&bdev->fence_lock); 1739 continue; 1740 } 1741 1742 if (no_wait) 1743 return -EBUSY; 1744 1745 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1746 sync_obj_arg = bo->sync_obj_arg; 1747 spin_unlock(&bdev->fence_lock); 1748 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg, 1749 lazy, interruptible); 1750 if (unlikely(ret != 0)) { 1751 driver->sync_obj_unref(&sync_obj); 1752 spin_lock(&bdev->fence_lock); 1753 return ret; 1754 } 1755 spin_lock(&bdev->fence_lock); 1756 if (likely(bo->sync_obj == sync_obj && 1757 bo->sync_obj_arg == sync_obj_arg)) { 1758 void *tmp_obj = bo->sync_obj; 1759 bo->sync_obj = NULL; 1760 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1761 &bo->priv_flags); 1762 spin_unlock(&bdev->fence_lock); 1763 driver->sync_obj_unref(&sync_obj); 1764 driver->sync_obj_unref(&tmp_obj); 1765 spin_lock(&bdev->fence_lock); 1766 } else { 1767 spin_unlock(&bdev->fence_lock); 1768 driver->sync_obj_unref(&sync_obj); 1769 spin_lock(&bdev->fence_lock); 1770 } 1771 } 1772 return 0; 1773} 1774EXPORT_SYMBOL(ttm_bo_wait); 1775 1776int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1777{ 1778 struct ttm_bo_device *bdev = bo->bdev; 1779 int ret = 0; 1780 1781 /* 1782 * Using ttm_bo_reserve makes sure the lru lists are updated. 1783 */ 1784 1785 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1786 if (unlikely(ret != 0)) 1787 return ret; 1788 spin_lock(&bdev->fence_lock); 1789 ret = ttm_bo_wait(bo, false, true, no_wait); 1790 spin_unlock(&bdev->fence_lock); 1791 if (likely(ret == 0)) 1792 atomic_inc(&bo->cpu_writers); 1793 ttm_bo_unreserve(bo); 1794 return ret; 1795} 1796EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1797 1798void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1799{ 1800 if (atomic_dec_and_test(&bo->cpu_writers)) 1801 wake_up_all(&bo->event_queue); 1802} 1803EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1804 1805/** 1806 * A buffer object shrink method that tries to swap out the first 1807 * buffer object on the bo_global::swap_lru list. 1808 */ 1809 1810static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1811{ 1812 struct ttm_bo_global *glob = 1813 container_of(shrink, struct ttm_bo_global, shrink); 1814 struct ttm_buffer_object *bo; 1815 int ret = -EBUSY; 1816 int put_count; 1817 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1818 1819 spin_lock(&glob->lru_lock); 1820 while (ret == -EBUSY) { 1821 if (unlikely(list_empty(&glob->swap_lru))) { 1822 spin_unlock(&glob->lru_lock); 1823 return -EBUSY; 1824 } 1825 1826 bo = list_first_entry(&glob->swap_lru, 1827 struct ttm_buffer_object, swap); 1828 kref_get(&bo->list_kref); 1829 1830 if (!list_empty(&bo->ddestroy)) { 1831 spin_unlock(&glob->lru_lock); 1832 (void) ttm_bo_cleanup_refs(bo, false, false, false); 1833 kref_put(&bo->list_kref, ttm_bo_release_list); 1834 continue; 1835 } 1836 1837 /** 1838 * Reserve buffer. Since we unlock while sleeping, we need 1839 * to re-check that nobody removed us from the swap-list while 1840 * we slept. 1841 */ 1842 1843 ret = ttm_bo_reserve_locked(bo, false, true, false, 0); 1844 if (unlikely(ret == -EBUSY)) { 1845 spin_unlock(&glob->lru_lock); 1846 ttm_bo_wait_unreserved(bo, false); 1847 kref_put(&bo->list_kref, ttm_bo_release_list); 1848 spin_lock(&glob->lru_lock); 1849 } 1850 } 1851 1852 BUG_ON(ret != 0); 1853 put_count = ttm_bo_del_from_lru(bo); 1854 spin_unlock(&glob->lru_lock); 1855 1856 ttm_bo_list_ref_sub(bo, put_count, true); 1857 1858 /** 1859 * Wait for GPU, then move to system cached. 1860 */ 1861 1862 spin_lock(&bo->bdev->fence_lock); 1863 ret = ttm_bo_wait(bo, false, false, false); 1864 spin_unlock(&bo->bdev->fence_lock); 1865 1866 if (unlikely(ret != 0)) 1867 goto out; 1868 1869 if ((bo->mem.placement & swap_placement) != swap_placement) { 1870 struct ttm_mem_reg evict_mem; 1871 1872 evict_mem = bo->mem; 1873 evict_mem.mm_node = NULL; 1874 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1875 evict_mem.mem_type = TTM_PL_SYSTEM; 1876 1877 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1878 false, false, false); 1879 if (unlikely(ret != 0)) 1880 goto out; 1881 } 1882 1883 ttm_bo_unmap_virtual(bo); 1884 1885 /** 1886 * Swap out. Buffer will be swapped in again as soon as 1887 * anyone tries to access a ttm page. 1888 */ 1889 1890 if (bo->bdev->driver->swap_notify) 1891 bo->bdev->driver->swap_notify(bo); 1892 1893 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1894out: 1895 1896 /** 1897 * 1898 * Unreserve without putting on LRU to avoid swapping out an 1899 * already swapped buffer. 1900 */ 1901 1902 atomic_set(&bo->reserved, 0); 1903 wake_up_all(&bo->event_queue); 1904 kref_put(&bo->list_kref, ttm_bo_release_list); 1905 return ret; 1906} 1907 1908void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1909{ 1910 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1911 ; 1912} 1913EXPORT_SYMBOL(ttm_bo_swapout_all); 1914