1/************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27/* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31#define pr_fmt(fmt) "[TTM] " fmt 32 33#include "ttm/ttm_module.h" 34#include "ttm/ttm_bo_driver.h" 35#include "ttm/ttm_placement.h" 36#include <linux/jiffies.h> 37#include <linux/slab.h> 38#include <linux/sched.h> 39#include <linux/mm.h> 40#include <linux/file.h> 41#include <linux/module.h> 42#include <linux/atomic.h> 43 44#define TTM_ASSERT_LOCKED(param) 45#define TTM_DEBUG(fmt, arg...) 46#define TTM_BO_HASH_ORDER 13 47 48static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); 49static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 50static void ttm_bo_global_kobj_release(struct kobject *kobj); 51 52static struct attribute ttm_bo_count = { 53 .name = "bo_count", 54 .mode = S_IRUGO 55}; 56 57static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) 58{ 59 int i; 60 61 for (i = 0; i <= TTM_PL_PRIV5; i++) 62 if (flags & (1 << i)) { 63 *mem_type = i; 64 return 0; 65 } 66 return -EINVAL; 67} 68 69static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 70{ 71 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 72 73 pr_err(" has_type: %d\n", man->has_type); 74 pr_err(" use_type: %d\n", man->use_type); 75 pr_err(" flags: 0x%08X\n", man->flags); 76 pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); 77 pr_err(" size: %llu\n", man->size); 78 pr_err(" available_caching: 0x%08X\n", man->available_caching); 79 pr_err(" default_caching: 0x%08X\n", man->default_caching); 80 if (mem_type != TTM_PL_SYSTEM) 81 (*man->func->debug)(man, TTM_PFX); 82} 83 84static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 85 struct ttm_placement *placement) 86{ 87 int i, ret, mem_type; 88 89 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 90 bo, bo->mem.num_pages, bo->mem.size >> 10, 91 bo->mem.size >> 20); 92 for (i = 0; i < placement->num_placement; i++) { 93 ret = ttm_mem_type_from_flags(placement->placement[i], 94 &mem_type); 95 if (ret) 96 return; 97 pr_err(" placement[%d]=0x%08X (%d)\n", 98 i, placement->placement[i], mem_type); 99 ttm_mem_type_debug(bo->bdev, mem_type); 100 } 101} 102 103static ssize_t ttm_bo_global_show(struct kobject *kobj, 104 struct attribute *attr, 105 char *buffer) 106{ 107 struct ttm_bo_global *glob = 108 container_of(kobj, struct ttm_bo_global, kobj); 109 110 return snprintf(buffer, PAGE_SIZE, "%lu\n", 111 (unsigned long) atomic_read(&glob->bo_count)); 112} 113 114static struct attribute *ttm_bo_global_attrs[] = { 115 &ttm_bo_count, 116 NULL 117}; 118 119static const struct sysfs_ops ttm_bo_global_ops = { 120 .show = &ttm_bo_global_show 121}; 122 123static struct kobj_type ttm_bo_glob_kobj_type = { 124 .release = &ttm_bo_global_kobj_release, 125 .sysfs_ops = &ttm_bo_global_ops, 126 .default_attrs = ttm_bo_global_attrs 127}; 128 129 130static inline uint32_t ttm_bo_type_flags(unsigned type) 131{ 132 return 1 << (type); 133} 134 135static void ttm_bo_release_list(struct kref *list_kref) 136{ 137 struct ttm_buffer_object *bo = 138 container_of(list_kref, struct ttm_buffer_object, list_kref); 139 struct ttm_bo_device *bdev = bo->bdev; 140 size_t acc_size = bo->acc_size; 141 142 BUG_ON(atomic_read(&bo->list_kref.refcount)); 143 BUG_ON(atomic_read(&bo->kref.refcount)); 144 BUG_ON(atomic_read(&bo->cpu_writers)); 145 BUG_ON(bo->sync_obj != NULL); 146 BUG_ON(bo->mem.mm_node != NULL); 147 BUG_ON(!list_empty(&bo->lru)); 148 BUG_ON(!list_empty(&bo->ddestroy)); 149 150 if (bo->ttm) 151 ttm_tt_destroy(bo->ttm); 152 atomic_dec(&bo->glob->bo_count); 153 if (bo->destroy) 154 bo->destroy(bo); 155 else { 156 kfree(bo); 157 } 158 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 159} 160 161int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) 162{ 163 if (interruptible) { 164 return wait_event_interruptible(bo->event_queue, 165 atomic_read(&bo->reserved) == 0); 166 } else { 167 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0); 168 return 0; 169 } 170} 171EXPORT_SYMBOL(ttm_bo_wait_unreserved); 172 173void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 174{ 175 struct ttm_bo_device *bdev = bo->bdev; 176 struct ttm_mem_type_manager *man; 177 178 BUG_ON(!atomic_read(&bo->reserved)); 179 180 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 181 182 BUG_ON(!list_empty(&bo->lru)); 183 184 man = &bdev->man[bo->mem.mem_type]; 185 list_add_tail(&bo->lru, &man->lru); 186 kref_get(&bo->list_kref); 187 188 if (bo->ttm != NULL) { 189 list_add_tail(&bo->swap, &bo->glob->swap_lru); 190 kref_get(&bo->list_kref); 191 } 192 } 193} 194 195int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 196{ 197 int put_count = 0; 198 199 if (!list_empty(&bo->swap)) { 200 list_del_init(&bo->swap); 201 ++put_count; 202 } 203 if (!list_empty(&bo->lru)) { 204 list_del_init(&bo->lru); 205 ++put_count; 206 } 207 208 /* 209 * TODO: Add a driver hook to delete from 210 * driver-specific LRU's here. 211 */ 212 213 return put_count; 214} 215 216int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, 217 bool interruptible, 218 bool no_wait, bool use_sequence, uint32_t sequence) 219{ 220 struct ttm_bo_global *glob = bo->glob; 221 int ret; 222 223 while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) { 224 /** 225 * Deadlock avoidance for multi-bo reserving. 226 */ 227 if (use_sequence && bo->seq_valid) { 228 /** 229 * We've already reserved this one. 230 */ 231 if (unlikely(sequence == bo->val_seq)) 232 return -EDEADLK; 233 /** 234 * Already reserved by a thread that will not back 235 * off for us. We need to back off. 236 */ 237 if (unlikely(sequence - bo->val_seq < (1 << 31))) 238 return -EAGAIN; 239 } 240 241 if (no_wait) 242 return -EBUSY; 243 244 spin_unlock(&glob->lru_lock); 245 ret = ttm_bo_wait_unreserved(bo, interruptible); 246 spin_lock(&glob->lru_lock); 247 248 if (unlikely(ret)) 249 return ret; 250 } 251 252 if (use_sequence) { 253 /** 254 * Wake up waiters that may need to recheck for deadlock, 255 * if we decreased the sequence number. 256 */ 257 if (unlikely((bo->val_seq - sequence < (1 << 31)) 258 || !bo->seq_valid)) 259 wake_up_all(&bo->event_queue); 260 261 bo->val_seq = sequence; 262 bo->seq_valid = true; 263 } else { 264 bo->seq_valid = false; 265 } 266 267 return 0; 268} 269EXPORT_SYMBOL(ttm_bo_reserve); 270 271static void ttm_bo_ref_bug(struct kref *list_kref) 272{ 273 BUG(); 274} 275 276void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, 277 bool never_free) 278{ 279 kref_sub(&bo->list_kref, count, 280 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); 281} 282 283int ttm_bo_reserve(struct ttm_buffer_object *bo, 284 bool interruptible, 285 bool no_wait, bool use_sequence, uint32_t sequence) 286{ 287 struct ttm_bo_global *glob = bo->glob; 288 int put_count = 0; 289 int ret; 290 291 spin_lock(&glob->lru_lock); 292 ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, 293 sequence); 294 if (likely(ret == 0)) 295 put_count = ttm_bo_del_from_lru(bo); 296 spin_unlock(&glob->lru_lock); 297 298 ttm_bo_list_ref_sub(bo, put_count, true); 299 300 return ret; 301} 302 303void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) 304{ 305 ttm_bo_add_to_lru(bo); 306 atomic_set(&bo->reserved, 0); 307 wake_up_all(&bo->event_queue); 308} 309 310void ttm_bo_unreserve(struct ttm_buffer_object *bo) 311{ 312 struct ttm_bo_global *glob = bo->glob; 313 314 spin_lock(&glob->lru_lock); 315 ttm_bo_unreserve_locked(bo); 316 spin_unlock(&glob->lru_lock); 317} 318EXPORT_SYMBOL(ttm_bo_unreserve); 319 320/* 321 * Call bo->mutex locked. 322 */ 323static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 324{ 325 struct ttm_bo_device *bdev = bo->bdev; 326 struct ttm_bo_global *glob = bo->glob; 327 int ret = 0; 328 uint32_t page_flags = 0; 329 330 TTM_ASSERT_LOCKED(&bo->mutex); 331 bo->ttm = NULL; 332 333 if (bdev->need_dma32) 334 page_flags |= TTM_PAGE_FLAG_DMA32; 335 336 switch (bo->type) { 337 case ttm_bo_type_device: 338 if (zero_alloc) 339 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 340 case ttm_bo_type_kernel: 341 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 342 page_flags, glob->dummy_read_page); 343 if (unlikely(bo->ttm == NULL)) 344 ret = -ENOMEM; 345 break; 346 default: 347 pr_err("Illegal buffer object type\n"); 348 ret = -EINVAL; 349 break; 350 } 351 352 return ret; 353} 354 355static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 356 struct ttm_mem_reg *mem, 357 bool evict, bool interruptible, 358 bool no_wait_reserve, bool no_wait_gpu) 359{ 360 struct ttm_bo_device *bdev = bo->bdev; 361 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 362 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 363 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 364 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 365 int ret = 0; 366 367 if (old_is_pci || new_is_pci || 368 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 369 ret = ttm_mem_io_lock(old_man, true); 370 if (unlikely(ret != 0)) 371 goto out_err; 372 ttm_bo_unmap_virtual_locked(bo); 373 ttm_mem_io_unlock(old_man); 374 } 375 376 /* 377 * Create and bind a ttm if required. 378 */ 379 380 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 381 if (bo->ttm == NULL) { 382 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 383 ret = ttm_bo_add_ttm(bo, zero); 384 if (ret) 385 goto out_err; 386 } 387 388 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 389 if (ret) 390 goto out_err; 391 392 if (mem->mem_type != TTM_PL_SYSTEM) { 393 ret = ttm_tt_bind(bo->ttm, mem); 394 if (ret) 395 goto out_err; 396 } 397 398 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 399 if (bdev->driver->move_notify) 400 bdev->driver->move_notify(bo, mem); 401 bo->mem = *mem; 402 mem->mm_node = NULL; 403 goto moved; 404 } 405 } 406 407 if (bdev->driver->move_notify) 408 bdev->driver->move_notify(bo, mem); 409 410 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 411 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 412 ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem); 413 else if (bdev->driver->move) 414 ret = bdev->driver->move(bo, evict, interruptible, 415 no_wait_reserve, no_wait_gpu, mem); 416 else 417 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem); 418 419 if (ret) { 420 if (bdev->driver->move_notify) { 421 struct ttm_mem_reg tmp_mem = *mem; 422 *mem = bo->mem; 423 bo->mem = tmp_mem; 424 bdev->driver->move_notify(bo, mem); 425 bo->mem = *mem; 426 } 427 428 goto out_err; 429 } 430 431moved: 432 if (bo->evicted) { 433 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 434 if (ret) 435 pr_err("Can not flush read caches\n"); 436 bo->evicted = false; 437 } 438 439 if (bo->mem.mm_node) { 440 bo->offset = (bo->mem.start << PAGE_SHIFT) + 441 bdev->man[bo->mem.mem_type].gpu_offset; 442 bo->cur_placement = bo->mem.placement; 443 } else 444 bo->offset = 0; 445 446 return 0; 447 448out_err: 449 new_man = &bdev->man[bo->mem.mem_type]; 450 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 451 ttm_tt_unbind(bo->ttm); 452 ttm_tt_destroy(bo->ttm); 453 bo->ttm = NULL; 454 } 455 456 return ret; 457} 458 459/** 460 * Call bo::reserved. 461 * Will release GPU memory type usage on destruction. 462 * This is the place to put in driver specific hooks to release 463 * driver private resources. 464 * Will release the bo::reserved lock. 465 */ 466 467static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 468{ 469 if (bo->bdev->driver->move_notify) 470 bo->bdev->driver->move_notify(bo, NULL); 471 472 if (bo->ttm) { 473 ttm_tt_unbind(bo->ttm); 474 ttm_tt_destroy(bo->ttm); 475 bo->ttm = NULL; 476 } 477 ttm_bo_mem_put(bo, &bo->mem); 478 479 atomic_set(&bo->reserved, 0); 480 481 /* 482 * Make processes trying to reserve really pick it up. 483 */ 484 smp_mb__after_atomic_dec(); 485 wake_up_all(&bo->event_queue); 486} 487 488static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 489{ 490 struct ttm_bo_device *bdev = bo->bdev; 491 struct ttm_bo_global *glob = bo->glob; 492 struct ttm_bo_driver *driver; 493 void *sync_obj = NULL; 494 void *sync_obj_arg; 495 int put_count; 496 int ret; 497 498 spin_lock(&bdev->fence_lock); 499 (void) ttm_bo_wait(bo, false, false, true); 500 if (!bo->sync_obj) { 501 502 spin_lock(&glob->lru_lock); 503 504 /** 505 * Lock inversion between bo:reserve and bdev::fence_lock here, 506 * but that's OK, since we're only trylocking. 507 */ 508 509 ret = ttm_bo_reserve_locked(bo, false, true, false, 0); 510 511 if (unlikely(ret == -EBUSY)) 512 goto queue; 513 514 spin_unlock(&bdev->fence_lock); 515 put_count = ttm_bo_del_from_lru(bo); 516 517 spin_unlock(&glob->lru_lock); 518 ttm_bo_cleanup_memtype_use(bo); 519 520 ttm_bo_list_ref_sub(bo, put_count, true); 521 522 return; 523 } else { 524 spin_lock(&glob->lru_lock); 525 } 526queue: 527 driver = bdev->driver; 528 if (bo->sync_obj) 529 sync_obj = driver->sync_obj_ref(bo->sync_obj); 530 sync_obj_arg = bo->sync_obj_arg; 531 532 kref_get(&bo->list_kref); 533 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 534 spin_unlock(&glob->lru_lock); 535 spin_unlock(&bdev->fence_lock); 536 537 if (sync_obj) { 538 driver->sync_obj_flush(sync_obj, sync_obj_arg); 539 driver->sync_obj_unref(&sync_obj); 540 } 541 schedule_delayed_work(&bdev->wq, 542 ((HZ / 100) < 1) ? 1 : HZ / 100); 543} 544 545/** 546 * function ttm_bo_cleanup_refs 547 * If bo idle, remove from delayed- and lru lists, and unref. 548 * If not idle, do nothing. 549 * 550 * @interruptible Any sleeps should occur interruptibly. 551 * @no_wait_reserve Never wait for reserve. Return -EBUSY instead. 552 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 553 */ 554 555static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 556 bool interruptible, 557 bool no_wait_reserve, 558 bool no_wait_gpu) 559{ 560 struct ttm_bo_device *bdev = bo->bdev; 561 struct ttm_bo_global *glob = bo->glob; 562 int put_count; 563 int ret = 0; 564 565retry: 566 spin_lock(&bdev->fence_lock); 567 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 568 spin_unlock(&bdev->fence_lock); 569 570 if (unlikely(ret != 0)) 571 return ret; 572 573 spin_lock(&glob->lru_lock); 574 575 if (unlikely(list_empty(&bo->ddestroy))) { 576 spin_unlock(&glob->lru_lock); 577 return 0; 578 } 579 580 ret = ttm_bo_reserve_locked(bo, interruptible, 581 no_wait_reserve, false, 0); 582 583 if (unlikely(ret != 0)) { 584 spin_unlock(&glob->lru_lock); 585 return ret; 586 } 587 588 /** 589 * We can re-check for sync object without taking 590 * the bo::lock since setting the sync object requires 591 * also bo::reserved. A busy object at this point may 592 * be caused by another thread recently starting an accelerated 593 * eviction. 594 */ 595 596 if (unlikely(bo->sync_obj)) { 597 atomic_set(&bo->reserved, 0); 598 wake_up_all(&bo->event_queue); 599 spin_unlock(&glob->lru_lock); 600 goto retry; 601 } 602 603 put_count = ttm_bo_del_from_lru(bo); 604 list_del_init(&bo->ddestroy); 605 ++put_count; 606 607 spin_unlock(&glob->lru_lock); 608 ttm_bo_cleanup_memtype_use(bo); 609 610 ttm_bo_list_ref_sub(bo, put_count, true); 611 612 return 0; 613} 614 615/** 616 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 617 * encountered buffers. 618 */ 619 620static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 621{ 622 struct ttm_bo_global *glob = bdev->glob; 623 struct ttm_buffer_object *entry = NULL; 624 int ret = 0; 625 626 spin_lock(&glob->lru_lock); 627 if (list_empty(&bdev->ddestroy)) 628 goto out_unlock; 629 630 entry = list_first_entry(&bdev->ddestroy, 631 struct ttm_buffer_object, ddestroy); 632 kref_get(&entry->list_kref); 633 634 for (;;) { 635 struct ttm_buffer_object *nentry = NULL; 636 637 if (entry->ddestroy.next != &bdev->ddestroy) { 638 nentry = list_first_entry(&entry->ddestroy, 639 struct ttm_buffer_object, ddestroy); 640 kref_get(&nentry->list_kref); 641 } 642 643 spin_unlock(&glob->lru_lock); 644 ret = ttm_bo_cleanup_refs(entry, false, !remove_all, 645 !remove_all); 646 kref_put(&entry->list_kref, ttm_bo_release_list); 647 entry = nentry; 648 649 if (ret || !entry) 650 goto out; 651 652 spin_lock(&glob->lru_lock); 653 if (list_empty(&entry->ddestroy)) 654 break; 655 } 656 657out_unlock: 658 spin_unlock(&glob->lru_lock); 659out: 660 if (entry) 661 kref_put(&entry->list_kref, ttm_bo_release_list); 662 return ret; 663} 664 665static void ttm_bo_delayed_workqueue(struct work_struct *work) 666{ 667 struct ttm_bo_device *bdev = 668 container_of(work, struct ttm_bo_device, wq.work); 669 670 if (ttm_bo_delayed_delete(bdev, false)) { 671 schedule_delayed_work(&bdev->wq, 672 ((HZ / 100) < 1) ? 1 : HZ / 100); 673 } 674} 675 676static void ttm_bo_release(struct kref *kref) 677{ 678 struct ttm_buffer_object *bo = 679 container_of(kref, struct ttm_buffer_object, kref); 680 struct ttm_bo_device *bdev = bo->bdev; 681 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 682 683 if (likely(bo->vm_node != NULL)) { 684 rb_erase(&bo->vm_rb, &bdev->addr_space_rb); 685 drm_mm_put_block(bo->vm_node); 686 bo->vm_node = NULL; 687 } 688 write_unlock(&bdev->vm_lock); 689 ttm_mem_io_lock(man, false); 690 ttm_mem_io_free_vm(bo); 691 ttm_mem_io_unlock(man); 692 ttm_bo_cleanup_refs_or_queue(bo); 693 kref_put(&bo->list_kref, ttm_bo_release_list); 694 write_lock(&bdev->vm_lock); 695} 696 697void ttm_bo_unref(struct ttm_buffer_object **p_bo) 698{ 699 struct ttm_buffer_object *bo = *p_bo; 700 struct ttm_bo_device *bdev = bo->bdev; 701 702 *p_bo = NULL; 703 write_lock(&bdev->vm_lock); 704 kref_put(&bo->kref, ttm_bo_release); 705 write_unlock(&bdev->vm_lock); 706} 707EXPORT_SYMBOL(ttm_bo_unref); 708 709int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 710{ 711 return cancel_delayed_work_sync(&bdev->wq); 712} 713EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 714 715void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 716{ 717 if (resched) 718 schedule_delayed_work(&bdev->wq, 719 ((HZ / 100) < 1) ? 1 : HZ / 100); 720} 721EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 722 723static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 724 bool no_wait_reserve, bool no_wait_gpu) 725{ 726 struct ttm_bo_device *bdev = bo->bdev; 727 struct ttm_mem_reg evict_mem; 728 struct ttm_placement placement; 729 int ret = 0; 730 731 spin_lock(&bdev->fence_lock); 732 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 733 spin_unlock(&bdev->fence_lock); 734 735 if (unlikely(ret != 0)) { 736 if (ret != -ERESTARTSYS) { 737 pr_err("Failed to expire sync object before buffer eviction\n"); 738 } 739 goto out; 740 } 741 742 BUG_ON(!atomic_read(&bo->reserved)); 743 744 evict_mem = bo->mem; 745 evict_mem.mm_node = NULL; 746 evict_mem.bus.io_reserved_vm = false; 747 evict_mem.bus.io_reserved_count = 0; 748 749 placement.fpfn = 0; 750 placement.lpfn = 0; 751 placement.num_placement = 0; 752 placement.num_busy_placement = 0; 753 bdev->driver->evict_flags(bo, &placement); 754 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 755 no_wait_reserve, no_wait_gpu); 756 if (ret) { 757 if (ret != -ERESTARTSYS) { 758 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 759 bo); 760 ttm_bo_mem_space_debug(bo, &placement); 761 } 762 goto out; 763 } 764 765 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 766 no_wait_reserve, no_wait_gpu); 767 if (ret) { 768 if (ret != -ERESTARTSYS) 769 pr_err("Buffer eviction failed\n"); 770 ttm_bo_mem_put(bo, &evict_mem); 771 goto out; 772 } 773 bo->evicted = true; 774out: 775 return ret; 776} 777 778static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 779 uint32_t mem_type, 780 bool interruptible, bool no_wait_reserve, 781 bool no_wait_gpu) 782{ 783 struct ttm_bo_global *glob = bdev->glob; 784 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 785 struct ttm_buffer_object *bo; 786 int ret, put_count = 0; 787 788retry: 789 spin_lock(&glob->lru_lock); 790 if (list_empty(&man->lru)) { 791 spin_unlock(&glob->lru_lock); 792 return -EBUSY; 793 } 794 795 bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); 796 kref_get(&bo->list_kref); 797 798 if (!list_empty(&bo->ddestroy)) { 799 spin_unlock(&glob->lru_lock); 800 ret = ttm_bo_cleanup_refs(bo, interruptible, 801 no_wait_reserve, no_wait_gpu); 802 kref_put(&bo->list_kref, ttm_bo_release_list); 803 804 if (likely(ret == 0 || ret == -ERESTARTSYS)) 805 return ret; 806 807 goto retry; 808 } 809 810 ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); 811 812 if (unlikely(ret == -EBUSY)) { 813 spin_unlock(&glob->lru_lock); 814 if (likely(!no_wait_gpu)) 815 ret = ttm_bo_wait_unreserved(bo, interruptible); 816 817 kref_put(&bo->list_kref, ttm_bo_release_list); 818 819 /** 820 * We *need* to retry after releasing the lru lock. 821 */ 822 823 if (unlikely(ret != 0)) 824 return ret; 825 goto retry; 826 } 827 828 put_count = ttm_bo_del_from_lru(bo); 829 spin_unlock(&glob->lru_lock); 830 831 BUG_ON(ret != 0); 832 833 ttm_bo_list_ref_sub(bo, put_count, true); 834 835 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); 836 ttm_bo_unreserve(bo); 837 838 kref_put(&bo->list_kref, ttm_bo_release_list); 839 return ret; 840} 841 842void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 843{ 844 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 845 846 if (mem->mm_node) 847 (*man->func->put_node)(man, mem); 848} 849EXPORT_SYMBOL(ttm_bo_mem_put); 850 851/** 852 * Repeatedly evict memory from the LRU for @mem_type until we create enough 853 * space, or we've evicted everything and there isn't enough space. 854 */ 855static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 856 uint32_t mem_type, 857 struct ttm_placement *placement, 858 struct ttm_mem_reg *mem, 859 bool interruptible, 860 bool no_wait_reserve, 861 bool no_wait_gpu) 862{ 863 struct ttm_bo_device *bdev = bo->bdev; 864 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 865 int ret; 866 867 do { 868 ret = (*man->func->get_node)(man, bo, placement, mem); 869 if (unlikely(ret != 0)) 870 return ret; 871 if (mem->mm_node) 872 break; 873 ret = ttm_mem_evict_first(bdev, mem_type, interruptible, 874 no_wait_reserve, no_wait_gpu); 875 if (unlikely(ret != 0)) 876 return ret; 877 } while (1); 878 if (mem->mm_node == NULL) 879 return -ENOMEM; 880 mem->mem_type = mem_type; 881 return 0; 882} 883 884static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 885 uint32_t cur_placement, 886 uint32_t proposed_placement) 887{ 888 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 889 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 890 891 /** 892 * Keep current caching if possible. 893 */ 894 895 if ((cur_placement & caching) != 0) 896 result |= (cur_placement & caching); 897 else if ((man->default_caching & caching) != 0) 898 result |= man->default_caching; 899 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 900 result |= TTM_PL_FLAG_CACHED; 901 else if ((TTM_PL_FLAG_WC & caching) != 0) 902 result |= TTM_PL_FLAG_WC; 903 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 904 result |= TTM_PL_FLAG_UNCACHED; 905 906 return result; 907} 908 909static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 910 uint32_t mem_type, 911 uint32_t proposed_placement, 912 uint32_t *masked_placement) 913{ 914 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 915 916 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 917 return false; 918 919 if ((proposed_placement & man->available_caching) == 0) 920 return false; 921 922 cur_flags |= (proposed_placement & man->available_caching); 923 924 *masked_placement = cur_flags; 925 return true; 926} 927 928/** 929 * Creates space for memory region @mem according to its type. 930 * 931 * This function first searches for free space in compatible memory types in 932 * the priority order defined by the driver. If free space isn't found, then 933 * ttm_bo_mem_force_space is attempted in priority order to evict and find 934 * space. 935 */ 936int ttm_bo_mem_space(struct ttm_buffer_object *bo, 937 struct ttm_placement *placement, 938 struct ttm_mem_reg *mem, 939 bool interruptible, bool no_wait_reserve, 940 bool no_wait_gpu) 941{ 942 struct ttm_bo_device *bdev = bo->bdev; 943 struct ttm_mem_type_manager *man; 944 uint32_t mem_type = TTM_PL_SYSTEM; 945 uint32_t cur_flags = 0; 946 bool type_found = false; 947 bool type_ok = false; 948 bool has_erestartsys = false; 949 int i, ret; 950 951 mem->mm_node = NULL; 952 for (i = 0; i < placement->num_placement; ++i) { 953 ret = ttm_mem_type_from_flags(placement->placement[i], 954 &mem_type); 955 if (ret) 956 return ret; 957 man = &bdev->man[mem_type]; 958 959 type_ok = ttm_bo_mt_compatible(man, 960 mem_type, 961 placement->placement[i], 962 &cur_flags); 963 964 if (!type_ok) 965 continue; 966 967 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 968 cur_flags); 969 /* 970 * Use the access and other non-mapping-related flag bits from 971 * the memory placement flags to the current flags 972 */ 973 ttm_flag_masked(&cur_flags, placement->placement[i], 974 ~TTM_PL_MASK_MEMTYPE); 975 976 if (mem_type == TTM_PL_SYSTEM) 977 break; 978 979 if (man->has_type && man->use_type) { 980 type_found = true; 981 ret = (*man->func->get_node)(man, bo, placement, mem); 982 if (unlikely(ret)) 983 return ret; 984 } 985 if (mem->mm_node) 986 break; 987 } 988 989 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 990 mem->mem_type = mem_type; 991 mem->placement = cur_flags; 992 return 0; 993 } 994 995 if (!type_found) 996 return -EINVAL; 997 998 for (i = 0; i < placement->num_busy_placement; ++i) { 999 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 1000 &mem_type); 1001 if (ret) 1002 return ret; 1003 man = &bdev->man[mem_type]; 1004 if (!man->has_type) 1005 continue; 1006 if (!ttm_bo_mt_compatible(man, 1007 mem_type, 1008 placement->busy_placement[i], 1009 &cur_flags)) 1010 continue; 1011 1012 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1013 cur_flags); 1014 /* 1015 * Use the access and other non-mapping-related flag bits from 1016 * the memory placement flags to the current flags 1017 */ 1018 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 1019 ~TTM_PL_MASK_MEMTYPE); 1020 1021 1022 if (mem_type == TTM_PL_SYSTEM) { 1023 mem->mem_type = mem_type; 1024 mem->placement = cur_flags; 1025 mem->mm_node = NULL; 1026 return 0; 1027 } 1028 1029 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 1030 interruptible, no_wait_reserve, no_wait_gpu); 1031 if (ret == 0 && mem->mm_node) { 1032 mem->placement = cur_flags; 1033 return 0; 1034 } 1035 if (ret == -ERESTARTSYS) 1036 has_erestartsys = true; 1037 } 1038 ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 1039 return ret; 1040} 1041EXPORT_SYMBOL(ttm_bo_mem_space); 1042 1043int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait) 1044{ 1045 if ((atomic_read(&bo->cpu_writers) > 0) && no_wait) 1046 return -EBUSY; 1047 1048 return wait_event_interruptible(bo->event_queue, 1049 atomic_read(&bo->cpu_writers) == 0); 1050} 1051EXPORT_SYMBOL(ttm_bo_wait_cpu); 1052 1053int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1054 struct ttm_placement *placement, 1055 bool interruptible, bool no_wait_reserve, 1056 bool no_wait_gpu) 1057{ 1058 int ret = 0; 1059 struct ttm_mem_reg mem; 1060 struct ttm_bo_device *bdev = bo->bdev; 1061 1062 BUG_ON(!atomic_read(&bo->reserved)); 1063 1064 /* 1065 * FIXME: It's possible to pipeline buffer moves. 1066 * Have the driver move function wait for idle when necessary, 1067 * instead of doing it here. 1068 */ 1069 spin_lock(&bdev->fence_lock); 1070 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1071 spin_unlock(&bdev->fence_lock); 1072 if (ret) 1073 return ret; 1074 mem.num_pages = bo->num_pages; 1075 mem.size = mem.num_pages << PAGE_SHIFT; 1076 mem.page_alignment = bo->mem.page_alignment; 1077 mem.bus.io_reserved_vm = false; 1078 mem.bus.io_reserved_count = 0; 1079 /* 1080 * Determine where to move the buffer. 1081 */ 1082 ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu); 1083 if (ret) 1084 goto out_unlock; 1085 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu); 1086out_unlock: 1087 if (ret && mem.mm_node) 1088 ttm_bo_mem_put(bo, &mem); 1089 return ret; 1090} 1091 1092static int ttm_bo_mem_compat(struct ttm_placement *placement, 1093 struct ttm_mem_reg *mem) 1094{ 1095 int i; 1096 1097 if (mem->mm_node && placement->lpfn != 0 && 1098 (mem->start < placement->fpfn || 1099 mem->start + mem->num_pages > placement->lpfn)) 1100 return -1; 1101 1102 for (i = 0; i < placement->num_placement; i++) { 1103 if ((placement->placement[i] & mem->placement & 1104 TTM_PL_MASK_CACHING) && 1105 (placement->placement[i] & mem->placement & 1106 TTM_PL_MASK_MEM)) 1107 return i; 1108 } 1109 return -1; 1110} 1111 1112int ttm_bo_validate(struct ttm_buffer_object *bo, 1113 struct ttm_placement *placement, 1114 bool interruptible, bool no_wait_reserve, 1115 bool no_wait_gpu) 1116{ 1117 int ret; 1118 1119 BUG_ON(!atomic_read(&bo->reserved)); 1120 /* Check that range is valid */ 1121 if (placement->lpfn || placement->fpfn) 1122 if (placement->fpfn > placement->lpfn || 1123 (placement->lpfn - placement->fpfn) < bo->num_pages) 1124 return -EINVAL; 1125 /* 1126 * Check whether we need to move buffer. 1127 */ 1128 ret = ttm_bo_mem_compat(placement, &bo->mem); 1129 if (ret < 0) { 1130 ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu); 1131 if (ret) 1132 return ret; 1133 } else { 1134 /* 1135 * Use the access and other non-mapping-related flag bits from 1136 * the compatible memory placement flags to the active flags 1137 */ 1138 ttm_flag_masked(&bo->mem.placement, placement->placement[ret], 1139 ~TTM_PL_MASK_MEMTYPE); 1140 } 1141 /* 1142 * We might need to add a TTM. 1143 */ 1144 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1145 ret = ttm_bo_add_ttm(bo, true); 1146 if (ret) 1147 return ret; 1148 } 1149 return 0; 1150} 1151EXPORT_SYMBOL(ttm_bo_validate); 1152 1153int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1154 struct ttm_placement *placement) 1155{ 1156 BUG_ON((placement->fpfn || placement->lpfn) && 1157 (bo->mem.num_pages > (placement->lpfn - placement->fpfn))); 1158 1159 return 0; 1160} 1161 1162int ttm_bo_init(struct ttm_bo_device *bdev, 1163 struct ttm_buffer_object *bo, 1164 unsigned long size, 1165 enum ttm_bo_type type, 1166 struct ttm_placement *placement, 1167 uint32_t page_alignment, 1168 unsigned long buffer_start, 1169 bool interruptible, 1170 struct file *persistent_swap_storage, 1171 size_t acc_size, 1172 void (*destroy) (struct ttm_buffer_object *)) 1173{ 1174 int ret = 0; 1175 unsigned long num_pages; 1176 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1177 1178 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1179 if (ret) { 1180 pr_err("Out of kernel memory\n"); 1181 if (destroy) 1182 (*destroy)(bo); 1183 else 1184 kfree(bo); 1185 return -ENOMEM; 1186 } 1187 1188 size += buffer_start & ~PAGE_MASK; 1189 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1190 if (num_pages == 0) { 1191 pr_err("Illegal buffer object size\n"); 1192 if (destroy) 1193 (*destroy)(bo); 1194 else 1195 kfree(bo); 1196 ttm_mem_global_free(mem_glob, acc_size); 1197 return -EINVAL; 1198 } 1199 bo->destroy = destroy; 1200 1201 kref_init(&bo->kref); 1202 kref_init(&bo->list_kref); 1203 atomic_set(&bo->cpu_writers, 0); 1204 atomic_set(&bo->reserved, 1); 1205 init_waitqueue_head(&bo->event_queue); 1206 INIT_LIST_HEAD(&bo->lru); 1207 INIT_LIST_HEAD(&bo->ddestroy); 1208 INIT_LIST_HEAD(&bo->swap); 1209 INIT_LIST_HEAD(&bo->io_reserve_lru); 1210 bo->bdev = bdev; 1211 bo->glob = bdev->glob; 1212 bo->type = type; 1213 bo->num_pages = num_pages; 1214 bo->mem.size = num_pages << PAGE_SHIFT; 1215 bo->mem.mem_type = TTM_PL_SYSTEM; 1216 bo->mem.num_pages = bo->num_pages; 1217 bo->mem.mm_node = NULL; 1218 bo->mem.page_alignment = page_alignment; 1219 bo->mem.bus.io_reserved_vm = false; 1220 bo->mem.bus.io_reserved_count = 0; 1221 bo->buffer_start = buffer_start & PAGE_MASK; 1222 bo->priv_flags = 0; 1223 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1224 bo->seq_valid = false; 1225 bo->persistent_swap_storage = persistent_swap_storage; 1226 bo->acc_size = acc_size; 1227 atomic_inc(&bo->glob->bo_count); 1228 1229 ret = ttm_bo_check_placement(bo, placement); 1230 if (unlikely(ret != 0)) 1231 goto out_err; 1232 1233 /* 1234 * For ttm_bo_type_device buffers, allocate 1235 * address space from the device. 1236 */ 1237 if (bo->type == ttm_bo_type_device) { 1238 ret = ttm_bo_setup_vm(bo); 1239 if (ret) 1240 goto out_err; 1241 } 1242 1243 ret = ttm_bo_validate(bo, placement, interruptible, false, false); 1244 if (ret) 1245 goto out_err; 1246 1247 ttm_bo_unreserve(bo); 1248 return 0; 1249 1250out_err: 1251 ttm_bo_unreserve(bo); 1252 ttm_bo_unref(&bo); 1253 1254 return ret; 1255} 1256EXPORT_SYMBOL(ttm_bo_init); 1257 1258size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1259 unsigned long bo_size, 1260 unsigned struct_size) 1261{ 1262 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1263 size_t size = 0; 1264 1265 size += ttm_round_pot(struct_size); 1266 size += PAGE_ALIGN(npages * sizeof(void *)); 1267 size += ttm_round_pot(sizeof(struct ttm_tt)); 1268 return size; 1269} 1270EXPORT_SYMBOL(ttm_bo_acc_size); 1271 1272size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1273 unsigned long bo_size, 1274 unsigned struct_size) 1275{ 1276 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1277 size_t size = 0; 1278 1279 size += ttm_round_pot(struct_size); 1280 size += PAGE_ALIGN(npages * sizeof(void *)); 1281 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1282 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1283 return size; 1284} 1285EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1286 1287int ttm_bo_create(struct ttm_bo_device *bdev, 1288 unsigned long size, 1289 enum ttm_bo_type type, 1290 struct ttm_placement *placement, 1291 uint32_t page_alignment, 1292 unsigned long buffer_start, 1293 bool interruptible, 1294 struct file *persistent_swap_storage, 1295 struct ttm_buffer_object **p_bo) 1296{ 1297 struct ttm_buffer_object *bo; 1298 size_t acc_size; 1299 int ret; 1300 1301 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1302 if (unlikely(bo == NULL)) 1303 return -ENOMEM; 1304 1305 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1306 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1307 buffer_start, interruptible, 1308 persistent_swap_storage, acc_size, NULL); 1309 if (likely(ret == 0)) 1310 *p_bo = bo; 1311 1312 return ret; 1313} 1314EXPORT_SYMBOL(ttm_bo_create); 1315 1316static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1317 unsigned mem_type, bool allow_errors) 1318{ 1319 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1320 struct ttm_bo_global *glob = bdev->glob; 1321 int ret; 1322 1323 /* 1324 * Can't use standard list traversal since we're unlocking. 1325 */ 1326 1327 spin_lock(&glob->lru_lock); 1328 while (!list_empty(&man->lru)) { 1329 spin_unlock(&glob->lru_lock); 1330 ret = ttm_mem_evict_first(bdev, mem_type, false, false, false); 1331 if (ret) { 1332 if (allow_errors) { 1333 return ret; 1334 } else { 1335 pr_err("Cleanup eviction failed\n"); 1336 } 1337 } 1338 spin_lock(&glob->lru_lock); 1339 } 1340 spin_unlock(&glob->lru_lock); 1341 return 0; 1342} 1343 1344int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1345{ 1346 struct ttm_mem_type_manager *man; 1347 int ret = -EINVAL; 1348 1349 if (mem_type >= TTM_NUM_MEM_TYPES) { 1350 pr_err("Illegal memory type %d\n", mem_type); 1351 return ret; 1352 } 1353 man = &bdev->man[mem_type]; 1354 1355 if (!man->has_type) { 1356 pr_err("Trying to take down uninitialized memory manager type %u\n", 1357 mem_type); 1358 return ret; 1359 } 1360 1361 man->use_type = false; 1362 man->has_type = false; 1363 1364 ret = 0; 1365 if (mem_type > 0) { 1366 ttm_bo_force_list_clean(bdev, mem_type, false); 1367 1368 ret = (*man->func->takedown)(man); 1369 } 1370 1371 return ret; 1372} 1373EXPORT_SYMBOL(ttm_bo_clean_mm); 1374 1375int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1376{ 1377 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1378 1379 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1380 pr_err("Illegal memory manager memory type %u\n", mem_type); 1381 return -EINVAL; 1382 } 1383 1384 if (!man->has_type) { 1385 pr_err("Memory type %u has not been initialized\n", mem_type); 1386 return 0; 1387 } 1388 1389 return ttm_bo_force_list_clean(bdev, mem_type, true); 1390} 1391EXPORT_SYMBOL(ttm_bo_evict_mm); 1392 1393int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1394 unsigned long p_size) 1395{ 1396 int ret = -EINVAL; 1397 struct ttm_mem_type_manager *man; 1398 1399 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1400 man = &bdev->man[type]; 1401 BUG_ON(man->has_type); 1402 man->io_reserve_fastpath = true; 1403 man->use_io_reserve_lru = false; 1404 mutex_init(&man->io_reserve_mutex); 1405 INIT_LIST_HEAD(&man->io_reserve_lru); 1406 1407 ret = bdev->driver->init_mem_type(bdev, type, man); 1408 if (ret) 1409 return ret; 1410 man->bdev = bdev; 1411 1412 ret = 0; 1413 if (type != TTM_PL_SYSTEM) { 1414 ret = (*man->func->init)(man, p_size); 1415 if (ret) 1416 return ret; 1417 } 1418 man->has_type = true; 1419 man->use_type = true; 1420 man->size = p_size; 1421 1422 INIT_LIST_HEAD(&man->lru); 1423 1424 return 0; 1425} 1426EXPORT_SYMBOL(ttm_bo_init_mm); 1427 1428static void ttm_bo_global_kobj_release(struct kobject *kobj) 1429{ 1430 struct ttm_bo_global *glob = 1431 container_of(kobj, struct ttm_bo_global, kobj); 1432 1433 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1434 __free_page(glob->dummy_read_page); 1435 kfree(glob); 1436} 1437 1438void ttm_bo_global_release(struct drm_global_reference *ref) 1439{ 1440 struct ttm_bo_global *glob = ref->object; 1441 1442 kobject_del(&glob->kobj); 1443 kobject_put(&glob->kobj); 1444} 1445EXPORT_SYMBOL(ttm_bo_global_release); 1446 1447int ttm_bo_global_init(struct drm_global_reference *ref) 1448{ 1449 struct ttm_bo_global_ref *bo_ref = 1450 container_of(ref, struct ttm_bo_global_ref, ref); 1451 struct ttm_bo_global *glob = ref->object; 1452 int ret; 1453 1454 mutex_init(&glob->device_list_mutex); 1455 spin_lock_init(&glob->lru_lock); 1456 glob->mem_glob = bo_ref->mem_glob; 1457 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1458 1459 if (unlikely(glob->dummy_read_page == NULL)) { 1460 ret = -ENOMEM; 1461 goto out_no_drp; 1462 } 1463 1464 INIT_LIST_HEAD(&glob->swap_lru); 1465 INIT_LIST_HEAD(&glob->device_list); 1466 1467 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1468 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1469 if (unlikely(ret != 0)) { 1470 pr_err("Could not register buffer object swapout\n"); 1471 goto out_no_shrink; 1472 } 1473 1474 atomic_set(&glob->bo_count, 0); 1475 1476 ret = kobject_init_and_add( 1477 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1478 if (unlikely(ret != 0)) 1479 kobject_put(&glob->kobj); 1480 return ret; 1481out_no_shrink: 1482 __free_page(glob->dummy_read_page); 1483out_no_drp: 1484 kfree(glob); 1485 return ret; 1486} 1487EXPORT_SYMBOL(ttm_bo_global_init); 1488 1489 1490int ttm_bo_device_release(struct ttm_bo_device *bdev) 1491{ 1492 int ret = 0; 1493 unsigned i = TTM_NUM_MEM_TYPES; 1494 struct ttm_mem_type_manager *man; 1495 struct ttm_bo_global *glob = bdev->glob; 1496 1497 while (i--) { 1498 man = &bdev->man[i]; 1499 if (man->has_type) { 1500 man->use_type = false; 1501 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1502 ret = -EBUSY; 1503 pr_err("DRM memory manager type %d is not clean\n", 1504 i); 1505 } 1506 man->has_type = false; 1507 } 1508 } 1509 1510 mutex_lock(&glob->device_list_mutex); 1511 list_del(&bdev->device_list); 1512 mutex_unlock(&glob->device_list_mutex); 1513 1514 cancel_delayed_work_sync(&bdev->wq); 1515 1516 while (ttm_bo_delayed_delete(bdev, true)) 1517 ; 1518 1519 spin_lock(&glob->lru_lock); 1520 if (list_empty(&bdev->ddestroy)) 1521 TTM_DEBUG("Delayed destroy list was clean\n"); 1522 1523 if (list_empty(&bdev->man[0].lru)) 1524 TTM_DEBUG("Swap list was clean\n"); 1525 spin_unlock(&glob->lru_lock); 1526 1527 BUG_ON(!drm_mm_clean(&bdev->addr_space_mm)); 1528 write_lock(&bdev->vm_lock); 1529 drm_mm_takedown(&bdev->addr_space_mm); 1530 write_unlock(&bdev->vm_lock); 1531 1532 return ret; 1533} 1534EXPORT_SYMBOL(ttm_bo_device_release); 1535 1536int ttm_bo_device_init(struct ttm_bo_device *bdev, 1537 struct ttm_bo_global *glob, 1538 struct ttm_bo_driver *driver, 1539 uint64_t file_page_offset, 1540 bool need_dma32) 1541{ 1542 int ret = -EINVAL; 1543 1544 rwlock_init(&bdev->vm_lock); 1545 bdev->driver = driver; 1546 1547 memset(bdev->man, 0, sizeof(bdev->man)); 1548 1549 /* 1550 * Initialize the system memory buffer type. 1551 * Other types need to be driver / IOCTL initialized. 1552 */ 1553 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1554 if (unlikely(ret != 0)) 1555 goto out_no_sys; 1556 1557 bdev->addr_space_rb = RB_ROOT; 1558 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); 1559 if (unlikely(ret != 0)) 1560 goto out_no_addr_mm; 1561 1562 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1563 bdev->nice_mode = true; 1564 INIT_LIST_HEAD(&bdev->ddestroy); 1565 bdev->dev_mapping = NULL; 1566 bdev->glob = glob; 1567 bdev->need_dma32 = need_dma32; 1568 bdev->val_seq = 0; 1569 spin_lock_init(&bdev->fence_lock); 1570 mutex_lock(&glob->device_list_mutex); 1571 list_add_tail(&bdev->device_list, &glob->device_list); 1572 mutex_unlock(&glob->device_list_mutex); 1573 1574 return 0; 1575out_no_addr_mm: 1576 ttm_bo_clean_mm(bdev, 0); 1577out_no_sys: 1578 return ret; 1579} 1580EXPORT_SYMBOL(ttm_bo_device_init); 1581 1582/* 1583 * buffer object vm functions. 1584 */ 1585 1586bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1587{ 1588 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1589 1590 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1591 if (mem->mem_type == TTM_PL_SYSTEM) 1592 return false; 1593 1594 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1595 return false; 1596 1597 if (mem->placement & TTM_PL_FLAG_CACHED) 1598 return false; 1599 } 1600 return true; 1601} 1602 1603void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1604{ 1605 struct ttm_bo_device *bdev = bo->bdev; 1606 loff_t offset = (loff_t) bo->addr_space_offset; 1607 loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT; 1608 1609 if (!bdev->dev_mapping) 1610 return; 1611 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); 1612 ttm_mem_io_free_vm(bo); 1613} 1614 1615void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1616{ 1617 struct ttm_bo_device *bdev = bo->bdev; 1618 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1619 1620 ttm_mem_io_lock(man, false); 1621 ttm_bo_unmap_virtual_locked(bo); 1622 ttm_mem_io_unlock(man); 1623} 1624 1625 1626EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1627 1628static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1629{ 1630 struct ttm_bo_device *bdev = bo->bdev; 1631 struct rb_node **cur = &bdev->addr_space_rb.rb_node; 1632 struct rb_node *parent = NULL; 1633 struct ttm_buffer_object *cur_bo; 1634 unsigned long offset = bo->vm_node->start; 1635 unsigned long cur_offset; 1636 1637 while (*cur) { 1638 parent = *cur; 1639 cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb); 1640 cur_offset = cur_bo->vm_node->start; 1641 if (offset < cur_offset) 1642 cur = &parent->rb_left; 1643 else if (offset > cur_offset) 1644 cur = &parent->rb_right; 1645 else 1646 BUG(); 1647 } 1648 1649 rb_link_node(&bo->vm_rb, parent, cur); 1650 rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb); 1651} 1652 1653/** 1654 * ttm_bo_setup_vm: 1655 * 1656 * @bo: the buffer to allocate address space for 1657 * 1658 * Allocate address space in the drm device so that applications 1659 * can mmap the buffer and access the contents. This only 1660 * applies to ttm_bo_type_device objects as others are not 1661 * placed in the drm device address space. 1662 */ 1663 1664static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1665{ 1666 struct ttm_bo_device *bdev = bo->bdev; 1667 int ret; 1668 1669retry_pre_get: 1670 ret = drm_mm_pre_get(&bdev->addr_space_mm); 1671 if (unlikely(ret != 0)) 1672 return ret; 1673 1674 write_lock(&bdev->vm_lock); 1675 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, 1676 bo->mem.num_pages, 0, 0); 1677 1678 if (unlikely(bo->vm_node == NULL)) { 1679 ret = -ENOMEM; 1680 goto out_unlock; 1681 } 1682 1683 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, 1684 bo->mem.num_pages, 0); 1685 1686 if (unlikely(bo->vm_node == NULL)) { 1687 write_unlock(&bdev->vm_lock); 1688 goto retry_pre_get; 1689 } 1690 1691 ttm_bo_vm_insert_rb(bo); 1692 write_unlock(&bdev->vm_lock); 1693 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; 1694 1695 return 0; 1696out_unlock: 1697 write_unlock(&bdev->vm_lock); 1698 return ret; 1699} 1700 1701int ttm_bo_wait(struct ttm_buffer_object *bo, 1702 bool lazy, bool interruptible, bool no_wait) 1703{ 1704 struct ttm_bo_driver *driver = bo->bdev->driver; 1705 struct ttm_bo_device *bdev = bo->bdev; 1706 void *sync_obj; 1707 void *sync_obj_arg; 1708 int ret = 0; 1709 1710 if (likely(bo->sync_obj == NULL)) 1711 return 0; 1712 1713 while (bo->sync_obj) { 1714 1715 if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) { 1716 void *tmp_obj = bo->sync_obj; 1717 bo->sync_obj = NULL; 1718 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1719 spin_unlock(&bdev->fence_lock); 1720 driver->sync_obj_unref(&tmp_obj); 1721 spin_lock(&bdev->fence_lock); 1722 continue; 1723 } 1724 1725 if (no_wait) 1726 return -EBUSY; 1727 1728 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1729 sync_obj_arg = bo->sync_obj_arg; 1730 spin_unlock(&bdev->fence_lock); 1731 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg, 1732 lazy, interruptible); 1733 if (unlikely(ret != 0)) { 1734 driver->sync_obj_unref(&sync_obj); 1735 spin_lock(&bdev->fence_lock); 1736 return ret; 1737 } 1738 spin_lock(&bdev->fence_lock); 1739 if (likely(bo->sync_obj == sync_obj && 1740 bo->sync_obj_arg == sync_obj_arg)) { 1741 void *tmp_obj = bo->sync_obj; 1742 bo->sync_obj = NULL; 1743 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1744 &bo->priv_flags); 1745 spin_unlock(&bdev->fence_lock); 1746 driver->sync_obj_unref(&sync_obj); 1747 driver->sync_obj_unref(&tmp_obj); 1748 spin_lock(&bdev->fence_lock); 1749 } else { 1750 spin_unlock(&bdev->fence_lock); 1751 driver->sync_obj_unref(&sync_obj); 1752 spin_lock(&bdev->fence_lock); 1753 } 1754 } 1755 return 0; 1756} 1757EXPORT_SYMBOL(ttm_bo_wait); 1758 1759int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1760{ 1761 struct ttm_bo_device *bdev = bo->bdev; 1762 int ret = 0; 1763 1764 /* 1765 * Using ttm_bo_reserve makes sure the lru lists are updated. 1766 */ 1767 1768 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1769 if (unlikely(ret != 0)) 1770 return ret; 1771 spin_lock(&bdev->fence_lock); 1772 ret = ttm_bo_wait(bo, false, true, no_wait); 1773 spin_unlock(&bdev->fence_lock); 1774 if (likely(ret == 0)) 1775 atomic_inc(&bo->cpu_writers); 1776 ttm_bo_unreserve(bo); 1777 return ret; 1778} 1779EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1780 1781void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1782{ 1783 if (atomic_dec_and_test(&bo->cpu_writers)) 1784 wake_up_all(&bo->event_queue); 1785} 1786EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1787 1788/** 1789 * A buffer object shrink method that tries to swap out the first 1790 * buffer object on the bo_global::swap_lru list. 1791 */ 1792 1793static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1794{ 1795 struct ttm_bo_global *glob = 1796 container_of(shrink, struct ttm_bo_global, shrink); 1797 struct ttm_buffer_object *bo; 1798 int ret = -EBUSY; 1799 int put_count; 1800 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1801 1802 spin_lock(&glob->lru_lock); 1803 while (ret == -EBUSY) { 1804 if (unlikely(list_empty(&glob->swap_lru))) { 1805 spin_unlock(&glob->lru_lock); 1806 return -EBUSY; 1807 } 1808 1809 bo = list_first_entry(&glob->swap_lru, 1810 struct ttm_buffer_object, swap); 1811 kref_get(&bo->list_kref); 1812 1813 if (!list_empty(&bo->ddestroy)) { 1814 spin_unlock(&glob->lru_lock); 1815 (void) ttm_bo_cleanup_refs(bo, false, false, false); 1816 kref_put(&bo->list_kref, ttm_bo_release_list); 1817 spin_lock(&glob->lru_lock); 1818 continue; 1819 } 1820 1821 /** 1822 * Reserve buffer. Since we unlock while sleeping, we need 1823 * to re-check that nobody removed us from the swap-list while 1824 * we slept. 1825 */ 1826 1827 ret = ttm_bo_reserve_locked(bo, false, true, false, 0); 1828 if (unlikely(ret == -EBUSY)) { 1829 spin_unlock(&glob->lru_lock); 1830 ttm_bo_wait_unreserved(bo, false); 1831 kref_put(&bo->list_kref, ttm_bo_release_list); 1832 spin_lock(&glob->lru_lock); 1833 } 1834 } 1835 1836 BUG_ON(ret != 0); 1837 put_count = ttm_bo_del_from_lru(bo); 1838 spin_unlock(&glob->lru_lock); 1839 1840 ttm_bo_list_ref_sub(bo, put_count, true); 1841 1842 /** 1843 * Wait for GPU, then move to system cached. 1844 */ 1845 1846 spin_lock(&bo->bdev->fence_lock); 1847 ret = ttm_bo_wait(bo, false, false, false); 1848 spin_unlock(&bo->bdev->fence_lock); 1849 1850 if (unlikely(ret != 0)) 1851 goto out; 1852 1853 if ((bo->mem.placement & swap_placement) != swap_placement) { 1854 struct ttm_mem_reg evict_mem; 1855 1856 evict_mem = bo->mem; 1857 evict_mem.mm_node = NULL; 1858 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1859 evict_mem.mem_type = TTM_PL_SYSTEM; 1860 1861 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1862 false, false, false); 1863 if (unlikely(ret != 0)) 1864 goto out; 1865 } 1866 1867 ttm_bo_unmap_virtual(bo); 1868 1869 /** 1870 * Swap out. Buffer will be swapped in again as soon as 1871 * anyone tries to access a ttm page. 1872 */ 1873 1874 if (bo->bdev->driver->swap_notify) 1875 bo->bdev->driver->swap_notify(bo); 1876 1877 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1878out: 1879 1880 /** 1881 * 1882 * Unreserve without putting on LRU to avoid swapping out an 1883 * already swapped buffer. 1884 */ 1885 1886 atomic_set(&bo->reserved, 0); 1887 wake_up_all(&bo->event_queue); 1888 kref_put(&bo->list_kref, ttm_bo_release_list); 1889 return ret; 1890} 1891 1892void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1893{ 1894 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1895 ; 1896} 1897EXPORT_SYMBOL(ttm_bo_swapout_all); 1898