1771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse/* 2771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Copyright 2009 Jerome Glisse. 3771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * All Rights Reserved. 4771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 5771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Permission is hereby granted, free of charge, to any person obtaining a 6771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * copy of this software and associated documentation files (the 7771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * "Software"), to deal in the Software without restriction, including 8771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * without limitation the rights to use, copy, modify, merge, publish, 9771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * distribute, sub license, and/or sell copies of the Software, and to 10771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * permit persons to whom the Software is furnished to do so, subject to 11771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * the following conditions: 12771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 13771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * USE OR OTHER DEALINGS IN THE SOFTWARE. 20771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 21771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * The above copyright notice and this permission notice (including the 22771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * next paragraph) shall be included in all copies or substantial portions 23771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * of the Software. 24771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 25771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse */ 26771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse/* 27771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Authors: 28771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Jerome Glisse <glisse@freedesktop.org> 29771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Dave Airlie 31771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse */ 32771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <ttm/ttm_bo_api.h> 33771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <ttm/ttm_bo_driver.h> 34771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <ttm/ttm_placement.h> 35771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <ttm/ttm_module.h> 368d7cddcd7f9e847cec44851ca53fd3e81e846c49Pauli Nieminen#include <ttm/ttm_page_alloc.h> 37771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <drm/drmP.h> 38771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <drm/radeon_drm.h> 39fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie#include <linux/seq_file.h> 405a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 41771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include "radeon_reg.h" 42771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include "radeon.h" 43771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 44771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) 45771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 46fa8a123855e20068204982596b8fafceb1a67f0bDave Airliestatic int radeon_ttm_debugfs_init(struct radeon_device *rdev); 47fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie 48771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) 49771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 50771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_mman *mman; 51771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev; 52771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 53771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse mman = container_of(bdev, struct radeon_mman, bdev); 54771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev = container_of(mman, struct radeon_device, mman); 55771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return rdev; 56771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 57771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 58771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 59771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse/* 60771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Global memory. 61771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse */ 62ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airliestatic int radeon_ttm_mem_global_init(struct drm_global_reference *ref) 63771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 64771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return ttm_mem_global_init(ref->object); 65771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 66771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 67ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airliestatic void radeon_ttm_mem_global_release(struct drm_global_reference *ref) 68771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 69771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse ttm_mem_global_release(ref->object); 70771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 71771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 72771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_ttm_global_init(struct radeon_device *rdev) 73771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 74ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airlie struct drm_global_reference *global_ref; 75771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse int r; 76771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 77771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev->mman.mem_global_referenced = false; 78771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse global_ref = &rdev->mman.mem_global_ref; 79ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airlie global_ref->global_type = DRM_GLOBAL_TTM_MEM; 80771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse global_ref->size = sizeof(struct ttm_mem_global); 81771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse global_ref->init = &radeon_ttm_mem_global_init; 82771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse global_ref->release = &radeon_ttm_mem_global_release; 83ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airlie r = drm_global_item_ref(global_ref); 84771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r != 0) { 85a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom DRM_ERROR("Failed setting up TTM memory accounting " 86a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom "subsystem.\n"); 87771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 88771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 89a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom 90a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom rdev->mman.bo_global_ref.mem_glob = 91a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom rdev->mman.mem_global_ref.object; 92a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom global_ref = &rdev->mman.bo_global_ref.ref; 93ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airlie global_ref->global_type = DRM_GLOBAL_TTM_BO; 947f5f4db2d50ea1af8f160686d2e97bbfa5102b4fThomas Hellstrom global_ref->size = sizeof(struct ttm_bo_global); 95a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom global_ref->init = &ttm_bo_global_init; 96a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom global_ref->release = &ttm_bo_global_release; 97ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airlie r = drm_global_item_ref(global_ref); 98a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom if (r != 0) { 99a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom DRM_ERROR("Failed setting up TTM BO subsystem.\n"); 100ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airlie drm_global_item_unref(&rdev->mman.mem_global_ref); 101a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom return r; 102a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom } 103a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom 104771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev->mman.mem_global_referenced = true; 105771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 106771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 107771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 108771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic void radeon_ttm_global_fini(struct radeon_device *rdev) 109771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 110771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (rdev->mman.mem_global_referenced) { 111ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airlie drm_global_item_unref(&rdev->mman.bo_global_ref.ref); 112ba4420c224c2808f2661cf8428f43ceef7a73a4aDave Airlie drm_global_item_unref(&rdev->mman.mem_global_ref); 113771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev->mman.mem_global_referenced = false; 114771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 115771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 116771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 117771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 118771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 119771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 120771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 121771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 122771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 123771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_type_manager *man) 124771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 125771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev; 126771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 127771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev = radeon_get_rdev(bdev); 128771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 129771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse switch (type) { 130771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse case TTM_PL_SYSTEM: 131771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse /* System memory */ 132771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 133771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->available_caching = TTM_PL_MASK_CACHING; 134771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->default_caching = TTM_PL_FLAG_CACHED; 135771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse break; 136771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse case TTM_PL_TT: 137d961db75ce86a84f1f04e91ad1014653ed7d9f46Ben Skeggs man->func = &ttm_bo_manager_func; 138d594e46ace22afa1621254f6f669e65430048153Jerome Glisse man->gpu_offset = rdev->mc.gtt_start; 139771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->available_caching = TTM_PL_MASK_CACHING; 140771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->default_caching = TTM_PL_FLAG_CACHED; 14155c93278ec03c349af7b01933655b31c7f740df4Michel Dänzer man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; 142771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#if __OS_HAS_AGP 143771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (rdev->flags & RADEON_IS_AGP) { 144771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) { 145771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("AGP is not enabled for memory type %u\n", 146771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse (unsigned)type); 147771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -EINVAL; 148771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 14955c93278ec03c349af7b01933655b31c7f740df4Michel Dänzer if (!rdev->ddev->agp->cant_use_aperture) 1500a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 151771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->available_caching = TTM_PL_FLAG_UNCACHED | 152771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse TTM_PL_FLAG_WC; 153771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->default_caching = TTM_PL_FLAG_WC; 154771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 1550c321c79627189204d7d0bf65ab19f5ac419abedJerome Glisse#endif 156771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse break; 157771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse case TTM_PL_VRAM: 158771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse /* "On-card" video ram */ 159d961db75ce86a84f1f04e91ad1014653ed7d9f46Ben Skeggs man->func = &ttm_bo_manager_func; 160d594e46ace22afa1621254f6f669e65430048153Jerome Glisse man->gpu_offset = rdev->mc.vram_start; 161771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->flags = TTM_MEMTYPE_FLAG_FIXED | 162771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse TTM_MEMTYPE_FLAG_MAPPABLE; 163771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; 164771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse man->default_caching = TTM_PL_FLAG_WC; 165771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse break; 166771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse default: 167771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); 168771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -EINVAL; 169771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 170771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 171771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 172771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 173312ea8da049a1830aa50c6e00002e50e30df476eJerome Glissestatic void radeon_evict_flags(struct ttm_buffer_object *bo, 174312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse struct ttm_placement *placement) 175771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 176d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse struct radeon_bo *rbo; 177d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse static u32 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; 178d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse 179d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse if (!radeon_ttm_bo_is_radeon_bo(bo)) { 180d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse placement->fpfn = 0; 181d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse placement->lpfn = 0; 182d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse placement->placement = &placements; 183d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse placement->busy_placement = &placements; 184d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse placement->num_placement = 1; 185d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse placement->num_busy_placement = 1; 186d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse return; 187d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse } 188d03d858970a158fa82d9ba0d502bdea0bdb3ad33Jerome Glisse rbo = container_of(bo, struct radeon_bo, tbo); 189771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse switch (bo->mem.mem_type) { 190312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse case TTM_PL_VRAM: 191e32eb50dbe43862606a51caa94368ec6bd019434Christian König if (rbo->rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready == false) 1929270eb1b496cb002d75f49ef82c9ef4cbd22a5a0Dave Airlie radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 1939270eb1b496cb002d75f49ef82c9ef4cbd22a5a0Dave Airlie else 1949270eb1b496cb002d75f49ef82c9ef4cbd22a5a0Dave Airlie radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 195312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse break; 196312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse case TTM_PL_TT: 197771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse default: 198312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 199771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 200eaa5fd1a66fefd7cc918d80250d66fa48b10b81fJerome Glisse *placement = rbo->placement; 201771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 202771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 203771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) 204771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 205771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 206771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 207771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 208771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic void radeon_move_null(struct ttm_buffer_object *bo, 209771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg *new_mem) 210771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 211771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg *old_mem = &bo->mem; 212771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 213771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse BUG_ON(old_mem->mm_node != NULL); 214771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse *old_mem = *new_mem; 215771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse new_mem->mm_node = NULL; 216771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 217771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 218771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_move_blit(struct ttm_buffer_object *bo, 2199d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool evict, int no_wait_reserve, bool no_wait_gpu, 2209d87fa2138d06ff400551800d67d522625033e35Jerome Glisse struct ttm_mem_reg *new_mem, 2219d87fa2138d06ff400551800d67d522625033e35Jerome Glisse struct ttm_mem_reg *old_mem) 222771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 223771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev; 224771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse uint64_t old_start, new_start; 225771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_fence *fence; 2263000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher int r, i; 227771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 228771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev = radeon_get_rdev(bo->bdev); 22927cd77694bfa2e123cb7440507f8ddd762de6c38Alex Deucher r = radeon_fence_create(rdev, &fence, radeon_copy_ring_index(rdev)); 230771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(r)) { 231771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 232771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 233d961db75ce86a84f1f04e91ad1014653ed7d9f46Ben Skeggs old_start = old_mem->start << PAGE_SHIFT; 234d961db75ce86a84f1f04e91ad1014653ed7d9f46Ben Skeggs new_start = new_mem->start << PAGE_SHIFT; 235771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 236771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse switch (old_mem->mem_type) { 237771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse case TTM_PL_VRAM: 238d594e46ace22afa1621254f6f669e65430048153Jerome Glisse old_start += rdev->mc.vram_start; 239771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse break; 240771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse case TTM_PL_TT: 241d594e46ace22afa1621254f6f669e65430048153Jerome Glisse old_start += rdev->mc.gtt_start; 242771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse break; 243771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse default: 244771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 245771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -EINVAL; 246771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 247771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse switch (new_mem->mem_type) { 248771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse case TTM_PL_VRAM: 249d594e46ace22afa1621254f6f669e65430048153Jerome Glisse new_start += rdev->mc.vram_start; 250771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse break; 251771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse case TTM_PL_TT: 252d594e46ace22afa1621254f6f669e65430048153Jerome Glisse new_start += rdev->mc.gtt_start; 253771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse break; 254771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse default: 255771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 256771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -EINVAL; 257771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 25827cd77694bfa2e123cb7440507f8ddd762de6c38Alex Deucher if (!rdev->ring[radeon_copy_ring_index(rdev)].ready) { 2593000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher DRM_ERROR("Trying to move memory with ring turned off.\n"); 260771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -EINVAL; 261771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 262003cefe0c238e683a29d2207dba945b508cd45b7Alex Deucher 263003cefe0c238e683a29d2207dba945b508cd45b7Alex Deucher BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); 264003cefe0c238e683a29d2207dba945b508cd45b7Alex Deucher 2653000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher /* sync other rings */ 2663000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher if (rdev->family >= CHIP_R600) { 2673000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher for (i = 0; i < RADEON_NUM_RINGS; ++i) { 2683000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher /* no need to sync to our own or unused rings */ 26927cd77694bfa2e123cb7440507f8ddd762de6c38Alex Deucher if (i == radeon_copy_ring_index(rdev) || !rdev->ring[i].ready) 2703000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher continue; 2713000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher 2723000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher if (!fence->semaphore) { 2733000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher r = radeon_semaphore_create(rdev, &fence->semaphore); 2743000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher /* FIXME: handle semaphore error */ 2753000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher if (r) 2763000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher continue; 2773000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher } 2783000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher 2793000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher r = radeon_ring_lock(rdev, &rdev->ring[i], 3); 2803000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher /* FIXME: handle ring lock error */ 2813000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher if (r) 2823000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher continue; 2833000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher radeon_semaphore_emit_signal(rdev, i, fence->semaphore); 2843000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher radeon_ring_unlock_commit(rdev, &rdev->ring[i]); 2853000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher 28627cd77694bfa2e123cb7440507f8ddd762de6c38Alex Deucher r = radeon_ring_lock(rdev, &rdev->ring[radeon_copy_ring_index(rdev)], 3); 2873000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher /* FIXME: handle ring lock error */ 2883000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher if (r) 2893000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher continue; 29027cd77694bfa2e123cb7440507f8ddd762de6c38Alex Deucher radeon_semaphore_emit_wait(rdev, radeon_copy_ring_index(rdev), fence->semaphore); 29127cd77694bfa2e123cb7440507f8ddd762de6c38Alex Deucher radeon_ring_unlock_commit(rdev, &rdev->ring[radeon_copy_ring_index(rdev)]); 2923000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher } 2933000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher } 2943000bf393302a8c786e9ebfc778050cb0d6226c4Alex Deucher 295003cefe0c238e683a29d2207dba945b508cd45b7Alex Deucher r = radeon_copy(rdev, old_start, new_start, 296003cefe0c238e683a29d2207dba945b508cd45b7Alex Deucher new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ 297003cefe0c238e683a29d2207dba945b508cd45b7Alex Deucher fence); 298771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse /* FIXME: handle copy error */ 299771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL, 3009d87fa2138d06ff400551800d67d522625033e35Jerome Glisse evict, no_wait_reserve, no_wait_gpu, new_mem); 301771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_fence_unref(&fence); 302771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 303771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 304771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 305771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_move_vram_ram(struct ttm_buffer_object *bo, 3069d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool evict, bool interruptible, 3079d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool no_wait_reserve, bool no_wait_gpu, 308771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg *new_mem) 309771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 310771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev; 311771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg *old_mem = &bo->mem; 312771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg tmp_mem; 313312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse u32 placements; 314312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse struct ttm_placement placement; 315771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse int r; 316771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 317771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev = radeon_get_rdev(bo->bdev); 318771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse tmp_mem = *new_mem; 319771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse tmp_mem.mm_node = NULL; 320312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.fpfn = 0; 321312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.lpfn = 0; 322312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.num_placement = 1; 323312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.placement = &placements; 324312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.num_busy_placement = 1; 325312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.busy_placement = &placements; 326312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 327312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse r = ttm_bo_mem_space(bo, &placement, &tmp_mem, 3289d87fa2138d06ff400551800d67d522625033e35Jerome Glisse interruptible, no_wait_reserve, no_wait_gpu); 329771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(r)) { 330771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 331771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 332df67bed92fa86ef926da8b62a6da68722388ff72Dave Airlie 333df67bed92fa86ef926da8b62a6da68722388ff72Dave Airlie r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); 334df67bed92fa86ef926da8b62a6da68722388ff72Dave Airlie if (unlikely(r)) { 335df67bed92fa86ef926da8b62a6da68722388ff72Dave Airlie goto out_cleanup; 336df67bed92fa86ef926da8b62a6da68722388ff72Dave Airlie } 337df67bed92fa86ef926da8b62a6da68722388ff72Dave Airlie 338771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = ttm_tt_bind(bo->ttm, &tmp_mem); 339771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(r)) { 340771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse goto out_cleanup; 341771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 3429d87fa2138d06ff400551800d67d522625033e35Jerome Glisse r = radeon_move_blit(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem, old_mem); 343771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(r)) { 344771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse goto out_cleanup; 345771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 3469d87fa2138d06ff400551800d67d522625033e35Jerome Glisse r = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem); 347771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisseout_cleanup: 34842311ff90dc8746bd81427b2ed6efda9af791b77Ben Skeggs ttm_bo_mem_put(bo, &tmp_mem); 349771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 350771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 351771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 352771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_move_ram_vram(struct ttm_buffer_object *bo, 3539d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool evict, bool interruptible, 3549d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool no_wait_reserve, bool no_wait_gpu, 355771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg *new_mem) 356771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 357771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev; 358771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg *old_mem = &bo->mem; 359771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg tmp_mem; 360312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse struct ttm_placement placement; 361312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse u32 placements; 362771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse int r; 363771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 364771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev = radeon_get_rdev(bo->bdev); 365771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse tmp_mem = *new_mem; 366771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse tmp_mem.mm_node = NULL; 367312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.fpfn = 0; 368312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.lpfn = 0; 369312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.num_placement = 1; 370312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.placement = &placements; 371312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.num_busy_placement = 1; 372312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placement.busy_placement = &placements; 373312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 3749d87fa2138d06ff400551800d67d522625033e35Jerome Glisse r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_reserve, no_wait_gpu); 375771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(r)) { 376771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 377771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 3789d87fa2138d06ff400551800d67d522625033e35Jerome Glisse r = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem); 379771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(r)) { 380771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse goto out_cleanup; 381771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 3829d87fa2138d06ff400551800d67d522625033e35Jerome Glisse r = radeon_move_blit(bo, true, no_wait_reserve, no_wait_gpu, new_mem, old_mem); 383771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(r)) { 384771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse goto out_cleanup; 385771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 386771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisseout_cleanup: 38742311ff90dc8746bd81427b2ed6efda9af791b77Ben Skeggs ttm_bo_mem_put(bo, &tmp_mem); 388771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 389771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 390771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 391771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_bo_move(struct ttm_buffer_object *bo, 3929d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool evict, bool interruptible, 3939d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool no_wait_reserve, bool no_wait_gpu, 3949d87fa2138d06ff400551800d67d522625033e35Jerome Glisse struct ttm_mem_reg *new_mem) 395771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 396771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev; 397771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_mem_reg *old_mem = &bo->mem; 398771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse int r; 399771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 400771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev = radeon_get_rdev(bo->bdev); 401771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 402771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_move_null(bo, new_mem); 403771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 404771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 405771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if ((old_mem->mem_type == TTM_PL_TT && 406771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse new_mem->mem_type == TTM_PL_SYSTEM) || 407771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse (old_mem->mem_type == TTM_PL_SYSTEM && 408771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse new_mem->mem_type == TTM_PL_TT)) { 409af901ca181d92aac3a7dc265144a9081a86d8f39André Goddard Rosa /* bind is enough */ 410771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_move_null(bo, new_mem); 411771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 412771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 41327cd77694bfa2e123cb7440507f8ddd762de6c38Alex Deucher if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || 41427cd77694bfa2e123cb7440507f8ddd762de6c38Alex Deucher rdev->asic->copy.copy == NULL) { 415771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse /* use memcpy */ 4161ab2e1059916b917af19e4137a4222988bd7a169Michel Dänzer goto memcpy; 417771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 418771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 419771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (old_mem->mem_type == TTM_PL_VRAM && 420771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse new_mem->mem_type == TTM_PL_SYSTEM) { 4211ab2e1059916b917af19e4137a4222988bd7a169Michel Dänzer r = radeon_move_vram_ram(bo, evict, interruptible, 4229d87fa2138d06ff400551800d67d522625033e35Jerome Glisse no_wait_reserve, no_wait_gpu, new_mem); 423771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } else if (old_mem->mem_type == TTM_PL_SYSTEM && 424771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse new_mem->mem_type == TTM_PL_VRAM) { 4251ab2e1059916b917af19e4137a4222988bd7a169Michel Dänzer r = radeon_move_ram_vram(bo, evict, interruptible, 4269d87fa2138d06ff400551800d67d522625033e35Jerome Glisse no_wait_reserve, no_wait_gpu, new_mem); 427771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } else { 4289d87fa2138d06ff400551800d67d522625033e35Jerome Glisse r = radeon_move_blit(bo, evict, no_wait_reserve, no_wait_gpu, new_mem, old_mem); 429771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 4301ab2e1059916b917af19e4137a4222988bd7a169Michel Dänzer 4311ab2e1059916b917af19e4137a4222988bd7a169Michel Dänzer if (r) { 4321ab2e1059916b917af19e4137a4222988bd7a169Michel Dänzermemcpy: 4339d87fa2138d06ff400551800d67d522625033e35Jerome Glisse r = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 4341ab2e1059916b917af19e4137a4222988bd7a169Michel Dänzer } 435771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 436771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 437771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 4380a2d50e3a8faaf36cde36920431586090411ea15Jerome Glissestatic int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 4390a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse{ 4400a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 4410a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse struct radeon_device *rdev = radeon_get_rdev(bdev); 4420a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse 4430a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse mem->bus.addr = NULL; 4440a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse mem->bus.offset = 0; 4450a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse mem->bus.size = mem->num_pages << PAGE_SHIFT; 4460a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse mem->bus.base = 0; 4470a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse mem->bus.is_iomem = false; 4480a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 4490a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse return -EINVAL; 4500a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse switch (mem->mem_type) { 4510a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse case TTM_PL_SYSTEM: 4520a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse /* system memory */ 4530a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse return 0; 4540a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse case TTM_PL_TT: 4550a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse#if __OS_HAS_AGP 4560a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse if (rdev->flags & RADEON_IS_AGP) { 4570a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse /* RADEON_IS_AGP is set only if AGP is active */ 458d961db75ce86a84f1f04e91ad1014653ed7d9f46Ben Skeggs mem->bus.offset = mem->start << PAGE_SHIFT; 4590a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse mem->bus.base = rdev->mc.agp_base; 460365048ff7f977c5983d67b63c47502c5964840e9Michel Dänzer mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; 4610a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse } 4620a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse#endif 4630a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse break; 4640a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse case TTM_PL_VRAM: 465d961db75ce86a84f1f04e91ad1014653ed7d9f46Ben Skeggs mem->bus.offset = mem->start << PAGE_SHIFT; 4660a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse /* check if it's visible */ 4670a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size) 4680a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse return -EINVAL; 4690a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse mem->bus.base = rdev->mc.aper_base; 4700a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse mem->bus.is_iomem = true; 471ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook#ifdef __alpha__ 472ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook /* 473ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook * Alpha: use bus.addr to hold the ioremap() return, 474ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook * so we can modify bus.base below. 475ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook */ 476ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook if (mem->placement & TTM_PL_FLAG_WC) 477ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook mem->bus.addr = 478ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook ioremap_wc(mem->bus.base + mem->bus.offset, 479ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook mem->bus.size); 480ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook else 481ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook mem->bus.addr = 482ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook ioremap_nocache(mem->bus.base + mem->bus.offset, 483ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook mem->bus.size); 484ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook 485ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook /* 486ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook * Alpha: Use just the bus offset plus 487ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook * the hose/domain memory base for bus.base. 488ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook * It then can be used to build PTEs for VRAM 489ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook * access, as done in ttm_bo_vm_fault(). 490ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook */ 491ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook mem->bus.base = (mem->bus.base & 0x0ffffffffUL) + 492ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook rdev->ddev->hose->dense_mem_base; 493ffb57c4b8612c31204b06713770f6df4b8a94e4fJay Estabrook#endif 4940a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse break; 4950a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse default: 4960a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse return -EINVAL; 4970a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse } 4980a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse return 0; 4990a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse} 5000a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse 5010a2d50e3a8faaf36cde36920431586090411ea15Jerome Glissestatic void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 5020a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse{ 5030a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse} 5040a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse 505771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_sync_obj_wait(void *sync_obj, void *sync_arg, 506771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse bool lazy, bool interruptible) 507771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 508771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible); 509771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 510771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 511771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_sync_obj_flush(void *sync_obj, void *sync_arg) 512771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 513771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 514771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 515771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 516771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic void radeon_sync_obj_unref(void **sync_obj) 517771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 518771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_fence_unref((struct radeon_fence **)sync_obj); 519771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 520771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 521771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic void *radeon_sync_obj_ref(void *sync_obj) 522771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 523771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return radeon_fence_ref((struct radeon_fence *)sync_obj); 524771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 525771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 526771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic bool radeon_sync_obj_signaled(void *sync_obj, void *sync_arg) 527771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 528771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return radeon_fence_signaled((struct radeon_fence *)sync_obj); 529771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 530771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 531649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse/* 532649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse * TTM backend functions. 533649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse */ 534649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glissestruct radeon_ttm_tt { 5358e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse struct ttm_dma_tt ttm; 536649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse struct radeon_device *rdev; 537649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse u64 offset; 538649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse}; 539649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 540649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glissestatic int radeon_ttm_backend_bind(struct ttm_tt *ttm, 541649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse struct ttm_mem_reg *bo_mem) 542649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse{ 5438e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse struct radeon_ttm_tt *gtt = (void*)ttm; 544649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse int r; 545649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 546649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); 547649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse if (!ttm->num_pages) { 548649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", 549649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse ttm->num_pages, bo_mem, ttm); 550649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse } 551649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse r = radeon_gart_bind(gtt->rdev, gtt->offset, 5528e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse ttm->num_pages, ttm->pages, gtt->ttm.dma_address); 553649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse if (r) { 554649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse DRM_ERROR("failed to bind %lu pages at 0x%08X\n", 555649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse ttm->num_pages, (unsigned)gtt->offset); 556649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse return r; 557649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse } 558649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse return 0; 559649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse} 560649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 561649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glissestatic int radeon_ttm_backend_unbind(struct ttm_tt *ttm) 562649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse{ 5638e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse struct radeon_ttm_tt *gtt = (void *)ttm; 564649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 565649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse radeon_gart_unbind(gtt->rdev, gtt->offset, ttm->num_pages); 566649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse return 0; 567649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse} 568649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 569649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glissestatic void radeon_ttm_backend_destroy(struct ttm_tt *ttm) 570649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse{ 5718e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse struct radeon_ttm_tt *gtt = (void *)ttm; 572649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 5738e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse ttm_dma_tt_fini(>t->ttm); 574649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse kfree(gtt); 575649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse} 576649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 577649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glissestatic struct ttm_backend_func radeon_backend_func = { 578649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse .bind = &radeon_ttm_backend_bind, 579649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse .unbind = &radeon_ttm_backend_unbind, 580649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse .destroy = &radeon_ttm_backend_destroy, 581649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse}; 582649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 583649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glissestruct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev, 584649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse unsigned long size, uint32_t page_flags, 585649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse struct page *dummy_read_page) 586649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse{ 587649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse struct radeon_device *rdev; 588649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse struct radeon_ttm_tt *gtt; 589649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 590649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse rdev = radeon_get_rdev(bdev); 591649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse#if __OS_HAS_AGP 592649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse if (rdev->flags & RADEON_IS_AGP) { 593649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse return ttm_agp_tt_create(bdev, rdev->ddev->agp->bridge, 594649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse size, page_flags, dummy_read_page); 595649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse } 596649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse#endif 597649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 598649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL); 599649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse if (gtt == NULL) { 600649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse return NULL; 601649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse } 6028e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse gtt->ttm.ttm.func = &radeon_backend_func; 603649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse gtt->rdev = rdev; 6048e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse if (ttm_dma_tt_init(>t->ttm, bdev, size, page_flags, dummy_read_page)) { 6058e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse kfree(gtt); 606649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse return NULL; 607649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse } 6088e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse return >t->ttm.ttm; 609649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse} 610649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 611c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilkstatic int radeon_ttm_tt_populate(struct ttm_tt *ttm) 612c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk{ 613c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk struct radeon_device *rdev; 6148e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse struct radeon_ttm_tt *gtt = (void *)ttm; 615c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk unsigned i; 616c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk int r; 617c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 618c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk if (ttm->state != tt_unpopulated) 619c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk return 0; 620c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 621c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk rdev = radeon_get_rdev(ttm->bdev); 622dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse#if __OS_HAS_AGP 623dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse if (rdev->flags & RADEON_IS_AGP) { 624dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse return ttm_agp_tt_populate(ttm); 625dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse } 626dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse#endif 627c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 628c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk#ifdef CONFIG_SWIOTLB 629c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk if (swiotlb_nr_tbl()) { 6308e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse return ttm_dma_populate(>t->ttm, rdev->dev); 631c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 632c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk#endif 633c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 634c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk r = ttm_pool_populate(ttm); 635c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk if (r) { 636c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk return r; 637c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 638c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 639c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk for (i = 0; i < ttm->num_pages; i++) { 6408e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i], 6418e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse 0, PAGE_SIZE, 6428e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse PCI_DMA_BIDIRECTIONAL); 6438e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) { 644c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk while (--i) { 6458e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], 646c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 6478e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse gtt->ttm.dma_address[i] = 0; 648c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 649c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk ttm_pool_unpopulate(ttm); 650c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk return -EFAULT; 651c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 652c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 653c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk return 0; 654c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk} 655c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 656c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilkstatic void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) 657c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk{ 658c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk struct radeon_device *rdev; 6598e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse struct radeon_ttm_tt *gtt = (void *)ttm; 660c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk unsigned i; 661c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 662c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk rdev = radeon_get_rdev(ttm->bdev); 663dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse#if __OS_HAS_AGP 664dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse if (rdev->flags & RADEON_IS_AGP) { 665dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse ttm_agp_tt_unpopulate(ttm); 666dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse return; 667dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse } 668dea7e0ac45fd28f90bbc38ff226d36a9f788efbfJerome Glisse#endif 669c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 670c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk#ifdef CONFIG_SWIOTLB 671c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk if (swiotlb_nr_tbl()) { 6728e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse ttm_dma_unpopulate(>t->ttm, rdev->dev); 673c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk return; 674c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 675c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk#endif 676c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 677c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk for (i = 0; i < ttm->num_pages; i++) { 6788e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse if (gtt->ttm.dma_address[i]) { 6798e7e70522d760c4ccd4cd370ebfa0ba69e006c6eJerome Glisse pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], 680c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 681c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 682c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 683c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk 684c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk ttm_pool_unpopulate(ttm); 685c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk} 686649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse 687771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic struct ttm_bo_driver radeon_bo_driver = { 688649bf3ca77343e3be1e0af8e21356fa569b1abd9Jerome Glisse .ttm_tt_create = &radeon_ttm_tt_create, 689c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk .ttm_tt_populate = &radeon_ttm_tt_populate, 690c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, 691771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .invalidate_caches = &radeon_invalidate_caches, 692771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .init_mem_type = &radeon_init_mem_type, 693771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .evict_flags = &radeon_evict_flags, 694771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .move = &radeon_bo_move, 695771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .verify_access = &radeon_verify_access, 696771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .sync_obj_signaled = &radeon_sync_obj_signaled, 697771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .sync_obj_wait = &radeon_sync_obj_wait, 698771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .sync_obj_flush = &radeon_sync_obj_flush, 699771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .sync_obj_unref = &radeon_sync_obj_unref, 700771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse .sync_obj_ref = &radeon_sync_obj_ref, 701e024e11070a0a0dc7163ce1ec2da354a638bdbedDave Airlie .move_notify = &radeon_bo_move_notify, 702e024e11070a0a0dc7163ce1ec2da354a638bdbedDave Airlie .fault_reserve_notify = &radeon_bo_fault_reserve_notify, 7030a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse .io_mem_reserve = &radeon_ttm_io_mem_reserve, 7040a2d50e3a8faaf36cde36920431586090411ea15Jerome Glisse .io_mem_free = &radeon_ttm_io_mem_free, 705771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse}; 706771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 707771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisseint radeon_ttm_init(struct radeon_device *rdev) 708771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 709771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse int r; 710771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 711771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = radeon_ttm_global_init(rdev); 712771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 713771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 714771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 715771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse /* No others user of address space so set it to 0 */ 716771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = ttm_bo_device_init(&rdev->mman.bdev, 717a987fcaa805fcb24ba885c2e29fd4fdb6816f08fThomas Hellstrom rdev->mman.bo_global_ref.ref.object, 718ad49f501867cba87e1e45e5ebae0b12435d68bf1Dave Airlie &radeon_bo_driver, DRM_FILE_PAGE_OFFSET, 719ad49f501867cba87e1e45e5ebae0b12435d68bf1Dave Airlie rdev->need_dma32); 720771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 721771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 722771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 723771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 7240a0c7596c643239e8d4c3eaaba43b74a96f2411eJerome Glisse rdev->mman.initialized = true; 7254c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM, 726312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse rdev->mc.real_vram_size >> PAGE_SHIFT); 727771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 728771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("Failed initializing VRAM heap.\n"); 729771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 730771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 731441921d5309cfe098747d9840fd71bdc6ca2a93bDaniel Vetter r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, 7324c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse RADEON_GEM_DOMAIN_VRAM, 7334c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse &rdev->stollen_vga_memory); 734771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 735771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 736771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 7374c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse r = radeon_bo_reserve(rdev->stollen_vga_memory, false); 7384c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse if (r) 7394c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse return r; 7404c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); 7414c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse radeon_bo_unreserve(rdev->stollen_vga_memory); 742771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 7434c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse radeon_bo_unref(&rdev->stollen_vga_memory); 744771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 745771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 746771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_INFO("radeon: %uM of VRAM memory ready\n", 7473ce0a23d2d253185df24e22e3d5f89800bb3dd1cJerome Glisse (unsigned)rdev->mc.real_vram_size / (1024 * 1024)); 7484c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, 749312ea8da049a1830aa50c6e00002e50e30df476eJerome Glisse rdev->mc.gtt_size >> PAGE_SHIFT); 750771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 751771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("Failed initializing GTT heap.\n"); 752771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 753771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 754771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_INFO("radeon: %uM of GTT memory ready.\n", 7553ce0a23d2d253185df24e22e3d5f89800bb3dd1cJerome Glisse (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); 756771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) { 757771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping; 758771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 759fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie 760fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie r = radeon_ttm_debugfs_init(rdev); 761fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie if (r) { 762fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie DRM_ERROR("Failed to init debugfs\n"); 763fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie return r; 764fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie } 765771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 766771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 767771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 768771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissevoid radeon_ttm_fini(struct radeon_device *rdev) 769771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 7704c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse int r; 7714c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse 7720a0c7596c643239e8d4c3eaaba43b74a96f2411eJerome Glisse if (!rdev->mman.initialized) 7730a0c7596c643239e8d4c3eaaba43b74a96f2411eJerome Glisse return; 774771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (rdev->stollen_vga_memory) { 7754c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse r = radeon_bo_reserve(rdev->stollen_vga_memory, false); 7764c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse if (r == 0) { 7774c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse radeon_bo_unpin(rdev->stollen_vga_memory); 7784c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse radeon_bo_unreserve(rdev->stollen_vga_memory); 7794c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse } 7804c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse radeon_bo_unref(&rdev->stollen_vga_memory); 781771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 782771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM); 783771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT); 784771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse ttm_bo_device_release(&rdev->mman.bdev); 785771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_gart_fini(rdev); 786771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_ttm_global_fini(rdev); 7870a0c7596c643239e8d4c3eaaba43b74a96f2411eJerome Glisse rdev->mman.initialized = false; 788771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_INFO("radeon: ttm finalized\n"); 789771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 790771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 7915359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie/* this should only be called at bootup or when userspace 7925359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie * isn't running */ 7935359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlievoid radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) 7945359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie{ 7955359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie struct ttm_mem_type_manager *man; 7965359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie 7975359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie if (!rdev->mman.initialized) 7985359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie return; 7995359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie 8005359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie man = &rdev->mman.bdev.man[TTM_PL_VRAM]; 8015359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 8025359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie man->size = size >> PAGE_SHIFT; 8035359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie} 8045359533801e3dd3abca5b7d3d985b0b33fd9fe8bDave Airlie 805771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic struct vm_operations_struct radeon_ttm_vm_ops; 806f0f37e2f77731b3473fa6bd5ee53255d9a9cdb40Alexey Dobriyanstatic const struct vm_operations_struct *ttm_vm_ops = NULL; 807771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 808771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 809771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 810771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct ttm_buffer_object *bo; 8115876dd249e8e47c730cac090bf6edd88e5f04327Matthew Garrett struct radeon_device *rdev; 812771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse int r; 813771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 8145876dd249e8e47c730cac090bf6edd88e5f04327Matthew Garrett bo = (struct ttm_buffer_object *)vma->vm_private_data; 815771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (bo == NULL) { 816771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return VM_FAULT_NOPAGE; 817771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 8185876dd249e8e47c730cac090bf6edd88e5f04327Matthew Garrett rdev = radeon_get_rdev(bo->bdev); 8195876dd249e8e47c730cac090bf6edd88e5f04327Matthew Garrett mutex_lock(&rdev->vram_mutex); 820771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = ttm_vm_ops->fault(vma, vmf); 8215876dd249e8e47c730cac090bf6edd88e5f04327Matthew Garrett mutex_unlock(&rdev->vram_mutex); 822771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 823771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 824771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 825771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisseint radeon_mmap(struct file *filp, struct vm_area_struct *vma) 826771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 827771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct drm_file *file_priv; 828771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev; 829771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse int r; 830771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 831771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) { 832771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return drm_mmap(filp, vma); 833771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 834771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 83540b3be3fcaf8e0d8f7d8cee266dc4af3251d814bJoe Perches file_priv = filp->private_data; 836771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse rdev = file_priv->minor->dev->dev_private; 837771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (rdev == NULL) { 838771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -EINVAL; 839771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 840771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); 841771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(r != 0)) { 842771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 843771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 844771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (unlikely(ttm_vm_ops == NULL)) { 845771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse ttm_vm_ops = vma->vm_ops; 846771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_ttm_vm_ops = *ttm_vm_ops; 847771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_ttm_vm_ops.fault = &radeon_ttm_fault; 848771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 849771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse vma->vm_ops = &radeon_ttm_vm_ops; 850771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 851771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 852771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 853771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 854fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie#define RADEON_DEBUGFS_MEM_TYPES 2 855fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie 856fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie#if defined(CONFIG_DEBUG_FS) 857fa8a123855e20068204982596b8fafceb1a67f0bDave Airliestatic int radeon_mm_dump_table(struct seq_file *m, void *data) 858fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie{ 859fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie struct drm_info_node *node = (struct drm_info_node *)m->private; 860fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie struct drm_mm *mm = (struct drm_mm *)node->info_ent->data; 861fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie struct drm_device *dev = node->minor->dev; 862fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie struct radeon_device *rdev = dev->dev_private; 863fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie int ret; 864fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie struct ttm_bo_global *glob = rdev->mman.bdev.glob; 865fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie 866fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie spin_lock(&glob->lru_lock); 867fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie ret = drm_mm_dump_table(m, mm); 868fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie spin_unlock(&glob->lru_lock); 869fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie return ret; 870fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie} 871fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie#endif 872fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie 873fa8a123855e20068204982596b8fafceb1a67f0bDave Airliestatic int radeon_ttm_debugfs_init(struct radeon_device *rdev) 874fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie{ 875f4e45d02e4135043fe98bc21be38527c516ad990Mikael Pettersson#if defined(CONFIG_DEBUG_FS) 876c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES+2]; 877c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES+2][32]; 878fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie unsigned i; 879fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie 880fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) { 881fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie if (i == 0) 882fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie sprintf(radeon_mem_types_names[i], "radeon_vram_mm"); 883fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie else 884fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie sprintf(radeon_mem_types_names[i], "radeon_gtt_mm"); 885fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie radeon_mem_types_list[i].name = radeon_mem_types_names[i]; 886fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie radeon_mem_types_list[i].show = &radeon_mm_dump_table; 887fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie radeon_mem_types_list[i].driver_features = 0; 888fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie if (i == 0) 88916f9fdcbcce74102bed9a4b7ccc1fb05b5dd6ca3Dave Airlie radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_VRAM].priv; 890fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie else 89116f9fdcbcce74102bed9a4b7ccc1fb05b5dd6ca3Dave Airlie radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_TT].priv; 892fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie 893fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie } 8948d7cddcd7f9e847cec44851ca53fd3e81e846c49Pauli Nieminen /* Add ttm page pool to debugfs */ 8958d7cddcd7f9e847cec44851ca53fd3e81e846c49Pauli Nieminen sprintf(radeon_mem_types_names[i], "ttm_page_pool"); 8968d7cddcd7f9e847cec44851ca53fd3e81e846c49Pauli Nieminen radeon_mem_types_list[i].name = radeon_mem_types_names[i]; 8978d7cddcd7f9e847cec44851ca53fd3e81e846c49Pauli Nieminen radeon_mem_types_list[i].show = &ttm_page_alloc_debugfs; 8988d7cddcd7f9e847cec44851ca53fd3e81e846c49Pauli Nieminen radeon_mem_types_list[i].driver_features = 0; 899c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk radeon_mem_types_list[i++].data = NULL; 900c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk#ifdef CONFIG_SWIOTLB 901c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk if (swiotlb_nr_tbl()) { 902c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk sprintf(radeon_mem_types_names[i], "ttm_dma_page_pool"); 903c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk radeon_mem_types_list[i].name = radeon_mem_types_names[i]; 904c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk radeon_mem_types_list[i].show = &ttm_dma_page_alloc_debugfs; 905c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk radeon_mem_types_list[i].driver_features = 0; 906c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk radeon_mem_types_list[i++].data = NULL; 907c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk } 908c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk#endif 909c52494f69538f6fe1a234972f024011b17a48329Konrad Rzeszutek Wilk return radeon_debugfs_add_files(rdev, radeon_mem_types_list, i); 910fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie 911fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie#endif 912fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie return 0; 913fa8a123855e20068204982596b8fafceb1a67f0bDave Airlie} 914