1ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom/************************************************************************** 2ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 3ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 4ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * All Rights Reserved. 5ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 6ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Permission is hereby granted, free of charge, to any person obtaining a 7ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * copy of this software and associated documentation files (the 8ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * "Software"), to deal in the Software without restriction, including 9ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * without limitation the rights to use, copy, modify, merge, publish, 10ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * distribute, sub license, and/or sell copies of the Software, and to 11ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * permit persons to whom the Software is furnished to do so, subject to 12ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * the following conditions: 13ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 14ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * The above copyright notice and this permission notice (including the 15ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * next paragraph) shall be included in all copies or substantial portions 16ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * of the Software. 17ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 18ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * USE OR OTHER DEALINGS IN THE SOFTWARE. 25ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 26ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom **************************************************************************/ 27ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom/* 28ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 30ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 31ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include "ttm/ttm_bo_driver.h" 32ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include "ttm/ttm_placement.h" 33ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/io.h> 34ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/highmem.h> 35ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/wait.h> 365a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 37ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/vmalloc.h> 38ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/module.h> 39ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 40ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromvoid ttm_bo_free_old_node(struct ttm_buffer_object *bo) 41ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 4242311ff90dc8746bd81427b2ed6efda9af791b77Ben Skeggs ttm_bo_mem_put(bo, &bo->mem); 43ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 44ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 45ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_move_ttm(struct ttm_buffer_object *bo, 469d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool evict, bool no_wait_reserve, 479d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool no_wait_gpu, struct ttm_mem_reg *new_mem) 48ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 49ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_tt *ttm = bo->ttm; 50ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *old_mem = &bo->mem; 51ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 52ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 53ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (old_mem->mem_type != TTM_PL_SYSTEM) { 54ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_unbind(ttm); 55ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_free_old_node(bo); 56ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, 57ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom TTM_PL_MASK_MEM); 58ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom old_mem->mem_type = TTM_PL_SYSTEM; 59ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 60ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 61ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); 62ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(ret != 0)) 63ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 64ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 65ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (new_mem->mem_type != TTM_PL_SYSTEM) { 66ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_tt_bind(ttm, new_mem); 67ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(ret != 0)) 68ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 69ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 70ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 71ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *old_mem = *new_mem; 72ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom new_mem->mm_node = NULL; 73110b20c3ddcfa98cc932aef3af2d59b4e0841f08Austin Yuan 74ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 75ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 76ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_move_ttm); 77ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 78eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromint ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) 7982c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse{ 80eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (likely(man->io_reserve_fastpath)) 81eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return 0; 82eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 83eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (interruptible) 84eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return mutex_lock_interruptible(&man->io_reserve_mutex); 85eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 86eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom mutex_lock(&man->io_reserve_mutex); 87eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return 0; 88eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom} 8982c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse 90eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromvoid ttm_mem_io_unlock(struct ttm_mem_type_manager *man) 91eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom{ 92eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (likely(man->io_reserve_fastpath)) 93eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return; 94eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 95eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom mutex_unlock(&man->io_reserve_mutex); 96eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom} 97eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 98eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromstatic int ttm_mem_io_evict(struct ttm_mem_type_manager *man) 99eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom{ 100eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_buffer_object *bo; 101eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 102eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) 103eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return -EAGAIN; 104eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 105eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom bo = list_first_entry(&man->io_reserve_lru, 106eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_buffer_object, 107eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom io_reserve_lru); 108eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom list_del_init(&bo->io_reserve_lru); 109eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_bo_unmap_virtual_locked(bo); 110eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 111eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return 0; 112eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom} 113eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 114eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromstatic int ttm_mem_io_reserve(struct ttm_bo_device *bdev, 115eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_reg *mem) 116eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom{ 117eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 118eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom int ret = 0; 119eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 120eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (!bdev->driver->io_mem_reserve) 121eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return 0; 122eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (likely(man->io_reserve_fastpath)) 123eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return bdev->driver->io_mem_reserve(bdev, mem); 124eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 125eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (bdev->driver->io_mem_reserve && 126eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom mem->bus.io_reserved_count++ == 0) { 127eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromretry: 1280c321c79627189204d7d0bf65ab19f5ac419abedJerome Glisse ret = bdev->driver->io_mem_reserve(bdev, mem); 129eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (ret == -EAGAIN) { 130eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ret = ttm_mem_io_evict(man); 131eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (ret == 0) 132eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom goto retry; 133eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom } 134eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom } 135eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return ret; 136eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom} 137eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 138eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromstatic void ttm_mem_io_free(struct ttm_bo_device *bdev, 139eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_reg *mem) 140eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom{ 141eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 142eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 143eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (likely(man->io_reserve_fastpath)) 144eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom return; 145eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 146eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (bdev->driver->io_mem_reserve && 147eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom --mem->bus.io_reserved_count == 0 && 148eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom bdev->driver->io_mem_free) 149eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom bdev->driver->io_mem_free(bdev, mem); 150eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 151eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom} 152eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 153eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromint ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) 154eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom{ 155eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_reg *mem = &bo->mem; 156eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom int ret; 157eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 158eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (!mem->bus.io_reserved_vm) { 159eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_type_manager *man = 160eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom &bo->bdev->man[mem->mem_type]; 161eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 162eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ret = ttm_mem_io_reserve(bo->bdev, mem); 16382c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse if (unlikely(ret != 0)) 16482c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse return ret; 165eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom mem->bus.io_reserved_vm = true; 166eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (man->use_io_reserve_lru) 167eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom list_add_tail(&bo->io_reserve_lru, 168eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom &man->io_reserve_lru); 16982c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse } 17082c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse return 0; 17182c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse} 17282c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse 173eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromvoid ttm_mem_io_free_vm(struct ttm_buffer_object *bo) 17482c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse{ 175eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_reg *mem = &bo->mem; 176eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 177eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (mem->bus.io_reserved_vm) { 178eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom mem->bus.io_reserved_vm = false; 179eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom list_del_init(&bo->io_reserve_lru); 180eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_io_free(bo->bdev, mem); 18182c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse } 18282c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse} 18382c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse 184ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 185ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void **virtual) 186ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 187eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 188ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 189ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *addr; 190ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 191ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *virtual = NULL; 192eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom (void) ttm_mem_io_lock(man, false); 19382c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse ret = ttm_mem_io_reserve(bdev, mem); 194eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_io_unlock(man); 1959e51159c14c29ebd485a45ba56f148e180d62c29Jerome Glisse if (ret || !mem->bus.is_iomem) 196ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 197ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 19882c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse if (mem->bus.addr) { 19982c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse addr = mem->bus.addr; 20082c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse } else { 201ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (mem->placement & TTM_PL_FLAG_WC) 20282c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size); 203ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 20482c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size); 20582c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse if (!addr) { 206eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom (void) ttm_mem_io_lock(man, false); 20782c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse ttm_mem_io_free(bdev, mem); 208eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_io_unlock(man); 209ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 21082c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse } 211ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 212ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *virtual = addr; 213ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 214ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 215ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 216ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromvoid ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 217ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *virtual) 218ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 219ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_type_manager *man; 220ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 221ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom man = &bdev->man[mem->mem_type]; 222ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 2230c321c79627189204d7d0bf65ab19f5ac419abedJerome Glisse if (virtual && mem->bus.addr == NULL) 224ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom iounmap(virtual); 225eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom (void) ttm_mem_io_lock(man, false); 22682c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse ttm_mem_io_free(bdev, mem); 227eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_io_unlock(man); 228ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 229ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 230ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_copy_io_page(void *dst, void *src, unsigned long page) 231ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 232ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom uint32_t *dstP = 233ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 234ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom uint32_t *srcP = 235ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 236ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 237ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int i; 238ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 239ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom iowrite32(ioread32(srcP++), dstP++); 240ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 241ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 242ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 243ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 244542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom unsigned long page, 245542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom pgprot_t prot) 246ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 247b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse struct page *d = ttm->pages[page]; 248ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *dst; 249ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 250ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!d) 251ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 252ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 253ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 254542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom 255542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#ifdef CONFIG_X86 2563e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra dst = kmap_atomic_prot(d, prot); 257542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#else 2586d0897ba58139523d37e97855ee0fe2d78629da6Thomas Hellstrom if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) 259542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom dst = vmap(&d, 1, 0, prot); 260542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom else 261542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom dst = kmap(d); 262542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#endif 263ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!dst) 264ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 265ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 266ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom memcpy_fromio(dst, src, PAGE_SIZE); 267542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom 268542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#ifdef CONFIG_X86 2693e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra kunmap_atomic(dst); 270542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#else 2716d0897ba58139523d37e97855ee0fe2d78629da6Thomas Hellstrom if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) 272542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom vunmap(dst); 273542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom else 274542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom kunmap(d); 275542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#endif 276542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom 277ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 278ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 279ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 280ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 281542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom unsigned long page, 282542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom pgprot_t prot) 283ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 284b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse struct page *s = ttm->pages[page]; 285ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *src; 286ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 287ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!s) 288ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 289ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 290ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 291542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#ifdef CONFIG_X86 2923e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra src = kmap_atomic_prot(s, prot); 293542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#else 2946d0897ba58139523d37e97855ee0fe2d78629da6Thomas Hellstrom if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) 295542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom src = vmap(&s, 1, 0, prot); 296542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom else 297542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom src = kmap(s); 298542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#endif 299ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!src) 300ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 301ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 302ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom memcpy_toio(dst, src, PAGE_SIZE); 303542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom 304542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#ifdef CONFIG_X86 3053e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra kunmap_atomic(src); 306542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#else 3076d0897ba58139523d37e97855ee0fe2d78629da6Thomas Hellstrom if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) 308542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom vunmap(src); 309542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom else 310542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom kunmap(s); 311542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom#endif 312542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom 313ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 314ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 315ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 316ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 3179d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool evict, bool no_wait_reserve, bool no_wait_gpu, 3189d87fa2138d06ff400551800d67d522625033e35Jerome Glisse struct ttm_mem_reg *new_mem) 319ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 320ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 321ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 322ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_tt *ttm = bo->ttm; 323ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *old_mem = &bo->mem; 324e22469ca88a8f1f6fe47adbf5e5ce0906aec07cdThomas Hellstrom struct ttm_mem_reg old_copy = *old_mem; 325ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *old_iomap; 326ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *new_iomap; 327ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 328ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long i; 329ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long page; 330ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long add = 0; 331ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int dir; 332ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 333ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); 334ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 335ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 336ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); 337ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 338ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out; 339ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 340ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (old_iomap == NULL && new_iomap == NULL) 341ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out2; 342ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (old_iomap == NULL && ttm == NULL) 343ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out2; 344ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 345b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse if (ttm->state == tt_unpopulated) { 346b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse ret = ttm->bdev->driver->ttm_tt_populate(ttm); 347b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse if (ret) 348b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse goto out1; 349b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse } 350b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse 351ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom add = 0; 352ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom dir = 1; 353ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 354ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if ((old_mem->mem_type == new_mem->mem_type) && 355d961db75ce86a84f1f04e91ad1014653ed7d9f46Ben Skeggs (new_mem->start < old_mem->start + old_mem->size)) { 356ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom dir = -1; 357ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom add = new_mem->num_pages - 1; 358ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 359ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 360ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom for (i = 0; i < new_mem->num_pages; ++i) { 361ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom page = i * dir + add; 362542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom if (old_iomap == NULL) { 363542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom pgprot_t prot = ttm_io_prot(old_mem->placement, 364542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom PAGE_KERNEL); 365542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, 366542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom prot); 367542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom } else if (new_iomap == NULL) { 368542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom pgprot_t prot = ttm_io_prot(new_mem->placement, 369542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom PAGE_KERNEL); 370542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, 371542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom prot); 372542c6f6df51327dbb180cf4d9b34827e147efe17Thomas Hellstrom } else 373ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_copy_io_page(new_iomap, old_iomap, page); 374ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 375ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out1; 376ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 377ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom mb(); 378ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromout2: 379eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom old_copy = *old_mem; 380ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *old_mem = *new_mem; 381ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom new_mem->mm_node = NULL; 382ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 383ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { 384ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_unbind(ttm); 385ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_destroy(ttm); 386ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->ttm = NULL; 387ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 388ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 389ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromout1: 390eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); 391ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromout: 392ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 393b921bae2eedc806b118a03d986cf0be9ffd3af40Ben Skeggs ttm_bo_mem_put(bo, &old_copy); 394ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 395ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 396ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_move_memcpy); 397ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 398ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic void ttm_transfered_destroy(struct ttm_buffer_object *bo) 399ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 400ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kfree(bo); 401ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 402ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 403ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom/** 404ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * ttm_buffer_object_transfer 405ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 406ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * @bo: A pointer to a struct ttm_buffer_object. 407ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 408ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * holding the data of @bo with the old placement. 409ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 410ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * This is a utility function that may be called after an accelerated move 411ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * has been scheduled. A new buffer object is created as a placeholder for 412ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * the old data while it's being copied. When that buffer object is idle, 413ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * it can be destroyed, releasing the space of the old placement. 414ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Returns: 415ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * !0: Failure. 416ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 417ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 418ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 419ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object **new_obj) 420ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 421ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object *fbo; 422ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 423ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_driver *driver = bdev->driver; 424ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 425ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo = kzalloc(sizeof(*fbo), GFP_KERNEL); 426ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!fbo) 427ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 428ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 429ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *fbo = *bo; 430ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 431ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /** 432ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Fix up members that we shouldn't copy directly: 433ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * TODO: Explicit member copy would probably be better here. 434ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 435ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 436ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom init_waitqueue_head(&fbo->event_queue); 437ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom INIT_LIST_HEAD(&fbo->ddestroy); 438ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom INIT_LIST_HEAD(&fbo->lru); 439ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom INIT_LIST_HEAD(&fbo->swap); 440eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom INIT_LIST_HEAD(&fbo->io_reserve_lru); 441ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo->vm_node = NULL; 4420fbecd400dd0a82d465b3086f209681e8c54cb0fFrancisco Jerez atomic_set(&fbo->cpu_writers, 0); 443ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 444ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); 445ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kref_init(&fbo->list_kref); 446ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kref_init(&fbo->kref); 447ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo->destroy = &ttm_transfered_destroy; 44857de4ba959b290f0b8cf36ecd5e7f1b29d4b8a12Jerome Glisse fbo->acc_size = 0; 449ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 450ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *new_obj = fbo; 451ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 452ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 453ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 454ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrompgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) 455ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 456ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#if defined(__i386__) || defined(__x86_64__) 457ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (caching_flags & TTM_PL_FLAG_WC) 458ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_writecombine(tmp); 459ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else if (boot_cpu_data.x86 > 3) 460ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_noncached(tmp); 461ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 462ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#elif defined(__powerpc__) 463ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!(caching_flags & TTM_PL_FLAG_CACHED)) { 464ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom pgprot_val(tmp) |= _PAGE_NO_CACHE; 465ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (caching_flags & TTM_PL_FLAG_UNCACHED) 466ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom pgprot_val(tmp) |= _PAGE_GUARDED; 467ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 468ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#endif 469ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#if defined(__ia64__) 470ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (caching_flags & TTM_PL_FLAG_WC) 471ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_writecombine(tmp); 472ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 473ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_noncached(tmp); 474ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#endif 475ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#if defined(__sparc__) 476ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!(caching_flags & TTM_PL_FLAG_CACHED)) 477ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_noncached(tmp); 478ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#endif 479ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return tmp; 480ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 4814bfd75cb08a362cb1df35dc6a5032d12843c6d87Thomas HellstromEXPORT_SYMBOL(ttm_io_prot); 482ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 483ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_bo_ioremap(struct ttm_buffer_object *bo, 48482c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse unsigned long offset, 48582c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse unsigned long size, 486ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_kmap_obj *map) 487ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 488ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *mem = &bo->mem; 489ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 49082c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse if (bo->mem.bus.addr) { 491ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->bo_kmap_type = ttm_bo_map_premapped; 49282c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 493ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 494ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->bo_kmap_type = ttm_bo_map_iomap; 495ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (mem->placement & TTM_PL_FLAG_WC) 49682c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset, 49782c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse size); 498ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 49982c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset, 50082c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse size); 501ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 502ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return (!map->virtual) ? -ENOMEM : 0; 503ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 504ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 505ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 506ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long start_page, 507ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long num_pages, 508ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_kmap_obj *map) 509ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 510ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot; 511ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_tt *ttm = bo->ttm; 512b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse int ret; 513ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 514ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom BUG_ON(!ttm); 515b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse 516b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse if (ttm->state == tt_unpopulated) { 517b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse ret = ttm->bdev->driver->ttm_tt_populate(ttm); 518b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse if (ret) 519b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse return ret; 520b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse } 521b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse 522ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { 523ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 524ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * We're mapping a single page, and the desired 525ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * page protection is consistent with the bo. 526ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 527ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 528ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->bo_kmap_type = ttm_bo_map_kmap; 529b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse map->page = ttm->pages[start_page]; 530ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = kmap(map->page); 531ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 532ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 533ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * We need to use vmap to get the desired page protection 534af901ca181d92aac3a7dc265144a9081a86d8f39André Goddard Rosa * or to make the buffer object look contiguous. 535ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 536ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom prot = (mem->placement & TTM_PL_FLAG_CACHED) ? 537ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom PAGE_KERNEL : 538ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_io_prot(mem->placement, PAGE_KERNEL); 539ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->bo_kmap_type = ttm_bo_map_vmap; 540ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = vmap(ttm->pages + start_page, num_pages, 541ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 0, prot); 542ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 543ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return (!map->virtual) ? -ENOMEM : 0; 544ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 545ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 546ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_kmap(struct ttm_buffer_object *bo, 547ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long start_page, unsigned long num_pages, 548ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_kmap_obj *map) 549ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 550eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_type_manager *man = 551eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom &bo->bdev->man[bo->mem.mem_type]; 55282c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse unsigned long offset, size; 553ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 554ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 555ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom BUG_ON(!list_empty(&bo->swap)); 556ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = NULL; 55782c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse map->bo = bo; 558ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (num_pages > bo->num_pages) 559ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EINVAL; 560ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (start_page > bo->num_pages) 561ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EINVAL; 562ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#if 0 563ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) 564ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EPERM; 565ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#endif 566eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom (void) ttm_mem_io_lock(man, false); 56782c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 568eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_io_unlock(man); 569ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 570ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 57182c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse if (!bo->mem.bus.is_iomem) { 572ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 573ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 57482c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse offset = start_page << PAGE_SHIFT; 57582c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse size = num_pages << PAGE_SHIFT; 57682c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse return ttm_bo_ioremap(bo, offset, size, map); 577ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 578ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 579ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_kmap); 580ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 581ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromvoid ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 582ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 583eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_buffer_object *bo = map->bo; 584eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_type_manager *man = 585eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom &bo->bdev->man[bo->mem.mem_type]; 586eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom 587ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!map->virtual) 588ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return; 589ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom switch (map->bo_kmap_type) { 590ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom case ttm_bo_map_iomap: 591ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom iounmap(map->virtual); 592ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 593ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom case ttm_bo_map_vmap: 594ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom vunmap(map->virtual); 595ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 596ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom case ttm_bo_map_kmap: 597ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kunmap(map->page); 598ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 599ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom case ttm_bo_map_premapped: 600ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 601ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom default: 602ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom BUG(); 603ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 604eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom (void) ttm_mem_io_lock(man, false); 605eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 606eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_io_unlock(man); 607ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = NULL; 608ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->page = NULL; 609ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 610ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_kunmap); 611ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 612ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 613ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *sync_obj, 614ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *sync_obj_arg, 6159d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool evict, bool no_wait_reserve, 6169d87fa2138d06ff400551800d67d522625033e35Jerome Glisse bool no_wait_gpu, 617ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *new_mem) 618ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 619ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 620ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_driver *driver = bdev->driver; 621ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 622ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *old_mem = &bo->mem; 623ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 624ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object *ghost_obj; 625ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *tmp_obj = NULL; 626ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 627702adba22433c175e8429a47760f35ca16caf1cdThomas Hellstrom spin_lock(&bdev->fence_lock); 628ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (bo->sync_obj) { 629ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp_obj = bo->sync_obj; 630ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->sync_obj = NULL; 631ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 632ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->sync_obj = driver->sync_obj_ref(sync_obj); 633ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->sync_obj_arg = sync_obj_arg; 634ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (evict) { 635ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_bo_wait(bo, false, false, false); 636702adba22433c175e8429a47760f35ca16caf1cdThomas Hellstrom spin_unlock(&bdev->fence_lock); 6374677f15c60421d48566c48c3149474e64977f071Thomas Hellstrom if (tmp_obj) 6384677f15c60421d48566c48c3149474e64977f071Thomas Hellstrom driver->sync_obj_unref(&tmp_obj); 639ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 640ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 641ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 642ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && 643ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (bo->ttm != NULL)) { 644ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_unbind(bo->ttm); 645ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_destroy(bo->ttm); 646ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->ttm = NULL; 647ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 648eac2095398668f989a3dd8d00be1b87850d78c01Ben Skeggs ttm_bo_free_old_node(bo); 649ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 650ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /** 651ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * This should help pipeline ordinary buffer moves. 652ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 653ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Hang old buffer memory on a new buffer object, 654ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * and leave it to be released when the GPU 655ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * operation has completed. 656ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 657ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 658ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 659702adba22433c175e8429a47760f35ca16caf1cdThomas Hellstrom spin_unlock(&bdev->fence_lock); 6604677f15c60421d48566c48c3149474e64977f071Thomas Hellstrom if (tmp_obj) 6614677f15c60421d48566c48c3149474e64977f071Thomas Hellstrom driver->sync_obj_unref(&tmp_obj); 662ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 663ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_buffer_object_transfer(bo, &ghost_obj); 664ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 665ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 666ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 667ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /** 668ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * If we're not moving to fixed memory, the TTM object 669ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * needs to stay alive. Otherwhise hang it on the ghost 670ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * bo to be unbound and destroyed. 671ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 672ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 673ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) 674ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ghost_obj->ttm = NULL; 675ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 676ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->ttm = NULL; 677ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 678ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_unreserve(ghost_obj); 679ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_unref(&ghost_obj); 680ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 681ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 682ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *old_mem = *new_mem; 683ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom new_mem->mm_node = NULL; 684110b20c3ddcfa98cc932aef3af2d59b4e0841f08Austin Yuan 685ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 686ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 687ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_move_accel_cleanup); 688