ttm_bo_util.c revision ba4e7d973dd09b66912ac4c0856add8b0703a997
1ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom/************************************************************************** 2ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 3ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 4ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * All Rights Reserved. 5ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 6ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Permission is hereby granted, free of charge, to any person obtaining a 7ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * copy of this software and associated documentation files (the 8ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * "Software"), to deal in the Software without restriction, including 9ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * without limitation the rights to use, copy, modify, merge, publish, 10ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * distribute, sub license, and/or sell copies of the Software, and to 11ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * permit persons to whom the Software is furnished to do so, subject to 12ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * the following conditions: 13ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 14ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * The above copyright notice and this permission notice (including the 15ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * next paragraph) shall be included in all copies or substantial portions 16ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * of the Software. 17ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 18ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * USE OR OTHER DEALINGS IN THE SOFTWARE. 25ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 26ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom **************************************************************************/ 27ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom/* 28ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 30ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 31ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include "ttm/ttm_bo_driver.h" 32ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include "ttm/ttm_placement.h" 33ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/io.h> 34ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/highmem.h> 35ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/wait.h> 36ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/vmalloc.h> 37ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/version.h> 38ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/module.h> 39ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 40ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromvoid ttm_bo_free_old_node(struct ttm_buffer_object *bo) 41ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 42ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *old_mem = &bo->mem; 43ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 44ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (old_mem->mm_node) { 45ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom spin_lock(&bo->bdev->lru_lock); 46ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom drm_mm_put_block(old_mem->mm_node); 47ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom spin_unlock(&bo->bdev->lru_lock); 48ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 49ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom old_mem->mm_node = NULL; 50ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 51ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 52ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_move_ttm(struct ttm_buffer_object *bo, 53ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bool evict, bool no_wait, struct ttm_mem_reg *new_mem) 54ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 55ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_tt *ttm = bo->ttm; 56ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *old_mem = &bo->mem; 57ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom uint32_t save_flags = old_mem->placement; 58ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 59ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 60ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (old_mem->mem_type != TTM_PL_SYSTEM) { 61ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_unbind(ttm); 62ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_free_old_node(bo); 63ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, 64ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom TTM_PL_MASK_MEM); 65ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom old_mem->mem_type = TTM_PL_SYSTEM; 66ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom save_flags = old_mem->placement; 67ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 68ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 69ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); 70ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(ret != 0)) 71ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 72ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 73ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (new_mem->mem_type != TTM_PL_SYSTEM) { 74ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_tt_bind(ttm, new_mem); 75ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(ret != 0)) 76ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 77ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 78ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 79ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *old_mem = *new_mem; 80ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom new_mem->mm_node = NULL; 81ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE); 82ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 83ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 84ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_move_ttm); 85ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 86ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 87ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void **virtual) 88ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 89ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 90ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_offset; 91ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_size; 92ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_base; 93ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 94ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *addr; 95ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 96ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *virtual = NULL; 97ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset, &bus_size); 98ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret || bus_size == 0) 99ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 100ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 101ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) 102ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom addr = (void *)(((u8 *) man->io_addr) + bus_offset); 103ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else { 104ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (mem->placement & TTM_PL_FLAG_WC) 105ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom addr = ioremap_wc(bus_base + bus_offset, bus_size); 106ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 107ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom addr = ioremap_nocache(bus_base + bus_offset, bus_size); 108ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!addr) 109ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 110ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 111ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *virtual = addr; 112ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 113ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 114ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 115ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromvoid ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 116ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *virtual) 117ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 118ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_type_manager *man; 119ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 120ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom man = &bdev->man[mem->mem_type]; 121ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 122ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (virtual && (man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) 123ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom iounmap(virtual); 124ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 125ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 126ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_copy_io_page(void *dst, void *src, unsigned long page) 127ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 128ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom uint32_t *dstP = 129ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 130ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom uint32_t *srcP = 131ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 132ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 133ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int i; 134ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 135ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom iowrite32(ioread32(srcP++), dstP++); 136ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 137ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 138ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 139ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 140ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long page) 141ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 142ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct page *d = ttm_tt_get_page(ttm, page); 143ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *dst; 144ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 145ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!d) 146ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 147ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 148ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 149ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom dst = kmap(d); 150ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!dst) 151ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 152ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 153ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom memcpy_fromio(dst, src, PAGE_SIZE); 154ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kunmap(d); 155ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 156ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 157ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 158ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 159ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long page) 160ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 161ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct page *s = ttm_tt_get_page(ttm, page); 162ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *src; 163ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 164ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!s) 165ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 166ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 167ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 168ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom src = kmap(s); 169ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!src) 170ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 171ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 172ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom memcpy_toio(dst, src, PAGE_SIZE); 173ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kunmap(s); 174ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 175ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 176ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 177ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 178ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bool evict, bool no_wait, struct ttm_mem_reg *new_mem) 179ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 180ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 181ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 182ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_tt *ttm = bo->ttm; 183ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *old_mem = &bo->mem; 184ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg old_copy = *old_mem; 185ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *old_iomap; 186ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *new_iomap; 187ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 188ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom uint32_t save_flags = old_mem->placement; 189ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long i; 190ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long page; 191ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long add = 0; 192ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int dir; 193ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 194ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); 195ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 196ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 197ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); 198ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 199ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out; 200ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 201ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (old_iomap == NULL && new_iomap == NULL) 202ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out2; 203ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (old_iomap == NULL && ttm == NULL) 204ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out2; 205ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 206ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom add = 0; 207ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom dir = 1; 208ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 209ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if ((old_mem->mem_type == new_mem->mem_type) && 210ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (new_mem->mm_node->start < 211ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom old_mem->mm_node->start + old_mem->mm_node->size)) { 212ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom dir = -1; 213ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom add = new_mem->num_pages - 1; 214ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 215ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 216ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom for (i = 0; i < new_mem->num_pages; ++i) { 217ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom page = i * dir + add; 218ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (old_iomap == NULL) 219ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_copy_ttm_io_page(ttm, new_iomap, page); 220ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else if (new_iomap == NULL) 221ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_copy_io_ttm_page(ttm, old_iomap, page); 222ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 223ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_copy_io_page(new_iomap, old_iomap, page); 224ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 225ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out1; 226ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 227ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom mb(); 228ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromout2: 229ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_free_old_node(bo); 230ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 231ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *old_mem = *new_mem; 232ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom new_mem->mm_node = NULL; 233ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE); 234ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 235ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { 236ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_unbind(ttm); 237ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_destroy(ttm); 238ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->ttm = NULL; 239ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 240ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 241ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromout1: 242ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_mem_reg_iounmap(bdev, new_mem, new_iomap); 243ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromout: 244ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 245ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 246ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 247ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_move_memcpy); 248ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 249ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic void ttm_transfered_destroy(struct ttm_buffer_object *bo) 250ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 251ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kfree(bo); 252ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 253ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 254ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom/** 255ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * ttm_buffer_object_transfer 256ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 257ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * @bo: A pointer to a struct ttm_buffer_object. 258ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 259ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * holding the data of @bo with the old placement. 260ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 261ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * This is a utility function that may be called after an accelerated move 262ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * has been scheduled. A new buffer object is created as a placeholder for 263ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * the old data while it's being copied. When that buffer object is idle, 264ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * it can be destroyed, releasing the space of the old placement. 265ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Returns: 266ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * !0: Failure. 267ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 268ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 269ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 270ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object **new_obj) 271ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 272ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object *fbo; 273ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 274ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_driver *driver = bdev->driver; 275ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 276ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo = kzalloc(sizeof(*fbo), GFP_KERNEL); 277ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!fbo) 278ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 279ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 280ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *fbo = *bo; 281ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 282ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /** 283ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Fix up members that we shouldn't copy directly: 284ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * TODO: Explicit member copy would probably be better here. 285ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 286ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 287ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom spin_lock_init(&fbo->lock); 288ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom init_waitqueue_head(&fbo->event_queue); 289ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom INIT_LIST_HEAD(&fbo->ddestroy); 290ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom INIT_LIST_HEAD(&fbo->lru); 291ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom INIT_LIST_HEAD(&fbo->swap); 292ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo->vm_node = NULL; 293ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 294ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); 295ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (fbo->mem.mm_node) 296ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo->mem.mm_node->private = (void *)fbo; 297ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kref_init(&fbo->list_kref); 298ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kref_init(&fbo->kref); 299ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom fbo->destroy = &ttm_transfered_destroy; 300ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 301ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *new_obj = fbo; 302ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 303ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 304ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 305ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrompgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) 306ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 307ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#if defined(__i386__) || defined(__x86_64__) 308ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (caching_flags & TTM_PL_FLAG_WC) 309ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_writecombine(tmp); 310ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else if (boot_cpu_data.x86 > 3) 311ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_noncached(tmp); 312ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 313ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#elif defined(__powerpc__) 314ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!(caching_flags & TTM_PL_FLAG_CACHED)) { 315ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom pgprot_val(tmp) |= _PAGE_NO_CACHE; 316ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (caching_flags & TTM_PL_FLAG_UNCACHED) 317ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom pgprot_val(tmp) |= _PAGE_GUARDED; 318ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 319ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#endif 320ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#if defined(__ia64__) 321ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (caching_flags & TTM_PL_FLAG_WC) 322ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_writecombine(tmp); 323ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 324ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_noncached(tmp); 325ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#endif 326ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#if defined(__sparc__) 327ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!(caching_flags & TTM_PL_FLAG_CACHED)) 328ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp = pgprot_noncached(tmp); 329ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#endif 330ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return tmp; 331ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 332ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 333ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_bo_ioremap(struct ttm_buffer_object *bo, 334ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_base, 335ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_offset, 336ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_size, 337ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_kmap_obj *map) 338ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 339ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 340ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *mem = &bo->mem; 341ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 342ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 343ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) { 344ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->bo_kmap_type = ttm_bo_map_premapped; 345ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = (void *)(((u8 *) man->io_addr) + bus_offset); 346ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 347ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->bo_kmap_type = ttm_bo_map_iomap; 348ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (mem->placement & TTM_PL_FLAG_WC) 349ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = ioremap_wc(bus_base + bus_offset, 350ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bus_size); 351ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 352ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = ioremap_nocache(bus_base + bus_offset, 353ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bus_size); 354ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 355ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return (!map->virtual) ? -ENOMEM : 0; 356ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 357ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 358ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 359ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long start_page, 360ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long num_pages, 361ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_kmap_obj *map) 362ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 363ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot; 364ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_tt *ttm = bo->ttm; 365ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct page *d; 366ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int i; 367ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 368ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom BUG_ON(!ttm); 369ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { 370ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 371ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * We're mapping a single page, and the desired 372ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * page protection is consistent with the bo. 373ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 374ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 375ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->bo_kmap_type = ttm_bo_map_kmap; 376ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->page = ttm_tt_get_page(ttm, start_page); 377ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = kmap(map->page); 378ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 379ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 380ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Populate the part we're mapping; 381ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 382ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom for (i = start_page; i < start_page + num_pages; ++i) { 383ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom d = ttm_tt_get_page(ttm, i); 384ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!d) 385ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -ENOMEM; 386ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 387ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 388ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 389ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * We need to use vmap to get the desired page protection 390ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * or to make the buffer object look contigous. 391ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 392ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom prot = (mem->placement & TTM_PL_FLAG_CACHED) ? 393ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom PAGE_KERNEL : 394ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_io_prot(mem->placement, PAGE_KERNEL); 395ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->bo_kmap_type = ttm_bo_map_vmap; 396ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = vmap(ttm->pages + start_page, num_pages, 397ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 0, prot); 398ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 399ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return (!map->virtual) ? -ENOMEM : 0; 400ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 401ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 402ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_kmap(struct ttm_buffer_object *bo, 403ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long start_page, unsigned long num_pages, 404ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_kmap_obj *map) 405ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 406ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 407ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_base; 408ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_offset; 409ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_size; 410ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 411ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom BUG_ON(!list_empty(&bo->swap)); 412ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = NULL; 413ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (num_pages > bo->num_pages) 414ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EINVAL; 415ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (start_page > bo->num_pages) 416ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EINVAL; 417ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#if 0 418ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) 419ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EPERM; 420ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#endif 421ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_bo_pci_offset(bo->bdev, &bo->mem, &bus_base, 422ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom &bus_offset, &bus_size); 423ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 424ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 425ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (bus_size == 0) { 426ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 427ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 428ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bus_offset += start_page << PAGE_SHIFT; 429ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bus_size = num_pages << PAGE_SHIFT; 430ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ttm_bo_ioremap(bo, bus_base, bus_offset, bus_size, map); 431ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 432ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 433ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_kmap); 434ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 435ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromvoid ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 436ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 437ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!map->virtual) 438ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return; 439ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom switch (map->bo_kmap_type) { 440ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom case ttm_bo_map_iomap: 441ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom iounmap(map->virtual); 442ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 443ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom case ttm_bo_map_vmap: 444ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom vunmap(map->virtual); 445ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 446ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom case ttm_bo_map_kmap: 447ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom kunmap(map->page); 448ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 449ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom case ttm_bo_map_premapped: 450ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 451ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom default: 452ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom BUG(); 453ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 454ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->virtual = NULL; 455ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom map->page = NULL; 456ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 457ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_kunmap); 458ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 459ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_pfn_prot(struct ttm_buffer_object *bo, 460ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long dst_offset, 461ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long *pfn, pgprot_t *prot) 462ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 463ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *mem = &bo->mem; 464ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 465ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_offset; 466ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_size; 467ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long bus_base; 468ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 469ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset, 470ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom &bus_size); 471ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 472ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EINVAL; 473ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (bus_size != 0) 474ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *pfn = (bus_base + bus_offset + dst_offset) >> PAGE_SHIFT; 475ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 476ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!bo->ttm) 477ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EINVAL; 478ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 479ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *pfn = page_to_pfn(ttm_tt_get_page(bo->ttm, 480ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom dst_offset >> 481ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom PAGE_SHIFT)); 482ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *prot = (mem->placement & TTM_PL_FLAG_CACHED) ? 483ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom PAGE_KERNEL : ttm_io_prot(mem->placement, PAGE_KERNEL); 484ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 485ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 486ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 487ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 488ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 489ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *sync_obj, 490ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *sync_obj_arg, 491ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bool evict, bool no_wait, 492ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *new_mem) 493ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 494ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 495ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_driver *driver = bdev->driver; 496ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 497ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_mem_reg *old_mem = &bo->mem; 498ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 499ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom uint32_t save_flags = old_mem->placement; 500ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object *ghost_obj; 501ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom void *tmp_obj = NULL; 502ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 503ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom spin_lock(&bo->lock); 504ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (bo->sync_obj) { 505ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom tmp_obj = bo->sync_obj; 506ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->sync_obj = NULL; 507ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 508ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->sync_obj = driver->sync_obj_ref(sync_obj); 509ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->sync_obj_arg = sync_obj_arg; 510ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (evict) { 511ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_bo_wait(bo, false, false, false); 512ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom spin_unlock(&bo->lock); 513ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom driver->sync_obj_unref(&bo->sync_obj); 514ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 515ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 516ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 517ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 518ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_free_old_node(bo); 519ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && 520ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (bo->ttm != NULL)) { 521ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_unbind(bo->ttm); 522ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_tt_destroy(bo->ttm); 523ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->ttm = NULL; 524ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 525ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 526ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /** 527ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * This should help pipeline ordinary buffer moves. 528ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 529ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Hang old buffer memory on a new buffer object, 530ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * and leave it to be released when the GPU 531ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * operation has completed. 532ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 533ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 534ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 535ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom spin_unlock(&bo->lock); 536ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 537ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = ttm_buffer_object_transfer(bo, &ghost_obj); 538ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (ret) 539ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 540ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 541ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /** 542ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * If we're not moving to fixed memory, the TTM object 543ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * needs to stay alive. Otherwhise hang it on the ghost 544ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * bo to be unbound and destroyed. 545ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 546ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 547ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) 548ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ghost_obj->ttm = NULL; 549ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else 550ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom bo->ttm = NULL; 551ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 552ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_unreserve(ghost_obj); 553ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_unref(&ghost_obj); 554ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 555ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 556ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom *old_mem = *new_mem; 557ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom new_mem->mm_node = NULL; 558ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE); 559ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 560ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 561ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_move_accel_cleanup); 562