1ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom/************************************************************************** 2ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 3ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * All Rights Reserved. 5ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 6ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Permission is hereby granted, free of charge, to any person obtaining a 7ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * copy of this software and associated documentation files (the 8ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * "Software"), to deal in the Software without restriction, including 9ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * without limitation the rights to use, copy, modify, merge, publish, 10ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * distribute, sub license, and/or sell copies of the Software, and to 11ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * permit persons to whom the Software is furnished to do so, subject to 12ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * the following conditions: 13ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 14ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * The above copyright notice and this permission notice (including the 15ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * next paragraph) shall be included in all copies or substantial portions 16ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * of the Software. 17ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 18ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * USE OR OTHER DEALINGS IN THE SOFTWARE. 25ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * 26ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom **************************************************************************/ 27ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom/* 28ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 30ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 3125d0479a5925562fbf999afb5a8daa3f501c729dJoe Perches#define pr_fmt(fmt) "[TTM] " fmt 3225d0479a5925562fbf999afb5a8daa3f501c729dJoe Perches 33ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <ttm/ttm_module.h> 34ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <ttm/ttm_bo_driver.h> 35ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <ttm/ttm_placement.h> 3672525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann#include <drm/drm_vma_manager.h> 37ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/mm.h> 38ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/rbtree.h> 39ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/module.h> 40ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#include <linux/uaccess.h> 41ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 42ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom#define TTM_BO_VM_NUM_PREFAULT 16 43ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 44cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstromstatic int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, 45cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom struct vm_area_struct *vma, 46cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom struct vm_fault *vmf) 47cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom{ 48cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom int ret = 0; 49cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom 50cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags))) 51cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom goto out_unlock; 52cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom 53cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom /* 54cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom * Quick non-stalling check for idle. 55cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom */ 56cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom ret = ttm_bo_wait(bo, false, false, true); 57cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom if (likely(ret == 0)) 58cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom goto out_unlock; 59cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom 60cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom /* 61cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom * If possible, avoid waiting for GPU with mmap_sem 62cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom * held. 63cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom */ 64cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { 65cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom ret = VM_FAULT_RETRY; 66cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 67cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom goto out_unlock; 68cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom 69cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom up_read(&vma->vm_mm->mmap_sem); 70cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom (void) ttm_bo_wait(bo, false, true, false); 71cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom goto out_unlock; 72cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom } 73cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom 74cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom /* 75cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom * Ordinary wait. 76cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom */ 77cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom ret = ttm_bo_wait(bo, false, true, false); 78cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom if (unlikely(ret != 0)) 79cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS : 80cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom VM_FAULT_NOPAGE; 81cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom 82cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstromout_unlock: 83cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom return ret; 84cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom} 85cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom 86ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 87ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 88ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object *bo = (struct ttm_buffer_object *) 89ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom vma->vm_private_data; 90ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev = bo->bdev; 91ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long page_offset; 92ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long page_last; 93ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long pfn; 94ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_tt *ttm = NULL; 95ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct page *page; 96ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 97ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int i; 98ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom unsigned long address = (unsigned long)vmf->virtual_address; 99ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int retval = VM_FAULT_NOPAGE; 100eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom struct ttm_mem_type_manager *man = 101eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom &bdev->man[bo->mem.mem_type]; 1023943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom struct vm_area_struct cvma; 103ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 104ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 105ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Work around locking order reversal in fault / nopfn 106ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * between mmap_sem and bo_reserve: Perform a trylock operation 107c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom * for reserve, and if it fails, retry the fault after waiting 108c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom * for the buffer to become unreserved. 109ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 110c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom ret = ttm_bo_reserve(bo, true, true, false, NULL); 111ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(ret != 0)) { 112c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom if (ret != -EBUSY) 113c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom return VM_FAULT_NOPAGE; 114c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom 115c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { 116c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 117c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom up_read(&vma->vm_mm->mmap_sem); 118c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom (void) ttm_bo_wait_unreserved(bo); 119c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom } 120c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom 121c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom return VM_FAULT_RETRY; 122c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom } 123c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom 124c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom /* 125c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom * If we'd want to change locking order to 126c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom * mmap_sem -> bo::reserve, we'd use a blocking reserve here 127c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom * instead of retrying the fault... 128c58f009e01c918717379c206a63baa66f56a77f9Thomas Hellstrom */ 129ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return VM_FAULT_NOPAGE; 130ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 131ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 132667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom /* 133667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom * Refuse to fault imported pages. This should be handled 134667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom * (if at all) by redirecting mmap to the exporter. 135667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom */ 136667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { 137667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom retval = VM_FAULT_SIGBUS; 138667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom goto out_unlock; 139667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom } 140667a50db0477d47fdff01c666f5ee1ce26b5264cThomas Hellstrom 14182c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse if (bdev->driver->fault_reserve_notify) { 14282c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse ret = bdev->driver->fault_reserve_notify(bo); 14382c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse switch (ret) { 14482c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse case 0: 14582c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse break; 14682c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse case -EBUSY: 14782c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse case -ERESTARTSYS: 14882c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse retval = VM_FAULT_NOPAGE; 14982c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse goto out_unlock; 15082c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse default: 15182c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse retval = VM_FAULT_SIGBUS; 15282c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse goto out_unlock; 15382c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse } 15482c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse } 155e024e11070a0a0dc7163ce1ec2da354a638bdbedDave Airlie 156ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 157ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Wait for buffer data in transit, due to a pipelined 158ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * move. 159ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 160cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom ret = ttm_bo_vm_fault_idle(bo, vma, vmf); 161cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom if (unlikely(ret != 0)) { 162cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom retval = ret; 163cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom goto out_unlock; 164cbe12e74ee4e29b6cb4e63fa284e80b73ad57926Thomas Hellstrom } 165ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 166eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ret = ttm_mem_io_lock(man, true); 167eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (unlikely(ret != 0)) { 168eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom retval = VM_FAULT_NOPAGE; 169ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out_unlock; 170ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 171eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ret = ttm_mem_io_reserve_vm(bo); 172eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom if (unlikely(ret != 0)) { 173eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom retval = VM_FAULT_SIGBUS; 174eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom goto out_io_unlock; 175eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom } 176ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 177ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + 178d386735588c3e22129c2bc6eb64fc1d37a8f805cThomas Hellstrom vma->vm_pgoff - drm_vma_node_start(&bo->vma_node); 179d386735588c3e22129c2bc6eb64fc1d37a8f805cThomas Hellstrom page_last = vma_pages(vma) + vma->vm_pgoff - 180d386735588c3e22129c2bc6eb64fc1d37a8f805cThomas Hellstrom drm_vma_node_start(&bo->vma_node); 181ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 182ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(page_offset >= bo->num_pages)) { 183ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom retval = VM_FAULT_SIGBUS; 184eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom goto out_io_unlock; 185ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 186ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 187ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 1883943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom * Make a local vma copy to modify the page_prot member 1893943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom * and vm_flags if necessary. The vma parameter is protected 1903943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom * by mmap_sem in write mode. 191ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 1923943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom cvma = *vma; 1933943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags); 1943943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom 19582c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse if (bo->mem.bus.is_iomem) { 1963943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, 1973943875e7b73fdd94dd9e911d69f0cee9ab66a89Thomas Hellstrom cvma.vm_page_prot); 198ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else { 199ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm = bo->ttm; 20094318d50ffc84a1ebaf1a83a0a56bbbaf415bacfBenjamin Herrenschmidt cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, 20194318d50ffc84a1ebaf1a83a0a56bbbaf415bacfBenjamin Herrenschmidt cvma.vm_page_prot); 202b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse 203b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse /* Allocate all page at once, most common usage */ 204b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse if (ttm->bdev->driver->ttm_tt_populate(ttm)) { 205b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse retval = VM_FAULT_OOM; 206b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse goto out_io_unlock; 207b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse } 208ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 209ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 210ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 211ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Speculatively prefault a number of pages. Only error on 212ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * first page. 213ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 214ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { 21582c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse if (bo->mem.bus.is_iomem) 21682c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset; 217ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else { 218b1e5f172325547270f35e7d1e42416a606e1dbd2Jerome Glisse page = ttm->pages[page_offset]; 219ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(!page && i == 0)) { 220ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom retval = VM_FAULT_OOM; 221eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom goto out_io_unlock; 222ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } else if (unlikely(!page)) { 223ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 224ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 22558aa6622d32af7d2c08d45085f44c54554a16ed7Thomas Hellstrom page->mapping = vma->vm_file->f_mapping; 22658aa6622d32af7d2c08d45085f44c54554a16ed7Thomas Hellstrom page->index = drm_vma_node_start(&bo->vma_node) + 22758aa6622d32af7d2c08d45085f44c54554a16ed7Thomas Hellstrom page_offset; 228ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom pfn = page_to_pfn(page); 229ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 230ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 2317dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom if (vma->vm_flags & VM_MIXEDMAP) 2327dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom ret = vm_insert_mixed(&cvma, address, pfn); 2337dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom else 2347dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom ret = vm_insert_pfn(&cvma, address, pfn); 2357dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom 236ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 237ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Somebody beat us to this PTE or prefaulting to 238ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * an already populated PTE, or prefaulting error. 239ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 240ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 241ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) 242ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 243ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom else if (unlikely(ret != 0)) { 244ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom retval = 245ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; 246eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom goto out_io_unlock; 247ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 248ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 249ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom address += PAGE_SIZE; 250ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(++page_offset >= page_last)) 251ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom break; 252ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 253eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstromout_io_unlock: 254eba67093f535322cb4f1c4b737319c0907a0c81dThomas Hellstrom ttm_mem_io_unlock(man); 255ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromout_unlock: 256ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_unreserve(bo); 257ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return retval; 258ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 259ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 260ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic void ttm_bo_vm_open(struct vm_area_struct *vma) 261ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 262ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object *bo = 263ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (struct ttm_buffer_object *)vma->vm_private_data; 264ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 26558aa6622d32af7d2c08d45085f44c54554a16ed7Thomas Hellstrom WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); 26658aa6622d32af7d2c08d45085f44c54554a16ed7Thomas Hellstrom 267ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom (void)ttm_bo_reference(bo); 268ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 269ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 270ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromstatic void ttm_bo_vm_close(struct vm_area_struct *vma) 271ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 27282c5da6bf8b55a931b042fb531083863d26c8020Jerome Glisse struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data; 273ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 274ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_unref(&bo); 275ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom vma->vm_private_data = NULL; 276ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 277ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 278f0f37e2f77731b3473fa6bd5ee53255d9a9cdb40Alexey Dobriyanstatic const struct vm_operations_struct ttm_bo_vm_ops = { 279ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom .fault = ttm_bo_vm_fault, 280ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom .open = ttm_bo_vm_open, 281ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom .close = ttm_bo_vm_close 282ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom}; 283ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 28472525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmannstatic struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, 28572525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann unsigned long offset, 28672525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann unsigned long pages) 28772525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann{ 28872525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann struct drm_vma_offset_node *node; 28972525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann struct ttm_buffer_object *bo = NULL; 29072525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann 29172525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann drm_vma_offset_lock_lookup(&bdev->vma_manager); 29272525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann 29372525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages); 29472525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann if (likely(node)) { 29572525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann bo = container_of(node, struct ttm_buffer_object, vma_node); 29672525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann if (!kref_get_unless_zero(&bo->kref)) 29772525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann bo = NULL; 29872525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann } 29972525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann 30072525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann drm_vma_offset_unlock_lookup(&bdev->vma_manager); 30172525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann 30272525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann if (!bo) 30372525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann pr_err("Could not find buffer object to map\n"); 30472525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann 30572525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann return bo; 30672525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann} 30772525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann 308ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, 309ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_device *bdev) 310ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 311ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_bo_driver *driver; 312ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom struct ttm_buffer_object *bo; 313ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom int ret; 314ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 31572525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); 31672525b3f333de54fa0c42ef87f27861e41478f1eDavid Herrmann if (unlikely(!bo)) 317ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EINVAL; 318ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 319ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom driver = bo->bdev->driver; 320ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(!driver->verify_access)) { 321ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = -EPERM; 322ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out_unref; 323ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom } 324ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ret = driver->verify_access(bo, filp); 325ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (unlikely(ret != 0)) 326ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom goto out_unref; 327ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 328ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom vma->vm_ops = &ttm_bo_vm_ops; 329ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 330ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom /* 331ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * Note: We're transferring the bo reference to 332ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom * vma->vm_private_data here. 333ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom */ 334ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 335ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom vma->vm_private_data = bo; 3367dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom 3377dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom /* 3380e6d6ec02f867fe8d1f785312b7343b21052322fThomas Hellstrom * We'd like to use VM_PFNMAP on shared mappings, where 3390e6d6ec02f867fe8d1f785312b7343b21052322fThomas Hellstrom * (vma->vm_flags & VM_SHARED) != 0, for performance reasons, 3400e6d6ec02f867fe8d1f785312b7343b21052322fThomas Hellstrom * but for some reason VM_PFNMAP + x86 PAT + write-combine is very 3410e6d6ec02f867fe8d1f785312b7343b21052322fThomas Hellstrom * bad for performance. Until that has been sorted out, use 3420e6d6ec02f867fe8d1f785312b7343b21052322fThomas Hellstrom * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 3437dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom */ 3440e6d6ec02f867fe8d1f785312b7343b21052322fThomas Hellstrom vma->vm_flags |= VM_MIXEDMAP; 3457dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; 346ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 347ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromout_unref: 348ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom ttm_bo_unref(&bo); 349ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return ret; 350ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 351ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_bo_mmap); 352ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 353ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstromint ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) 354ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom{ 355ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom if (vma->vm_pgoff != 0) 356ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return -EACCES; 357ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom 358ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom vma->vm_ops = &ttm_bo_vm_ops; 359ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom vma->vm_private_data = ttm_bo_reference(bo); 3600e6d6ec02f867fe8d1f785312b7343b21052322fThomas Hellstrom vma->vm_flags |= VM_MIXEDMAP; 3617dfe8b6187f43d791229b1f8eef9b7fa872f7195Thomas Hellstrom vma->vm_flags |= VM_IO | VM_DONTEXPAND; 362ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom return 0; 363ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas Hellstrom} 364ba4e7d973dd09b66912ac4c0856add8b0703a997Thomas HellstromEXPORT_SYMBOL(ttm_fbdev_mmap); 365