1142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 2142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * SN Platform GRU Driver 3142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 4142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * FAULT HANDLER FOR GRU DETECTED TLB MISSES 5142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 6142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * This file contains code that handles TLB misses within the GRU. 7142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * These misses are reported either via interrupts or user polling of 8142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * the user CB. 9142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 10142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. 11142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 12142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * This program is free software; you can redistribute it and/or modify 13142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * it under the terms of the GNU General Public License as published by 14142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * the Free Software Foundation; either version 2 of the License, or 15142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * (at your option) any later version. 16142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 17142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * This program is distributed in the hope that it will be useful, 18142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * but WITHOUT ANY WARRANTY; without even the implied warranty of 19142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * GNU General Public License for more details. 21142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 22142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * You should have received a copy of the GNU General Public License 23142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * along with this program; if not, write to the Free Software 24142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 25142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 26142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 27142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <linux/kernel.h> 28142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <linux/errno.h> 29142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <linux/spinlock.h> 30142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <linux/mm.h> 31142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <linux/hugetlb.h> 32142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <linux/device.h> 33142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <linux/io.h> 34142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <linux/uaccess.h> 35bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner#include <linux/security.h> 36268bb0ce3e87872cb9290c322b0d35bce230d88fLinus Torvalds#include <linux/prefetch.h> 37142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <asm/pgtable.h> 38142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include "gru.h" 39142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include "grutables.h" 40142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include "grulib.h" 41142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include "gru_instructions.h" 42142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#include <asm/uv/uv_hub.h> 43142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 449c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner/* Return codes for vtop functions */ 459c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner#define VTOP_SUCCESS 0 469c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner#define VTOP_INVALID -1 479c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner#define VTOP_RETRY -2 489c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner 499c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner 50142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 51142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Test if a physical address is a valid GRU GSEG address 52142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 53142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstatic inline int is_gru_paddr(unsigned long paddr) 54142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 55142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return paddr >= gru_start_paddr && paddr < gru_end_paddr; 56142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 57142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 58142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 59142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Find the vma of a GRU segment. Caller must hold mmap_sem. 60142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 61142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstruct vm_area_struct *gru_find_vma(unsigned long vaddr) 62142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 63142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct vm_area_struct *vma; 64142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 65142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner vma = find_vma(current->mm, vaddr); 66142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (vma && vma->vm_start <= vaddr && vma->vm_ops == &gru_vm_ops) 67142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return vma; 68142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return NULL; 69142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 70142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 71142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 72142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Find and lock the gts that contains the specified user vaddr. 73142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 74142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Returns: 75142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * - *gts with the mmap_sem locked for read and the GTS locked. 76142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * - NULL if vaddr invalid OR is not a valid GSEG vaddr. 77142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 78142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 79142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstatic struct gru_thread_state *gru_find_lock_gts(unsigned long vaddr) 80142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 81142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct mm_struct *mm = current->mm; 82142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct vm_area_struct *vma; 83142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_thread_state *gts = NULL; 84142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 85142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner down_read(&mm->mmap_sem); 86142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner vma = gru_find_vma(vaddr); 87142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (vma) 88142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gts = gru_find_thread_state(vma, TSID(vaddr, vma)); 89142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (gts) 90142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner mutex_lock(>s->ts_ctxlock); 91142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner else 92142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner up_read(&mm->mmap_sem); 93142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return gts; 94142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 95142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 96142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstatic struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr) 97142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 98142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct mm_struct *mm = current->mm; 99142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct vm_area_struct *vma; 100e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner struct gru_thread_state *gts = ERR_PTR(-EINVAL); 101142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 102142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner down_write(&mm->mmap_sem); 103142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner vma = gru_find_vma(vaddr); 104e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner if (!vma) 105e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner goto err; 106e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner 107e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner gts = gru_alloc_thread_state(vma, TSID(vaddr, vma)); 108e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner if (IS_ERR(gts)) 109e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner goto err; 110e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner mutex_lock(>s->ts_ctxlock); 111e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner downgrade_write(&mm->mmap_sem); 112e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner return gts; 113142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 114e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steinererr: 115e006043a4d2da52bba9fd9cb7e5a22e2951ff69bJack Steiner up_write(&mm->mmap_sem); 116142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return gts; 117142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 118142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 119142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 120142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Unlock a GTS that was previously locked with gru_find_lock_gts(). 121142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 122142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstatic void gru_unlock_gts(struct gru_thread_state *gts) 123142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 124142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner mutex_unlock(>s->ts_ctxlock); 125142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner up_read(¤t->mm->mmap_sem); 126142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 127142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 128142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 129142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Set a CB.istatus to active using a user virtual address. This must be done 130142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * just prior to a TFH RESTART. The new cb.istatus is an in-cache status ONLY. 131142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * If the line is evicted, the status may be lost. The in-cache update 132142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * is necessary to prevent the user from seeing a stale cb.istatus that will 133142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * change as soon as the TFH restart is complete. Races may cause an 134142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * occasional failure to clear the cb.istatus, but that is ok. 135142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 136b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steinerstatic void gru_cb_set_istatus_active(struct gru_instruction_bits *cbk) 137142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 138b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner if (cbk) { 139b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner cbk->istatus = CBS_ACTIVE; 140142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } 141142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 142142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 143142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 144142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Read & clear a TFM 145142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 146142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * The GRU has an array of fault maps. A map is private to a cpu 147142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Only one cpu will be accessing a cpu's fault map. 148142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 149142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * This function scans the cpu-private fault map & clears all bits that 150142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * are set. The function returns a bitmap that indicates the bits that 151142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * were cleared. Note that sense the maps may be updated asynchronously by 152142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * the GRU, atomic operations must be used to clear bits. 153142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 154142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstatic void get_clear_fault_map(struct gru_state *gru, 1554a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner struct gru_tlb_fault_map *imap, 1564a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner struct gru_tlb_fault_map *dmap) 157142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 158142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner unsigned long i, k; 159142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_tlb_fault_map *tfm; 160142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 161142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfm = get_tfm_for_cpu(gru, gru_cpu_fault_map_id()); 162142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner prefetchw(tfm); /* Helps on hardware, required for emulator */ 163142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner for (i = 0; i < BITS_TO_LONGS(GRU_NUM_CBE); i++) { 164142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner k = tfm->fault_bits[i]; 165142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (k) 166142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner k = xchg(&tfm->fault_bits[i], 0UL); 1674a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner imap->fault_bits[i] = k; 1684a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner k = tfm->done_bits[i]; 1694a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner if (k) 1704a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner k = xchg(&tfm->done_bits[i], 0UL); 1714a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner dmap->fault_bits[i] = k; 172142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } 173142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 174142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* 175142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Not functionally required but helps performance. (Required 176142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * on emulator) 177142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 178142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_flush_cache(tfm); 179142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 180142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 181142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 182142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Atomic (interrupt context) & non-atomic (user context) functions to 183142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * convert a vaddr into a physical address. The size of the page 184142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * is returned in pageshift. 185142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * returns: 186142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 0 - successful 187142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * < 0 - error code 188142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 1 - (atomic only) try again in non-atomic context 189142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 190142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstatic int non_atomic_pte_lookup(struct vm_area_struct *vma, 191142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner unsigned long vaddr, int write, 192142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner unsigned long *paddr, int *pageshift) 193142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 194142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct page *page; 195142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 19674ccd09526a0254c362adcc6c862354d724aa803Jack Steiner#ifdef CONFIG_HUGETLB_PAGE 19774ccd09526a0254c362adcc6c862354d724aa803Jack Steiner *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT; 19874ccd09526a0254c362adcc6c862354d724aa803Jack Steiner#else 199142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner *pageshift = PAGE_SHIFT; 20074ccd09526a0254c362adcc6c862354d724aa803Jack Steiner#endif 201142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (get_user_pages 202142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner (current, current->mm, vaddr, 1, write, 0, &page, NULL) <= 0) 203142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EFAULT; 204142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner *paddr = page_to_phys(page); 205142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner put_page(page); 206142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 0; 207142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 208142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 209142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 210142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * atomic_pte_lookup 211142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 212142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Convert a user virtual address to a physical address 213142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Only supports Intel large pages (2MB only) on x86_64. 214142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * ZZZ - hugepage support is incomplete 215923f7f6970bd448b8e88b9e4be10fd01fc7106a4Jack Steiner * 216923f7f6970bd448b8e88b9e4be10fd01fc7106a4Jack Steiner * NOTE: mmap_sem is already held on entry to this function. This 217923f7f6970bd448b8e88b9e4be10fd01fc7106a4Jack Steiner * guarantees existence of the page tables. 218142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 219142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstatic int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, 220142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner int write, unsigned long *paddr, int *pageshift) 221142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 222142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pgd_t *pgdp; 223142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pmd_t *pmdp; 224142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pud_t *pudp; 225142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pte_t pte; 226142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 227142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pgdp = pgd_offset(vma->vm_mm, vaddr); 228142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (unlikely(pgd_none(*pgdp))) 229142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto err; 230142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 231142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pudp = pud_offset(pgdp, vaddr); 232142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (unlikely(pud_none(*pudp))) 233142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto err; 234142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 235142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pmdp = pmd_offset(pudp, vaddr); 236142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (unlikely(pmd_none(*pmdp))) 237142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto err; 238142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#ifdef CONFIG_X86_64 239142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (unlikely(pmd_large(*pmdp))) 240142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pte = *(pte_t *) pmdp; 241142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner else 242142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner#endif 243142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner pte = *pte_offset_kernel(pmdp, vaddr); 244142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 245142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (unlikely(!pte_present(pte) || 246142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner (write && (!pte_write(pte) || !pte_dirty(pte))))) 247142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 1; 248142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 249142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner *paddr = pte_pfn(pte) << PAGE_SHIFT; 250023a407f1cdc8b86903495d7ce0d2abae0f8bce5Jack Steiner#ifdef CONFIG_HUGETLB_PAGE 251142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT; 252023a407f1cdc8b86903495d7ce0d2abae0f8bce5Jack Steiner#else 253023a407f1cdc8b86903495d7ce0d2abae0f8bce5Jack Steiner *pageshift = PAGE_SHIFT; 254023a407f1cdc8b86903495d7ce0d2abae0f8bce5Jack Steiner#endif 255142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 0; 256142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 257142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinererr: 258142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 1; 259142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 260142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 261ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steinerstatic int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr, 262ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner int write, int atomic, unsigned long *gpa, int *pageshift) 263ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner{ 264ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner struct mm_struct *mm = gts->ts_mm; 265ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner struct vm_area_struct *vma; 266ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner unsigned long paddr; 267ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner int ret, ps; 268ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner 269ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner vma = find_vma(mm, vaddr); 270ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner if (!vma) 271ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner goto inval; 272ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner 273ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner /* 274ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner * Atomic lookup is faster & usually works even if called in non-atomic 275ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner * context. 276ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner */ 277ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner rmb(); /* Must/check ms_range_active before loading PTEs */ 278ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps); 279ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner if (ret) { 280ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner if (atomic) 281ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner goto upm; 282ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) 283ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner goto inval; 284ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner } 285ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner if (is_gru_paddr(paddr)) 286ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner goto inval; 287ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner paddr = paddr & ~((1UL << ps) - 1); 288ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner *gpa = uv_soc_phys_ram_to_gpa(paddr); 289ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner *pageshift = ps; 2909c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner return VTOP_SUCCESS; 291ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner 292ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steinerinval: 2939c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner return VTOP_INVALID; 294ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steinerupm: 2959c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner return VTOP_RETRY; 296ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner} 297ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner 298ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner 299142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 300c550222f64265c9384c45d7f691445386025a82bJack Steiner * Flush a CBE from cache. The CBE is clean in the cache. Dirty the 301c550222f64265c9384c45d7f691445386025a82bJack Steiner * CBE cacheline so that the line will be written back to home agent. 302c550222f64265c9384c45d7f691445386025a82bJack Steiner * Otherwise the line may be silently dropped. This has no impact 303c550222f64265c9384c45d7f691445386025a82bJack Steiner * except on performance. 304c550222f64265c9384c45d7f691445386025a82bJack Steiner */ 305c550222f64265c9384c45d7f691445386025a82bJack Steinerstatic void gru_flush_cache_cbe(struct gru_control_block_extended *cbe) 306c550222f64265c9384c45d7f691445386025a82bJack Steiner{ 307c550222f64265c9384c45d7f691445386025a82bJack Steiner if (unlikely(cbe)) { 308c550222f64265c9384c45d7f691445386025a82bJack Steiner cbe->cbrexecstatus = 0; /* make CL dirty */ 309c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache(cbe); 310c550222f64265c9384c45d7f691445386025a82bJack Steiner } 311c550222f64265c9384c45d7f691445386025a82bJack Steiner} 312c550222f64265c9384c45d7f691445386025a82bJack Steiner 313c550222f64265c9384c45d7f691445386025a82bJack Steiner/* 314c550222f64265c9384c45d7f691445386025a82bJack Steiner * Preload the TLB with entries that may be required. Currently, preloading 315c550222f64265c9384c45d7f691445386025a82bJack Steiner * is implemented only for BCOPY. Preload <tlb_preload_count> pages OR to 316c550222f64265c9384c45d7f691445386025a82bJack Steiner * the end of the bcopy tranfer, whichever is smaller. 317c550222f64265c9384c45d7f691445386025a82bJack Steiner */ 318c550222f64265c9384c45d7f691445386025a82bJack Steinerstatic void gru_preload_tlb(struct gru_state *gru, 319c550222f64265c9384c45d7f691445386025a82bJack Steiner struct gru_thread_state *gts, int atomic, 320c550222f64265c9384c45d7f691445386025a82bJack Steiner unsigned long fault_vaddr, int asid, int write, 321c550222f64265c9384c45d7f691445386025a82bJack Steiner unsigned char tlb_preload_count, 322c550222f64265c9384c45d7f691445386025a82bJack Steiner struct gru_tlb_fault_handle *tfh, 323c550222f64265c9384c45d7f691445386025a82bJack Steiner struct gru_control_block_extended *cbe) 324c550222f64265c9384c45d7f691445386025a82bJack Steiner{ 325c550222f64265c9384c45d7f691445386025a82bJack Steiner unsigned long vaddr = 0, gpa; 326c550222f64265c9384c45d7f691445386025a82bJack Steiner int ret, pageshift; 327c550222f64265c9384c45d7f691445386025a82bJack Steiner 328c550222f64265c9384c45d7f691445386025a82bJack Steiner if (cbe->opccpy != OP_BCOPY) 329c550222f64265c9384c45d7f691445386025a82bJack Steiner return; 330c550222f64265c9384c45d7f691445386025a82bJack Steiner 331c550222f64265c9384c45d7f691445386025a82bJack Steiner if (fault_vaddr == cbe->cbe_baddr0) 332c550222f64265c9384c45d7f691445386025a82bJack Steiner vaddr = fault_vaddr + GRU_CACHE_LINE_BYTES * cbe->cbe_src_cl - 1; 333c550222f64265c9384c45d7f691445386025a82bJack Steiner else if (fault_vaddr == cbe->cbe_baddr1) 334c550222f64265c9384c45d7f691445386025a82bJack Steiner vaddr = fault_vaddr + (1 << cbe->xtypecpy) * cbe->cbe_nelemcur - 1; 335c550222f64265c9384c45d7f691445386025a82bJack Steiner 336c550222f64265c9384c45d7f691445386025a82bJack Steiner fault_vaddr &= PAGE_MASK; 337c550222f64265c9384c45d7f691445386025a82bJack Steiner vaddr &= PAGE_MASK; 338c550222f64265c9384c45d7f691445386025a82bJack Steiner vaddr = min(vaddr, fault_vaddr + tlb_preload_count * PAGE_SIZE); 339c550222f64265c9384c45d7f691445386025a82bJack Steiner 340c550222f64265c9384c45d7f691445386025a82bJack Steiner while (vaddr > fault_vaddr) { 341c550222f64265c9384c45d7f691445386025a82bJack Steiner ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); 342c550222f64265c9384c45d7f691445386025a82bJack Steiner if (ret || tfh_write_only(tfh, gpa, GAA_RAM, vaddr, asid, write, 343c550222f64265c9384c45d7f691445386025a82bJack Steiner GRU_PAGESIZE(pageshift))) 344c550222f64265c9384c45d7f691445386025a82bJack Steiner return; 345c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_dbg(grudev, 346c550222f64265c9384c45d7f691445386025a82bJack Steiner "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, rw %d, ps %d, gpa 0x%lx\n", 347c550222f64265c9384c45d7f691445386025a82bJack Steiner atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, 348c550222f64265c9384c45d7f691445386025a82bJack Steiner vaddr, asid, write, pageshift, gpa); 349c550222f64265c9384c45d7f691445386025a82bJack Steiner vaddr -= PAGE_SIZE; 350c550222f64265c9384c45d7f691445386025a82bJack Steiner STAT(tlb_preload_page); 351c550222f64265c9384c45d7f691445386025a82bJack Steiner } 352c550222f64265c9384c45d7f691445386025a82bJack Steiner} 353c550222f64265c9384c45d7f691445386025a82bJack Steiner 354c550222f64265c9384c45d7f691445386025a82bJack Steiner/* 355142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Drop a TLB entry into the GRU. The fault is described by info in an TFH. 356142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Input: 357142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * cb Address of user CBR. Null if not running in user context 358142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Return: 359142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 0 = dropin, exception, or switch to UPM successful 360142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 1 = range invalidate active 361142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * < 0 = error code 362142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * 363142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 3642ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steinerstatic int gru_try_dropin(struct gru_state *gru, 3652ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner struct gru_thread_state *gts, 366142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_tlb_fault_handle *tfh, 367b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner struct gru_instruction_bits *cbk) 368142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 369c550222f64265c9384c45d7f691445386025a82bJack Steiner struct gru_control_block_extended *cbe = NULL; 370c550222f64265c9384c45d7f691445386025a82bJack Steiner unsigned char tlb_preload_count = gts->ts_tlb_preload_count; 371563447d7eb04c9b382f90a132be126a21a635647Jack Steiner int pageshift = 0, asid, write, ret, atomic = !cbk, indexway; 372ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner unsigned long gpa = 0, vaddr = 0; 373142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 374142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* 375142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * NOTE: The GRU contains magic hardware that eliminates races between 376142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * TLB invalidates and TLB dropins. If an invalidate occurs 377142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * in the window between reading the TFH and the subsequent TLB dropin, 378142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * the dropin is ignored. This eliminates the need for additional locks. 379142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 380142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 381142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* 382c550222f64265c9384c45d7f691445386025a82bJack Steiner * Prefetch the CBE if doing TLB preloading 383c550222f64265c9384c45d7f691445386025a82bJack Steiner */ 384c550222f64265c9384c45d7f691445386025a82bJack Steiner if (unlikely(tlb_preload_count)) { 385c550222f64265c9384c45d7f691445386025a82bJack Steiner cbe = gru_tfh_to_cbe(tfh); 386c550222f64265c9384c45d7f691445386025a82bJack Steiner prefetchw(cbe); 387c550222f64265c9384c45d7f691445386025a82bJack Steiner } 388c550222f64265c9384c45d7f691445386025a82bJack Steiner 389c550222f64265c9384c45d7f691445386025a82bJack Steiner /* 390142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Error if TFH state is IDLE or FMM mode & the user issuing a UPM call. 391142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Might be a hardware race OR a stupid user. Ignore FMM because FMM 392142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * is a transient state. 393142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 394270952a907220c0331fdaecbb55df892921c5e2dJack Steiner if (tfh->status != TFHSTATUS_EXCEPTION) { 395270952a907220c0331fdaecbb55df892921c5e2dJack Steiner gru_flush_cache(tfh); 39667bf04a5c2574e9495f660f418f6df776821d578Jack Steiner sync_core(); 397270952a907220c0331fdaecbb55df892921c5e2dJack Steiner if (tfh->status != TFHSTATUS_EXCEPTION) 398270952a907220c0331fdaecbb55df892921c5e2dJack Steiner goto failnoexception; 399270952a907220c0331fdaecbb55df892921c5e2dJack Steiner STAT(tfh_stale_on_fault); 400270952a907220c0331fdaecbb55df892921c5e2dJack Steiner } 401142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (tfh->state == TFHSTATE_IDLE) 402142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto failidle; 403b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner if (tfh->state == TFHSTATE_MISS_FMM && cbk) 404142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto failfmm; 405142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 406142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner write = (tfh->cause & TFHCAUSE_TLB_MOD) != 0; 407142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner vaddr = tfh->missvaddr; 408142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner asid = tfh->missasid; 409563447d7eb04c9b382f90a132be126a21a635647Jack Steiner indexway = tfh->indexway; 410142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (asid == 0) 411142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto failnoasid; 412142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 413142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner rmb(); /* TFH must be cache resident before reading ms_range_active */ 414142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 415142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* 416142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * TFH is cache resident - at least briefly. Fail the dropin 417142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * if a range invalidate is active. 418142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 419142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (atomic_read(>s->ts_gms->ms_range_active)) 420142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto failactive; 421142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 422ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); 4239c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner if (ret == VTOP_INVALID) 424142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto failinval; 4259c13cb331f7054d537d4caa094d3ebbba544833aJack Steiner if (ret == VTOP_RETRY) 426ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582cJack Steiner goto failupm; 427142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 4287b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) { 4297b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift); 43099f7c229b32bdf7424fbeb1d0d1b3883e14e97d0Jack Steiner if (atomic || !gru_update_cch(gts)) { 4317b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner gts->ts_force_cch_reload = 1; 4327b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner goto failupm; 4337b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner } 4347b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner } 435c550222f64265c9384c45d7f691445386025a82bJack Steiner 436c550222f64265c9384c45d7f691445386025a82bJack Steiner if (unlikely(cbe) && pageshift == PAGE_SHIFT) { 4372ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe); 438c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 439c550222f64265c9384c45d7f691445386025a82bJack Steiner } 440c550222f64265c9384c45d7f691445386025a82bJack Steiner 441b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner gru_cb_set_istatus_active(cbk); 4425958ab88f721d312b531d17705fc3ed54102fa05Jack Steiner gts->ustats.tlbdropin++; 443142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write, 444142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner GRU_PAGESIZE(pageshift)); 445142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, 446563447d7eb04c9b382f90a132be126a21a635647Jack Steiner "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x," 447563447d7eb04c9b382f90a132be126a21a635647Jack Steiner " rw %d, ps %d, gpa 0x%lx\n", 4482ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, vaddr, asid, 449563447d7eb04c9b382f90a132be126a21a635647Jack Steiner indexway, write, pageshift, gpa); 450563447d7eb04c9b382f90a132be126a21a635647Jack Steiner STAT(tlb_dropin); 451142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 0; 452142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 453142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerfailnoasid: 454142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* No asid (delayed unload). */ 455142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(tlb_dropin_fail_no_asid); 456142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, "FAILED no_asid tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr); 457b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner if (!cbk) 458142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh_user_polling_mode(tfh); 459142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner else 460142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_flush_cache(tfh); 461c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 462142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EAGAIN; 463142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 464142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerfailupm: 465142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* Atomic failure switch CBR to UPM */ 466142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh_user_polling_mode(tfh); 467c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 468142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(tlb_dropin_fail_upm); 469142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr); 470142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 1; 471142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 472142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerfailfmm: 473142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* FMM state on UPM call */ 474fe5bb6b00c3a9374841d651e01694fe4190a677eJack Steiner gru_flush_cache(tfh); 475c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 476142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(tlb_dropin_fail_fmm); 477142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state); 478142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 0; 479142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 480cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steinerfailnoexception: 481cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner /* TFH status did not show exception pending */ 482cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner gru_flush_cache(tfh); 483c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 484b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner if (cbk) 485b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner gru_flush_cache(cbk); 486cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner STAT(tlb_dropin_fail_no_exception); 487b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner gru_dbg(grudev, "FAILED non-exception tfh: 0x%p, status %d, state %d\n", 488b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner tfh, tfh->status, tfh->state); 489cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner return 0; 490cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner 491142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerfailidle: 492cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner /* TFH state was idle - no miss pending */ 493142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_flush_cache(tfh); 494c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 495b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner if (cbk) 496b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner gru_flush_cache(cbk); 497142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(tlb_dropin_fail_idle); 498142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, "FAILED idle tfh: 0x%p, state %d\n", tfh, tfh->state); 499142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 0; 500142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 501142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerfailinval: 502142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* All errors (atomic & non-atomic) switch CBR to EXCEPTION state */ 503142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh_exception(tfh); 504c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 505142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(tlb_dropin_fail_invalid); 506142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr); 507142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EFAULT; 508142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 509142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerfailactive: 510142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* Range invalidate active. Switch to UPM iff atomic */ 511b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner if (!cbk) 512142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh_user_polling_mode(tfh); 513142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner else 514142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_flush_cache(tfh); 515c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 516142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(tlb_dropin_fail_range_active); 517142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, "FAILED range active: tfh 0x%p, vaddr 0x%lx\n", 518142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh, vaddr); 519142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 1; 520142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 521142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 522142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 523142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Process an external interrupt from the GRU. This interrupt is 524142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * caused by a TLB miss. 525142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Note that this is the interrupt handler that is registered with linux 526142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * interrupt handlers. 527142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 5284107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steinerstatic irqreturn_t gru_intr(int chiplet, int blade) 529142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 530142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_state *gru; 5314a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner struct gru_tlb_fault_map imap, dmap; 532142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_thread_state *gts; 533142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_tlb_fault_handle *tfh = NULL; 5342ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner struct completion *cmp; 535142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner int cbrnum, ctxnum; 536142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 537142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(intr); 538142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 5394107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner gru = &gru_base[blade]->bs_grus[chiplet]; 540142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (!gru) { 5414107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner dev_err(grudev, "GRU: invalid interrupt: cpu %d, chiplet %d\n", 5424107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner raw_smp_processor_id(), chiplet); 543142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return IRQ_NONE; 544142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } 5454a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner get_clear_fault_map(gru, &imap, &dmap); 5464107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner gru_dbg(grudev, 5474107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner "cpu %d, chiplet %d, gid %d, imap %016lx %016lx, dmap %016lx %016lx\n", 5484107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner smp_processor_id(), chiplet, gru->gs_gid, 5494107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner imap.fault_bits[0], imap.fault_bits[1], 5504107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner dmap.fault_bits[0], dmap.fault_bits[1]); 5514a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner 5524a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) { 553563447d7eb04c9b382f90a132be126a21a635647Jack Steiner STAT(intr_cbr); 5542ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner cmp = gru->gs_blade->bs_async_wq; 5552ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner if (cmp) 5562ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner complete(cmp); 5574a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n", 5582ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner gru->gs_gid, cbrnum, cmp ? cmp->done : -1); 5594a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner } 560142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 5614a7a17c1188a878e9f00e4ca8dc724c7cff17606Jack Steiner for_each_cbr_in_tfm(cbrnum, imap.fault_bits) { 562563447d7eb04c9b382f90a132be126a21a635647Jack Steiner STAT(intr_tfh); 563142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh = get_tfh_by_index(gru, cbrnum); 564142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner prefetchw(tfh); /* Helps on hdw, required for emulator */ 565142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 566142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* 567142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * When hardware sets a bit in the faultmap, it implicitly 568142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * locks the GRU context so that it cannot be unloaded. 569142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * The gts cannot change until a TFH start/writestart command 570142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * is issued. 571142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 572142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner ctxnum = tfh->ctxnum; 573142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gts = gru->gs_gts[ctxnum]; 574142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 5752ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner /* Spurious interrupts can cause this. Ignore. */ 5762ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner if (!gts) { 5772ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner STAT(intr_spurious); 5782ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner continue; 5792ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner } 5802ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner 581142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* 582142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * This is running in interrupt context. Trylock the mmap_sem. 583142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * If it fails, retry the fault in user context. 584142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 5855958ab88f721d312b531d17705fc3ed54102fa05Jack Steiner gts->ustats.fmm_tlbmiss++; 586cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner if (!gts->ts_force_cch_reload && 587cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner down_read_trylock(>s->ts_mm->mmap_sem)) { 5882ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner gru_try_dropin(gru, gts, tfh, NULL); 589142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner up_read(>s->ts_mm->mmap_sem); 590142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } else { 591142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh_user_polling_mode(tfh); 592438846043af39fef286f68b197d42bd75f49898eJack Steiner STAT(intr_mm_lock_failed); 593142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } 594142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } 595142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return IRQ_HANDLED; 596142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 597142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 5984107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steinerirqreturn_t gru0_intr(int irq, void *dev_id) 5994107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner{ 6004107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner return gru_intr(0, uv_numa_blade_id()); 6014107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner} 6024107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner 6034107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steinerirqreturn_t gru1_intr(int irq, void *dev_id) 6044107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner{ 6054107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner return gru_intr(1, uv_numa_blade_id()); 6064107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner} 6074107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner 6084107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steinerirqreturn_t gru_intr_mblade(int irq, void *dev_id) 6094107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner{ 6104107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner int blade; 6114107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner 6124107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner for_each_possible_blade(blade) { 6134107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner if (uv_blade_nr_possible_cpus(blade)) 6144107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner continue; 6154107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner gru_intr(0, blade); 6164107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner gru_intr(1, blade); 6174107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner } 6184107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner return IRQ_HANDLED; 6194107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner} 6204107e1d38a23028c2a3bc23dd948265dbe6becbaJack Steiner 621142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 622142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerstatic int gru_user_dropin(struct gru_thread_state *gts, 623142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_tlb_fault_handle *tfh, 624b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner void *cb) 625142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 626142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_mm_struct *gms = gts->ts_gms; 627142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner int ret; 628142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 6295958ab88f721d312b531d17705fc3ed54102fa05Jack Steiner gts->ustats.upm_tlbmiss++; 630142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner while (1) { 631142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner wait_event(gms->ms_wait_queue, 632142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner atomic_read(&gms->ms_range_active) == 0); 633142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner prefetchw(tfh); /* Helps on hdw, required for emulator */ 6342ce4d4c937ce4c14704f02ae60e2e07e3906c68aJack Steiner ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb); 635142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (ret <= 0) 636142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return ret; 637142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(call_os_wait_queue); 638142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } 639142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 640142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 641142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 642142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * This interface is called as a result of a user detecting a "call OS" bit 643142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * in a user CB. Normally means that a TLB fault has occurred. 644142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * cb - user virtual address of the CB 645142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 646142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerint gru_handle_user_call_os(unsigned long cb) 647142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 648142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_tlb_fault_handle *tfh; 649142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_thread_state *gts; 650b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner void *cbk; 651142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner int ucbnum, cbrnum, ret = -EINVAL; 652142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 653142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(call_os); 654142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 655142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner /* sanity check the cb pointer */ 656142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner ucbnum = get_cb_number((void *)cb); 657142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if ((cb & (GRU_HANDLE_STRIDE - 1)) || ucbnum >= GRU_NUM_CB) 658142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EINVAL; 659142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 660142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gts = gru_find_lock_gts(cb); 661142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (!gts) 662142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EINVAL; 663563447d7eb04c9b382f90a132be126a21a635647Jack Steiner gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts); 664142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 665fe5bb6b00c3a9374841d651e01694fe4190a677eJack Steiner if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) 666142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner goto exit; 667142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 66855484c45dbeca2eec7642932ec3f60f8a2d4bdbfJack Steiner gru_check_context_placement(gts); 669fe5bb6b00c3a9374841d651e01694fe4190a677eJack Steiner 6707b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner /* 6717b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner * CCH may contain stale data if ts_force_cch_reload is set. 6727b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner */ 6737b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner if (gts->ts_gru && gts->ts_force_cch_reload) { 6747b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner gts->ts_force_cch_reload = 0; 67599f7c229b32bdf7424fbeb1d0d1b3883e14e97d0Jack Steiner gru_update_cch(gts); 6767b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner } 6777b8274e93fbabc7534aa51f95551c30aecdd0066Jack Steiner 678142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner ret = -EAGAIN; 679142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner cbrnum = thread_cbr_number(gts, ucbnum); 68099f7c229b32bdf7424fbeb1d0d1b3883e14e97d0Jack Steiner if (gts->ts_gru) { 681142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner tfh = get_tfh_by_index(gts->ts_gru, cbrnum); 682b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner cbk = get_gseg_base_address_cb(gts->ts_gru->gs_gru_base_vaddr, 683b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner gts->ts_ctxnum, ucbnum); 684b61fc69bb61838e99b368d208e6e2ee363eba2a1Jack Steiner ret = gru_user_dropin(gts, tfh, cbk); 685142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } 686142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerexit: 687142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_unlock_gts(gts); 688142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return ret; 689142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 690142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 691142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 692142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Fetch the exception detail information for a CB that terminated with 693142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * an exception. 694142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 695142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerint gru_get_exception_detail(unsigned long arg) 696142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 697142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct control_block_extended_exc_detail excdet; 698142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_control_block_extended *cbe; 699142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_thread_state *gts; 700142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner int ucbnum, cbrnum, ret; 701142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 702142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(user_exception); 703142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (copy_from_user(&excdet, (void __user *)arg, sizeof(excdet))) 704142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EFAULT; 705142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 706142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gts = gru_find_lock_gts(excdet.cb); 707142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (!gts) 708142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EINVAL; 709142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 710563447d7eb04c9b382f90a132be126a21a635647Jack Steiner gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", excdet.cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts); 711fe5bb6b00c3a9374841d651e01694fe4190a677eJack Steiner ucbnum = get_cb_number((void *)excdet.cb); 712fe5bb6b00c3a9374841d651e01694fe4190a677eJack Steiner if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) { 713fe5bb6b00c3a9374841d651e01694fe4190a677eJack Steiner ret = -EINVAL; 714fe5bb6b00c3a9374841d651e01694fe4190a677eJack Steiner } else if (gts->ts_gru) { 715142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner cbrnum = thread_cbr_number(gts, ucbnum); 716142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner cbe = get_cbe_by_index(gts->ts_gru, cbrnum); 7171a2c09e3b41e334b6651d53b39cfe8ceefbc45f8Jack Steiner gru_flush_cache(cbe); /* CBE not coherent */ 71867bf04a5c2574e9495f660f418f6df776821d578Jack Steiner sync_core(); /* make sure we are have current data */ 719142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner excdet.opc = cbe->opccpy; 720142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner excdet.exopc = cbe->exopccpy; 721142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner excdet.ecause = cbe->ecause; 722142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner excdet.exceptdet0 = cbe->idef1upd; 723142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner excdet.exceptdet1 = cbe->idef3upd; 724cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner excdet.cbrstate = cbe->cbrstate; 725cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner excdet.cbrexecstatus = cbe->cbrexecstatus; 726c550222f64265c9384c45d7f691445386025a82bJack Steiner gru_flush_cache_cbe(cbe); 727142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner ret = 0; 728142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } else { 729142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner ret = -EAGAIN; 730142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner } 731142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_unlock_gts(gts); 732142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 733cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner gru_dbg(grudev, 734cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner "cb 0x%lx, op %d, exopc %d, cbrstate %d, cbrexecstatus 0x%x, ecause 0x%x, " 735cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner "exdet0 0x%lx, exdet1 0x%x\n", 736cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner excdet.cb, excdet.opc, excdet.exopc, excdet.cbrstate, excdet.cbrexecstatus, 737cd1334f03f7b799bc6893b511daf2080e8f73863Jack Steiner excdet.ecause, excdet.exceptdet0, excdet.exceptdet1); 738142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (!ret && copy_to_user((void __user *)arg, &excdet, sizeof(excdet))) 739142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner ret = -EFAULT; 740142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return ret; 741142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 742142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 743142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 744142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * User request to unload a context. Content is saved for possible reload. 745142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 746bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steinerstatic int gru_unload_all_contexts(void) 747bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner{ 748bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner struct gru_thread_state *gts; 749bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner struct gru_state *gru; 750e1c3219d06447dab8a87b96df1211d158e21071eJack Steiner int gid, ctxnum; 751bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner 752bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner if (!capable(CAP_SYS_ADMIN)) 753bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner return -EPERM; 754e1c3219d06447dab8a87b96df1211d158e21071eJack Steiner foreach_gid(gid) { 755bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner gru = GID_TO_GRU(gid); 756bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner spin_lock(&gru->gs_lock); 757bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) { 758bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner gts = gru->gs_gts[ctxnum]; 759bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner if (gts && mutex_trylock(>s->ts_ctxlock)) { 760bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner spin_unlock(&gru->gs_lock); 761bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner gru_unload_context(gts, 1); 762d57c82b10709bbb1deb7eb26cf42abcde8851e4dJack Steiner mutex_unlock(>s->ts_ctxlock); 763bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner spin_lock(&gru->gs_lock); 764bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner } 765bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner } 766bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner spin_unlock(&gru->gs_lock); 767bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner } 768bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner return 0; 769bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner} 770bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner 771142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerint gru_user_unload_context(unsigned long arg) 772142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 773142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_thread_state *gts; 774142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_unload_context_req req; 775142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 776142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(user_unload_context); 777142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (copy_from_user(&req, (void __user *)arg, sizeof(req))) 778142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EFAULT; 779142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 780142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, "gseg 0x%lx\n", req.gseg); 781142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 782bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner if (!req.gseg) 783bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner return gru_unload_all_contexts(); 784bb04aa78ec1393295c30edad154e6f6e906955f3Jack Steiner 785142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gts = gru_find_lock_gts(req.gseg); 786142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (!gts) 787142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EINVAL; 788142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 789142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (gts->ts_gru) 790142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_unload_context(gts, 1); 791142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_unlock_gts(gts); 792142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 793142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 0; 794142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 795142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 796142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 797142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * User request to flush a range of virtual addresses from the GRU TLB 798142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * (Mainly for testing). 799142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 800142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steinerint gru_user_flush_tlb(unsigned long arg) 801142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 802142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_thread_state *gts; 803142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_flush_tlb_req req; 8041926ee85a903d189c5702eed6531be321e33eb47Jack Steiner struct gru_mm_struct *gms; 805142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 806142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner STAT(user_flush_tlb); 807142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (copy_from_user(&req, (void __user *)arg, sizeof(req))) 808142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EFAULT; 809142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 810142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_dbg(grudev, "gseg 0x%lx, vaddr 0x%lx, len 0x%lx\n", req.gseg, 811142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner req.vaddr, req.len); 812142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 813142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gts = gru_find_lock_gts(req.gseg); 814142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner if (!gts) 815142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return -EINVAL; 816142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 8171926ee85a903d189c5702eed6531be321e33eb47Jack Steiner gms = gts->ts_gms; 818142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_unlock_gts(gts); 8191926ee85a903d189c5702eed6531be321e33eb47Jack Steiner gru_flush_tlb_range(gms, req.vaddr, req.len); 820142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 821142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner return 0; 822142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 823142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 824142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner/* 8257e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner * Fetch GSEG statisticss 8267e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner */ 8277e796a72a2691d7094fd62da61097294d0d59ce4Jack Steinerlong gru_get_gseg_statistics(unsigned long arg) 8287e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner{ 8297e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner struct gru_thread_state *gts; 8307e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner struct gru_get_gseg_statistics_req req; 8317e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner 8327e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner if (copy_from_user(&req, (void __user *)arg, sizeof(req))) 8337e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner return -EFAULT; 8347e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner 835091f1a10614db89a108cda980425799730d73d8aJack Steiner /* 836091f1a10614db89a108cda980425799730d73d8aJack Steiner * The library creates arrays of contexts for threaded programs. 837091f1a10614db89a108cda980425799730d73d8aJack Steiner * If no gts exists in the array, the context has never been used & all 838091f1a10614db89a108cda980425799730d73d8aJack Steiner * statistics are implicitly 0. 839091f1a10614db89a108cda980425799730d73d8aJack Steiner */ 8407e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner gts = gru_find_lock_gts(req.gseg); 8417e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner if (gts) { 8427e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner memcpy(&req.stats, >s->ustats, sizeof(gts->ustats)); 8437e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner gru_unlock_gts(gts); 8447e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner } else { 8457e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner memset(&req.stats, 0, sizeof(gts->ustats)); 8467e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner } 8477e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner 8487e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner if (copy_to_user((void __user *)arg, &req, sizeof(req))) 8497e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner return -EFAULT; 8507e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner 8517e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner return 0; 8527e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner} 8537e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner 8547e796a72a2691d7094fd62da61097294d0d59ce4Jack Steiner/* 855142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Register the current task as the user of the GSEG slice. 856142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner * Needed for TLB fault interrupt targeting. 857142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner */ 85892b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steinerint gru_set_context_option(unsigned long arg) 859142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner{ 860142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner struct gru_thread_state *gts; 86192b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner struct gru_set_context_option_req req; 86292b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner int ret = 0; 863142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 86492b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner STAT(set_context_option); 86592b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner if (copy_from_user(&req, (void __user *)arg, sizeof(req))) 86692b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner return -EFAULT; 86792b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner gru_dbg(grudev, "op %d, gseg 0x%lx, value1 0x%lx\n", req.op, req.gseg, req.val1); 86892b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner 869c550222f64265c9384c45d7f691445386025a82bJack Steiner gts = gru_find_lock_gts(req.gseg); 870c550222f64265c9384c45d7f691445386025a82bJack Steiner if (!gts) { 871c550222f64265c9384c45d7f691445386025a82bJack Steiner gts = gru_alloc_locked_gts(req.gseg); 872c550222f64265c9384c45d7f691445386025a82bJack Steiner if (IS_ERR(gts)) 873c550222f64265c9384c45d7f691445386025a82bJack Steiner return PTR_ERR(gts); 874c550222f64265c9384c45d7f691445386025a82bJack Steiner } 875142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 87692b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner switch (req.op) { 877518e5cd4aae476042bdee511e0e00c8670c0df42Jack Steiner case sco_blade_chiplet: 878518e5cd4aae476042bdee511e0e00c8670c0df42Jack Steiner /* Select blade/chiplet for GRU context */ 879a7d0dabb3e863a1d4018235c1384b4318322116aDimitri Sivanich if (req.val0 < -1 || req.val0 >= GRU_CHIPLETS_PER_HUB || 880a7d0dabb3e863a1d4018235c1384b4318322116aDimitri Sivanich req.val1 < -1 || req.val1 >= GRU_MAX_BLADES || 881a7d0dabb3e863a1d4018235c1384b4318322116aDimitri Sivanich (req.val1 >= 0 && !gru_base[req.val1])) { 882518e5cd4aae476042bdee511e0e00c8670c0df42Jack Steiner ret = -EINVAL; 883518e5cd4aae476042bdee511e0e00c8670c0df42Jack Steiner } else { 884518e5cd4aae476042bdee511e0e00c8670c0df42Jack Steiner gts->ts_user_blade_id = req.val1; 885518e5cd4aae476042bdee511e0e00c8670c0df42Jack Steiner gts->ts_user_chiplet_id = req.val0; 88655484c45dbeca2eec7642932ec3f60f8a2d4bdbfJack Steiner gru_check_context_placement(gts); 887518e5cd4aae476042bdee511e0e00c8670c0df42Jack Steiner } 888518e5cd4aae476042bdee511e0e00c8670c0df42Jack Steiner break; 88992b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner case sco_gseg_owner: 89092b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner /* Register the current task as the GSEG owner */ 89192b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner gts->ts_tgid_owner = current->tgid; 89292b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner break; 893b1b19fcfa417cf62447413d6e8b9b6598adf00b9Jack Steiner case sco_cch_req_slice: 894b1b19fcfa417cf62447413d6e8b9b6598adf00b9Jack Steiner /* Set the CCH slice option */ 895b1b19fcfa417cf62447413d6e8b9b6598adf00b9Jack Steiner gts->ts_cch_req_slice = req.val1 & 3; 896b1b19fcfa417cf62447413d6e8b9b6598adf00b9Jack Steiner break; 89792b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner default: 89892b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner ret = -EINVAL; 89992b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner } 900142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner gru_unlock_gts(gts); 901142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner 90292b39388eeb45326feb0fa8bd69dbbce66c9efbfJack Steiner return ret; 903142586409c8be7dc071bb94d7cd2d69ccfd99b6bJack Steiner} 904