15ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy/* 25ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * VFIO: IOMMU DMA mapping support for TCE on POWER 35ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * 45ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * Copyright (C) 2013 IBM Corp. All rights reserved. 55ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * Author: Alexey Kardashevskiy <aik@ozlabs.ru> 65ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * 75ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * This program is free software; you can redistribute it and/or modify 85ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * it under the terms of the GNU General Public License version 2 as 95ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * published by the Free Software Foundation. 105ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * 115ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * Derived from original vfio_iommu_type1.c: 125ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 135ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * Author: Alex Williamson <alex.williamson@redhat.com> 145ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy */ 155ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 165ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#include <linux/module.h> 175ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#include <linux/pci.h> 185ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#include <linux/slab.h> 195ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#include <linux/uaccess.h> 205ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#include <linux/err.h> 215ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#include <linux/vfio.h> 225ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#include <asm/iommu.h> 235ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#include <asm/tce.h> 245ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 255ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#define DRIVER_VERSION "0.1" 265ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#define DRIVER_AUTHOR "aik@ozlabs.ru" 275ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy#define DRIVER_DESC "VFIO IOMMU SPAPR TCE" 285ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 295ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic void tce_iommu_detach_group(void *iommu_data, 305ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_group *iommu_group); 315ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 325ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy/* 335ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation 345ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * 355ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * This code handles mapping and unmapping of user data buffers 365ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * into DMA'ble space using the IOMMU 375ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy */ 385ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 395ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy/* 405ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * The container descriptor supports only a single group per container. 415ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * Required by the API as the container is not supplied with the IOMMU group 425ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * at the moment of initialization. 435ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy */ 445ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystruct tce_container { 455ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct mutex lock; 465ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_table *tbl; 475ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy bool enabled; 485ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy}; 495ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 505ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic int tce_iommu_enable(struct tce_container *container) 515ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 525ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy int ret = 0; 535ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy unsigned long locked, lock_limit, npages; 545ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_table *tbl = container->tbl; 555ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 565ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (!container->tbl) 575ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -ENXIO; 585ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 595ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (!current->mm) 605ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -ESRCH; /* process exited */ 615ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 625ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (container->enabled) 635ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EBUSY; 645ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 655ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy /* 665ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * When userspace pages are mapped into the IOMMU, they are effectively 675ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * locked memory, so, theoretically, we need to update the accounting 685ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * of locked pages on each map and unmap. For powerpc, the map unmap 695ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * paths can be very hot, though, and the accounting would kill 705ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * performance, especially since it would be difficult to impossible 715ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * to handle the accounting in real mode only. 725ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * 735ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * To address that, rather than precisely accounting every page, we 745ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * instead account for a worst case on locked memory when the iommu is 755ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * enabled and disabled. The worst case upper bound on locked memory 765ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * is the size of the whole iommu window, which is usually relatively 775ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * small (compared to total memory sizes) on POWER hardware. 785ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * 795ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits, 805ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * that would effectively kill the guest at random points, much better 815ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy * enforcing the limit based on the max that the guest can map. 825ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy */ 835ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy down_write(¤t->mm->mmap_sem); 84e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple npages = (tbl->it_size << IOMMU_PAGE_SHIFT_4K) >> PAGE_SHIFT; 855ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy locked = current->mm->locked_vm + npages; 865ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 875ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (locked > lock_limit && !capable(CAP_IPC_LOCK)) { 885ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy pr_warn("RLIMIT_MEMLOCK (%ld) exceeded\n", 895ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy rlimit(RLIMIT_MEMLOCK)); 905ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = -ENOMEM; 915ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } else { 925ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 935ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy current->mm->locked_vm += npages; 945ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy container->enabled = true; 955ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 965ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy up_write(¤t->mm->mmap_sem); 975ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 985ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ret; 995ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 1005ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1015ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic void tce_iommu_disable(struct tce_container *container) 1025ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 1035ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (!container->enabled) 1045ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return; 1055ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1065ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy container->enabled = false; 1075ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1085ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (!container->tbl || !current->mm) 1095ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return; 1105ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1115ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy down_write(¤t->mm->mmap_sem); 1125ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy current->mm->locked_vm -= (container->tbl->it_size << 113e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple IOMMU_PAGE_SHIFT_4K) >> PAGE_SHIFT; 1145ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy up_write(¤t->mm->mmap_sem); 1155ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 1165ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1175ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic void *tce_iommu_open(unsigned long arg) 1185ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 1195ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct tce_container *container; 1205ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1215ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (arg != VFIO_SPAPR_TCE_IOMMU) { 1225ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy pr_err("tce_vfio: Wrong IOMMU type\n"); 1235ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ERR_PTR(-EINVAL); 1245ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 1255ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1265ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy container = kzalloc(sizeof(*container), GFP_KERNEL); 1275ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (!container) 1285ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ERR_PTR(-ENOMEM); 1295ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1305ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_init(&container->lock); 1315ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1325ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return container; 1335ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 1345ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1355ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic void tce_iommu_release(void *iommu_data) 1365ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 1375ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct tce_container *container = iommu_data; 1385ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1395ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy WARN_ON(container->tbl && !container->tbl->it_group); 1405ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy tce_iommu_disable(container); 1415ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1425ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (container->tbl && container->tbl->it_group) 1435ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy tce_iommu_detach_group(iommu_data, container->tbl->it_group); 1445ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1455ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_destroy(&container->lock); 1465ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1475ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy kfree(container); 1485ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 1495ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1505ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic long tce_iommu_ioctl(void *iommu_data, 1515ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy unsigned int cmd, unsigned long arg) 1525ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 1535ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct tce_container *container = iommu_data; 1545ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy unsigned long minsz; 1555ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy long ret; 1565ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1575ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy switch (cmd) { 1585ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy case VFIO_CHECK_EXTENSION: 1591b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan switch (arg) { 1601b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan case VFIO_SPAPR_TCE_IOMMU: 1611b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan ret = 1; 1621b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan break; 1631b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan default: 1641b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan ret = vfio_spapr_iommu_eeh_ioctl(NULL, cmd, arg); 1651b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan break; 1661b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan } 1671b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan 1681b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan return (ret < 0) ? 0 : ret; 1695ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1705ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy case VFIO_IOMMU_SPAPR_TCE_GET_INFO: { 1715ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct vfio_iommu_spapr_tce_info info; 1725ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_table *tbl = container->tbl; 1735ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1745ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (WARN_ON(!tbl)) 1755ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -ENXIO; 1765ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1775ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_spapr_tce_info, 1785ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy dma32_window_size); 1795ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1805ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (copy_from_user(&info, (void __user *)arg, minsz)) 1815ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EFAULT; 1825ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1835ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (info.argsz < minsz) 1845ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EINVAL; 1855ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 186e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple info.dma32_window_start = tbl->it_offset << IOMMU_PAGE_SHIFT_4K; 187e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple info.dma32_window_size = tbl->it_size << IOMMU_PAGE_SHIFT_4K; 1885ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy info.flags = 0; 1895ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1905ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (copy_to_user((void __user *)arg, &info, minsz)) 1915ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EFAULT; 1925ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 1935ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return 0; 1945ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 1955ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy case VFIO_IOMMU_MAP_DMA: { 1965ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct vfio_iommu_type1_dma_map param; 1975ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_table *tbl = container->tbl; 1985ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy unsigned long tce, i; 1995ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2005ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (!tbl) 2015ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -ENXIO; 2025ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2035ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy BUG_ON(!tbl->it_group); 2045ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2055ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); 2065ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2075ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 2085ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EFAULT; 2095ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2105ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (param.argsz < minsz) 2115ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EINVAL; 2125ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2135ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ | 2145ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy VFIO_DMA_MAP_FLAG_WRITE)) 2155ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EINVAL; 2165ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 217e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple if ((param.size & ~IOMMU_PAGE_MASK_4K) || 218e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple (param.vaddr & ~IOMMU_PAGE_MASK_4K)) 2195ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EINVAL; 2205ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2215ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy /* iova is checked by the IOMMU API */ 2225ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy tce = param.vaddr; 2235ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_READ) 2245ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy tce |= TCE_PCI_READ; 2255ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (param.flags & VFIO_DMA_MAP_FLAG_WRITE) 2265ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy tce |= TCE_PCI_WRITE; 2275ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2285ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = iommu_tce_put_param_check(tbl, param.iova, tce); 2295ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (ret) 2305ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ret; 2315ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 232e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple for (i = 0; i < (param.size >> IOMMU_PAGE_SHIFT_4K); ++i) { 2335ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = iommu_put_tce_user_mode(tbl, 234e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple (param.iova >> IOMMU_PAGE_SHIFT_4K) + i, 2355ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy tce); 2365ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (ret) 2375ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy break; 238e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple tce += IOMMU_PAGE_SIZE_4K; 2395ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 2405ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (ret) 2415ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_clear_tces_and_put_pages(tbl, 242e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple param.iova >> IOMMU_PAGE_SHIFT_4K, i); 2435ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2445ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_flush_tce(tbl); 2455ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2465ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ret; 2475ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 2485ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy case VFIO_IOMMU_UNMAP_DMA: { 2495ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct vfio_iommu_type1_dma_unmap param; 2505ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_table *tbl = container->tbl; 2515ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2525ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (WARN_ON(!tbl)) 2535ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -ENXIO; 2545ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2555ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, 2565ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy size); 2575ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2585ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (copy_from_user(¶m, (void __user *)arg, minsz)) 2595ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EFAULT; 2605ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2615ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (param.argsz < minsz) 2625ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EINVAL; 2635ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2645ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy /* No flag is supported now */ 2655ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (param.flags) 2665ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EINVAL; 2675ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 268e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple if (param.size & ~IOMMU_PAGE_MASK_4K) 2695ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -EINVAL; 2705ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2715ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = iommu_tce_clear_param_check(tbl, param.iova, 0, 272e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple param.size >> IOMMU_PAGE_SHIFT_4K); 2735ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (ret) 2745ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ret; 2755ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2765ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = iommu_clear_tces_and_put_pages(tbl, 277e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple param.iova >> IOMMU_PAGE_SHIFT_4K, 278e589a4404fa06730355de204d3d136ed9bbc7deaAlistair Popple param.size >> IOMMU_PAGE_SHIFT_4K); 2795ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_flush_tce(tbl); 2805ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2815ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ret; 2825ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 2835ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy case VFIO_IOMMU_ENABLE: 2845ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_lock(&container->lock); 2855ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = tce_iommu_enable(container); 2865ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_unlock(&container->lock); 2875ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ret; 2885ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2895ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 2905ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy case VFIO_IOMMU_DISABLE: 2915ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_lock(&container->lock); 2925ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy tce_iommu_disable(container); 2935ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_unlock(&container->lock); 2945ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return 0; 2951b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan case VFIO_EEH_PE_OP: 2961b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan if (!container->tbl || !container->tbl->it_group) 2971b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan return -ENODEV; 2981b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan 2991b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan return vfio_spapr_iommu_eeh_ioctl(container->tbl->it_group, 3001b69be5e8afc634f39ad695a6ab6aad0cf0975c7Gavin Shan cmd, arg); 3015ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 3025ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3035ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return -ENOTTY; 3045ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 3055ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3065ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic int tce_iommu_attach_group(void *iommu_data, 3075ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_group *iommu_group) 3085ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 3095ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy int ret; 3105ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct tce_container *container = iommu_data; 3115ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_table *tbl = iommu_group_get_iommudata(iommu_group); 3125ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3135ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy BUG_ON(!tbl); 3145ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_lock(&container->lock); 3155ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3165ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n", 3175ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_group_id(iommu_group), iommu_group); */ 3185ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (container->tbl) { 3195ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy pr_warn("tce_vfio: Only one group per IOMMU container is allowed, existing id=%d, attaching id=%d\n", 3205ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_group_id(container->tbl->it_group), 3215ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_group_id(iommu_group)); 3225ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = -EBUSY; 3235ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } else if (container->enabled) { 3245ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy pr_err("tce_vfio: attaching group #%u to enabled container\n", 3255ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_group_id(iommu_group)); 3265ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = -EBUSY; 3275ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } else { 3285ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy ret = iommu_take_ownership(tbl); 3295ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (!ret) 3305ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy container->tbl = tbl; 3315ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 3325ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3335ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_unlock(&container->lock); 3345ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3355ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return ret; 3365ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 3375ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3385ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic void tce_iommu_detach_group(void *iommu_data, 3395ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_group *iommu_group) 3405ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 3415ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct tce_container *container = iommu_data; 3425ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy struct iommu_table *tbl = iommu_group_get_iommudata(iommu_group); 3435ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3445ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy BUG_ON(!tbl); 3455ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_lock(&container->lock); 3465ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (tbl != container->tbl) { 3475ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy pr_warn("tce_vfio: detaching group #%u, expected group is #%u\n", 3485ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_group_id(iommu_group), 3495ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_group_id(tbl->it_group)); 3505ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } else { 3515ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy if (container->enabled) { 3525ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy pr_warn("tce_vfio: detaching group #%u from enabled container, forcing disable\n", 3535ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_group_id(tbl->it_group)); 3545ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy tce_iommu_disable(container); 3555ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 3565ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3575ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy /* pr_debug("tce_vfio: detaching group #%u from iommu %p\n", 3585ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_group_id(iommu_group), iommu_group); */ 3595ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy container->tbl = NULL; 3605ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy iommu_release_ownership(tbl); 3615ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy } 3625ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy mutex_unlock(&container->lock); 3635ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 3645ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3655ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiyconst struct vfio_iommu_driver_ops tce_iommu_driver_ops = { 3665ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy .name = "iommu-vfio-powerpc", 3675ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy .owner = THIS_MODULE, 3685ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy .open = tce_iommu_open, 3695ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy .release = tce_iommu_release, 3705ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy .ioctl = tce_iommu_ioctl, 3715ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy .attach_group = tce_iommu_attach_group, 3725ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy .detach_group = tce_iommu_detach_group, 3735ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy}; 3745ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3755ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic int __init tce_iommu_init(void) 3765ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 3775ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy return vfio_register_iommu_driver(&tce_iommu_driver_ops); 3785ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 3795ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3805ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiystatic void __exit tce_iommu_cleanup(void) 3815ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy{ 3825ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy vfio_unregister_iommu_driver(&tce_iommu_driver_ops); 3835ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy} 3845ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3855ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiymodule_init(tce_iommu_init); 3865ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiymodule_exit(tce_iommu_cleanup); 3875ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 3885ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey KardashevskiyMODULE_VERSION(DRIVER_VERSION); 3895ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey KardashevskiyMODULE_LICENSE("GPL v2"); 3905ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey KardashevskiyMODULE_AUTHOR(DRIVER_AUTHOR); 3915ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey KardashevskiyMODULE_DESCRIPTION(DRIVER_DESC); 3925ffd229c02731a91d08ca21e76b503c5bbb5c095Alexey Kardashevskiy 393