1/* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2007 - 2009 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 */ 22 23/* 24 * This driver supports an Intel I/OAT DMA engine, which does asynchronous 25 * copy operations. 26 */ 27 28#include <linux/init.h> 29#include <linux/module.h> 30#include <linux/pci.h> 31#include <linux/interrupt.h> 32#include <linux/dca.h> 33#include <linux/slab.h> 34#include "dma.h" 35#include "dma_v2.h" 36#include "registers.h" 37#include "hw.h" 38 39MODULE_VERSION(IOAT_DMA_VERSION); 40MODULE_LICENSE("Dual BSD/GPL"); 41MODULE_AUTHOR("Intel Corporation"); 42 43static struct pci_device_id ioat_pci_tbl[] = { 44 /* I/OAT v1 platforms */ 45 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, 46 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB) }, 47 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) }, 48 { PCI_VDEVICE(UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) }, 49 50 /* I/OAT v2 platforms */ 51 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) }, 52 53 /* I/OAT v3 platforms */ 54 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) }, 55 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) }, 56 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) }, 57 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) }, 58 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) }, 59 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) }, 60 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) }, 61 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) }, 62 63 /* I/OAT v3.2 platforms */ 64 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) }, 65 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) }, 66 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) }, 67 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) }, 68 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) }, 69 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) }, 70 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) }, 71 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) }, 72 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) }, 73 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) }, 74 75 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) }, 76 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) }, 77 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) }, 78 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) }, 79 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) }, 80 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) }, 81 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) }, 82 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) }, 83 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) }, 84 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) }, 85 86 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) }, 87 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) }, 88 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) }, 89 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) }, 90 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) }, 91 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) }, 92 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) }, 93 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) }, 94 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) }, 95 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) }, 96 97 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) }, 98 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) }, 99 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) }, 100 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) }, 101 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) }, 102 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) }, 103 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) }, 104 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) }, 105 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) }, 106 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) }, 107 108 /* I/OAT v3.3 platforms */ 109 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) }, 110 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) }, 111 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) }, 112 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) }, 113 114 { 0, } 115}; 116MODULE_DEVICE_TABLE(pci, ioat_pci_tbl); 117 118static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id); 119static void ioat_remove(struct pci_dev *pdev); 120 121static int ioat_dca_enabled = 1; 122module_param(ioat_dca_enabled, int, 0644); 123MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); 124 125struct kmem_cache *ioat2_cache; 126struct kmem_cache *ioat3_sed_cache; 127 128#define DRV_NAME "ioatdma" 129 130static struct pci_driver ioat_pci_driver = { 131 .name = DRV_NAME, 132 .id_table = ioat_pci_tbl, 133 .probe = ioat_pci_probe, 134 .remove = ioat_remove, 135}; 136 137static struct ioatdma_device * 138alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase) 139{ 140 struct device *dev = &pdev->dev; 141 struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); 142 143 if (!d) 144 return NULL; 145 d->pdev = pdev; 146 d->reg_base = iobase; 147 return d; 148} 149 150static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 151{ 152 void __iomem * const *iomap; 153 struct device *dev = &pdev->dev; 154 struct ioatdma_device *device; 155 int err; 156 157 err = pcim_enable_device(pdev); 158 if (err) 159 return err; 160 161 err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME); 162 if (err) 163 return err; 164 iomap = pcim_iomap_table(pdev); 165 if (!iomap) 166 return -ENOMEM; 167 168 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 169 if (err) 170 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 171 if (err) 172 return err; 173 174 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 175 if (err) 176 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 177 if (err) 178 return err; 179 180 device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]); 181 if (!device) 182 return -ENOMEM; 183 pci_set_master(pdev); 184 pci_set_drvdata(pdev, device); 185 186 device->version = readb(device->reg_base + IOAT_VER_OFFSET); 187 if (device->version == IOAT_VER_1_2) 188 err = ioat1_dma_probe(device, ioat_dca_enabled); 189 else if (device->version == IOAT_VER_2_0) 190 err = ioat2_dma_probe(device, ioat_dca_enabled); 191 else if (device->version >= IOAT_VER_3_0) 192 err = ioat3_dma_probe(device, ioat_dca_enabled); 193 else 194 return -ENODEV; 195 196 if (err) { 197 dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n"); 198 return -ENODEV; 199 } 200 201 return 0; 202} 203 204static void ioat_remove(struct pci_dev *pdev) 205{ 206 struct ioatdma_device *device = pci_get_drvdata(pdev); 207 208 if (!device) 209 return; 210 211 dev_err(&pdev->dev, "Removing dma and dca services\n"); 212 if (device->dca) { 213 unregister_dca_provider(device->dca, &pdev->dev); 214 free_dca_provider(device->dca); 215 device->dca = NULL; 216 } 217 ioat_dma_remove(device); 218} 219 220static int __init ioat_init_module(void) 221{ 222 int err = -ENOMEM; 223 224 pr_info("%s: Intel(R) QuickData Technology Driver %s\n", 225 DRV_NAME, IOAT_DMA_VERSION); 226 227 ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent), 228 0, SLAB_HWCACHE_ALIGN, NULL); 229 if (!ioat2_cache) 230 return -ENOMEM; 231 232 ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0); 233 if (!ioat3_sed_cache) 234 goto err_ioat2_cache; 235 236 err = pci_register_driver(&ioat_pci_driver); 237 if (err) 238 goto err_ioat3_cache; 239 240 return 0; 241 242 err_ioat3_cache: 243 kmem_cache_destroy(ioat3_sed_cache); 244 245 err_ioat2_cache: 246 kmem_cache_destroy(ioat2_cache); 247 248 return err; 249} 250module_init(ioat_init_module); 251 252static void __exit ioat_exit_module(void) 253{ 254 pci_unregister_driver(&ioat_pci_driver); 255 kmem_cache_destroy(ioat2_cache); 256} 257module_exit(ioat_exit_module); 258