vfio_pci_intrs.c revision 9dbdfd23b7638d054f3b0e70c64dfb9f297f2a9f
189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson/* 289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * VFIO PCI interrupt handling 389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * 489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Author: Alex Williamson <alex.williamson@redhat.com> 689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * 789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * This program is free software; you can redistribute it and/or modify 889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * it under the terms of the GNU General Public License version 2 as 989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * published by the Free Software Foundation. 1089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * 1189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Derived from original vfio: 1289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Copyright 2010 Cisco Systems, Inc. All rights reserved. 1389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Author: Tom Lyon, pugs@cisco.com 1489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 1589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 1689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/device.h> 1789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/interrupt.h> 1889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/eventfd.h> 1989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/pci.h> 2089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/file.h> 2189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/poll.h> 2289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/vfio.h> 2389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/wait.h> 2489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include <linux/workqueue.h> 2589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 2689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson#include "vfio_pci_private.h" 2789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 2889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson/* 2989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * IRQfd - generic 3089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 3189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstruct virqfd { 3289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct vfio_pci_device *vdev; 3389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct eventfd_ctx *eventfd; 3489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int (*handler)(struct vfio_pci_device *, void *); 3589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson void (*thread)(struct vfio_pci_device *, void *); 3689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson void *data; 3789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct work_struct inject; 3889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson wait_queue_t wait; 3989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson poll_table pt; 4089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct work_struct shutdown; 4189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct virqfd **pvirqfd; 4289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson}; 4389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 4489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic struct workqueue_struct *vfio_irqfd_cleanup_wq; 4589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 4689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonint __init vfio_pci_virqfd_init(void) 4789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 4889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_irqfd_cleanup_wq = 4989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson create_singlethread_workqueue("vfio-irqfd-cleanup"); 5089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vfio_irqfd_cleanup_wq) 5189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOMEM; 5289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 5389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 5489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 5589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 5689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonvoid vfio_pci_virqfd_exit(void) 5789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 5889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson destroy_workqueue(vfio_irqfd_cleanup_wq); 5989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 6089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 6189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic void virqfd_deactivate(struct virqfd *virqfd) 6289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 6389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown); 6489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 6589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 6689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key) 6789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 6889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct virqfd *virqfd = container_of(wait, struct virqfd, wait); 6989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned long flags = (unsigned long)key; 7089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 7189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (flags & POLLIN) { 7289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* An event has been signaled, call function */ 7389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if ((!virqfd->handler || 7489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->handler(virqfd->vdev, virqfd->data)) && 7589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->thread) 7689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson schedule_work(&virqfd->inject); 7789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 7889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 79b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson if (flags & POLLHUP) { 80b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson unsigned long flags; 81b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson spin_lock_irqsave(&virqfd->vdev->irqlock, flags); 82b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson 83b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson /* 84b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * The eventfd is closing, if the virqfd has not yet been 85b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * queued for release, as determined by testing whether the 86b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * vdev pointer to it is still valid, queue it now. As 87b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * with kvm irqfds, we know we won't race against the virqfd 88b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * going away because we hold wqh->lock to get here. 89b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson */ 90b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson if (*(virqfd->pvirqfd) == virqfd) { 91b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson *(virqfd->pvirqfd) = NULL; 92b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson virqfd_deactivate(virqfd); 93b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson } 94b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson 95b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson spin_unlock_irqrestore(&virqfd->vdev->irqlock, flags); 96b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson } 9789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 9889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 9989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 10089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 10189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic void virqfd_ptable_queue_proc(struct file *file, 10289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson wait_queue_head_t *wqh, poll_table *pt) 10389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 10489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct virqfd *virqfd = container_of(pt, struct virqfd, pt); 10589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson add_wait_queue(wqh, &virqfd->wait); 10689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 10789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 10889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic void virqfd_shutdown(struct work_struct *work) 10989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 11089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct virqfd *virqfd = container_of(work, struct virqfd, shutdown); 11189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson u64 cnt; 11289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 11389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt); 11489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson flush_work(&virqfd->inject); 11589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_ctx_put(virqfd->eventfd); 11689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 11789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(virqfd); 11889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 11989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 12089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic void virqfd_inject(struct work_struct *work) 12189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 12289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct virqfd *virqfd = container_of(work, struct virqfd, inject); 12389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (virqfd->thread) 12489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->thread(virqfd->vdev, virqfd->data); 12589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 12689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 12789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int virqfd_enable(struct vfio_pci_device *vdev, 12889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int (*handler)(struct vfio_pci_device *, void *), 12989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson void (*thread)(struct vfio_pci_device *, void *), 13089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson void *data, struct virqfd **pvirqfd, int fd) 13189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 13289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct file *file = NULL; 13389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct eventfd_ctx *ctx = NULL; 13489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct virqfd *virqfd; 13589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int ret = 0; 13689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned int events; 13789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 13889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL); 13989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!virqfd) 14089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOMEM; 14189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 14289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->pvirqfd = pvirqfd; 14389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->vdev = vdev; 14489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->handler = handler; 14589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->thread = thread; 14689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->data = data; 14789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 14889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson INIT_WORK(&virqfd->shutdown, virqfd_shutdown); 14989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson INIT_WORK(&virqfd->inject, virqfd_inject); 15089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 15189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson file = eventfd_fget(fd); 15289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (IS_ERR(file)) { 15389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = PTR_ERR(file); 15489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson goto fail; 15589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 15689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 15789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ctx = eventfd_ctx_fileget(file); 15889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (IS_ERR(ctx)) { 15989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = PTR_ERR(ctx); 16089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson goto fail; 16189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 16289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 16389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson virqfd->eventfd = ctx; 16489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 16589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 166b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * virqfds can be released by closing the eventfd or directly 167b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * through ioctl. These are both done through a workqueue, so 168b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * we update the pointer to the virqfd under lock to avoid 169b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * pushing multiple jobs to release the same virqfd. 170b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson */ 171b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson spin_lock_irq(&vdev->irqlock); 172b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson 173b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson if (*pvirqfd) { 174b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson spin_unlock_irq(&vdev->irqlock); 175b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson ret = -EBUSY; 176b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson goto fail; 177b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson } 178b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson *pvirqfd = virqfd; 179b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson 180b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson spin_unlock_irq(&vdev->irqlock); 181b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson 182b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson /* 18389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Install our own custom wake-up handling so we are notified via 18489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * a callback whenever someone signals the underlying eventfd. 18589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 18689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup); 18789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc); 18889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 18989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson events = file->f_op->poll(file, &virqfd->pt); 19089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 19189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 19289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Check if there was an event already pending on the eventfd 19389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * before we registered and trigger it as if we didn't miss it. 19489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 19589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (events & POLLIN) { 19689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if ((!handler || handler(vdev, data)) && thread) 19789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson schedule_work(&virqfd->inject); 19889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 19989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 20089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 20189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Do not drop the file until the irqfd is fully initialized, 20289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * otherwise we might race against the POLLHUP. 20389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 20489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson fput(file); 20589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 20689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 20789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 20889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonfail: 20989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ctx && !IS_ERR(ctx)) 21089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_ctx_put(ctx); 21189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 21289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (file && !IS_ERR(file)) 21389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson fput(file); 21489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 21589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(virqfd); 21689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 21789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 21889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 21989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 220b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamsonstatic void virqfd_disable(struct vfio_pci_device *vdev, 221b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson struct virqfd **pvirqfd) 22289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 223b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson unsigned long flags; 224b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson 225b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson spin_lock_irqsave(&vdev->irqlock, flags); 226b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson 227b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson if (*pvirqfd) { 228b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson virqfd_deactivate(*pvirqfd); 229b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson *pvirqfd = NULL; 230b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson } 23189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 232b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson spin_unlock_irqrestore(&vdev->irqlock, flags); 23389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 234b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson /* 235b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * Block until we know all outstanding shutdown jobs have completed. 236b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * Even if we don't queue the job, flush the wq to be sure it's 237b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson * been released. 238b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson */ 23989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson flush_workqueue(vfio_irqfd_cleanup_wq); 24089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 24189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 24289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson/* 24389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * INTx 24489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 24589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic void vfio_send_intx_eventfd(struct vfio_pci_device *vdev, void *unused) 24689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 24789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (likely(is_intx(vdev) && !vdev->virq_disabled)) 24889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_signal(vdev->ctx[0].trigger, 1); 24989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 25089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 25189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonvoid vfio_pci_intx_mask(struct vfio_pci_device *vdev) 25289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 25389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct pci_dev *pdev = vdev->pdev; 25489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned long flags; 25589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 25689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson spin_lock_irqsave(&vdev->irqlock, flags); 25789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 25889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 25989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Masking can come from interrupt, ioctl, or config space 26089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * via INTx disable. The latter means this can get called 26189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * even when not using intx delivery. In this case, just 26289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * try to have the physical bit follow the virtual bit. 26389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 26489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (unlikely(!is_intx(vdev))) { 26589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vdev->pci_2_3) 26689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson pci_intx(pdev, 0); 26789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (!vdev->ctx[0].masked) { 26889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 26989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Can't use check_and_mask here because we always want to 27089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * mask, not just when something is pending. 27189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 27289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vdev->pci_2_3) 27389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson pci_intx(pdev, 0); 27489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson else 27589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson disable_irq_nosync(pdev->irq); 27689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 27789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[0].masked = true; 27889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 27989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 28089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson spin_unlock_irqrestore(&vdev->irqlock, flags); 28189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 28289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 28389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson/* 28489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * If this is triggered by an eventfd, we can't call eventfd_signal 28589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * or else we'll deadlock on the eventfd wait queue. Return >0 when 28689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * a signal is necessary, which can then be handled via a work queue 28789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * or directly depending on the caller. 28889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 28989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonint vfio_pci_intx_unmask_handler(struct vfio_pci_device *vdev, void *unused) 29089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 29189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct pci_dev *pdev = vdev->pdev; 29289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned long flags; 29389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int ret = 0; 29489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 29589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson spin_lock_irqsave(&vdev->irqlock, flags); 29689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 29789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 29889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Unmasking comes from ioctl or config, so again, have the 29989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * physical bit follow the virtual even when not using INTx. 30089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 30189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (unlikely(!is_intx(vdev))) { 30289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vdev->pci_2_3) 30389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson pci_intx(pdev, 1); 30489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (vdev->ctx[0].masked && !vdev->virq_disabled) { 30589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 30689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * A pending interrupt here would immediately trigger, 30789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * but we can avoid that overhead by just re-sending 30889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * the interrupt to the user. 30989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 31089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vdev->pci_2_3) { 31189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!pci_check_and_unmask_intx(pdev)) 31289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = 1; 31389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else 31489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson enable_irq(pdev->irq); 31589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 31689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[0].masked = (ret > 0); 31789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 31889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 31989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson spin_unlock_irqrestore(&vdev->irqlock, flags); 32089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 32189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 32289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 32389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 32489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonvoid vfio_pci_intx_unmask(struct vfio_pci_device *vdev) 32589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 32689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0) 32789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_send_intx_eventfd(vdev, NULL); 32889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 32989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 33089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic irqreturn_t vfio_intx_handler(int irq, void *dev_id) 33189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 33289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct vfio_pci_device *vdev = dev_id; 33389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned long flags; 33489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int ret = IRQ_NONE; 33589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 33689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson spin_lock_irqsave(&vdev->irqlock, flags); 33789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 33889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->pci_2_3) { 33989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson disable_irq_nosync(vdev->pdev->irq); 34089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[0].masked = true; 34189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = IRQ_HANDLED; 34289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (!vdev->ctx[0].masked && /* may be shared */ 34389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson pci_check_and_mask_intx(vdev->pdev)) { 34489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[0].masked = true; 34589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = IRQ_HANDLED; 34689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 34789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 34889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson spin_unlock_irqrestore(&vdev->irqlock, flags); 34989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 35089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret == IRQ_HANDLED) 35189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_send_intx_eventfd(vdev, NULL); 35289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 35389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 35489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 35589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 35689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_intx_enable(struct vfio_pci_device *vdev) 35789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 35889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!is_irq_none(vdev)) 35989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 36089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 36189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->pdev->irq) 36289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENODEV; 36389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 36489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx = kzalloc(sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL); 36589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->ctx) 36689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOMEM; 36789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 36889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->num_ctx = 1; 36989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; 37089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 37189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 37289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 37389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 37489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd) 37589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 37689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct pci_dev *pdev = vdev->pdev; 37789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned long irqflags = IRQF_SHARED; 37889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct eventfd_ctx *trigger; 37989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned long flags; 38089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int ret; 38189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 38289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vdev->ctx[0].trigger) { 38389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson free_irq(pdev->irq, vdev); 38489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx[0].name); 38589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_ctx_put(vdev->ctx[0].trigger); 38689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[0].trigger = NULL; 38789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 38889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 38989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (fd < 0) /* Disable only */ 39089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 39189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 39289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[0].name = kasprintf(GFP_KERNEL, "vfio-intx(%s)", 39389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson pci_name(pdev)); 39489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->ctx[0].name) 39589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOMEM; 39689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 39789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson trigger = eventfd_ctx_fdget(fd); 39889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (IS_ERR(trigger)) { 39989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx[0].name); 40089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return PTR_ERR(trigger); 40189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 40289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 4039dbdfd23b7638d054f3b0e70c64dfb9f297f2a9fAlex Williamson vdev->ctx[0].trigger = trigger; 4049dbdfd23b7638d054f3b0e70c64dfb9f297f2a9fAlex Williamson 40589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->pci_2_3) 40689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson irqflags = 0; 40789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 40889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = request_irq(pdev->irq, vfio_intx_handler, 40989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson irqflags, vdev->ctx[0].name, vdev); 41089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) { 4119dbdfd23b7638d054f3b0e70c64dfb9f297f2a9fAlex Williamson vdev->ctx[0].trigger = NULL; 41289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx[0].name); 41389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_ctx_put(trigger); 41489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 41589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 41689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 41789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 41889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * INTx disable will stick across the new irq setup, 41989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * disable_irq won't. 42089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 42189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson spin_lock_irqsave(&vdev->irqlock, flags); 42289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->pci_2_3 && (vdev->ctx[0].masked || vdev->virq_disabled)) 42389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson disable_irq_nosync(pdev->irq); 42489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson spin_unlock_irqrestore(&vdev->irqlock, flags); 42589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 42689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 42789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 42889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 42989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic void vfio_intx_disable(struct vfio_pci_device *vdev) 43089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 43189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_intx_set_signal(vdev, -1); 432b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson virqfd_disable(vdev, &vdev->ctx[0].unmask); 433b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson virqfd_disable(vdev, &vdev->ctx[0].mask); 43489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->irq_type = VFIO_PCI_NUM_IRQS; 43589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->num_ctx = 0; 43689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx); 43789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 43889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 43989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson/* 44089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * MSI/MSI-X 44189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 44289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic irqreturn_t vfio_msihandler(int irq, void *arg) 44389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 44489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct eventfd_ctx *trigger = arg; 44589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 44689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_signal(trigger, 1); 44789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return IRQ_HANDLED; 44889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 44989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 45089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix) 45189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 45289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct pci_dev *pdev = vdev->pdev; 45389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int ret; 45489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 45589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!is_irq_none(vdev)) 45689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 45789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 45889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx = kzalloc(nvec * sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL); 45989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->ctx) 46089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOMEM; 46189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 46289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (msix) { 46389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int i; 46489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 46589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->msix = kzalloc(nvec * sizeof(struct msix_entry), 46689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson GFP_KERNEL); 46789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->msix) { 46889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx); 46989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOMEM; 47089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 47189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 47289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson for (i = 0; i < nvec; i++) 47389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->msix[i].entry = i; 47489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 47589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = pci_enable_msix(pdev, vdev->msix, nvec); 47689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) { 47789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->msix); 47889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx); 47989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 48089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 48189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else { 48289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = pci_enable_msi_block(pdev, nvec); 48389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) { 48489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx); 48589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 48689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 48789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 48889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 48989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->num_ctx = nvec; 49089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : 49189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson VFIO_PCI_MSI_IRQ_INDEX; 49289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 49389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!msix) { 49489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* 49589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * Compute the virtual hardware field for max msi vectors - 49689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * it is the log base 2 of the number of vectors. 49789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 49889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->msi_qmax = fls(nvec * 2 - 1) - 1; 49989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 50089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 50189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 50289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 50389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 50489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, 50589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int vector, int fd, bool msix) 50689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 50789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct pci_dev *pdev = vdev->pdev; 50889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int irq = msix ? vdev->msix[vector].vector : pdev->irq + vector; 50989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson char *name = msix ? "vfio-msix" : "vfio-msi"; 51089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct eventfd_ctx *trigger; 51189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int ret; 51289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 51389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vector >= vdev->num_ctx) 51489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 51589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 51689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vdev->ctx[vector].trigger) { 51789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson free_irq(irq, vdev->ctx[vector].trigger); 51889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx[vector].name); 51989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_ctx_put(vdev->ctx[vector].trigger); 52089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[vector].trigger = NULL; 52189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 52289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 52389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (fd < 0) 52489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 52589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 52689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "%s[%d](%s)", 52789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson name, vector, pci_name(pdev)); 52889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->ctx[vector].name) 52989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOMEM; 53089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 53189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson trigger = eventfd_ctx_fdget(fd); 53289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (IS_ERR(trigger)) { 53389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx[vector].name); 53489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return PTR_ERR(trigger); 53589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 53689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 53789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = request_irq(irq, vfio_msihandler, 0, 53889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[vector].name, trigger); 53989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) { 54089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx[vector].name); 54189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_ctx_put(trigger); 54289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 54389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 54489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 54589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->ctx[vector].trigger = trigger; 54689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 54789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 54889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 54989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 55089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start, 55189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned count, int32_t *fds, bool msix) 55289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 55389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int i, j, ret = 0; 55489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 55589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (start + count > vdev->num_ctx) 55689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 55789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 55889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson for (i = 0, j = start; i < count && !ret; i++, j++) { 55989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int fd = fds ? fds[i] : -1; 56089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = vfio_msi_set_vector_signal(vdev, j, fd, msix); 56189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 56289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 56389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) { 56489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson for (--j; j >= start; j--) 56589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_msi_set_vector_signal(vdev, j, -1, msix); 56689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 56789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 56889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 56989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 57089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 57189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix) 57289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 57389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson struct pci_dev *pdev = vdev->pdev; 57489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int i; 57589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 57689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix); 57789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 57889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson for (i = 0; i < vdev->num_ctx; i++) { 579b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson virqfd_disable(vdev, &vdev->ctx[i].unmask); 580b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson virqfd_disable(vdev, &vdev->ctx[i].mask); 58189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 58289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 58389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (msix) { 58489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson pci_disable_msix(vdev->pdev); 58589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->msix); 58689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else 58789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson pci_disable_msi(pdev); 58889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 58989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->irq_type = VFIO_PCI_NUM_IRQS; 59089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vdev->num_ctx = 0; 59189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson kfree(vdev->ctx); 59289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 59389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 59489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson/* 59589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson * IOCTL support 59689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson */ 59789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev, 59889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned index, unsigned start, 59989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned count, uint32_t flags, void *data) 60089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 60189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!is_intx(vdev) || start != 0 || count != 1) 60289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 60389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 60489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (flags & VFIO_IRQ_SET_DATA_NONE) { 60589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_pci_intx_unmask(vdev); 60689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 60789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson uint8_t unmask = *(uint8_t *)data; 60889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (unmask) 60989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_pci_intx_unmask(vdev); 61089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 61189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int32_t fd = *(int32_t *)data; 61289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (fd >= 0) 61389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return virqfd_enable(vdev, vfio_pci_intx_unmask_handler, 61489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_send_intx_eventfd, NULL, 61589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson &vdev->ctx[0].unmask, fd); 61689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 617b68e7fa879cd3b1126a7c455d9da1b70299efc0dAlex Williamson virqfd_disable(vdev, &vdev->ctx[0].unmask); 61889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 61989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 62089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 62189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 62289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 62389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_pci_set_intx_mask(struct vfio_pci_device *vdev, 62489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned index, unsigned start, 62589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned count, uint32_t flags, void *data) 62689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 62789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!is_intx(vdev) || start != 0 || count != 1) 62889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 62989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 63089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (flags & VFIO_IRQ_SET_DATA_NONE) { 63189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_pci_intx_mask(vdev); 63289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 63389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson uint8_t mask = *(uint8_t *)data; 63489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (mask) 63589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_pci_intx_mask(vdev); 63689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 63789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOTTY; /* XXX implement me */ 63889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 63989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 64089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 64189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 64289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 64389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_pci_set_intx_trigger(struct vfio_pci_device *vdev, 64489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned index, unsigned start, 64589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned count, uint32_t flags, void *data) 64689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 64789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 64889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_intx_disable(vdev); 64989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 65089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 65189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 65289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) 65389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 65489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 65589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 65689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int32_t fd = *(int32_t *)data; 65789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int ret; 65889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 65989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (is_intx(vdev)) 66089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return vfio_intx_set_signal(vdev, fd); 66189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 66289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = vfio_intx_enable(vdev); 66389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) 66489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 66589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 66689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = vfio_intx_set_signal(vdev, fd); 66789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) 66889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_intx_disable(vdev); 66989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 67089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 67189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 67289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 67389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!is_intx(vdev)) 67489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 67589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 67689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (flags & VFIO_IRQ_SET_DATA_NONE) { 67789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_send_intx_eventfd(vdev, NULL); 67889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 67989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson uint8_t trigger = *(uint8_t *)data; 68089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (trigger) 68189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_send_intx_eventfd(vdev, NULL); 68289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 68389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 68489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 68589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 68689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonstatic int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev, 68789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned index, unsigned start, 68889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned count, uint32_t flags, void *data) 68989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 69089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int i; 69189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; 69289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 69389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (irq_is(vdev, index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 69489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_msi_disable(vdev, msix); 69589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 69689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 69789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 69889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!(irq_is(vdev, index) || is_irq_none(vdev))) 69989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 70089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 70189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 70289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int32_t *fds = data; 70389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int ret; 70489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 70589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (vdev->irq_type == index) 70689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return vfio_msi_set_block(vdev, start, count, 70789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson fds, msix); 70889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 70989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = vfio_msi_enable(vdev, start + count, msix); 71089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) 71189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 71289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 71389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson ret = vfio_msi_set_block(vdev, start, count, fds, msix); 71489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (ret) 71589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson vfio_msi_disable(vdev, msix); 71689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 71789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return ret; 71889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 71989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 72089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!irq_is(vdev, index) || start + count > vdev->num_ctx) 72189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -EINVAL; 72289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 72389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson for (i = start; i < start + count; i++) { 72489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!vdev->ctx[i].trigger) 72589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson continue; 72689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (flags & VFIO_IRQ_SET_DATA_NONE) { 72789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_signal(vdev->ctx[i].trigger, 1); 72889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 72989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson uint8_t *bools = data; 73089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (bools[i - start]) 73189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson eventfd_signal(vdev->ctx[i].trigger, 1); 73289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 73389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 73489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return 0; 73589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 73689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 73789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamsonint vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags, 73889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned index, unsigned start, unsigned count, 73989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson void *data) 74089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson{ 74189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson int (*func)(struct vfio_pci_device *vdev, unsigned index, 74289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson unsigned start, unsigned count, uint32_t flags, 74389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson void *data) = NULL; 74489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 74589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson switch (index) { 74689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_PCI_INTX_IRQ_INDEX: 74789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 74889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_IRQ_SET_ACTION_MASK: 74989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson func = vfio_pci_set_intx_mask; 75089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson break; 75189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_IRQ_SET_ACTION_UNMASK: 75289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson func = vfio_pci_set_intx_unmask; 75389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson break; 75489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_IRQ_SET_ACTION_TRIGGER: 75589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson func = vfio_pci_set_intx_trigger; 75689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson break; 75789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 75889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson break; 75989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_PCI_MSI_IRQ_INDEX: 76089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_PCI_MSIX_IRQ_INDEX: 76189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 76289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_IRQ_SET_ACTION_MASK: 76389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_IRQ_SET_ACTION_UNMASK: 76489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson /* XXX Need masking support exported */ 76589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson break; 76689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson case VFIO_IRQ_SET_ACTION_TRIGGER: 76789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson func = vfio_pci_set_msi_trigger; 76889e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson break; 76989e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 77089e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson break; 77189e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson } 77289e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 77389e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson if (!func) 77489e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return -ENOTTY; 77589e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson 77689e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson return func(vdev, index, start, count, flags, data); 77789e1f7d4c66d85f42c3d52ea3866eb10cadf6153Alex Williamson} 778