1/* 2 * Virtio PCI driver 3 * 4 * This module allows virtio devices to be used over a virtual PCI device. 5 * This can be used with QEMU based VMMs like KVM or Xen. 6 * 7 * Copyright IBM Corp. 2007 8 * 9 * Authors: 10 * Anthony Liguori <aliguori@us.ibm.com> 11 * 12 * This work is licensed under the terms of the GNU GPL, version 2 or later. 13 * See the COPYING file in the top-level directory. 14 * 15 */ 16 17#include <linux/module.h> 18#include <linux/list.h> 19#include <linux/pci.h> 20#include <linux/slab.h> 21#include <linux/interrupt.h> 22#include <linux/virtio.h> 23#include <linux/virtio_config.h> 24#include <linux/virtio_ring.h> 25#include <linux/virtio_pci.h> 26#include <linux/highmem.h> 27#include <linux/spinlock.h> 28 29MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>"); 30MODULE_DESCRIPTION("virtio-pci"); 31MODULE_LICENSE("GPL"); 32MODULE_VERSION("1"); 33 34/* Our device structure */ 35struct virtio_pci_device 36{ 37 struct virtio_device vdev; 38 struct pci_dev *pci_dev; 39 40 /* the IO mapping for the PCI config space */ 41 void __iomem *ioaddr; 42 43 /* a list of queues so we can dispatch IRQs */ 44 spinlock_t lock; 45 struct list_head virtqueues; 46 47 /* MSI-X support */ 48 int msix_enabled; 49 int intx_enabled; 50 struct msix_entry *msix_entries; 51 /* Name strings for interrupts. This size should be enough, 52 * and I'm too lazy to allocate each name separately. */ 53 char (*msix_names)[256]; 54 /* Number of available vectors */ 55 unsigned msix_vectors; 56 /* Vectors allocated, excluding per-vq vectors if any */ 57 unsigned msix_used_vectors; 58 59 /* Status saved during hibernate/restore */ 60 u8 saved_status; 61 62 /* Whether we have vector per vq */ 63 bool per_vq_vectors; 64}; 65 66/* Constants for MSI-X */ 67/* Use first vector for configuration changes, second and the rest for 68 * virtqueues Thus, we need at least 2 vectors for MSI. */ 69enum { 70 VP_MSIX_CONFIG_VECTOR = 0, 71 VP_MSIX_VQ_VECTOR = 1, 72}; 73 74struct virtio_pci_vq_info 75{ 76 /* the actual virtqueue */ 77 struct virtqueue *vq; 78 79 /* the number of entries in the queue */ 80 int num; 81 82 /* the index of the queue */ 83 int queue_index; 84 85 /* the virtual address of the ring queue */ 86 void *queue; 87 88 /* the list node for the virtqueues list */ 89 struct list_head node; 90 91 /* MSI-X vector (or none) */ 92 unsigned msix_vector; 93}; 94 95/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */ 96static struct pci_device_id virtio_pci_id_table[] = { 97 { 0x1af4, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, 98 { 0 }, 99}; 100 101MODULE_DEVICE_TABLE(pci, virtio_pci_id_table); 102 103/* Convert a generic virtio device to our structure */ 104static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev) 105{ 106 return container_of(vdev, struct virtio_pci_device, vdev); 107} 108 109/* virtio config->get_features() implementation */ 110static u32 vp_get_features(struct virtio_device *vdev) 111{ 112 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 113 114 /* When someone needs more than 32 feature bits, we'll need to 115 * steal a bit to indicate that the rest are somewhere else. */ 116 return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES); 117} 118 119/* virtio config->finalize_features() implementation */ 120static void vp_finalize_features(struct virtio_device *vdev) 121{ 122 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 123 124 /* Give virtio_ring a chance to accept features. */ 125 vring_transport_features(vdev); 126 127 /* We only support 32 feature bits. */ 128 BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1); 129 iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES); 130} 131 132/* virtio config->get() implementation */ 133static void vp_get(struct virtio_device *vdev, unsigned offset, 134 void *buf, unsigned len) 135{ 136 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 137 void __iomem *ioaddr = vp_dev->ioaddr + 138 VIRTIO_PCI_CONFIG(vp_dev) + offset; 139 u8 *ptr = buf; 140 int i; 141 142 for (i = 0; i < len; i++) 143 ptr[i] = ioread8(ioaddr + i); 144} 145 146/* the config->set() implementation. it's symmetric to the config->get() 147 * implementation */ 148static void vp_set(struct virtio_device *vdev, unsigned offset, 149 const void *buf, unsigned len) 150{ 151 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 152 void __iomem *ioaddr = vp_dev->ioaddr + 153 VIRTIO_PCI_CONFIG(vp_dev) + offset; 154 const u8 *ptr = buf; 155 int i; 156 157 for (i = 0; i < len; i++) 158 iowrite8(ptr[i], ioaddr + i); 159} 160 161/* config->{get,set}_status() implementations */ 162static u8 vp_get_status(struct virtio_device *vdev) 163{ 164 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 165 return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS); 166} 167 168static void vp_set_status(struct virtio_device *vdev, u8 status) 169{ 170 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 171 /* We should never be setting status to 0. */ 172 BUG_ON(status == 0); 173 iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS); 174} 175 176/* wait for pending irq handlers */ 177static void vp_synchronize_vectors(struct virtio_device *vdev) 178{ 179 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 180 int i; 181 182 if (vp_dev->intx_enabled) 183 synchronize_irq(vp_dev->pci_dev->irq); 184 185 for (i = 0; i < vp_dev->msix_vectors; ++i) 186 synchronize_irq(vp_dev->msix_entries[i].vector); 187} 188 189static void vp_reset(struct virtio_device *vdev) 190{ 191 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 192 /* 0 status means a reset. */ 193 iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS); 194 /* Flush out the status write, and flush in device writes, 195 * including MSi-X interrupts, if any. */ 196 ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS); 197 /* Flush pending VQ/configuration callbacks. */ 198 vp_synchronize_vectors(vdev); 199} 200 201/* the notify function used when creating a virt queue */ 202static void vp_notify(struct virtqueue *vq) 203{ 204 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 205 struct virtio_pci_vq_info *info = vq->priv; 206 207 /* we write the queue's selector into the notification register to 208 * signal the other end */ 209 iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY); 210} 211 212/* Handle a configuration change: Tell driver if it wants to know. */ 213static irqreturn_t vp_config_changed(int irq, void *opaque) 214{ 215 struct virtio_pci_device *vp_dev = opaque; 216 struct virtio_driver *drv; 217 drv = container_of(vp_dev->vdev.dev.driver, 218 struct virtio_driver, driver); 219 220 if (drv && drv->config_changed) 221 drv->config_changed(&vp_dev->vdev); 222 return IRQ_HANDLED; 223} 224 225/* Notify all virtqueues on an interrupt. */ 226static irqreturn_t vp_vring_interrupt(int irq, void *opaque) 227{ 228 struct virtio_pci_device *vp_dev = opaque; 229 struct virtio_pci_vq_info *info; 230 irqreturn_t ret = IRQ_NONE; 231 unsigned long flags; 232 233 spin_lock_irqsave(&vp_dev->lock, flags); 234 list_for_each_entry(info, &vp_dev->virtqueues, node) { 235 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED) 236 ret = IRQ_HANDLED; 237 } 238 spin_unlock_irqrestore(&vp_dev->lock, flags); 239 240 return ret; 241} 242 243/* A small wrapper to also acknowledge the interrupt when it's handled. 244 * I really need an EIO hook for the vring so I can ack the interrupt once we 245 * know that we'll be handling the IRQ but before we invoke the callback since 246 * the callback may notify the host which results in the host attempting to 247 * raise an interrupt that we would then mask once we acknowledged the 248 * interrupt. */ 249static irqreturn_t vp_interrupt(int irq, void *opaque) 250{ 251 struct virtio_pci_device *vp_dev = opaque; 252 u8 isr; 253 254 /* reading the ISR has the effect of also clearing it so it's very 255 * important to save off the value. */ 256 isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR); 257 258 /* It's definitely not us if the ISR was not high */ 259 if (!isr) 260 return IRQ_NONE; 261 262 /* Configuration change? Tell driver if it wants to know. */ 263 if (isr & VIRTIO_PCI_ISR_CONFIG) 264 vp_config_changed(irq, opaque); 265 266 return vp_vring_interrupt(irq, opaque); 267} 268 269static void vp_free_vectors(struct virtio_device *vdev) 270{ 271 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 272 int i; 273 274 if (vp_dev->intx_enabled) { 275 free_irq(vp_dev->pci_dev->irq, vp_dev); 276 vp_dev->intx_enabled = 0; 277 } 278 279 for (i = 0; i < vp_dev->msix_used_vectors; ++i) 280 free_irq(vp_dev->msix_entries[i].vector, vp_dev); 281 282 if (vp_dev->msix_enabled) { 283 /* Disable the vector used for configuration */ 284 iowrite16(VIRTIO_MSI_NO_VECTOR, 285 vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR); 286 /* Flush the write out to device */ 287 ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR); 288 289 pci_disable_msix(vp_dev->pci_dev); 290 vp_dev->msix_enabled = 0; 291 vp_dev->msix_vectors = 0; 292 } 293 294 vp_dev->msix_used_vectors = 0; 295 kfree(vp_dev->msix_names); 296 vp_dev->msix_names = NULL; 297 kfree(vp_dev->msix_entries); 298 vp_dev->msix_entries = NULL; 299} 300 301static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, 302 bool per_vq_vectors) 303{ 304 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 305 const char *name = dev_name(&vp_dev->vdev.dev); 306 unsigned i, v; 307 int err = -ENOMEM; 308 309 vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries, 310 GFP_KERNEL); 311 if (!vp_dev->msix_entries) 312 goto error; 313 vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names, 314 GFP_KERNEL); 315 if (!vp_dev->msix_names) 316 goto error; 317 318 for (i = 0; i < nvectors; ++i) 319 vp_dev->msix_entries[i].entry = i; 320 321 /* pci_enable_msix returns positive if we can't get this many. */ 322 err = pci_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries, nvectors); 323 if (err > 0) 324 err = -ENOSPC; 325 if (err) 326 goto error; 327 vp_dev->msix_vectors = nvectors; 328 vp_dev->msix_enabled = 1; 329 330 /* Set the vector used for configuration */ 331 v = vp_dev->msix_used_vectors; 332 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names, 333 "%s-config", name); 334 err = request_irq(vp_dev->msix_entries[v].vector, 335 vp_config_changed, 0, vp_dev->msix_names[v], 336 vp_dev); 337 if (err) 338 goto error; 339 ++vp_dev->msix_used_vectors; 340 341 iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR); 342 /* Verify we had enough resources to assign the vector */ 343 v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR); 344 if (v == VIRTIO_MSI_NO_VECTOR) { 345 err = -EBUSY; 346 goto error; 347 } 348 349 if (!per_vq_vectors) { 350 /* Shared vector for all VQs */ 351 v = vp_dev->msix_used_vectors; 352 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names, 353 "%s-virtqueues", name); 354 err = request_irq(vp_dev->msix_entries[v].vector, 355 vp_vring_interrupt, 0, vp_dev->msix_names[v], 356 vp_dev); 357 if (err) 358 goto error; 359 ++vp_dev->msix_used_vectors; 360 } 361 return 0; 362error: 363 vp_free_vectors(vdev); 364 return err; 365} 366 367static int vp_request_intx(struct virtio_device *vdev) 368{ 369 int err; 370 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 371 372 err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, 373 IRQF_SHARED, dev_name(&vdev->dev), vp_dev); 374 if (!err) 375 vp_dev->intx_enabled = 1; 376 return err; 377} 378 379static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index, 380 void (*callback)(struct virtqueue *vq), 381 const char *name, 382 u16 msix_vec) 383{ 384 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 385 struct virtio_pci_vq_info *info; 386 struct virtqueue *vq; 387 unsigned long flags, size; 388 u16 num; 389 int err; 390 391 /* Select the queue we're interested in */ 392 iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL); 393 394 /* Check if queue is either not available or already active. */ 395 num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM); 396 if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN)) 397 return ERR_PTR(-ENOENT); 398 399 /* allocate and fill out our structure the represents an active 400 * queue */ 401 info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL); 402 if (!info) 403 return ERR_PTR(-ENOMEM); 404 405 info->queue_index = index; 406 info->num = num; 407 info->msix_vector = msix_vec; 408 409 size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN)); 410 info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO); 411 if (info->queue == NULL) { 412 err = -ENOMEM; 413 goto out_info; 414 } 415 416 /* activate the queue */ 417 iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT, 418 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); 419 420 /* create the vring */ 421 vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev, 422 true, info->queue, vp_notify, callback, name); 423 if (!vq) { 424 err = -ENOMEM; 425 goto out_activate_queue; 426 } 427 428 vq->priv = info; 429 info->vq = vq; 430 431 if (msix_vec != VIRTIO_MSI_NO_VECTOR) { 432 iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR); 433 msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR); 434 if (msix_vec == VIRTIO_MSI_NO_VECTOR) { 435 err = -EBUSY; 436 goto out_assign; 437 } 438 } 439 440 if (callback) { 441 spin_lock_irqsave(&vp_dev->lock, flags); 442 list_add(&info->node, &vp_dev->virtqueues); 443 spin_unlock_irqrestore(&vp_dev->lock, flags); 444 } else { 445 INIT_LIST_HEAD(&info->node); 446 } 447 448 return vq; 449 450out_assign: 451 vring_del_virtqueue(vq); 452out_activate_queue: 453 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); 454 free_pages_exact(info->queue, size); 455out_info: 456 kfree(info); 457 return ERR_PTR(err); 458} 459 460static void vp_del_vq(struct virtqueue *vq) 461{ 462 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 463 struct virtio_pci_vq_info *info = vq->priv; 464 unsigned long flags, size; 465 466 spin_lock_irqsave(&vp_dev->lock, flags); 467 list_del(&info->node); 468 spin_unlock_irqrestore(&vp_dev->lock, flags); 469 470 iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL); 471 472 if (vp_dev->msix_enabled) { 473 iowrite16(VIRTIO_MSI_NO_VECTOR, 474 vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR); 475 /* Flush the write out to device */ 476 ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR); 477 } 478 479 vring_del_virtqueue(vq); 480 481 /* Select and deactivate the queue */ 482 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); 483 484 size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN)); 485 free_pages_exact(info->queue, size); 486 kfree(info); 487} 488 489/* the config->del_vqs() implementation */ 490static void vp_del_vqs(struct virtio_device *vdev) 491{ 492 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 493 struct virtqueue *vq, *n; 494 struct virtio_pci_vq_info *info; 495 496 list_for_each_entry_safe(vq, n, &vdev->vqs, list) { 497 info = vq->priv; 498 if (vp_dev->per_vq_vectors && 499 info->msix_vector != VIRTIO_MSI_NO_VECTOR) 500 free_irq(vp_dev->msix_entries[info->msix_vector].vector, 501 vq); 502 vp_del_vq(vq); 503 } 504 vp_dev->per_vq_vectors = false; 505 506 vp_free_vectors(vdev); 507} 508 509static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs, 510 struct virtqueue *vqs[], 511 vq_callback_t *callbacks[], 512 const char *names[], 513 bool use_msix, 514 bool per_vq_vectors) 515{ 516 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 517 u16 msix_vec; 518 int i, err, nvectors, allocated_vectors; 519 520 if (!use_msix) { 521 /* Old style: one normal interrupt for change and all vqs. */ 522 err = vp_request_intx(vdev); 523 if (err) 524 goto error_request; 525 } else { 526 if (per_vq_vectors) { 527 /* Best option: one for change interrupt, one per vq. */ 528 nvectors = 1; 529 for (i = 0; i < nvqs; ++i) 530 if (callbacks[i]) 531 ++nvectors; 532 } else { 533 /* Second best: one for change, shared for all vqs. */ 534 nvectors = 2; 535 } 536 537 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors); 538 if (err) 539 goto error_request; 540 } 541 542 vp_dev->per_vq_vectors = per_vq_vectors; 543 allocated_vectors = vp_dev->msix_used_vectors; 544 for (i = 0; i < nvqs; ++i) { 545 if (!callbacks[i] || !vp_dev->msix_enabled) 546 msix_vec = VIRTIO_MSI_NO_VECTOR; 547 else if (vp_dev->per_vq_vectors) 548 msix_vec = allocated_vectors++; 549 else 550 msix_vec = VP_MSIX_VQ_VECTOR; 551 vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec); 552 if (IS_ERR(vqs[i])) { 553 err = PTR_ERR(vqs[i]); 554 goto error_find; 555 } 556 557 if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR) 558 continue; 559 560 /* allocate per-vq irq if available and necessary */ 561 snprintf(vp_dev->msix_names[msix_vec], 562 sizeof *vp_dev->msix_names, 563 "%s-%s", 564 dev_name(&vp_dev->vdev.dev), names[i]); 565 err = request_irq(vp_dev->msix_entries[msix_vec].vector, 566 vring_interrupt, 0, 567 vp_dev->msix_names[msix_vec], 568 vqs[i]); 569 if (err) { 570 vp_del_vq(vqs[i]); 571 goto error_find; 572 } 573 } 574 return 0; 575 576error_find: 577 vp_del_vqs(vdev); 578 579error_request: 580 return err; 581} 582 583/* the config->find_vqs() implementation */ 584static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs, 585 struct virtqueue *vqs[], 586 vq_callback_t *callbacks[], 587 const char *names[]) 588{ 589 int err; 590 591 /* Try MSI-X with one vector per queue. */ 592 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true); 593 if (!err) 594 return 0; 595 /* Fallback: MSI-X with one vector for config, one shared for queues. */ 596 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, 597 true, false); 598 if (!err) 599 return 0; 600 /* Finally fall back to regular interrupts. */ 601 return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, 602 false, false); 603} 604 605static const char *vp_bus_name(struct virtio_device *vdev) 606{ 607 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 608 609 return pci_name(vp_dev->pci_dev); 610} 611 612static struct virtio_config_ops virtio_pci_config_ops = { 613 .get = vp_get, 614 .set = vp_set, 615 .get_status = vp_get_status, 616 .set_status = vp_set_status, 617 .reset = vp_reset, 618 .find_vqs = vp_find_vqs, 619 .del_vqs = vp_del_vqs, 620 .get_features = vp_get_features, 621 .finalize_features = vp_finalize_features, 622 .bus_name = vp_bus_name, 623}; 624 625static void virtio_pci_release_dev(struct device *_d) 626{ 627 /* 628 * No need for a release method as we allocate/free 629 * all devices together with the pci devices. 630 * Provide an empty one to avoid getting a warning from core. 631 */ 632} 633 634/* the PCI probing function */ 635static int __devinit virtio_pci_probe(struct pci_dev *pci_dev, 636 const struct pci_device_id *id) 637{ 638 struct virtio_pci_device *vp_dev; 639 int err; 640 641 /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */ 642 if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f) 643 return -ENODEV; 644 645 if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) { 646 printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n", 647 VIRTIO_PCI_ABI_VERSION, pci_dev->revision); 648 return -ENODEV; 649 } 650 651 /* allocate our structure and fill it out */ 652 vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL); 653 if (vp_dev == NULL) 654 return -ENOMEM; 655 656 vp_dev->vdev.dev.parent = &pci_dev->dev; 657 vp_dev->vdev.dev.release = virtio_pci_release_dev; 658 vp_dev->vdev.config = &virtio_pci_config_ops; 659 vp_dev->pci_dev = pci_dev; 660 INIT_LIST_HEAD(&vp_dev->virtqueues); 661 spin_lock_init(&vp_dev->lock); 662 663 /* Disable MSI/MSIX to bring device to a known good state. */ 664 pci_msi_off(pci_dev); 665 666 /* enable the device */ 667 err = pci_enable_device(pci_dev); 668 if (err) 669 goto out; 670 671 err = pci_request_regions(pci_dev, "virtio-pci"); 672 if (err) 673 goto out_enable_device; 674 675 vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0); 676 if (vp_dev->ioaddr == NULL) 677 goto out_req_regions; 678 679 pci_set_drvdata(pci_dev, vp_dev); 680 pci_set_master(pci_dev); 681 682 /* we use the subsystem vendor/device id as the virtio vendor/device 683 * id. this allows us to use the same PCI vendor/device id for all 684 * virtio devices and to identify the particular virtio driver by 685 * the subsystem ids */ 686 vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor; 687 vp_dev->vdev.id.device = pci_dev->subsystem_device; 688 689 /* finally register the virtio device */ 690 err = register_virtio_device(&vp_dev->vdev); 691 if (err) 692 goto out_set_drvdata; 693 694 return 0; 695 696out_set_drvdata: 697 pci_set_drvdata(pci_dev, NULL); 698 pci_iounmap(pci_dev, vp_dev->ioaddr); 699out_req_regions: 700 pci_release_regions(pci_dev); 701out_enable_device: 702 pci_disable_device(pci_dev); 703out: 704 kfree(vp_dev); 705 return err; 706} 707 708static void __devexit virtio_pci_remove(struct pci_dev *pci_dev) 709{ 710 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 711 712 unregister_virtio_device(&vp_dev->vdev); 713 714 vp_del_vqs(&vp_dev->vdev); 715 pci_set_drvdata(pci_dev, NULL); 716 pci_iounmap(pci_dev, vp_dev->ioaddr); 717 pci_release_regions(pci_dev); 718 pci_disable_device(pci_dev); 719 kfree(vp_dev); 720} 721 722#ifdef CONFIG_PM 723static int virtio_pci_suspend(struct device *dev) 724{ 725 struct pci_dev *pci_dev = to_pci_dev(dev); 726 727 pci_save_state(pci_dev); 728 pci_set_power_state(pci_dev, PCI_D3hot); 729 return 0; 730} 731 732static int virtio_pci_resume(struct device *dev) 733{ 734 struct pci_dev *pci_dev = to_pci_dev(dev); 735 736 pci_restore_state(pci_dev); 737 pci_set_power_state(pci_dev, PCI_D0); 738 return 0; 739} 740 741static int virtio_pci_freeze(struct device *dev) 742{ 743 struct pci_dev *pci_dev = to_pci_dev(dev); 744 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 745 struct virtio_driver *drv; 746 int ret; 747 748 drv = container_of(vp_dev->vdev.dev.driver, 749 struct virtio_driver, driver); 750 751 ret = 0; 752 vp_dev->saved_status = vp_get_status(&vp_dev->vdev); 753 if (drv && drv->freeze) 754 ret = drv->freeze(&vp_dev->vdev); 755 756 if (!ret) 757 pci_disable_device(pci_dev); 758 return ret; 759} 760 761static int restore_common(struct device *dev) 762{ 763 struct pci_dev *pci_dev = to_pci_dev(dev); 764 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 765 int ret; 766 767 ret = pci_enable_device(pci_dev); 768 if (ret) 769 return ret; 770 pci_set_master(pci_dev); 771 vp_finalize_features(&vp_dev->vdev); 772 773 return ret; 774} 775 776static int virtio_pci_thaw(struct device *dev) 777{ 778 struct pci_dev *pci_dev = to_pci_dev(dev); 779 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 780 struct virtio_driver *drv; 781 int ret; 782 783 ret = restore_common(dev); 784 if (ret) 785 return ret; 786 787 drv = container_of(vp_dev->vdev.dev.driver, 788 struct virtio_driver, driver); 789 790 if (drv && drv->thaw) 791 ret = drv->thaw(&vp_dev->vdev); 792 else if (drv && drv->restore) 793 ret = drv->restore(&vp_dev->vdev); 794 795 /* Finally, tell the device we're all set */ 796 if (!ret) 797 vp_set_status(&vp_dev->vdev, vp_dev->saved_status); 798 799 return ret; 800} 801 802static int virtio_pci_restore(struct device *dev) 803{ 804 struct pci_dev *pci_dev = to_pci_dev(dev); 805 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 806 struct virtio_driver *drv; 807 int ret; 808 809 drv = container_of(vp_dev->vdev.dev.driver, 810 struct virtio_driver, driver); 811 812 ret = restore_common(dev); 813 if (!ret && drv && drv->restore) 814 ret = drv->restore(&vp_dev->vdev); 815 816 /* Finally, tell the device we're all set */ 817 if (!ret) 818 vp_set_status(&vp_dev->vdev, vp_dev->saved_status); 819 820 return ret; 821} 822 823static const struct dev_pm_ops virtio_pci_pm_ops = { 824 .suspend = virtio_pci_suspend, 825 .resume = virtio_pci_resume, 826 .freeze = virtio_pci_freeze, 827 .thaw = virtio_pci_thaw, 828 .restore = virtio_pci_restore, 829 .poweroff = virtio_pci_suspend, 830}; 831#endif 832 833static struct pci_driver virtio_pci_driver = { 834 .name = "virtio-pci", 835 .id_table = virtio_pci_id_table, 836 .probe = virtio_pci_probe, 837 .remove = __devexit_p(virtio_pci_remove), 838#ifdef CONFIG_PM 839 .driver.pm = &virtio_pci_pm_ops, 840#endif 841}; 842 843static int __init virtio_pci_init(void) 844{ 845 return pci_register_driver(&virtio_pci_driver); 846} 847 848module_init(virtio_pci_init); 849 850static void __exit virtio_pci_exit(void) 851{ 852 pci_unregister_driver(&virtio_pci_driver); 853} 854 855module_exit(virtio_pci_exit); 856