msi.c revision f7e6600d762bf7c04b48c8d9bd0ab26d04a8d11f
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/mm.h> 10#include <linux/irq.h> 11#include <linux/interrupt.h> 12#include <linux/init.h> 13#include <linux/config.h> 14#include <linux/ioport.h> 15#include <linux/smp_lock.h> 16#include <linux/pci.h> 17#include <linux/proc_fs.h> 18 19#include <asm/errno.h> 20#include <asm/io.h> 21#include <asm/smp.h> 22 23#include "pci.h" 24#include "msi.h" 25 26static DEFINE_SPINLOCK(msi_lock); 27static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL }; 28static kmem_cache_t* msi_cachep; 29 30static int pci_msi_enable = 1; 31static int last_alloc_vector; 32static int nr_released_vectors; 33static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS; 34static int nr_msix_devices; 35 36#ifndef CONFIG_X86_IO_APIC 37int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1}; 38#endif 39 40static struct msi_ops *msi_ops; 41 42int 43msi_register(struct msi_ops *ops) 44{ 45 msi_ops = ops; 46 return 0; 47} 48 49static void msi_cache_ctor(void *p, kmem_cache_t *cache, unsigned long flags) 50{ 51 memset(p, 0, NR_IRQS * sizeof(struct msi_desc)); 52} 53 54static int msi_cache_init(void) 55{ 56 msi_cachep = kmem_cache_create("msi_cache", 57 NR_IRQS * sizeof(struct msi_desc), 58 0, SLAB_HWCACHE_ALIGN, msi_cache_ctor, NULL); 59 if (!msi_cachep) 60 return -ENOMEM; 61 62 return 0; 63} 64 65static void msi_set_mask_bit(unsigned int vector, int flag) 66{ 67 struct msi_desc *entry; 68 69 entry = (struct msi_desc *)msi_desc[vector]; 70 if (!entry || !entry->dev || !entry->mask_base) 71 return; 72 switch (entry->msi_attrib.type) { 73 case PCI_CAP_ID_MSI: 74 { 75 int pos; 76 u32 mask_bits; 77 78 pos = (long)entry->mask_base; 79 pci_read_config_dword(entry->dev, pos, &mask_bits); 80 mask_bits &= ~(1); 81 mask_bits |= flag; 82 pci_write_config_dword(entry->dev, pos, mask_bits); 83 break; 84 } 85 case PCI_CAP_ID_MSIX: 86 { 87 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 88 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 89 writel(flag, entry->mask_base + offset); 90 break; 91 } 92 default: 93 break; 94 } 95} 96 97#ifdef CONFIG_SMP 98static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask) 99{ 100 struct msi_desc *entry; 101 u32 address_hi, address_lo; 102 unsigned int irq = vector; 103 unsigned int dest_cpu = first_cpu(cpu_mask); 104 105 entry = (struct msi_desc *)msi_desc[vector]; 106 if (!entry || !entry->dev) 107 return; 108 109 switch (entry->msi_attrib.type) { 110 case PCI_CAP_ID_MSI: 111 { 112 int pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI); 113 114 if (!pos) 115 return; 116 117 pci_read_config_dword(entry->dev, msi_upper_address_reg(pos), 118 &address_hi); 119 pci_read_config_dword(entry->dev, msi_lower_address_reg(pos), 120 &address_lo); 121 122 msi_ops->target(vector, dest_cpu, &address_hi, &address_lo); 123 124 pci_write_config_dword(entry->dev, msi_upper_address_reg(pos), 125 address_hi); 126 pci_write_config_dword(entry->dev, msi_lower_address_reg(pos), 127 address_lo); 128 set_native_irq_info(irq, cpu_mask); 129 break; 130 } 131 case PCI_CAP_ID_MSIX: 132 { 133 int offset_hi = 134 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 135 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET; 136 int offset_lo = 137 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 138 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET; 139 140 address_hi = readl(entry->mask_base + offset_hi); 141 address_lo = readl(entry->mask_base + offset_lo); 142 143 msi_ops->target(vector, dest_cpu, &address_hi, &address_lo); 144 145 writel(address_hi, entry->mask_base + offset_hi); 146 writel(address_lo, entry->mask_base + offset_lo); 147 set_native_irq_info(irq, cpu_mask); 148 break; 149 } 150 default: 151 break; 152 } 153} 154#else 155#define set_msi_affinity NULL 156#endif /* CONFIG_SMP */ 157 158static void mask_MSI_irq(unsigned int vector) 159{ 160 msi_set_mask_bit(vector, 1); 161} 162 163static void unmask_MSI_irq(unsigned int vector) 164{ 165 msi_set_mask_bit(vector, 0); 166} 167 168static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector) 169{ 170 struct msi_desc *entry; 171 unsigned long flags; 172 173 spin_lock_irqsave(&msi_lock, flags); 174 entry = msi_desc[vector]; 175 if (!entry || !entry->dev) { 176 spin_unlock_irqrestore(&msi_lock, flags); 177 return 0; 178 } 179 entry->msi_attrib.state = 1; /* Mark it active */ 180 spin_unlock_irqrestore(&msi_lock, flags); 181 182 return 0; /* never anything pending */ 183} 184 185static unsigned int startup_msi_irq_w_maskbit(unsigned int vector) 186{ 187 startup_msi_irq_wo_maskbit(vector); 188 unmask_MSI_irq(vector); 189 return 0; /* never anything pending */ 190} 191 192static void shutdown_msi_irq(unsigned int vector) 193{ 194 struct msi_desc *entry; 195 unsigned long flags; 196 197 spin_lock_irqsave(&msi_lock, flags); 198 entry = msi_desc[vector]; 199 if (entry && entry->dev) 200 entry->msi_attrib.state = 0; /* Mark it not active */ 201 spin_unlock_irqrestore(&msi_lock, flags); 202} 203 204static void end_msi_irq_wo_maskbit(unsigned int vector) 205{ 206 move_native_irq(vector); 207 ack_APIC_irq(); 208} 209 210static void end_msi_irq_w_maskbit(unsigned int vector) 211{ 212 move_native_irq(vector); 213 unmask_MSI_irq(vector); 214 ack_APIC_irq(); 215} 216 217static void do_nothing(unsigned int vector) 218{ 219} 220 221/* 222 * Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices, 223 * which implement the MSI-X Capability Structure. 224 */ 225static struct hw_interrupt_type msix_irq_type = { 226 .typename = "PCI-MSI-X", 227 .startup = startup_msi_irq_w_maskbit, 228 .shutdown = shutdown_msi_irq, 229 .enable = unmask_MSI_irq, 230 .disable = mask_MSI_irq, 231 .ack = mask_MSI_irq, 232 .end = end_msi_irq_w_maskbit, 233 .set_affinity = set_msi_affinity 234}; 235 236/* 237 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices, 238 * which implement the MSI Capability Structure with 239 * Mask-and-Pending Bits. 240 */ 241static struct hw_interrupt_type msi_irq_w_maskbit_type = { 242 .typename = "PCI-MSI", 243 .startup = startup_msi_irq_w_maskbit, 244 .shutdown = shutdown_msi_irq, 245 .enable = unmask_MSI_irq, 246 .disable = mask_MSI_irq, 247 .ack = mask_MSI_irq, 248 .end = end_msi_irq_w_maskbit, 249 .set_affinity = set_msi_affinity 250}; 251 252/* 253 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices, 254 * which implement the MSI Capability Structure without 255 * Mask-and-Pending Bits. 256 */ 257static struct hw_interrupt_type msi_irq_wo_maskbit_type = { 258 .typename = "PCI-MSI", 259 .startup = startup_msi_irq_wo_maskbit, 260 .shutdown = shutdown_msi_irq, 261 .enable = do_nothing, 262 .disable = do_nothing, 263 .ack = do_nothing, 264 .end = end_msi_irq_wo_maskbit, 265 .set_affinity = set_msi_affinity 266}; 267 268static int msi_free_vector(struct pci_dev* dev, int vector, int reassign); 269static int assign_msi_vector(void) 270{ 271 static int new_vector_avail = 1; 272 int vector; 273 unsigned long flags; 274 275 /* 276 * msi_lock is provided to ensure that successful allocation of MSI 277 * vector is assigned unique among drivers. 278 */ 279 spin_lock_irqsave(&msi_lock, flags); 280 281 if (!new_vector_avail) { 282 int free_vector = 0; 283 284 /* 285 * vector_irq[] = -1 indicates that this specific vector is: 286 * - assigned for MSI (since MSI have no associated IRQ) or 287 * - assigned for legacy if less than 16, or 288 * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping 289 * vector_irq[] = 0 indicates that this vector, previously 290 * assigned for MSI, is freed by hotplug removed operations. 291 * This vector will be reused for any subsequent hotplug added 292 * operations. 293 * vector_irq[] > 0 indicates that this vector is assigned for 294 * IOxAPIC IRQs. This vector and its value provides a 1-to-1 295 * vector-to-IOxAPIC IRQ mapping. 296 */ 297 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) { 298 if (vector_irq[vector] != 0) 299 continue; 300 free_vector = vector; 301 if (!msi_desc[vector]) 302 break; 303 else 304 continue; 305 } 306 if (!free_vector) { 307 spin_unlock_irqrestore(&msi_lock, flags); 308 return -EBUSY; 309 } 310 vector_irq[free_vector] = -1; 311 nr_released_vectors--; 312 spin_unlock_irqrestore(&msi_lock, flags); 313 if (msi_desc[free_vector] != NULL) { 314 struct pci_dev *dev; 315 int tail; 316 317 /* free all linked vectors before re-assign */ 318 do { 319 spin_lock_irqsave(&msi_lock, flags); 320 dev = msi_desc[free_vector]->dev; 321 tail = msi_desc[free_vector]->link.tail; 322 spin_unlock_irqrestore(&msi_lock, flags); 323 msi_free_vector(dev, tail, 1); 324 } while (free_vector != tail); 325 } 326 327 return free_vector; 328 } 329 vector = assign_irq_vector(AUTO_ASSIGN); 330 last_alloc_vector = vector; 331 if (vector == LAST_DEVICE_VECTOR) 332 new_vector_avail = 0; 333 334 spin_unlock_irqrestore(&msi_lock, flags); 335 return vector; 336} 337 338static int get_new_vector(void) 339{ 340 int vector = assign_msi_vector(); 341 342 if (vector > 0) 343 set_intr_gate(vector, interrupt[vector]); 344 345 return vector; 346} 347 348static int msi_init(void) 349{ 350 static int status = -ENOMEM; 351 352 if (!status) 353 return status; 354 355 if (pci_msi_quirk) { 356 pci_msi_enable = 0; 357 printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n"); 358 status = -EINVAL; 359 return status; 360 } 361 362 status = msi_arch_init(); 363 if (status < 0) { 364 pci_msi_enable = 0; 365 printk(KERN_WARNING 366 "PCI: MSI arch init failed. MSI disabled.\n"); 367 return status; 368 } 369 370 if (! msi_ops) { 371 printk(KERN_WARNING 372 "PCI: MSI ops not registered. MSI disabled.\n"); 373 status = -EINVAL; 374 return status; 375 } 376 377 last_alloc_vector = assign_irq_vector(AUTO_ASSIGN); 378 status = msi_cache_init(); 379 if (status < 0) { 380 pci_msi_enable = 0; 381 printk(KERN_WARNING "PCI: MSI cache init failed\n"); 382 return status; 383 } 384 385 if (last_alloc_vector < 0) { 386 pci_msi_enable = 0; 387 printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n"); 388 status = -EBUSY; 389 return status; 390 } 391 vector_irq[last_alloc_vector] = 0; 392 nr_released_vectors++; 393 394 return status; 395} 396 397static int get_msi_vector(struct pci_dev *dev) 398{ 399 return get_new_vector(); 400} 401 402static struct msi_desc* alloc_msi_entry(void) 403{ 404 struct msi_desc *entry; 405 406 entry = kmem_cache_alloc(msi_cachep, SLAB_KERNEL); 407 if (!entry) 408 return NULL; 409 410 memset(entry, 0, sizeof(struct msi_desc)); 411 entry->link.tail = entry->link.head = 0; /* single message */ 412 entry->dev = NULL; 413 414 return entry; 415} 416 417static void attach_msi_entry(struct msi_desc *entry, int vector) 418{ 419 unsigned long flags; 420 421 spin_lock_irqsave(&msi_lock, flags); 422 msi_desc[vector] = entry; 423 spin_unlock_irqrestore(&msi_lock, flags); 424} 425 426static void irq_handler_init(int cap_id, int pos, int mask) 427{ 428 unsigned long flags; 429 430 spin_lock_irqsave(&irq_desc[pos].lock, flags); 431 if (cap_id == PCI_CAP_ID_MSIX) 432 irq_desc[pos].handler = &msix_irq_type; 433 else { 434 if (!mask) 435 irq_desc[pos].handler = &msi_irq_wo_maskbit_type; 436 else 437 irq_desc[pos].handler = &msi_irq_w_maskbit_type; 438 } 439 spin_unlock_irqrestore(&irq_desc[pos].lock, flags); 440} 441 442static void enable_msi_mode(struct pci_dev *dev, int pos, int type) 443{ 444 u16 control; 445 446 pci_read_config_word(dev, msi_control_reg(pos), &control); 447 if (type == PCI_CAP_ID_MSI) { 448 /* Set enabled bits to single MSI & enable MSI_enable bit */ 449 msi_enable(control, 1); 450 pci_write_config_word(dev, msi_control_reg(pos), control); 451 dev->msi_enabled = 1; 452 } else { 453 msix_enable(control); 454 pci_write_config_word(dev, msi_control_reg(pos), control); 455 dev->msix_enabled = 1; 456 } 457 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) { 458 /* PCI Express Endpoint device detected */ 459 pci_intx(dev, 0); /* disable intx */ 460 } 461} 462 463void disable_msi_mode(struct pci_dev *dev, int pos, int type) 464{ 465 u16 control; 466 467 pci_read_config_word(dev, msi_control_reg(pos), &control); 468 if (type == PCI_CAP_ID_MSI) { 469 /* Set enabled bits to single MSI & enable MSI_enable bit */ 470 msi_disable(control); 471 pci_write_config_word(dev, msi_control_reg(pos), control); 472 dev->msi_enabled = 0; 473 } else { 474 msix_disable(control); 475 pci_write_config_word(dev, msi_control_reg(pos), control); 476 dev->msix_enabled = 0; 477 } 478 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) { 479 /* PCI Express Endpoint device detected */ 480 pci_intx(dev, 1); /* enable intx */ 481 } 482} 483 484static int msi_lookup_vector(struct pci_dev *dev, int type) 485{ 486 int vector; 487 unsigned long flags; 488 489 spin_lock_irqsave(&msi_lock, flags); 490 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) { 491 if (!msi_desc[vector] || msi_desc[vector]->dev != dev || 492 msi_desc[vector]->msi_attrib.type != type || 493 msi_desc[vector]->msi_attrib.default_vector != dev->irq) 494 continue; 495 spin_unlock_irqrestore(&msi_lock, flags); 496 /* This pre-assigned MSI vector for this device 497 already exits. Override dev->irq with this vector */ 498 dev->irq = vector; 499 return 0; 500 } 501 spin_unlock_irqrestore(&msi_lock, flags); 502 503 return -EACCES; 504} 505 506void pci_scan_msi_device(struct pci_dev *dev) 507{ 508 if (!dev) 509 return; 510 511 if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0) 512 nr_msix_devices++; 513 else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0) 514 nr_reserved_vectors++; 515} 516 517#ifdef CONFIG_PM 518int pci_save_msi_state(struct pci_dev *dev) 519{ 520 int pos, i = 0; 521 u16 control; 522 struct pci_cap_saved_state *save_state; 523 u32 *cap; 524 525 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 526 if (pos <= 0 || dev->no_msi) 527 return 0; 528 529 pci_read_config_word(dev, msi_control_reg(pos), &control); 530 if (!(control & PCI_MSI_FLAGS_ENABLE)) 531 return 0; 532 533 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u32) * 5, 534 GFP_KERNEL); 535 if (!save_state) { 536 printk(KERN_ERR "Out of memory in pci_save_msi_state\n"); 537 return -ENOMEM; 538 } 539 cap = &save_state->data[0]; 540 541 pci_read_config_dword(dev, pos, &cap[i++]); 542 control = cap[0] >> 16; 543 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, &cap[i++]); 544 if (control & PCI_MSI_FLAGS_64BIT) { 545 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, &cap[i++]); 546 pci_read_config_dword(dev, pos + PCI_MSI_DATA_64, &cap[i++]); 547 } else 548 pci_read_config_dword(dev, pos + PCI_MSI_DATA_32, &cap[i++]); 549 if (control & PCI_MSI_FLAGS_MASKBIT) 550 pci_read_config_dword(dev, pos + PCI_MSI_MASK_BIT, &cap[i++]); 551 save_state->cap_nr = PCI_CAP_ID_MSI; 552 pci_add_saved_cap(dev, save_state); 553 return 0; 554} 555 556void pci_restore_msi_state(struct pci_dev *dev) 557{ 558 int i = 0, pos; 559 u16 control; 560 struct pci_cap_saved_state *save_state; 561 u32 *cap; 562 563 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSI); 564 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 565 if (!save_state || pos <= 0) 566 return; 567 cap = &save_state->data[0]; 568 569 control = cap[i++] >> 16; 570 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, cap[i++]); 571 if (control & PCI_MSI_FLAGS_64BIT) { 572 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, cap[i++]); 573 pci_write_config_dword(dev, pos + PCI_MSI_DATA_64, cap[i++]); 574 } else 575 pci_write_config_dword(dev, pos + PCI_MSI_DATA_32, cap[i++]); 576 if (control & PCI_MSI_FLAGS_MASKBIT) 577 pci_write_config_dword(dev, pos + PCI_MSI_MASK_BIT, cap[i++]); 578 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 579 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 580 pci_remove_saved_cap(save_state); 581 kfree(save_state); 582} 583 584int pci_save_msix_state(struct pci_dev *dev) 585{ 586 int pos; 587 int temp; 588 int vector, head, tail = 0; 589 u16 control; 590 struct pci_cap_saved_state *save_state; 591 592 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 593 if (pos <= 0 || dev->no_msi) 594 return 0; 595 596 /* save the capability */ 597 pci_read_config_word(dev, msi_control_reg(pos), &control); 598 if (!(control & PCI_MSIX_FLAGS_ENABLE)) 599 return 0; 600 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u16), 601 GFP_KERNEL); 602 if (!save_state) { 603 printk(KERN_ERR "Out of memory in pci_save_msix_state\n"); 604 return -ENOMEM; 605 } 606 *((u16 *)&save_state->data[0]) = control; 607 608 /* save the table */ 609 temp = dev->irq; 610 if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 611 kfree(save_state); 612 return -EINVAL; 613 } 614 615 vector = head = dev->irq; 616 while (head != tail) { 617 int j; 618 void __iomem *base; 619 struct msi_desc *entry; 620 621 entry = msi_desc[vector]; 622 base = entry->mask_base; 623 j = entry->msi_attrib.entry_nr; 624 625 entry->address_lo_save = 626 readl(base + j * PCI_MSIX_ENTRY_SIZE + 627 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 628 entry->address_hi_save = 629 readl(base + j * PCI_MSIX_ENTRY_SIZE + 630 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 631 entry->data_save = 632 readl(base + j * PCI_MSIX_ENTRY_SIZE + 633 PCI_MSIX_ENTRY_DATA_OFFSET); 634 635 tail = msi_desc[vector]->link.tail; 636 vector = tail; 637 } 638 dev->irq = temp; 639 640 save_state->cap_nr = PCI_CAP_ID_MSIX; 641 pci_add_saved_cap(dev, save_state); 642 return 0; 643} 644 645void pci_restore_msix_state(struct pci_dev *dev) 646{ 647 u16 save; 648 int pos; 649 int vector, head, tail = 0; 650 void __iomem *base; 651 int j; 652 struct msi_desc *entry; 653 int temp; 654 struct pci_cap_saved_state *save_state; 655 656 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSIX); 657 if (!save_state) 658 return; 659 save = *((u16 *)&save_state->data[0]); 660 pci_remove_saved_cap(save_state); 661 kfree(save_state); 662 663 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 664 if (pos <= 0) 665 return; 666 667 /* route the table */ 668 temp = dev->irq; 669 if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) 670 return; 671 vector = head = dev->irq; 672 while (head != tail) { 673 entry = msi_desc[vector]; 674 base = entry->mask_base; 675 j = entry->msi_attrib.entry_nr; 676 677 writel(entry->address_lo_save, 678 base + j * PCI_MSIX_ENTRY_SIZE + 679 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 680 writel(entry->address_hi_save, 681 base + j * PCI_MSIX_ENTRY_SIZE + 682 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 683 writel(entry->data_save, 684 base + j * PCI_MSIX_ENTRY_SIZE + 685 PCI_MSIX_ENTRY_DATA_OFFSET); 686 687 tail = msi_desc[vector]->link.tail; 688 vector = tail; 689 } 690 dev->irq = temp; 691 692 pci_write_config_word(dev, msi_control_reg(pos), save); 693 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 694} 695#endif 696 697static int msi_register_init(struct pci_dev *dev, struct msi_desc *entry) 698{ 699 int status; 700 u32 address_hi; 701 u32 address_lo; 702 u32 data; 703 int pos, vector = dev->irq; 704 u16 control; 705 706 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 707 pci_read_config_word(dev, msi_control_reg(pos), &control); 708 709 /* Configure MSI capability structure */ 710 status = msi_ops->setup(dev, vector, &address_hi, &address_lo, &data); 711 if (status < 0) 712 return status; 713 714 pci_write_config_dword(dev, msi_lower_address_reg(pos), address_lo); 715 if (is_64bit_address(control)) { 716 pci_write_config_dword(dev, 717 msi_upper_address_reg(pos), address_hi); 718 pci_write_config_word(dev, 719 msi_data_reg(pos, 1), data); 720 } else 721 pci_write_config_word(dev, 722 msi_data_reg(pos, 0), data); 723 if (entry->msi_attrib.maskbit) { 724 unsigned int maskbits, temp; 725 /* All MSIs are unmasked by default, Mask them all */ 726 pci_read_config_dword(dev, 727 msi_mask_bits_reg(pos, is_64bit_address(control)), 728 &maskbits); 729 temp = (1 << multi_msi_capable(control)); 730 temp = ((temp - 1) & ~temp); 731 maskbits |= temp; 732 pci_write_config_dword(dev, 733 msi_mask_bits_reg(pos, is_64bit_address(control)), 734 maskbits); 735 } 736 737 return 0; 738} 739 740/** 741 * msi_capability_init - configure device's MSI capability structure 742 * @dev: pointer to the pci_dev data structure of MSI device function 743 * 744 * Setup the MSI capability structure of device function with a single 745 * MSI vector, regardless of device function is capable of handling 746 * multiple messages. A return of zero indicates the successful setup 747 * of an entry zero with the new MSI vector or non-zero for otherwise. 748 **/ 749static int msi_capability_init(struct pci_dev *dev) 750{ 751 int status; 752 struct msi_desc *entry; 753 int pos, vector; 754 u16 control; 755 756 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 757 pci_read_config_word(dev, msi_control_reg(pos), &control); 758 /* MSI Entry Initialization */ 759 entry = alloc_msi_entry(); 760 if (!entry) 761 return -ENOMEM; 762 763 vector = get_msi_vector(dev); 764 if (vector < 0) { 765 kmem_cache_free(msi_cachep, entry); 766 return -EBUSY; 767 } 768 entry->link.head = vector; 769 entry->link.tail = vector; 770 entry->msi_attrib.type = PCI_CAP_ID_MSI; 771 entry->msi_attrib.state = 0; /* Mark it not active */ 772 entry->msi_attrib.entry_nr = 0; 773 entry->msi_attrib.maskbit = is_mask_bit_support(control); 774 entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */ 775 dev->irq = vector; 776 entry->dev = dev; 777 if (is_mask_bit_support(control)) { 778 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, 779 is_64bit_address(control)); 780 } 781 /* Replace with MSI handler */ 782 irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit); 783 /* Configure MSI capability structure */ 784 status = msi_register_init(dev, entry); 785 if (status != 0) { 786 dev->irq = entry->msi_attrib.default_vector; 787 kmem_cache_free(msi_cachep, entry); 788 return status; 789 } 790 791 attach_msi_entry(entry, vector); 792 /* Set MSI enabled bits */ 793 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 794 795 return 0; 796} 797 798/** 799 * msix_capability_init - configure device's MSI-X capability 800 * @dev: pointer to the pci_dev data structure of MSI-X device function 801 * @entries: pointer to an array of struct msix_entry entries 802 * @nvec: number of @entries 803 * 804 * Setup the MSI-X capability structure of device function with a 805 * single MSI-X vector. A return of zero indicates the successful setup of 806 * requested MSI-X entries with allocated vectors or non-zero for otherwise. 807 **/ 808static int msix_capability_init(struct pci_dev *dev, 809 struct msix_entry *entries, int nvec) 810{ 811 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL; 812 u32 address_hi; 813 u32 address_lo; 814 u32 data; 815 int status; 816 int vector, pos, i, j, nr_entries, temp = 0; 817 unsigned long phys_addr; 818 u32 table_offset; 819 u16 control; 820 u8 bir; 821 void __iomem *base; 822 823 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 824 /* Request & Map MSI-X table region */ 825 pci_read_config_word(dev, msi_control_reg(pos), &control); 826 nr_entries = multi_msix_capable(control); 827 828 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 829 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 830 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 831 phys_addr = pci_resource_start (dev, bir) + table_offset; 832 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 833 if (base == NULL) 834 return -ENOMEM; 835 836 /* MSI-X Table Initialization */ 837 for (i = 0; i < nvec; i++) { 838 entry = alloc_msi_entry(); 839 if (!entry) 840 break; 841 vector = get_msi_vector(dev); 842 if (vector < 0) { 843 kmem_cache_free(msi_cachep, entry); 844 break; 845 } 846 847 j = entries[i].entry; 848 entries[i].vector = vector; 849 entry->msi_attrib.type = PCI_CAP_ID_MSIX; 850 entry->msi_attrib.state = 0; /* Mark it not active */ 851 entry->msi_attrib.entry_nr = j; 852 entry->msi_attrib.maskbit = 1; 853 entry->msi_attrib.default_vector = dev->irq; 854 entry->dev = dev; 855 entry->mask_base = base; 856 if (!head) { 857 entry->link.head = vector; 858 entry->link.tail = vector; 859 head = entry; 860 } else { 861 entry->link.head = temp; 862 entry->link.tail = tail->link.tail; 863 tail->link.tail = vector; 864 head->link.head = vector; 865 } 866 temp = vector; 867 tail = entry; 868 /* Replace with MSI-X handler */ 869 irq_handler_init(PCI_CAP_ID_MSIX, vector, 1); 870 /* Configure MSI-X capability structure */ 871 status = msi_ops->setup(dev, vector, 872 &address_hi, 873 &address_lo, 874 &data); 875 if (status < 0) 876 break; 877 878 writel(address_lo, 879 base + j * PCI_MSIX_ENTRY_SIZE + 880 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 881 writel(address_hi, 882 base + j * PCI_MSIX_ENTRY_SIZE + 883 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 884 writel(data, 885 base + j * PCI_MSIX_ENTRY_SIZE + 886 PCI_MSIX_ENTRY_DATA_OFFSET); 887 attach_msi_entry(entry, vector); 888 } 889 if (i != nvec) { 890 i--; 891 for (; i >= 0; i--) { 892 vector = (entries + i)->vector; 893 msi_free_vector(dev, vector, 0); 894 (entries + i)->vector = 0; 895 } 896 return -EBUSY; 897 } 898 /* Set MSI-X enabled bits */ 899 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 900 901 return 0; 902} 903 904/** 905 * pci_enable_msi - configure device's MSI capability structure 906 * @dev: pointer to the pci_dev data structure of MSI device function 907 * 908 * Setup the MSI capability structure of device function with 909 * a single MSI vector upon its software driver call to request for 910 * MSI mode enabled on its hardware device function. A return of zero 911 * indicates the successful setup of an entry zero with the new MSI 912 * vector or non-zero for otherwise. 913 **/ 914int pci_enable_msi(struct pci_dev* dev) 915{ 916 struct pci_bus *bus; 917 int pos, temp, status = -EINVAL; 918 u16 control; 919 920 if (!pci_msi_enable || !dev) 921 return status; 922 923 if (dev->no_msi) 924 return status; 925 926 for (bus = dev->bus; bus; bus = bus->parent) 927 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 928 return -EINVAL; 929 930 temp = dev->irq; 931 932 status = msi_init(); 933 if (status < 0) 934 return status; 935 936 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 937 if (!pos) 938 return -EINVAL; 939 940 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 941 /* Lookup Sucess */ 942 unsigned long flags; 943 944 pci_read_config_word(dev, msi_control_reg(pos), &control); 945 if (control & PCI_MSI_FLAGS_ENABLE) 946 return 0; /* Already in MSI mode */ 947 spin_lock_irqsave(&msi_lock, flags); 948 if (!vector_irq[dev->irq]) { 949 msi_desc[dev->irq]->msi_attrib.state = 0; 950 vector_irq[dev->irq] = -1; 951 nr_released_vectors--; 952 spin_unlock_irqrestore(&msi_lock, flags); 953 status = msi_register_init(dev, msi_desc[dev->irq]); 954 if (status == 0) 955 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 956 return status; 957 } 958 spin_unlock_irqrestore(&msi_lock, flags); 959 dev->irq = temp; 960 } 961 /* Check whether driver already requested for MSI-X vectors */ 962 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 963 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 964 printk(KERN_INFO "PCI: %s: Can't enable MSI. " 965 "Device already has MSI-X vectors assigned\n", 966 pci_name(dev)); 967 dev->irq = temp; 968 return -EINVAL; 969 } 970 status = msi_capability_init(dev); 971 if (!status) { 972 if (!pos) 973 nr_reserved_vectors--; /* Only MSI capable */ 974 else if (nr_msix_devices > 0) 975 nr_msix_devices--; /* Both MSI and MSI-X capable, 976 but choose enabling MSI */ 977 } 978 979 return status; 980} 981 982void pci_disable_msi(struct pci_dev* dev) 983{ 984 struct msi_desc *entry; 985 int pos, default_vector; 986 u16 control; 987 unsigned long flags; 988 989 if (!pci_msi_enable) 990 return; 991 if (!dev) 992 return; 993 994 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 995 if (!pos) 996 return; 997 998 pci_read_config_word(dev, msi_control_reg(pos), &control); 999 if (!(control & PCI_MSI_FLAGS_ENABLE)) 1000 return; 1001 1002 spin_lock_irqsave(&msi_lock, flags); 1003 entry = msi_desc[dev->irq]; 1004 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { 1005 spin_unlock_irqrestore(&msi_lock, flags); 1006 return; 1007 } 1008 if (entry->msi_attrib.state) { 1009 spin_unlock_irqrestore(&msi_lock, flags); 1010 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without " 1011 "free_irq() on MSI vector %d\n", 1012 pci_name(dev), dev->irq); 1013 BUG_ON(entry->msi_attrib.state > 0); 1014 } else { 1015 vector_irq[dev->irq] = 0; /* free it */ 1016 nr_released_vectors++; 1017 default_vector = entry->msi_attrib.default_vector; 1018 spin_unlock_irqrestore(&msi_lock, flags); 1019 /* Restore dev->irq to its default pin-assertion vector */ 1020 dev->irq = default_vector; 1021 disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI), 1022 PCI_CAP_ID_MSI); 1023 } 1024} 1025 1026static int msi_free_vector(struct pci_dev* dev, int vector, int reassign) 1027{ 1028 struct msi_desc *entry; 1029 int head, entry_nr, type; 1030 void __iomem *base; 1031 unsigned long flags; 1032 1033 msi_ops->teardown(vector); 1034 1035 spin_lock_irqsave(&msi_lock, flags); 1036 entry = msi_desc[vector]; 1037 if (!entry || entry->dev != dev) { 1038 spin_unlock_irqrestore(&msi_lock, flags); 1039 return -EINVAL; 1040 } 1041 type = entry->msi_attrib.type; 1042 entry_nr = entry->msi_attrib.entry_nr; 1043 head = entry->link.head; 1044 base = entry->mask_base; 1045 msi_desc[entry->link.head]->link.tail = entry->link.tail; 1046 msi_desc[entry->link.tail]->link.head = entry->link.head; 1047 entry->dev = NULL; 1048 if (!reassign) { 1049 vector_irq[vector] = 0; 1050 nr_released_vectors++; 1051 } 1052 msi_desc[vector] = NULL; 1053 spin_unlock_irqrestore(&msi_lock, flags); 1054 1055 kmem_cache_free(msi_cachep, entry); 1056 1057 if (type == PCI_CAP_ID_MSIX) { 1058 if (!reassign) 1059 writel(1, base + 1060 entry_nr * PCI_MSIX_ENTRY_SIZE + 1061 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); 1062 1063 if (head == vector) 1064 iounmap(base); 1065 } 1066 1067 return 0; 1068} 1069 1070static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec) 1071{ 1072 int vector = head, tail = 0; 1073 int i, j = 0, nr_entries = 0; 1074 void __iomem *base; 1075 unsigned long flags; 1076 1077 spin_lock_irqsave(&msi_lock, flags); 1078 while (head != tail) { 1079 nr_entries++; 1080 tail = msi_desc[vector]->link.tail; 1081 if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr) 1082 j = vector; 1083 vector = tail; 1084 } 1085 if (*nvec > nr_entries) { 1086 spin_unlock_irqrestore(&msi_lock, flags); 1087 *nvec = nr_entries; 1088 return -EINVAL; 1089 } 1090 vector = ((j > 0) ? j : head); 1091 for (i = 0; i < *nvec; i++) { 1092 j = msi_desc[vector]->msi_attrib.entry_nr; 1093 msi_desc[vector]->msi_attrib.state = 0; /* Mark it not active */ 1094 vector_irq[vector] = -1; /* Mark it busy */ 1095 nr_released_vectors--; 1096 entries[i].vector = vector; 1097 if (j != (entries + i)->entry) { 1098 base = msi_desc[vector]->mask_base; 1099 msi_desc[vector]->msi_attrib.entry_nr = 1100 (entries + i)->entry; 1101 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE + 1102 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base + 1103 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE + 1104 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 1105 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE + 1106 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base + 1107 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE + 1108 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 1109 writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE + 1110 PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector, 1111 base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE + 1112 PCI_MSIX_ENTRY_DATA_OFFSET); 1113 } 1114 vector = msi_desc[vector]->link.tail; 1115 } 1116 spin_unlock_irqrestore(&msi_lock, flags); 1117 1118 return 0; 1119} 1120 1121/** 1122 * pci_enable_msix - configure device's MSI-X capability structure 1123 * @dev: pointer to the pci_dev data structure of MSI-X device function 1124 * @entries: pointer to an array of MSI-X entries 1125 * @nvec: number of MSI-X vectors requested for allocation by device driver 1126 * 1127 * Setup the MSI-X capability structure of device function with the number 1128 * of requested vectors upon its software driver call to request for 1129 * MSI-X mode enabled on its hardware device function. A return of zero 1130 * indicates the successful configuration of MSI-X capability structure 1131 * with new allocated MSI-X vectors. A return of < 0 indicates a failure. 1132 * Or a return of > 0 indicates that driver request is exceeding the number 1133 * of vectors available. Driver should use the returned value to re-send 1134 * its request. 1135 **/ 1136int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) 1137{ 1138 struct pci_bus *bus; 1139 int status, pos, nr_entries, free_vectors; 1140 int i, j, temp; 1141 u16 control; 1142 unsigned long flags; 1143 1144 if (!pci_msi_enable || !dev || !entries) 1145 return -EINVAL; 1146 1147 if (dev->no_msi) 1148 return -EINVAL; 1149 1150 for (bus = dev->bus; bus; bus = bus->parent) 1151 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 1152 return -EINVAL; 1153 1154 status = msi_init(); 1155 if (status < 0) 1156 return status; 1157 1158 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 1159 if (!pos) 1160 return -EINVAL; 1161 1162 pci_read_config_word(dev, msi_control_reg(pos), &control); 1163 if (control & PCI_MSIX_FLAGS_ENABLE) 1164 return -EINVAL; /* Already in MSI-X mode */ 1165 1166 nr_entries = multi_msix_capable(control); 1167 if (nvec > nr_entries) 1168 return -EINVAL; 1169 1170 /* Check for any invalid entries */ 1171 for (i = 0; i < nvec; i++) { 1172 if (entries[i].entry >= nr_entries) 1173 return -EINVAL; /* invalid entry */ 1174 for (j = i + 1; j < nvec; j++) { 1175 if (entries[i].entry == entries[j].entry) 1176 return -EINVAL; /* duplicate entry */ 1177 } 1178 } 1179 temp = dev->irq; 1180 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 1181 /* Lookup Sucess */ 1182 nr_entries = nvec; 1183 /* Reroute MSI-X table */ 1184 if (reroute_msix_table(dev->irq, entries, &nr_entries)) { 1185 /* #requested > #previous-assigned */ 1186 dev->irq = temp; 1187 return nr_entries; 1188 } 1189 dev->irq = temp; 1190 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 1191 return 0; 1192 } 1193 /* Check whether driver already requested for MSI vector */ 1194 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 && 1195 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 1196 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " 1197 "Device already has an MSI vector assigned\n", 1198 pci_name(dev)); 1199 dev->irq = temp; 1200 return -EINVAL; 1201 } 1202 1203 spin_lock_irqsave(&msi_lock, flags); 1204 /* 1205 * msi_lock is provided to ensure that enough vectors resources are 1206 * available before granting. 1207 */ 1208 free_vectors = pci_vector_resources(last_alloc_vector, 1209 nr_released_vectors); 1210 /* Ensure that each MSI/MSI-X device has one vector reserved by 1211 default to avoid any MSI-X driver to take all available 1212 resources */ 1213 free_vectors -= nr_reserved_vectors; 1214 /* Find the average of free vectors among MSI-X devices */ 1215 if (nr_msix_devices > 0) 1216 free_vectors /= nr_msix_devices; 1217 spin_unlock_irqrestore(&msi_lock, flags); 1218 1219 if (nvec > free_vectors) { 1220 if (free_vectors > 0) 1221 return free_vectors; 1222 else 1223 return -EBUSY; 1224 } 1225 1226 status = msix_capability_init(dev, entries, nvec); 1227 if (!status && nr_msix_devices > 0) 1228 nr_msix_devices--; 1229 1230 return status; 1231} 1232 1233void pci_disable_msix(struct pci_dev* dev) 1234{ 1235 int pos, temp; 1236 u16 control; 1237 1238 if (!pci_msi_enable) 1239 return; 1240 if (!dev) 1241 return; 1242 1243 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 1244 if (!pos) 1245 return; 1246 1247 pci_read_config_word(dev, msi_control_reg(pos), &control); 1248 if (!(control & PCI_MSIX_FLAGS_ENABLE)) 1249 return; 1250 1251 temp = dev->irq; 1252 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 1253 int state, vector, head, tail = 0, warning = 0; 1254 unsigned long flags; 1255 1256 vector = head = dev->irq; 1257 spin_lock_irqsave(&msi_lock, flags); 1258 while (head != tail) { 1259 state = msi_desc[vector]->msi_attrib.state; 1260 if (state) 1261 warning = 1; 1262 else { 1263 vector_irq[vector] = 0; /* free it */ 1264 nr_released_vectors++; 1265 } 1266 tail = msi_desc[vector]->link.tail; 1267 vector = tail; 1268 } 1269 spin_unlock_irqrestore(&msi_lock, flags); 1270 if (warning) { 1271 dev->irq = temp; 1272 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without " 1273 "free_irq() on all MSI-X vectors\n", 1274 pci_name(dev)); 1275 BUG_ON(warning > 0); 1276 } else { 1277 dev->irq = temp; 1278 disable_msi_mode(dev, 1279 pci_find_capability(dev, PCI_CAP_ID_MSIX), 1280 PCI_CAP_ID_MSIX); 1281 1282 } 1283 } 1284} 1285 1286/** 1287 * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state 1288 * @dev: pointer to the pci_dev data structure of MSI(X) device function 1289 * 1290 * Being called during hotplug remove, from which the device function 1291 * is hot-removed. All previous assigned MSI/MSI-X vectors, if 1292 * allocated for this device function, are reclaimed to unused state, 1293 * which may be used later on. 1294 **/ 1295void msi_remove_pci_irq_vectors(struct pci_dev* dev) 1296{ 1297 int state, pos, temp; 1298 unsigned long flags; 1299 1300 if (!pci_msi_enable || !dev) 1301 return; 1302 1303 temp = dev->irq; /* Save IOAPIC IRQ */ 1304 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 1305 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 1306 spin_lock_irqsave(&msi_lock, flags); 1307 state = msi_desc[dev->irq]->msi_attrib.state; 1308 spin_unlock_irqrestore(&msi_lock, flags); 1309 if (state) { 1310 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 1311 "called without free_irq() on MSI vector %d\n", 1312 pci_name(dev), dev->irq); 1313 BUG_ON(state > 0); 1314 } else /* Release MSI vector assigned to this device */ 1315 msi_free_vector(dev, dev->irq, 0); 1316 dev->irq = temp; /* Restore IOAPIC IRQ */ 1317 } 1318 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 1319 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 1320 int vector, head, tail = 0, warning = 0; 1321 void __iomem *base = NULL; 1322 1323 vector = head = dev->irq; 1324 while (head != tail) { 1325 spin_lock_irqsave(&msi_lock, flags); 1326 state = msi_desc[vector]->msi_attrib.state; 1327 tail = msi_desc[vector]->link.tail; 1328 base = msi_desc[vector]->mask_base; 1329 spin_unlock_irqrestore(&msi_lock, flags); 1330 if (state) 1331 warning = 1; 1332 else if (vector != head) /* Release MSI-X vector */ 1333 msi_free_vector(dev, vector, 0); 1334 vector = tail; 1335 } 1336 msi_free_vector(dev, vector, 0); 1337 if (warning) { 1338 iounmap(base); 1339 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 1340 "called without free_irq() on all MSI-X vectors\n", 1341 pci_name(dev)); 1342 BUG_ON(warning > 0); 1343 } 1344 dev->irq = temp; /* Restore IOAPIC IRQ */ 1345 } 1346} 1347 1348void pci_no_msi(void) 1349{ 1350 pci_msi_enable = 0; 1351} 1352 1353EXPORT_SYMBOL(pci_enable_msi); 1354EXPORT_SYMBOL(pci_disable_msi); 1355EXPORT_SYMBOL(pci_enable_msix); 1356EXPORT_SYMBOL(pci_disable_msix); 1357