msi.c revision 5b912c108c8b1fcecbfe13d6d9a183db97b682d3
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/err.h> 10#include <linux/mm.h> 11#include <linux/irq.h> 12#include <linux/interrupt.h> 13#include <linux/init.h> 14#include <linux/ioport.h> 15#include <linux/smp_lock.h> 16#include <linux/pci.h> 17#include <linux/proc_fs.h> 18#include <linux/msi.h> 19 20#include <asm/errno.h> 21#include <asm/io.h> 22#include <asm/smp.h> 23 24#include "pci.h" 25#include "msi.h" 26 27static struct kmem_cache* msi_cachep; 28 29static int pci_msi_enable = 1; 30 31static int msi_cache_init(void) 32{ 33 msi_cachep = kmem_cache_create("msi_cache", sizeof(struct msi_desc), 34 0, SLAB_HWCACHE_ALIGN, NULL, NULL); 35 if (!msi_cachep) 36 return -ENOMEM; 37 38 return 0; 39} 40 41static void msi_set_mask_bit(unsigned int irq, int flag) 42{ 43 struct msi_desc *entry; 44 45 entry = get_irq_msi(irq); 46 BUG_ON(!entry || !entry->dev); 47 switch (entry->msi_attrib.type) { 48 case PCI_CAP_ID_MSI: 49 if (entry->msi_attrib.maskbit) { 50 int pos; 51 u32 mask_bits; 52 53 pos = (long)entry->mask_base; 54 pci_read_config_dword(entry->dev, pos, &mask_bits); 55 mask_bits &= ~(1); 56 mask_bits |= flag; 57 pci_write_config_dword(entry->dev, pos, mask_bits); 58 } 59 break; 60 case PCI_CAP_ID_MSIX: 61 { 62 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 63 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 64 writel(flag, entry->mask_base + offset); 65 break; 66 } 67 default: 68 BUG(); 69 break; 70 } 71} 72 73void read_msi_msg(unsigned int irq, struct msi_msg *msg) 74{ 75 struct msi_desc *entry = get_irq_msi(irq); 76 switch(entry->msi_attrib.type) { 77 case PCI_CAP_ID_MSI: 78 { 79 struct pci_dev *dev = entry->dev; 80 int pos = entry->msi_attrib.pos; 81 u16 data; 82 83 pci_read_config_dword(dev, msi_lower_address_reg(pos), 84 &msg->address_lo); 85 if (entry->msi_attrib.is_64) { 86 pci_read_config_dword(dev, msi_upper_address_reg(pos), 87 &msg->address_hi); 88 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 89 } else { 90 msg->address_hi = 0; 91 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 92 } 93 msg->data = data; 94 break; 95 } 96 case PCI_CAP_ID_MSIX: 97 { 98 void __iomem *base; 99 base = entry->mask_base + 100 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 101 102 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 103 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 104 msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET); 105 break; 106 } 107 default: 108 BUG(); 109 } 110} 111 112void write_msi_msg(unsigned int irq, struct msi_msg *msg) 113{ 114 struct msi_desc *entry = get_irq_msi(irq); 115 switch (entry->msi_attrib.type) { 116 case PCI_CAP_ID_MSI: 117 { 118 struct pci_dev *dev = entry->dev; 119 int pos = entry->msi_attrib.pos; 120 121 pci_write_config_dword(dev, msi_lower_address_reg(pos), 122 msg->address_lo); 123 if (entry->msi_attrib.is_64) { 124 pci_write_config_dword(dev, msi_upper_address_reg(pos), 125 msg->address_hi); 126 pci_write_config_word(dev, msi_data_reg(pos, 1), 127 msg->data); 128 } else { 129 pci_write_config_word(dev, msi_data_reg(pos, 0), 130 msg->data); 131 } 132 break; 133 } 134 case PCI_CAP_ID_MSIX: 135 { 136 void __iomem *base; 137 base = entry->mask_base + 138 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 139 140 writel(msg->address_lo, 141 base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 142 writel(msg->address_hi, 143 base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 144 writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET); 145 break; 146 } 147 default: 148 BUG(); 149 } 150} 151 152void mask_msi_irq(unsigned int irq) 153{ 154 msi_set_mask_bit(irq, 1); 155} 156 157void unmask_msi_irq(unsigned int irq) 158{ 159 msi_set_mask_bit(irq, 0); 160} 161 162static int msi_free_irq(struct pci_dev* dev, int irq); 163 164static int msi_init(void) 165{ 166 static int status = -ENOMEM; 167 168 if (!status) 169 return status; 170 171 status = msi_cache_init(); 172 if (status < 0) { 173 pci_msi_enable = 0; 174 printk(KERN_WARNING "PCI: MSI cache init failed\n"); 175 return status; 176 } 177 178 return status; 179} 180 181static struct msi_desc* alloc_msi_entry(void) 182{ 183 struct msi_desc *entry; 184 185 entry = kmem_cache_zalloc(msi_cachep, GFP_KERNEL); 186 if (!entry) 187 return NULL; 188 189 entry->link.tail = entry->link.head = 0; /* single message */ 190 entry->dev = NULL; 191 192 return entry; 193} 194 195static int create_msi_irq(void) 196{ 197 struct msi_desc *entry; 198 int irq; 199 200 entry = alloc_msi_entry(); 201 if (!entry) 202 return -ENOMEM; 203 204 irq = create_irq(); 205 if (irq < 0) { 206 kmem_cache_free(msi_cachep, entry); 207 return -EBUSY; 208 } 209 210 set_irq_msi(irq, entry); 211 212 return irq; 213} 214 215static void destroy_msi_irq(unsigned int irq) 216{ 217 struct msi_desc *entry; 218 219 entry = get_irq_msi(irq); 220 set_irq_chip(irq, NULL); 221 set_irq_msi(irq, NULL); 222 destroy_irq(irq); 223 kmem_cache_free(msi_cachep, entry); 224} 225 226static void enable_msi_mode(struct pci_dev *dev, int pos, int type) 227{ 228 u16 control; 229 230 pci_read_config_word(dev, msi_control_reg(pos), &control); 231 if (type == PCI_CAP_ID_MSI) { 232 /* Set enabled bits to single MSI & enable MSI_enable bit */ 233 msi_enable(control, 1); 234 pci_write_config_word(dev, msi_control_reg(pos), control); 235 dev->msi_enabled = 1; 236 } else { 237 msix_enable(control); 238 pci_write_config_word(dev, msi_control_reg(pos), control); 239 dev->msix_enabled = 1; 240 } 241 242 pci_intx(dev, 0); /* disable intx */ 243} 244 245void disable_msi_mode(struct pci_dev *dev, int pos, int type) 246{ 247 u16 control; 248 249 pci_read_config_word(dev, msi_control_reg(pos), &control); 250 if (type == PCI_CAP_ID_MSI) { 251 /* Set enabled bits to single MSI & enable MSI_enable bit */ 252 msi_disable(control); 253 pci_write_config_word(dev, msi_control_reg(pos), control); 254 dev->msi_enabled = 0; 255 } else { 256 msix_disable(control); 257 pci_write_config_word(dev, msi_control_reg(pos), control); 258 dev->msix_enabled = 0; 259 } 260 261 pci_intx(dev, 1); /* enable intx */ 262} 263 264#ifdef CONFIG_PM 265static int __pci_save_msi_state(struct pci_dev *dev) 266{ 267 int pos, i = 0; 268 u16 control; 269 struct pci_cap_saved_state *save_state; 270 u32 *cap; 271 272 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 273 if (pos <= 0 || dev->no_msi) 274 return 0; 275 276 pci_read_config_word(dev, msi_control_reg(pos), &control); 277 if (!(control & PCI_MSI_FLAGS_ENABLE)) 278 return 0; 279 280 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u32) * 5, 281 GFP_KERNEL); 282 if (!save_state) { 283 printk(KERN_ERR "Out of memory in pci_save_msi_state\n"); 284 return -ENOMEM; 285 } 286 cap = &save_state->data[0]; 287 288 pci_read_config_dword(dev, pos, &cap[i++]); 289 control = cap[0] >> 16; 290 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, &cap[i++]); 291 if (control & PCI_MSI_FLAGS_64BIT) { 292 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, &cap[i++]); 293 pci_read_config_dword(dev, pos + PCI_MSI_DATA_64, &cap[i++]); 294 } else 295 pci_read_config_dword(dev, pos + PCI_MSI_DATA_32, &cap[i++]); 296 if (control & PCI_MSI_FLAGS_MASKBIT) 297 pci_read_config_dword(dev, pos + PCI_MSI_MASK_BIT, &cap[i++]); 298 save_state->cap_nr = PCI_CAP_ID_MSI; 299 pci_add_saved_cap(dev, save_state); 300 return 0; 301} 302 303static void __pci_restore_msi_state(struct pci_dev *dev) 304{ 305 int i = 0, pos; 306 u16 control; 307 struct pci_cap_saved_state *save_state; 308 u32 *cap; 309 310 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSI); 311 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 312 if (!save_state || pos <= 0) 313 return; 314 cap = &save_state->data[0]; 315 316 control = cap[i++] >> 16; 317 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, cap[i++]); 318 if (control & PCI_MSI_FLAGS_64BIT) { 319 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, cap[i++]); 320 pci_write_config_dword(dev, pos + PCI_MSI_DATA_64, cap[i++]); 321 } else 322 pci_write_config_dword(dev, pos + PCI_MSI_DATA_32, cap[i++]); 323 if (control & PCI_MSI_FLAGS_MASKBIT) 324 pci_write_config_dword(dev, pos + PCI_MSI_MASK_BIT, cap[i++]); 325 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 326 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 327 pci_remove_saved_cap(save_state); 328 kfree(save_state); 329} 330 331static int __pci_save_msix_state(struct pci_dev *dev) 332{ 333 int pos; 334 int irq, head, tail = 0; 335 u16 control; 336 struct pci_cap_saved_state *save_state; 337 338 if (!dev->msix_enabled) 339 return 0; 340 341 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 342 if (pos <= 0 || dev->no_msi) 343 return 0; 344 345 /* save the capability */ 346 pci_read_config_word(dev, msi_control_reg(pos), &control); 347 if (!(control & PCI_MSIX_FLAGS_ENABLE)) 348 return 0; 349 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u16), 350 GFP_KERNEL); 351 if (!save_state) { 352 printk(KERN_ERR "Out of memory in pci_save_msix_state\n"); 353 return -ENOMEM; 354 } 355 *((u16 *)&save_state->data[0]) = control; 356 357 /* save the table */ 358 irq = head = dev->first_msi_irq; 359 while (head != tail) { 360 struct msi_desc *entry; 361 362 entry = get_irq_msi(irq); 363 read_msi_msg(irq, &entry->msg_save); 364 365 tail = entry->link.tail; 366 irq = tail; 367 } 368 369 save_state->cap_nr = PCI_CAP_ID_MSIX; 370 pci_add_saved_cap(dev, save_state); 371 return 0; 372} 373 374int pci_save_msi_state(struct pci_dev *dev) 375{ 376 int rc; 377 378 rc = __pci_save_msi_state(dev); 379 if (rc) 380 return rc; 381 382 rc = __pci_save_msix_state(dev); 383 384 return rc; 385} 386 387static void __pci_restore_msix_state(struct pci_dev *dev) 388{ 389 u16 save; 390 int pos; 391 int irq, head, tail = 0; 392 struct msi_desc *entry; 393 struct pci_cap_saved_state *save_state; 394 395 if (!dev->msix_enabled) 396 return; 397 398 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSIX); 399 if (!save_state) 400 return; 401 save = *((u16 *)&save_state->data[0]); 402 pci_remove_saved_cap(save_state); 403 kfree(save_state); 404 405 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 406 if (pos <= 0) 407 return; 408 409 /* route the table */ 410 irq = head = dev->first_msi_irq; 411 while (head != tail) { 412 entry = get_irq_msi(irq); 413 write_msi_msg(irq, &entry->msg_save); 414 415 tail = entry->link.tail; 416 irq = tail; 417 } 418 419 pci_write_config_word(dev, msi_control_reg(pos), save); 420 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 421} 422 423void pci_restore_msi_state(struct pci_dev *dev) 424{ 425 __pci_restore_msi_state(dev); 426 __pci_restore_msix_state(dev); 427} 428#endif /* CONFIG_PM */ 429 430/** 431 * msi_capability_init - configure device's MSI capability structure 432 * @dev: pointer to the pci_dev data structure of MSI device function 433 * 434 * Setup the MSI capability structure of device function with a single 435 * MSI irq, regardless of device function is capable of handling 436 * multiple messages. A return of zero indicates the successful setup 437 * of an entry zero with the new MSI irq or non-zero for otherwise. 438 **/ 439static int msi_capability_init(struct pci_dev *dev) 440{ 441 int status; 442 struct msi_desc *entry; 443 int pos, irq; 444 u16 control; 445 446 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 447 pci_read_config_word(dev, msi_control_reg(pos), &control); 448 /* MSI Entry Initialization */ 449 irq = create_msi_irq(); 450 if (irq < 0) 451 return irq; 452 453 entry = get_irq_msi(irq); 454 entry->link.head = irq; 455 entry->link.tail = irq; 456 entry->msi_attrib.type = PCI_CAP_ID_MSI; 457 entry->msi_attrib.is_64 = is_64bit_address(control); 458 entry->msi_attrib.entry_nr = 0; 459 entry->msi_attrib.maskbit = is_mask_bit_support(control); 460 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ 461 entry->msi_attrib.pos = pos; 462 if (is_mask_bit_support(control)) { 463 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, 464 is_64bit_address(control)); 465 } 466 entry->dev = dev; 467 if (entry->msi_attrib.maskbit) { 468 unsigned int maskbits, temp; 469 /* All MSIs are unmasked by default, Mask them all */ 470 pci_read_config_dword(dev, 471 msi_mask_bits_reg(pos, is_64bit_address(control)), 472 &maskbits); 473 temp = (1 << multi_msi_capable(control)); 474 temp = ((temp - 1) & ~temp); 475 maskbits |= temp; 476 pci_write_config_dword(dev, 477 msi_mask_bits_reg(pos, is_64bit_address(control)), 478 maskbits); 479 } 480 /* Configure MSI capability structure */ 481 status = arch_setup_msi_irq(irq, dev); 482 if (status < 0) { 483 destroy_msi_irq(irq); 484 return status; 485 } 486 487 dev->first_msi_irq = irq; 488 set_irq_msi(irq, entry); 489 /* Set MSI enabled bits */ 490 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 491 492 dev->irq = irq; 493 return 0; 494} 495 496/** 497 * msix_capability_init - configure device's MSI-X capability 498 * @dev: pointer to the pci_dev data structure of MSI-X device function 499 * @entries: pointer to an array of struct msix_entry entries 500 * @nvec: number of @entries 501 * 502 * Setup the MSI-X capability structure of device function with a 503 * single MSI-X irq. A return of zero indicates the successful setup of 504 * requested MSI-X entries with allocated irqs or non-zero for otherwise. 505 **/ 506static int msix_capability_init(struct pci_dev *dev, 507 struct msix_entry *entries, int nvec) 508{ 509 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL; 510 int status; 511 int irq, pos, i, j, nr_entries, temp = 0; 512 unsigned long phys_addr; 513 u32 table_offset; 514 u16 control; 515 u8 bir; 516 void __iomem *base; 517 518 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 519 /* Request & Map MSI-X table region */ 520 pci_read_config_word(dev, msi_control_reg(pos), &control); 521 nr_entries = multi_msix_capable(control); 522 523 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 524 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 525 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 526 phys_addr = pci_resource_start (dev, bir) + table_offset; 527 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 528 if (base == NULL) 529 return -ENOMEM; 530 531 /* MSI-X Table Initialization */ 532 for (i = 0; i < nvec; i++) { 533 irq = create_msi_irq(); 534 if (irq < 0) 535 break; 536 537 entry = get_irq_msi(irq); 538 j = entries[i].entry; 539 entries[i].vector = irq; 540 entry->msi_attrib.type = PCI_CAP_ID_MSIX; 541 entry->msi_attrib.is_64 = 1; 542 entry->msi_attrib.entry_nr = j; 543 entry->msi_attrib.maskbit = 1; 544 entry->msi_attrib.default_irq = dev->irq; 545 entry->msi_attrib.pos = pos; 546 entry->dev = dev; 547 entry->mask_base = base; 548 if (!head) { 549 entry->link.head = irq; 550 entry->link.tail = irq; 551 head = entry; 552 } else { 553 entry->link.head = temp; 554 entry->link.tail = tail->link.tail; 555 tail->link.tail = irq; 556 head->link.head = irq; 557 } 558 temp = irq; 559 tail = entry; 560 /* Configure MSI-X capability structure */ 561 status = arch_setup_msi_irq(irq, dev); 562 if (status < 0) { 563 destroy_msi_irq(irq); 564 break; 565 } 566 567 set_irq_msi(irq, entry); 568 } 569 if (i != nvec) { 570 int avail = i - 1; 571 i--; 572 for (; i >= 0; i--) { 573 irq = (entries + i)->vector; 574 msi_free_irq(dev, irq); 575 (entries + i)->vector = 0; 576 } 577 /* If we had some success report the number of irqs 578 * we succeeded in setting up. 579 */ 580 if (avail <= 0) 581 avail = -EBUSY; 582 return avail; 583 } 584 dev->first_msi_irq = entries[0].vector; 585 /* Set MSI-X enabled bits */ 586 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 587 588 return 0; 589} 590 591/** 592 * pci_msi_supported - check whether MSI may be enabled on device 593 * @dev: pointer to the pci_dev data structure of MSI device function 594 * 595 * Look at global flags, the device itself, and its parent busses 596 * to return 0 if MSI are supported for the device. 597 **/ 598static 599int pci_msi_supported(struct pci_dev * dev) 600{ 601 struct pci_bus *bus; 602 603 /* MSI must be globally enabled and supported by the device */ 604 if (!pci_msi_enable || !dev || dev->no_msi) 605 return -EINVAL; 606 607 /* Any bridge which does NOT route MSI transactions from it's 608 * secondary bus to it's primary bus must set NO_MSI flag on 609 * the secondary pci_bus. 610 * We expect only arch-specific PCI host bus controller driver 611 * or quirks for specific PCI bridges to be setting NO_MSI. 612 */ 613 for (bus = dev->bus; bus; bus = bus->parent) 614 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 615 return -EINVAL; 616 617 return 0; 618} 619 620/** 621 * pci_enable_msi - configure device's MSI capability structure 622 * @dev: pointer to the pci_dev data structure of MSI device function 623 * 624 * Setup the MSI capability structure of device function with 625 * a single MSI irq upon its software driver call to request for 626 * MSI mode enabled on its hardware device function. A return of zero 627 * indicates the successful setup of an entry zero with the new MSI 628 * irq or non-zero for otherwise. 629 **/ 630int pci_enable_msi(struct pci_dev* dev) 631{ 632 int pos, status; 633 634 if (pci_msi_supported(dev) < 0) 635 return -EINVAL; 636 637 status = msi_init(); 638 if (status < 0) 639 return status; 640 641 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 642 if (!pos) 643 return -EINVAL; 644 645 WARN_ON(!!dev->msi_enabled); 646 647 /* Check whether driver already requested for MSI-X irqs */ 648 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 649 if (pos > 0 && dev->msix_enabled) { 650 printk(KERN_INFO "PCI: %s: Can't enable MSI. " 651 "Device already has MSI-X enabled\n", 652 pci_name(dev)); 653 return -EINVAL; 654 } 655 status = msi_capability_init(dev); 656 return status; 657} 658 659void pci_disable_msi(struct pci_dev* dev) 660{ 661 struct msi_desc *entry; 662 int pos, default_irq; 663 u16 control; 664 665 if (!pci_msi_enable) 666 return; 667 if (!dev) 668 return; 669 670 if (!dev->msi_enabled) 671 return; 672 673 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 674 if (!pos) 675 return; 676 677 pci_read_config_word(dev, msi_control_reg(pos), &control); 678 if (!(control & PCI_MSI_FLAGS_ENABLE)) 679 return; 680 681 682 disable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 683 684 entry = get_irq_msi(dev->first_msi_irq); 685 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { 686 return; 687 } 688 if (irq_has_action(dev->first_msi_irq)) { 689 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without " 690 "free_irq() on MSI irq %d\n", 691 pci_name(dev), dev->first_msi_irq); 692 BUG_ON(irq_has_action(dev->first_msi_irq)); 693 } else { 694 default_irq = entry->msi_attrib.default_irq; 695 msi_free_irq(dev, dev->first_msi_irq); 696 697 /* Restore dev->irq to its default pin-assertion irq */ 698 dev->irq = default_irq; 699 } 700 dev->first_msi_irq = 0; 701} 702 703static int msi_free_irq(struct pci_dev* dev, int irq) 704{ 705 struct msi_desc *entry; 706 int head, entry_nr, type; 707 void __iomem *base; 708 709 arch_teardown_msi_irq(irq); 710 711 entry = get_irq_msi(irq); 712 if (!entry || entry->dev != dev) { 713 return -EINVAL; 714 } 715 type = entry->msi_attrib.type; 716 entry_nr = entry->msi_attrib.entry_nr; 717 head = entry->link.head; 718 base = entry->mask_base; 719 get_irq_msi(entry->link.head)->link.tail = entry->link.tail; 720 get_irq_msi(entry->link.tail)->link.head = entry->link.head; 721 entry->dev = NULL; 722 723 destroy_msi_irq(irq); 724 725 if (type == PCI_CAP_ID_MSIX) { 726 writel(1, base + entry_nr * PCI_MSIX_ENTRY_SIZE + 727 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); 728 729 if (head == irq) 730 iounmap(base); 731 } 732 733 return 0; 734} 735 736/** 737 * pci_enable_msix - configure device's MSI-X capability structure 738 * @dev: pointer to the pci_dev data structure of MSI-X device function 739 * @entries: pointer to an array of MSI-X entries 740 * @nvec: number of MSI-X irqs requested for allocation by device driver 741 * 742 * Setup the MSI-X capability structure of device function with the number 743 * of requested irqs upon its software driver call to request for 744 * MSI-X mode enabled on its hardware device function. A return of zero 745 * indicates the successful configuration of MSI-X capability structure 746 * with new allocated MSI-X irqs. A return of < 0 indicates a failure. 747 * Or a return of > 0 indicates that driver request is exceeding the number 748 * of irqs available. Driver should use the returned value to re-send 749 * its request. 750 **/ 751int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) 752{ 753 int status, pos, nr_entries; 754 int i, j; 755 u16 control; 756 757 if (!entries || pci_msi_supported(dev) < 0) 758 return -EINVAL; 759 760 status = msi_init(); 761 if (status < 0) 762 return status; 763 764 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 765 if (!pos) 766 return -EINVAL; 767 768 pci_read_config_word(dev, msi_control_reg(pos), &control); 769 nr_entries = multi_msix_capable(control); 770 if (nvec > nr_entries) 771 return -EINVAL; 772 773 /* Check for any invalid entries */ 774 for (i = 0; i < nvec; i++) { 775 if (entries[i].entry >= nr_entries) 776 return -EINVAL; /* invalid entry */ 777 for (j = i + 1; j < nvec; j++) { 778 if (entries[i].entry == entries[j].entry) 779 return -EINVAL; /* duplicate entry */ 780 } 781 } 782 WARN_ON(!!dev->msix_enabled); 783 784 /* Check whether driver already requested for MSI irq */ 785 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 && 786 dev->msi_enabled) { 787 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " 788 "Device already has an MSI irq assigned\n", 789 pci_name(dev)); 790 return -EINVAL; 791 } 792 status = msix_capability_init(dev, entries, nvec); 793 return status; 794} 795 796void pci_disable_msix(struct pci_dev* dev) 797{ 798 int irq, head, tail = 0, warning = 0; 799 int pos; 800 u16 control; 801 802 if (!pci_msi_enable) 803 return; 804 if (!dev) 805 return; 806 807 if (!dev->msix_enabled) 808 return; 809 810 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 811 if (!pos) 812 return; 813 814 pci_read_config_word(dev, msi_control_reg(pos), &control); 815 if (!(control & PCI_MSIX_FLAGS_ENABLE)) 816 return; 817 818 disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 819 820 irq = head = dev->first_msi_irq; 821 while (head != tail) { 822 tail = get_irq_msi(irq)->link.tail; 823 if (irq_has_action(irq)) 824 warning = 1; 825 else if (irq != head) /* Release MSI-X irq */ 826 msi_free_irq(dev, irq); 827 irq = tail; 828 } 829 msi_free_irq(dev, irq); 830 if (warning) { 831 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without " 832 "free_irq() on all MSI-X irqs\n", 833 pci_name(dev)); 834 BUG_ON(warning > 0); 835 } 836 dev->first_msi_irq = 0; 837} 838 839/** 840 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state 841 * @dev: pointer to the pci_dev data structure of MSI(X) device function 842 * 843 * Being called during hotplug remove, from which the device function 844 * is hot-removed. All previous assigned MSI/MSI-X irqs, if 845 * allocated for this device function, are reclaimed to unused state, 846 * which may be used later on. 847 **/ 848void msi_remove_pci_irq_vectors(struct pci_dev* dev) 849{ 850 if (!pci_msi_enable || !dev) 851 return; 852 853 if (dev->msi_enabled) { 854 if (irq_has_action(dev->first_msi_irq)) { 855 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 856 "called without free_irq() on MSI irq %d\n", 857 pci_name(dev), dev->first_msi_irq); 858 BUG_ON(irq_has_action(dev->first_msi_irq)); 859 } else /* Release MSI irq assigned to this device */ 860 msi_free_irq(dev, dev->first_msi_irq); 861 } 862 if (dev->msix_enabled) { 863 int irq, head, tail = 0, warning = 0; 864 void __iomem *base = NULL; 865 866 irq = head = dev->first_msi_irq; 867 while (head != tail) { 868 tail = get_irq_msi(irq)->link.tail; 869 base = get_irq_msi(irq)->mask_base; 870 if (irq_has_action(irq)) 871 warning = 1; 872 else if (irq != head) /* Release MSI-X irq */ 873 msi_free_irq(dev, irq); 874 irq = tail; 875 } 876 msi_free_irq(dev, irq); 877 if (warning) { 878 iounmap(base); 879 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 880 "called without free_irq() on all MSI-X irqs\n", 881 pci_name(dev)); 882 BUG_ON(warning > 0); 883 } 884 } 885} 886 887void pci_no_msi(void) 888{ 889 pci_msi_enable = 0; 890} 891 892EXPORT_SYMBOL(pci_enable_msi); 893EXPORT_SYMBOL(pci_disable_msi); 894EXPORT_SYMBOL(pci_enable_msix); 895EXPORT_SYMBOL(pci_disable_msix); 896