msi.c revision 1c659d61cfbd8dc3926688c1bbf12d80f4cfb5c2
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/err.h> 10#include <linux/mm.h> 11#include <linux/irq.h> 12#include <linux/interrupt.h> 13#include <linux/init.h> 14#include <linux/ioport.h> 15#include <linux/smp_lock.h> 16#include <linux/pci.h> 17#include <linux/proc_fs.h> 18#include <linux/msi.h> 19 20#include <asm/errno.h> 21#include <asm/io.h> 22#include <asm/smp.h> 23 24#include "pci.h" 25#include "msi.h" 26 27static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL }; 28static struct kmem_cache* msi_cachep; 29 30static int pci_msi_enable = 1; 31 32static int msi_cache_init(void) 33{ 34 msi_cachep = kmem_cache_create("msi_cache", sizeof(struct msi_desc), 35 0, SLAB_HWCACHE_ALIGN, NULL, NULL); 36 if (!msi_cachep) 37 return -ENOMEM; 38 39 return 0; 40} 41 42static void msi_set_mask_bit(unsigned int irq, int flag) 43{ 44 struct msi_desc *entry; 45 46 entry = msi_desc[irq]; 47 BUG_ON(!entry || !entry->dev); 48 switch (entry->msi_attrib.type) { 49 case PCI_CAP_ID_MSI: 50 if (entry->msi_attrib.maskbit) { 51 int pos; 52 u32 mask_bits; 53 54 pos = (long)entry->mask_base; 55 pci_read_config_dword(entry->dev, pos, &mask_bits); 56 mask_bits &= ~(1); 57 mask_bits |= flag; 58 pci_write_config_dword(entry->dev, pos, mask_bits); 59 } 60 break; 61 case PCI_CAP_ID_MSIX: 62 { 63 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 64 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 65 writel(flag, entry->mask_base + offset); 66 break; 67 } 68 default: 69 BUG(); 70 break; 71 } 72} 73 74void read_msi_msg(unsigned int irq, struct msi_msg *msg) 75{ 76 struct msi_desc *entry = get_irq_data(irq); 77 switch(entry->msi_attrib.type) { 78 case PCI_CAP_ID_MSI: 79 { 80 struct pci_dev *dev = entry->dev; 81 int pos = entry->msi_attrib.pos; 82 u16 data; 83 84 pci_read_config_dword(dev, msi_lower_address_reg(pos), 85 &msg->address_lo); 86 if (entry->msi_attrib.is_64) { 87 pci_read_config_dword(dev, msi_upper_address_reg(pos), 88 &msg->address_hi); 89 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 90 } else { 91 msg->address_hi = 0; 92 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 93 } 94 msg->data = data; 95 break; 96 } 97 case PCI_CAP_ID_MSIX: 98 { 99 void __iomem *base; 100 base = entry->mask_base + 101 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 102 103 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 104 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 105 msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET); 106 break; 107 } 108 default: 109 BUG(); 110 } 111} 112 113void write_msi_msg(unsigned int irq, struct msi_msg *msg) 114{ 115 struct msi_desc *entry = get_irq_data(irq); 116 switch (entry->msi_attrib.type) { 117 case PCI_CAP_ID_MSI: 118 { 119 struct pci_dev *dev = entry->dev; 120 int pos = entry->msi_attrib.pos; 121 122 pci_write_config_dword(dev, msi_lower_address_reg(pos), 123 msg->address_lo); 124 if (entry->msi_attrib.is_64) { 125 pci_write_config_dword(dev, msi_upper_address_reg(pos), 126 msg->address_hi); 127 pci_write_config_word(dev, msi_data_reg(pos, 1), 128 msg->data); 129 } else { 130 pci_write_config_word(dev, msi_data_reg(pos, 0), 131 msg->data); 132 } 133 break; 134 } 135 case PCI_CAP_ID_MSIX: 136 { 137 void __iomem *base; 138 base = entry->mask_base + 139 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 140 141 writel(msg->address_lo, 142 base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 143 writel(msg->address_hi, 144 base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 145 writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET); 146 break; 147 } 148 default: 149 BUG(); 150 } 151} 152 153void mask_msi_irq(unsigned int irq) 154{ 155 msi_set_mask_bit(irq, 1); 156} 157 158void unmask_msi_irq(unsigned int irq) 159{ 160 msi_set_mask_bit(irq, 0); 161} 162 163static int msi_free_irq(struct pci_dev* dev, int irq); 164 165static int msi_init(void) 166{ 167 static int status = -ENOMEM; 168 169 if (!status) 170 return status; 171 172 status = msi_cache_init(); 173 if (status < 0) { 174 pci_msi_enable = 0; 175 printk(KERN_WARNING "PCI: MSI cache init failed\n"); 176 return status; 177 } 178 179 return status; 180} 181 182static struct msi_desc* alloc_msi_entry(void) 183{ 184 struct msi_desc *entry; 185 186 entry = kmem_cache_zalloc(msi_cachep, GFP_KERNEL); 187 if (!entry) 188 return NULL; 189 190 entry->link.tail = entry->link.head = 0; /* single message */ 191 entry->dev = NULL; 192 193 return entry; 194} 195 196static int create_msi_irq(void) 197{ 198 struct msi_desc *entry; 199 int irq; 200 201 entry = alloc_msi_entry(); 202 if (!entry) 203 return -ENOMEM; 204 205 irq = create_irq(); 206 if (irq < 0) { 207 kmem_cache_free(msi_cachep, entry); 208 return -EBUSY; 209 } 210 211 set_irq_data(irq, entry); 212 213 return irq; 214} 215 216static void destroy_msi_irq(unsigned int irq) 217{ 218 struct msi_desc *entry; 219 220 entry = get_irq_data(irq); 221 set_irq_chip(irq, NULL); 222 set_irq_data(irq, NULL); 223 destroy_irq(irq); 224 kmem_cache_free(msi_cachep, entry); 225} 226 227static void enable_msi_mode(struct pci_dev *dev, int pos, int type) 228{ 229 u16 control; 230 231 pci_read_config_word(dev, msi_control_reg(pos), &control); 232 if (type == PCI_CAP_ID_MSI) { 233 /* Set enabled bits to single MSI & enable MSI_enable bit */ 234 msi_enable(control, 1); 235 pci_write_config_word(dev, msi_control_reg(pos), control); 236 dev->msi_enabled = 1; 237 } else { 238 msix_enable(control); 239 pci_write_config_word(dev, msi_control_reg(pos), control); 240 dev->msix_enabled = 1; 241 } 242 243 pci_intx(dev, 0); /* disable intx */ 244} 245 246void disable_msi_mode(struct pci_dev *dev, int pos, int type) 247{ 248 u16 control; 249 250 pci_read_config_word(dev, msi_control_reg(pos), &control); 251 if (type == PCI_CAP_ID_MSI) { 252 /* Set enabled bits to single MSI & enable MSI_enable bit */ 253 msi_disable(control); 254 pci_write_config_word(dev, msi_control_reg(pos), control); 255 dev->msi_enabled = 0; 256 } else { 257 msix_disable(control); 258 pci_write_config_word(dev, msi_control_reg(pos), control); 259 dev->msix_enabled = 0; 260 } 261 262 pci_intx(dev, 1); /* enable intx */ 263} 264 265#ifdef CONFIG_PM 266static int __pci_save_msi_state(struct pci_dev *dev) 267{ 268 int pos, i = 0; 269 u16 control; 270 struct pci_cap_saved_state *save_state; 271 u32 *cap; 272 273 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 274 if (pos <= 0 || dev->no_msi) 275 return 0; 276 277 pci_read_config_word(dev, msi_control_reg(pos), &control); 278 if (!(control & PCI_MSI_FLAGS_ENABLE)) 279 return 0; 280 281 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u32) * 5, 282 GFP_KERNEL); 283 if (!save_state) { 284 printk(KERN_ERR "Out of memory in pci_save_msi_state\n"); 285 return -ENOMEM; 286 } 287 cap = &save_state->data[0]; 288 289 pci_read_config_dword(dev, pos, &cap[i++]); 290 control = cap[0] >> 16; 291 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, &cap[i++]); 292 if (control & PCI_MSI_FLAGS_64BIT) { 293 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, &cap[i++]); 294 pci_read_config_dword(dev, pos + PCI_MSI_DATA_64, &cap[i++]); 295 } else 296 pci_read_config_dword(dev, pos + PCI_MSI_DATA_32, &cap[i++]); 297 if (control & PCI_MSI_FLAGS_MASKBIT) 298 pci_read_config_dword(dev, pos + PCI_MSI_MASK_BIT, &cap[i++]); 299 save_state->cap_nr = PCI_CAP_ID_MSI; 300 pci_add_saved_cap(dev, save_state); 301 return 0; 302} 303 304static void __pci_restore_msi_state(struct pci_dev *dev) 305{ 306 int i = 0, pos; 307 u16 control; 308 struct pci_cap_saved_state *save_state; 309 u32 *cap; 310 311 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSI); 312 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 313 if (!save_state || pos <= 0) 314 return; 315 cap = &save_state->data[0]; 316 317 control = cap[i++] >> 16; 318 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, cap[i++]); 319 if (control & PCI_MSI_FLAGS_64BIT) { 320 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, cap[i++]); 321 pci_write_config_dword(dev, pos + PCI_MSI_DATA_64, cap[i++]); 322 } else 323 pci_write_config_dword(dev, pos + PCI_MSI_DATA_32, cap[i++]); 324 if (control & PCI_MSI_FLAGS_MASKBIT) 325 pci_write_config_dword(dev, pos + PCI_MSI_MASK_BIT, cap[i++]); 326 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 327 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 328 pci_remove_saved_cap(save_state); 329 kfree(save_state); 330} 331 332static int __pci_save_msix_state(struct pci_dev *dev) 333{ 334 int pos; 335 int irq, head, tail = 0; 336 u16 control; 337 struct pci_cap_saved_state *save_state; 338 339 if (!dev->msix_enabled) 340 return 0; 341 342 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 343 if (pos <= 0 || dev->no_msi) 344 return 0; 345 346 /* save the capability */ 347 pci_read_config_word(dev, msi_control_reg(pos), &control); 348 if (!(control & PCI_MSIX_FLAGS_ENABLE)) 349 return 0; 350 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u16), 351 GFP_KERNEL); 352 if (!save_state) { 353 printk(KERN_ERR "Out of memory in pci_save_msix_state\n"); 354 return -ENOMEM; 355 } 356 *((u16 *)&save_state->data[0]) = control; 357 358 /* save the table */ 359 irq = head = dev->first_msi_irq; 360 while (head != tail) { 361 struct msi_desc *entry; 362 363 entry = msi_desc[irq]; 364 read_msi_msg(irq, &entry->msg_save); 365 366 tail = msi_desc[irq]->link.tail; 367 irq = tail; 368 } 369 370 save_state->cap_nr = PCI_CAP_ID_MSIX; 371 pci_add_saved_cap(dev, save_state); 372 return 0; 373} 374 375int pci_save_msi_state(struct pci_dev *dev) 376{ 377 int rc; 378 379 rc = __pci_save_msi_state(dev); 380 if (rc) 381 return rc; 382 383 rc = __pci_save_msix_state(dev); 384 385 return rc; 386} 387 388static void __pci_restore_msix_state(struct pci_dev *dev) 389{ 390 u16 save; 391 int pos; 392 int irq, head, tail = 0; 393 struct msi_desc *entry; 394 struct pci_cap_saved_state *save_state; 395 396 if (!dev->msix_enabled) 397 return; 398 399 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSIX); 400 if (!save_state) 401 return; 402 save = *((u16 *)&save_state->data[0]); 403 pci_remove_saved_cap(save_state); 404 kfree(save_state); 405 406 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 407 if (pos <= 0) 408 return; 409 410 /* route the table */ 411 irq = head = dev->first_msi_irq; 412 while (head != tail) { 413 entry = msi_desc[irq]; 414 write_msi_msg(irq, &entry->msg_save); 415 416 tail = msi_desc[irq]->link.tail; 417 irq = tail; 418 } 419 420 pci_write_config_word(dev, msi_control_reg(pos), save); 421 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 422} 423 424void pci_restore_msi_state(struct pci_dev *dev) 425{ 426 __pci_restore_msi_state(dev); 427 __pci_restore_msix_state(dev); 428} 429#endif /* CONFIG_PM */ 430 431/** 432 * msi_capability_init - configure device's MSI capability structure 433 * @dev: pointer to the pci_dev data structure of MSI device function 434 * 435 * Setup the MSI capability structure of device function with a single 436 * MSI irq, regardless of device function is capable of handling 437 * multiple messages. A return of zero indicates the successful setup 438 * of an entry zero with the new MSI irq or non-zero for otherwise. 439 **/ 440static int msi_capability_init(struct pci_dev *dev) 441{ 442 int status; 443 struct msi_desc *entry; 444 int pos, irq; 445 u16 control; 446 447 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 448 pci_read_config_word(dev, msi_control_reg(pos), &control); 449 /* MSI Entry Initialization */ 450 irq = create_msi_irq(); 451 if (irq < 0) 452 return irq; 453 454 entry = get_irq_data(irq); 455 entry->link.head = irq; 456 entry->link.tail = irq; 457 entry->msi_attrib.type = PCI_CAP_ID_MSI; 458 entry->msi_attrib.is_64 = is_64bit_address(control); 459 entry->msi_attrib.entry_nr = 0; 460 entry->msi_attrib.maskbit = is_mask_bit_support(control); 461 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ 462 entry->msi_attrib.pos = pos; 463 if (is_mask_bit_support(control)) { 464 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, 465 is_64bit_address(control)); 466 } 467 entry->dev = dev; 468 if (entry->msi_attrib.maskbit) { 469 unsigned int maskbits, temp; 470 /* All MSIs are unmasked by default, Mask them all */ 471 pci_read_config_dword(dev, 472 msi_mask_bits_reg(pos, is_64bit_address(control)), 473 &maskbits); 474 temp = (1 << multi_msi_capable(control)); 475 temp = ((temp - 1) & ~temp); 476 maskbits |= temp; 477 pci_write_config_dword(dev, 478 msi_mask_bits_reg(pos, is_64bit_address(control)), 479 maskbits); 480 } 481 /* Configure MSI capability structure */ 482 status = arch_setup_msi_irq(irq, dev); 483 if (status < 0) { 484 destroy_msi_irq(irq); 485 return status; 486 } 487 488 dev->first_msi_irq = irq; 489 msi_desc[irq] = entry; 490 /* Set MSI enabled bits */ 491 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 492 493 dev->irq = irq; 494 return 0; 495} 496 497/** 498 * msix_capability_init - configure device's MSI-X capability 499 * @dev: pointer to the pci_dev data structure of MSI-X device function 500 * @entries: pointer to an array of struct msix_entry entries 501 * @nvec: number of @entries 502 * 503 * Setup the MSI-X capability structure of device function with a 504 * single MSI-X irq. A return of zero indicates the successful setup of 505 * requested MSI-X entries with allocated irqs or non-zero for otherwise. 506 **/ 507static int msix_capability_init(struct pci_dev *dev, 508 struct msix_entry *entries, int nvec) 509{ 510 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL; 511 int status; 512 int irq, pos, i, j, nr_entries, temp = 0; 513 unsigned long phys_addr; 514 u32 table_offset; 515 u16 control; 516 u8 bir; 517 void __iomem *base; 518 519 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 520 /* Request & Map MSI-X table region */ 521 pci_read_config_word(dev, msi_control_reg(pos), &control); 522 nr_entries = multi_msix_capable(control); 523 524 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 525 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 526 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 527 phys_addr = pci_resource_start (dev, bir) + table_offset; 528 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 529 if (base == NULL) 530 return -ENOMEM; 531 532 /* MSI-X Table Initialization */ 533 for (i = 0; i < nvec; i++) { 534 irq = create_msi_irq(); 535 if (irq < 0) 536 break; 537 538 entry = get_irq_data(irq); 539 j = entries[i].entry; 540 entries[i].vector = irq; 541 entry->msi_attrib.type = PCI_CAP_ID_MSIX; 542 entry->msi_attrib.is_64 = 1; 543 entry->msi_attrib.entry_nr = j; 544 entry->msi_attrib.maskbit = 1; 545 entry->msi_attrib.default_irq = dev->irq; 546 entry->msi_attrib.pos = pos; 547 entry->dev = dev; 548 entry->mask_base = base; 549 if (!head) { 550 entry->link.head = irq; 551 entry->link.tail = irq; 552 head = entry; 553 } else { 554 entry->link.head = temp; 555 entry->link.tail = tail->link.tail; 556 tail->link.tail = irq; 557 head->link.head = irq; 558 } 559 temp = irq; 560 tail = entry; 561 /* Configure MSI-X capability structure */ 562 status = arch_setup_msi_irq(irq, dev); 563 if (status < 0) { 564 destroy_msi_irq(irq); 565 break; 566 } 567 568 msi_desc[irq] = entry; 569 } 570 if (i != nvec) { 571 int avail = i - 1; 572 i--; 573 for (; i >= 0; i--) { 574 irq = (entries + i)->vector; 575 msi_free_irq(dev, irq); 576 (entries + i)->vector = 0; 577 } 578 /* If we had some success report the number of irqs 579 * we succeeded in setting up. 580 */ 581 if (avail <= 0) 582 avail = -EBUSY; 583 return avail; 584 } 585 dev->first_msi_irq = entries[0].vector; 586 /* Set MSI-X enabled bits */ 587 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 588 589 return 0; 590} 591 592/** 593 * pci_msi_supported - check whether MSI may be enabled on device 594 * @dev: pointer to the pci_dev data structure of MSI device function 595 * 596 * Look at global flags, the device itself, and its parent busses 597 * to return 0 if MSI are supported for the device. 598 **/ 599static 600int pci_msi_supported(struct pci_dev * dev) 601{ 602 struct pci_bus *bus; 603 604 /* MSI must be globally enabled and supported by the device */ 605 if (!pci_msi_enable || !dev || dev->no_msi) 606 return -EINVAL; 607 608 /* Any bridge which does NOT route MSI transactions from it's 609 * secondary bus to it's primary bus must set NO_MSI flag on 610 * the secondary pci_bus. 611 * We expect only arch-specific PCI host bus controller driver 612 * or quirks for specific PCI bridges to be setting NO_MSI. 613 */ 614 for (bus = dev->bus; bus; bus = bus->parent) 615 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 616 return -EINVAL; 617 618 return 0; 619} 620 621/** 622 * pci_enable_msi - configure device's MSI capability structure 623 * @dev: pointer to the pci_dev data structure of MSI device function 624 * 625 * Setup the MSI capability structure of device function with 626 * a single MSI irq upon its software driver call to request for 627 * MSI mode enabled on its hardware device function. A return of zero 628 * indicates the successful setup of an entry zero with the new MSI 629 * irq or non-zero for otherwise. 630 **/ 631int pci_enable_msi(struct pci_dev* dev) 632{ 633 int pos, status; 634 635 if (pci_msi_supported(dev) < 0) 636 return -EINVAL; 637 638 status = msi_init(); 639 if (status < 0) 640 return status; 641 642 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 643 if (!pos) 644 return -EINVAL; 645 646 WARN_ON(!!dev->msi_enabled); 647 648 /* Check whether driver already requested for MSI-X irqs */ 649 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 650 if (pos > 0 && dev->msix_enabled) { 651 printk(KERN_INFO "PCI: %s: Can't enable MSI. " 652 "Device already has MSI-X enabled\n", 653 pci_name(dev)); 654 return -EINVAL; 655 } 656 status = msi_capability_init(dev); 657 return status; 658} 659 660void pci_disable_msi(struct pci_dev* dev) 661{ 662 struct msi_desc *entry; 663 int pos, default_irq; 664 u16 control; 665 666 if (!pci_msi_enable) 667 return; 668 if (!dev) 669 return; 670 671 if (!dev->msi_enabled) 672 return; 673 674 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 675 if (!pos) 676 return; 677 678 pci_read_config_word(dev, msi_control_reg(pos), &control); 679 if (!(control & PCI_MSI_FLAGS_ENABLE)) 680 return; 681 682 683 disable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 684 685 entry = msi_desc[dev->first_msi_irq]; 686 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { 687 return; 688 } 689 if (irq_has_action(dev->first_msi_irq)) { 690 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without " 691 "free_irq() on MSI irq %d\n", 692 pci_name(dev), dev->first_msi_irq); 693 BUG_ON(irq_has_action(dev->first_msi_irq)); 694 } else { 695 default_irq = entry->msi_attrib.default_irq; 696 msi_free_irq(dev, dev->first_msi_irq); 697 698 /* Restore dev->irq to its default pin-assertion irq */ 699 dev->irq = default_irq; 700 } 701 dev->first_msi_irq = 0; 702} 703 704static int msi_free_irq(struct pci_dev* dev, int irq) 705{ 706 struct msi_desc *entry; 707 int head, entry_nr, type; 708 void __iomem *base; 709 710 arch_teardown_msi_irq(irq); 711 712 entry = msi_desc[irq]; 713 if (!entry || entry->dev != dev) { 714 return -EINVAL; 715 } 716 type = entry->msi_attrib.type; 717 entry_nr = entry->msi_attrib.entry_nr; 718 head = entry->link.head; 719 base = entry->mask_base; 720 msi_desc[entry->link.head]->link.tail = entry->link.tail; 721 msi_desc[entry->link.tail]->link.head = entry->link.head; 722 entry->dev = NULL; 723 msi_desc[irq] = NULL; 724 725 destroy_msi_irq(irq); 726 727 if (type == PCI_CAP_ID_MSIX) { 728 writel(1, base + entry_nr * PCI_MSIX_ENTRY_SIZE + 729 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); 730 731 if (head == irq) 732 iounmap(base); 733 } 734 735 return 0; 736} 737 738/** 739 * pci_enable_msix - configure device's MSI-X capability structure 740 * @dev: pointer to the pci_dev data structure of MSI-X device function 741 * @entries: pointer to an array of MSI-X entries 742 * @nvec: number of MSI-X irqs requested for allocation by device driver 743 * 744 * Setup the MSI-X capability structure of device function with the number 745 * of requested irqs upon its software driver call to request for 746 * MSI-X mode enabled on its hardware device function. A return of zero 747 * indicates the successful configuration of MSI-X capability structure 748 * with new allocated MSI-X irqs. A return of < 0 indicates a failure. 749 * Or a return of > 0 indicates that driver request is exceeding the number 750 * of irqs available. Driver should use the returned value to re-send 751 * its request. 752 **/ 753int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) 754{ 755 int status, pos, nr_entries; 756 int i, j; 757 u16 control; 758 759 if (!entries || pci_msi_supported(dev) < 0) 760 return -EINVAL; 761 762 status = msi_init(); 763 if (status < 0) 764 return status; 765 766 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 767 if (!pos) 768 return -EINVAL; 769 770 pci_read_config_word(dev, msi_control_reg(pos), &control); 771 nr_entries = multi_msix_capable(control); 772 if (nvec > nr_entries) 773 return -EINVAL; 774 775 /* Check for any invalid entries */ 776 for (i = 0; i < nvec; i++) { 777 if (entries[i].entry >= nr_entries) 778 return -EINVAL; /* invalid entry */ 779 for (j = i + 1; j < nvec; j++) { 780 if (entries[i].entry == entries[j].entry) 781 return -EINVAL; /* duplicate entry */ 782 } 783 } 784 WARN_ON(!!dev->msix_enabled); 785 786 /* Check whether driver already requested for MSI irq */ 787 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 && 788 dev->msi_enabled) { 789 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " 790 "Device already has an MSI irq assigned\n", 791 pci_name(dev)); 792 return -EINVAL; 793 } 794 status = msix_capability_init(dev, entries, nvec); 795 return status; 796} 797 798void pci_disable_msix(struct pci_dev* dev) 799{ 800 int irq, head, tail = 0, warning = 0; 801 int pos; 802 u16 control; 803 804 if (!pci_msi_enable) 805 return; 806 if (!dev) 807 return; 808 809 if (!dev->msix_enabled) 810 return; 811 812 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 813 if (!pos) 814 return; 815 816 pci_read_config_word(dev, msi_control_reg(pos), &control); 817 if (!(control & PCI_MSIX_FLAGS_ENABLE)) 818 return; 819 820 disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 821 822 irq = head = dev->first_msi_irq; 823 while (head != tail) { 824 tail = msi_desc[irq]->link.tail; 825 if (irq_has_action(irq)) 826 warning = 1; 827 else if (irq != head) /* Release MSI-X irq */ 828 msi_free_irq(dev, irq); 829 irq = tail; 830 } 831 msi_free_irq(dev, irq); 832 if (warning) { 833 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without " 834 "free_irq() on all MSI-X irqs\n", 835 pci_name(dev)); 836 BUG_ON(warning > 0); 837 } 838 dev->first_msi_irq = 0; 839} 840 841/** 842 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state 843 * @dev: pointer to the pci_dev data structure of MSI(X) device function 844 * 845 * Being called during hotplug remove, from which the device function 846 * is hot-removed. All previous assigned MSI/MSI-X irqs, if 847 * allocated for this device function, are reclaimed to unused state, 848 * which may be used later on. 849 **/ 850void msi_remove_pci_irq_vectors(struct pci_dev* dev) 851{ 852 if (!pci_msi_enable || !dev) 853 return; 854 855 if (dev->msi_enabled) { 856 if (irq_has_action(dev->first_msi_irq)) { 857 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 858 "called without free_irq() on MSI irq %d\n", 859 pci_name(dev), dev->first_msi_irq); 860 BUG_ON(irq_has_action(dev->first_msi_irq)); 861 } else /* Release MSI irq assigned to this device */ 862 msi_free_irq(dev, dev->first_msi_irq); 863 } 864 if (dev->msix_enabled) { 865 int irq, head, tail = 0, warning = 0; 866 void __iomem *base = NULL; 867 868 irq = head = dev->first_msi_irq; 869 while (head != tail) { 870 tail = msi_desc[irq]->link.tail; 871 base = msi_desc[irq]->mask_base; 872 if (irq_has_action(irq)) 873 warning = 1; 874 else if (irq != head) /* Release MSI-X irq */ 875 msi_free_irq(dev, irq); 876 irq = tail; 877 } 878 msi_free_irq(dev, irq); 879 if (warning) { 880 iounmap(base); 881 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 882 "called without free_irq() on all MSI-X irqs\n", 883 pci_name(dev)); 884 BUG_ON(warning > 0); 885 } 886 } 887} 888 889void pci_no_msi(void) 890{ 891 pci_msi_enable = 0; 892} 893 894EXPORT_SYMBOL(pci_enable_msi); 895EXPORT_SYMBOL(pci_disable_msi); 896EXPORT_SYMBOL(pci_enable_msix); 897EXPORT_SYMBOL(pci_disable_msix); 898