msi.c revision 128bc5fced238752d01b5169077f2ec624b3d59b
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/err.h> 10#include <linux/mm.h> 11#include <linux/irq.h> 12#include <linux/interrupt.h> 13#include <linux/init.h> 14#include <linux/ioport.h> 15#include <linux/smp_lock.h> 16#include <linux/pci.h> 17#include <linux/proc_fs.h> 18#include <linux/msi.h> 19 20#include <asm/errno.h> 21#include <asm/io.h> 22#include <asm/smp.h> 23 24#include "pci.h" 25#include "msi.h" 26 27static int pci_msi_enable = 1; 28 29static void msi_set_enable(struct pci_dev *dev, int enable) 30{ 31 int pos; 32 u16 control; 33 34 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 35 if (pos) { 36 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 37 control &= ~PCI_MSI_FLAGS_ENABLE; 38 if (enable) 39 control |= PCI_MSI_FLAGS_ENABLE; 40 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 41 } 42} 43 44static void msix_set_enable(struct pci_dev *dev, int enable) 45{ 46 int pos; 47 u16 control; 48 49 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 50 if (pos) { 51 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 52 control &= ~PCI_MSIX_FLAGS_ENABLE; 53 if (enable) 54 control |= PCI_MSIX_FLAGS_ENABLE; 55 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 56 } 57} 58 59static void msix_flush_writes(unsigned int irq) 60{ 61 struct msi_desc *entry; 62 63 entry = get_irq_msi(irq); 64 BUG_ON(!entry || !entry->dev); 65 switch (entry->msi_attrib.type) { 66 case PCI_CAP_ID_MSI: 67 /* nothing to do */ 68 break; 69 case PCI_CAP_ID_MSIX: 70 { 71 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 72 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 73 readl(entry->mask_base + offset); 74 break; 75 } 76 default: 77 BUG(); 78 break; 79 } 80} 81 82static void msi_set_mask_bit(unsigned int irq, int flag) 83{ 84 struct msi_desc *entry; 85 86 entry = get_irq_msi(irq); 87 BUG_ON(!entry || !entry->dev); 88 switch (entry->msi_attrib.type) { 89 case PCI_CAP_ID_MSI: 90 if (entry->msi_attrib.maskbit) { 91 int pos; 92 u32 mask_bits; 93 94 pos = (long)entry->mask_base; 95 pci_read_config_dword(entry->dev, pos, &mask_bits); 96 mask_bits &= ~(1); 97 mask_bits |= flag; 98 pci_write_config_dword(entry->dev, pos, mask_bits); 99 } else { 100 msi_set_enable(entry->dev, !flag); 101 } 102 break; 103 case PCI_CAP_ID_MSIX: 104 { 105 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 106 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 107 writel(flag, entry->mask_base + offset); 108 readl(entry->mask_base + offset); 109 break; 110 } 111 default: 112 BUG(); 113 break; 114 } 115 entry->msi_attrib.masked = !!flag; 116} 117 118void read_msi_msg(unsigned int irq, struct msi_msg *msg) 119{ 120 struct msi_desc *entry = get_irq_msi(irq); 121 switch(entry->msi_attrib.type) { 122 case PCI_CAP_ID_MSI: 123 { 124 struct pci_dev *dev = entry->dev; 125 int pos = entry->msi_attrib.pos; 126 u16 data; 127 128 pci_read_config_dword(dev, msi_lower_address_reg(pos), 129 &msg->address_lo); 130 if (entry->msi_attrib.is_64) { 131 pci_read_config_dword(dev, msi_upper_address_reg(pos), 132 &msg->address_hi); 133 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 134 } else { 135 msg->address_hi = 0; 136 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 137 } 138 msg->data = data; 139 break; 140 } 141 case PCI_CAP_ID_MSIX: 142 { 143 void __iomem *base; 144 base = entry->mask_base + 145 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 146 147 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 148 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 149 msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET); 150 break; 151 } 152 default: 153 BUG(); 154 } 155} 156 157void write_msi_msg(unsigned int irq, struct msi_msg *msg) 158{ 159 struct msi_desc *entry = get_irq_msi(irq); 160 switch (entry->msi_attrib.type) { 161 case PCI_CAP_ID_MSI: 162 { 163 struct pci_dev *dev = entry->dev; 164 int pos = entry->msi_attrib.pos; 165 166 pci_write_config_dword(dev, msi_lower_address_reg(pos), 167 msg->address_lo); 168 if (entry->msi_attrib.is_64) { 169 pci_write_config_dword(dev, msi_upper_address_reg(pos), 170 msg->address_hi); 171 pci_write_config_word(dev, msi_data_reg(pos, 1), 172 msg->data); 173 } else { 174 pci_write_config_word(dev, msi_data_reg(pos, 0), 175 msg->data); 176 } 177 break; 178 } 179 case PCI_CAP_ID_MSIX: 180 { 181 void __iomem *base; 182 base = entry->mask_base + 183 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 184 185 writel(msg->address_lo, 186 base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 187 writel(msg->address_hi, 188 base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 189 writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET); 190 break; 191 } 192 default: 193 BUG(); 194 } 195 entry->msg = *msg; 196} 197 198void mask_msi_irq(unsigned int irq) 199{ 200 msi_set_mask_bit(irq, 1); 201 msix_flush_writes(irq); 202} 203 204void unmask_msi_irq(unsigned int irq) 205{ 206 msi_set_mask_bit(irq, 0); 207 msix_flush_writes(irq); 208} 209 210static int msi_free_irq(struct pci_dev* dev, int irq); 211 212 213static struct msi_desc* alloc_msi_entry(void) 214{ 215 struct msi_desc *entry; 216 217 entry = kzalloc(sizeof(struct msi_desc), GFP_KERNEL); 218 if (!entry) 219 return NULL; 220 221 entry->link.tail = entry->link.head = 0; /* single message */ 222 entry->dev = NULL; 223 224 return entry; 225} 226 227#ifdef CONFIG_PM 228static void __pci_restore_msi_state(struct pci_dev *dev) 229{ 230 int pos; 231 u16 control; 232 struct msi_desc *entry; 233 234 if (!dev->msi_enabled) 235 return; 236 237 entry = get_irq_msi(dev->irq); 238 pos = entry->msi_attrib.pos; 239 240 pci_intx(dev, 0); /* disable intx */ 241 msi_set_enable(dev, 0); 242 write_msi_msg(dev->irq, &entry->msg); 243 if (entry->msi_attrib.maskbit) 244 msi_set_mask_bit(dev->irq, entry->msi_attrib.masked); 245 246 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 247 control &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE); 248 if (entry->msi_attrib.maskbit || !entry->msi_attrib.masked) 249 control |= PCI_MSI_FLAGS_ENABLE; 250 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 251} 252 253static void __pci_restore_msix_state(struct pci_dev *dev) 254{ 255 int pos; 256 int irq, head, tail = 0; 257 struct msi_desc *entry; 258 u16 control; 259 260 if (!dev->msix_enabled) 261 return; 262 263 /* route the table */ 264 pci_intx(dev, 0); /* disable intx */ 265 msix_set_enable(dev, 0); 266 irq = head = dev->first_msi_irq; 267 entry = get_irq_msi(irq); 268 pos = entry->msi_attrib.pos; 269 while (head != tail) { 270 entry = get_irq_msi(irq); 271 write_msi_msg(irq, &entry->msg); 272 msi_set_mask_bit(irq, entry->msi_attrib.masked); 273 274 tail = entry->link.tail; 275 irq = tail; 276 } 277 278 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 279 control &= ~PCI_MSIX_FLAGS_MASKALL; 280 control |= PCI_MSIX_FLAGS_ENABLE; 281 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 282} 283 284void pci_restore_msi_state(struct pci_dev *dev) 285{ 286 __pci_restore_msi_state(dev); 287 __pci_restore_msix_state(dev); 288} 289#endif /* CONFIG_PM */ 290 291/** 292 * msi_capability_init - configure device's MSI capability structure 293 * @dev: pointer to the pci_dev data structure of MSI device function 294 * 295 * Setup the MSI capability structure of device function with a single 296 * MSI irq, regardless of device function is capable of handling 297 * multiple messages. A return of zero indicates the successful setup 298 * of an entry zero with the new MSI irq or non-zero for otherwise. 299 **/ 300static int msi_capability_init(struct pci_dev *dev) 301{ 302 struct msi_desc *entry; 303 int pos, irq; 304 u16 control; 305 306 msi_set_enable(dev, 0); /* Ensure msi is disabled as I set it up */ 307 308 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 309 pci_read_config_word(dev, msi_control_reg(pos), &control); 310 /* MSI Entry Initialization */ 311 entry = alloc_msi_entry(); 312 if (!entry) 313 return -ENOMEM; 314 315 entry->msi_attrib.type = PCI_CAP_ID_MSI; 316 entry->msi_attrib.is_64 = is_64bit_address(control); 317 entry->msi_attrib.entry_nr = 0; 318 entry->msi_attrib.maskbit = is_mask_bit_support(control); 319 entry->msi_attrib.masked = 1; 320 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ 321 entry->msi_attrib.pos = pos; 322 if (is_mask_bit_support(control)) { 323 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, 324 is_64bit_address(control)); 325 } 326 entry->dev = dev; 327 if (entry->msi_attrib.maskbit) { 328 unsigned int maskbits, temp; 329 /* All MSIs are unmasked by default, Mask them all */ 330 pci_read_config_dword(dev, 331 msi_mask_bits_reg(pos, is_64bit_address(control)), 332 &maskbits); 333 temp = (1 << multi_msi_capable(control)); 334 temp = ((temp - 1) & ~temp); 335 maskbits |= temp; 336 pci_write_config_dword(dev, 337 msi_mask_bits_reg(pos, is_64bit_address(control)), 338 maskbits); 339 } 340 /* Configure MSI capability structure */ 341 irq = arch_setup_msi_irq(dev, entry); 342 if (irq < 0) { 343 kfree(entry); 344 return irq; 345 } 346 entry->link.head = irq; 347 entry->link.tail = irq; 348 dev->first_msi_irq = irq; 349 set_irq_msi(irq, entry); 350 351 /* Set MSI enabled bits */ 352 pci_intx(dev, 0); /* disable intx */ 353 msi_set_enable(dev, 1); 354 dev->msi_enabled = 1; 355 356 dev->irq = irq; 357 return 0; 358} 359 360/** 361 * msix_capability_init - configure device's MSI-X capability 362 * @dev: pointer to the pci_dev data structure of MSI-X device function 363 * @entries: pointer to an array of struct msix_entry entries 364 * @nvec: number of @entries 365 * 366 * Setup the MSI-X capability structure of device function with a 367 * single MSI-X irq. A return of zero indicates the successful setup of 368 * requested MSI-X entries with allocated irqs or non-zero for otherwise. 369 **/ 370static int msix_capability_init(struct pci_dev *dev, 371 struct msix_entry *entries, int nvec) 372{ 373 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL; 374 int irq, pos, i, j, nr_entries, temp = 0; 375 unsigned long phys_addr; 376 u32 table_offset; 377 u16 control; 378 u8 bir; 379 void __iomem *base; 380 381 msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */ 382 383 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 384 /* Request & Map MSI-X table region */ 385 pci_read_config_word(dev, msi_control_reg(pos), &control); 386 nr_entries = multi_msix_capable(control); 387 388 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 389 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 390 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 391 phys_addr = pci_resource_start (dev, bir) + table_offset; 392 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 393 if (base == NULL) 394 return -ENOMEM; 395 396 /* MSI-X Table Initialization */ 397 for (i = 0; i < nvec; i++) { 398 entry = alloc_msi_entry(); 399 if (!entry) 400 break; 401 402 j = entries[i].entry; 403 entry->msi_attrib.type = PCI_CAP_ID_MSIX; 404 entry->msi_attrib.is_64 = 1; 405 entry->msi_attrib.entry_nr = j; 406 entry->msi_attrib.maskbit = 1; 407 entry->msi_attrib.masked = 1; 408 entry->msi_attrib.default_irq = dev->irq; 409 entry->msi_attrib.pos = pos; 410 entry->dev = dev; 411 entry->mask_base = base; 412 413 /* Configure MSI-X capability structure */ 414 irq = arch_setup_msi_irq(dev, entry); 415 if (irq < 0) { 416 kfree(entry); 417 break; 418 } 419 entries[i].vector = irq; 420 if (!head) { 421 entry->link.head = irq; 422 entry->link.tail = irq; 423 head = entry; 424 } else { 425 entry->link.head = temp; 426 entry->link.tail = tail->link.tail; 427 tail->link.tail = irq; 428 head->link.head = irq; 429 } 430 temp = irq; 431 tail = entry; 432 433 set_irq_msi(irq, entry); 434 } 435 if (i != nvec) { 436 int avail = i - 1; 437 i--; 438 for (; i >= 0; i--) { 439 irq = (entries + i)->vector; 440 msi_free_irq(dev, irq); 441 (entries + i)->vector = 0; 442 } 443 /* If we had some success report the number of irqs 444 * we succeeded in setting up. 445 */ 446 if (avail <= 0) 447 avail = -EBUSY; 448 return avail; 449 } 450 dev->first_msi_irq = entries[0].vector; 451 /* Set MSI-X enabled bits */ 452 pci_intx(dev, 0); /* disable intx */ 453 msix_set_enable(dev, 1); 454 dev->msix_enabled = 1; 455 456 return 0; 457} 458 459/** 460 * pci_msi_supported - check whether MSI may be enabled on device 461 * @dev: pointer to the pci_dev data structure of MSI device function 462 * @type: are we checking for MSI or MSI-X ? 463 * 464 * Look at global flags, the device itself, and its parent busses 465 * to return 0 if MSI are supported for the device. 466 **/ 467static 468int pci_msi_supported(struct pci_dev * dev, int type) 469{ 470 struct pci_bus *bus; 471 472 /* MSI must be globally enabled and supported by the device */ 473 if (!pci_msi_enable || !dev || dev->no_msi) 474 return -EINVAL; 475 476 /* Any bridge which does NOT route MSI transactions from it's 477 * secondary bus to it's primary bus must set NO_MSI flag on 478 * the secondary pci_bus. 479 * We expect only arch-specific PCI host bus controller driver 480 * or quirks for specific PCI bridges to be setting NO_MSI. 481 */ 482 for (bus = dev->bus; bus; bus = bus->parent) 483 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 484 return -EINVAL; 485 486 if (!pci_find_capability(dev, type)) 487 return -EINVAL; 488 489 return 0; 490} 491 492/** 493 * pci_enable_msi - configure device's MSI capability structure 494 * @dev: pointer to the pci_dev data structure of MSI device function 495 * 496 * Setup the MSI capability structure of device function with 497 * a single MSI irq upon its software driver call to request for 498 * MSI mode enabled on its hardware device function. A return of zero 499 * indicates the successful setup of an entry zero with the new MSI 500 * irq or non-zero for otherwise. 501 **/ 502int pci_enable_msi(struct pci_dev* dev) 503{ 504 int status; 505 506 if (pci_msi_supported(dev, PCI_CAP_ID_MSI)) 507 return -EINVAL; 508 509 WARN_ON(!!dev->msi_enabled); 510 511 /* Check whether driver already requested for MSI-X irqs */ 512 if (dev->msix_enabled) { 513 printk(KERN_INFO "PCI: %s: Can't enable MSI. " 514 "Device already has MSI-X enabled\n", 515 pci_name(dev)); 516 return -EINVAL; 517 } 518 status = msi_capability_init(dev); 519 return status; 520} 521EXPORT_SYMBOL(pci_enable_msi); 522 523void pci_disable_msi(struct pci_dev* dev) 524{ 525 struct msi_desc *entry; 526 int default_irq; 527 528 if (!pci_msi_enable || !dev || !dev->msi_enabled) 529 return; 530 531 msi_set_enable(dev, 0); 532 pci_intx(dev, 1); /* enable intx */ 533 dev->msi_enabled = 0; 534 535 entry = get_irq_msi(dev->first_msi_irq); 536 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { 537 return; 538 } 539 540 default_irq = entry->msi_attrib.default_irq; 541 msi_free_irq(dev, dev->first_msi_irq); 542 543 /* Restore dev->irq to its default pin-assertion irq */ 544 dev->irq = default_irq; 545 546 dev->first_msi_irq = 0; 547} 548EXPORT_SYMBOL(pci_disable_msi); 549 550static int msi_free_irq(struct pci_dev* dev, int irq) 551{ 552 struct msi_desc *entry; 553 int head, entry_nr, type; 554 void __iomem *base; 555 556 BUG_ON(irq_has_action(irq)); 557 558 entry = get_irq_msi(irq); 559 if (!entry || entry->dev != dev) { 560 return -EINVAL; 561 } 562 type = entry->msi_attrib.type; 563 entry_nr = entry->msi_attrib.entry_nr; 564 head = entry->link.head; 565 base = entry->mask_base; 566 get_irq_msi(entry->link.head)->link.tail = entry->link.tail; 567 get_irq_msi(entry->link.tail)->link.head = entry->link.head; 568 569 arch_teardown_msi_irq(irq); 570 kfree(entry); 571 572 if (type == PCI_CAP_ID_MSIX) { 573 writel(1, base + entry_nr * PCI_MSIX_ENTRY_SIZE + 574 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); 575 576 if (head == irq) 577 iounmap(base); 578 } 579 580 return 0; 581} 582 583/** 584 * pci_enable_msix - configure device's MSI-X capability structure 585 * @dev: pointer to the pci_dev data structure of MSI-X device function 586 * @entries: pointer to an array of MSI-X entries 587 * @nvec: number of MSI-X irqs requested for allocation by device driver 588 * 589 * Setup the MSI-X capability structure of device function with the number 590 * of requested irqs upon its software driver call to request for 591 * MSI-X mode enabled on its hardware device function. A return of zero 592 * indicates the successful configuration of MSI-X capability structure 593 * with new allocated MSI-X irqs. A return of < 0 indicates a failure. 594 * Or a return of > 0 indicates that driver request is exceeding the number 595 * of irqs available. Driver should use the returned value to re-send 596 * its request. 597 **/ 598int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) 599{ 600 int status, pos, nr_entries; 601 int i, j; 602 u16 control; 603 604 if (!entries || pci_msi_supported(dev, PCI_CAP_ID_MSIX)) 605 return -EINVAL; 606 607 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 608 pci_read_config_word(dev, msi_control_reg(pos), &control); 609 nr_entries = multi_msix_capable(control); 610 if (nvec > nr_entries) 611 return -EINVAL; 612 613 /* Check for any invalid entries */ 614 for (i = 0; i < nvec; i++) { 615 if (entries[i].entry >= nr_entries) 616 return -EINVAL; /* invalid entry */ 617 for (j = i + 1; j < nvec; j++) { 618 if (entries[i].entry == entries[j].entry) 619 return -EINVAL; /* duplicate entry */ 620 } 621 } 622 WARN_ON(!!dev->msix_enabled); 623 624 /* Check whether driver already requested for MSI irq */ 625 if (dev->msi_enabled) { 626 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " 627 "Device already has an MSI irq assigned\n", 628 pci_name(dev)); 629 return -EINVAL; 630 } 631 status = msix_capability_init(dev, entries, nvec); 632 return status; 633} 634EXPORT_SYMBOL(pci_enable_msix); 635 636static void msix_free_all_irqs(struct pci_dev *dev) 637{ 638 int irq, head, tail = 0; 639 640 irq = head = dev->first_msi_irq; 641 while (head != tail) { 642 tail = get_irq_msi(irq)->link.tail; 643 644 if (irq != head) 645 msi_free_irq(dev, irq); 646 irq = tail; 647 } 648 msi_free_irq(dev, irq); 649 dev->first_msi_irq = 0; 650} 651 652void pci_disable_msix(struct pci_dev* dev) 653{ 654 if (!pci_msi_enable || !dev || !dev->msix_enabled) 655 return; 656 657 msix_set_enable(dev, 0); 658 pci_intx(dev, 1); /* enable intx */ 659 dev->msix_enabled = 0; 660 661 msix_free_all_irqs(dev); 662} 663EXPORT_SYMBOL(pci_disable_msix); 664 665/** 666 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state 667 * @dev: pointer to the pci_dev data structure of MSI(X) device function 668 * 669 * Being called during hotplug remove, from which the device function 670 * is hot-removed. All previous assigned MSI/MSI-X irqs, if 671 * allocated for this device function, are reclaimed to unused state, 672 * which may be used later on. 673 **/ 674void msi_remove_pci_irq_vectors(struct pci_dev* dev) 675{ 676 if (!pci_msi_enable || !dev) 677 return; 678 679 if (dev->msi_enabled) 680 msi_free_irq(dev, dev->first_msi_irq); 681 682 if (dev->msix_enabled) 683 msix_free_all_irqs(dev); 684} 685 686void pci_no_msi(void) 687{ 688 pci_msi_enable = 0; 689} 690