msi.c revision 4aa9bc955d61fdf03b5f9cee67db188fe1ffa8b7
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/err.h> 10#include <linux/mm.h> 11#include <linux/irq.h> 12#include <linux/interrupt.h> 13#include <linux/init.h> 14#include <linux/ioport.h> 15#include <linux/smp_lock.h> 16#include <linux/pci.h> 17#include <linux/proc_fs.h> 18#include <linux/msi.h> 19 20#include <asm/errno.h> 21#include <asm/io.h> 22#include <asm/smp.h> 23 24#include "pci.h" 25#include "msi.h" 26 27static int pci_msi_enable = 1; 28 29static void msi_set_enable(struct pci_dev *dev, int enable) 30{ 31 int pos; 32 u16 control; 33 34 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 35 if (pos) { 36 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 37 control &= ~PCI_MSI_FLAGS_ENABLE; 38 if (enable) 39 control |= PCI_MSI_FLAGS_ENABLE; 40 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 41 } 42} 43 44static void msix_set_enable(struct pci_dev *dev, int enable) 45{ 46 int pos; 47 u16 control; 48 49 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 50 if (pos) { 51 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 52 control &= ~PCI_MSIX_FLAGS_ENABLE; 53 if (enable) 54 control |= PCI_MSIX_FLAGS_ENABLE; 55 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 56 } 57} 58 59static void msix_flush_writes(unsigned int irq) 60{ 61 struct msi_desc *entry; 62 63 entry = get_irq_msi(irq); 64 BUG_ON(!entry || !entry->dev); 65 switch (entry->msi_attrib.type) { 66 case PCI_CAP_ID_MSI: 67 /* nothing to do */ 68 break; 69 case PCI_CAP_ID_MSIX: 70 { 71 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 72 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 73 readl(entry->mask_base + offset); 74 break; 75 } 76 default: 77 BUG(); 78 break; 79 } 80} 81 82static void msi_set_mask_bit(unsigned int irq, int flag) 83{ 84 struct msi_desc *entry; 85 86 entry = get_irq_msi(irq); 87 BUG_ON(!entry || !entry->dev); 88 switch (entry->msi_attrib.type) { 89 case PCI_CAP_ID_MSI: 90 if (entry->msi_attrib.maskbit) { 91 int pos; 92 u32 mask_bits; 93 94 pos = (long)entry->mask_base; 95 pci_read_config_dword(entry->dev, pos, &mask_bits); 96 mask_bits &= ~(1); 97 mask_bits |= flag; 98 pci_write_config_dword(entry->dev, pos, mask_bits); 99 } else { 100 msi_set_enable(entry->dev, !flag); 101 } 102 break; 103 case PCI_CAP_ID_MSIX: 104 { 105 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 106 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 107 writel(flag, entry->mask_base + offset); 108 readl(entry->mask_base + offset); 109 break; 110 } 111 default: 112 BUG(); 113 break; 114 } 115 entry->msi_attrib.masked = !!flag; 116} 117 118void read_msi_msg(unsigned int irq, struct msi_msg *msg) 119{ 120 struct msi_desc *entry = get_irq_msi(irq); 121 switch(entry->msi_attrib.type) { 122 case PCI_CAP_ID_MSI: 123 { 124 struct pci_dev *dev = entry->dev; 125 int pos = entry->msi_attrib.pos; 126 u16 data; 127 128 pci_read_config_dword(dev, msi_lower_address_reg(pos), 129 &msg->address_lo); 130 if (entry->msi_attrib.is_64) { 131 pci_read_config_dword(dev, msi_upper_address_reg(pos), 132 &msg->address_hi); 133 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 134 } else { 135 msg->address_hi = 0; 136 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 137 } 138 msg->data = data; 139 break; 140 } 141 case PCI_CAP_ID_MSIX: 142 { 143 void __iomem *base; 144 base = entry->mask_base + 145 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 146 147 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 148 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 149 msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET); 150 break; 151 } 152 default: 153 BUG(); 154 } 155} 156 157void write_msi_msg(unsigned int irq, struct msi_msg *msg) 158{ 159 struct msi_desc *entry = get_irq_msi(irq); 160 switch (entry->msi_attrib.type) { 161 case PCI_CAP_ID_MSI: 162 { 163 struct pci_dev *dev = entry->dev; 164 int pos = entry->msi_attrib.pos; 165 166 pci_write_config_dword(dev, msi_lower_address_reg(pos), 167 msg->address_lo); 168 if (entry->msi_attrib.is_64) { 169 pci_write_config_dword(dev, msi_upper_address_reg(pos), 170 msg->address_hi); 171 pci_write_config_word(dev, msi_data_reg(pos, 1), 172 msg->data); 173 } else { 174 pci_write_config_word(dev, msi_data_reg(pos, 0), 175 msg->data); 176 } 177 break; 178 } 179 case PCI_CAP_ID_MSIX: 180 { 181 void __iomem *base; 182 base = entry->mask_base + 183 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 184 185 writel(msg->address_lo, 186 base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 187 writel(msg->address_hi, 188 base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 189 writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET); 190 break; 191 } 192 default: 193 BUG(); 194 } 195 entry->msg = *msg; 196} 197 198void mask_msi_irq(unsigned int irq) 199{ 200 msi_set_mask_bit(irq, 1); 201 msix_flush_writes(irq); 202} 203 204void unmask_msi_irq(unsigned int irq) 205{ 206 msi_set_mask_bit(irq, 0); 207 msix_flush_writes(irq); 208} 209 210static int msi_free_irq(struct pci_dev* dev, int irq); 211 212 213static struct msi_desc* alloc_msi_entry(void) 214{ 215 struct msi_desc *entry; 216 217 entry = kzalloc(sizeof(struct msi_desc), GFP_KERNEL); 218 if (!entry) 219 return NULL; 220 221 INIT_LIST_HEAD(&entry->list); 222 entry->irq = 0; 223 entry->dev = NULL; 224 225 return entry; 226} 227 228#ifdef CONFIG_PM 229static void __pci_restore_msi_state(struct pci_dev *dev) 230{ 231 int pos; 232 u16 control; 233 struct msi_desc *entry; 234 235 if (!dev->msi_enabled) 236 return; 237 238 entry = get_irq_msi(dev->irq); 239 pos = entry->msi_attrib.pos; 240 241 pci_intx(dev, 0); /* disable intx */ 242 msi_set_enable(dev, 0); 243 write_msi_msg(dev->irq, &entry->msg); 244 if (entry->msi_attrib.maskbit) 245 msi_set_mask_bit(dev->irq, entry->msi_attrib.masked); 246 247 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 248 control &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE); 249 if (entry->msi_attrib.maskbit || !entry->msi_attrib.masked) 250 control |= PCI_MSI_FLAGS_ENABLE; 251 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 252} 253 254static void __pci_restore_msix_state(struct pci_dev *dev) 255{ 256 int pos; 257 struct msi_desc *entry; 258 u16 control; 259 260 if (!dev->msix_enabled) 261 return; 262 263 /* route the table */ 264 pci_intx(dev, 0); /* disable intx */ 265 msix_set_enable(dev, 0); 266 267 list_for_each_entry(entry, &dev->msi_list, list) { 268 write_msi_msg(entry->irq, &entry->msg); 269 msi_set_mask_bit(entry->irq, entry->msi_attrib.masked); 270 } 271 272 entry = get_irq_msi(dev->first_msi_irq); 273 pos = entry->msi_attrib.pos; 274 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 275 control &= ~PCI_MSIX_FLAGS_MASKALL; 276 control |= PCI_MSIX_FLAGS_ENABLE; 277 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 278} 279 280void pci_restore_msi_state(struct pci_dev *dev) 281{ 282 __pci_restore_msi_state(dev); 283 __pci_restore_msix_state(dev); 284} 285#endif /* CONFIG_PM */ 286 287/** 288 * msi_capability_init - configure device's MSI capability structure 289 * @dev: pointer to the pci_dev data structure of MSI device function 290 * 291 * Setup the MSI capability structure of device function with a single 292 * MSI irq, regardless of device function is capable of handling 293 * multiple messages. A return of zero indicates the successful setup 294 * of an entry zero with the new MSI irq or non-zero for otherwise. 295 **/ 296static int msi_capability_init(struct pci_dev *dev) 297{ 298 struct msi_desc *entry; 299 int pos, irq; 300 u16 control; 301 302 msi_set_enable(dev, 0); /* Ensure msi is disabled as I set it up */ 303 304 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 305 pci_read_config_word(dev, msi_control_reg(pos), &control); 306 /* MSI Entry Initialization */ 307 entry = alloc_msi_entry(); 308 if (!entry) 309 return -ENOMEM; 310 311 entry->msi_attrib.type = PCI_CAP_ID_MSI; 312 entry->msi_attrib.is_64 = is_64bit_address(control); 313 entry->msi_attrib.entry_nr = 0; 314 entry->msi_attrib.maskbit = is_mask_bit_support(control); 315 entry->msi_attrib.masked = 1; 316 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ 317 entry->msi_attrib.pos = pos; 318 if (is_mask_bit_support(control)) { 319 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, 320 is_64bit_address(control)); 321 } 322 entry->dev = dev; 323 if (entry->msi_attrib.maskbit) { 324 unsigned int maskbits, temp; 325 /* All MSIs are unmasked by default, Mask them all */ 326 pci_read_config_dword(dev, 327 msi_mask_bits_reg(pos, is_64bit_address(control)), 328 &maskbits); 329 temp = (1 << multi_msi_capable(control)); 330 temp = ((temp - 1) & ~temp); 331 maskbits |= temp; 332 pci_write_config_dword(dev, 333 msi_mask_bits_reg(pos, is_64bit_address(control)), 334 maskbits); 335 } 336 /* Configure MSI capability structure */ 337 irq = arch_setup_msi_irq(dev, entry); 338 if (irq < 0) { 339 kfree(entry); 340 return irq; 341 } 342 entry->irq = irq; 343 list_add(&entry->list, &dev->msi_list); 344 dev->first_msi_irq = irq; 345 set_irq_msi(irq, entry); 346 347 /* Set MSI enabled bits */ 348 pci_intx(dev, 0); /* disable intx */ 349 msi_set_enable(dev, 1); 350 dev->msi_enabled = 1; 351 352 dev->irq = irq; 353 return 0; 354} 355 356/** 357 * msix_capability_init - configure device's MSI-X capability 358 * @dev: pointer to the pci_dev data structure of MSI-X device function 359 * @entries: pointer to an array of struct msix_entry entries 360 * @nvec: number of @entries 361 * 362 * Setup the MSI-X capability structure of device function with a 363 * single MSI-X irq. A return of zero indicates the successful setup of 364 * requested MSI-X entries with allocated irqs or non-zero for otherwise. 365 **/ 366static int msix_capability_init(struct pci_dev *dev, 367 struct msix_entry *entries, int nvec) 368{ 369 struct msi_desc *entry; 370 int irq, pos, i, j, nr_entries; 371 unsigned long phys_addr; 372 u32 table_offset; 373 u16 control; 374 u8 bir; 375 void __iomem *base; 376 377 msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */ 378 379 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 380 /* Request & Map MSI-X table region */ 381 pci_read_config_word(dev, msi_control_reg(pos), &control); 382 nr_entries = multi_msix_capable(control); 383 384 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 385 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 386 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 387 phys_addr = pci_resource_start (dev, bir) + table_offset; 388 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 389 if (base == NULL) 390 return -ENOMEM; 391 392 /* MSI-X Table Initialization */ 393 for (i = 0; i < nvec; i++) { 394 entry = alloc_msi_entry(); 395 if (!entry) 396 break; 397 398 j = entries[i].entry; 399 entry->msi_attrib.type = PCI_CAP_ID_MSIX; 400 entry->msi_attrib.is_64 = 1; 401 entry->msi_attrib.entry_nr = j; 402 entry->msi_attrib.maskbit = 1; 403 entry->msi_attrib.masked = 1; 404 entry->msi_attrib.default_irq = dev->irq; 405 entry->msi_attrib.pos = pos; 406 entry->dev = dev; 407 entry->mask_base = base; 408 409 /* Configure MSI-X capability structure */ 410 irq = arch_setup_msi_irq(dev, entry); 411 if (irq < 0) { 412 kfree(entry); 413 break; 414 } 415 entry->irq = irq; 416 entries[i].vector = irq; 417 list_add(&entry->list, &dev->msi_list); 418 419 set_irq_msi(irq, entry); 420 } 421 if (i != nvec) { 422 int avail = i - 1; 423 i--; 424 for (; i >= 0; i--) { 425 irq = (entries + i)->vector; 426 msi_free_irq(dev, irq); 427 (entries + i)->vector = 0; 428 } 429 /* If we had some success report the number of irqs 430 * we succeeded in setting up. 431 */ 432 if (avail <= 0) 433 avail = -EBUSY; 434 return avail; 435 } 436 dev->first_msi_irq = entries[0].vector; 437 /* Set MSI-X enabled bits */ 438 pci_intx(dev, 0); /* disable intx */ 439 msix_set_enable(dev, 1); 440 dev->msix_enabled = 1; 441 442 return 0; 443} 444 445/** 446 * pci_msi_check_device - check whether MSI may be enabled on a device 447 * @dev: pointer to the pci_dev data structure of MSI device function 448 * @nvec: how many MSIs have been requested ? 449 * @type: are we checking for MSI or MSI-X ? 450 * 451 * Look at global flags, the device itself, and its parent busses 452 * to determine if MSI/-X are supported for the device. If MSI/-X is 453 * supported return 0, else return an error code. 454 **/ 455static int pci_msi_check_device(struct pci_dev* dev, int nvec, int type) 456{ 457 struct pci_bus *bus; 458 int ret; 459 460 /* MSI must be globally enabled and supported by the device */ 461 if (!pci_msi_enable || !dev || dev->no_msi) 462 return -EINVAL; 463 464 /* Any bridge which does NOT route MSI transactions from it's 465 * secondary bus to it's primary bus must set NO_MSI flag on 466 * the secondary pci_bus. 467 * We expect only arch-specific PCI host bus controller driver 468 * or quirks for specific PCI bridges to be setting NO_MSI. 469 */ 470 for (bus = dev->bus; bus; bus = bus->parent) 471 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 472 return -EINVAL; 473 474 ret = arch_msi_check_device(dev, nvec, type); 475 if (ret) 476 return ret; 477 478 if (!pci_find_capability(dev, type)) 479 return -EINVAL; 480 481 return 0; 482} 483 484/** 485 * pci_enable_msi - configure device's MSI capability structure 486 * @dev: pointer to the pci_dev data structure of MSI device function 487 * 488 * Setup the MSI capability structure of device function with 489 * a single MSI irq upon its software driver call to request for 490 * MSI mode enabled on its hardware device function. A return of zero 491 * indicates the successful setup of an entry zero with the new MSI 492 * irq or non-zero for otherwise. 493 **/ 494int pci_enable_msi(struct pci_dev* dev) 495{ 496 int status; 497 498 status = pci_msi_check_device(dev, 1, PCI_CAP_ID_MSI); 499 if (status) 500 return status; 501 502 WARN_ON(!!dev->msi_enabled); 503 504 /* Check whether driver already requested for MSI-X irqs */ 505 if (dev->msix_enabled) { 506 printk(KERN_INFO "PCI: %s: Can't enable MSI. " 507 "Device already has MSI-X enabled\n", 508 pci_name(dev)); 509 return -EINVAL; 510 } 511 status = msi_capability_init(dev); 512 return status; 513} 514EXPORT_SYMBOL(pci_enable_msi); 515 516void pci_disable_msi(struct pci_dev* dev) 517{ 518 struct msi_desc *entry; 519 int default_irq; 520 521 if (!pci_msi_enable || !dev || !dev->msi_enabled) 522 return; 523 524 msi_set_enable(dev, 0); 525 pci_intx(dev, 1); /* enable intx */ 526 dev->msi_enabled = 0; 527 528 entry = get_irq_msi(dev->first_msi_irq); 529 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { 530 return; 531 } 532 533 default_irq = entry->msi_attrib.default_irq; 534 msi_free_irq(dev, dev->first_msi_irq); 535 536 /* Restore dev->irq to its default pin-assertion irq */ 537 dev->irq = default_irq; 538 539 dev->first_msi_irq = 0; 540} 541EXPORT_SYMBOL(pci_disable_msi); 542 543static int msi_free_irq(struct pci_dev* dev, int irq) 544{ 545 struct msi_desc *entry; 546 int entry_nr, type; 547 void __iomem *base; 548 549 BUG_ON(irq_has_action(irq)); 550 551 entry = get_irq_msi(irq); 552 if (!entry || entry->dev != dev) { 553 return -EINVAL; 554 } 555 type = entry->msi_attrib.type; 556 entry_nr = entry->msi_attrib.entry_nr; 557 base = entry->mask_base; 558 list_del(&entry->list); 559 560 arch_teardown_msi_irq(irq); 561 kfree(entry); 562 563 if (type == PCI_CAP_ID_MSIX) { 564 writel(1, base + entry_nr * PCI_MSIX_ENTRY_SIZE + 565 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); 566 567 if (list_empty(&dev->msi_list)) 568 iounmap(base); 569 } 570 571 return 0; 572} 573 574/** 575 * pci_enable_msix - configure device's MSI-X capability structure 576 * @dev: pointer to the pci_dev data structure of MSI-X device function 577 * @entries: pointer to an array of MSI-X entries 578 * @nvec: number of MSI-X irqs requested for allocation by device driver 579 * 580 * Setup the MSI-X capability structure of device function with the number 581 * of requested irqs upon its software driver call to request for 582 * MSI-X mode enabled on its hardware device function. A return of zero 583 * indicates the successful configuration of MSI-X capability structure 584 * with new allocated MSI-X irqs. A return of < 0 indicates a failure. 585 * Or a return of > 0 indicates that driver request is exceeding the number 586 * of irqs available. Driver should use the returned value to re-send 587 * its request. 588 **/ 589int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) 590{ 591 int status, pos, nr_entries; 592 int i, j; 593 u16 control; 594 595 if (!entries) 596 return -EINVAL; 597 598 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX); 599 if (status) 600 return status; 601 602 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 603 pci_read_config_word(dev, msi_control_reg(pos), &control); 604 nr_entries = multi_msix_capable(control); 605 if (nvec > nr_entries) 606 return -EINVAL; 607 608 /* Check for any invalid entries */ 609 for (i = 0; i < nvec; i++) { 610 if (entries[i].entry >= nr_entries) 611 return -EINVAL; /* invalid entry */ 612 for (j = i + 1; j < nvec; j++) { 613 if (entries[i].entry == entries[j].entry) 614 return -EINVAL; /* duplicate entry */ 615 } 616 } 617 WARN_ON(!!dev->msix_enabled); 618 619 /* Check whether driver already requested for MSI irq */ 620 if (dev->msi_enabled) { 621 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " 622 "Device already has an MSI irq assigned\n", 623 pci_name(dev)); 624 return -EINVAL; 625 } 626 status = msix_capability_init(dev, entries, nvec); 627 return status; 628} 629EXPORT_SYMBOL(pci_enable_msix); 630 631static void msix_free_all_irqs(struct pci_dev *dev) 632{ 633 struct msi_desc *entry; 634 635 list_for_each_entry(entry, &dev->msi_list, list) 636 msi_free_irq(dev, entry->irq); 637 dev->first_msi_irq = 0; 638} 639 640void pci_disable_msix(struct pci_dev* dev) 641{ 642 if (!pci_msi_enable || !dev || !dev->msix_enabled) 643 return; 644 645 msix_set_enable(dev, 0); 646 pci_intx(dev, 1); /* enable intx */ 647 dev->msix_enabled = 0; 648 649 msix_free_all_irqs(dev); 650} 651EXPORT_SYMBOL(pci_disable_msix); 652 653/** 654 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state 655 * @dev: pointer to the pci_dev data structure of MSI(X) device function 656 * 657 * Being called during hotplug remove, from which the device function 658 * is hot-removed. All previous assigned MSI/MSI-X irqs, if 659 * allocated for this device function, are reclaimed to unused state, 660 * which may be used later on. 661 **/ 662void msi_remove_pci_irq_vectors(struct pci_dev* dev) 663{ 664 if (!pci_msi_enable || !dev) 665 return; 666 667 if (dev->msi_enabled) 668 msi_free_irq(dev, dev->first_msi_irq); 669 670 if (dev->msix_enabled) 671 msix_free_all_irqs(dev); 672} 673 674void pci_no_msi(void) 675{ 676 pci_msi_enable = 0; 677} 678 679void pci_msi_init_pci_dev(struct pci_dev *dev) 680{ 681 INIT_LIST_HEAD(&dev->msi_list); 682} 683 684 685/* Arch hooks */ 686 687int __attribute__ ((weak)) 688arch_msi_check_device(struct pci_dev* dev, int nvec, int type) 689{ 690 return 0; 691} 692 693