msi.c revision 7fe3730de729b758e9f69b862b9255d998671b5f
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/err.h> 10#include <linux/mm.h> 11#include <linux/irq.h> 12#include <linux/interrupt.h> 13#include <linux/init.h> 14#include <linux/ioport.h> 15#include <linux/smp_lock.h> 16#include <linux/pci.h> 17#include <linux/proc_fs.h> 18#include <linux/msi.h> 19 20#include <asm/errno.h> 21#include <asm/io.h> 22#include <asm/smp.h> 23 24#include "pci.h" 25#include "msi.h" 26 27static int pci_msi_enable = 1; 28 29static void msi_set_enable(struct pci_dev *dev, int enable) 30{ 31 int pos; 32 u16 control; 33 34 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 35 if (pos) { 36 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 37 control &= ~PCI_MSI_FLAGS_ENABLE; 38 if (enable) 39 control |= PCI_MSI_FLAGS_ENABLE; 40 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 41 } 42} 43 44static void msix_set_enable(struct pci_dev *dev, int enable) 45{ 46 int pos; 47 u16 control; 48 49 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 50 if (pos) { 51 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 52 control &= ~PCI_MSIX_FLAGS_ENABLE; 53 if (enable) 54 control |= PCI_MSIX_FLAGS_ENABLE; 55 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 56 } 57} 58 59static void msix_flush_writes(unsigned int irq) 60{ 61 struct msi_desc *entry; 62 63 entry = get_irq_msi(irq); 64 BUG_ON(!entry || !entry->dev); 65 switch (entry->msi_attrib.type) { 66 case PCI_CAP_ID_MSI: 67 /* nothing to do */ 68 break; 69 case PCI_CAP_ID_MSIX: 70 { 71 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 72 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 73 readl(entry->mask_base + offset); 74 break; 75 } 76 default: 77 BUG(); 78 break; 79 } 80} 81 82static void msi_set_mask_bit(unsigned int irq, int flag) 83{ 84 struct msi_desc *entry; 85 86 entry = get_irq_msi(irq); 87 BUG_ON(!entry || !entry->dev); 88 switch (entry->msi_attrib.type) { 89 case PCI_CAP_ID_MSI: 90 if (entry->msi_attrib.maskbit) { 91 int pos; 92 u32 mask_bits; 93 94 pos = (long)entry->mask_base; 95 pci_read_config_dword(entry->dev, pos, &mask_bits); 96 mask_bits &= ~(1); 97 mask_bits |= flag; 98 pci_write_config_dword(entry->dev, pos, mask_bits); 99 } else { 100 msi_set_enable(entry->dev, !flag); 101 } 102 break; 103 case PCI_CAP_ID_MSIX: 104 { 105 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 106 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 107 writel(flag, entry->mask_base + offset); 108 readl(entry->mask_base + offset); 109 break; 110 } 111 default: 112 BUG(); 113 break; 114 } 115 entry->msi_attrib.masked = !!flag; 116} 117 118void read_msi_msg(unsigned int irq, struct msi_msg *msg) 119{ 120 struct msi_desc *entry = get_irq_msi(irq); 121 switch(entry->msi_attrib.type) { 122 case PCI_CAP_ID_MSI: 123 { 124 struct pci_dev *dev = entry->dev; 125 int pos = entry->msi_attrib.pos; 126 u16 data; 127 128 pci_read_config_dword(dev, msi_lower_address_reg(pos), 129 &msg->address_lo); 130 if (entry->msi_attrib.is_64) { 131 pci_read_config_dword(dev, msi_upper_address_reg(pos), 132 &msg->address_hi); 133 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 134 } else { 135 msg->address_hi = 0; 136 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 137 } 138 msg->data = data; 139 break; 140 } 141 case PCI_CAP_ID_MSIX: 142 { 143 void __iomem *base; 144 base = entry->mask_base + 145 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 146 147 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 148 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 149 msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET); 150 break; 151 } 152 default: 153 BUG(); 154 } 155} 156 157void write_msi_msg(unsigned int irq, struct msi_msg *msg) 158{ 159 struct msi_desc *entry = get_irq_msi(irq); 160 switch (entry->msi_attrib.type) { 161 case PCI_CAP_ID_MSI: 162 { 163 struct pci_dev *dev = entry->dev; 164 int pos = entry->msi_attrib.pos; 165 166 pci_write_config_dword(dev, msi_lower_address_reg(pos), 167 msg->address_lo); 168 if (entry->msi_attrib.is_64) { 169 pci_write_config_dword(dev, msi_upper_address_reg(pos), 170 msg->address_hi); 171 pci_write_config_word(dev, msi_data_reg(pos, 1), 172 msg->data); 173 } else { 174 pci_write_config_word(dev, msi_data_reg(pos, 0), 175 msg->data); 176 } 177 break; 178 } 179 case PCI_CAP_ID_MSIX: 180 { 181 void __iomem *base; 182 base = entry->mask_base + 183 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 184 185 writel(msg->address_lo, 186 base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 187 writel(msg->address_hi, 188 base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 189 writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET); 190 break; 191 } 192 default: 193 BUG(); 194 } 195 entry->msg = *msg; 196} 197 198void mask_msi_irq(unsigned int irq) 199{ 200 msi_set_mask_bit(irq, 1); 201 msix_flush_writes(irq); 202} 203 204void unmask_msi_irq(unsigned int irq) 205{ 206 msi_set_mask_bit(irq, 0); 207 msix_flush_writes(irq); 208} 209 210static int msi_free_irq(struct pci_dev* dev, int irq); 211 212 213static struct msi_desc* alloc_msi_entry(void) 214{ 215 struct msi_desc *entry; 216 217 entry = kzalloc(sizeof(struct msi_desc), GFP_KERNEL); 218 if (!entry) 219 return NULL; 220 221 INIT_LIST_HEAD(&entry->list); 222 entry->irq = 0; 223 entry->dev = NULL; 224 225 return entry; 226} 227 228#ifdef CONFIG_PM 229static void __pci_restore_msi_state(struct pci_dev *dev) 230{ 231 int pos; 232 u16 control; 233 struct msi_desc *entry; 234 235 if (!dev->msi_enabled) 236 return; 237 238 entry = get_irq_msi(dev->irq); 239 pos = entry->msi_attrib.pos; 240 241 pci_intx(dev, 0); /* disable intx */ 242 msi_set_enable(dev, 0); 243 write_msi_msg(dev->irq, &entry->msg); 244 if (entry->msi_attrib.maskbit) 245 msi_set_mask_bit(dev->irq, entry->msi_attrib.masked); 246 247 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 248 control &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE); 249 if (entry->msi_attrib.maskbit || !entry->msi_attrib.masked) 250 control |= PCI_MSI_FLAGS_ENABLE; 251 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 252} 253 254static void __pci_restore_msix_state(struct pci_dev *dev) 255{ 256 int pos; 257 struct msi_desc *entry; 258 u16 control; 259 260 if (!dev->msix_enabled) 261 return; 262 263 /* route the table */ 264 pci_intx(dev, 0); /* disable intx */ 265 msix_set_enable(dev, 0); 266 267 list_for_each_entry(entry, &dev->msi_list, list) { 268 write_msi_msg(entry->irq, &entry->msg); 269 msi_set_mask_bit(entry->irq, entry->msi_attrib.masked); 270 } 271 272 BUG_ON(list_empty(&dev->msi_list)); 273 entry = list_entry(dev->msi_list.next, struct msi_desc, list); 274 pos = entry->msi_attrib.pos; 275 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 276 control &= ~PCI_MSIX_FLAGS_MASKALL; 277 control |= PCI_MSIX_FLAGS_ENABLE; 278 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 279} 280 281void pci_restore_msi_state(struct pci_dev *dev) 282{ 283 __pci_restore_msi_state(dev); 284 __pci_restore_msix_state(dev); 285} 286#endif /* CONFIG_PM */ 287 288/** 289 * msi_capability_init - configure device's MSI capability structure 290 * @dev: pointer to the pci_dev data structure of MSI device function 291 * 292 * Setup the MSI capability structure of device function with a single 293 * MSI irq, regardless of device function is capable of handling 294 * multiple messages. A return of zero indicates the successful setup 295 * of an entry zero with the new MSI irq or non-zero for otherwise. 296 **/ 297static int msi_capability_init(struct pci_dev *dev) 298{ 299 struct msi_desc *entry; 300 int pos, ret; 301 u16 control; 302 303 msi_set_enable(dev, 0); /* Ensure msi is disabled as I set it up */ 304 305 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 306 pci_read_config_word(dev, msi_control_reg(pos), &control); 307 /* MSI Entry Initialization */ 308 entry = alloc_msi_entry(); 309 if (!entry) 310 return -ENOMEM; 311 312 entry->msi_attrib.type = PCI_CAP_ID_MSI; 313 entry->msi_attrib.is_64 = is_64bit_address(control); 314 entry->msi_attrib.entry_nr = 0; 315 entry->msi_attrib.maskbit = is_mask_bit_support(control); 316 entry->msi_attrib.masked = 1; 317 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ 318 entry->msi_attrib.pos = pos; 319 if (is_mask_bit_support(control)) { 320 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, 321 is_64bit_address(control)); 322 } 323 entry->dev = dev; 324 if (entry->msi_attrib.maskbit) { 325 unsigned int maskbits, temp; 326 /* All MSIs are unmasked by default, Mask them all */ 327 pci_read_config_dword(dev, 328 msi_mask_bits_reg(pos, is_64bit_address(control)), 329 &maskbits); 330 temp = (1 << multi_msi_capable(control)); 331 temp = ((temp - 1) & ~temp); 332 maskbits |= temp; 333 pci_write_config_dword(dev, 334 msi_mask_bits_reg(pos, is_64bit_address(control)), 335 maskbits); 336 } 337 /* Configure MSI capability structure */ 338 ret = arch_setup_msi_irq(dev, entry); 339 if (ret) { 340 kfree(entry); 341 return ret; 342 } 343 list_add(&entry->list, &dev->msi_list); 344 345 /* Set MSI enabled bits */ 346 pci_intx(dev, 0); /* disable intx */ 347 msi_set_enable(dev, 1); 348 dev->msi_enabled = 1; 349 350 dev->irq = entry->irq; 351 return 0; 352} 353 354/** 355 * msix_capability_init - configure device's MSI-X capability 356 * @dev: pointer to the pci_dev data structure of MSI-X device function 357 * @entries: pointer to an array of struct msix_entry entries 358 * @nvec: number of @entries 359 * 360 * Setup the MSI-X capability structure of device function with a 361 * single MSI-X irq. A return of zero indicates the successful setup of 362 * requested MSI-X entries with allocated irqs or non-zero for otherwise. 363 **/ 364static int msix_capability_init(struct pci_dev *dev, 365 struct msix_entry *entries, int nvec) 366{ 367 struct msi_desc *entry; 368 int irq, pos, i, j, nr_entries, ret; 369 unsigned long phys_addr; 370 u32 table_offset; 371 u16 control; 372 u8 bir; 373 void __iomem *base; 374 375 msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */ 376 377 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 378 /* Request & Map MSI-X table region */ 379 pci_read_config_word(dev, msi_control_reg(pos), &control); 380 nr_entries = multi_msix_capable(control); 381 382 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 383 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 384 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 385 phys_addr = pci_resource_start (dev, bir) + table_offset; 386 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 387 if (base == NULL) 388 return -ENOMEM; 389 390 /* MSI-X Table Initialization */ 391 for (i = 0; i < nvec; i++) { 392 entry = alloc_msi_entry(); 393 if (!entry) 394 break; 395 396 j = entries[i].entry; 397 entry->msi_attrib.type = PCI_CAP_ID_MSIX; 398 entry->msi_attrib.is_64 = 1; 399 entry->msi_attrib.entry_nr = j; 400 entry->msi_attrib.maskbit = 1; 401 entry->msi_attrib.masked = 1; 402 entry->msi_attrib.default_irq = dev->irq; 403 entry->msi_attrib.pos = pos; 404 entry->dev = dev; 405 entry->mask_base = base; 406 407 /* Configure MSI-X capability structure */ 408 ret = arch_setup_msi_irq(dev, entry); 409 if (ret) { 410 kfree(entry); 411 break; 412 } 413 entries[i].vector = entry->irq; 414 list_add(&entry->list, &dev->msi_list); 415 } 416 if (i != nvec) { 417 int avail = i - 1; 418 i--; 419 for (; i >= 0; i--) { 420 irq = (entries + i)->vector; 421 msi_free_irq(dev, irq); 422 (entries + i)->vector = 0; 423 } 424 /* If we had some success report the number of irqs 425 * we succeeded in setting up. 426 */ 427 if (avail <= 0) 428 avail = -EBUSY; 429 return avail; 430 } 431 /* Set MSI-X enabled bits */ 432 pci_intx(dev, 0); /* disable intx */ 433 msix_set_enable(dev, 1); 434 dev->msix_enabled = 1; 435 436 return 0; 437} 438 439/** 440 * pci_msi_check_device - check whether MSI may be enabled on a device 441 * @dev: pointer to the pci_dev data structure of MSI device function 442 * @nvec: how many MSIs have been requested ? 443 * @type: are we checking for MSI or MSI-X ? 444 * 445 * Look at global flags, the device itself, and its parent busses 446 * to determine if MSI/-X are supported for the device. If MSI/-X is 447 * supported return 0, else return an error code. 448 **/ 449static int pci_msi_check_device(struct pci_dev* dev, int nvec, int type) 450{ 451 struct pci_bus *bus; 452 int ret; 453 454 /* MSI must be globally enabled and supported by the device */ 455 if (!pci_msi_enable || !dev || dev->no_msi) 456 return -EINVAL; 457 458 /* 459 * You can't ask to have 0 or less MSIs configured. 460 * a) it's stupid .. 461 * b) the list manipulation code assumes nvec >= 1. 462 */ 463 if (nvec < 1) 464 return -ERANGE; 465 466 /* Any bridge which does NOT route MSI transactions from it's 467 * secondary bus to it's primary bus must set NO_MSI flag on 468 * the secondary pci_bus. 469 * We expect only arch-specific PCI host bus controller driver 470 * or quirks for specific PCI bridges to be setting NO_MSI. 471 */ 472 for (bus = dev->bus; bus; bus = bus->parent) 473 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 474 return -EINVAL; 475 476 ret = arch_msi_check_device(dev, nvec, type); 477 if (ret) 478 return ret; 479 480 if (!pci_find_capability(dev, type)) 481 return -EINVAL; 482 483 return 0; 484} 485 486/** 487 * pci_enable_msi - configure device's MSI capability structure 488 * @dev: pointer to the pci_dev data structure of MSI device function 489 * 490 * Setup the MSI capability structure of device function with 491 * a single MSI irq upon its software driver call to request for 492 * MSI mode enabled on its hardware device function. A return of zero 493 * indicates the successful setup of an entry zero with the new MSI 494 * irq or non-zero for otherwise. 495 **/ 496int pci_enable_msi(struct pci_dev* dev) 497{ 498 int status; 499 500 status = pci_msi_check_device(dev, 1, PCI_CAP_ID_MSI); 501 if (status) 502 return status; 503 504 WARN_ON(!!dev->msi_enabled); 505 506 /* Check whether driver already requested for MSI-X irqs */ 507 if (dev->msix_enabled) { 508 printk(KERN_INFO "PCI: %s: Can't enable MSI. " 509 "Device already has MSI-X enabled\n", 510 pci_name(dev)); 511 return -EINVAL; 512 } 513 status = msi_capability_init(dev); 514 return status; 515} 516EXPORT_SYMBOL(pci_enable_msi); 517 518void pci_disable_msi(struct pci_dev* dev) 519{ 520 struct msi_desc *entry; 521 int default_irq; 522 523 if (!pci_msi_enable || !dev || !dev->msi_enabled) 524 return; 525 526 msi_set_enable(dev, 0); 527 pci_intx(dev, 1); /* enable intx */ 528 dev->msi_enabled = 0; 529 530 BUG_ON(list_empty(&dev->msi_list)); 531 entry = list_entry(dev->msi_list.next, struct msi_desc, list); 532 if (!entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { 533 return; 534 } 535 536 default_irq = entry->msi_attrib.default_irq; 537 msi_free_irq(dev, entry->irq); 538 539 /* Restore dev->irq to its default pin-assertion irq */ 540 dev->irq = default_irq; 541} 542EXPORT_SYMBOL(pci_disable_msi); 543 544static int msi_free_irq(struct pci_dev* dev, int irq) 545{ 546 struct msi_desc *entry; 547 int entry_nr, type; 548 void __iomem *base; 549 550 BUG_ON(irq_has_action(irq)); 551 552 entry = get_irq_msi(irq); 553 if (!entry || entry->dev != dev) { 554 return -EINVAL; 555 } 556 type = entry->msi_attrib.type; 557 entry_nr = entry->msi_attrib.entry_nr; 558 base = entry->mask_base; 559 list_del(&entry->list); 560 561 arch_teardown_msi_irq(irq); 562 kfree(entry); 563 564 if (type == PCI_CAP_ID_MSIX) { 565 writel(1, base + entry_nr * PCI_MSIX_ENTRY_SIZE + 566 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); 567 568 if (list_empty(&dev->msi_list)) 569 iounmap(base); 570 } 571 572 return 0; 573} 574 575/** 576 * pci_enable_msix - configure device's MSI-X capability structure 577 * @dev: pointer to the pci_dev data structure of MSI-X device function 578 * @entries: pointer to an array of MSI-X entries 579 * @nvec: number of MSI-X irqs requested for allocation by device driver 580 * 581 * Setup the MSI-X capability structure of device function with the number 582 * of requested irqs upon its software driver call to request for 583 * MSI-X mode enabled on its hardware device function. A return of zero 584 * indicates the successful configuration of MSI-X capability structure 585 * with new allocated MSI-X irqs. A return of < 0 indicates a failure. 586 * Or a return of > 0 indicates that driver request is exceeding the number 587 * of irqs available. Driver should use the returned value to re-send 588 * its request. 589 **/ 590int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) 591{ 592 int status, pos, nr_entries; 593 int i, j; 594 u16 control; 595 596 if (!entries) 597 return -EINVAL; 598 599 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX); 600 if (status) 601 return status; 602 603 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 604 pci_read_config_word(dev, msi_control_reg(pos), &control); 605 nr_entries = multi_msix_capable(control); 606 if (nvec > nr_entries) 607 return -EINVAL; 608 609 /* Check for any invalid entries */ 610 for (i = 0; i < nvec; i++) { 611 if (entries[i].entry >= nr_entries) 612 return -EINVAL; /* invalid entry */ 613 for (j = i + 1; j < nvec; j++) { 614 if (entries[i].entry == entries[j].entry) 615 return -EINVAL; /* duplicate entry */ 616 } 617 } 618 WARN_ON(!!dev->msix_enabled); 619 620 /* Check whether driver already requested for MSI irq */ 621 if (dev->msi_enabled) { 622 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " 623 "Device already has an MSI irq assigned\n", 624 pci_name(dev)); 625 return -EINVAL; 626 } 627 status = msix_capability_init(dev, entries, nvec); 628 return status; 629} 630EXPORT_SYMBOL(pci_enable_msix); 631 632static void msix_free_all_irqs(struct pci_dev *dev) 633{ 634 struct msi_desc *entry; 635 636 list_for_each_entry(entry, &dev->msi_list, list) 637 msi_free_irq(dev, entry->irq); 638} 639 640void pci_disable_msix(struct pci_dev* dev) 641{ 642 if (!pci_msi_enable || !dev || !dev->msix_enabled) 643 return; 644 645 msix_set_enable(dev, 0); 646 pci_intx(dev, 1); /* enable intx */ 647 dev->msix_enabled = 0; 648 649 msix_free_all_irqs(dev); 650} 651EXPORT_SYMBOL(pci_disable_msix); 652 653/** 654 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state 655 * @dev: pointer to the pci_dev data structure of MSI(X) device function 656 * 657 * Being called during hotplug remove, from which the device function 658 * is hot-removed. All previous assigned MSI/MSI-X irqs, if 659 * allocated for this device function, are reclaimed to unused state, 660 * which may be used later on. 661 **/ 662void msi_remove_pci_irq_vectors(struct pci_dev* dev) 663{ 664 if (!pci_msi_enable || !dev) 665 return; 666 667 if (dev->msi_enabled) { 668 struct msi_desc *entry; 669 BUG_ON(list_empty(&dev->msi_list)); 670 entry = list_entry(dev->msi_list.next, struct msi_desc, list); 671 msi_free_irq(dev, entry->irq); 672 } 673 674 if (dev->msix_enabled) 675 msix_free_all_irqs(dev); 676} 677 678void pci_no_msi(void) 679{ 680 pci_msi_enable = 0; 681} 682 683void pci_msi_init_pci_dev(struct pci_dev *dev) 684{ 685 INIT_LIST_HEAD(&dev->msi_list); 686} 687 688 689/* Arch hooks */ 690 691int __attribute__ ((weak)) 692arch_msi_check_device(struct pci_dev* dev, int nvec, int type) 693{ 694 return 0; 695} 696 697