msi.c revision 39431acb1a4c464e62471cb3058b8ffffb9244db
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/err.h> 10#include <linux/mm.h> 11#include <linux/irq.h> 12#include <linux/interrupt.h> 13#include <linux/init.h> 14#include <linux/ioport.h> 15#include <linux/pci.h> 16#include <linux/proc_fs.h> 17#include <linux/msi.h> 18#include <linux/smp.h> 19#include <linux/errno.h> 20#include <linux/io.h> 21#include <linux/slab.h> 22 23#include "pci.h" 24#include "msi.h" 25 26static int pci_msi_enable = 1; 27 28/* Arch hooks */ 29 30#ifndef arch_msi_check_device 31int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) 32{ 33 return 0; 34} 35#endif 36 37#ifndef arch_setup_msi_irqs 38int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) 39{ 40 struct msi_desc *entry; 41 int ret; 42 43 /* 44 * If an architecture wants to support multiple MSI, it needs to 45 * override arch_setup_msi_irqs() 46 */ 47 if (type == PCI_CAP_ID_MSI && nvec > 1) 48 return 1; 49 50 list_for_each_entry(entry, &dev->msi_list, list) { 51 ret = arch_setup_msi_irq(dev, entry); 52 if (ret < 0) 53 return ret; 54 if (ret > 0) 55 return -ENOSPC; 56 } 57 58 return 0; 59} 60#endif 61 62#ifndef arch_teardown_msi_irqs 63void arch_teardown_msi_irqs(struct pci_dev *dev) 64{ 65 struct msi_desc *entry; 66 67 list_for_each_entry(entry, &dev->msi_list, list) { 68 int i, nvec; 69 if (entry->irq == 0) 70 continue; 71 nvec = 1 << entry->msi_attrib.multiple; 72 for (i = 0; i < nvec; i++) 73 arch_teardown_msi_irq(entry->irq + i); 74 } 75} 76#endif 77 78static void msi_set_enable(struct pci_dev *dev, int pos, int enable) 79{ 80 u16 control; 81 82 BUG_ON(!pos); 83 84 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 85 control &= ~PCI_MSI_FLAGS_ENABLE; 86 if (enable) 87 control |= PCI_MSI_FLAGS_ENABLE; 88 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 89} 90 91static void msix_set_enable(struct pci_dev *dev, int enable) 92{ 93 int pos; 94 u16 control; 95 96 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 97 if (pos) { 98 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 99 control &= ~PCI_MSIX_FLAGS_ENABLE; 100 if (enable) 101 control |= PCI_MSIX_FLAGS_ENABLE; 102 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 103 } 104} 105 106static inline __attribute_const__ u32 msi_mask(unsigned x) 107{ 108 /* Don't shift by >= width of type */ 109 if (x >= 5) 110 return 0xffffffff; 111 return (1 << (1 << x)) - 1; 112} 113 114static inline __attribute_const__ u32 msi_capable_mask(u16 control) 115{ 116 return msi_mask((control >> 1) & 7); 117} 118 119static inline __attribute_const__ u32 msi_enabled_mask(u16 control) 120{ 121 return msi_mask((control >> 4) & 7); 122} 123 124/* 125 * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to 126 * mask all MSI interrupts by clearing the MSI enable bit does not work 127 * reliably as devices without an INTx disable bit will then generate a 128 * level IRQ which will never be cleared. 129 */ 130static u32 __msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) 131{ 132 u32 mask_bits = desc->masked; 133 134 if (!desc->msi_attrib.maskbit) 135 return 0; 136 137 mask_bits &= ~mask; 138 mask_bits |= flag; 139 pci_write_config_dword(desc->dev, desc->mask_pos, mask_bits); 140 141 return mask_bits; 142} 143 144static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) 145{ 146 desc->masked = __msi_mask_irq(desc, mask, flag); 147} 148 149/* 150 * This internal function does not flush PCI writes to the device. 151 * All users must ensure that they read from the device before either 152 * assuming that the device state is up to date, or returning out of this 153 * file. This saves a few milliseconds when initialising devices with lots 154 * of MSI-X interrupts. 155 */ 156static u32 __msix_mask_irq(struct msi_desc *desc, u32 flag) 157{ 158 u32 mask_bits = desc->masked; 159 unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 160 PCI_MSIX_ENTRY_VECTOR_CTRL; 161 mask_bits &= ~1; 162 mask_bits |= flag; 163 writel(mask_bits, desc->mask_base + offset); 164 165 return mask_bits; 166} 167 168static void msix_mask_irq(struct msi_desc *desc, u32 flag) 169{ 170 desc->masked = __msix_mask_irq(desc, flag); 171} 172 173static void msi_set_mask_bit(struct irq_data *data, u32 flag) 174{ 175 struct msi_desc *desc = irq_data_get_msi(data); 176 177 if (desc->msi_attrib.is_msix) { 178 msix_mask_irq(desc, flag); 179 readl(desc->mask_base); /* Flush write to device */ 180 } else { 181 unsigned offset = data->irq - desc->dev->irq; 182 msi_mask_irq(desc, 1 << offset, flag << offset); 183 } 184} 185 186void mask_msi_irq(struct irq_data *data) 187{ 188 msi_set_mask_bit(data, 1); 189} 190 191void unmask_msi_irq(struct irq_data *data) 192{ 193 msi_set_mask_bit(data, 0); 194} 195 196void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 197{ 198 BUG_ON(entry->dev->current_state != PCI_D0); 199 200 if (entry->msi_attrib.is_msix) { 201 void __iomem *base = entry->mask_base + 202 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 203 204 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); 205 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); 206 msg->data = readl(base + PCI_MSIX_ENTRY_DATA); 207 } else { 208 struct pci_dev *dev = entry->dev; 209 int pos = entry->msi_attrib.pos; 210 u16 data; 211 212 pci_read_config_dword(dev, msi_lower_address_reg(pos), 213 &msg->address_lo); 214 if (entry->msi_attrib.is_64) { 215 pci_read_config_dword(dev, msi_upper_address_reg(pos), 216 &msg->address_hi); 217 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 218 } else { 219 msg->address_hi = 0; 220 pci_read_config_word(dev, msi_data_reg(pos, 0), &data); 221 } 222 msg->data = data; 223 } 224} 225 226void read_msi_msg(unsigned int irq, struct msi_msg *msg) 227{ 228 struct msi_desc *entry = get_irq_msi(irq); 229 230 __read_msi_msg(entry, msg); 231} 232 233void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 234{ 235 /* Assert that the cache is valid, assuming that 236 * valid messages are not all-zeroes. */ 237 BUG_ON(!(entry->msg.address_hi | entry->msg.address_lo | 238 entry->msg.data)); 239 240 *msg = entry->msg; 241} 242 243void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) 244{ 245 struct msi_desc *entry = get_irq_msi(irq); 246 247 __get_cached_msi_msg(entry, msg); 248} 249 250void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 251{ 252 if (entry->dev->current_state != PCI_D0) { 253 /* Don't touch the hardware now */ 254 } else if (entry->msi_attrib.is_msix) { 255 void __iomem *base; 256 base = entry->mask_base + 257 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 258 259 writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); 260 writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); 261 writel(msg->data, base + PCI_MSIX_ENTRY_DATA); 262 } else { 263 struct pci_dev *dev = entry->dev; 264 int pos = entry->msi_attrib.pos; 265 u16 msgctl; 266 267 pci_read_config_word(dev, msi_control_reg(pos), &msgctl); 268 msgctl &= ~PCI_MSI_FLAGS_QSIZE; 269 msgctl |= entry->msi_attrib.multiple << 4; 270 pci_write_config_word(dev, msi_control_reg(pos), msgctl); 271 272 pci_write_config_dword(dev, msi_lower_address_reg(pos), 273 msg->address_lo); 274 if (entry->msi_attrib.is_64) { 275 pci_write_config_dword(dev, msi_upper_address_reg(pos), 276 msg->address_hi); 277 pci_write_config_word(dev, msi_data_reg(pos, 1), 278 msg->data); 279 } else { 280 pci_write_config_word(dev, msi_data_reg(pos, 0), 281 msg->data); 282 } 283 } 284 entry->msg = *msg; 285} 286 287void write_msi_msg(unsigned int irq, struct msi_msg *msg) 288{ 289 struct msi_desc *entry = get_irq_msi(irq); 290 291 __write_msi_msg(entry, msg); 292} 293 294static void free_msi_irqs(struct pci_dev *dev) 295{ 296 struct msi_desc *entry, *tmp; 297 298 list_for_each_entry(entry, &dev->msi_list, list) { 299 int i, nvec; 300 if (!entry->irq) 301 continue; 302 nvec = 1 << entry->msi_attrib.multiple; 303 for (i = 0; i < nvec; i++) 304 BUG_ON(irq_has_action(entry->irq + i)); 305 } 306 307 arch_teardown_msi_irqs(dev); 308 309 list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) { 310 if (entry->msi_attrib.is_msix) { 311 if (list_is_last(&entry->list, &dev->msi_list)) 312 iounmap(entry->mask_base); 313 } 314 list_del(&entry->list); 315 kfree(entry); 316 } 317} 318 319static struct msi_desc *alloc_msi_entry(struct pci_dev *dev) 320{ 321 struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); 322 if (!desc) 323 return NULL; 324 325 INIT_LIST_HEAD(&desc->list); 326 desc->dev = dev; 327 328 return desc; 329} 330 331static void pci_intx_for_msi(struct pci_dev *dev, int enable) 332{ 333 if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG)) 334 pci_intx(dev, enable); 335} 336 337static void __pci_restore_msi_state(struct pci_dev *dev) 338{ 339 int pos; 340 u16 control; 341 struct msi_desc *entry; 342 343 if (!dev->msi_enabled) 344 return; 345 346 entry = get_irq_msi(dev->irq); 347 pos = entry->msi_attrib.pos; 348 349 pci_intx_for_msi(dev, 0); 350 msi_set_enable(dev, pos, 0); 351 write_msi_msg(dev->irq, &entry->msg); 352 353 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 354 msi_mask_irq(entry, msi_capable_mask(control), entry->masked); 355 control &= ~PCI_MSI_FLAGS_QSIZE; 356 control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; 357 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 358} 359 360static void __pci_restore_msix_state(struct pci_dev *dev) 361{ 362 int pos; 363 struct msi_desc *entry; 364 u16 control; 365 366 if (!dev->msix_enabled) 367 return; 368 BUG_ON(list_empty(&dev->msi_list)); 369 entry = list_first_entry(&dev->msi_list, struct msi_desc, list); 370 pos = entry->msi_attrib.pos; 371 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 372 373 /* route the table */ 374 pci_intx_for_msi(dev, 0); 375 control |= PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL; 376 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 377 378 list_for_each_entry(entry, &dev->msi_list, list) { 379 write_msi_msg(entry->irq, &entry->msg); 380 msix_mask_irq(entry, entry->masked); 381 } 382 383 control &= ~PCI_MSIX_FLAGS_MASKALL; 384 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 385} 386 387void pci_restore_msi_state(struct pci_dev *dev) 388{ 389 __pci_restore_msi_state(dev); 390 __pci_restore_msix_state(dev); 391} 392EXPORT_SYMBOL_GPL(pci_restore_msi_state); 393 394/** 395 * msi_capability_init - configure device's MSI capability structure 396 * @dev: pointer to the pci_dev data structure of MSI device function 397 * @nvec: number of interrupts to allocate 398 * 399 * Setup the MSI capability structure of the device with the requested 400 * number of interrupts. A return value of zero indicates the successful 401 * setup of an entry with the new MSI irq. A negative return value indicates 402 * an error, and a positive return value indicates the number of interrupts 403 * which could have been allocated. 404 */ 405static int msi_capability_init(struct pci_dev *dev, int nvec) 406{ 407 struct msi_desc *entry; 408 int pos, ret; 409 u16 control; 410 unsigned mask; 411 412 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 413 msi_set_enable(dev, pos, 0); /* Disable MSI during set up */ 414 415 pci_read_config_word(dev, msi_control_reg(pos), &control); 416 /* MSI Entry Initialization */ 417 entry = alloc_msi_entry(dev); 418 if (!entry) 419 return -ENOMEM; 420 421 entry->msi_attrib.is_msix = 0; 422 entry->msi_attrib.is_64 = is_64bit_address(control); 423 entry->msi_attrib.entry_nr = 0; 424 entry->msi_attrib.maskbit = is_mask_bit_support(control); 425 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ 426 entry->msi_attrib.pos = pos; 427 428 entry->mask_pos = msi_mask_reg(pos, entry->msi_attrib.is_64); 429 /* All MSIs are unmasked by default, Mask them all */ 430 if (entry->msi_attrib.maskbit) 431 pci_read_config_dword(dev, entry->mask_pos, &entry->masked); 432 mask = msi_capable_mask(control); 433 msi_mask_irq(entry, mask, mask); 434 435 list_add_tail(&entry->list, &dev->msi_list); 436 437 /* Configure MSI capability structure */ 438 ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); 439 if (ret) { 440 msi_mask_irq(entry, mask, ~mask); 441 free_msi_irqs(dev); 442 return ret; 443 } 444 445 /* Set MSI enabled bits */ 446 pci_intx_for_msi(dev, 0); 447 msi_set_enable(dev, pos, 1); 448 dev->msi_enabled = 1; 449 450 dev->irq = entry->irq; 451 return 0; 452} 453 454static void __iomem *msix_map_region(struct pci_dev *dev, unsigned pos, 455 unsigned nr_entries) 456{ 457 resource_size_t phys_addr; 458 u32 table_offset; 459 u8 bir; 460 461 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 462 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 463 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 464 phys_addr = pci_resource_start(dev, bir) + table_offset; 465 466 return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 467} 468 469static int msix_setup_entries(struct pci_dev *dev, unsigned pos, 470 void __iomem *base, struct msix_entry *entries, 471 int nvec) 472{ 473 struct msi_desc *entry; 474 int i; 475 476 for (i = 0; i < nvec; i++) { 477 entry = alloc_msi_entry(dev); 478 if (!entry) { 479 if (!i) 480 iounmap(base); 481 else 482 free_msi_irqs(dev); 483 /* No enough memory. Don't try again */ 484 return -ENOMEM; 485 } 486 487 entry->msi_attrib.is_msix = 1; 488 entry->msi_attrib.is_64 = 1; 489 entry->msi_attrib.entry_nr = entries[i].entry; 490 entry->msi_attrib.default_irq = dev->irq; 491 entry->msi_attrib.pos = pos; 492 entry->mask_base = base; 493 494 list_add_tail(&entry->list, &dev->msi_list); 495 } 496 497 return 0; 498} 499 500static void msix_program_entries(struct pci_dev *dev, 501 struct msix_entry *entries) 502{ 503 struct msi_desc *entry; 504 int i = 0; 505 506 list_for_each_entry(entry, &dev->msi_list, list) { 507 int offset = entries[i].entry * PCI_MSIX_ENTRY_SIZE + 508 PCI_MSIX_ENTRY_VECTOR_CTRL; 509 510 entries[i].vector = entry->irq; 511 set_irq_msi(entry->irq, entry); 512 entry->masked = readl(entry->mask_base + offset); 513 msix_mask_irq(entry, 1); 514 i++; 515 } 516} 517 518/** 519 * msix_capability_init - configure device's MSI-X capability 520 * @dev: pointer to the pci_dev data structure of MSI-X device function 521 * @entries: pointer to an array of struct msix_entry entries 522 * @nvec: number of @entries 523 * 524 * Setup the MSI-X capability structure of device function with a 525 * single MSI-X irq. A return of zero indicates the successful setup of 526 * requested MSI-X entries with allocated irqs or non-zero for otherwise. 527 **/ 528static int msix_capability_init(struct pci_dev *dev, 529 struct msix_entry *entries, int nvec) 530{ 531 int pos, ret; 532 u16 control; 533 void __iomem *base; 534 535 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 536 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 537 538 /* Ensure MSI-X is disabled while it is set up */ 539 control &= ~PCI_MSIX_FLAGS_ENABLE; 540 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 541 542 /* Request & Map MSI-X table region */ 543 base = msix_map_region(dev, pos, multi_msix_capable(control)); 544 if (!base) 545 return -ENOMEM; 546 547 ret = msix_setup_entries(dev, pos, base, entries, nvec); 548 if (ret) 549 return ret; 550 551 ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); 552 if (ret) 553 goto error; 554 555 /* 556 * Some devices require MSI-X to be enabled before we can touch the 557 * MSI-X registers. We need to mask all the vectors to prevent 558 * interrupts coming in before they're fully set up. 559 */ 560 control |= PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE; 561 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 562 563 msix_program_entries(dev, entries); 564 565 /* Set MSI-X enabled bits and unmask the function */ 566 pci_intx_for_msi(dev, 0); 567 dev->msix_enabled = 1; 568 569 control &= ~PCI_MSIX_FLAGS_MASKALL; 570 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 571 572 return 0; 573 574error: 575 if (ret < 0) { 576 /* 577 * If we had some success, report the number of irqs 578 * we succeeded in setting up. 579 */ 580 struct msi_desc *entry; 581 int avail = 0; 582 583 list_for_each_entry(entry, &dev->msi_list, list) { 584 if (entry->irq != 0) 585 avail++; 586 } 587 if (avail != 0) 588 ret = avail; 589 } 590 591 free_msi_irqs(dev); 592 593 return ret; 594} 595 596/** 597 * pci_msi_check_device - check whether MSI may be enabled on a device 598 * @dev: pointer to the pci_dev data structure of MSI device function 599 * @nvec: how many MSIs have been requested ? 600 * @type: are we checking for MSI or MSI-X ? 601 * 602 * Look at global flags, the device itself, and its parent busses 603 * to determine if MSI/-X are supported for the device. If MSI/-X is 604 * supported return 0, else return an error code. 605 **/ 606static int pci_msi_check_device(struct pci_dev *dev, int nvec, int type) 607{ 608 struct pci_bus *bus; 609 int ret; 610 611 /* MSI must be globally enabled and supported by the device */ 612 if (!pci_msi_enable || !dev || dev->no_msi) 613 return -EINVAL; 614 615 /* 616 * You can't ask to have 0 or less MSIs configured. 617 * a) it's stupid .. 618 * b) the list manipulation code assumes nvec >= 1. 619 */ 620 if (nvec < 1) 621 return -ERANGE; 622 623 /* 624 * Any bridge which does NOT route MSI transactions from its 625 * secondary bus to its primary bus must set NO_MSI flag on 626 * the secondary pci_bus. 627 * We expect only arch-specific PCI host bus controller driver 628 * or quirks for specific PCI bridges to be setting NO_MSI. 629 */ 630 for (bus = dev->bus; bus; bus = bus->parent) 631 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 632 return -EINVAL; 633 634 ret = arch_msi_check_device(dev, nvec, type); 635 if (ret) 636 return ret; 637 638 if (!pci_find_capability(dev, type)) 639 return -EINVAL; 640 641 return 0; 642} 643 644/** 645 * pci_enable_msi_block - configure device's MSI capability structure 646 * @dev: device to configure 647 * @nvec: number of interrupts to configure 648 * 649 * Allocate IRQs for a device with the MSI capability. 650 * This function returns a negative errno if an error occurs. If it 651 * is unable to allocate the number of interrupts requested, it returns 652 * the number of interrupts it might be able to allocate. If it successfully 653 * allocates at least the number of interrupts requested, it returns 0 and 654 * updates the @dev's irq member to the lowest new interrupt number; the 655 * other interrupt numbers allocated to this device are consecutive. 656 */ 657int pci_enable_msi_block(struct pci_dev *dev, unsigned int nvec) 658{ 659 int status, pos, maxvec; 660 u16 msgctl; 661 662 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 663 if (!pos) 664 return -EINVAL; 665 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); 666 maxvec = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1); 667 if (nvec > maxvec) 668 return maxvec; 669 670 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSI); 671 if (status) 672 return status; 673 674 WARN_ON(!!dev->msi_enabled); 675 676 /* Check whether driver already requested MSI-X irqs */ 677 if (dev->msix_enabled) { 678 dev_info(&dev->dev, "can't enable MSI " 679 "(MSI-X already enabled)\n"); 680 return -EINVAL; 681 } 682 683 status = msi_capability_init(dev, nvec); 684 return status; 685} 686EXPORT_SYMBOL(pci_enable_msi_block); 687 688void pci_msi_shutdown(struct pci_dev *dev) 689{ 690 struct msi_desc *desc; 691 u32 mask; 692 u16 ctrl; 693 unsigned pos; 694 695 if (!pci_msi_enable || !dev || !dev->msi_enabled) 696 return; 697 698 BUG_ON(list_empty(&dev->msi_list)); 699 desc = list_first_entry(&dev->msi_list, struct msi_desc, list); 700 pos = desc->msi_attrib.pos; 701 702 msi_set_enable(dev, pos, 0); 703 pci_intx_for_msi(dev, 1); 704 dev->msi_enabled = 0; 705 706 /* Return the device with MSI unmasked as initial states */ 707 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &ctrl); 708 mask = msi_capable_mask(ctrl); 709 /* Keep cached state to be restored */ 710 __msi_mask_irq(desc, mask, ~mask); 711 712 /* Restore dev->irq to its default pin-assertion irq */ 713 dev->irq = desc->msi_attrib.default_irq; 714} 715 716void pci_disable_msi(struct pci_dev *dev) 717{ 718 if (!pci_msi_enable || !dev || !dev->msi_enabled) 719 return; 720 721 pci_msi_shutdown(dev); 722 free_msi_irqs(dev); 723} 724EXPORT_SYMBOL(pci_disable_msi); 725 726/** 727 * pci_msix_table_size - return the number of device's MSI-X table entries 728 * @dev: pointer to the pci_dev data structure of MSI-X device function 729 */ 730int pci_msix_table_size(struct pci_dev *dev) 731{ 732 int pos; 733 u16 control; 734 735 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 736 if (!pos) 737 return 0; 738 739 pci_read_config_word(dev, msi_control_reg(pos), &control); 740 return multi_msix_capable(control); 741} 742 743/** 744 * pci_enable_msix - configure device's MSI-X capability structure 745 * @dev: pointer to the pci_dev data structure of MSI-X device function 746 * @entries: pointer to an array of MSI-X entries 747 * @nvec: number of MSI-X irqs requested for allocation by device driver 748 * 749 * Setup the MSI-X capability structure of device function with the number 750 * of requested irqs upon its software driver call to request for 751 * MSI-X mode enabled on its hardware device function. A return of zero 752 * indicates the successful configuration of MSI-X capability structure 753 * with new allocated MSI-X irqs. A return of < 0 indicates a failure. 754 * Or a return of > 0 indicates that driver request is exceeding the number 755 * of irqs or MSI-X vectors available. Driver should use the returned value to 756 * re-send its request. 757 **/ 758int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec) 759{ 760 int status, nr_entries; 761 int i, j; 762 763 if (!entries) 764 return -EINVAL; 765 766 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX); 767 if (status) 768 return status; 769 770 nr_entries = pci_msix_table_size(dev); 771 if (nvec > nr_entries) 772 return nr_entries; 773 774 /* Check for any invalid entries */ 775 for (i = 0; i < nvec; i++) { 776 if (entries[i].entry >= nr_entries) 777 return -EINVAL; /* invalid entry */ 778 for (j = i + 1; j < nvec; j++) { 779 if (entries[i].entry == entries[j].entry) 780 return -EINVAL; /* duplicate entry */ 781 } 782 } 783 WARN_ON(!!dev->msix_enabled); 784 785 /* Check whether driver already requested for MSI irq */ 786 if (dev->msi_enabled) { 787 dev_info(&dev->dev, "can't enable MSI-X " 788 "(MSI IRQ already assigned)\n"); 789 return -EINVAL; 790 } 791 status = msix_capability_init(dev, entries, nvec); 792 return status; 793} 794EXPORT_SYMBOL(pci_enable_msix); 795 796void pci_msix_shutdown(struct pci_dev *dev) 797{ 798 struct msi_desc *entry; 799 800 if (!pci_msi_enable || !dev || !dev->msix_enabled) 801 return; 802 803 /* Return the device with MSI-X masked as initial states */ 804 list_for_each_entry(entry, &dev->msi_list, list) { 805 /* Keep cached states to be restored */ 806 __msix_mask_irq(entry, 1); 807 } 808 809 msix_set_enable(dev, 0); 810 pci_intx_for_msi(dev, 1); 811 dev->msix_enabled = 0; 812} 813 814void pci_disable_msix(struct pci_dev *dev) 815{ 816 if (!pci_msi_enable || !dev || !dev->msix_enabled) 817 return; 818 819 pci_msix_shutdown(dev); 820 free_msi_irqs(dev); 821} 822EXPORT_SYMBOL(pci_disable_msix); 823 824/** 825 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state 826 * @dev: pointer to the pci_dev data structure of MSI(X) device function 827 * 828 * Being called during hotplug remove, from which the device function 829 * is hot-removed. All previous assigned MSI/MSI-X irqs, if 830 * allocated for this device function, are reclaimed to unused state, 831 * which may be used later on. 832 **/ 833void msi_remove_pci_irq_vectors(struct pci_dev *dev) 834{ 835 if (!pci_msi_enable || !dev) 836 return; 837 838 if (dev->msi_enabled || dev->msix_enabled) 839 free_msi_irqs(dev); 840} 841 842void pci_no_msi(void) 843{ 844 pci_msi_enable = 0; 845} 846 847/** 848 * pci_msi_enabled - is MSI enabled? 849 * 850 * Returns true if MSI has not been disabled by the command-line option 851 * pci=nomsi. 852 **/ 853int pci_msi_enabled(void) 854{ 855 return pci_msi_enable; 856} 857EXPORT_SYMBOL(pci_msi_enabled); 858 859void pci_msi_init_pci_dev(struct pci_dev *dev) 860{ 861 INIT_LIST_HEAD(&dev->msi_list); 862} 863