msi.c revision 30da55242818a8ca08583188ebcbaccd283ad4d9
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/err.h> 10#include <linux/mm.h> 11#include <linux/irq.h> 12#include <linux/interrupt.h> 13#include <linux/init.h> 14#include <linux/ioport.h> 15#include <linux/pci.h> 16#include <linux/proc_fs.h> 17#include <linux/msi.h> 18#include <linux/smp.h> 19#include <linux/errno.h> 20#include <linux/io.h> 21#include <linux/slab.h> 22 23#include "pci.h" 24#include "msi.h" 25 26static int pci_msi_enable = 1; 27 28/* Arch hooks */ 29 30#ifndef arch_msi_check_device 31int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) 32{ 33 return 0; 34} 35#endif 36 37#ifndef arch_setup_msi_irqs 38int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) 39{ 40 struct msi_desc *entry; 41 int ret; 42 43 /* 44 * If an architecture wants to support multiple MSI, it needs to 45 * override arch_setup_msi_irqs() 46 */ 47 if (type == PCI_CAP_ID_MSI && nvec > 1) 48 return 1; 49 50 list_for_each_entry(entry, &dev->msi_list, list) { 51 ret = arch_setup_msi_irq(dev, entry); 52 if (ret < 0) 53 return ret; 54 if (ret > 0) 55 return -ENOSPC; 56 } 57 58 return 0; 59} 60#endif 61 62#ifndef arch_teardown_msi_irqs 63void arch_teardown_msi_irqs(struct pci_dev *dev) 64{ 65 struct msi_desc *entry; 66 67 list_for_each_entry(entry, &dev->msi_list, list) { 68 int i, nvec; 69 if (entry->irq == 0) 70 continue; 71 nvec = 1 << entry->msi_attrib.multiple; 72 for (i = 0; i < nvec; i++) 73 arch_teardown_msi_irq(entry->irq + i); 74 } 75} 76#endif 77 78static void msi_set_enable(struct pci_dev *dev, int pos, int enable) 79{ 80 u16 control; 81 82 BUG_ON(!pos); 83 84 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 85 control &= ~PCI_MSI_FLAGS_ENABLE; 86 if (enable) 87 control |= PCI_MSI_FLAGS_ENABLE; 88 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 89} 90 91static void msix_set_enable(struct pci_dev *dev, int enable) 92{ 93 int pos; 94 u16 control; 95 96 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 97 if (pos) { 98 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 99 control &= ~PCI_MSIX_FLAGS_ENABLE; 100 if (enable) 101 control |= PCI_MSIX_FLAGS_ENABLE; 102 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 103 } 104} 105 106static inline __attribute_const__ u32 msi_mask(unsigned x) 107{ 108 /* Don't shift by >= width of type */ 109 if (x >= 5) 110 return 0xffffffff; 111 return (1 << (1 << x)) - 1; 112} 113 114static inline __attribute_const__ u32 msi_capable_mask(u16 control) 115{ 116 return msi_mask((control >> 1) & 7); 117} 118 119static inline __attribute_const__ u32 msi_enabled_mask(u16 control) 120{ 121 return msi_mask((control >> 4) & 7); 122} 123 124/* 125 * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to 126 * mask all MSI interrupts by clearing the MSI enable bit does not work 127 * reliably as devices without an INTx disable bit will then generate a 128 * level IRQ which will never be cleared. 129 */ 130static u32 __msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) 131{ 132 u32 mask_bits = desc->masked; 133 134 if (!desc->msi_attrib.maskbit) 135 return 0; 136 137 mask_bits &= ~mask; 138 mask_bits |= flag; 139 pci_write_config_dword(desc->dev, desc->mask_pos, mask_bits); 140 141 return mask_bits; 142} 143 144static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) 145{ 146 desc->masked = __msi_mask_irq(desc, mask, flag); 147} 148 149/* 150 * This internal function does not flush PCI writes to the device. 151 * All users must ensure that they read from the device before either 152 * assuming that the device state is up to date, or returning out of this 153 * file. This saves a few milliseconds when initialising devices with lots 154 * of MSI-X interrupts. 155 */ 156static u32 __msix_mask_irq(struct msi_desc *desc, u32 flag) 157{ 158 u32 mask_bits = desc->masked; 159 unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 160 PCI_MSIX_ENTRY_VECTOR_CTRL; 161 mask_bits &= ~1; 162 mask_bits |= flag; 163 writel(mask_bits, desc->mask_base + offset); 164 165 return mask_bits; 166} 167 168static void msix_mask_irq(struct msi_desc *desc, u32 flag) 169{ 170 desc->masked = __msix_mask_irq(desc, flag); 171} 172 173static void msi_set_mask_bit(unsigned irq, u32 flag) 174{ 175 struct msi_desc *desc = get_irq_msi(irq); 176 177 if (desc->msi_attrib.is_msix) { 178 msix_mask_irq(desc, flag); 179 readl(desc->mask_base); /* Flush write to device */ 180 } else { 181 unsigned offset = irq - desc->dev->irq; 182 msi_mask_irq(desc, 1 << offset, flag << offset); 183 } 184} 185 186void mask_msi_irq(unsigned int irq) 187{ 188 msi_set_mask_bit(irq, 1); 189} 190 191void unmask_msi_irq(unsigned int irq) 192{ 193 msi_set_mask_bit(irq, 0); 194} 195 196void read_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg) 197{ 198 struct msi_desc *entry = get_irq_desc_msi(desc); 199 200 BUG_ON(entry->dev->current_state != PCI_D0); 201 202 if (entry->msi_attrib.is_msix) { 203 void __iomem *base = entry->mask_base + 204 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 205 206 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); 207 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); 208 msg->data = readl(base + PCI_MSIX_ENTRY_DATA); 209 } else { 210 struct pci_dev *dev = entry->dev; 211 int pos = entry->msi_attrib.pos; 212 u16 data; 213 214 pci_read_config_dword(dev, msi_lower_address_reg(pos), 215 &msg->address_lo); 216 if (entry->msi_attrib.is_64) { 217 pci_read_config_dword(dev, msi_upper_address_reg(pos), 218 &msg->address_hi); 219 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 220 } else { 221 msg->address_hi = 0; 222 pci_read_config_word(dev, msi_data_reg(pos, 0), &data); 223 } 224 msg->data = data; 225 } 226} 227 228void read_msi_msg(unsigned int irq, struct msi_msg *msg) 229{ 230 struct irq_desc *desc = irq_to_desc(irq); 231 232 read_msi_msg_desc(desc, msg); 233} 234 235void get_cached_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg) 236{ 237 struct msi_desc *entry = get_irq_desc_msi(desc); 238 239 /* Assert that the cache is valid, assuming that 240 * valid messages are not all-zeroes. */ 241 BUG_ON(!(entry->msg.address_hi | entry->msg.address_lo | 242 entry->msg.data)); 243 244 *msg = entry->msg; 245} 246 247void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) 248{ 249 struct irq_desc *desc = irq_to_desc(irq); 250 251 get_cached_msi_msg_desc(desc, msg); 252} 253 254void write_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg) 255{ 256 struct msi_desc *entry = get_irq_desc_msi(desc); 257 258 if (entry->dev->current_state != PCI_D0) { 259 /* Don't touch the hardware now */ 260 } else if (entry->msi_attrib.is_msix) { 261 void __iomem *base; 262 base = entry->mask_base + 263 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 264 265 writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); 266 writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); 267 writel(msg->data, base + PCI_MSIX_ENTRY_DATA); 268 } else { 269 struct pci_dev *dev = entry->dev; 270 int pos = entry->msi_attrib.pos; 271 u16 msgctl; 272 273 pci_read_config_word(dev, msi_control_reg(pos), &msgctl); 274 msgctl &= ~PCI_MSI_FLAGS_QSIZE; 275 msgctl |= entry->msi_attrib.multiple << 4; 276 pci_write_config_word(dev, msi_control_reg(pos), msgctl); 277 278 pci_write_config_dword(dev, msi_lower_address_reg(pos), 279 msg->address_lo); 280 if (entry->msi_attrib.is_64) { 281 pci_write_config_dword(dev, msi_upper_address_reg(pos), 282 msg->address_hi); 283 pci_write_config_word(dev, msi_data_reg(pos, 1), 284 msg->data); 285 } else { 286 pci_write_config_word(dev, msi_data_reg(pos, 0), 287 msg->data); 288 } 289 } 290 entry->msg = *msg; 291} 292 293void write_msi_msg(unsigned int irq, struct msi_msg *msg) 294{ 295 struct irq_desc *desc = irq_to_desc(irq); 296 297 write_msi_msg_desc(desc, msg); 298} 299 300static void free_msi_irqs(struct pci_dev *dev) 301{ 302 struct msi_desc *entry, *tmp; 303 304 list_for_each_entry(entry, &dev->msi_list, list) { 305 int i, nvec; 306 if (!entry->irq) 307 continue; 308 nvec = 1 << entry->msi_attrib.multiple; 309 for (i = 0; i < nvec; i++) 310 BUG_ON(irq_has_action(entry->irq + i)); 311 } 312 313 arch_teardown_msi_irqs(dev); 314 315 list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) { 316 if (entry->msi_attrib.is_msix) { 317 if (list_is_last(&entry->list, &dev->msi_list)) 318 iounmap(entry->mask_base); 319 } 320 list_del(&entry->list); 321 kfree(entry); 322 } 323} 324 325static struct msi_desc *alloc_msi_entry(struct pci_dev *dev) 326{ 327 struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); 328 if (!desc) 329 return NULL; 330 331 INIT_LIST_HEAD(&desc->list); 332 desc->dev = dev; 333 334 return desc; 335} 336 337static void pci_intx_for_msi(struct pci_dev *dev, int enable) 338{ 339 if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG)) 340 pci_intx(dev, enable); 341} 342 343static void __pci_restore_msi_state(struct pci_dev *dev) 344{ 345 int pos; 346 u16 control; 347 struct msi_desc *entry; 348 349 if (!dev->msi_enabled) 350 return; 351 352 entry = get_irq_msi(dev->irq); 353 pos = entry->msi_attrib.pos; 354 355 pci_intx_for_msi(dev, 0); 356 msi_set_enable(dev, pos, 0); 357 write_msi_msg(dev->irq, &entry->msg); 358 359 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 360 msi_mask_irq(entry, msi_capable_mask(control), entry->masked); 361 control &= ~PCI_MSI_FLAGS_QSIZE; 362 control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; 363 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 364} 365 366static void __pci_restore_msix_state(struct pci_dev *dev) 367{ 368 int pos; 369 struct msi_desc *entry; 370 u16 control; 371 372 if (!dev->msix_enabled) 373 return; 374 BUG_ON(list_empty(&dev->msi_list)); 375 entry = list_first_entry(&dev->msi_list, struct msi_desc, list); 376 pos = entry->msi_attrib.pos; 377 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 378 379 /* route the table */ 380 pci_intx_for_msi(dev, 0); 381 control |= PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL; 382 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 383 384 list_for_each_entry(entry, &dev->msi_list, list) { 385 write_msi_msg(entry->irq, &entry->msg); 386 msix_mask_irq(entry, entry->masked); 387 } 388 389 control &= ~PCI_MSIX_FLAGS_MASKALL; 390 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 391} 392 393void pci_restore_msi_state(struct pci_dev *dev) 394{ 395 __pci_restore_msi_state(dev); 396 __pci_restore_msix_state(dev); 397} 398EXPORT_SYMBOL_GPL(pci_restore_msi_state); 399 400/** 401 * msi_capability_init - configure device's MSI capability structure 402 * @dev: pointer to the pci_dev data structure of MSI device function 403 * @nvec: number of interrupts to allocate 404 * 405 * Setup the MSI capability structure of the device with the requested 406 * number of interrupts. A return value of zero indicates the successful 407 * setup of an entry with the new MSI irq. A negative return value indicates 408 * an error, and a positive return value indicates the number of interrupts 409 * which could have been allocated. 410 */ 411static int msi_capability_init(struct pci_dev *dev, int nvec) 412{ 413 struct msi_desc *entry; 414 int pos, ret; 415 u16 control; 416 unsigned mask; 417 418 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 419 msi_set_enable(dev, pos, 0); /* Disable MSI during set up */ 420 421 pci_read_config_word(dev, msi_control_reg(pos), &control); 422 /* MSI Entry Initialization */ 423 entry = alloc_msi_entry(dev); 424 if (!entry) 425 return -ENOMEM; 426 427 entry->msi_attrib.is_msix = 0; 428 entry->msi_attrib.is_64 = is_64bit_address(control); 429 entry->msi_attrib.entry_nr = 0; 430 entry->msi_attrib.maskbit = is_mask_bit_support(control); 431 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ 432 entry->msi_attrib.pos = pos; 433 434 entry->mask_pos = msi_mask_reg(pos, entry->msi_attrib.is_64); 435 /* All MSIs are unmasked by default, Mask them all */ 436 if (entry->msi_attrib.maskbit) 437 pci_read_config_dword(dev, entry->mask_pos, &entry->masked); 438 mask = msi_capable_mask(control); 439 msi_mask_irq(entry, mask, mask); 440 441 list_add_tail(&entry->list, &dev->msi_list); 442 443 /* Configure MSI capability structure */ 444 ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); 445 if (ret) { 446 msi_mask_irq(entry, mask, ~mask); 447 free_msi_irqs(dev); 448 return ret; 449 } 450 451 /* Set MSI enabled bits */ 452 pci_intx_for_msi(dev, 0); 453 msi_set_enable(dev, pos, 1); 454 dev->msi_enabled = 1; 455 456 dev->irq = entry->irq; 457 return 0; 458} 459 460static void __iomem *msix_map_region(struct pci_dev *dev, unsigned pos, 461 unsigned nr_entries) 462{ 463 resource_size_t phys_addr; 464 u32 table_offset; 465 u8 bir; 466 467 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 468 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 469 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 470 phys_addr = pci_resource_start(dev, bir) + table_offset; 471 472 return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 473} 474 475static int msix_setup_entries(struct pci_dev *dev, unsigned pos, 476 void __iomem *base, struct msix_entry *entries, 477 int nvec) 478{ 479 struct msi_desc *entry; 480 int i; 481 482 for (i = 0; i < nvec; i++) { 483 entry = alloc_msi_entry(dev); 484 if (!entry) { 485 if (!i) 486 iounmap(base); 487 else 488 free_msi_irqs(dev); 489 /* No enough memory. Don't try again */ 490 return -ENOMEM; 491 } 492 493 entry->msi_attrib.is_msix = 1; 494 entry->msi_attrib.is_64 = 1; 495 entry->msi_attrib.entry_nr = entries[i].entry; 496 entry->msi_attrib.default_irq = dev->irq; 497 entry->msi_attrib.pos = pos; 498 entry->mask_base = base; 499 500 list_add_tail(&entry->list, &dev->msi_list); 501 } 502 503 return 0; 504} 505 506static void msix_program_entries(struct pci_dev *dev, 507 struct msix_entry *entries) 508{ 509 struct msi_desc *entry; 510 int i = 0; 511 512 list_for_each_entry(entry, &dev->msi_list, list) { 513 int offset = entries[i].entry * PCI_MSIX_ENTRY_SIZE + 514 PCI_MSIX_ENTRY_VECTOR_CTRL; 515 516 entries[i].vector = entry->irq; 517 set_irq_msi(entry->irq, entry); 518 entry->masked = readl(entry->mask_base + offset); 519 msix_mask_irq(entry, 1); 520 i++; 521 } 522} 523 524/** 525 * msix_capability_init - configure device's MSI-X capability 526 * @dev: pointer to the pci_dev data structure of MSI-X device function 527 * @entries: pointer to an array of struct msix_entry entries 528 * @nvec: number of @entries 529 * 530 * Setup the MSI-X capability structure of device function with a 531 * single MSI-X irq. A return of zero indicates the successful setup of 532 * requested MSI-X entries with allocated irqs or non-zero for otherwise. 533 **/ 534static int msix_capability_init(struct pci_dev *dev, 535 struct msix_entry *entries, int nvec) 536{ 537 int pos, ret; 538 u16 control; 539 void __iomem *base; 540 541 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 542 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 543 544 /* Ensure MSI-X is disabled while it is set up */ 545 control &= ~PCI_MSIX_FLAGS_ENABLE; 546 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 547 548 /* Request & Map MSI-X table region */ 549 base = msix_map_region(dev, pos, multi_msix_capable(control)); 550 if (!base) 551 return -ENOMEM; 552 553 ret = msix_setup_entries(dev, pos, base, entries, nvec); 554 if (ret) 555 return ret; 556 557 ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); 558 if (ret) 559 goto error; 560 561 /* 562 * Some devices require MSI-X to be enabled before we can touch the 563 * MSI-X registers. We need to mask all the vectors to prevent 564 * interrupts coming in before they're fully set up. 565 */ 566 control |= PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE; 567 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 568 569 msix_program_entries(dev, entries); 570 571 /* Set MSI-X enabled bits and unmask the function */ 572 pci_intx_for_msi(dev, 0); 573 dev->msix_enabled = 1; 574 575 control &= ~PCI_MSIX_FLAGS_MASKALL; 576 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 577 578 return 0; 579 580error: 581 if (ret < 0) { 582 /* 583 * If we had some success, report the number of irqs 584 * we succeeded in setting up. 585 */ 586 struct msi_desc *entry; 587 int avail = 0; 588 589 list_for_each_entry(entry, &dev->msi_list, list) { 590 if (entry->irq != 0) 591 avail++; 592 } 593 if (avail != 0) 594 ret = avail; 595 } 596 597 free_msi_irqs(dev); 598 599 return ret; 600} 601 602/** 603 * pci_msi_check_device - check whether MSI may be enabled on a device 604 * @dev: pointer to the pci_dev data structure of MSI device function 605 * @nvec: how many MSIs have been requested ? 606 * @type: are we checking for MSI or MSI-X ? 607 * 608 * Look at global flags, the device itself, and its parent busses 609 * to determine if MSI/-X are supported for the device. If MSI/-X is 610 * supported return 0, else return an error code. 611 **/ 612static int pci_msi_check_device(struct pci_dev *dev, int nvec, int type) 613{ 614 struct pci_bus *bus; 615 int ret; 616 617 /* MSI must be globally enabled and supported by the device */ 618 if (!pci_msi_enable || !dev || dev->no_msi) 619 return -EINVAL; 620 621 /* 622 * You can't ask to have 0 or less MSIs configured. 623 * a) it's stupid .. 624 * b) the list manipulation code assumes nvec >= 1. 625 */ 626 if (nvec < 1) 627 return -ERANGE; 628 629 /* 630 * Any bridge which does NOT route MSI transactions from its 631 * secondary bus to its primary bus must set NO_MSI flag on 632 * the secondary pci_bus. 633 * We expect only arch-specific PCI host bus controller driver 634 * or quirks for specific PCI bridges to be setting NO_MSI. 635 */ 636 for (bus = dev->bus; bus; bus = bus->parent) 637 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 638 return -EINVAL; 639 640 ret = arch_msi_check_device(dev, nvec, type); 641 if (ret) 642 return ret; 643 644 if (!pci_find_capability(dev, type)) 645 return -EINVAL; 646 647 return 0; 648} 649 650/** 651 * pci_enable_msi_block - configure device's MSI capability structure 652 * @dev: device to configure 653 * @nvec: number of interrupts to configure 654 * 655 * Allocate IRQs for a device with the MSI capability. 656 * This function returns a negative errno if an error occurs. If it 657 * is unable to allocate the number of interrupts requested, it returns 658 * the number of interrupts it might be able to allocate. If it successfully 659 * allocates at least the number of interrupts requested, it returns 0 and 660 * updates the @dev's irq member to the lowest new interrupt number; the 661 * other interrupt numbers allocated to this device are consecutive. 662 */ 663int pci_enable_msi_block(struct pci_dev *dev, unsigned int nvec) 664{ 665 int status, pos, maxvec; 666 u16 msgctl; 667 668 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 669 if (!pos) 670 return -EINVAL; 671 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); 672 maxvec = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1); 673 if (nvec > maxvec) 674 return maxvec; 675 676 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSI); 677 if (status) 678 return status; 679 680 WARN_ON(!!dev->msi_enabled); 681 682 /* Check whether driver already requested MSI-X irqs */ 683 if (dev->msix_enabled) { 684 dev_info(&dev->dev, "can't enable MSI " 685 "(MSI-X already enabled)\n"); 686 return -EINVAL; 687 } 688 689 status = msi_capability_init(dev, nvec); 690 return status; 691} 692EXPORT_SYMBOL(pci_enable_msi_block); 693 694void pci_msi_shutdown(struct pci_dev *dev) 695{ 696 struct msi_desc *desc; 697 u32 mask; 698 u16 ctrl; 699 unsigned pos; 700 701 if (!pci_msi_enable || !dev || !dev->msi_enabled) 702 return; 703 704 BUG_ON(list_empty(&dev->msi_list)); 705 desc = list_first_entry(&dev->msi_list, struct msi_desc, list); 706 pos = desc->msi_attrib.pos; 707 708 msi_set_enable(dev, pos, 0); 709 pci_intx_for_msi(dev, 1); 710 dev->msi_enabled = 0; 711 712 /* Return the device with MSI unmasked as initial states */ 713 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &ctrl); 714 mask = msi_capable_mask(ctrl); 715 /* Keep cached state to be restored */ 716 __msi_mask_irq(desc, mask, ~mask); 717 718 /* Restore dev->irq to its default pin-assertion irq */ 719 dev->irq = desc->msi_attrib.default_irq; 720} 721 722void pci_disable_msi(struct pci_dev *dev) 723{ 724 if (!pci_msi_enable || !dev || !dev->msi_enabled) 725 return; 726 727 pci_msi_shutdown(dev); 728 free_msi_irqs(dev); 729} 730EXPORT_SYMBOL(pci_disable_msi); 731 732/** 733 * pci_msix_table_size - return the number of device's MSI-X table entries 734 * @dev: pointer to the pci_dev data structure of MSI-X device function 735 */ 736int pci_msix_table_size(struct pci_dev *dev) 737{ 738 int pos; 739 u16 control; 740 741 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 742 if (!pos) 743 return 0; 744 745 pci_read_config_word(dev, msi_control_reg(pos), &control); 746 return multi_msix_capable(control); 747} 748 749/** 750 * pci_enable_msix - configure device's MSI-X capability structure 751 * @dev: pointer to the pci_dev data structure of MSI-X device function 752 * @entries: pointer to an array of MSI-X entries 753 * @nvec: number of MSI-X irqs requested for allocation by device driver 754 * 755 * Setup the MSI-X capability structure of device function with the number 756 * of requested irqs upon its software driver call to request for 757 * MSI-X mode enabled on its hardware device function. A return of zero 758 * indicates the successful configuration of MSI-X capability structure 759 * with new allocated MSI-X irqs. A return of < 0 indicates a failure. 760 * Or a return of > 0 indicates that driver request is exceeding the number 761 * of irqs or MSI-X vectors available. Driver should use the returned value to 762 * re-send its request. 763 **/ 764int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec) 765{ 766 int status, nr_entries; 767 int i, j; 768 769 if (!entries) 770 return -EINVAL; 771 772 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX); 773 if (status) 774 return status; 775 776 nr_entries = pci_msix_table_size(dev); 777 if (nvec > nr_entries) 778 return nr_entries; 779 780 /* Check for any invalid entries */ 781 for (i = 0; i < nvec; i++) { 782 if (entries[i].entry >= nr_entries) 783 return -EINVAL; /* invalid entry */ 784 for (j = i + 1; j < nvec; j++) { 785 if (entries[i].entry == entries[j].entry) 786 return -EINVAL; /* duplicate entry */ 787 } 788 } 789 WARN_ON(!!dev->msix_enabled); 790 791 /* Check whether driver already requested for MSI irq */ 792 if (dev->msi_enabled) { 793 dev_info(&dev->dev, "can't enable MSI-X " 794 "(MSI IRQ already assigned)\n"); 795 return -EINVAL; 796 } 797 status = msix_capability_init(dev, entries, nvec); 798 return status; 799} 800EXPORT_SYMBOL(pci_enable_msix); 801 802void pci_msix_shutdown(struct pci_dev *dev) 803{ 804 struct msi_desc *entry; 805 806 if (!pci_msi_enable || !dev || !dev->msix_enabled) 807 return; 808 809 /* Return the device with MSI-X masked as initial states */ 810 list_for_each_entry(entry, &dev->msi_list, list) { 811 /* Keep cached states to be restored */ 812 __msix_mask_irq(entry, 1); 813 } 814 815 msix_set_enable(dev, 0); 816 pci_intx_for_msi(dev, 1); 817 dev->msix_enabled = 0; 818} 819 820void pci_disable_msix(struct pci_dev *dev) 821{ 822 if (!pci_msi_enable || !dev || !dev->msix_enabled) 823 return; 824 825 pci_msix_shutdown(dev); 826 free_msi_irqs(dev); 827} 828EXPORT_SYMBOL(pci_disable_msix); 829 830/** 831 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state 832 * @dev: pointer to the pci_dev data structure of MSI(X) device function 833 * 834 * Being called during hotplug remove, from which the device function 835 * is hot-removed. All previous assigned MSI/MSI-X irqs, if 836 * allocated for this device function, are reclaimed to unused state, 837 * which may be used later on. 838 **/ 839void msi_remove_pci_irq_vectors(struct pci_dev *dev) 840{ 841 if (!pci_msi_enable || !dev) 842 return; 843 844 if (dev->msi_enabled || dev->msix_enabled) 845 free_msi_irqs(dev); 846} 847 848void pci_no_msi(void) 849{ 850 pci_msi_enable = 0; 851} 852 853/** 854 * pci_msi_enabled - is MSI enabled? 855 * 856 * Returns true if MSI has not been disabled by the command-line option 857 * pci=nomsi. 858 **/ 859int pci_msi_enabled(void) 860{ 861 return pci_msi_enable; 862} 863EXPORT_SYMBOL(pci_msi_enabled); 864 865void pci_msi_init_pci_dev(struct pci_dev *dev) 866{ 867 INIT_LIST_HEAD(&dev->msi_list); 868} 869