msi.c revision dced35aeb0367dda2636ee9ee914bda14510dcc9
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/err.h> 10#include <linux/mm.h> 11#include <linux/irq.h> 12#include <linux/interrupt.h> 13#include <linux/init.h> 14#include <linux/ioport.h> 15#include <linux/pci.h> 16#include <linux/proc_fs.h> 17#include <linux/msi.h> 18#include <linux/smp.h> 19#include <linux/errno.h> 20#include <linux/io.h> 21#include <linux/slab.h> 22 23#include "pci.h" 24#include "msi.h" 25 26static int pci_msi_enable = 1; 27 28/* Arch hooks */ 29 30#ifndef arch_msi_check_device 31int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) 32{ 33 return 0; 34} 35#endif 36 37#ifndef arch_setup_msi_irqs 38# define arch_setup_msi_irqs default_setup_msi_irqs 39# define HAVE_DEFAULT_MSI_SETUP_IRQS 40#endif 41 42#ifdef HAVE_DEFAULT_MSI_SETUP_IRQS 43int default_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) 44{ 45 struct msi_desc *entry; 46 int ret; 47 48 /* 49 * If an architecture wants to support multiple MSI, it needs to 50 * override arch_setup_msi_irqs() 51 */ 52 if (type == PCI_CAP_ID_MSI && nvec > 1) 53 return 1; 54 55 list_for_each_entry(entry, &dev->msi_list, list) { 56 ret = arch_setup_msi_irq(dev, entry); 57 if (ret < 0) 58 return ret; 59 if (ret > 0) 60 return -ENOSPC; 61 } 62 63 return 0; 64} 65#endif 66 67#ifndef arch_teardown_msi_irqs 68# define arch_teardown_msi_irqs default_teardown_msi_irqs 69# define HAVE_DEFAULT_MSI_TEARDOWN_IRQS 70#endif 71 72#ifdef HAVE_DEFAULT_MSI_TEARDOWN_IRQS 73void default_teardown_msi_irqs(struct pci_dev *dev) 74{ 75 struct msi_desc *entry; 76 77 list_for_each_entry(entry, &dev->msi_list, list) { 78 int i, nvec; 79 if (entry->irq == 0) 80 continue; 81 nvec = 1 << entry->msi_attrib.multiple; 82 for (i = 0; i < nvec; i++) 83 arch_teardown_msi_irq(entry->irq + i); 84 } 85} 86#endif 87 88static void msi_set_enable(struct pci_dev *dev, int pos, int enable) 89{ 90 u16 control; 91 92 BUG_ON(!pos); 93 94 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 95 control &= ~PCI_MSI_FLAGS_ENABLE; 96 if (enable) 97 control |= PCI_MSI_FLAGS_ENABLE; 98 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 99} 100 101static void msix_set_enable(struct pci_dev *dev, int enable) 102{ 103 int pos; 104 u16 control; 105 106 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 107 if (pos) { 108 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 109 control &= ~PCI_MSIX_FLAGS_ENABLE; 110 if (enable) 111 control |= PCI_MSIX_FLAGS_ENABLE; 112 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 113 } 114} 115 116static inline __attribute_const__ u32 msi_mask(unsigned x) 117{ 118 /* Don't shift by >= width of type */ 119 if (x >= 5) 120 return 0xffffffff; 121 return (1 << (1 << x)) - 1; 122} 123 124static inline __attribute_const__ u32 msi_capable_mask(u16 control) 125{ 126 return msi_mask((control >> 1) & 7); 127} 128 129static inline __attribute_const__ u32 msi_enabled_mask(u16 control) 130{ 131 return msi_mask((control >> 4) & 7); 132} 133 134/* 135 * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to 136 * mask all MSI interrupts by clearing the MSI enable bit does not work 137 * reliably as devices without an INTx disable bit will then generate a 138 * level IRQ which will never be cleared. 139 */ 140static u32 __msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) 141{ 142 u32 mask_bits = desc->masked; 143 144 if (!desc->msi_attrib.maskbit) 145 return 0; 146 147 mask_bits &= ~mask; 148 mask_bits |= flag; 149 pci_write_config_dword(desc->dev, desc->mask_pos, mask_bits); 150 151 return mask_bits; 152} 153 154static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) 155{ 156 desc->masked = __msi_mask_irq(desc, mask, flag); 157} 158 159/* 160 * This internal function does not flush PCI writes to the device. 161 * All users must ensure that they read from the device before either 162 * assuming that the device state is up to date, or returning out of this 163 * file. This saves a few milliseconds when initialising devices with lots 164 * of MSI-X interrupts. 165 */ 166static u32 __msix_mask_irq(struct msi_desc *desc, u32 flag) 167{ 168 u32 mask_bits = desc->masked; 169 unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 170 PCI_MSIX_ENTRY_VECTOR_CTRL; 171 mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; 172 if (flag) 173 mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT; 174 writel(mask_bits, desc->mask_base + offset); 175 176 return mask_bits; 177} 178 179static void msix_mask_irq(struct msi_desc *desc, u32 flag) 180{ 181 desc->masked = __msix_mask_irq(desc, flag); 182} 183 184static void msi_set_mask_bit(struct irq_data *data, u32 flag) 185{ 186 struct msi_desc *desc = irq_data_get_msi(data); 187 188 if (desc->msi_attrib.is_msix) { 189 msix_mask_irq(desc, flag); 190 readl(desc->mask_base); /* Flush write to device */ 191 } else { 192 unsigned offset = data->irq - desc->dev->irq; 193 msi_mask_irq(desc, 1 << offset, flag << offset); 194 } 195} 196 197void mask_msi_irq(struct irq_data *data) 198{ 199 msi_set_mask_bit(data, 1); 200} 201 202void unmask_msi_irq(struct irq_data *data) 203{ 204 msi_set_mask_bit(data, 0); 205} 206 207void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 208{ 209 BUG_ON(entry->dev->current_state != PCI_D0); 210 211 if (entry->msi_attrib.is_msix) { 212 void __iomem *base = entry->mask_base + 213 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 214 215 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); 216 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); 217 msg->data = readl(base + PCI_MSIX_ENTRY_DATA); 218 } else { 219 struct pci_dev *dev = entry->dev; 220 int pos = entry->msi_attrib.pos; 221 u16 data; 222 223 pci_read_config_dword(dev, msi_lower_address_reg(pos), 224 &msg->address_lo); 225 if (entry->msi_attrib.is_64) { 226 pci_read_config_dword(dev, msi_upper_address_reg(pos), 227 &msg->address_hi); 228 pci_read_config_word(dev, msi_data_reg(pos, 1), &data); 229 } else { 230 msg->address_hi = 0; 231 pci_read_config_word(dev, msi_data_reg(pos, 0), &data); 232 } 233 msg->data = data; 234 } 235} 236 237void read_msi_msg(unsigned int irq, struct msi_msg *msg) 238{ 239 struct msi_desc *entry = irq_get_msi_desc(irq); 240 241 __read_msi_msg(entry, msg); 242} 243 244void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 245{ 246 /* Assert that the cache is valid, assuming that 247 * valid messages are not all-zeroes. */ 248 BUG_ON(!(entry->msg.address_hi | entry->msg.address_lo | 249 entry->msg.data)); 250 251 *msg = entry->msg; 252} 253 254void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) 255{ 256 struct msi_desc *entry = irq_get_msi_desc(irq); 257 258 __get_cached_msi_msg(entry, msg); 259} 260 261void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 262{ 263 if (entry->dev->current_state != PCI_D0) { 264 /* Don't touch the hardware now */ 265 } else if (entry->msi_attrib.is_msix) { 266 void __iomem *base; 267 base = entry->mask_base + 268 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; 269 270 writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); 271 writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); 272 writel(msg->data, base + PCI_MSIX_ENTRY_DATA); 273 } else { 274 struct pci_dev *dev = entry->dev; 275 int pos = entry->msi_attrib.pos; 276 u16 msgctl; 277 278 pci_read_config_word(dev, msi_control_reg(pos), &msgctl); 279 msgctl &= ~PCI_MSI_FLAGS_QSIZE; 280 msgctl |= entry->msi_attrib.multiple << 4; 281 pci_write_config_word(dev, msi_control_reg(pos), msgctl); 282 283 pci_write_config_dword(dev, msi_lower_address_reg(pos), 284 msg->address_lo); 285 if (entry->msi_attrib.is_64) { 286 pci_write_config_dword(dev, msi_upper_address_reg(pos), 287 msg->address_hi); 288 pci_write_config_word(dev, msi_data_reg(pos, 1), 289 msg->data); 290 } else { 291 pci_write_config_word(dev, msi_data_reg(pos, 0), 292 msg->data); 293 } 294 } 295 entry->msg = *msg; 296} 297 298void write_msi_msg(unsigned int irq, struct msi_msg *msg) 299{ 300 struct msi_desc *entry = irq_get_msi_desc(irq); 301 302 __write_msi_msg(entry, msg); 303} 304 305static void free_msi_irqs(struct pci_dev *dev) 306{ 307 struct msi_desc *entry, *tmp; 308 309 list_for_each_entry(entry, &dev->msi_list, list) { 310 int i, nvec; 311 if (!entry->irq) 312 continue; 313 nvec = 1 << entry->msi_attrib.multiple; 314 for (i = 0; i < nvec; i++) 315 BUG_ON(irq_has_action(entry->irq + i)); 316 } 317 318 arch_teardown_msi_irqs(dev); 319 320 list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) { 321 if (entry->msi_attrib.is_msix) { 322 if (list_is_last(&entry->list, &dev->msi_list)) 323 iounmap(entry->mask_base); 324 } 325 list_del(&entry->list); 326 kfree(entry); 327 } 328} 329 330static struct msi_desc *alloc_msi_entry(struct pci_dev *dev) 331{ 332 struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); 333 if (!desc) 334 return NULL; 335 336 INIT_LIST_HEAD(&desc->list); 337 desc->dev = dev; 338 339 return desc; 340} 341 342static void pci_intx_for_msi(struct pci_dev *dev, int enable) 343{ 344 if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG)) 345 pci_intx(dev, enable); 346} 347 348static void __pci_restore_msi_state(struct pci_dev *dev) 349{ 350 int pos; 351 u16 control; 352 struct msi_desc *entry; 353 354 if (!dev->msi_enabled) 355 return; 356 357 entry = irq_get_msi_desc(dev->irq); 358 pos = entry->msi_attrib.pos; 359 360 pci_intx_for_msi(dev, 0); 361 msi_set_enable(dev, pos, 0); 362 write_msi_msg(dev->irq, &entry->msg); 363 364 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); 365 msi_mask_irq(entry, msi_capable_mask(control), entry->masked); 366 control &= ~PCI_MSI_FLAGS_QSIZE; 367 control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; 368 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control); 369} 370 371static void __pci_restore_msix_state(struct pci_dev *dev) 372{ 373 int pos; 374 struct msi_desc *entry; 375 u16 control; 376 377 if (!dev->msix_enabled) 378 return; 379 BUG_ON(list_empty(&dev->msi_list)); 380 entry = list_first_entry(&dev->msi_list, struct msi_desc, list); 381 pos = entry->msi_attrib.pos; 382 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 383 384 /* route the table */ 385 pci_intx_for_msi(dev, 0); 386 control |= PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL; 387 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 388 389 list_for_each_entry(entry, &dev->msi_list, list) { 390 write_msi_msg(entry->irq, &entry->msg); 391 msix_mask_irq(entry, entry->masked); 392 } 393 394 control &= ~PCI_MSIX_FLAGS_MASKALL; 395 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 396} 397 398void pci_restore_msi_state(struct pci_dev *dev) 399{ 400 __pci_restore_msi_state(dev); 401 __pci_restore_msix_state(dev); 402} 403EXPORT_SYMBOL_GPL(pci_restore_msi_state); 404 405/** 406 * msi_capability_init - configure device's MSI capability structure 407 * @dev: pointer to the pci_dev data structure of MSI device function 408 * @nvec: number of interrupts to allocate 409 * 410 * Setup the MSI capability structure of the device with the requested 411 * number of interrupts. A return value of zero indicates the successful 412 * setup of an entry with the new MSI irq. A negative return value indicates 413 * an error, and a positive return value indicates the number of interrupts 414 * which could have been allocated. 415 */ 416static int msi_capability_init(struct pci_dev *dev, int nvec) 417{ 418 struct msi_desc *entry; 419 int pos, ret; 420 u16 control; 421 unsigned mask; 422 423 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 424 msi_set_enable(dev, pos, 0); /* Disable MSI during set up */ 425 426 pci_read_config_word(dev, msi_control_reg(pos), &control); 427 /* MSI Entry Initialization */ 428 entry = alloc_msi_entry(dev); 429 if (!entry) 430 return -ENOMEM; 431 432 entry->msi_attrib.is_msix = 0; 433 entry->msi_attrib.is_64 = is_64bit_address(control); 434 entry->msi_attrib.entry_nr = 0; 435 entry->msi_attrib.maskbit = is_mask_bit_support(control); 436 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ 437 entry->msi_attrib.pos = pos; 438 439 entry->mask_pos = msi_mask_reg(pos, entry->msi_attrib.is_64); 440 /* All MSIs are unmasked by default, Mask them all */ 441 if (entry->msi_attrib.maskbit) 442 pci_read_config_dword(dev, entry->mask_pos, &entry->masked); 443 mask = msi_capable_mask(control); 444 msi_mask_irq(entry, mask, mask); 445 446 list_add_tail(&entry->list, &dev->msi_list); 447 448 /* Configure MSI capability structure */ 449 ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); 450 if (ret) { 451 msi_mask_irq(entry, mask, ~mask); 452 free_msi_irqs(dev); 453 return ret; 454 } 455 456 /* Set MSI enabled bits */ 457 pci_intx_for_msi(dev, 0); 458 msi_set_enable(dev, pos, 1); 459 dev->msi_enabled = 1; 460 461 dev->irq = entry->irq; 462 return 0; 463} 464 465static void __iomem *msix_map_region(struct pci_dev *dev, unsigned pos, 466 unsigned nr_entries) 467{ 468 resource_size_t phys_addr; 469 u32 table_offset; 470 u8 bir; 471 472 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset); 473 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 474 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK; 475 phys_addr = pci_resource_start(dev, bir) + table_offset; 476 477 return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 478} 479 480static int msix_setup_entries(struct pci_dev *dev, unsigned pos, 481 void __iomem *base, struct msix_entry *entries, 482 int nvec) 483{ 484 struct msi_desc *entry; 485 int i; 486 487 for (i = 0; i < nvec; i++) { 488 entry = alloc_msi_entry(dev); 489 if (!entry) { 490 if (!i) 491 iounmap(base); 492 else 493 free_msi_irqs(dev); 494 /* No enough memory. Don't try again */ 495 return -ENOMEM; 496 } 497 498 entry->msi_attrib.is_msix = 1; 499 entry->msi_attrib.is_64 = 1; 500 entry->msi_attrib.entry_nr = entries[i].entry; 501 entry->msi_attrib.default_irq = dev->irq; 502 entry->msi_attrib.pos = pos; 503 entry->mask_base = base; 504 505 list_add_tail(&entry->list, &dev->msi_list); 506 } 507 508 return 0; 509} 510 511static void msix_program_entries(struct pci_dev *dev, 512 struct msix_entry *entries) 513{ 514 struct msi_desc *entry; 515 int i = 0; 516 517 list_for_each_entry(entry, &dev->msi_list, list) { 518 int offset = entries[i].entry * PCI_MSIX_ENTRY_SIZE + 519 PCI_MSIX_ENTRY_VECTOR_CTRL; 520 521 entries[i].vector = entry->irq; 522 irq_set_msi_desc(entry->irq, entry); 523 entry->masked = readl(entry->mask_base + offset); 524 msix_mask_irq(entry, 1); 525 i++; 526 } 527} 528 529/** 530 * msix_capability_init - configure device's MSI-X capability 531 * @dev: pointer to the pci_dev data structure of MSI-X device function 532 * @entries: pointer to an array of struct msix_entry entries 533 * @nvec: number of @entries 534 * 535 * Setup the MSI-X capability structure of device function with a 536 * single MSI-X irq. A return of zero indicates the successful setup of 537 * requested MSI-X entries with allocated irqs or non-zero for otherwise. 538 **/ 539static int msix_capability_init(struct pci_dev *dev, 540 struct msix_entry *entries, int nvec) 541{ 542 int pos, ret; 543 u16 control; 544 void __iomem *base; 545 546 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 547 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control); 548 549 /* Ensure MSI-X is disabled while it is set up */ 550 control &= ~PCI_MSIX_FLAGS_ENABLE; 551 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 552 553 /* Request & Map MSI-X table region */ 554 base = msix_map_region(dev, pos, multi_msix_capable(control)); 555 if (!base) 556 return -ENOMEM; 557 558 ret = msix_setup_entries(dev, pos, base, entries, nvec); 559 if (ret) 560 return ret; 561 562 ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); 563 if (ret) 564 goto error; 565 566 /* 567 * Some devices require MSI-X to be enabled before we can touch the 568 * MSI-X registers. We need to mask all the vectors to prevent 569 * interrupts coming in before they're fully set up. 570 */ 571 control |= PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE; 572 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 573 574 msix_program_entries(dev, entries); 575 576 /* Set MSI-X enabled bits and unmask the function */ 577 pci_intx_for_msi(dev, 0); 578 dev->msix_enabled = 1; 579 580 control &= ~PCI_MSIX_FLAGS_MASKALL; 581 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); 582 583 return 0; 584 585error: 586 if (ret < 0) { 587 /* 588 * If we had some success, report the number of irqs 589 * we succeeded in setting up. 590 */ 591 struct msi_desc *entry; 592 int avail = 0; 593 594 list_for_each_entry(entry, &dev->msi_list, list) { 595 if (entry->irq != 0) 596 avail++; 597 } 598 if (avail != 0) 599 ret = avail; 600 } 601 602 free_msi_irqs(dev); 603 604 return ret; 605} 606 607/** 608 * pci_msi_check_device - check whether MSI may be enabled on a device 609 * @dev: pointer to the pci_dev data structure of MSI device function 610 * @nvec: how many MSIs have been requested ? 611 * @type: are we checking for MSI or MSI-X ? 612 * 613 * Look at global flags, the device itself, and its parent busses 614 * to determine if MSI/-X are supported for the device. If MSI/-X is 615 * supported return 0, else return an error code. 616 **/ 617static int pci_msi_check_device(struct pci_dev *dev, int nvec, int type) 618{ 619 struct pci_bus *bus; 620 int ret; 621 622 /* MSI must be globally enabled and supported by the device */ 623 if (!pci_msi_enable || !dev || dev->no_msi) 624 return -EINVAL; 625 626 /* 627 * You can't ask to have 0 or less MSIs configured. 628 * a) it's stupid .. 629 * b) the list manipulation code assumes nvec >= 1. 630 */ 631 if (nvec < 1) 632 return -ERANGE; 633 634 /* 635 * Any bridge which does NOT route MSI transactions from its 636 * secondary bus to its primary bus must set NO_MSI flag on 637 * the secondary pci_bus. 638 * We expect only arch-specific PCI host bus controller driver 639 * or quirks for specific PCI bridges to be setting NO_MSI. 640 */ 641 for (bus = dev->bus; bus; bus = bus->parent) 642 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 643 return -EINVAL; 644 645 ret = arch_msi_check_device(dev, nvec, type); 646 if (ret) 647 return ret; 648 649 if (!pci_find_capability(dev, type)) 650 return -EINVAL; 651 652 return 0; 653} 654 655/** 656 * pci_enable_msi_block - configure device's MSI capability structure 657 * @dev: device to configure 658 * @nvec: number of interrupts to configure 659 * 660 * Allocate IRQs for a device with the MSI capability. 661 * This function returns a negative errno if an error occurs. If it 662 * is unable to allocate the number of interrupts requested, it returns 663 * the number of interrupts it might be able to allocate. If it successfully 664 * allocates at least the number of interrupts requested, it returns 0 and 665 * updates the @dev's irq member to the lowest new interrupt number; the 666 * other interrupt numbers allocated to this device are consecutive. 667 */ 668int pci_enable_msi_block(struct pci_dev *dev, unsigned int nvec) 669{ 670 int status, pos, maxvec; 671 u16 msgctl; 672 673 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 674 if (!pos) 675 return -EINVAL; 676 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); 677 maxvec = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1); 678 if (nvec > maxvec) 679 return maxvec; 680 681 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSI); 682 if (status) 683 return status; 684 685 WARN_ON(!!dev->msi_enabled); 686 687 /* Check whether driver already requested MSI-X irqs */ 688 if (dev->msix_enabled) { 689 dev_info(&dev->dev, "can't enable MSI " 690 "(MSI-X already enabled)\n"); 691 return -EINVAL; 692 } 693 694 status = msi_capability_init(dev, nvec); 695 return status; 696} 697EXPORT_SYMBOL(pci_enable_msi_block); 698 699void pci_msi_shutdown(struct pci_dev *dev) 700{ 701 struct msi_desc *desc; 702 u32 mask; 703 u16 ctrl; 704 unsigned pos; 705 706 if (!pci_msi_enable || !dev || !dev->msi_enabled) 707 return; 708 709 BUG_ON(list_empty(&dev->msi_list)); 710 desc = list_first_entry(&dev->msi_list, struct msi_desc, list); 711 pos = desc->msi_attrib.pos; 712 713 msi_set_enable(dev, pos, 0); 714 pci_intx_for_msi(dev, 1); 715 dev->msi_enabled = 0; 716 717 /* Return the device with MSI unmasked as initial states */ 718 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &ctrl); 719 mask = msi_capable_mask(ctrl); 720 /* Keep cached state to be restored */ 721 __msi_mask_irq(desc, mask, ~mask); 722 723 /* Restore dev->irq to its default pin-assertion irq */ 724 dev->irq = desc->msi_attrib.default_irq; 725} 726 727void pci_disable_msi(struct pci_dev *dev) 728{ 729 if (!pci_msi_enable || !dev || !dev->msi_enabled) 730 return; 731 732 pci_msi_shutdown(dev); 733 free_msi_irqs(dev); 734} 735EXPORT_SYMBOL(pci_disable_msi); 736 737/** 738 * pci_msix_table_size - return the number of device's MSI-X table entries 739 * @dev: pointer to the pci_dev data structure of MSI-X device function 740 */ 741int pci_msix_table_size(struct pci_dev *dev) 742{ 743 int pos; 744 u16 control; 745 746 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 747 if (!pos) 748 return 0; 749 750 pci_read_config_word(dev, msi_control_reg(pos), &control); 751 return multi_msix_capable(control); 752} 753 754/** 755 * pci_enable_msix - configure device's MSI-X capability structure 756 * @dev: pointer to the pci_dev data structure of MSI-X device function 757 * @entries: pointer to an array of MSI-X entries 758 * @nvec: number of MSI-X irqs requested for allocation by device driver 759 * 760 * Setup the MSI-X capability structure of device function with the number 761 * of requested irqs upon its software driver call to request for 762 * MSI-X mode enabled on its hardware device function. A return of zero 763 * indicates the successful configuration of MSI-X capability structure 764 * with new allocated MSI-X irqs. A return of < 0 indicates a failure. 765 * Or a return of > 0 indicates that driver request is exceeding the number 766 * of irqs or MSI-X vectors available. Driver should use the returned value to 767 * re-send its request. 768 **/ 769int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec) 770{ 771 int status, nr_entries; 772 int i, j; 773 774 if (!entries) 775 return -EINVAL; 776 777 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX); 778 if (status) 779 return status; 780 781 nr_entries = pci_msix_table_size(dev); 782 if (nvec > nr_entries) 783 return nr_entries; 784 785 /* Check for any invalid entries */ 786 for (i = 0; i < nvec; i++) { 787 if (entries[i].entry >= nr_entries) 788 return -EINVAL; /* invalid entry */ 789 for (j = i + 1; j < nvec; j++) { 790 if (entries[i].entry == entries[j].entry) 791 return -EINVAL; /* duplicate entry */ 792 } 793 } 794 WARN_ON(!!dev->msix_enabled); 795 796 /* Check whether driver already requested for MSI irq */ 797 if (dev->msi_enabled) { 798 dev_info(&dev->dev, "can't enable MSI-X " 799 "(MSI IRQ already assigned)\n"); 800 return -EINVAL; 801 } 802 status = msix_capability_init(dev, entries, nvec); 803 return status; 804} 805EXPORT_SYMBOL(pci_enable_msix); 806 807void pci_msix_shutdown(struct pci_dev *dev) 808{ 809 struct msi_desc *entry; 810 811 if (!pci_msi_enable || !dev || !dev->msix_enabled) 812 return; 813 814 /* Return the device with MSI-X masked as initial states */ 815 list_for_each_entry(entry, &dev->msi_list, list) { 816 /* Keep cached states to be restored */ 817 __msix_mask_irq(entry, 1); 818 } 819 820 msix_set_enable(dev, 0); 821 pci_intx_for_msi(dev, 1); 822 dev->msix_enabled = 0; 823} 824 825void pci_disable_msix(struct pci_dev *dev) 826{ 827 if (!pci_msi_enable || !dev || !dev->msix_enabled) 828 return; 829 830 pci_msix_shutdown(dev); 831 free_msi_irqs(dev); 832} 833EXPORT_SYMBOL(pci_disable_msix); 834 835/** 836 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state 837 * @dev: pointer to the pci_dev data structure of MSI(X) device function 838 * 839 * Being called during hotplug remove, from which the device function 840 * is hot-removed. All previous assigned MSI/MSI-X irqs, if 841 * allocated for this device function, are reclaimed to unused state, 842 * which may be used later on. 843 **/ 844void msi_remove_pci_irq_vectors(struct pci_dev *dev) 845{ 846 if (!pci_msi_enable || !dev) 847 return; 848 849 if (dev->msi_enabled || dev->msix_enabled) 850 free_msi_irqs(dev); 851} 852 853void pci_no_msi(void) 854{ 855 pci_msi_enable = 0; 856} 857 858/** 859 * pci_msi_enabled - is MSI enabled? 860 * 861 * Returns true if MSI has not been disabled by the command-line option 862 * pci=nomsi. 863 **/ 864int pci_msi_enabled(void) 865{ 866 return pci_msi_enable; 867} 868EXPORT_SYMBOL(pci_msi_enabled); 869 870void pci_msi_init_pci_dev(struct pci_dev *dev) 871{ 872 INIT_LIST_HEAD(&dev->msi_list); 873} 874