msi.c revision 4602b88d9743b5f20655de8078fb42e9fd25581f
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/mm.h> 10#include <linux/irq.h> 11#include <linux/interrupt.h> 12#include <linux/init.h> 13#include <linux/config.h> 14#include <linux/ioport.h> 15#include <linux/smp_lock.h> 16#include <linux/pci.h> 17#include <linux/proc_fs.h> 18 19#include <asm/errno.h> 20#include <asm/io.h> 21#include <asm/smp.h> 22 23#include "pci.h" 24#include "msi.h" 25 26static DEFINE_SPINLOCK(msi_lock); 27static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL }; 28static kmem_cache_t* msi_cachep; 29 30static int pci_msi_enable = 1; 31static int last_alloc_vector; 32static int nr_released_vectors; 33static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS; 34static int nr_msix_devices; 35 36#ifndef CONFIG_X86_IO_APIC 37int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1}; 38u8 irq_vector[NR_IRQ_VECTORS] = { FIRST_DEVICE_VECTOR , 0 }; 39#endif 40 41static void msi_cache_ctor(void *p, kmem_cache_t *cache, unsigned long flags) 42{ 43 memset(p, 0, NR_IRQS * sizeof(struct msi_desc)); 44} 45 46static int msi_cache_init(void) 47{ 48 msi_cachep = kmem_cache_create("msi_cache", 49 NR_IRQS * sizeof(struct msi_desc), 50 0, SLAB_HWCACHE_ALIGN, msi_cache_ctor, NULL); 51 if (!msi_cachep) 52 return -ENOMEM; 53 54 return 0; 55} 56 57static void msi_set_mask_bit(unsigned int vector, int flag) 58{ 59 struct msi_desc *entry; 60 61 entry = (struct msi_desc *)msi_desc[vector]; 62 if (!entry || !entry->dev || !entry->mask_base) 63 return; 64 switch (entry->msi_attrib.type) { 65 case PCI_CAP_ID_MSI: 66 { 67 int pos; 68 u32 mask_bits; 69 70 pos = (long)entry->mask_base; 71 pci_read_config_dword(entry->dev, pos, &mask_bits); 72 mask_bits &= ~(1); 73 mask_bits |= flag; 74 pci_write_config_dword(entry->dev, pos, mask_bits); 75 break; 76 } 77 case PCI_CAP_ID_MSIX: 78 { 79 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 80 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 81 writel(flag, entry->mask_base + offset); 82 break; 83 } 84 default: 85 break; 86 } 87} 88 89#ifdef CONFIG_SMP 90static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask) 91{ 92 struct msi_desc *entry; 93 struct msg_address address; 94 95 entry = (struct msi_desc *)msi_desc[vector]; 96 if (!entry || !entry->dev) 97 return; 98 99 switch (entry->msi_attrib.type) { 100 case PCI_CAP_ID_MSI: 101 { 102 int pos; 103 104 if (!(pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI))) 105 return; 106 107 pci_read_config_dword(entry->dev, msi_lower_address_reg(pos), 108 &address.lo_address.value); 109 address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK; 110 address.lo_address.value |= (cpu_mask_to_apicid(cpu_mask) << 111 MSI_TARGET_CPU_SHIFT); 112 entry->msi_attrib.current_cpu = cpu_mask_to_apicid(cpu_mask); 113 pci_write_config_dword(entry->dev, msi_lower_address_reg(pos), 114 address.lo_address.value); 115 break; 116 } 117 case PCI_CAP_ID_MSIX: 118 { 119 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 120 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET; 121 122 address.lo_address.value = readl(entry->mask_base + offset); 123 address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK; 124 address.lo_address.value |= (cpu_mask_to_apicid(cpu_mask) << 125 MSI_TARGET_CPU_SHIFT); 126 entry->msi_attrib.current_cpu = cpu_mask_to_apicid(cpu_mask); 127 writel(address.lo_address.value, entry->mask_base + offset); 128 break; 129 } 130 default: 131 break; 132 } 133} 134 135#ifdef CONFIG_IRQBALANCE 136static inline void move_msi(int vector) 137{ 138 if (!cpus_empty(pending_irq_balance_cpumask[vector])) { 139 set_msi_affinity(vector, pending_irq_balance_cpumask[vector]); 140 cpus_clear(pending_irq_balance_cpumask[vector]); 141 } 142} 143#endif /* CONFIG_IRQBALANCE */ 144#endif /* CONFIG_SMP */ 145 146static void mask_MSI_irq(unsigned int vector) 147{ 148 msi_set_mask_bit(vector, 1); 149} 150 151static void unmask_MSI_irq(unsigned int vector) 152{ 153 msi_set_mask_bit(vector, 0); 154} 155 156static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector) 157{ 158 struct msi_desc *entry; 159 unsigned long flags; 160 161 spin_lock_irqsave(&msi_lock, flags); 162 entry = msi_desc[vector]; 163 if (!entry || !entry->dev) { 164 spin_unlock_irqrestore(&msi_lock, flags); 165 return 0; 166 } 167 entry->msi_attrib.state = 1; /* Mark it active */ 168 spin_unlock_irqrestore(&msi_lock, flags); 169 170 return 0; /* never anything pending */ 171} 172 173static unsigned int startup_msi_irq_w_maskbit(unsigned int vector) 174{ 175 startup_msi_irq_wo_maskbit(vector); 176 unmask_MSI_irq(vector); 177 return 0; /* never anything pending */ 178} 179 180static void shutdown_msi_irq(unsigned int vector) 181{ 182 struct msi_desc *entry; 183 unsigned long flags; 184 185 spin_lock_irqsave(&msi_lock, flags); 186 entry = msi_desc[vector]; 187 if (entry && entry->dev) 188 entry->msi_attrib.state = 0; /* Mark it not active */ 189 spin_unlock_irqrestore(&msi_lock, flags); 190} 191 192static void end_msi_irq_wo_maskbit(unsigned int vector) 193{ 194 move_msi(vector); 195 ack_APIC_irq(); 196} 197 198static void end_msi_irq_w_maskbit(unsigned int vector) 199{ 200 move_msi(vector); 201 unmask_MSI_irq(vector); 202 ack_APIC_irq(); 203} 204 205static void do_nothing(unsigned int vector) 206{ 207} 208 209/* 210 * Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices, 211 * which implement the MSI-X Capability Structure. 212 */ 213static struct hw_interrupt_type msix_irq_type = { 214 .typename = "PCI-MSI-X", 215 .startup = startup_msi_irq_w_maskbit, 216 .shutdown = shutdown_msi_irq, 217 .enable = unmask_MSI_irq, 218 .disable = mask_MSI_irq, 219 .ack = mask_MSI_irq, 220 .end = end_msi_irq_w_maskbit, 221 .set_affinity = set_msi_irq_affinity 222}; 223 224/* 225 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices, 226 * which implement the MSI Capability Structure with 227 * Mask-and-Pending Bits. 228 */ 229static struct hw_interrupt_type msi_irq_w_maskbit_type = { 230 .typename = "PCI-MSI", 231 .startup = startup_msi_irq_w_maskbit, 232 .shutdown = shutdown_msi_irq, 233 .enable = unmask_MSI_irq, 234 .disable = mask_MSI_irq, 235 .ack = mask_MSI_irq, 236 .end = end_msi_irq_w_maskbit, 237 .set_affinity = set_msi_irq_affinity 238}; 239 240/* 241 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices, 242 * which implement the MSI Capability Structure without 243 * Mask-and-Pending Bits. 244 */ 245static struct hw_interrupt_type msi_irq_wo_maskbit_type = { 246 .typename = "PCI-MSI", 247 .startup = startup_msi_irq_wo_maskbit, 248 .shutdown = shutdown_msi_irq, 249 .enable = do_nothing, 250 .disable = do_nothing, 251 .ack = do_nothing, 252 .end = end_msi_irq_wo_maskbit, 253 .set_affinity = set_msi_irq_affinity 254}; 255 256static void msi_data_init(struct msg_data *msi_data, 257 unsigned int vector) 258{ 259 memset(msi_data, 0, sizeof(struct msg_data)); 260 msi_data->vector = (u8)vector; 261 msi_data->delivery_mode = MSI_DELIVERY_MODE; 262 msi_data->level = MSI_LEVEL_MODE; 263 msi_data->trigger = MSI_TRIGGER_MODE; 264} 265 266static void msi_address_init(struct msg_address *msi_address) 267{ 268 unsigned int dest_id; 269 270 memset(msi_address, 0, sizeof(struct msg_address)); 271 msi_address->hi_address = (u32)0; 272 dest_id = (MSI_ADDRESS_HEADER << MSI_ADDRESS_HEADER_SHIFT); 273 msi_address->lo_address.u.dest_mode = MSI_DEST_MODE; 274 msi_address->lo_address.u.redirection_hint = MSI_REDIRECTION_HINT_MODE; 275 msi_address->lo_address.u.dest_id = dest_id; 276 msi_address->lo_address.value |= (MSI_TARGET_CPU << MSI_TARGET_CPU_SHIFT); 277} 278 279static int msi_free_vector(struct pci_dev* dev, int vector, int reassign); 280static int assign_msi_vector(void) 281{ 282 static int new_vector_avail = 1; 283 int vector; 284 unsigned long flags; 285 286 /* 287 * msi_lock is provided to ensure that successful allocation of MSI 288 * vector is assigned unique among drivers. 289 */ 290 spin_lock_irqsave(&msi_lock, flags); 291 292 if (!new_vector_avail) { 293 int free_vector = 0; 294 295 /* 296 * vector_irq[] = -1 indicates that this specific vector is: 297 * - assigned for MSI (since MSI have no associated IRQ) or 298 * - assigned for legacy if less than 16, or 299 * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping 300 * vector_irq[] = 0 indicates that this vector, previously 301 * assigned for MSI, is freed by hotplug removed operations. 302 * This vector will be reused for any subsequent hotplug added 303 * operations. 304 * vector_irq[] > 0 indicates that this vector is assigned for 305 * IOxAPIC IRQs. This vector and its value provides a 1-to-1 306 * vector-to-IOxAPIC IRQ mapping. 307 */ 308 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) { 309 if (vector_irq[vector] != 0) 310 continue; 311 free_vector = vector; 312 if (!msi_desc[vector]) 313 break; 314 else 315 continue; 316 } 317 if (!free_vector) { 318 spin_unlock_irqrestore(&msi_lock, flags); 319 return -EBUSY; 320 } 321 vector_irq[free_vector] = -1; 322 nr_released_vectors--; 323 spin_unlock_irqrestore(&msi_lock, flags); 324 if (msi_desc[free_vector] != NULL) { 325 struct pci_dev *dev; 326 int tail; 327 328 /* free all linked vectors before re-assign */ 329 do { 330 spin_lock_irqsave(&msi_lock, flags); 331 dev = msi_desc[free_vector]->dev; 332 tail = msi_desc[free_vector]->link.tail; 333 spin_unlock_irqrestore(&msi_lock, flags); 334 msi_free_vector(dev, tail, 1); 335 } while (free_vector != tail); 336 } 337 338 return free_vector; 339 } 340 vector = assign_irq_vector(AUTO_ASSIGN); 341 last_alloc_vector = vector; 342 if (vector == LAST_DEVICE_VECTOR) 343 new_vector_avail = 0; 344 345 spin_unlock_irqrestore(&msi_lock, flags); 346 return vector; 347} 348 349static int get_new_vector(void) 350{ 351 int vector; 352 353 if ((vector = assign_msi_vector()) > 0) 354 set_intr_gate(vector, interrupt[vector]); 355 356 return vector; 357} 358 359static int msi_init(void) 360{ 361 static int status = -ENOMEM; 362 363 if (!status) 364 return status; 365 366 if (pci_msi_quirk) { 367 pci_msi_enable = 0; 368 printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n"); 369 status = -EINVAL; 370 return status; 371 } 372 373 if ((status = msi_cache_init()) < 0) { 374 pci_msi_enable = 0; 375 printk(KERN_WARNING "PCI: MSI cache init failed\n"); 376 return status; 377 } 378 last_alloc_vector = assign_irq_vector(AUTO_ASSIGN); 379 if (last_alloc_vector < 0) { 380 pci_msi_enable = 0; 381 printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n"); 382 status = -EBUSY; 383 return status; 384 } 385 vector_irq[last_alloc_vector] = 0; 386 nr_released_vectors++; 387 388 return status; 389} 390 391static int get_msi_vector(struct pci_dev *dev) 392{ 393 return get_new_vector(); 394} 395 396static struct msi_desc* alloc_msi_entry(void) 397{ 398 struct msi_desc *entry; 399 400 entry = kmem_cache_alloc(msi_cachep, SLAB_KERNEL); 401 if (!entry) 402 return NULL; 403 404 memset(entry, 0, sizeof(struct msi_desc)); 405 entry->link.tail = entry->link.head = 0; /* single message */ 406 entry->dev = NULL; 407 408 return entry; 409} 410 411static void attach_msi_entry(struct msi_desc *entry, int vector) 412{ 413 unsigned long flags; 414 415 spin_lock_irqsave(&msi_lock, flags); 416 msi_desc[vector] = entry; 417 spin_unlock_irqrestore(&msi_lock, flags); 418} 419 420static void irq_handler_init(int cap_id, int pos, int mask) 421{ 422 spin_lock(&irq_desc[pos].lock); 423 if (cap_id == PCI_CAP_ID_MSIX) 424 irq_desc[pos].handler = &msix_irq_type; 425 else { 426 if (!mask) 427 irq_desc[pos].handler = &msi_irq_wo_maskbit_type; 428 else 429 irq_desc[pos].handler = &msi_irq_w_maskbit_type; 430 } 431 spin_unlock(&irq_desc[pos].lock); 432} 433 434static void enable_msi_mode(struct pci_dev *dev, int pos, int type) 435{ 436 u16 control; 437 438 pci_read_config_word(dev, msi_control_reg(pos), &control); 439 if (type == PCI_CAP_ID_MSI) { 440 /* Set enabled bits to single MSI & enable MSI_enable bit */ 441 msi_enable(control, 1); 442 pci_write_config_word(dev, msi_control_reg(pos), control); 443 } else { 444 msix_enable(control); 445 pci_write_config_word(dev, msi_control_reg(pos), control); 446 } 447 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) { 448 /* PCI Express Endpoint device detected */ 449 u16 cmd; 450 pci_read_config_word(dev, PCI_COMMAND, &cmd); 451 cmd |= PCI_COMMAND_INTX_DISABLE; 452 pci_write_config_word(dev, PCI_COMMAND, cmd); 453 } 454} 455 456void disable_msi_mode(struct pci_dev *dev, int pos, int type) 457{ 458 u16 control; 459 460 pci_read_config_word(dev, msi_control_reg(pos), &control); 461 if (type == PCI_CAP_ID_MSI) { 462 /* Set enabled bits to single MSI & enable MSI_enable bit */ 463 msi_disable(control); 464 pci_write_config_word(dev, msi_control_reg(pos), control); 465 } else { 466 msix_disable(control); 467 pci_write_config_word(dev, msi_control_reg(pos), control); 468 } 469 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) { 470 /* PCI Express Endpoint device detected */ 471 u16 cmd; 472 pci_read_config_word(dev, PCI_COMMAND, &cmd); 473 cmd &= ~PCI_COMMAND_INTX_DISABLE; 474 pci_write_config_word(dev, PCI_COMMAND, cmd); 475 } 476} 477 478static int msi_lookup_vector(struct pci_dev *dev, int type) 479{ 480 int vector; 481 unsigned long flags; 482 483 spin_lock_irqsave(&msi_lock, flags); 484 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) { 485 if (!msi_desc[vector] || msi_desc[vector]->dev != dev || 486 msi_desc[vector]->msi_attrib.type != type || 487 msi_desc[vector]->msi_attrib.default_vector != dev->irq) 488 continue; 489 spin_unlock_irqrestore(&msi_lock, flags); 490 /* This pre-assigned MSI vector for this device 491 already exits. Override dev->irq with this vector */ 492 dev->irq = vector; 493 return 0; 494 } 495 spin_unlock_irqrestore(&msi_lock, flags); 496 497 return -EACCES; 498} 499 500void pci_scan_msi_device(struct pci_dev *dev) 501{ 502 if (!dev) 503 return; 504 505 if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0) 506 nr_msix_devices++; 507 else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0) 508 nr_reserved_vectors++; 509} 510 511/** 512 * msi_capability_init - configure device's MSI capability structure 513 * @dev: pointer to the pci_dev data structure of MSI device function 514 * 515 * Setup the MSI capability structure of device function with a single 516 * MSI vector, regardless of device function is capable of handling 517 * multiple messages. A return of zero indicates the successful setup 518 * of an entry zero with the new MSI vector or non-zero for otherwise. 519 **/ 520static int msi_capability_init(struct pci_dev *dev) 521{ 522 struct msi_desc *entry; 523 struct msg_address address; 524 struct msg_data data; 525 int pos, vector; 526 u16 control; 527 528 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 529 pci_read_config_word(dev, msi_control_reg(pos), &control); 530 /* MSI Entry Initialization */ 531 if (!(entry = alloc_msi_entry())) 532 return -ENOMEM; 533 534 if ((vector = get_msi_vector(dev)) < 0) { 535 kmem_cache_free(msi_cachep, entry); 536 return -EBUSY; 537 } 538 entry->link.head = vector; 539 entry->link.tail = vector; 540 entry->msi_attrib.type = PCI_CAP_ID_MSI; 541 entry->msi_attrib.state = 0; /* Mark it not active */ 542 entry->msi_attrib.entry_nr = 0; 543 entry->msi_attrib.maskbit = is_mask_bit_support(control); 544 entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */ 545 dev->irq = vector; 546 entry->dev = dev; 547 if (is_mask_bit_support(control)) { 548 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, 549 is_64bit_address(control)); 550 } 551 /* Replace with MSI handler */ 552 irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit); 553 /* Configure MSI capability structure */ 554 msi_address_init(&address); 555 msi_data_init(&data, vector); 556 entry->msi_attrib.current_cpu = ((address.lo_address.u.dest_id >> 557 MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK); 558 pci_write_config_dword(dev, msi_lower_address_reg(pos), 559 address.lo_address.value); 560 if (is_64bit_address(control)) { 561 pci_write_config_dword(dev, 562 msi_upper_address_reg(pos), address.hi_address); 563 pci_write_config_word(dev, 564 msi_data_reg(pos, 1), *((u32*)&data)); 565 } else 566 pci_write_config_word(dev, 567 msi_data_reg(pos, 0), *((u32*)&data)); 568 if (entry->msi_attrib.maskbit) { 569 unsigned int maskbits, temp; 570 /* All MSIs are unmasked by default, Mask them all */ 571 pci_read_config_dword(dev, 572 msi_mask_bits_reg(pos, is_64bit_address(control)), 573 &maskbits); 574 temp = (1 << multi_msi_capable(control)); 575 temp = ((temp - 1) & ~temp); 576 maskbits |= temp; 577 pci_write_config_dword(dev, 578 msi_mask_bits_reg(pos, is_64bit_address(control)), 579 maskbits); 580 } 581 attach_msi_entry(entry, vector); 582 /* Set MSI enabled bits */ 583 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 584 585 return 0; 586} 587 588/** 589 * msix_capability_init - configure device's MSI-X capability 590 * @dev: pointer to the pci_dev data structure of MSI-X device function 591 * 592 * Setup the MSI-X capability structure of device function with a 593 * single MSI-X vector. A return of zero indicates the successful setup of 594 * requested MSI-X entries with allocated vectors or non-zero for otherwise. 595 **/ 596static int msix_capability_init(struct pci_dev *dev, 597 struct msix_entry *entries, int nvec) 598{ 599 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL; 600 struct msg_address address; 601 struct msg_data data; 602 int vector, pos, i, j, nr_entries, temp = 0; 603 u32 phys_addr, table_offset; 604 u16 control; 605 u8 bir; 606 void __iomem *base; 607 608 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 609 /* Request & Map MSI-X table region */ 610 pci_read_config_word(dev, msi_control_reg(pos), &control); 611 nr_entries = multi_msix_capable(control); 612 pci_read_config_dword(dev, msix_table_offset_reg(pos), 613 &table_offset); 614 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 615 phys_addr = pci_resource_start (dev, bir); 616 phys_addr += (u32)(table_offset & ~PCI_MSIX_FLAGS_BIRMASK); 617 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 618 if (base == NULL) 619 return -ENOMEM; 620 621 /* MSI-X Table Initialization */ 622 for (i = 0; i < nvec; i++) { 623 entry = alloc_msi_entry(); 624 if (!entry) 625 break; 626 if ((vector = get_msi_vector(dev)) < 0) 627 break; 628 629 j = entries[i].entry; 630 entries[i].vector = vector; 631 entry->msi_attrib.type = PCI_CAP_ID_MSIX; 632 entry->msi_attrib.state = 0; /* Mark it not active */ 633 entry->msi_attrib.entry_nr = j; 634 entry->msi_attrib.maskbit = 1; 635 entry->msi_attrib.default_vector = dev->irq; 636 entry->dev = dev; 637 entry->mask_base = base; 638 if (!head) { 639 entry->link.head = vector; 640 entry->link.tail = vector; 641 head = entry; 642 } else { 643 entry->link.head = temp; 644 entry->link.tail = tail->link.tail; 645 tail->link.tail = vector; 646 head->link.head = vector; 647 } 648 temp = vector; 649 tail = entry; 650 /* Replace with MSI-X handler */ 651 irq_handler_init(PCI_CAP_ID_MSIX, vector, 1); 652 /* Configure MSI-X capability structure */ 653 msi_address_init(&address); 654 msi_data_init(&data, vector); 655 entry->msi_attrib.current_cpu = 656 ((address.lo_address.u.dest_id >> 657 MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK); 658 writel(address.lo_address.value, 659 base + j * PCI_MSIX_ENTRY_SIZE + 660 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 661 writel(address.hi_address, 662 base + j * PCI_MSIX_ENTRY_SIZE + 663 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 664 writel(*(u32*)&data, 665 base + j * PCI_MSIX_ENTRY_SIZE + 666 PCI_MSIX_ENTRY_DATA_OFFSET); 667 attach_msi_entry(entry, vector); 668 } 669 if (i != nvec) { 670 i--; 671 for (; i >= 0; i--) { 672 vector = (entries + i)->vector; 673 msi_free_vector(dev, vector, 0); 674 (entries + i)->vector = 0; 675 } 676 return -EBUSY; 677 } 678 /* Set MSI-X enabled bits */ 679 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 680 681 return 0; 682} 683 684/** 685 * pci_enable_msi - configure device's MSI capability structure 686 * @dev: pointer to the pci_dev data structure of MSI device function 687 * 688 * Setup the MSI capability structure of device function with 689 * a single MSI vector upon its software driver call to request for 690 * MSI mode enabled on its hardware device function. A return of zero 691 * indicates the successful setup of an entry zero with the new MSI 692 * vector or non-zero for otherwise. 693 **/ 694int pci_enable_msi(struct pci_dev* dev) 695{ 696 int pos, temp, status = -EINVAL; 697 u16 control; 698 699 if (!pci_msi_enable || !dev) 700 return status; 701 702 if (dev->no_msi) 703 return status; 704 705 temp = dev->irq; 706 707 if ((status = msi_init()) < 0) 708 return status; 709 710 if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSI))) 711 return -EINVAL; 712 713 pci_read_config_word(dev, msi_control_reg(pos), &control); 714 if (control & PCI_MSI_FLAGS_ENABLE) 715 return 0; /* Already in MSI mode */ 716 717 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 718 /* Lookup Sucess */ 719 unsigned long flags; 720 721 spin_lock_irqsave(&msi_lock, flags); 722 if (!vector_irq[dev->irq]) { 723 msi_desc[dev->irq]->msi_attrib.state = 0; 724 vector_irq[dev->irq] = -1; 725 nr_released_vectors--; 726 spin_unlock_irqrestore(&msi_lock, flags); 727 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 728 return 0; 729 } 730 spin_unlock_irqrestore(&msi_lock, flags); 731 dev->irq = temp; 732 } 733 /* Check whether driver already requested for MSI-X vectors */ 734 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 && 735 !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 736 printk(KERN_INFO "PCI: %s: Can't enable MSI. " 737 "Device already has MSI-X vectors assigned\n", 738 pci_name(dev)); 739 dev->irq = temp; 740 return -EINVAL; 741 } 742 status = msi_capability_init(dev); 743 if (!status) { 744 if (!pos) 745 nr_reserved_vectors--; /* Only MSI capable */ 746 else if (nr_msix_devices > 0) 747 nr_msix_devices--; /* Both MSI and MSI-X capable, 748 but choose enabling MSI */ 749 } 750 751 return status; 752} 753 754void pci_disable_msi(struct pci_dev* dev) 755{ 756 struct msi_desc *entry; 757 int pos, default_vector; 758 u16 control; 759 unsigned long flags; 760 761 if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSI))) 762 return; 763 764 pci_read_config_word(dev, msi_control_reg(pos), &control); 765 if (!(control & PCI_MSI_FLAGS_ENABLE)) 766 return; 767 768 spin_lock_irqsave(&msi_lock, flags); 769 entry = msi_desc[dev->irq]; 770 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { 771 spin_unlock_irqrestore(&msi_lock, flags); 772 return; 773 } 774 if (entry->msi_attrib.state) { 775 spin_unlock_irqrestore(&msi_lock, flags); 776 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without " 777 "free_irq() on MSI vector %d\n", 778 pci_name(dev), dev->irq); 779 BUG_ON(entry->msi_attrib.state > 0); 780 } else { 781 vector_irq[dev->irq] = 0; /* free it */ 782 nr_released_vectors++; 783 default_vector = entry->msi_attrib.default_vector; 784 spin_unlock_irqrestore(&msi_lock, flags); 785 /* Restore dev->irq to its default pin-assertion vector */ 786 dev->irq = default_vector; 787 disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI), 788 PCI_CAP_ID_MSI); 789 } 790} 791 792static int msi_free_vector(struct pci_dev* dev, int vector, int reassign) 793{ 794 struct msi_desc *entry; 795 int head, entry_nr, type; 796 void __iomem *base; 797 unsigned long flags; 798 799 spin_lock_irqsave(&msi_lock, flags); 800 entry = msi_desc[vector]; 801 if (!entry || entry->dev != dev) { 802 spin_unlock_irqrestore(&msi_lock, flags); 803 return -EINVAL; 804 } 805 type = entry->msi_attrib.type; 806 entry_nr = entry->msi_attrib.entry_nr; 807 head = entry->link.head; 808 base = entry->mask_base; 809 msi_desc[entry->link.head]->link.tail = entry->link.tail; 810 msi_desc[entry->link.tail]->link.head = entry->link.head; 811 entry->dev = NULL; 812 if (!reassign) { 813 vector_irq[vector] = 0; 814 nr_released_vectors++; 815 } 816 msi_desc[vector] = NULL; 817 spin_unlock_irqrestore(&msi_lock, flags); 818 819 kmem_cache_free(msi_cachep, entry); 820 821 if (type == PCI_CAP_ID_MSIX) { 822 if (!reassign) 823 writel(1, base + 824 entry_nr * PCI_MSIX_ENTRY_SIZE + 825 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); 826 827 if (head == vector) { 828 /* 829 * Detect last MSI-X vector to be released. 830 * Release the MSI-X memory-mapped table. 831 */ 832 int pos, nr_entries; 833 u32 phys_addr, table_offset; 834 u16 control; 835 u8 bir; 836 837 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 838 pci_read_config_word(dev, msi_control_reg(pos), 839 &control); 840 nr_entries = multi_msix_capable(control); 841 pci_read_config_dword(dev, msix_table_offset_reg(pos), 842 &table_offset); 843 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 844 phys_addr = pci_resource_start (dev, bir); 845 phys_addr += (u32)(table_offset & 846 ~PCI_MSIX_FLAGS_BIRMASK); 847 iounmap(base); 848 } 849 } 850 851 return 0; 852} 853 854static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec) 855{ 856 int vector = head, tail = 0; 857 int i, j = 0, nr_entries = 0; 858 void __iomem *base; 859 unsigned long flags; 860 861 spin_lock_irqsave(&msi_lock, flags); 862 while (head != tail) { 863 nr_entries++; 864 tail = msi_desc[vector]->link.tail; 865 if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr) 866 j = vector; 867 vector = tail; 868 } 869 if (*nvec > nr_entries) { 870 spin_unlock_irqrestore(&msi_lock, flags); 871 *nvec = nr_entries; 872 return -EINVAL; 873 } 874 vector = ((j > 0) ? j : head); 875 for (i = 0; i < *nvec; i++) { 876 j = msi_desc[vector]->msi_attrib.entry_nr; 877 msi_desc[vector]->msi_attrib.state = 0; /* Mark it not active */ 878 vector_irq[vector] = -1; /* Mark it busy */ 879 nr_released_vectors--; 880 entries[i].vector = vector; 881 if (j != (entries + i)->entry) { 882 base = msi_desc[vector]->mask_base; 883 msi_desc[vector]->msi_attrib.entry_nr = 884 (entries + i)->entry; 885 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE + 886 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base + 887 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE + 888 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 889 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE + 890 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base + 891 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE + 892 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 893 writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE + 894 PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector, 895 base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE + 896 PCI_MSIX_ENTRY_DATA_OFFSET); 897 } 898 vector = msi_desc[vector]->link.tail; 899 } 900 spin_unlock_irqrestore(&msi_lock, flags); 901 902 return 0; 903} 904 905/** 906 * pci_enable_msix - configure device's MSI-X capability structure 907 * @dev: pointer to the pci_dev data structure of MSI-X device function 908 * @entries: pointer to an array of MSI-X entries 909 * @nvec: number of MSI-X vectors requested for allocation by device driver 910 * 911 * Setup the MSI-X capability structure of device function with the number 912 * of requested vectors upon its software driver call to request for 913 * MSI-X mode enabled on its hardware device function. A return of zero 914 * indicates the successful configuration of MSI-X capability structure 915 * with new allocated MSI-X vectors. A return of < 0 indicates a failure. 916 * Or a return of > 0 indicates that driver request is exceeding the number 917 * of vectors available. Driver should use the returned value to re-send 918 * its request. 919 **/ 920int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) 921{ 922 int status, pos, nr_entries, free_vectors; 923 int i, j, temp; 924 u16 control; 925 unsigned long flags; 926 927 if (!pci_msi_enable || !dev || !entries) 928 return -EINVAL; 929 930 if ((status = msi_init()) < 0) 931 return status; 932 933 if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX))) 934 return -EINVAL; 935 936 pci_read_config_word(dev, msi_control_reg(pos), &control); 937 if (control & PCI_MSIX_FLAGS_ENABLE) 938 return -EINVAL; /* Already in MSI-X mode */ 939 940 nr_entries = multi_msix_capable(control); 941 if (nvec > nr_entries) 942 return -EINVAL; 943 944 /* Check for any invalid entries */ 945 for (i = 0; i < nvec; i++) { 946 if (entries[i].entry >= nr_entries) 947 return -EINVAL; /* invalid entry */ 948 for (j = i + 1; j < nvec; j++) { 949 if (entries[i].entry == entries[j].entry) 950 return -EINVAL; /* duplicate entry */ 951 } 952 } 953 temp = dev->irq; 954 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 955 /* Lookup Sucess */ 956 nr_entries = nvec; 957 /* Reroute MSI-X table */ 958 if (reroute_msix_table(dev->irq, entries, &nr_entries)) { 959 /* #requested > #previous-assigned */ 960 dev->irq = temp; 961 return nr_entries; 962 } 963 dev->irq = temp; 964 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 965 return 0; 966 } 967 /* Check whether driver already requested for MSI vector */ 968 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 && 969 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 970 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " 971 "Device already has an MSI vector assigned\n", 972 pci_name(dev)); 973 dev->irq = temp; 974 return -EINVAL; 975 } 976 977 spin_lock_irqsave(&msi_lock, flags); 978 /* 979 * msi_lock is provided to ensure that enough vectors resources are 980 * available before granting. 981 */ 982 free_vectors = pci_vector_resources(last_alloc_vector, 983 nr_released_vectors); 984 /* Ensure that each MSI/MSI-X device has one vector reserved by 985 default to avoid any MSI-X driver to take all available 986 resources */ 987 free_vectors -= nr_reserved_vectors; 988 /* Find the average of free vectors among MSI-X devices */ 989 if (nr_msix_devices > 0) 990 free_vectors /= nr_msix_devices; 991 spin_unlock_irqrestore(&msi_lock, flags); 992 993 if (nvec > free_vectors) { 994 if (free_vectors > 0) 995 return free_vectors; 996 else 997 return -EBUSY; 998 } 999 1000 status = msix_capability_init(dev, entries, nvec); 1001 if (!status && nr_msix_devices > 0) 1002 nr_msix_devices--; 1003 1004 return status; 1005} 1006 1007void pci_disable_msix(struct pci_dev* dev) 1008{ 1009 int pos, temp; 1010 u16 control; 1011 1012 if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX))) 1013 return; 1014 1015 pci_read_config_word(dev, msi_control_reg(pos), &control); 1016 if (!(control & PCI_MSIX_FLAGS_ENABLE)) 1017 return; 1018 1019 temp = dev->irq; 1020 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 1021 int state, vector, head, tail = 0, warning = 0; 1022 unsigned long flags; 1023 1024 vector = head = dev->irq; 1025 spin_lock_irqsave(&msi_lock, flags); 1026 while (head != tail) { 1027 state = msi_desc[vector]->msi_attrib.state; 1028 if (state) 1029 warning = 1; 1030 else { 1031 vector_irq[vector] = 0; /* free it */ 1032 nr_released_vectors++; 1033 } 1034 tail = msi_desc[vector]->link.tail; 1035 vector = tail; 1036 } 1037 spin_unlock_irqrestore(&msi_lock, flags); 1038 if (warning) { 1039 dev->irq = temp; 1040 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without " 1041 "free_irq() on all MSI-X vectors\n", 1042 pci_name(dev)); 1043 BUG_ON(warning > 0); 1044 } else { 1045 dev->irq = temp; 1046 disable_msi_mode(dev, 1047 pci_find_capability(dev, PCI_CAP_ID_MSIX), 1048 PCI_CAP_ID_MSIX); 1049 1050 } 1051 } 1052} 1053 1054/** 1055 * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state 1056 * @dev: pointer to the pci_dev data structure of MSI(X) device function 1057 * 1058 * Being called during hotplug remove, from which the device function 1059 * is hot-removed. All previous assigned MSI/MSI-X vectors, if 1060 * allocated for this device function, are reclaimed to unused state, 1061 * which may be used later on. 1062 **/ 1063void msi_remove_pci_irq_vectors(struct pci_dev* dev) 1064{ 1065 int state, pos, temp; 1066 unsigned long flags; 1067 1068 if (!pci_msi_enable || !dev) 1069 return; 1070 1071 temp = dev->irq; /* Save IOAPIC IRQ */ 1072 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSI)) > 0 && 1073 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 1074 spin_lock_irqsave(&msi_lock, flags); 1075 state = msi_desc[dev->irq]->msi_attrib.state; 1076 spin_unlock_irqrestore(&msi_lock, flags); 1077 if (state) { 1078 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 1079 "called without free_irq() on MSI vector %d\n", 1080 pci_name(dev), dev->irq); 1081 BUG_ON(state > 0); 1082 } else /* Release MSI vector assigned to this device */ 1083 msi_free_vector(dev, dev->irq, 0); 1084 dev->irq = temp; /* Restore IOAPIC IRQ */ 1085 } 1086 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 && 1087 !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 1088 int vector, head, tail = 0, warning = 0; 1089 void __iomem *base = NULL; 1090 1091 vector = head = dev->irq; 1092 while (head != tail) { 1093 spin_lock_irqsave(&msi_lock, flags); 1094 state = msi_desc[vector]->msi_attrib.state; 1095 tail = msi_desc[vector]->link.tail; 1096 base = msi_desc[vector]->mask_base; 1097 spin_unlock_irqrestore(&msi_lock, flags); 1098 if (state) 1099 warning = 1; 1100 else if (vector != head) /* Release MSI-X vector */ 1101 msi_free_vector(dev, vector, 0); 1102 vector = tail; 1103 } 1104 msi_free_vector(dev, vector, 0); 1105 if (warning) { 1106 /* Force to release the MSI-X memory-mapped table */ 1107 u32 phys_addr, table_offset; 1108 u16 control; 1109 u8 bir; 1110 1111 pci_read_config_word(dev, msi_control_reg(pos), 1112 &control); 1113 pci_read_config_dword(dev, msix_table_offset_reg(pos), 1114 &table_offset); 1115 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 1116 phys_addr = pci_resource_start (dev, bir); 1117 phys_addr += (u32)(table_offset & 1118 ~PCI_MSIX_FLAGS_BIRMASK); 1119 iounmap(base); 1120 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 1121 "called without free_irq() on all MSI-X vectors\n", 1122 pci_name(dev)); 1123 BUG_ON(warning > 0); 1124 } 1125 dev->irq = temp; /* Restore IOAPIC IRQ */ 1126 } 1127} 1128 1129EXPORT_SYMBOL(pci_enable_msi); 1130EXPORT_SYMBOL(pci_disable_msi); 1131EXPORT_SYMBOL(pci_enable_msix); 1132EXPORT_SYMBOL(pci_disable_msix); 1133