msi.c revision a04ce0ffcaf561994ecf382cd3caad75556dc499
1/* 2 * File: msi.c 3 * Purpose: PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 */ 8 9#include <linux/mm.h> 10#include <linux/irq.h> 11#include <linux/interrupt.h> 12#include <linux/init.h> 13#include <linux/config.h> 14#include <linux/ioport.h> 15#include <linux/smp_lock.h> 16#include <linux/pci.h> 17#include <linux/proc_fs.h> 18 19#include <asm/errno.h> 20#include <asm/io.h> 21#include <asm/smp.h> 22 23#include "pci.h" 24#include "msi.h" 25 26static DEFINE_SPINLOCK(msi_lock); 27static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL }; 28static kmem_cache_t* msi_cachep; 29 30static int pci_msi_enable = 1; 31static int last_alloc_vector; 32static int nr_released_vectors; 33static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS; 34static int nr_msix_devices; 35 36#ifndef CONFIG_X86_IO_APIC 37int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1}; 38u8 irq_vector[NR_IRQ_VECTORS] = { FIRST_DEVICE_VECTOR , 0 }; 39#endif 40 41static void msi_cache_ctor(void *p, kmem_cache_t *cache, unsigned long flags) 42{ 43 memset(p, 0, NR_IRQS * sizeof(struct msi_desc)); 44} 45 46static int msi_cache_init(void) 47{ 48 msi_cachep = kmem_cache_create("msi_cache", 49 NR_IRQS * sizeof(struct msi_desc), 50 0, SLAB_HWCACHE_ALIGN, msi_cache_ctor, NULL); 51 if (!msi_cachep) 52 return -ENOMEM; 53 54 return 0; 55} 56 57static void msi_set_mask_bit(unsigned int vector, int flag) 58{ 59 struct msi_desc *entry; 60 61 entry = (struct msi_desc *)msi_desc[vector]; 62 if (!entry || !entry->dev || !entry->mask_base) 63 return; 64 switch (entry->msi_attrib.type) { 65 case PCI_CAP_ID_MSI: 66 { 67 int pos; 68 u32 mask_bits; 69 70 pos = (long)entry->mask_base; 71 pci_read_config_dword(entry->dev, pos, &mask_bits); 72 mask_bits &= ~(1); 73 mask_bits |= flag; 74 pci_write_config_dword(entry->dev, pos, mask_bits); 75 break; 76 } 77 case PCI_CAP_ID_MSIX: 78 { 79 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 80 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; 81 writel(flag, entry->mask_base + offset); 82 break; 83 } 84 default: 85 break; 86 } 87} 88 89#ifdef CONFIG_SMP 90static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask) 91{ 92 struct msi_desc *entry; 93 struct msg_address address; 94 unsigned int irq = vector; 95 96 entry = (struct msi_desc *)msi_desc[vector]; 97 if (!entry || !entry->dev) 98 return; 99 100 switch (entry->msi_attrib.type) { 101 case PCI_CAP_ID_MSI: 102 { 103 int pos; 104 105 if (!(pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI))) 106 return; 107 108 pci_read_config_dword(entry->dev, msi_lower_address_reg(pos), 109 &address.lo_address.value); 110 address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK; 111 address.lo_address.value |= (cpu_mask_to_apicid(cpu_mask) << 112 MSI_TARGET_CPU_SHIFT); 113 entry->msi_attrib.current_cpu = cpu_mask_to_apicid(cpu_mask); 114 pci_write_config_dword(entry->dev, msi_lower_address_reg(pos), 115 address.lo_address.value); 116 set_native_irq_info(irq, cpu_mask); 117 break; 118 } 119 case PCI_CAP_ID_MSIX: 120 { 121 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + 122 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET; 123 124 address.lo_address.value = readl(entry->mask_base + offset); 125 address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK; 126 address.lo_address.value |= (cpu_mask_to_apicid(cpu_mask) << 127 MSI_TARGET_CPU_SHIFT); 128 entry->msi_attrib.current_cpu = cpu_mask_to_apicid(cpu_mask); 129 writel(address.lo_address.value, entry->mask_base + offset); 130 set_native_irq_info(irq, cpu_mask); 131 break; 132 } 133 default: 134 break; 135 } 136} 137#endif /* CONFIG_SMP */ 138 139static void mask_MSI_irq(unsigned int vector) 140{ 141 msi_set_mask_bit(vector, 1); 142} 143 144static void unmask_MSI_irq(unsigned int vector) 145{ 146 msi_set_mask_bit(vector, 0); 147} 148 149static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector) 150{ 151 struct msi_desc *entry; 152 unsigned long flags; 153 154 spin_lock_irqsave(&msi_lock, flags); 155 entry = msi_desc[vector]; 156 if (!entry || !entry->dev) { 157 spin_unlock_irqrestore(&msi_lock, flags); 158 return 0; 159 } 160 entry->msi_attrib.state = 1; /* Mark it active */ 161 spin_unlock_irqrestore(&msi_lock, flags); 162 163 return 0; /* never anything pending */ 164} 165 166static unsigned int startup_msi_irq_w_maskbit(unsigned int vector) 167{ 168 startup_msi_irq_wo_maskbit(vector); 169 unmask_MSI_irq(vector); 170 return 0; /* never anything pending */ 171} 172 173static void shutdown_msi_irq(unsigned int vector) 174{ 175 struct msi_desc *entry; 176 unsigned long flags; 177 178 spin_lock_irqsave(&msi_lock, flags); 179 entry = msi_desc[vector]; 180 if (entry && entry->dev) 181 entry->msi_attrib.state = 0; /* Mark it not active */ 182 spin_unlock_irqrestore(&msi_lock, flags); 183} 184 185static void end_msi_irq_wo_maskbit(unsigned int vector) 186{ 187 move_native_irq(vector); 188 ack_APIC_irq(); 189} 190 191static void end_msi_irq_w_maskbit(unsigned int vector) 192{ 193 move_native_irq(vector); 194 unmask_MSI_irq(vector); 195 ack_APIC_irq(); 196} 197 198static void do_nothing(unsigned int vector) 199{ 200} 201 202/* 203 * Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices, 204 * which implement the MSI-X Capability Structure. 205 */ 206static struct hw_interrupt_type msix_irq_type = { 207 .typename = "PCI-MSI-X", 208 .startup = startup_msi_irq_w_maskbit, 209 .shutdown = shutdown_msi_irq, 210 .enable = unmask_MSI_irq, 211 .disable = mask_MSI_irq, 212 .ack = mask_MSI_irq, 213 .end = end_msi_irq_w_maskbit, 214 .set_affinity = set_msi_irq_affinity 215}; 216 217/* 218 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices, 219 * which implement the MSI Capability Structure with 220 * Mask-and-Pending Bits. 221 */ 222static struct hw_interrupt_type msi_irq_w_maskbit_type = { 223 .typename = "PCI-MSI", 224 .startup = startup_msi_irq_w_maskbit, 225 .shutdown = shutdown_msi_irq, 226 .enable = unmask_MSI_irq, 227 .disable = mask_MSI_irq, 228 .ack = mask_MSI_irq, 229 .end = end_msi_irq_w_maskbit, 230 .set_affinity = set_msi_irq_affinity 231}; 232 233/* 234 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices, 235 * which implement the MSI Capability Structure without 236 * Mask-and-Pending Bits. 237 */ 238static struct hw_interrupt_type msi_irq_wo_maskbit_type = { 239 .typename = "PCI-MSI", 240 .startup = startup_msi_irq_wo_maskbit, 241 .shutdown = shutdown_msi_irq, 242 .enable = do_nothing, 243 .disable = do_nothing, 244 .ack = do_nothing, 245 .end = end_msi_irq_wo_maskbit, 246 .set_affinity = set_msi_irq_affinity 247}; 248 249static void msi_data_init(struct msg_data *msi_data, 250 unsigned int vector) 251{ 252 memset(msi_data, 0, sizeof(struct msg_data)); 253 msi_data->vector = (u8)vector; 254 msi_data->delivery_mode = MSI_DELIVERY_MODE; 255 msi_data->level = MSI_LEVEL_MODE; 256 msi_data->trigger = MSI_TRIGGER_MODE; 257} 258 259static void msi_address_init(struct msg_address *msi_address) 260{ 261 unsigned int dest_id; 262 263 memset(msi_address, 0, sizeof(struct msg_address)); 264 msi_address->hi_address = (u32)0; 265 dest_id = (MSI_ADDRESS_HEADER << MSI_ADDRESS_HEADER_SHIFT); 266 msi_address->lo_address.u.dest_mode = MSI_DEST_MODE; 267 msi_address->lo_address.u.redirection_hint = MSI_REDIRECTION_HINT_MODE; 268 msi_address->lo_address.u.dest_id = dest_id; 269 msi_address->lo_address.value |= (MSI_TARGET_CPU << MSI_TARGET_CPU_SHIFT); 270} 271 272static int msi_free_vector(struct pci_dev* dev, int vector, int reassign); 273static int assign_msi_vector(void) 274{ 275 static int new_vector_avail = 1; 276 int vector; 277 unsigned long flags; 278 279 /* 280 * msi_lock is provided to ensure that successful allocation of MSI 281 * vector is assigned unique among drivers. 282 */ 283 spin_lock_irqsave(&msi_lock, flags); 284 285 if (!new_vector_avail) { 286 int free_vector = 0; 287 288 /* 289 * vector_irq[] = -1 indicates that this specific vector is: 290 * - assigned for MSI (since MSI have no associated IRQ) or 291 * - assigned for legacy if less than 16, or 292 * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping 293 * vector_irq[] = 0 indicates that this vector, previously 294 * assigned for MSI, is freed by hotplug removed operations. 295 * This vector will be reused for any subsequent hotplug added 296 * operations. 297 * vector_irq[] > 0 indicates that this vector is assigned for 298 * IOxAPIC IRQs. This vector and its value provides a 1-to-1 299 * vector-to-IOxAPIC IRQ mapping. 300 */ 301 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) { 302 if (vector_irq[vector] != 0) 303 continue; 304 free_vector = vector; 305 if (!msi_desc[vector]) 306 break; 307 else 308 continue; 309 } 310 if (!free_vector) { 311 spin_unlock_irqrestore(&msi_lock, flags); 312 return -EBUSY; 313 } 314 vector_irq[free_vector] = -1; 315 nr_released_vectors--; 316 spin_unlock_irqrestore(&msi_lock, flags); 317 if (msi_desc[free_vector] != NULL) { 318 struct pci_dev *dev; 319 int tail; 320 321 /* free all linked vectors before re-assign */ 322 do { 323 spin_lock_irqsave(&msi_lock, flags); 324 dev = msi_desc[free_vector]->dev; 325 tail = msi_desc[free_vector]->link.tail; 326 spin_unlock_irqrestore(&msi_lock, flags); 327 msi_free_vector(dev, tail, 1); 328 } while (free_vector != tail); 329 } 330 331 return free_vector; 332 } 333 vector = assign_irq_vector(AUTO_ASSIGN); 334 last_alloc_vector = vector; 335 if (vector == LAST_DEVICE_VECTOR) 336 new_vector_avail = 0; 337 338 spin_unlock_irqrestore(&msi_lock, flags); 339 return vector; 340} 341 342static int get_new_vector(void) 343{ 344 int vector; 345 346 if ((vector = assign_msi_vector()) > 0) 347 set_intr_gate(vector, interrupt[vector]); 348 349 return vector; 350} 351 352static int msi_init(void) 353{ 354 static int status = -ENOMEM; 355 356 if (!status) 357 return status; 358 359 if (pci_msi_quirk) { 360 pci_msi_enable = 0; 361 printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n"); 362 status = -EINVAL; 363 return status; 364 } 365 366 if ((status = msi_cache_init()) < 0) { 367 pci_msi_enable = 0; 368 printk(KERN_WARNING "PCI: MSI cache init failed\n"); 369 return status; 370 } 371 last_alloc_vector = assign_irq_vector(AUTO_ASSIGN); 372 if (last_alloc_vector < 0) { 373 pci_msi_enable = 0; 374 printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n"); 375 status = -EBUSY; 376 return status; 377 } 378 vector_irq[last_alloc_vector] = 0; 379 nr_released_vectors++; 380 381 return status; 382} 383 384static int get_msi_vector(struct pci_dev *dev) 385{ 386 return get_new_vector(); 387} 388 389static struct msi_desc* alloc_msi_entry(void) 390{ 391 struct msi_desc *entry; 392 393 entry = kmem_cache_alloc(msi_cachep, SLAB_KERNEL); 394 if (!entry) 395 return NULL; 396 397 memset(entry, 0, sizeof(struct msi_desc)); 398 entry->link.tail = entry->link.head = 0; /* single message */ 399 entry->dev = NULL; 400 401 return entry; 402} 403 404static void attach_msi_entry(struct msi_desc *entry, int vector) 405{ 406 unsigned long flags; 407 408 spin_lock_irqsave(&msi_lock, flags); 409 msi_desc[vector] = entry; 410 spin_unlock_irqrestore(&msi_lock, flags); 411} 412 413static void irq_handler_init(int cap_id, int pos, int mask) 414{ 415 spin_lock(&irq_desc[pos].lock); 416 if (cap_id == PCI_CAP_ID_MSIX) 417 irq_desc[pos].handler = &msix_irq_type; 418 else { 419 if (!mask) 420 irq_desc[pos].handler = &msi_irq_wo_maskbit_type; 421 else 422 irq_desc[pos].handler = &msi_irq_w_maskbit_type; 423 } 424 spin_unlock(&irq_desc[pos].lock); 425} 426 427static void enable_msi_mode(struct pci_dev *dev, int pos, int type) 428{ 429 u16 control; 430 431 pci_read_config_word(dev, msi_control_reg(pos), &control); 432 if (type == PCI_CAP_ID_MSI) { 433 /* Set enabled bits to single MSI & enable MSI_enable bit */ 434 msi_enable(control, 1); 435 pci_write_config_word(dev, msi_control_reg(pos), control); 436 } else { 437 msix_enable(control); 438 pci_write_config_word(dev, msi_control_reg(pos), control); 439 } 440 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) { 441 /* PCI Express Endpoint device detected */ 442 pci_intx(dev, 0); /* disable intx */ 443 } 444} 445 446void disable_msi_mode(struct pci_dev *dev, int pos, int type) 447{ 448 u16 control; 449 450 pci_read_config_word(dev, msi_control_reg(pos), &control); 451 if (type == PCI_CAP_ID_MSI) { 452 /* Set enabled bits to single MSI & enable MSI_enable bit */ 453 msi_disable(control); 454 pci_write_config_word(dev, msi_control_reg(pos), control); 455 } else { 456 msix_disable(control); 457 pci_write_config_word(dev, msi_control_reg(pos), control); 458 } 459 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) { 460 /* PCI Express Endpoint device detected */ 461 pci_intx(dev, 1); /* enable intx */ 462 } 463} 464 465static int msi_lookup_vector(struct pci_dev *dev, int type) 466{ 467 int vector; 468 unsigned long flags; 469 470 spin_lock_irqsave(&msi_lock, flags); 471 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) { 472 if (!msi_desc[vector] || msi_desc[vector]->dev != dev || 473 msi_desc[vector]->msi_attrib.type != type || 474 msi_desc[vector]->msi_attrib.default_vector != dev->irq) 475 continue; 476 spin_unlock_irqrestore(&msi_lock, flags); 477 /* This pre-assigned MSI vector for this device 478 already exits. Override dev->irq with this vector */ 479 dev->irq = vector; 480 return 0; 481 } 482 spin_unlock_irqrestore(&msi_lock, flags); 483 484 return -EACCES; 485} 486 487void pci_scan_msi_device(struct pci_dev *dev) 488{ 489 if (!dev) 490 return; 491 492 if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0) 493 nr_msix_devices++; 494 else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0) 495 nr_reserved_vectors++; 496} 497 498/** 499 * msi_capability_init - configure device's MSI capability structure 500 * @dev: pointer to the pci_dev data structure of MSI device function 501 * 502 * Setup the MSI capability structure of device function with a single 503 * MSI vector, regardless of device function is capable of handling 504 * multiple messages. A return of zero indicates the successful setup 505 * of an entry zero with the new MSI vector or non-zero for otherwise. 506 **/ 507static int msi_capability_init(struct pci_dev *dev) 508{ 509 struct msi_desc *entry; 510 struct msg_address address; 511 struct msg_data data; 512 int pos, vector; 513 u16 control; 514 515 pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 516 pci_read_config_word(dev, msi_control_reg(pos), &control); 517 /* MSI Entry Initialization */ 518 if (!(entry = alloc_msi_entry())) 519 return -ENOMEM; 520 521 if ((vector = get_msi_vector(dev)) < 0) { 522 kmem_cache_free(msi_cachep, entry); 523 return -EBUSY; 524 } 525 entry->link.head = vector; 526 entry->link.tail = vector; 527 entry->msi_attrib.type = PCI_CAP_ID_MSI; 528 entry->msi_attrib.state = 0; /* Mark it not active */ 529 entry->msi_attrib.entry_nr = 0; 530 entry->msi_attrib.maskbit = is_mask_bit_support(control); 531 entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */ 532 dev->irq = vector; 533 entry->dev = dev; 534 if (is_mask_bit_support(control)) { 535 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, 536 is_64bit_address(control)); 537 } 538 /* Replace with MSI handler */ 539 irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit); 540 /* Configure MSI capability structure */ 541 msi_address_init(&address); 542 msi_data_init(&data, vector); 543 entry->msi_attrib.current_cpu = ((address.lo_address.u.dest_id >> 544 MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK); 545 pci_write_config_dword(dev, msi_lower_address_reg(pos), 546 address.lo_address.value); 547 if (is_64bit_address(control)) { 548 pci_write_config_dword(dev, 549 msi_upper_address_reg(pos), address.hi_address); 550 pci_write_config_word(dev, 551 msi_data_reg(pos, 1), *((u32*)&data)); 552 } else 553 pci_write_config_word(dev, 554 msi_data_reg(pos, 0), *((u32*)&data)); 555 if (entry->msi_attrib.maskbit) { 556 unsigned int maskbits, temp; 557 /* All MSIs are unmasked by default, Mask them all */ 558 pci_read_config_dword(dev, 559 msi_mask_bits_reg(pos, is_64bit_address(control)), 560 &maskbits); 561 temp = (1 << multi_msi_capable(control)); 562 temp = ((temp - 1) & ~temp); 563 maskbits |= temp; 564 pci_write_config_dword(dev, 565 msi_mask_bits_reg(pos, is_64bit_address(control)), 566 maskbits); 567 } 568 attach_msi_entry(entry, vector); 569 /* Set MSI enabled bits */ 570 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 571 572 return 0; 573} 574 575/** 576 * msix_capability_init - configure device's MSI-X capability 577 * @dev: pointer to the pci_dev data structure of MSI-X device function 578 * 579 * Setup the MSI-X capability structure of device function with a 580 * single MSI-X vector. A return of zero indicates the successful setup of 581 * requested MSI-X entries with allocated vectors or non-zero for otherwise. 582 **/ 583static int msix_capability_init(struct pci_dev *dev, 584 struct msix_entry *entries, int nvec) 585{ 586 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL; 587 struct msg_address address; 588 struct msg_data data; 589 int vector, pos, i, j, nr_entries, temp = 0; 590 u32 phys_addr, table_offset; 591 u16 control; 592 u8 bir; 593 void __iomem *base; 594 595 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 596 /* Request & Map MSI-X table region */ 597 pci_read_config_word(dev, msi_control_reg(pos), &control); 598 nr_entries = multi_msix_capable(control); 599 pci_read_config_dword(dev, msix_table_offset_reg(pos), 600 &table_offset); 601 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 602 phys_addr = pci_resource_start (dev, bir); 603 phys_addr += (u32)(table_offset & ~PCI_MSIX_FLAGS_BIRMASK); 604 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 605 if (base == NULL) 606 return -ENOMEM; 607 608 /* MSI-X Table Initialization */ 609 for (i = 0; i < nvec; i++) { 610 entry = alloc_msi_entry(); 611 if (!entry) 612 break; 613 if ((vector = get_msi_vector(dev)) < 0) 614 break; 615 616 j = entries[i].entry; 617 entries[i].vector = vector; 618 entry->msi_attrib.type = PCI_CAP_ID_MSIX; 619 entry->msi_attrib.state = 0; /* Mark it not active */ 620 entry->msi_attrib.entry_nr = j; 621 entry->msi_attrib.maskbit = 1; 622 entry->msi_attrib.default_vector = dev->irq; 623 entry->dev = dev; 624 entry->mask_base = base; 625 if (!head) { 626 entry->link.head = vector; 627 entry->link.tail = vector; 628 head = entry; 629 } else { 630 entry->link.head = temp; 631 entry->link.tail = tail->link.tail; 632 tail->link.tail = vector; 633 head->link.head = vector; 634 } 635 temp = vector; 636 tail = entry; 637 /* Replace with MSI-X handler */ 638 irq_handler_init(PCI_CAP_ID_MSIX, vector, 1); 639 /* Configure MSI-X capability structure */ 640 msi_address_init(&address); 641 msi_data_init(&data, vector); 642 entry->msi_attrib.current_cpu = 643 ((address.lo_address.u.dest_id >> 644 MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK); 645 writel(address.lo_address.value, 646 base + j * PCI_MSIX_ENTRY_SIZE + 647 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 648 writel(address.hi_address, 649 base + j * PCI_MSIX_ENTRY_SIZE + 650 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 651 writel(*(u32*)&data, 652 base + j * PCI_MSIX_ENTRY_SIZE + 653 PCI_MSIX_ENTRY_DATA_OFFSET); 654 attach_msi_entry(entry, vector); 655 } 656 if (i != nvec) { 657 i--; 658 for (; i >= 0; i--) { 659 vector = (entries + i)->vector; 660 msi_free_vector(dev, vector, 0); 661 (entries + i)->vector = 0; 662 } 663 return -EBUSY; 664 } 665 /* Set MSI-X enabled bits */ 666 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 667 668 return 0; 669} 670 671/** 672 * pci_enable_msi - configure device's MSI capability structure 673 * @dev: pointer to the pci_dev data structure of MSI device function 674 * 675 * Setup the MSI capability structure of device function with 676 * a single MSI vector upon its software driver call to request for 677 * MSI mode enabled on its hardware device function. A return of zero 678 * indicates the successful setup of an entry zero with the new MSI 679 * vector or non-zero for otherwise. 680 **/ 681int pci_enable_msi(struct pci_dev* dev) 682{ 683 int pos, temp, status = -EINVAL; 684 u16 control; 685 686 if (!pci_msi_enable || !dev) 687 return status; 688 689 if (dev->no_msi) 690 return status; 691 692 temp = dev->irq; 693 694 if ((status = msi_init()) < 0) 695 return status; 696 697 if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSI))) 698 return -EINVAL; 699 700 pci_read_config_word(dev, msi_control_reg(pos), &control); 701 if (control & PCI_MSI_FLAGS_ENABLE) 702 return 0; /* Already in MSI mode */ 703 704 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 705 /* Lookup Sucess */ 706 unsigned long flags; 707 708 spin_lock_irqsave(&msi_lock, flags); 709 if (!vector_irq[dev->irq]) { 710 msi_desc[dev->irq]->msi_attrib.state = 0; 711 vector_irq[dev->irq] = -1; 712 nr_released_vectors--; 713 spin_unlock_irqrestore(&msi_lock, flags); 714 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); 715 return 0; 716 } 717 spin_unlock_irqrestore(&msi_lock, flags); 718 dev->irq = temp; 719 } 720 /* Check whether driver already requested for MSI-X vectors */ 721 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 && 722 !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 723 printk(KERN_INFO "PCI: %s: Can't enable MSI. " 724 "Device already has MSI-X vectors assigned\n", 725 pci_name(dev)); 726 dev->irq = temp; 727 return -EINVAL; 728 } 729 status = msi_capability_init(dev); 730 if (!status) { 731 if (!pos) 732 nr_reserved_vectors--; /* Only MSI capable */ 733 else if (nr_msix_devices > 0) 734 nr_msix_devices--; /* Both MSI and MSI-X capable, 735 but choose enabling MSI */ 736 } 737 738 return status; 739} 740 741void pci_disable_msi(struct pci_dev* dev) 742{ 743 struct msi_desc *entry; 744 int pos, default_vector; 745 u16 control; 746 unsigned long flags; 747 748 if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSI))) 749 return; 750 751 pci_read_config_word(dev, msi_control_reg(pos), &control); 752 if (!(control & PCI_MSI_FLAGS_ENABLE)) 753 return; 754 755 spin_lock_irqsave(&msi_lock, flags); 756 entry = msi_desc[dev->irq]; 757 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { 758 spin_unlock_irqrestore(&msi_lock, flags); 759 return; 760 } 761 if (entry->msi_attrib.state) { 762 spin_unlock_irqrestore(&msi_lock, flags); 763 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without " 764 "free_irq() on MSI vector %d\n", 765 pci_name(dev), dev->irq); 766 BUG_ON(entry->msi_attrib.state > 0); 767 } else { 768 vector_irq[dev->irq] = 0; /* free it */ 769 nr_released_vectors++; 770 default_vector = entry->msi_attrib.default_vector; 771 spin_unlock_irqrestore(&msi_lock, flags); 772 /* Restore dev->irq to its default pin-assertion vector */ 773 dev->irq = default_vector; 774 disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI), 775 PCI_CAP_ID_MSI); 776 } 777} 778 779static int msi_free_vector(struct pci_dev* dev, int vector, int reassign) 780{ 781 struct msi_desc *entry; 782 int head, entry_nr, type; 783 void __iomem *base; 784 unsigned long flags; 785 786 spin_lock_irqsave(&msi_lock, flags); 787 entry = msi_desc[vector]; 788 if (!entry || entry->dev != dev) { 789 spin_unlock_irqrestore(&msi_lock, flags); 790 return -EINVAL; 791 } 792 type = entry->msi_attrib.type; 793 entry_nr = entry->msi_attrib.entry_nr; 794 head = entry->link.head; 795 base = entry->mask_base; 796 msi_desc[entry->link.head]->link.tail = entry->link.tail; 797 msi_desc[entry->link.tail]->link.head = entry->link.head; 798 entry->dev = NULL; 799 if (!reassign) { 800 vector_irq[vector] = 0; 801 nr_released_vectors++; 802 } 803 msi_desc[vector] = NULL; 804 spin_unlock_irqrestore(&msi_lock, flags); 805 806 kmem_cache_free(msi_cachep, entry); 807 808 if (type == PCI_CAP_ID_MSIX) { 809 if (!reassign) 810 writel(1, base + 811 entry_nr * PCI_MSIX_ENTRY_SIZE + 812 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); 813 814 if (head == vector) { 815 /* 816 * Detect last MSI-X vector to be released. 817 * Release the MSI-X memory-mapped table. 818 */ 819 int pos, nr_entries; 820 u32 phys_addr, table_offset; 821 u16 control; 822 u8 bir; 823 824 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); 825 pci_read_config_word(dev, msi_control_reg(pos), 826 &control); 827 nr_entries = multi_msix_capable(control); 828 pci_read_config_dword(dev, msix_table_offset_reg(pos), 829 &table_offset); 830 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 831 phys_addr = pci_resource_start (dev, bir); 832 phys_addr += (u32)(table_offset & 833 ~PCI_MSIX_FLAGS_BIRMASK); 834 iounmap(base); 835 } 836 } 837 838 return 0; 839} 840 841static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec) 842{ 843 int vector = head, tail = 0; 844 int i, j = 0, nr_entries = 0; 845 void __iomem *base; 846 unsigned long flags; 847 848 spin_lock_irqsave(&msi_lock, flags); 849 while (head != tail) { 850 nr_entries++; 851 tail = msi_desc[vector]->link.tail; 852 if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr) 853 j = vector; 854 vector = tail; 855 } 856 if (*nvec > nr_entries) { 857 spin_unlock_irqrestore(&msi_lock, flags); 858 *nvec = nr_entries; 859 return -EINVAL; 860 } 861 vector = ((j > 0) ? j : head); 862 for (i = 0; i < *nvec; i++) { 863 j = msi_desc[vector]->msi_attrib.entry_nr; 864 msi_desc[vector]->msi_attrib.state = 0; /* Mark it not active */ 865 vector_irq[vector] = -1; /* Mark it busy */ 866 nr_released_vectors--; 867 entries[i].vector = vector; 868 if (j != (entries + i)->entry) { 869 base = msi_desc[vector]->mask_base; 870 msi_desc[vector]->msi_attrib.entry_nr = 871 (entries + i)->entry; 872 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE + 873 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base + 874 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE + 875 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); 876 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE + 877 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base + 878 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE + 879 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); 880 writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE + 881 PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector, 882 base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE + 883 PCI_MSIX_ENTRY_DATA_OFFSET); 884 } 885 vector = msi_desc[vector]->link.tail; 886 } 887 spin_unlock_irqrestore(&msi_lock, flags); 888 889 return 0; 890} 891 892/** 893 * pci_enable_msix - configure device's MSI-X capability structure 894 * @dev: pointer to the pci_dev data structure of MSI-X device function 895 * @entries: pointer to an array of MSI-X entries 896 * @nvec: number of MSI-X vectors requested for allocation by device driver 897 * 898 * Setup the MSI-X capability structure of device function with the number 899 * of requested vectors upon its software driver call to request for 900 * MSI-X mode enabled on its hardware device function. A return of zero 901 * indicates the successful configuration of MSI-X capability structure 902 * with new allocated MSI-X vectors. A return of < 0 indicates a failure. 903 * Or a return of > 0 indicates that driver request is exceeding the number 904 * of vectors available. Driver should use the returned value to re-send 905 * its request. 906 **/ 907int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) 908{ 909 int status, pos, nr_entries, free_vectors; 910 int i, j, temp; 911 u16 control; 912 unsigned long flags; 913 914 if (!pci_msi_enable || !dev || !entries) 915 return -EINVAL; 916 917 if ((status = msi_init()) < 0) 918 return status; 919 920 if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX))) 921 return -EINVAL; 922 923 pci_read_config_word(dev, msi_control_reg(pos), &control); 924 if (control & PCI_MSIX_FLAGS_ENABLE) 925 return -EINVAL; /* Already in MSI-X mode */ 926 927 nr_entries = multi_msix_capable(control); 928 if (nvec > nr_entries) 929 return -EINVAL; 930 931 /* Check for any invalid entries */ 932 for (i = 0; i < nvec; i++) { 933 if (entries[i].entry >= nr_entries) 934 return -EINVAL; /* invalid entry */ 935 for (j = i + 1; j < nvec; j++) { 936 if (entries[i].entry == entries[j].entry) 937 return -EINVAL; /* duplicate entry */ 938 } 939 } 940 temp = dev->irq; 941 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 942 /* Lookup Sucess */ 943 nr_entries = nvec; 944 /* Reroute MSI-X table */ 945 if (reroute_msix_table(dev->irq, entries, &nr_entries)) { 946 /* #requested > #previous-assigned */ 947 dev->irq = temp; 948 return nr_entries; 949 } 950 dev->irq = temp; 951 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); 952 return 0; 953 } 954 /* Check whether driver already requested for MSI vector */ 955 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 && 956 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 957 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " 958 "Device already has an MSI vector assigned\n", 959 pci_name(dev)); 960 dev->irq = temp; 961 return -EINVAL; 962 } 963 964 spin_lock_irqsave(&msi_lock, flags); 965 /* 966 * msi_lock is provided to ensure that enough vectors resources are 967 * available before granting. 968 */ 969 free_vectors = pci_vector_resources(last_alloc_vector, 970 nr_released_vectors); 971 /* Ensure that each MSI/MSI-X device has one vector reserved by 972 default to avoid any MSI-X driver to take all available 973 resources */ 974 free_vectors -= nr_reserved_vectors; 975 /* Find the average of free vectors among MSI-X devices */ 976 if (nr_msix_devices > 0) 977 free_vectors /= nr_msix_devices; 978 spin_unlock_irqrestore(&msi_lock, flags); 979 980 if (nvec > free_vectors) { 981 if (free_vectors > 0) 982 return free_vectors; 983 else 984 return -EBUSY; 985 } 986 987 status = msix_capability_init(dev, entries, nvec); 988 if (!status && nr_msix_devices > 0) 989 nr_msix_devices--; 990 991 return status; 992} 993 994void pci_disable_msix(struct pci_dev* dev) 995{ 996 int pos, temp; 997 u16 control; 998 999 if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX))) 1000 return; 1001 1002 pci_read_config_word(dev, msi_control_reg(pos), &control); 1003 if (!(control & PCI_MSIX_FLAGS_ENABLE)) 1004 return; 1005 1006 temp = dev->irq; 1007 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 1008 int state, vector, head, tail = 0, warning = 0; 1009 unsigned long flags; 1010 1011 vector = head = dev->irq; 1012 spin_lock_irqsave(&msi_lock, flags); 1013 while (head != tail) { 1014 state = msi_desc[vector]->msi_attrib.state; 1015 if (state) 1016 warning = 1; 1017 else { 1018 vector_irq[vector] = 0; /* free it */ 1019 nr_released_vectors++; 1020 } 1021 tail = msi_desc[vector]->link.tail; 1022 vector = tail; 1023 } 1024 spin_unlock_irqrestore(&msi_lock, flags); 1025 if (warning) { 1026 dev->irq = temp; 1027 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without " 1028 "free_irq() on all MSI-X vectors\n", 1029 pci_name(dev)); 1030 BUG_ON(warning > 0); 1031 } else { 1032 dev->irq = temp; 1033 disable_msi_mode(dev, 1034 pci_find_capability(dev, PCI_CAP_ID_MSIX), 1035 PCI_CAP_ID_MSIX); 1036 1037 } 1038 } 1039} 1040 1041/** 1042 * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state 1043 * @dev: pointer to the pci_dev data structure of MSI(X) device function 1044 * 1045 * Being called during hotplug remove, from which the device function 1046 * is hot-removed. All previous assigned MSI/MSI-X vectors, if 1047 * allocated for this device function, are reclaimed to unused state, 1048 * which may be used later on. 1049 **/ 1050void msi_remove_pci_irq_vectors(struct pci_dev* dev) 1051{ 1052 int state, pos, temp; 1053 unsigned long flags; 1054 1055 if (!pci_msi_enable || !dev) 1056 return; 1057 1058 temp = dev->irq; /* Save IOAPIC IRQ */ 1059 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSI)) > 0 && 1060 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { 1061 spin_lock_irqsave(&msi_lock, flags); 1062 state = msi_desc[dev->irq]->msi_attrib.state; 1063 spin_unlock_irqrestore(&msi_lock, flags); 1064 if (state) { 1065 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 1066 "called without free_irq() on MSI vector %d\n", 1067 pci_name(dev), dev->irq); 1068 BUG_ON(state > 0); 1069 } else /* Release MSI vector assigned to this device */ 1070 msi_free_vector(dev, dev->irq, 0); 1071 dev->irq = temp; /* Restore IOAPIC IRQ */ 1072 } 1073 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 && 1074 !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { 1075 int vector, head, tail = 0, warning = 0; 1076 void __iomem *base = NULL; 1077 1078 vector = head = dev->irq; 1079 while (head != tail) { 1080 spin_lock_irqsave(&msi_lock, flags); 1081 state = msi_desc[vector]->msi_attrib.state; 1082 tail = msi_desc[vector]->link.tail; 1083 base = msi_desc[vector]->mask_base; 1084 spin_unlock_irqrestore(&msi_lock, flags); 1085 if (state) 1086 warning = 1; 1087 else if (vector != head) /* Release MSI-X vector */ 1088 msi_free_vector(dev, vector, 0); 1089 vector = tail; 1090 } 1091 msi_free_vector(dev, vector, 0); 1092 if (warning) { 1093 /* Force to release the MSI-X memory-mapped table */ 1094 u32 phys_addr, table_offset; 1095 u16 control; 1096 u8 bir; 1097 1098 pci_read_config_word(dev, msi_control_reg(pos), 1099 &control); 1100 pci_read_config_dword(dev, msix_table_offset_reg(pos), 1101 &table_offset); 1102 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); 1103 phys_addr = pci_resource_start (dev, bir); 1104 phys_addr += (u32)(table_offset & 1105 ~PCI_MSIX_FLAGS_BIRMASK); 1106 iounmap(base); 1107 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " 1108 "called without free_irq() on all MSI-X vectors\n", 1109 pci_name(dev)); 1110 BUG_ON(warning > 0); 1111 } 1112 dev->irq = temp; /* Restore IOAPIC IRQ */ 1113 } 1114} 1115 1116EXPORT_SYMBOL(pci_enable_msi); 1117EXPORT_SYMBOL(pci_disable_msi); 1118EXPORT_SYMBOL(pci_enable_msix); 1119EXPORT_SYMBOL(pci_disable_msix); 1120