1/* 2 * File: drivers/pci/pcie/aspm.c 3 * Enabling PCIe link L0s/L1 state and Clock Power Management 4 * 5 * Copyright (C) 2007 Intel 6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) 7 * Copyright (C) Shaohua Li (shaohua.li@intel.com) 8 */ 9 10#include <linux/kernel.h> 11#include <linux/module.h> 12#include <linux/moduleparam.h> 13#include <linux/pci.h> 14#include <linux/pci_regs.h> 15#include <linux/errno.h> 16#include <linux/pm.h> 17#include <linux/init.h> 18#include <linux/slab.h> 19#include <linux/jiffies.h> 20#include <linux/delay.h> 21#include <linux/pci-aspm.h> 22#include "../pci.h" 23 24#ifdef MODULE_PARAM_PREFIX 25#undef MODULE_PARAM_PREFIX 26#endif 27#define MODULE_PARAM_PREFIX "pcie_aspm." 28 29/* Note: those are not register definitions */ 30#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ 31#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ 32#define ASPM_STATE_L1 (4) /* L1 state */ 33#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) 34#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1) 35 36struct aspm_latency { 37 u32 l0s; /* L0s latency (nsec) */ 38 u32 l1; /* L1 latency (nsec) */ 39}; 40 41struct pcie_link_state { 42 struct pci_dev *pdev; /* Upstream component of the Link */ 43 struct pcie_link_state *root; /* pointer to the root port link */ 44 struct pcie_link_state *parent; /* pointer to the parent Link state */ 45 struct list_head sibling; /* node in link_list */ 46 struct list_head children; /* list of child link states */ 47 struct list_head link; /* node in parent's children list */ 48 49 /* ASPM state */ 50 u32 aspm_support:3; /* Supported ASPM state */ 51 u32 aspm_enabled:3; /* Enabled ASPM state */ 52 u32 aspm_capable:3; /* Capable ASPM state with latency */ 53 u32 aspm_default:3; /* Default ASPM state by BIOS */ 54 u32 aspm_disable:3; /* Disabled ASPM state */ 55 56 /* Clock PM state */ 57 u32 clkpm_capable:1; /* Clock PM capable? */ 58 u32 clkpm_enabled:1; /* Current Clock PM state */ 59 u32 clkpm_default:1; /* Default Clock PM state by BIOS */ 60 61 /* Exit latencies */ 62 struct aspm_latency latency_up; /* Upstream direction exit latency */ 63 struct aspm_latency latency_dw; /* Downstream direction exit latency */ 64 /* 65 * Endpoint acceptable latencies. A pcie downstream port only 66 * has one slot under it, so at most there are 8 functions. 67 */ 68 struct aspm_latency acceptable[8]; 69}; 70 71static int aspm_disabled, aspm_force; 72static bool aspm_support_enabled = true; 73static DEFINE_MUTEX(aspm_lock); 74static LIST_HEAD(link_list); 75 76#define POLICY_DEFAULT 0 /* BIOS default setting */ 77#define POLICY_PERFORMANCE 1 /* high performance */ 78#define POLICY_POWERSAVE 2 /* high power saving */ 79static int aspm_policy; 80static const char *policy_str[] = { 81 [POLICY_DEFAULT] = "default", 82 [POLICY_PERFORMANCE] = "performance", 83 [POLICY_POWERSAVE] = "powersave" 84}; 85 86#define LINK_RETRAIN_TIMEOUT HZ 87 88static int policy_to_aspm_state(struct pcie_link_state *link) 89{ 90 switch (aspm_policy) { 91 case POLICY_PERFORMANCE: 92 /* Disable ASPM and Clock PM */ 93 return 0; 94 case POLICY_POWERSAVE: 95 /* Enable ASPM L0s/L1 */ 96 return ASPM_STATE_ALL; 97 case POLICY_DEFAULT: 98 return link->aspm_default; 99 } 100 return 0; 101} 102 103static int policy_to_clkpm_state(struct pcie_link_state *link) 104{ 105 switch (aspm_policy) { 106 case POLICY_PERFORMANCE: 107 /* Disable ASPM and Clock PM */ 108 return 0; 109 case POLICY_POWERSAVE: 110 /* Disable Clock PM */ 111 return 1; 112 case POLICY_DEFAULT: 113 return link->clkpm_default; 114 } 115 return 0; 116} 117 118static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) 119{ 120 int pos; 121 u16 reg16; 122 struct pci_dev *child; 123 struct pci_bus *linkbus = link->pdev->subordinate; 124 125 list_for_each_entry(child, &linkbus->devices, bus_list) { 126 pos = pci_pcie_cap(child); 127 if (!pos) 128 return; 129 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16); 130 if (enable) 131 reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN; 132 else 133 reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN; 134 pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16); 135 } 136 link->clkpm_enabled = !!enable; 137} 138 139static void pcie_set_clkpm(struct pcie_link_state *link, int enable) 140{ 141 /* Don't enable Clock PM if the link is not Clock PM capable */ 142 if (!link->clkpm_capable && enable) 143 enable = 0; 144 /* Need nothing if the specified equals to current state */ 145 if (link->clkpm_enabled == enable) 146 return; 147 pcie_set_clkpm_nocheck(link, enable); 148} 149 150static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) 151{ 152 int pos, capable = 1, enabled = 1; 153 u32 reg32; 154 u16 reg16; 155 struct pci_dev *child; 156 struct pci_bus *linkbus = link->pdev->subordinate; 157 158 /* All functions should have the same cap and state, take the worst */ 159 list_for_each_entry(child, &linkbus->devices, bus_list) { 160 pos = pci_pcie_cap(child); 161 if (!pos) 162 return; 163 pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, ®32); 164 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 165 capable = 0; 166 enabled = 0; 167 break; 168 } 169 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16); 170 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 171 enabled = 0; 172 } 173 link->clkpm_enabled = enabled; 174 link->clkpm_default = enabled; 175 link->clkpm_capable = (blacklist) ? 0 : capable; 176} 177 178/* 179 * pcie_aspm_configure_common_clock: check if the 2 ends of a link 180 * could use common clock. If they are, configure them to use the 181 * common clock. That will reduce the ASPM state exit latency. 182 */ 183static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) 184{ 185 int ppos, cpos, same_clock = 1; 186 u16 reg16, parent_reg, child_reg[8]; 187 unsigned long start_jiffies; 188 struct pci_dev *child, *parent = link->pdev; 189 struct pci_bus *linkbus = parent->subordinate; 190 /* 191 * All functions of a slot should have the same Slot Clock 192 * Configuration, so just check one function 193 */ 194 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 195 BUG_ON(!pci_is_pcie(child)); 196 197 /* Check downstream component if bit Slot Clock Configuration is 1 */ 198 cpos = pci_pcie_cap(child); 199 pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, ®16); 200 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 201 same_clock = 0; 202 203 /* Check upstream component if bit Slot Clock Configuration is 1 */ 204 ppos = pci_pcie_cap(parent); 205 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16); 206 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 207 same_clock = 0; 208 209 /* Configure downstream component, all functions */ 210 list_for_each_entry(child, &linkbus->devices, bus_list) { 211 cpos = pci_pcie_cap(child); 212 pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, ®16); 213 child_reg[PCI_FUNC(child->devfn)] = reg16; 214 if (same_clock) 215 reg16 |= PCI_EXP_LNKCTL_CCC; 216 else 217 reg16 &= ~PCI_EXP_LNKCTL_CCC; 218 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16); 219 } 220 221 /* Configure upstream component */ 222 pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, ®16); 223 parent_reg = reg16; 224 if (same_clock) 225 reg16 |= PCI_EXP_LNKCTL_CCC; 226 else 227 reg16 &= ~PCI_EXP_LNKCTL_CCC; 228 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16); 229 230 /* Retrain link */ 231 reg16 |= PCI_EXP_LNKCTL_RL; 232 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16); 233 234 /* Wait for link training end. Break out after waiting for timeout */ 235 start_jiffies = jiffies; 236 for (;;) { 237 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16); 238 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 239 break; 240 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) 241 break; 242 msleep(1); 243 } 244 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 245 return; 246 247 /* Training failed. Restore common clock configurations */ 248 dev_printk(KERN_ERR, &parent->dev, 249 "ASPM: Could not configure common clock\n"); 250 list_for_each_entry(child, &linkbus->devices, bus_list) { 251 cpos = pci_pcie_cap(child); 252 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, 253 child_reg[PCI_FUNC(child->devfn)]); 254 } 255 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg); 256} 257 258/* Convert L0s latency encoding to ns */ 259static u32 calc_l0s_latency(u32 encoding) 260{ 261 if (encoding == 0x7) 262 return (5 * 1000); /* > 4us */ 263 return (64 << encoding); 264} 265 266/* Convert L0s acceptable latency encoding to ns */ 267static u32 calc_l0s_acceptable(u32 encoding) 268{ 269 if (encoding == 0x7) 270 return -1U; 271 return (64 << encoding); 272} 273 274/* Convert L1 latency encoding to ns */ 275static u32 calc_l1_latency(u32 encoding) 276{ 277 if (encoding == 0x7) 278 return (65 * 1000); /* > 64us */ 279 return (1000 << encoding); 280} 281 282/* Convert L1 acceptable latency encoding to ns */ 283static u32 calc_l1_acceptable(u32 encoding) 284{ 285 if (encoding == 0x7) 286 return -1U; 287 return (1000 << encoding); 288} 289 290struct aspm_register_info { 291 u32 support:2; 292 u32 enabled:2; 293 u32 latency_encoding_l0s; 294 u32 latency_encoding_l1; 295}; 296 297static void pcie_get_aspm_reg(struct pci_dev *pdev, 298 struct aspm_register_info *info) 299{ 300 int pos; 301 u16 reg16; 302 u32 reg32; 303 304 pos = pci_pcie_cap(pdev); 305 pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32); 306 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; 307 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; 308 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; 309 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); 310 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; 311} 312 313static void pcie_aspm_check_latency(struct pci_dev *endpoint) 314{ 315 u32 latency, l1_switch_latency = 0; 316 struct aspm_latency *acceptable; 317 struct pcie_link_state *link; 318 319 /* Device not in D0 doesn't need latency check */ 320 if ((endpoint->current_state != PCI_D0) && 321 (endpoint->current_state != PCI_UNKNOWN)) 322 return; 323 324 link = endpoint->bus->self->link_state; 325 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; 326 327 while (link) { 328 /* Check upstream direction L0s latency */ 329 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && 330 (link->latency_up.l0s > acceptable->l0s)) 331 link->aspm_capable &= ~ASPM_STATE_L0S_UP; 332 333 /* Check downstream direction L0s latency */ 334 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && 335 (link->latency_dw.l0s > acceptable->l0s)) 336 link->aspm_capable &= ~ASPM_STATE_L0S_DW; 337 /* 338 * Check L1 latency. 339 * Every switch on the path to root complex need 1 340 * more microsecond for L1. Spec doesn't mention L0s. 341 */ 342 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); 343 if ((link->aspm_capable & ASPM_STATE_L1) && 344 (latency + l1_switch_latency > acceptable->l1)) 345 link->aspm_capable &= ~ASPM_STATE_L1; 346 l1_switch_latency += 1000; 347 348 link = link->parent; 349 } 350} 351 352static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 353{ 354 struct pci_dev *child, *parent = link->pdev; 355 struct pci_bus *linkbus = parent->subordinate; 356 struct aspm_register_info upreg, dwreg; 357 358 if (blacklist) { 359 /* Set enabled/disable so that we will disable ASPM later */ 360 link->aspm_enabled = ASPM_STATE_ALL; 361 link->aspm_disable = ASPM_STATE_ALL; 362 return; 363 } 364 365 /* Configure common clock before checking latencies */ 366 pcie_aspm_configure_common_clock(link); 367 368 /* Get upstream/downstream components' register state */ 369 pcie_get_aspm_reg(parent, &upreg); 370 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 371 pcie_get_aspm_reg(child, &dwreg); 372 373 /* 374 * Setup L0s state 375 * 376 * Note that we must not enable L0s in either direction on a 377 * given link unless components on both sides of the link each 378 * support L0s. 379 */ 380 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) 381 link->aspm_support |= ASPM_STATE_L0S; 382 if (dwreg.enabled & PCIE_LINK_STATE_L0S) 383 link->aspm_enabled |= ASPM_STATE_L0S_UP; 384 if (upreg.enabled & PCIE_LINK_STATE_L0S) 385 link->aspm_enabled |= ASPM_STATE_L0S_DW; 386 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); 387 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); 388 389 /* Setup L1 state */ 390 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) 391 link->aspm_support |= ASPM_STATE_L1; 392 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) 393 link->aspm_enabled |= ASPM_STATE_L1; 394 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); 395 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); 396 397 /* Save default state */ 398 link->aspm_default = link->aspm_enabled; 399 400 /* Setup initial capable state. Will be updated later */ 401 link->aspm_capable = link->aspm_support; 402 /* 403 * If the downstream component has pci bridge function, don't 404 * do ASPM for now. 405 */ 406 list_for_each_entry(child, &linkbus->devices, bus_list) { 407 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) { 408 link->aspm_disable = ASPM_STATE_ALL; 409 break; 410 } 411 } 412 413 /* Get and check endpoint acceptable latencies */ 414 list_for_each_entry(child, &linkbus->devices, bus_list) { 415 int pos; 416 u32 reg32, encoding; 417 struct aspm_latency *acceptable = 418 &link->acceptable[PCI_FUNC(child->devfn)]; 419 420 if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT && 421 child->pcie_type != PCI_EXP_TYPE_LEG_END) 422 continue; 423 424 pos = pci_pcie_cap(child); 425 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32); 426 /* Calculate endpoint L0s acceptable latency */ 427 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; 428 acceptable->l0s = calc_l0s_acceptable(encoding); 429 /* Calculate endpoint L1 acceptable latency */ 430 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; 431 acceptable->l1 = calc_l1_acceptable(encoding); 432 433 pcie_aspm_check_latency(child); 434 } 435} 436 437static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) 438{ 439 u16 reg16; 440 int pos = pci_pcie_cap(pdev); 441 442 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); 443 reg16 &= ~0x3; 444 reg16 |= val; 445 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); 446} 447 448static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) 449{ 450 u32 upstream = 0, dwstream = 0; 451 struct pci_dev *child, *parent = link->pdev; 452 struct pci_bus *linkbus = parent->subordinate; 453 454 /* Nothing to do if the link is already in the requested state */ 455 state &= (link->aspm_capable & ~link->aspm_disable); 456 if (link->aspm_enabled == state) 457 return; 458 /* Convert ASPM state to upstream/downstream ASPM register state */ 459 if (state & ASPM_STATE_L0S_UP) 460 dwstream |= PCIE_LINK_STATE_L0S; 461 if (state & ASPM_STATE_L0S_DW) 462 upstream |= PCIE_LINK_STATE_L0S; 463 if (state & ASPM_STATE_L1) { 464 upstream |= PCIE_LINK_STATE_L1; 465 dwstream |= PCIE_LINK_STATE_L1; 466 } 467 /* 468 * Spec 2.0 suggests all functions should be configured the 469 * same setting for ASPM. Enabling ASPM L1 should be done in 470 * upstream component first and then downstream, and vice 471 * versa for disabling ASPM L1. Spec doesn't mention L0S. 472 */ 473 if (state & ASPM_STATE_L1) 474 pcie_config_aspm_dev(parent, upstream); 475 list_for_each_entry(child, &linkbus->devices, bus_list) 476 pcie_config_aspm_dev(child, dwstream); 477 if (!(state & ASPM_STATE_L1)) 478 pcie_config_aspm_dev(parent, upstream); 479 480 link->aspm_enabled = state; 481} 482 483static void pcie_config_aspm_path(struct pcie_link_state *link) 484{ 485 while (link) { 486 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 487 link = link->parent; 488 } 489} 490 491static void free_link_state(struct pcie_link_state *link) 492{ 493 link->pdev->link_state = NULL; 494 kfree(link); 495} 496 497static int pcie_aspm_sanity_check(struct pci_dev *pdev) 498{ 499 struct pci_dev *child; 500 int pos; 501 u32 reg32; 502 503 /* 504 * Some functions in a slot might not all be PCIe functions, 505 * very strange. Disable ASPM for the whole slot 506 */ 507 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { 508 pos = pci_pcie_cap(child); 509 if (!pos) 510 return -EINVAL; 511 /* 512 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 513 * RBER bit to determine if a function is 1.1 version device 514 */ 515 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32); 516 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 517 dev_printk(KERN_INFO, &child->dev, "disabling ASPM" 518 " on pre-1.1 PCIe device. You can enable it" 519 " with 'pcie_aspm=force'\n"); 520 return -EINVAL; 521 } 522 } 523 return 0; 524} 525 526static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) 527{ 528 struct pcie_link_state *link; 529 530 link = kzalloc(sizeof(*link), GFP_KERNEL); 531 if (!link) 532 return NULL; 533 INIT_LIST_HEAD(&link->sibling); 534 INIT_LIST_HEAD(&link->children); 535 INIT_LIST_HEAD(&link->link); 536 link->pdev = pdev; 537 if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) { 538 struct pcie_link_state *parent; 539 parent = pdev->bus->parent->self->link_state; 540 if (!parent) { 541 kfree(link); 542 return NULL; 543 } 544 link->parent = parent; 545 list_add(&link->link, &parent->children); 546 } 547 /* Setup a pointer to the root port link */ 548 if (!link->parent) 549 link->root = link; 550 else 551 link->root = link->parent->root; 552 553 list_add(&link->sibling, &link_list); 554 pdev->link_state = link; 555 return link; 556} 557 558/* 559 * pcie_aspm_init_link_state: Initiate PCI express link state. 560 * It is called after the pcie and its children devices are scaned. 561 * @pdev: the root port or switch downstream port 562 */ 563void pcie_aspm_init_link_state(struct pci_dev *pdev) 564{ 565 struct pcie_link_state *link; 566 int blacklist = !!pcie_aspm_sanity_check(pdev); 567 568 if (!pci_is_pcie(pdev) || pdev->link_state) 569 return; 570 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && 571 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) 572 return; 573 574 /* VIA has a strange chipset, root port is under a bridge */ 575 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT && 576 pdev->bus->self) 577 return; 578 579 down_read(&pci_bus_sem); 580 if (list_empty(&pdev->subordinate->devices)) 581 goto out; 582 583 mutex_lock(&aspm_lock); 584 link = alloc_pcie_link_state(pdev); 585 if (!link) 586 goto unlock; 587 /* 588 * Setup initial ASPM state. Note that we need to configure 589 * upstream links also because capable state of them can be 590 * update through pcie_aspm_cap_init(). 591 */ 592 pcie_aspm_cap_init(link, blacklist); 593 594 /* Setup initial Clock PM state */ 595 pcie_clkpm_cap_init(link, blacklist); 596 597 /* 598 * At this stage drivers haven't had an opportunity to change the 599 * link policy setting. Enabling ASPM on broken hardware can cripple 600 * it even before the driver has had a chance to disable ASPM, so 601 * default to a safe level right now. If we're enabling ASPM beyond 602 * the BIOS's expectation, we'll do so once pci_enable_device() is 603 * called. 604 */ 605 if (aspm_policy != POLICY_POWERSAVE) { 606 pcie_config_aspm_path(link); 607 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 608 } 609 610unlock: 611 mutex_unlock(&aspm_lock); 612out: 613 up_read(&pci_bus_sem); 614} 615 616/* Recheck latencies and update aspm_capable for links under the root */ 617static void pcie_update_aspm_capable(struct pcie_link_state *root) 618{ 619 struct pcie_link_state *link; 620 BUG_ON(root->parent); 621 list_for_each_entry(link, &link_list, sibling) { 622 if (link->root != root) 623 continue; 624 link->aspm_capable = link->aspm_support; 625 } 626 list_for_each_entry(link, &link_list, sibling) { 627 struct pci_dev *child; 628 struct pci_bus *linkbus = link->pdev->subordinate; 629 if (link->root != root) 630 continue; 631 list_for_each_entry(child, &linkbus->devices, bus_list) { 632 if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT) && 633 (child->pcie_type != PCI_EXP_TYPE_LEG_END)) 634 continue; 635 pcie_aspm_check_latency(child); 636 } 637 } 638} 639 640/* @pdev: the endpoint device */ 641void pcie_aspm_exit_link_state(struct pci_dev *pdev) 642{ 643 struct pci_dev *parent = pdev->bus->self; 644 struct pcie_link_state *link, *root, *parent_link; 645 646 if (!pci_is_pcie(pdev) || !parent || !parent->link_state) 647 return; 648 if ((parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && 649 (parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)) 650 return; 651 652 down_read(&pci_bus_sem); 653 mutex_lock(&aspm_lock); 654 /* 655 * All PCIe functions are in one slot, remove one function will remove 656 * the whole slot, so just wait until we are the last function left. 657 */ 658 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) 659 goto out; 660 661 link = parent->link_state; 662 root = link->root; 663 parent_link = link->parent; 664 665 /* All functions are removed, so just disable ASPM for the link */ 666 pcie_config_aspm_link(link, 0); 667 list_del(&link->sibling); 668 list_del(&link->link); 669 /* Clock PM is for endpoint device */ 670 free_link_state(link); 671 672 /* Recheck latencies and configure upstream links */ 673 if (parent_link) { 674 pcie_update_aspm_capable(root); 675 pcie_config_aspm_path(parent_link); 676 } 677out: 678 mutex_unlock(&aspm_lock); 679 up_read(&pci_bus_sem); 680} 681 682/* @pdev: the root port or switch downstream port */ 683void pcie_aspm_pm_state_change(struct pci_dev *pdev) 684{ 685 struct pcie_link_state *link = pdev->link_state; 686 687 if (aspm_disabled || !pci_is_pcie(pdev) || !link) 688 return; 689 if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && 690 (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)) 691 return; 692 /* 693 * Devices changed PM state, we should recheck if latency 694 * meets all functions' requirement 695 */ 696 down_read(&pci_bus_sem); 697 mutex_lock(&aspm_lock); 698 pcie_update_aspm_capable(link->root); 699 pcie_config_aspm_path(link); 700 mutex_unlock(&aspm_lock); 701 up_read(&pci_bus_sem); 702} 703 704void pcie_aspm_powersave_config_link(struct pci_dev *pdev) 705{ 706 struct pcie_link_state *link = pdev->link_state; 707 708 if (aspm_disabled || !pci_is_pcie(pdev) || !link) 709 return; 710 711 if (aspm_policy != POLICY_POWERSAVE) 712 return; 713 714 if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && 715 (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)) 716 return; 717 718 down_read(&pci_bus_sem); 719 mutex_lock(&aspm_lock); 720 pcie_config_aspm_path(link); 721 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 722 mutex_unlock(&aspm_lock); 723 up_read(&pci_bus_sem); 724} 725 726/* 727 * pci_disable_link_state - disable pci device's link state, so the link will 728 * never enter specific states 729 */ 730static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem, 731 bool force) 732{ 733 struct pci_dev *parent = pdev->bus->self; 734 struct pcie_link_state *link; 735 736 if (aspm_disabled && !force) 737 return; 738 739 if (!pci_is_pcie(pdev)) 740 return; 741 742 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT || 743 pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) 744 parent = pdev; 745 if (!parent || !parent->link_state) 746 return; 747 748 if (sem) 749 down_read(&pci_bus_sem); 750 mutex_lock(&aspm_lock); 751 link = parent->link_state; 752 if (state & PCIE_LINK_STATE_L0S) 753 link->aspm_disable |= ASPM_STATE_L0S; 754 if (state & PCIE_LINK_STATE_L1) 755 link->aspm_disable |= ASPM_STATE_L1; 756 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 757 758 if (state & PCIE_LINK_STATE_CLKPM) { 759 link->clkpm_capable = 0; 760 pcie_set_clkpm(link, 0); 761 } 762 mutex_unlock(&aspm_lock); 763 if (sem) 764 up_read(&pci_bus_sem); 765} 766 767void pci_disable_link_state_locked(struct pci_dev *pdev, int state) 768{ 769 __pci_disable_link_state(pdev, state, false, false); 770} 771EXPORT_SYMBOL(pci_disable_link_state_locked); 772 773void pci_disable_link_state(struct pci_dev *pdev, int state) 774{ 775 __pci_disable_link_state(pdev, state, true, false); 776} 777EXPORT_SYMBOL(pci_disable_link_state); 778 779void pcie_clear_aspm(struct pci_bus *bus) 780{ 781 struct pci_dev *child; 782 783 /* 784 * Clear any ASPM setup that the firmware has carried out on this bus 785 */ 786 list_for_each_entry(child, &bus->devices, bus_list) { 787 __pci_disable_link_state(child, PCIE_LINK_STATE_L0S | 788 PCIE_LINK_STATE_L1 | 789 PCIE_LINK_STATE_CLKPM, 790 false, true); 791 } 792} 793 794static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) 795{ 796 int i; 797 struct pcie_link_state *link; 798 799 if (aspm_disabled) 800 return -EPERM; 801 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 802 if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) 803 break; 804 if (i >= ARRAY_SIZE(policy_str)) 805 return -EINVAL; 806 if (i == aspm_policy) 807 return 0; 808 809 down_read(&pci_bus_sem); 810 mutex_lock(&aspm_lock); 811 aspm_policy = i; 812 list_for_each_entry(link, &link_list, sibling) { 813 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 814 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 815 } 816 mutex_unlock(&aspm_lock); 817 up_read(&pci_bus_sem); 818 return 0; 819} 820 821static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) 822{ 823 int i, cnt = 0; 824 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 825 if (i == aspm_policy) 826 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); 827 else 828 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); 829 return cnt; 830} 831 832module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, 833 NULL, 0644); 834 835#ifdef CONFIG_PCIEASPM_DEBUG 836static ssize_t link_state_show(struct device *dev, 837 struct device_attribute *attr, 838 char *buf) 839{ 840 struct pci_dev *pci_device = to_pci_dev(dev); 841 struct pcie_link_state *link_state = pci_device->link_state; 842 843 return sprintf(buf, "%d\n", link_state->aspm_enabled); 844} 845 846static ssize_t link_state_store(struct device *dev, 847 struct device_attribute *attr, 848 const char *buf, 849 size_t n) 850{ 851 struct pci_dev *pdev = to_pci_dev(dev); 852 struct pcie_link_state *link, *root = pdev->link_state->root; 853 u32 val = buf[0] - '0', state = 0; 854 855 if (aspm_disabled) 856 return -EPERM; 857 if (n < 1 || val > 3) 858 return -EINVAL; 859 860 /* Convert requested state to ASPM state */ 861 if (val & PCIE_LINK_STATE_L0S) 862 state |= ASPM_STATE_L0S; 863 if (val & PCIE_LINK_STATE_L1) 864 state |= ASPM_STATE_L1; 865 866 down_read(&pci_bus_sem); 867 mutex_lock(&aspm_lock); 868 list_for_each_entry(link, &link_list, sibling) { 869 if (link->root != root) 870 continue; 871 pcie_config_aspm_link(link, state); 872 } 873 mutex_unlock(&aspm_lock); 874 up_read(&pci_bus_sem); 875 return n; 876} 877 878static ssize_t clk_ctl_show(struct device *dev, 879 struct device_attribute *attr, 880 char *buf) 881{ 882 struct pci_dev *pci_device = to_pci_dev(dev); 883 struct pcie_link_state *link_state = pci_device->link_state; 884 885 return sprintf(buf, "%d\n", link_state->clkpm_enabled); 886} 887 888static ssize_t clk_ctl_store(struct device *dev, 889 struct device_attribute *attr, 890 const char *buf, 891 size_t n) 892{ 893 struct pci_dev *pdev = to_pci_dev(dev); 894 int state; 895 896 if (n < 1) 897 return -EINVAL; 898 state = buf[0]-'0'; 899 900 down_read(&pci_bus_sem); 901 mutex_lock(&aspm_lock); 902 pcie_set_clkpm_nocheck(pdev->link_state, !!state); 903 mutex_unlock(&aspm_lock); 904 up_read(&pci_bus_sem); 905 906 return n; 907} 908 909static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store); 910static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store); 911 912static char power_group[] = "power"; 913void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) 914{ 915 struct pcie_link_state *link_state = pdev->link_state; 916 917 if (!pci_is_pcie(pdev) || 918 (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && 919 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) 920 return; 921 922 if (link_state->aspm_support) 923 sysfs_add_file_to_group(&pdev->dev.kobj, 924 &dev_attr_link_state.attr, power_group); 925 if (link_state->clkpm_capable) 926 sysfs_add_file_to_group(&pdev->dev.kobj, 927 &dev_attr_clk_ctl.attr, power_group); 928} 929 930void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) 931{ 932 struct pcie_link_state *link_state = pdev->link_state; 933 934 if (!pci_is_pcie(pdev) || 935 (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && 936 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) 937 return; 938 939 if (link_state->aspm_support) 940 sysfs_remove_file_from_group(&pdev->dev.kobj, 941 &dev_attr_link_state.attr, power_group); 942 if (link_state->clkpm_capable) 943 sysfs_remove_file_from_group(&pdev->dev.kobj, 944 &dev_attr_clk_ctl.attr, power_group); 945} 946#endif 947 948static int __init pcie_aspm_disable(char *str) 949{ 950 if (!strcmp(str, "off")) { 951 aspm_policy = POLICY_DEFAULT; 952 aspm_disabled = 1; 953 aspm_support_enabled = false; 954 printk(KERN_INFO "PCIe ASPM is disabled\n"); 955 } else if (!strcmp(str, "force")) { 956 aspm_force = 1; 957 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); 958 } 959 return 1; 960} 961 962__setup("pcie_aspm=", pcie_aspm_disable); 963 964void pcie_no_aspm(void) 965{ 966 /* 967 * Disabling ASPM is intended to prevent the kernel from modifying 968 * existing hardware state, not to clear existing state. To that end: 969 * (a) set policy to POLICY_DEFAULT in order to avoid changing state 970 * (b) prevent userspace from changing policy 971 */ 972 if (!aspm_force) { 973 aspm_policy = POLICY_DEFAULT; 974 aspm_disabled = 1; 975 } 976} 977 978/** 979 * pcie_aspm_enabled - is PCIe ASPM enabled? 980 * 981 * Returns true if ASPM has not been disabled by the command-line option 982 * pcie_aspm=off. 983 **/ 984int pcie_aspm_enabled(void) 985{ 986 return !aspm_disabled; 987} 988EXPORT_SYMBOL(pcie_aspm_enabled); 989 990bool pcie_aspm_support_enabled(void) 991{ 992 return aspm_support_enabled; 993} 994EXPORT_SYMBOL(pcie_aspm_support_enabled); 995