17d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li/* 27d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * File: drivers/pci/pcie/aspm.c 345e829ea412760d2404d7dfc42528df46aedbf62Stefan Assmann * Enabling PCIe link L0s/L1 state and Clock Power Management 47d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * 57d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * Copyright (C) 2007 Intel 67d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) 77d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * Copyright (C) Shaohua Li (shaohua.li@intel.com) 87d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li */ 97d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 107d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/kernel.h> 117d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/module.h> 127d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/moduleparam.h> 137d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/pci.h> 147d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/pci_regs.h> 157d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/errno.h> 167d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/pm.h> 177d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/init.h> 187d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/slab.h> 192a42d9dba7842422ffb2c02e75288a8bc2fd5065Thomas Renninger#include <linux/jiffies.h> 20987a4c783a8bbf3baf554e6b8ff588b26e06e020Andrew Patterson#include <linux/delay.h> 217d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include <linux/pci-aspm.h> 227d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#include "../pci.h" 237d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 247d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#ifdef MODULE_PARAM_PREFIX 257d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#undef MODULE_PARAM_PREFIX 267d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#endif 277d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#define MODULE_PARAM_PREFIX "pcie_aspm." 287d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 29ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige/* Note: those are not register definitions */ 30ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ 31ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ 32ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige#define ASPM_STATE_L1 (4) /* L1 state */ 33ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) 34ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1) 35ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige 36b6c2e54d3ea27719b920faf15db92dfe0260f0d2Kenji Kaneshigestruct aspm_latency { 37b6c2e54d3ea27719b920faf15db92dfe0260f0d2Kenji Kaneshige u32 l0s; /* L0s latency (nsec) */ 38b6c2e54d3ea27719b920faf15db92dfe0260f0d2Kenji Kaneshige u32 l1; /* L1 latency (nsec) */ 397d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li}; 407d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 417d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listruct pcie_link_state { 425cde89d80172a393e49077d2450545b97ac8d972Kenji Kaneshige struct pci_dev *pdev; /* Upstream component of the Link */ 435c92ffb1ecc7f13267cdef5dda8a838937912c93Kenji Kaneshige struct pcie_link_state *root; /* pointer to the root port link */ 445cde89d80172a393e49077d2450545b97ac8d972Kenji Kaneshige struct pcie_link_state *parent; /* pointer to the parent Link state */ 455cde89d80172a393e49077d2450545b97ac8d972Kenji Kaneshige struct list_head sibling; /* node in link_list */ 465cde89d80172a393e49077d2450545b97ac8d972Kenji Kaneshige struct list_head children; /* list of child link states */ 475cde89d80172a393e49077d2450545b97ac8d972Kenji Kaneshige struct list_head link; /* node in parent's children list */ 487d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 497d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* ASPM state */ 50ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 aspm_support:3; /* Supported ASPM state */ 51ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 aspm_enabled:3; /* Enabled ASPM state */ 52ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 aspm_capable:3; /* Capable ASPM state with latency */ 53ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 aspm_default:3; /* Default ASPM state by BIOS */ 54ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 aspm_disable:3; /* Disabled ASPM state */ 5580bfdbe370d56a1448c7078cd6d286b89241a72eKenji Kaneshige 564d246e458918d469ad645fd5f937ac22982e0466Kenji Kaneshige /* Clock PM state */ 574d246e458918d469ad645fd5f937ac22982e0466Kenji Kaneshige u32 clkpm_capable:1; /* Clock PM capable? */ 584d246e458918d469ad645fd5f937ac22982e0466Kenji Kaneshige u32 clkpm_enabled:1; /* Current Clock PM state */ 594d246e458918d469ad645fd5f937ac22982e0466Kenji Kaneshige u32 clkpm_default:1; /* Default Clock PM state by BIOS */ 604d246e458918d469ad645fd5f937ac22982e0466Kenji Kaneshige 61ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige /* Exit latencies */ 62ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige struct aspm_latency latency_up; /* Upstream direction exit latency */ 63ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige struct aspm_latency latency_dw; /* Downstream direction exit latency */ 647d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* 65b6c2e54d3ea27719b920faf15db92dfe0260f0d2Kenji Kaneshige * Endpoint acceptable latencies. A pcie downstream port only 66b6c2e54d3ea27719b920faf15db92dfe0260f0d2Kenji Kaneshige * has one slot under it, so at most there are 8 functions. 677d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li */ 68b6c2e54d3ea27719b920faf15db92dfe0260f0d2Kenji Kaneshige struct aspm_latency acceptable[8]; 697d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li}; 707d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 713c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrettstatic int aspm_disabled, aspm_force; 728b8bae901ce23addbdcdb54fa1696fb2d049feb5Rafael J. Wysockistatic bool aspm_support_enabled = true; 737d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic DEFINE_MUTEX(aspm_lock); 747d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic LIST_HEAD(link_list); 757d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 767d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#define POLICY_DEFAULT 0 /* BIOS default setting */ 777d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#define POLICY_PERFORMANCE 1 /* high performance */ 787d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#define POLICY_POWERSAVE 2 /* high power saving */ 79ad71c96213a68dfe6d761e3ff7ac7ac267fd612aMatthew Garrett 80ad71c96213a68dfe6d761e3ff7ac7ac267fd612aMatthew Garrett#ifdef CONFIG_PCIEASPM_PERFORMANCE 81ad71c96213a68dfe6d761e3ff7ac7ac267fd612aMatthew Garrettstatic int aspm_policy = POLICY_PERFORMANCE; 82ad71c96213a68dfe6d761e3ff7ac7ac267fd612aMatthew Garrett#elif defined CONFIG_PCIEASPM_POWERSAVE 83ad71c96213a68dfe6d761e3ff7ac7ac267fd612aMatthew Garrettstatic int aspm_policy = POLICY_POWERSAVE; 84ad71c96213a68dfe6d761e3ff7ac7ac267fd612aMatthew Garrett#else 857d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic int aspm_policy; 86ad71c96213a68dfe6d761e3ff7ac7ac267fd612aMatthew Garrett#endif 87ad71c96213a68dfe6d761e3ff7ac7ac267fd612aMatthew Garrett 887d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic const char *policy_str[] = { 897d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li [POLICY_DEFAULT] = "default", 907d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li [POLICY_PERFORMANCE] = "performance", 917d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li [POLICY_POWERSAVE] = "powersave" 927d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li}; 937d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 94987a4c783a8bbf3baf554e6b8ff588b26e06e020Andrew Patterson#define LINK_RETRAIN_TIMEOUT HZ 95987a4c783a8bbf3baf554e6b8ff588b26e06e020Andrew Patterson 965aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshigestatic int policy_to_aspm_state(struct pcie_link_state *link) 977d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 987d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li switch (aspm_policy) { 997d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li case POLICY_PERFORMANCE: 1007d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Disable ASPM and Clock PM */ 1017d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return 0; 1027d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li case POLICY_POWERSAVE: 1037d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Enable ASPM L0s/L1 */ 104ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige return ASPM_STATE_ALL; 1057d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li case POLICY_DEFAULT: 1065aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige return link->aspm_default; 1077d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 1087d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return 0; 1097d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 1107d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 1115aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshigestatic int policy_to_clkpm_state(struct pcie_link_state *link) 1127d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 1137d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li switch (aspm_policy) { 1147d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li case POLICY_PERFORMANCE: 1157d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Disable ASPM and Clock PM */ 1167d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return 0; 1177d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li case POLICY_POWERSAVE: 1187d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Disable Clock PM */ 1197d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return 1; 1207d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li case POLICY_DEFAULT: 1215aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige return link->clkpm_default; 1227d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 1237d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return 0; 1247d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 1257d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 126430842e29d396928989c0a45e05025e988004d79Kenji Kaneshigestatic void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) 1277d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 1287d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li int pos; 1297d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li u16 reg16; 1305aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_dev *child; 1315aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_bus *linkbus = link->pdev->subordinate; 1327d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 1335aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige list_for_each_entry(child, &linkbus->devices, bus_list) { 134db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige pos = pci_pcie_cap(child); 1357d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!pos) 1367d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 1375aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16); 1387d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (enable) 1397d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN; 1407d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li else 1417d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN; 1425aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16); 1437d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 1445aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige link->clkpm_enabled = !!enable; 1457d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 1467d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 147430842e29d396928989c0a45e05025e988004d79Kenji Kaneshigestatic void pcie_set_clkpm(struct pcie_link_state *link, int enable) 148430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige{ 149430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige /* Don't enable Clock PM if the link is not Clock PM capable */ 150430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige if (!link->clkpm_capable && enable) 1512f671e2dbff6eb5ef4e2600adbec550c13b8fe72Matthew Garrett enable = 0; 152430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige /* Need nothing if the specified equals to current state */ 153430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige if (link->clkpm_enabled == enable) 154430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige return; 155430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige pcie_set_clkpm_nocheck(link, enable); 156430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige} 157430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige 1588d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshigestatic void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) 1597d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 1605aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige int pos, capable = 1, enabled = 1; 1617d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li u32 reg32; 1627d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li u16 reg16; 1635aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_dev *child; 1645aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_bus *linkbus = link->pdev->subordinate; 1657d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 1667d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* All functions should have the same cap and state, take the worst */ 1675aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige list_for_each_entry(child, &linkbus->devices, bus_list) { 168db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige pos = pci_pcie_cap(child); 1697d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!pos) 1707d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 1715aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, ®32); 1727d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 1737d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li capable = 0; 1747d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li enabled = 0; 1757d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li break; 1767d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 1775aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16); 1787d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 1797d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li enabled = 0; 1807d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 1815aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige link->clkpm_enabled = enabled; 1825aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige link->clkpm_default = enabled; 1838d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige link->clkpm_capable = (blacklist) ? 0 : capable; 18446bbdfa44cfc0d352148a0dc33ba9f6db02ccdf0Shaohua Li} 18546bbdfa44cfc0d352148a0dc33ba9f6db02ccdf0Shaohua Li 1867d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li/* 1877d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * pcie_aspm_configure_common_clock: check if the 2 ends of a link 1887d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * could use common clock. If they are, configure them to use the 1897d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * common clock. That will reduce the ASPM state exit latency. 1907d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li */ 1915aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshigestatic void pcie_aspm_configure_common_clock(struct pcie_link_state *link) 1927d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 1935aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige int ppos, cpos, same_clock = 1; 1945aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige u16 reg16, parent_reg, child_reg[8]; 1952a42d9dba7842422ffb2c02e75288a8bc2fd5065Thomas Renninger unsigned long start_jiffies; 1965aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_dev *child, *parent = link->pdev; 1975aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_bus *linkbus = parent->subordinate; 1987d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* 1995aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige * All functions of a slot should have the same Slot Clock 2007d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * Configuration, so just check one function 2015aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige */ 2025aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 2038b06477dc4fcdfc21442ad334d3f3e335225ea0cKenji Kaneshige BUG_ON(!pci_is_pcie(child)); 2047d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2057d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Check downstream component if bit Slot Clock Configuration is 1 */ 206db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige cpos = pci_pcie_cap(child); 2075aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, ®16); 2087d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 2097d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li same_clock = 0; 2107d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2117d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Check upstream component if bit Slot Clock Configuration is 1 */ 212db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige ppos = pci_pcie_cap(parent); 2135aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16); 2147d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 2157d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li same_clock = 0; 2167d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2177d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Configure downstream component, all functions */ 2185aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige list_for_each_entry(child, &linkbus->devices, bus_list) { 219db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige cpos = pci_pcie_cap(child); 2205aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, ®16); 2215aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige child_reg[PCI_FUNC(child->devfn)] = reg16; 2227d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (same_clock) 2237d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li reg16 |= PCI_EXP_LNKCTL_CCC; 2247d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li else 2257d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li reg16 &= ~PCI_EXP_LNKCTL_CCC; 2265aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16); 2277d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 2287d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2297d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Configure upstream component */ 2305aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, ®16); 2312a42d9dba7842422ffb2c02e75288a8bc2fd5065Thomas Renninger parent_reg = reg16; 2327d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (same_clock) 2337d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li reg16 |= PCI_EXP_LNKCTL_CCC; 2347d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li else 2357d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li reg16 &= ~PCI_EXP_LNKCTL_CCC; 2365aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16); 2377d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2385aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige /* Retrain link */ 2397d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li reg16 |= PCI_EXP_LNKCTL_RL; 2405aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16); 2417d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2425aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige /* Wait for link training end. Break out after waiting for timeout */ 2432a42d9dba7842422ffb2c02e75288a8bc2fd5065Thomas Renninger start_jiffies = jiffies; 244987a4c783a8bbf3baf554e6b8ff588b26e06e020Andrew Patterson for (;;) { 2455aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16); 2467d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!(reg16 & PCI_EXP_LNKSTA_LT)) 2477d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li break; 248987a4c783a8bbf3baf554e6b8ff588b26e06e020Andrew Patterson if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) 249987a4c783a8bbf3baf554e6b8ff588b26e06e020Andrew Patterson break; 250987a4c783a8bbf3baf554e6b8ff588b26e06e020Andrew Patterson msleep(1); 2517d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 2525aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige if (!(reg16 & PCI_EXP_LNKSTA_LT)) 2535aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige return; 2545aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige 2555aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige /* Training failed. Restore common clock configurations */ 2565aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige dev_printk(KERN_ERR, &parent->dev, 2575aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige "ASPM: Could not configure common clock\n"); 2585aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige list_for_each_entry(child, &linkbus->devices, bus_list) { 259db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige cpos = pci_pcie_cap(child); 2605aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, 2615aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige child_reg[PCI_FUNC(child->devfn)]); 2622a42d9dba7842422ffb2c02e75288a8bc2fd5065Thomas Renninger } 2635aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg); 2647d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 2657d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2665e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige/* Convert L0s latency encoding to ns */ 2675e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshigestatic u32 calc_l0s_latency(u32 encoding) 2687d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 2695e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige if (encoding == 0x7) 2705e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige return (5 * 1000); /* > 4us */ 2715e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige return (64 << encoding); 2725e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige} 2737d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2745e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige/* Convert L0s acceptable latency encoding to ns */ 2755e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshigestatic u32 calc_l0s_acceptable(u32 encoding) 2765e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige{ 2775e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige if (encoding == 0x7) 2785e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige return -1U; 2795e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige return (64 << encoding); 2807d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 2817d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2825e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige/* Convert L1 latency encoding to ns */ 2835e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshigestatic u32 calc_l1_latency(u32 encoding) 2847d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 2855e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige if (encoding == 0x7) 2865e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige return (65 * 1000); /* > 64us */ 2875e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige return (1000 << encoding); 2885e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige} 2897d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 2905e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige/* Convert L1 acceptable latency encoding to ns */ 2915e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshigestatic u32 calc_l1_acceptable(u32 encoding) 2925e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige{ 2935e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige if (encoding == 0x7) 2945e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige return -1U; 2955e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige return (1000 << encoding); 2967d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 2977d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 298ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshigestruct aspm_register_info { 299ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 support:2; 300ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 enabled:2; 301ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 latency_encoding_l0s; 302ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 latency_encoding_l1; 303ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige}; 304ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige 305ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshigestatic void pcie_get_aspm_reg(struct pci_dev *pdev, 306ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige struct aspm_register_info *info) 3077d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 3087d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li int pos; 3097d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li u16 reg16; 310ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 reg32; 3117d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 312db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige pos = pci_pcie_cap(pdev); 3137d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32); 314ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; 315ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; 316ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; 3177d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); 318ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; 3197d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 3207d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 32107d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshigestatic void pcie_aspm_check_latency(struct pci_dev *endpoint) 32207d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige{ 323ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 latency, l1_switch_latency = 0; 32407d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige struct aspm_latency *acceptable; 32507d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige struct pcie_link_state *link; 32607d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 32707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige /* Device not in D0 doesn't need latency check */ 32807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige if ((endpoint->current_state != PCI_D0) && 32907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige (endpoint->current_state != PCI_UNKNOWN)) 33007d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige return; 33107d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 33207d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige link = endpoint->bus->self->link_state; 33307d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; 33407d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 33507d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige while (link) { 336ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige /* Check upstream direction L0s latency */ 337ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if ((link->aspm_capable & ASPM_STATE_L0S_UP) && 338ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige (link->latency_up.l0s > acceptable->l0s)) 339ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_capable &= ~ASPM_STATE_L0S_UP; 340ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige 341ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige /* Check downstream direction L0s latency */ 342ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if ((link->aspm_capable & ASPM_STATE_L0S_DW) && 343ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige (link->latency_dw.l0s > acceptable->l0s)) 344ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_capable &= ~ASPM_STATE_L0S_DW; 34507d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige /* 34607d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige * Check L1 latency. 34707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige * Every switch on the path to root complex need 1 34807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige * more microsecond for L1. Spec doesn't mention L0s. 34907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige */ 350ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); 351ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if ((link->aspm_capable & ASPM_STATE_L1) && 352ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige (latency + l1_switch_latency > acceptable->l1)) 353ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_capable &= ~ASPM_STATE_L1; 35407d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige l1_switch_latency += 1000; 35507d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 35607d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige link = link->parent; 35707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige } 35807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige} 35907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 3608d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshigestatic void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 3617d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 3625aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_dev *child, *parent = link->pdev; 3635aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_bus *linkbus = parent->subordinate; 364ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige struct aspm_register_info upreg, dwreg; 3657d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 3668d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige if (blacklist) { 367f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige /* Set enabled/disable so that we will disable ASPM later */ 368ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_enabled = ASPM_STATE_ALL; 369ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_disable = ASPM_STATE_ALL; 3708d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige return; 3718d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige } 3728d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige 3738d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige /* Configure common clock before checking latencies */ 3748d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige pcie_aspm_configure_common_clock(link); 3758d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige 376ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige /* Get upstream/downstream components' register state */ 377ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige pcie_get_aspm_reg(parent, &upreg); 3785aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 379ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige pcie_get_aspm_reg(child, &dwreg); 380ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige 381ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige /* 382ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige * Setup L0s state 383ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige * 384ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige * Note that we must not enable L0s in either direction on a 385ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige * given link unless components on both sides of the link each 386ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige * support L0s. 387ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige */ 388ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) 389ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_support |= ASPM_STATE_L0S; 390ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (dwreg.enabled & PCIE_LINK_STATE_L0S) 391ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_enabled |= ASPM_STATE_L0S_UP; 392ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (upreg.enabled & PCIE_LINK_STATE_L0S) 393ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_enabled |= ASPM_STATE_L0S_DW; 394ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); 395ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); 396ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige 397ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige /* Setup L1 state */ 398ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) 399ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_support |= ASPM_STATE_L1; 400ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) 401ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_enabled |= ASPM_STATE_L1; 402ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); 403ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); 4045aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige 405b127bd55d9cd9d5b40278b30645669d6d46933bcKenji Kaneshige /* Save default state */ 406b127bd55d9cd9d5b40278b30645669d6d46933bcKenji Kaneshige link->aspm_default = link->aspm_enabled; 40707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 40807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige /* Setup initial capable state. Will be updated later */ 40907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige link->aspm_capable = link->aspm_support; 410f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige /* 411f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige * If the downstream component has pci bridge function, don't 412f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige * do ASPM for now. 413f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige */ 414f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige list_for_each_entry(child, &linkbus->devices, bus_list) { 415f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) { 416ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_disable = ASPM_STATE_ALL; 417f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige break; 418f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige } 419f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige } 420b127bd55d9cd9d5b40278b30645669d6d46933bcKenji Kaneshige 421b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige /* Get and check endpoint acceptable latencies */ 4225aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige list_for_each_entry(child, &linkbus->devices, bus_list) { 4237d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li int pos; 4245e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige u32 reg32, encoding; 425b6c2e54d3ea27719b920faf15db92dfe0260f0d2Kenji Kaneshige struct aspm_latency *acceptable = 4265aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige &link->acceptable[PCI_FUNC(child->devfn)]; 4277d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 4285aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT && 4295aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige child->pcie_type != PCI_EXP_TYPE_LEG_END) 4307d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li continue; 4317d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 432db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige pos = pci_pcie_cap(child); 4335aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32); 43407d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige /* Calculate endpoint L0s acceptable latency */ 4355e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; 4365e0eaa7d3679c3ef8618803bc9311270e5816641Kenji Kaneshige acceptable->l0s = calc_l0s_acceptable(encoding); 43707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige /* Calculate endpoint L1 acceptable latency */ 43807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; 43907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige acceptable->l1 = calc_l1_acceptable(encoding); 44007d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 44107d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige pcie_aspm_check_latency(child); 4427d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 4437d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 4447d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 445ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshigestatic void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) 4467d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 4477d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li u16 reg16; 448db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige int pos = pci_pcie_cap(pdev); 4497d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 4507d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); 4517d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li reg16 &= ~0x3; 452ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige reg16 |= val; 4537d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); 4547d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 4557d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 456b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshigestatic void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) 4577d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 458ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 upstream = 0, dwstream = 0; 4595aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_dev *child, *parent = link->pdev; 4605aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_bus *linkbus = parent->subordinate; 4617d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 462f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige /* Nothing to do if the link is already in the requested state */ 463b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige state &= (link->aspm_capable & ~link->aspm_disable); 464f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige if (link->aspm_enabled == state) 465f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige return; 466ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige /* Convert ASPM state to upstream/downstream ASPM register state */ 467ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (state & ASPM_STATE_L0S_UP) 468ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige dwstream |= PCIE_LINK_STATE_L0S; 469ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (state & ASPM_STATE_L0S_DW) 470ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige upstream |= PCIE_LINK_STATE_L0S; 471ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (state & ASPM_STATE_L1) { 472ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige upstream |= PCIE_LINK_STATE_L1; 473ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige dwstream |= PCIE_LINK_STATE_L1; 474ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige } 4757d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* 4765aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige * Spec 2.0 suggests all functions should be configured the 4775aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige * same setting for ASPM. Enabling ASPM L1 should be done in 4785aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige * upstream component first and then downstream, and vice 4795aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige * versa for disabling ASPM L1. Spec doesn't mention L0S. 4807d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li */ 481ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (state & ASPM_STATE_L1) 482ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige pcie_config_aspm_dev(parent, upstream); 4835aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige list_for_each_entry(child, &linkbus->devices, bus_list) 484ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige pcie_config_aspm_dev(child, dwstream); 485ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (!(state & ASPM_STATE_L1)) 486ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige pcie_config_aspm_dev(parent, upstream); 4877d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 4885aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige link->aspm_enabled = state; 4897d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 4907d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 491b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshigestatic void pcie_config_aspm_path(struct pcie_link_state *link) 4927d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 493b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige while (link) { 494b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_config_aspm_link(link, policy_to_aspm_state(link)); 495b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige link = link->parent; 49646bbdfa44cfc0d352148a0dc33ba9f6db02ccdf0Shaohua Li } 4977d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 4987d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 4995aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshigestatic void free_link_state(struct pcie_link_state *link) 5007d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 5015aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige link->pdev->link_state = NULL; 5025aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige kfree(link); 5037d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 5047d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 505ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Listatic int pcie_aspm_sanity_check(struct pci_dev *pdev) 506ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Li{ 5073647584d9ef35c9ec4abefdbea29959c26c54f13Kenji Kaneshige struct pci_dev *child; 5083647584d9ef35c9ec4abefdbea29959c26c54f13Kenji Kaneshige int pos; 509149e16372a2066c5474d8a8db9b252afd57eb427Shaohua Li u32 reg32; 5102f671e2dbff6eb5ef4e2600adbec550c13b8fe72Matthew Garrett 511ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Li /* 51245e829ea412760d2404d7dfc42528df46aedbf62Stefan Assmann * Some functions in a slot might not all be PCIe functions, 5133647584d9ef35c9ec4abefdbea29959c26c54f13Kenji Kaneshige * very strange. Disable ASPM for the whole slot 514ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Li */ 5153647584d9ef35c9ec4abefdbea29959c26c54f13Kenji Kaneshige list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { 516db9538a7495e33f3571c0e791c7678bc0c6ef50fKenji Kaneshige pos = pci_pcie_cap(child); 5173647584d9ef35c9ec4abefdbea29959c26c54f13Kenji Kaneshige if (!pos) 518ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Li return -EINVAL; 519c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett 520c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett /* 521c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett * If ASPM is disabled then we're not going to change 522c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett * the BIOS state. It's safe to continue even if it's a 523c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett * pre-1.1 device 524c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett */ 525c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett 526c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett if (aspm_disabled) 527c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett continue; 528c9651e70ad0aa499814817cbf3cc1d0b806ed3a1Matthew Garrett 529149e16372a2066c5474d8a8db9b252afd57eb427Shaohua Li /* 530149e16372a2066c5474d8a8db9b252afd57eb427Shaohua Li * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 531149e16372a2066c5474d8a8db9b252afd57eb427Shaohua Li * RBER bit to determine if a function is 1.1 version device 532149e16372a2066c5474d8a8db9b252afd57eb427Shaohua Li */ 5333647584d9ef35c9ec4abefdbea29959c26c54f13Kenji Kaneshige pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32); 534e1f4f59d1ab9ebac44830d6ae450fb358ac559d3Sitsofe Wheeler if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 5353647584d9ef35c9ec4abefdbea29959c26c54f13Kenji Kaneshige dev_printk(KERN_INFO, &child->dev, "disabling ASPM" 536f393d9b130423a7a47c751b26df07ceaa5dc76a9Vincent Legoll " on pre-1.1 PCIe device. You can enable it" 537f393d9b130423a7a47c751b26df07ceaa5dc76a9Vincent Legoll " with 'pcie_aspm=force'\n"); 538149e16372a2066c5474d8a8db9b252afd57eb427Shaohua Li return -EINVAL; 539149e16372a2066c5474d8a8db9b252afd57eb427Shaohua Li } 540ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Li } 541ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Li return 0; 542ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Li} 543ddc9753fcddfe5f9885dc133824962c047252b43Shaohua Li 544b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshigestatic struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) 5458d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige{ 5468d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige struct pcie_link_state *link; 5478d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige 5488d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige link = kzalloc(sizeof(*link), GFP_KERNEL); 5498d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige if (!link) 5508d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige return NULL; 5518d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige INIT_LIST_HEAD(&link->sibling); 5528d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige INIT_LIST_HEAD(&link->children); 5538d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige INIT_LIST_HEAD(&link->link); 5548d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige link->pdev = pdev; 5558d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) { 5568d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige struct pcie_link_state *parent; 5578d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige parent = pdev->bus->parent->self->link_state; 5588d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige if (!parent) { 5598d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige kfree(link); 5608d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige return NULL; 5618d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige } 5628d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige link->parent = parent; 5638d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige list_add(&link->link, &parent->children); 5648d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige } 5655c92ffb1ecc7f13267cdef5dda8a838937912c93Kenji Kaneshige /* Setup a pointer to the root port link */ 5665c92ffb1ecc7f13267cdef5dda8a838937912c93Kenji Kaneshige if (!link->parent) 5675c92ffb1ecc7f13267cdef5dda8a838937912c93Kenji Kaneshige link->root = link; 5685c92ffb1ecc7f13267cdef5dda8a838937912c93Kenji Kaneshige else 5695c92ffb1ecc7f13267cdef5dda8a838937912c93Kenji Kaneshige link->root = link->parent->root; 5705c92ffb1ecc7f13267cdef5dda8a838937912c93Kenji Kaneshige 5718d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige list_add(&link->sibling, &link_list); 5728d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige pdev->link_state = link; 5738d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige return link; 5748d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige} 5758d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige 5767d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li/* 5777d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * pcie_aspm_init_link_state: Initiate PCI express link state. 5787d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * It is called after the pcie and its children devices are scaned. 5797d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * @pdev: the root port or switch downstream port 5807d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li */ 5817d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Livoid pcie_aspm_init_link_state(struct pci_dev *pdev) 5827d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 5838d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige struct pcie_link_state *link; 584b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige int blacklist = !!pcie_aspm_sanity_check(pdev); 5857d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 5862f671e2dbff6eb5ef4e2600adbec550c13b8fe72Matthew Garrett if (!pci_is_pcie(pdev) || pdev->link_state) 5877d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 5887d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && 5898d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) 5907d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 5918d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige 5928e822df700694ca6850d1e0c122fd7004b2778d8Shaohua Li /* VIA has a strange chipset, root port is under a bridge */ 5938e822df700694ca6850d1e0c122fd7004b2778d8Shaohua Li if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT && 5948d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige pdev->bus->self) 5958e822df700694ca6850d1e0c122fd7004b2778d8Shaohua Li return; 5968d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige 5977d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li down_read(&pci_bus_sem); 5987d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (list_empty(&pdev->subordinate->devices)) 5997d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li goto out; 6007d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 6017d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_lock(&aspm_lock); 602b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige link = alloc_pcie_link_state(pdev); 6038d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige if (!link) 6048d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige goto unlock; 6058d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige /* 606b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige * Setup initial ASPM state. Note that we need to configure 607b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige * upstream links also because capable state of them can be 608b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige * update through pcie_aspm_cap_init(). 6098d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige */ 610b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_aspm_cap_init(link, blacklist); 6117d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 6128d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshige /* Setup initial Clock PM state */ 613b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_clkpm_cap_init(link, blacklist); 61441cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett 61541cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett /* 61641cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett * At this stage drivers haven't had an opportunity to change the 61741cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett * link policy setting. Enabling ASPM on broken hardware can cripple 61841cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett * it even before the driver has had a chance to disable ASPM, so 61941cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett * default to a safe level right now. If we're enabling ASPM beyond 62041cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett * the BIOS's expectation, we'll do so once pci_enable_device() is 62141cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett * called. 62241cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett */ 6233c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett if (aspm_policy != POLICY_POWERSAVE) { 62441cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett pcie_config_aspm_path(link); 62541cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett pcie_set_clkpm(link, policy_to_clkpm_state(link)); 62641cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett } 62741cd766b065970ff6f6c89dd1cf55fa706c84a3dMatthew Garrett 6288d349ace9a5c2a8404bcf4a371fe170480ffbebbKenji Kaneshigeunlock: 6297d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_unlock(&aspm_lock); 6307d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Liout: 6317d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li up_read(&pci_bus_sem); 6327d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 6337d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 63407d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige/* Recheck latencies and update aspm_capable for links under the root */ 63507d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshigestatic void pcie_update_aspm_capable(struct pcie_link_state *root) 63607d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige{ 63707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige struct pcie_link_state *link; 63807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige BUG_ON(root->parent); 63907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige list_for_each_entry(link, &link_list, sibling) { 64007d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige if (link->root != root) 64107d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige continue; 64207d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige link->aspm_capable = link->aspm_support; 64307d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige } 64407d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige list_for_each_entry(link, &link_list, sibling) { 64507d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige struct pci_dev *child; 64607d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige struct pci_bus *linkbus = link->pdev->subordinate; 64707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige if (link->root != root) 64807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige continue; 64907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige list_for_each_entry(child, &linkbus->devices, bus_list) { 65007d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT) && 65107d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige (child->pcie_type != PCI_EXP_TYPE_LEG_END)) 65207d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige continue; 65307d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige pcie_aspm_check_latency(child); 65407d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige } 65507d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige } 65607d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige} 65707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 6587d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li/* @pdev: the endpoint device */ 6597d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Livoid pcie_aspm_exit_link_state(struct pci_dev *pdev) 6607d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 6617d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct pci_dev *parent = pdev->bus->self; 662b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige struct pcie_link_state *link, *root, *parent_link; 6637d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 6643c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett if (!pci_is_pcie(pdev) || !parent || !parent->link_state) 6657d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 666b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige if ((parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && 667b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige (parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)) 6687d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 669fc87e919c0ce8e213edf2ffca17f384f059873d3Kenji Kaneshige 6707d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li down_read(&pci_bus_sem); 6717d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_lock(&aspm_lock); 6727d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* 6737d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * All PCIe functions are in one slot, remove one function will remove 6743419c75e15f82c3ab09bd944fddbde72c9e4b3eaAlex Chiang * the whole slot, so just wait until we are the last function left. 6757d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li */ 6763419c75e15f82c3ab09bd944fddbde72c9e4b3eaAlex Chiang if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) 6777d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li goto out; 6787d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 679fc87e919c0ce8e213edf2ffca17f384f059873d3Kenji Kaneshige link = parent->link_state; 68007d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige root = link->root; 681b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige parent_link = link->parent; 682fc87e919c0ce8e213edf2ffca17f384f059873d3Kenji Kaneshige 6837d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* All functions are removed, so just disable ASPM for the link */ 684b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_config_aspm_link(link, 0); 685fc87e919c0ce8e213edf2ffca17f384f059873d3Kenji Kaneshige list_del(&link->sibling); 686fc87e919c0ce8e213edf2ffca17f384f059873d3Kenji Kaneshige list_del(&link->link); 6877d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* Clock PM is for endpoint device */ 688fc87e919c0ce8e213edf2ffca17f384f059873d3Kenji Kaneshige free_link_state(link); 68907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige 69007d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige /* Recheck latencies and configure upstream links */ 691b26a34aa4792b3db2500b8a98cb7702765c1a92eKenji Kaneshige if (parent_link) { 692b26a34aa4792b3db2500b8a98cb7702765c1a92eKenji Kaneshige pcie_update_aspm_capable(root); 693b26a34aa4792b3db2500b8a98cb7702765c1a92eKenji Kaneshige pcie_config_aspm_path(parent_link); 694b26a34aa4792b3db2500b8a98cb7702765c1a92eKenji Kaneshige } 6957d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Liout: 6967d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_unlock(&aspm_lock); 6977d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li up_read(&pci_bus_sem); 6987d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 6997d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 7007d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li/* @pdev: the root port or switch downstream port */ 7017d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Livoid pcie_aspm_pm_state_change(struct pci_dev *pdev) 7027d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 70307d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige struct pcie_link_state *link = pdev->link_state; 7047d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 7058b06477dc4fcdfc21442ad334d3f3e335225ea0cKenji Kaneshige if (aspm_disabled || !pci_is_pcie(pdev) || !link) 7067d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 70707d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && 70807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)) 7097d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 7107d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li /* 71107d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige * Devices changed PM state, we should recheck if latency 71207d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige * meets all functions' requirement 7137d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li */ 71407d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige down_read(&pci_bus_sem); 71507d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige mutex_lock(&aspm_lock); 71607d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige pcie_update_aspm_capable(link->root); 717b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_config_aspm_path(link); 71807d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige mutex_unlock(&aspm_lock); 71907d92760d2ee542fe932f4e8b5807dd98481d1fdKenji Kaneshige up_read(&pci_bus_sem); 7207d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 7217d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 7221a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkarvoid pcie_aspm_powersave_config_link(struct pci_dev *pdev) 7231a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar{ 7241a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar struct pcie_link_state *link = pdev->link_state; 7251a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar 7261a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar if (aspm_disabled || !pci_is_pcie(pdev) || !link) 7271a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar return; 7281a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar 7291a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar if (aspm_policy != POLICY_POWERSAVE) 7301a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar return; 7311a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar 7321a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && 7331a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)) 7341a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar return; 7351a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar 7361a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar down_read(&pci_bus_sem); 7371a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar mutex_lock(&aspm_lock); 7381a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar pcie_config_aspm_path(link); 7391a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar pcie_set_clkpm(link, policy_to_clkpm_state(link)); 7401a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar mutex_unlock(&aspm_lock); 7411a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar up_read(&pci_bus_sem); 7421a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar} 7431a680b7c325882188865f05b9a88d32f75f26495Naga Chumbalkar 7447d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li/* 7457d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * pci_disable_link_state - disable pci device's link state, so the link will 7467d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li * never enter specific states 7477d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li */ 7483c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrettstatic void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem, 7493c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett bool force) 7507d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 7517d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct pci_dev *parent = pdev->bus->self; 752f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige struct pcie_link_state *link; 7537d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 7543c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett if (aspm_disabled && !force) 7553c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett return; 7563c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett 7573c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett if (!pci_is_pcie(pdev)) 7587d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 7593c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett 7607d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT || 7617d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) 7627d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li parent = pdev; 7637d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!parent || !parent->link_state) 7647d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 7657d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 7669f728f53dd70396f3183d2f0861022259471824bYinghai Lu if (sem) 7679f728f53dd70396f3183d2f0861022259471824bYinghai Lu down_read(&pci_bus_sem); 7687d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_lock(&aspm_lock); 769f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige link = parent->link_state; 770ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (state & PCIE_LINK_STATE_L0S) 771ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_disable |= ASPM_STATE_L0S; 772ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (state & PCIE_LINK_STATE_L1) 773ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige link->aspm_disable |= ASPM_STATE_L1; 774b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_config_aspm_link(link, policy_to_aspm_state(link)); 775b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige 776430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige if (state & PCIE_LINK_STATE_CLKPM) { 777f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige link->clkpm_capable = 0; 778f1c0ca29ae72bc0c10282eada66c8a792ee98482Kenji Kaneshige pcie_set_clkpm(link, 0); 779430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige } 7807d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_unlock(&aspm_lock); 7819f728f53dd70396f3183d2f0861022259471824bYinghai Lu if (sem) 7829f728f53dd70396f3183d2f0861022259471824bYinghai Lu up_read(&pci_bus_sem); 7839f728f53dd70396f3183d2f0861022259471824bYinghai Lu} 7849f728f53dd70396f3183d2f0861022259471824bYinghai Lu 7859f728f53dd70396f3183d2f0861022259471824bYinghai Luvoid pci_disable_link_state_locked(struct pci_dev *pdev, int state) 7869f728f53dd70396f3183d2f0861022259471824bYinghai Lu{ 7873c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett __pci_disable_link_state(pdev, state, false, false); 7889f728f53dd70396f3183d2f0861022259471824bYinghai Lu} 7899f728f53dd70396f3183d2f0861022259471824bYinghai LuEXPORT_SYMBOL(pci_disable_link_state_locked); 7909f728f53dd70396f3183d2f0861022259471824bYinghai Lu 7919f728f53dd70396f3183d2f0861022259471824bYinghai Luvoid pci_disable_link_state(struct pci_dev *pdev, int state) 7929f728f53dd70396f3183d2f0861022259471824bYinghai Lu{ 7933c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett __pci_disable_link_state(pdev, state, true, false); 7947d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 7957d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua LiEXPORT_SYMBOL(pci_disable_link_state); 7967d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 7973c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrettvoid pcie_clear_aspm(struct pci_bus *bus) 7983c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett{ 7993c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett struct pci_dev *child; 8003c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett 8013c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett /* 8023c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett * Clear any ASPM setup that the firmware has carried out on this bus 8033c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett */ 8043c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett list_for_each_entry(child, &bus->devices, bus_list) { 8053c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett __pci_disable_link_state(child, PCIE_LINK_STATE_L0S | 8063c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett PCIE_LINK_STATE_L1 | 8073c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett PCIE_LINK_STATE_CLKPM, 8083c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett false, true); 8093c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett } 8103c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett} 8113c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett 8127d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) 8137d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 8147d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li int i; 815b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige struct pcie_link_state *link; 8167d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 817bbfa306a1e5d9618231aa0de3d52a8eb1219d0c3Naga Chumbalkar if (aspm_disabled) 818bbfa306a1e5d9618231aa0de3d52a8eb1219d0c3Naga Chumbalkar return -EPERM; 8197d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li for (i = 0; i < ARRAY_SIZE(policy_str); i++) 8207d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) 8217d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li break; 8227d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (i >= ARRAY_SIZE(policy_str)) 8237d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return -EINVAL; 8247d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (i == aspm_policy) 8257d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return 0; 8267d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 8277d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li down_read(&pci_bus_sem); 8287d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_lock(&aspm_lock); 8297d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li aspm_policy = i; 830b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige list_for_each_entry(link, &link_list, sibling) { 831b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_config_aspm_link(link, policy_to_aspm_state(link)); 832b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_set_clkpm(link, policy_to_clkpm_state(link)); 8337d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li } 8347d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_unlock(&aspm_lock); 8357d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li up_read(&pci_bus_sem); 8367d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return 0; 8377d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 8387d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 8397d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) 8407d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 8417d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li int i, cnt = 0; 8427d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li for (i = 0; i < ARRAY_SIZE(policy_str); i++) 8437d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (i == aspm_policy) 8447d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); 8457d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li else 8467d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); 8477d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return cnt; 8487d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 8497d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 8507d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Limodule_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, 8517d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li NULL, 0644); 8527d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 8537d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#ifdef CONFIG_PCIEASPM_DEBUG 8547d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic ssize_t link_state_show(struct device *dev, 8557d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct device_attribute *attr, 8567d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li char *buf) 8577d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 8587d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct pci_dev *pci_device = to_pci_dev(dev); 8597d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct pcie_link_state *link_state = pci_device->link_state; 8607d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 86180bfdbe370d56a1448c7078cd6d286b89241a72eKenji Kaneshige return sprintf(buf, "%d\n", link_state->aspm_enabled); 8627d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 8637d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 8647d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic ssize_t link_state_store(struct device *dev, 8657d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct device_attribute *attr, 8667d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li const char *buf, 8677d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li size_t n) 8687d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 8695aa63583cbec27482c6f1d761a0509f59b7969a8Kenji Kaneshige struct pci_dev *pdev = to_pci_dev(dev); 870b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige struct pcie_link_state *link, *root = pdev->link_state->root; 871ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige u32 val = buf[0] - '0', state = 0; 8727d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 873bbfa306a1e5d9618231aa0de3d52a8eb1219d0c3Naga Chumbalkar if (aspm_disabled) 874bbfa306a1e5d9618231aa0de3d52a8eb1219d0c3Naga Chumbalkar return -EPERM; 875ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (n < 1 || val > 3) 8767d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return -EINVAL; 8777d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 878ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige /* Convert requested state to ASPM state */ 879ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (val & PCIE_LINK_STATE_L0S) 880ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige state |= ASPM_STATE_L0S; 881ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige if (val & PCIE_LINK_STATE_L1) 882ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige state |= ASPM_STATE_L1; 883ac18018a414a90d841ea81d38fecb913c0ec1880Kenji Kaneshige 884b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige down_read(&pci_bus_sem); 885b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige mutex_lock(&aspm_lock); 886b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige list_for_each_entry(link, &link_list, sibling) { 887b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige if (link->root != root) 888b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige continue; 889b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige pcie_config_aspm_link(link, state); 890b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige } 891b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige mutex_unlock(&aspm_lock); 892b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige up_read(&pci_bus_sem); 893b7206cbf024dd43c42f9585e2017db1c1facd566Kenji Kaneshige return n; 8947d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 8957d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 8967d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic ssize_t clk_ctl_show(struct device *dev, 8977d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct device_attribute *attr, 8987d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li char *buf) 8997d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 9007d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct pci_dev *pci_device = to_pci_dev(dev); 9017d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct pcie_link_state *link_state = pci_device->link_state; 9027d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9034d246e458918d469ad645fd5f937ac22982e0466Kenji Kaneshige return sprintf(buf, "%d\n", link_state->clkpm_enabled); 9047d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 9057d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9067d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic ssize_t clk_ctl_store(struct device *dev, 9077d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct device_attribute *attr, 9087d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li const char *buf, 9097d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li size_t n) 9107d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 911430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige struct pci_dev *pdev = to_pci_dev(dev); 9127d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li int state; 9137d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9147d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li if (n < 1) 9157d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return -EINVAL; 9167d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li state = buf[0]-'0'; 9177d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9187d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li down_read(&pci_bus_sem); 9197d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_lock(&aspm_lock); 920430842e29d396928989c0a45e05025e988004d79Kenji Kaneshige pcie_set_clkpm_nocheck(pdev->link_state, !!state); 9217d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li mutex_unlock(&aspm_lock); 9227d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li up_read(&pci_bus_sem); 9237d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9247d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return n; 9257d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 9267d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9277d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store); 9287d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store); 9297d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9307d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic char power_group[] = "power"; 9317d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Livoid pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) 9327d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 9337d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct pcie_link_state *link_state = pdev->link_state; 9347d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9358b06477dc4fcdfc21442ad334d3f3e335225ea0cKenji Kaneshige if (!pci_is_pcie(pdev) || 9368b06477dc4fcdfc21442ad334d3f3e335225ea0cKenji Kaneshige (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && 9378b06477dc4fcdfc21442ad334d3f3e335225ea0cKenji Kaneshige pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) 9387d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 9397d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 94080bfdbe370d56a1448c7078cd6d286b89241a72eKenji Kaneshige if (link_state->aspm_support) 9417d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li sysfs_add_file_to_group(&pdev->dev.kobj, 9427d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li &dev_attr_link_state.attr, power_group); 9434d246e458918d469ad645fd5f937ac22982e0466Kenji Kaneshige if (link_state->clkpm_capable) 9447d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li sysfs_add_file_to_group(&pdev->dev.kobj, 9457d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li &dev_attr_clk_ctl.attr, power_group); 9467d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 9477d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9487d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Livoid pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) 9497d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 9507d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li struct pcie_link_state *link_state = pdev->link_state; 9517d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9528b06477dc4fcdfc21442ad334d3f3e335225ea0cKenji Kaneshige if (!pci_is_pcie(pdev) || 9538b06477dc4fcdfc21442ad334d3f3e335225ea0cKenji Kaneshige (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && 9548b06477dc4fcdfc21442ad334d3f3e335225ea0cKenji Kaneshige pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) 9557d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return; 9567d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 95780bfdbe370d56a1448c7078cd6d286b89241a72eKenji Kaneshige if (link_state->aspm_support) 9587d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li sysfs_remove_file_from_group(&pdev->dev.kobj, 9597d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li &dev_attr_link_state.attr, power_group); 9604d246e458918d469ad645fd5f937ac22982e0466Kenji Kaneshige if (link_state->clkpm_capable) 9617d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li sysfs_remove_file_from_group(&pdev->dev.kobj, 9627d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li &dev_attr_clk_ctl.attr, power_group); 9637d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 9647d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li#endif 9657d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9667d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Listatic int __init pcie_aspm_disable(char *str) 9677d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 968d6d385743463f38a0da899cd4607e526ad9a049fShaohua Li if (!strcmp(str, "off")) { 9693c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett aspm_policy = POLICY_DEFAULT; 970d6d385743463f38a0da899cd4607e526ad9a049fShaohua Li aspm_disabled = 1; 9718b8bae901ce23addbdcdb54fa1696fb2d049feb5Rafael J. Wysocki aspm_support_enabled = false; 972d6d385743463f38a0da899cd4607e526ad9a049fShaohua Li printk(KERN_INFO "PCIe ASPM is disabled\n"); 973d6d385743463f38a0da899cd4607e526ad9a049fShaohua Li } else if (!strcmp(str, "force")) { 974d6d385743463f38a0da899cd4607e526ad9a049fShaohua Li aspm_force = 1; 9758072ba1ba7fe9f48ad9f424829863214484dfc2fMichael Witten printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); 976d6d385743463f38a0da899cd4607e526ad9a049fShaohua Li } 9777d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li return 1; 9787d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 9797d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 980d6d385743463f38a0da899cd4607e526ad9a049fShaohua Li__setup("pcie_aspm=", pcie_aspm_disable); 9817d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 9825fde244d39b88625ac578d83e6625138714de031Shaohua Livoid pcie_no_aspm(void) 9835fde244d39b88625ac578d83e6625138714de031Shaohua Li{ 9843c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett /* 9853c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett * Disabling ASPM is intended to prevent the kernel from modifying 9863c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett * existing hardware state, not to clear existing state. To that end: 9873c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett * (a) set policy to POLICY_DEFAULT in order to avoid changing state 9883c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett * (b) prevent userspace from changing policy 9893c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett */ 9903c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett if (!aspm_force) { 9913c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett aspm_policy = POLICY_DEFAULT; 992d6d385743463f38a0da899cd4607e526ad9a049fShaohua Li aspm_disabled = 1; 9933c076351c4027a56d5005a39a0b518a4ba393ce2Matthew Garrett } 9945fde244d39b88625ac578d83e6625138714de031Shaohua Li} 9955fde244d39b88625ac578d83e6625138714de031Shaohua Li 9963e1b16002af29758b6bc9c38939d43838d9335bcAndrew Patterson/** 9973e1b16002af29758b6bc9c38939d43838d9335bcAndrew Patterson * pcie_aspm_enabled - is PCIe ASPM enabled? 9983e1b16002af29758b6bc9c38939d43838d9335bcAndrew Patterson * 9993e1b16002af29758b6bc9c38939d43838d9335bcAndrew Patterson * Returns true if ASPM has not been disabled by the command-line option 10003e1b16002af29758b6bc9c38939d43838d9335bcAndrew Patterson * pcie_aspm=off. 10013e1b16002af29758b6bc9c38939d43838d9335bcAndrew Patterson **/ 10023e1b16002af29758b6bc9c38939d43838d9335bcAndrew Pattersonint pcie_aspm_enabled(void) 10037d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li{ 10043e1b16002af29758b6bc9c38939d43838d9335bcAndrew Patterson return !aspm_disabled; 10057d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li} 10063e1b16002af29758b6bc9c38939d43838d9335bcAndrew PattersonEXPORT_SYMBOL(pcie_aspm_enabled); 10077d715a6c1ae5785d00fb9a876b5abdfc43abc44bShaohua Li 10088b8bae901ce23addbdcdb54fa1696fb2d049feb5Rafael J. Wysockibool pcie_aspm_support_enabled(void) 10098b8bae901ce23addbdcdb54fa1696fb2d049feb5Rafael J. Wysocki{ 10108b8bae901ce23addbdcdb54fa1696fb2d049feb5Rafael J. Wysocki return aspm_support_enabled; 10118b8bae901ce23addbdcdb54fa1696fb2d049feb5Rafael J. Wysocki} 10128b8bae901ce23addbdcdb54fa1696fb2d049feb5Rafael J. WysockiEXPORT_SYMBOL(pcie_aspm_support_enabled); 1013