1/*
   2 * File:	drivers/pci/pcie/aspm.c
   3 * Enabling PCIe link L0s/L1 state and Clock Power Management
   4 *
   5 * Copyright (C) 2007 Intel
   6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
   7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
   8 */
   9
  10#include <linux/kernel.h>
  11#include <linux/module.h>
  12#include <linux/moduleparam.h>
  13#include <linux/pci.h>
  14#include <linux/pci_regs.h>
  15#include <linux/errno.h>
  16#include <linux/pm.h>
  17#include <linux/init.h>
  18#include <linux/slab.h>
  19#include <linux/jiffies.h>
  20#include <linux/delay.h>
  21#include <linux/pci-aspm.h>
  22#include "../pci.h"
  23
  24#ifdef MODULE_PARAM_PREFIX
  25#undef MODULE_PARAM_PREFIX
  26#endif
  27#define MODULE_PARAM_PREFIX "pcie_aspm."
  28
  29/* Note: those are not register definitions */
  30#define ASPM_STATE_L0S_UP	(1)	/* Upstream direction L0s state */
  31#define ASPM_STATE_L0S_DW	(2)	/* Downstream direction L0s state */
  32#define ASPM_STATE_L1		(4)	/* L1 state */
 
 
 
 
 
 
 
 
  33#define ASPM_STATE_L0S		(ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
  34#define ASPM_STATE_ALL		(ASPM_STATE_L0S | ASPM_STATE_L1)
 
  35
  36struct aspm_latency {
  37	u32 l0s;			/* L0s latency (nsec) */
  38	u32 l1;				/* L1 latency (nsec) */
  39};
  40
  41struct pcie_link_state {
  42	struct pci_dev *pdev;		/* Upstream component of the Link */
 
  43	struct pcie_link_state *root;	/* pointer to the root port link */
  44	struct pcie_link_state *parent;	/* pointer to the parent Link state */
  45	struct list_head sibling;	/* node in link_list */
  46	struct list_head children;	/* list of child link states */
  47	struct list_head link;		/* node in parent's children list */
  48
  49	/* ASPM state */
  50	u32 aspm_support:3;		/* Supported ASPM state */
  51	u32 aspm_enabled:3;		/* Enabled ASPM state */
  52	u32 aspm_capable:3;		/* Capable ASPM state with latency */
  53	u32 aspm_default:3;		/* Default ASPM state by BIOS */
  54	u32 aspm_disable:3;		/* Disabled ASPM state */
  55
  56	/* Clock PM state */
  57	u32 clkpm_capable:1;		/* Clock PM capable? */
  58	u32 clkpm_enabled:1;		/* Current Clock PM state */
  59	u32 clkpm_default:1;		/* Default Clock PM state by BIOS */
 
  60
  61	/* Exit latencies */
  62	struct aspm_latency latency_up;	/* Upstream direction exit latency */
  63	struct aspm_latency latency_dw;	/* Downstream direction exit latency */
  64	/*
  65	 * Endpoint acceptable latencies. A pcie downstream port only
  66	 * has one slot under it, so at most there are 8 functions.
  67	 */
  68	struct aspm_latency acceptable[8];
 
 
 
 
 
 
 
 
  69};
  70
  71static int aspm_disabled, aspm_force;
  72static bool aspm_support_enabled = true;
  73static DEFINE_MUTEX(aspm_lock);
  74static LIST_HEAD(link_list);
  75
  76#define POLICY_DEFAULT 0	/* BIOS default setting */
  77#define POLICY_PERFORMANCE 1	/* high performance */
  78#define POLICY_POWERSAVE 2	/* high power saving */
 
  79
  80#ifdef CONFIG_PCIEASPM_PERFORMANCE
  81static int aspm_policy = POLICY_PERFORMANCE;
  82#elif defined CONFIG_PCIEASPM_POWERSAVE
  83static int aspm_policy = POLICY_POWERSAVE;
 
 
  84#else
  85static int aspm_policy;
  86#endif
  87
  88static const char *policy_str[] = {
  89	[POLICY_DEFAULT] = "default",
  90	[POLICY_PERFORMANCE] = "performance",
  91	[POLICY_POWERSAVE] = "powersave"
 
  92};
  93
  94#define LINK_RETRAIN_TIMEOUT HZ
  95
  96static int policy_to_aspm_state(struct pcie_link_state *link)
  97{
  98	switch (aspm_policy) {
  99	case POLICY_PERFORMANCE:
 100		/* Disable ASPM and Clock PM */
 101		return 0;
 102	case POLICY_POWERSAVE:
 103		/* Enable ASPM L0s/L1 */
 
 
 
 104		return ASPM_STATE_ALL;
 105	case POLICY_DEFAULT:
 106		return link->aspm_default;
 107	}
 108	return 0;
 109}
 110
 111static int policy_to_clkpm_state(struct pcie_link_state *link)
 112{
 113	switch (aspm_policy) {
 114	case POLICY_PERFORMANCE:
 115		/* Disable ASPM and Clock PM */
 116		return 0;
 117	case POLICY_POWERSAVE:
 118		/* Disable Clock PM */
 
 119		return 1;
 120	case POLICY_DEFAULT:
 121		return link->clkpm_default;
 122	}
 123	return 0;
 124}
 125
 126static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
 127{
 128	int pos;
 129	u16 reg16;
 130	struct pci_dev *child;
 131	struct pci_bus *linkbus = link->pdev->subordinate;
 
 132
 133	list_for_each_entry(child, &linkbus->devices, bus_list) {
 134		pos = pci_pcie_cap(child);
 135		if (!pos)
 136			return;
 137		pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
 138		if (enable)
 139			reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
 140		else
 141			reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
 142		pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
 143	}
 144	link->clkpm_enabled = !!enable;
 145}
 146
 147static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
 148{
 149	/* Don't enable Clock PM if the link is not Clock PM capable */
 150	if (!link->clkpm_capable && enable)
 
 
 
 151		enable = 0;
 152	/* Need nothing if the specified equals to current state */
 153	if (link->clkpm_enabled == enable)
 154		return;
 155	pcie_set_clkpm_nocheck(link, enable);
 156}
 157
 158static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
 159{
 160	int pos, capable = 1, enabled = 1;
 161	u32 reg32;
 162	u16 reg16;
 163	struct pci_dev *child;
 164	struct pci_bus *linkbus = link->pdev->subordinate;
 165
 166	/* All functions should have the same cap and state, take the worst */
 167	list_for_each_entry(child, &linkbus->devices, bus_list) {
 168		pos = pci_pcie_cap(child);
 169		if (!pos)
 170			return;
 171		pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, &reg32);
 172		if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
 173			capable = 0;
 174			enabled = 0;
 175			break;
 176		}
 177		pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
 178		if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
 179			enabled = 0;
 180	}
 181	link->clkpm_enabled = enabled;
 182	link->clkpm_default = enabled;
 183	link->clkpm_capable = (blacklist) ? 0 : capable;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 184}
 185
 186/*
 187 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
 188 *   could use common clock. If they are, configure them to use the
 189 *   common clock. That will reduce the ASPM state exit latency.
 190 */
 191static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
 192{
 193	int ppos, cpos, same_clock = 1;
 194	u16 reg16, parent_reg, child_reg[8];
 195	unsigned long start_jiffies;
 196	struct pci_dev *child, *parent = link->pdev;
 197	struct pci_bus *linkbus = parent->subordinate;
 198	/*
 199	 * All functions of a slot should have the same Slot Clock
 200	 * Configuration, so just check one function
 201	 */
 202	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
 203	BUG_ON(!pci_is_pcie(child));
 204
 205	/* Check downstream component if bit Slot Clock Configuration is 1 */
 206	cpos = pci_pcie_cap(child);
 207	pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, &reg16);
 208	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
 209		same_clock = 0;
 210
 211	/* Check upstream component if bit Slot Clock Configuration is 1 */
 212	ppos = pci_pcie_cap(parent);
 213	pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
 214	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
 215		same_clock = 0;
 216
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 217	/* Configure downstream component, all functions */
 218	list_for_each_entry(child, &linkbus->devices, bus_list) {
 219		cpos = pci_pcie_cap(child);
 220		pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, &reg16);
 221		child_reg[PCI_FUNC(child->devfn)] = reg16;
 222		if (same_clock)
 223			reg16 |= PCI_EXP_LNKCTL_CCC;
 224		else
 225			reg16 &= ~PCI_EXP_LNKCTL_CCC;
 226		pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
 227	}
 228
 229	/* Configure upstream component */
 230	pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, &reg16);
 231	parent_reg = reg16;
 232	if (same_clock)
 233		reg16 |= PCI_EXP_LNKCTL_CCC;
 234	else
 235		reg16 &= ~PCI_EXP_LNKCTL_CCC;
 236	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
 237
 238	/* Retrain link */
 239	reg16 |= PCI_EXP_LNKCTL_RL;
 240	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
 241
 242	/* Wait for link training end. Break out after waiting for timeout */
 243	start_jiffies = jiffies;
 244	for (;;) {
 245		pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
 246		if (!(reg16 & PCI_EXP_LNKSTA_LT))
 247			break;
 248		if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
 249			break;
 250		msleep(1);
 251	}
 252	if (!(reg16 & PCI_EXP_LNKSTA_LT))
 253		return;
 254
 255	/* Training failed. Restore common clock configurations */
 256	dev_printk(KERN_ERR, &parent->dev,
 257		   "ASPM: Could not configure common clock\n");
 258	list_for_each_entry(child, &linkbus->devices, bus_list) {
 259		cpos = pci_pcie_cap(child);
 260		pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
 261				      child_reg[PCI_FUNC(child->devfn)]);
 262	}
 263	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
 264}
 265
 266/* Convert L0s latency encoding to ns */
 267static u32 calc_l0s_latency(u32 encoding)
 268{
 269	if (encoding == 0x7)
 270		return (5 * 1000);	/* > 4us */
 271	return (64 << encoding);
 272}
 273
 274/* Convert L0s acceptable latency encoding to ns */
 275static u32 calc_l0s_acceptable(u32 encoding)
 276{
 277	if (encoding == 0x7)
 278		return -1U;
 279	return (64 << encoding);
 280}
 281
 282/* Convert L1 latency encoding to ns */
 283static u32 calc_l1_latency(u32 encoding)
 284{
 285	if (encoding == 0x7)
 286		return (65 * 1000);	/* > 64us */
 287	return (1000 << encoding);
 288}
 289
 290/* Convert L1 acceptable latency encoding to ns */
 291static u32 calc_l1_acceptable(u32 encoding)
 292{
 293	if (encoding == 0x7)
 294		return -1U;
 295	return (1000 << encoding);
 296}
 297
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 298struct aspm_register_info {
 299	u32 support:2;
 300	u32 enabled:2;
 301	u32 latency_encoding_l0s;
 302	u32 latency_encoding_l1;
 
 
 
 
 
 
 303};
 304
 305static void pcie_get_aspm_reg(struct pci_dev *pdev,
 306			      struct aspm_register_info *info)
 307{
 308	int pos;
 309	u16 reg16;
 310	u32 reg32;
 311
 312	pos = pci_pcie_cap(pdev);
 313	pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, &reg32);
 314	info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
 315	info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
 316	info->latency_encoding_l1  = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
 317	pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
 318	info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 319}
 320
 321static void pcie_aspm_check_latency(struct pci_dev *endpoint)
 322{
 323	u32 latency, l1_switch_latency = 0;
 324	struct aspm_latency *acceptable;
 325	struct pcie_link_state *link;
 326
 327	/* Device not in D0 doesn't need latency check */
 328	if ((endpoint->current_state != PCI_D0) &&
 329	    (endpoint->current_state != PCI_UNKNOWN))
 330		return;
 331
 332	link = endpoint->bus->self->link_state;
 333	acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
 334
 335	while (link) {
 336		/* Check upstream direction L0s latency */
 337		if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
 338		    (link->latency_up.l0s > acceptable->l0s))
 339			link->aspm_capable &= ~ASPM_STATE_L0S_UP;
 340
 341		/* Check downstream direction L0s latency */
 342		if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
 343		    (link->latency_dw.l0s > acceptable->l0s))
 344			link->aspm_capable &= ~ASPM_STATE_L0S_DW;
 345		/*
 346		 * Check L1 latency.
 347		 * Every switch on the path to root complex need 1
 348		 * more microsecond for L1. Spec doesn't mention L0s.
 
 
 
 
 
 
 
 
 349		 */
 350		latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
 351		if ((link->aspm_capable & ASPM_STATE_L1) &&
 352		    (latency + l1_switch_latency > acceptable->l1))
 353			link->aspm_capable &= ~ASPM_STATE_L1;
 354		l1_switch_latency += 1000;
 355
 356		link = link->parent;
 357	}
 358}
 359
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 360static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
 361{
 362	struct pci_dev *child, *parent = link->pdev;
 363	struct pci_bus *linkbus = parent->subordinate;
 364	struct aspm_register_info upreg, dwreg;
 365
 366	if (blacklist) {
 367		/* Set enabled/disable so that we will disable ASPM later */
 368		link->aspm_enabled = ASPM_STATE_ALL;
 369		link->aspm_disable = ASPM_STATE_ALL;
 370		return;
 371	}
 372
 
 
 
 
 
 
 
 
 
 
 
 373	/* Configure common clock before checking latencies */
 374	pcie_aspm_configure_common_clock(link);
 375
 376	/* Get upstream/downstream components' register state */
 
 
 
 377	pcie_get_aspm_reg(parent, &upreg);
 378	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
 379	pcie_get_aspm_reg(child, &dwreg);
 380
 381	/*
 382	 * Setup L0s state
 383	 *
 384	 * Note that we must not enable L0s in either direction on a
 385	 * given link unless components on both sides of the link each
 386	 * support L0s.
 387	 */
 388	if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
 389		link->aspm_support |= ASPM_STATE_L0S;
 390	if (dwreg.enabled & PCIE_LINK_STATE_L0S)
 391		link->aspm_enabled |= ASPM_STATE_L0S_UP;
 392	if (upreg.enabled & PCIE_LINK_STATE_L0S)
 393		link->aspm_enabled |= ASPM_STATE_L0S_DW;
 394	link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
 395	link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
 396
 397	/* Setup L1 state */
 398	if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
 399		link->aspm_support |= ASPM_STATE_L1;
 400	if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
 401		link->aspm_enabled |= ASPM_STATE_L1;
 402	link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
 403	link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
 404
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 405	/* Save default state */
 406	link->aspm_default = link->aspm_enabled;
 407
 408	/* Setup initial capable state. Will be updated later */
 409	link->aspm_capable = link->aspm_support;
 410	/*
 411	 * If the downstream component has pci bridge function, don't
 412	 * do ASPM for now.
 413	 */
 414	list_for_each_entry(child, &linkbus->devices, bus_list) {
 415		if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
 416			link->aspm_disable = ASPM_STATE_ALL;
 417			break;
 418		}
 419	}
 420
 421	/* Get and check endpoint acceptable latencies */
 422	list_for_each_entry(child, &linkbus->devices, bus_list) {
 423		int pos;
 424		u32 reg32, encoding;
 425		struct aspm_latency *acceptable =
 426			&link->acceptable[PCI_FUNC(child->devfn)];
 427
 428		if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
 429		    child->pcie_type != PCI_EXP_TYPE_LEG_END)
 430			continue;
 431
 432		pos = pci_pcie_cap(child);
 433		pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
 434		/* Calculate endpoint L0s acceptable latency */
 435		encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
 436		acceptable->l0s = calc_l0s_acceptable(encoding);
 437		/* Calculate endpoint L1 acceptable latency */
 438		encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
 439		acceptable->l1 = calc_l1_acceptable(encoding);
 440
 441		pcie_aspm_check_latency(child);
 442	}
 443}
 444
 445static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
 
 446{
 447	u16 reg16;
 448	int pos = pci_pcie_cap(pdev);
 
 
 
 
 
 449
 450	pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
 451	reg16 &= ~0x3;
 452	reg16 |= val;
 453	pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 454}
 455
 456static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
 457{
 458	u32 upstream = 0, dwstream = 0;
 459	struct pci_dev *child, *parent = link->pdev;
 460	struct pci_bus *linkbus = parent->subordinate;
 461
 462	/* Nothing to do if the link is already in the requested state */
 463	state &= (link->aspm_capable & ~link->aspm_disable);
 
 
 
 
 
 
 
 
 
 
 
 
 464	if (link->aspm_enabled == state)
 465		return;
 466	/* Convert ASPM state to upstream/downstream ASPM register state */
 467	if (state & ASPM_STATE_L0S_UP)
 468		dwstream |= PCIE_LINK_STATE_L0S;
 469	if (state & ASPM_STATE_L0S_DW)
 470		upstream |= PCIE_LINK_STATE_L0S;
 471	if (state & ASPM_STATE_L1) {
 472		upstream |= PCIE_LINK_STATE_L1;
 473		dwstream |= PCIE_LINK_STATE_L1;
 474	}
 
 
 
 
 475	/*
 476	 * Spec 2.0 suggests all functions should be configured the
 477	 * same setting for ASPM. Enabling ASPM L1 should be done in
 478	 * upstream component first and then downstream, and vice
 479	 * versa for disabling ASPM L1. Spec doesn't mention L0S.
 480	 */
 481	if (state & ASPM_STATE_L1)
 482		pcie_config_aspm_dev(parent, upstream);
 483	list_for_each_entry(child, &linkbus->devices, bus_list)
 484		pcie_config_aspm_dev(child, dwstream);
 485	if (!(state & ASPM_STATE_L1))
 486		pcie_config_aspm_dev(parent, upstream);
 487
 488	link->aspm_enabled = state;
 489}
 490
 491static void pcie_config_aspm_path(struct pcie_link_state *link)
 492{
 493	while (link) {
 494		pcie_config_aspm_link(link, policy_to_aspm_state(link));
 495		link = link->parent;
 496	}
 497}
 498
 499static void free_link_state(struct pcie_link_state *link)
 500{
 501	link->pdev->link_state = NULL;
 502	kfree(link);
 503}
 504
 505static int pcie_aspm_sanity_check(struct pci_dev *pdev)
 506{
 507	struct pci_dev *child;
 508	int pos;
 509	u32 reg32;
 510
 511	/*
 512	 * Some functions in a slot might not all be PCIe functions,
 513	 * very strange. Disable ASPM for the whole slot
 514	 */
 515	list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
 516		pos = pci_pcie_cap(child);
 517		if (!pos)
 518			return -EINVAL;
 519
 520		/*
 521		 * If ASPM is disabled then we're not going to change
 522		 * the BIOS state. It's safe to continue even if it's a
 523		 * pre-1.1 device
 524		 */
 525
 526		if (aspm_disabled)
 527			continue;
 528
 529		/*
 530		 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
 531		 * RBER bit to determine if a function is 1.1 version device
 532		 */
 533		pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
 534		if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
 535			dev_printk(KERN_INFO, &child->dev, "disabling ASPM"
 536				" on pre-1.1 PCIe device.  You can enable it"
 537				" with 'pcie_aspm=force'\n");
 538			return -EINVAL;
 539		}
 540	}
 541	return 0;
 542}
 543
 544static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
 545{
 546	struct pcie_link_state *link;
 547
 548	link = kzalloc(sizeof(*link), GFP_KERNEL);
 549	if (!link)
 550		return NULL;
 
 551	INIT_LIST_HEAD(&link->sibling);
 552	INIT_LIST_HEAD(&link->children);
 553	INIT_LIST_HEAD(&link->link);
 554	link->pdev = pdev;
 555	if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 556		struct pcie_link_state *parent;
 
 557		parent = pdev->bus->parent->self->link_state;
 558		if (!parent) {
 559			kfree(link);
 560			return NULL;
 561		}
 
 562		link->parent = parent;
 563		list_add(&link->link, &parent->children);
 564	}
 565	/* Setup a pointer to the root port link */
 566	if (!link->parent)
 567		link->root = link;
 568	else
 569		link->root = link->parent->root;
 
 570
 571	list_add(&link->sibling, &link_list);
 572	pdev->link_state = link;
 573	return link;
 574}
 575
 
 
 
 
 
 
 
 
 576/*
 577 * pcie_aspm_init_link_state: Initiate PCI express link state.
 578 * It is called after the pcie and its children devices are scaned.
 579 * @pdev: the root port or switch downstream port
 580 */
 581void pcie_aspm_init_link_state(struct pci_dev *pdev)
 582{
 583	struct pcie_link_state *link;
 584	int blacklist = !!pcie_aspm_sanity_check(pdev);
 585
 586	if (!pci_is_pcie(pdev) || pdev->link_state)
 587		return;
 588	if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
 589	    pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
 
 
 
 
 
 
 
 
 590		return;
 591
 592	/* VIA has a strange chipset, root port is under a bridge */
 593	if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
 594	    pdev->bus->self)
 595		return;
 596
 597	down_read(&pci_bus_sem);
 598	if (list_empty(&pdev->subordinate->devices))
 599		goto out;
 600
 601	mutex_lock(&aspm_lock);
 602	link = alloc_pcie_link_state(pdev);
 603	if (!link)
 604		goto unlock;
 605	/*
 606	 * Setup initial ASPM state. Note that we need to configure
 607	 * upstream links also because capable state of them can be
 608	 * update through pcie_aspm_cap_init().
 609	 */
 610	pcie_aspm_cap_init(link, blacklist);
 611
 612	/* Setup initial Clock PM state */
 613	pcie_clkpm_cap_init(link, blacklist);
 614
 615	/*
 616	 * At this stage drivers haven't had an opportunity to change the
 617	 * link policy setting. Enabling ASPM on broken hardware can cripple
 618	 * it even before the driver has had a chance to disable ASPM, so
 619	 * default to a safe level right now. If we're enabling ASPM beyond
 620	 * the BIOS's expectation, we'll do so once pci_enable_device() is
 621	 * called.
 622	 */
 623	if (aspm_policy != POLICY_POWERSAVE) {
 
 624		pcie_config_aspm_path(link);
 625		pcie_set_clkpm(link, policy_to_clkpm_state(link));
 626	}
 627
 
 
 628unlock:
 629	mutex_unlock(&aspm_lock);
 630out:
 631	up_read(&pci_bus_sem);
 632}
 633
 634/* Recheck latencies and update aspm_capable for links under the root */
 635static void pcie_update_aspm_capable(struct pcie_link_state *root)
 636{
 637	struct pcie_link_state *link;
 638	BUG_ON(root->parent);
 639	list_for_each_entry(link, &link_list, sibling) {
 640		if (link->root != root)
 641			continue;
 642		link->aspm_capable = link->aspm_support;
 643	}
 644	list_for_each_entry(link, &link_list, sibling) {
 645		struct pci_dev *child;
 646		struct pci_bus *linkbus = link->pdev->subordinate;
 647		if (link->root != root)
 648			continue;
 649		list_for_each_entry(child, &linkbus->devices, bus_list) {
 650			if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT) &&
 651			    (child->pcie_type != PCI_EXP_TYPE_LEG_END))
 652				continue;
 653			pcie_aspm_check_latency(child);
 654		}
 655	}
 656}
 657
 658/* @pdev: the endpoint device */
 659void pcie_aspm_exit_link_state(struct pci_dev *pdev)
 660{
 661	struct pci_dev *parent = pdev->bus->self;
 662	struct pcie_link_state *link, *root, *parent_link;
 663
 664	if (!pci_is_pcie(pdev) || !parent || !parent->link_state)
 665		return;
 666	if ((parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
 667	    (parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
 668		return;
 669
 670	down_read(&pci_bus_sem);
 671	mutex_lock(&aspm_lock);
 672	/*
 673	 * All PCIe functions are in one slot, remove one function will remove
 674	 * the whole slot, so just wait until we are the last function left.
 675	 */
 676	if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
 677		goto out;
 678
 679	link = parent->link_state;
 680	root = link->root;
 681	parent_link = link->parent;
 682
 683	/* All functions are removed, so just disable ASPM for the link */
 684	pcie_config_aspm_link(link, 0);
 685	list_del(&link->sibling);
 686	list_del(&link->link);
 687	/* Clock PM is for endpoint device */
 688	free_link_state(link);
 689
 690	/* Recheck latencies and configure upstream links */
 691	if (parent_link) {
 692		pcie_update_aspm_capable(root);
 693		pcie_config_aspm_path(parent_link);
 694	}
 695out:
 696	mutex_unlock(&aspm_lock);
 697	up_read(&pci_bus_sem);
 698}
 699
 700/* @pdev: the root port or switch downstream port */
 701void pcie_aspm_pm_state_change(struct pci_dev *pdev)
 702{
 703	struct pcie_link_state *link = pdev->link_state;
 704
 705	if (aspm_disabled || !pci_is_pcie(pdev) || !link)
 706		return;
 707	if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
 708	    (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
 709		return;
 710	/*
 711	 * Devices changed PM state, we should recheck if latency
 712	 * meets all functions' requirement
 713	 */
 714	down_read(&pci_bus_sem);
 715	mutex_lock(&aspm_lock);
 716	pcie_update_aspm_capable(link->root);
 717	pcie_config_aspm_path(link);
 718	mutex_unlock(&aspm_lock);
 719	up_read(&pci_bus_sem);
 720}
 721
 722void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
 723{
 724	struct pcie_link_state *link = pdev->link_state;
 725
 726	if (aspm_disabled || !pci_is_pcie(pdev) || !link)
 727		return;
 728
 729	if (aspm_policy != POLICY_POWERSAVE)
 730		return;
 731
 732	if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
 733	    (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
 734		return;
 735
 736	down_read(&pci_bus_sem);
 737	mutex_lock(&aspm_lock);
 738	pcie_config_aspm_path(link);
 739	pcie_set_clkpm(link, policy_to_clkpm_state(link));
 740	mutex_unlock(&aspm_lock);
 741	up_read(&pci_bus_sem);
 742}
 743
 744/*
 745 * pci_disable_link_state - disable pci device's link state, so the link will
 746 * never enter specific states
 747 */
 748static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem,
 749				     bool force)
 750{
 751	struct pci_dev *parent = pdev->bus->self;
 752	struct pcie_link_state *link;
 753
 754	if (aspm_disabled && !force)
 755		return;
 756
 757	if (!pci_is_pcie(pdev))
 758		return;
 759
 760	if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
 761	    pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
 762		parent = pdev;
 763	if (!parent || !parent->link_state)
 764		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 765
 766	if (sem)
 767		down_read(&pci_bus_sem);
 768	mutex_lock(&aspm_lock);
 769	link = parent->link_state;
 770	if (state & PCIE_LINK_STATE_L0S)
 771		link->aspm_disable |= ASPM_STATE_L0S;
 772	if (state & PCIE_LINK_STATE_L1)
 773		link->aspm_disable |= ASPM_STATE_L1;
 
 
 
 
 
 
 
 
 
 774	pcie_config_aspm_link(link, policy_to_aspm_state(link));
 775
 776	if (state & PCIE_LINK_STATE_CLKPM) {
 777		link->clkpm_capable = 0;
 778		pcie_set_clkpm(link, 0);
 779	}
 780	mutex_unlock(&aspm_lock);
 781	if (sem)
 782		up_read(&pci_bus_sem);
 
 
 783}
 784
 785void pci_disable_link_state_locked(struct pci_dev *pdev, int state)
 786{
 787	__pci_disable_link_state(pdev, state, false, false);
 788}
 789EXPORT_SYMBOL(pci_disable_link_state_locked);
 790
 791void pci_disable_link_state(struct pci_dev *pdev, int state)
 
 
 
 
 
 
 
 
 
 792{
 793	__pci_disable_link_state(pdev, state, true, false);
 794}
 795EXPORT_SYMBOL(pci_disable_link_state);
 796
 797void pcie_clear_aspm(struct pci_bus *bus)
 798{
 799	struct pci_dev *child;
 800
 801	/*
 802	 * Clear any ASPM setup that the firmware has carried out on this bus
 803	 */
 804	list_for_each_entry(child, &bus->devices, bus_list) {
 805		__pci_disable_link_state(child, PCIE_LINK_STATE_L0S |
 806					 PCIE_LINK_STATE_L1 |
 807					 PCIE_LINK_STATE_CLKPM,
 808					 false, true);
 809	}
 810}
 811
 812static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
 813{
 814	int i;
 815	struct pcie_link_state *link;
 816
 817	if (aspm_disabled)
 818		return -EPERM;
 819	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
 820		if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
 821			break;
 822	if (i >= ARRAY_SIZE(policy_str))
 823		return -EINVAL;
 824	if (i == aspm_policy)
 825		return 0;
 826
 827	down_read(&pci_bus_sem);
 828	mutex_lock(&aspm_lock);
 829	aspm_policy = i;
 830	list_for_each_entry(link, &link_list, sibling) {
 831		pcie_config_aspm_link(link, policy_to_aspm_state(link));
 832		pcie_set_clkpm(link, policy_to_clkpm_state(link));
 833	}
 834	mutex_unlock(&aspm_lock);
 835	up_read(&pci_bus_sem);
 836	return 0;
 837}
 838
 839static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
 840{
 841	int i, cnt = 0;
 842	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
 843		if (i == aspm_policy)
 844			cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
 845		else
 846			cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
 
 847	return cnt;
 848}
 849
 850module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
 851	NULL, 0644);
 852
 853#ifdef CONFIG_PCIEASPM_DEBUG
 854static ssize_t link_state_show(struct device *dev,
 855		struct device_attribute *attr,
 856		char *buf)
 
 
 
 
 
 
 857{
 858	struct pci_dev *pci_device = to_pci_dev(dev);
 859	struct pcie_link_state *link_state = pci_device->link_state;
 860
 861	return sprintf(buf, "%d\n", link_state->aspm_enabled);
 
 
 
 862}
 
 863
 864static ssize_t link_state_store(struct device *dev,
 865		struct device_attribute *attr,
 866		const char *buf,
 867		size_t n)
 868{
 869	struct pci_dev *pdev = to_pci_dev(dev);
 870	struct pcie_link_state *link, *root = pdev->link_state->root;
 871	u32 val = buf[0] - '0', state = 0;
 872
 873	if (aspm_disabled)
 874		return -EPERM;
 875	if (n < 1 || val > 3)
 876		return -EINVAL;
 877
 878	/* Convert requested state to ASPM state */
 879	if (val & PCIE_LINK_STATE_L0S)
 880		state |= ASPM_STATE_L0S;
 881	if (val & PCIE_LINK_STATE_L1)
 882		state |= ASPM_STATE_L1;
 
 
 
 
 
 883
 884	down_read(&pci_bus_sem);
 885	mutex_lock(&aspm_lock);
 886	list_for_each_entry(link, &link_list, sibling) {
 887		if (link->root != root)
 888			continue;
 889		pcie_config_aspm_link(link, state);
 
 
 
 
 890	}
 
 
 
 891	mutex_unlock(&aspm_lock);
 892	up_read(&pci_bus_sem);
 893	return n;
 
 894}
 895
 896static ssize_t clk_ctl_show(struct device *dev,
 897		struct device_attribute *attr,
 898		char *buf)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 899{
 900	struct pci_dev *pci_device = to_pci_dev(dev);
 901	struct pcie_link_state *link_state = pci_device->link_state;
 902
 903	return sprintf(buf, "%d\n", link_state->clkpm_enabled);
 904}
 905
 906static ssize_t clk_ctl_store(struct device *dev,
 907		struct device_attribute *attr,
 908		const char *buf,
 909		size_t n)
 910{
 911	struct pci_dev *pdev = to_pci_dev(dev);
 912	int state;
 
 913
 914	if (n < 1)
 915		return -EINVAL;
 916	state = buf[0]-'0';
 917
 918	down_read(&pci_bus_sem);
 919	mutex_lock(&aspm_lock);
 920	pcie_set_clkpm_nocheck(pdev->link_state, !!state);
 
 
 
 921	mutex_unlock(&aspm_lock);
 922	up_read(&pci_bus_sem);
 923
 924	return n;
 925}
 926
 927static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
 928static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 929
 930static char power_group[] = "power";
 931void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
 932{
 933	struct pcie_link_state *link_state = pdev->link_state;
 934
 935	if (!pci_is_pcie(pdev) ||
 936	    (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
 937	     pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
 938		return;
 939
 940	if (link_state->aspm_support)
 941		sysfs_add_file_to_group(&pdev->dev.kobj,
 942			&dev_attr_link_state.attr, power_group);
 943	if (link_state->clkpm_capable)
 944		sysfs_add_file_to_group(&pdev->dev.kobj,
 945			&dev_attr_clk_ctl.attr, power_group);
 946}
 947
 948void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
 949{
 950	struct pcie_link_state *link_state = pdev->link_state;
 951
 952	if (!pci_is_pcie(pdev) ||
 953	    (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
 954	     pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
 955		return;
 956
 957	if (link_state->aspm_support)
 958		sysfs_remove_file_from_group(&pdev->dev.kobj,
 959			&dev_attr_link_state.attr, power_group);
 960	if (link_state->clkpm_capable)
 961		sysfs_remove_file_from_group(&pdev->dev.kobj,
 962			&dev_attr_clk_ctl.attr, power_group);
 963}
 964#endif
 
 
 
 
 
 965
 966static int __init pcie_aspm_disable(char *str)
 967{
 968	if (!strcmp(str, "off")) {
 969		aspm_policy = POLICY_DEFAULT;
 970		aspm_disabled = 1;
 971		aspm_support_enabled = false;
 972		printk(KERN_INFO "PCIe ASPM is disabled\n");
 973	} else if (!strcmp(str, "force")) {
 974		aspm_force = 1;
 975		printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
 976	}
 977	return 1;
 978}
 979
 980__setup("pcie_aspm=", pcie_aspm_disable);
 981
 982void pcie_no_aspm(void)
 983{
 984	/*
 985	 * Disabling ASPM is intended to prevent the kernel from modifying
 986	 * existing hardware state, not to clear existing state. To that end:
 987	 * (a) set policy to POLICY_DEFAULT in order to avoid changing state
 988	 * (b) prevent userspace from changing policy
 989	 */
 990	if (!aspm_force) {
 991		aspm_policy = POLICY_DEFAULT;
 992		aspm_disabled = 1;
 993	}
 994}
 995
 996/**
 997 * pcie_aspm_enabled - is PCIe ASPM enabled?
 998 *
 999 * Returns true if ASPM has not been disabled by the command-line option
1000 * pcie_aspm=off.
1001 **/
1002int pcie_aspm_enabled(void)
1003{
1004       return !aspm_disabled;
1005}
1006EXPORT_SYMBOL(pcie_aspm_enabled);
1007
1008bool pcie_aspm_support_enabled(void)
1009{
1010	return aspm_support_enabled;
1011}
1012EXPORT_SYMBOL(pcie_aspm_support_enabled);