1/*
  2 * File:	drivers/pci/pcie/aspm.c
  3 * Enabling PCIe link L0s/L1 state and Clock Power Management
  4 *
  5 * Copyright (C) 2007 Intel
  6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
  7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
  8 */
  9
 10#include <linux/kernel.h>
 11#include <linux/module.h>
 12#include <linux/moduleparam.h>
 13#include <linux/pci.h>
 14#include <linux/pci_regs.h>
 15#include <linux/errno.h>
 16#include <linux/pm.h>
 17#include <linux/init.h>
 18#include <linux/slab.h>
 19#include <linux/jiffies.h>
 20#include <linux/delay.h>
 21#include <linux/pci-aspm.h>
 22#include "../pci.h"
 23
 24#ifdef MODULE_PARAM_PREFIX
 25#undef MODULE_PARAM_PREFIX
 26#endif
 27#define MODULE_PARAM_PREFIX "pcie_aspm."
 28
 29/* Note: those are not register definitions */
 30#define ASPM_STATE_L0S_UP	(1)	/* Upstream direction L0s state */
 31#define ASPM_STATE_L0S_DW	(2)	/* Downstream direction L0s state */
 32#define ASPM_STATE_L1		(4)	/* L1 state */
 33#define ASPM_STATE_L0S		(ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
 34#define ASPM_STATE_ALL		(ASPM_STATE_L0S | ASPM_STATE_L1)
 35
 36struct aspm_latency {
 37	u32 l0s;			/* L0s latency (nsec) */
 38	u32 l1;				/* L1 latency (nsec) */
 39};
 40
 41struct pcie_link_state {
 42	struct pci_dev *pdev;		/* Upstream component of the Link */
 43	struct pcie_link_state *root;	/* pointer to the root port link */
 44	struct pcie_link_state *parent;	/* pointer to the parent Link state */
 45	struct list_head sibling;	/* node in link_list */
 46	struct list_head children;	/* list of child link states */
 47	struct list_head link;		/* node in parent's children list */
 48
 49	/* ASPM state */
 50	u32 aspm_support:3;		/* Supported ASPM state */
 51	u32 aspm_enabled:3;		/* Enabled ASPM state */
 52	u32 aspm_capable:3;		/* Capable ASPM state with latency */
 53	u32 aspm_default:3;		/* Default ASPM state by BIOS */
 54	u32 aspm_disable:3;		/* Disabled ASPM state */
 55
 56	/* Clock PM state */
 57	u32 clkpm_capable:1;		/* Clock PM capable? */
 58	u32 clkpm_enabled:1;		/* Current Clock PM state */
 59	u32 clkpm_default:1;		/* Default Clock PM state by BIOS */
 60
 61	/* Exit latencies */
 62	struct aspm_latency latency_up;	/* Upstream direction exit latency */
 63	struct aspm_latency latency_dw;	/* Downstream direction exit latency */
 64	/*
 65	 * Endpoint acceptable latencies. A pcie downstream port only
 66	 * has one slot under it, so at most there are 8 functions.
 67	 */
 68	struct aspm_latency acceptable[8];
 69};
 70
 71static int aspm_disabled, aspm_force;
 72static bool aspm_support_enabled = true;
 73static DEFINE_MUTEX(aspm_lock);
 74static LIST_HEAD(link_list);
 75
 76#define POLICY_DEFAULT 0	/* BIOS default setting */
 77#define POLICY_PERFORMANCE 1	/* high performance */
 78#define POLICY_POWERSAVE 2	/* high power saving */
 79
 80#ifdef CONFIG_PCIEASPM_PERFORMANCE
 81static int aspm_policy = POLICY_PERFORMANCE;
 82#elif defined CONFIG_PCIEASPM_POWERSAVE
 83static int aspm_policy = POLICY_POWERSAVE;
 84#else
 85static int aspm_policy;
 86#endif
 87
 88static const char *policy_str[] = {
 89	[POLICY_DEFAULT] = "default",
 90	[POLICY_PERFORMANCE] = "performance",
 91	[POLICY_POWERSAVE] = "powersave"
 92};
 93
 94#define LINK_RETRAIN_TIMEOUT HZ
 95
 96static int policy_to_aspm_state(struct pcie_link_state *link)
 97{
 98	switch (aspm_policy) {
 99	case POLICY_PERFORMANCE:
100		/* Disable ASPM and Clock PM */
101		return 0;
102	case POLICY_POWERSAVE:
103		/* Enable ASPM L0s/L1 */
104		return ASPM_STATE_ALL;
105	case POLICY_DEFAULT:
106		return link->aspm_default;
107	}
108	return 0;
109}
110
111static int policy_to_clkpm_state(struct pcie_link_state *link)
112{
113	switch (aspm_policy) {
114	case POLICY_PERFORMANCE:
115		/* Disable ASPM and Clock PM */
116		return 0;
117	case POLICY_POWERSAVE:
118		/* Disable Clock PM */
119		return 1;
120	case POLICY_DEFAULT:
121		return link->clkpm_default;
122	}
123	return 0;
124}
125
126static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
127{
128	struct pci_dev *child;
129	struct pci_bus *linkbus = link->pdev->subordinate;
130
131	list_for_each_entry(child, &linkbus->devices, bus_list) {
132		if (enable)
133			pcie_capability_set_word(child, PCI_EXP_LNKCTL,
134						 PCI_EXP_LNKCTL_CLKREQ_EN);
135		else
136			pcie_capability_clear_word(child, PCI_EXP_LNKCTL,
137						   PCI_EXP_LNKCTL_CLKREQ_EN);
138	}
139	link->clkpm_enabled = !!enable;
140}
141
142static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
143{
144	/* Don't enable Clock PM if the link is not Clock PM capable */
145	if (!link->clkpm_capable && enable)
146		enable = 0;
147	/* Need nothing if the specified equals to current state */
148	if (link->clkpm_enabled == enable)
149		return;
150	pcie_set_clkpm_nocheck(link, enable);
151}
152
153static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
154{
155	int capable = 1, enabled = 1;
156	u32 reg32;
157	u16 reg16;
158	struct pci_dev *child;
159	struct pci_bus *linkbus = link->pdev->subordinate;
160
161	/* All functions should have the same cap and state, take the worst */
162	list_for_each_entry(child, &linkbus->devices, bus_list) {
163		pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &reg32);
164		if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
165			capable = 0;
166			enabled = 0;
167			break;
168		}
169		pcie_capability_read_word(child, PCI_EXP_LNKCTL, &reg16);
170		if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
171			enabled = 0;
172	}
173	link->clkpm_enabled = enabled;
174	link->clkpm_default = enabled;
175	link->clkpm_capable = (blacklist) ? 0 : capable;
176}
177
178/*
179 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
180 *   could use common clock. If they are, configure them to use the
181 *   common clock. That will reduce the ASPM state exit latency.
182 */
183static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
184{
185	int same_clock = 1;
186	u16 reg16, parent_reg, child_reg[8];
187	unsigned long start_jiffies;
188	struct pci_dev *child, *parent = link->pdev;
189	struct pci_bus *linkbus = parent->subordinate;
190	/*
191	 * All functions of a slot should have the same Slot Clock
192	 * Configuration, so just check one function
193	 */
194	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
195	BUG_ON(!pci_is_pcie(child));
196
197	/* Check downstream component if bit Slot Clock Configuration is 1 */
198	pcie_capability_read_word(child, PCI_EXP_LNKSTA, &reg16);
199	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
200		same_clock = 0;
201
202	/* Check upstream component if bit Slot Clock Configuration is 1 */
203	pcie_capability_read_word(parent, PCI_EXP_LNKSTA, &reg16);
204	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
205		same_clock = 0;
206
207	/* Configure downstream component, all functions */
208	list_for_each_entry(child, &linkbus->devices, bus_list) {
209		pcie_capability_read_word(child, PCI_EXP_LNKCTL, &reg16);
210		child_reg[PCI_FUNC(child->devfn)] = reg16;
211		if (same_clock)
212			reg16 |= PCI_EXP_LNKCTL_CCC;
213		else
214			reg16 &= ~PCI_EXP_LNKCTL_CCC;
215		pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16);
216	}
217
218	/* Configure upstream component */
219	pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &reg16);
220	parent_reg = reg16;
221	if (same_clock)
222		reg16 |= PCI_EXP_LNKCTL_CCC;
223	else
224		reg16 &= ~PCI_EXP_LNKCTL_CCC;
225	pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
226
227	/* Retrain link */
228	reg16 |= PCI_EXP_LNKCTL_RL;
229	pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
230
231	/* Wait for link training end. Break out after waiting for timeout */
232	start_jiffies = jiffies;
233	for (;;) {
234		pcie_capability_read_word(parent, PCI_EXP_LNKSTA, &reg16);
235		if (!(reg16 & PCI_EXP_LNKSTA_LT))
236			break;
237		if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
238			break;
239		msleep(1);
240	}
241	if (!(reg16 & PCI_EXP_LNKSTA_LT))
242		return;
243
244	/* Training failed. Restore common clock configurations */
245	dev_err(&parent->dev, "ASPM: Could not configure common clock\n");
246	list_for_each_entry(child, &linkbus->devices, bus_list)
247		pcie_capability_write_word(child, PCI_EXP_LNKCTL,
248					   child_reg[PCI_FUNC(child->devfn)]);
249	pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg);
250}
251
252/* Convert L0s latency encoding to ns */
253static u32 calc_l0s_latency(u32 encoding)
254{
255	if (encoding == 0x7)
256		return (5 * 1000);	/* > 4us */
257	return (64 << encoding);
258}
259
260/* Convert L0s acceptable latency encoding to ns */
261static u32 calc_l0s_acceptable(u32 encoding)
262{
263	if (encoding == 0x7)
264		return -1U;
265	return (64 << encoding);
266}
267
268/* Convert L1 latency encoding to ns */
269static u32 calc_l1_latency(u32 encoding)
270{
271	if (encoding == 0x7)
272		return (65 * 1000);	/* > 64us */
273	return (1000 << encoding);
274}
275
276/* Convert L1 acceptable latency encoding to ns */
277static u32 calc_l1_acceptable(u32 encoding)
278{
279	if (encoding == 0x7)
280		return -1U;
281	return (1000 << encoding);
282}
283
284struct aspm_register_info {
285	u32 support:2;
286	u32 enabled:2;
287	u32 latency_encoding_l0s;
288	u32 latency_encoding_l1;
289};
290
291static void pcie_get_aspm_reg(struct pci_dev *pdev,
292			      struct aspm_register_info *info)
293{
294	u16 reg16;
295	u32 reg32;
296
297	pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &reg32);
298	info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
299	info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
300	info->latency_encoding_l1  = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
301	pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &reg16);
302	info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
303}
304
305static void pcie_aspm_check_latency(struct pci_dev *endpoint)
306{
307	u32 latency, l1_switch_latency = 0;
308	struct aspm_latency *acceptable;
309	struct pcie_link_state *link;
310
311	/* Device not in D0 doesn't need latency check */
312	if ((endpoint->current_state != PCI_D0) &&
313	    (endpoint->current_state != PCI_UNKNOWN))
314		return;
315
316	link = endpoint->bus->self->link_state;
317	acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
318
319	while (link) {
320		/* Check upstream direction L0s latency */
321		if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
322		    (link->latency_up.l0s > acceptable->l0s))
323			link->aspm_capable &= ~ASPM_STATE_L0S_UP;
324
325		/* Check downstream direction L0s latency */
326		if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
327		    (link->latency_dw.l0s > acceptable->l0s))
328			link->aspm_capable &= ~ASPM_STATE_L0S_DW;
329		/*
330		 * Check L1 latency.
331		 * Every switch on the path to root complex need 1
332		 * more microsecond for L1. Spec doesn't mention L0s.
333		 */
334		latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
335		if ((link->aspm_capable & ASPM_STATE_L1) &&
336		    (latency + l1_switch_latency > acceptable->l1))
337			link->aspm_capable &= ~ASPM_STATE_L1;
338		l1_switch_latency += 1000;
339
340		link = link->parent;
341	}
342}
343
344static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
345{
346	struct pci_dev *child, *parent = link->pdev;
347	struct pci_bus *linkbus = parent->subordinate;
348	struct aspm_register_info upreg, dwreg;
349
350	if (blacklist) {
351		/* Set enabled/disable so that we will disable ASPM later */
352		link->aspm_enabled = ASPM_STATE_ALL;
353		link->aspm_disable = ASPM_STATE_ALL;
354		return;
355	}
356
357	/* Configure common clock before checking latencies */
358	pcie_aspm_configure_common_clock(link);
359
360	/* Get upstream/downstream components' register state */
361	pcie_get_aspm_reg(parent, &upreg);
362	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
363	pcie_get_aspm_reg(child, &dwreg);
364
365	/*
366	 * Setup L0s state
367	 *
368	 * Note that we must not enable L0s in either direction on a
369	 * given link unless components on both sides of the link each
370	 * support L0s.
371	 */
372	if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
373		link->aspm_support |= ASPM_STATE_L0S;
374	if (dwreg.enabled & PCIE_LINK_STATE_L0S)
375		link->aspm_enabled |= ASPM_STATE_L0S_UP;
376	if (upreg.enabled & PCIE_LINK_STATE_L0S)
377		link->aspm_enabled |= ASPM_STATE_L0S_DW;
378	link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
379	link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
380
381	/* Setup L1 state */
382	if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
383		link->aspm_support |= ASPM_STATE_L1;
384	if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
385		link->aspm_enabled |= ASPM_STATE_L1;
386	link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
387	link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
388
389	/* Save default state */
390	link->aspm_default = link->aspm_enabled;
391
392	/* Setup initial capable state. Will be updated later */
393	link->aspm_capable = link->aspm_support;
394	/*
395	 * If the downstream component has pci bridge function, don't
396	 * do ASPM for now.
397	 */
398	list_for_each_entry(child, &linkbus->devices, bus_list) {
399		if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) {
400			link->aspm_disable = ASPM_STATE_ALL;
401			break;
402		}
403	}
404
405	/* Get and check endpoint acceptable latencies */
406	list_for_each_entry(child, &linkbus->devices, bus_list) {
407		u32 reg32, encoding;
408		struct aspm_latency *acceptable =
409			&link->acceptable[PCI_FUNC(child->devfn)];
410
411		if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT &&
412		    pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)
413			continue;
414
415		pcie_capability_read_dword(child, PCI_EXP_DEVCAP, &reg32);
416		/* Calculate endpoint L0s acceptable latency */
417		encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
418		acceptable->l0s = calc_l0s_acceptable(encoding);
419		/* Calculate endpoint L1 acceptable latency */
420		encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
421		acceptable->l1 = calc_l1_acceptable(encoding);
422
423		pcie_aspm_check_latency(child);
424	}
425}
426
427static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
428{
429	pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,
430					   PCI_EXP_LNKCTL_ASPMC, val);
431}
432
433static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
434{
435	u32 upstream = 0, dwstream = 0;
436	struct pci_dev *child, *parent = link->pdev;
437	struct pci_bus *linkbus = parent->subordinate;
438
439	/* Nothing to do if the link is already in the requested state */
440	state &= (link->aspm_capable & ~link->aspm_disable);
441	if (link->aspm_enabled == state)
442		return;
443	/* Convert ASPM state to upstream/downstream ASPM register state */
444	if (state & ASPM_STATE_L0S_UP)
445		dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;
446	if (state & ASPM_STATE_L0S_DW)
447		upstream |= PCI_EXP_LNKCTL_ASPM_L0S;
448	if (state & ASPM_STATE_L1) {
449		upstream |= PCI_EXP_LNKCTL_ASPM_L1;
450		dwstream |= PCI_EXP_LNKCTL_ASPM_L1;
451	}
452	/*
453	 * Spec 2.0 suggests all functions should be configured the
454	 * same setting for ASPM. Enabling ASPM L1 should be done in
455	 * upstream component first and then downstream, and vice
456	 * versa for disabling ASPM L1. Spec doesn't mention L0S.
457	 */
458	if (state & ASPM_STATE_L1)
459		pcie_config_aspm_dev(parent, upstream);
460	list_for_each_entry(child, &linkbus->devices, bus_list)
461		pcie_config_aspm_dev(child, dwstream);
462	if (!(state & ASPM_STATE_L1))
463		pcie_config_aspm_dev(parent, upstream);
464
465	link->aspm_enabled = state;
466}
467
468static void pcie_config_aspm_path(struct pcie_link_state *link)
469{
470	while (link) {
471		pcie_config_aspm_link(link, policy_to_aspm_state(link));
472		link = link->parent;
473	}
474}
475
476static void free_link_state(struct pcie_link_state *link)
477{
478	link->pdev->link_state = NULL;
479	kfree(link);
480}
481
482static int pcie_aspm_sanity_check(struct pci_dev *pdev)
483{
484	struct pci_dev *child;
485	u32 reg32;
486
487	/*
488	 * Some functions in a slot might not all be PCIe functions,
489	 * very strange. Disable ASPM for the whole slot
490	 */
491	list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
492		if (!pci_is_pcie(child))
493			return -EINVAL;
494
495		/*
496		 * If ASPM is disabled then we're not going to change
497		 * the BIOS state. It's safe to continue even if it's a
498		 * pre-1.1 device
499		 */
500
501		if (aspm_disabled)
502			continue;
503
504		/*
505		 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
506		 * RBER bit to determine if a function is 1.1 version device
507		 */
508		pcie_capability_read_dword(child, PCI_EXP_DEVCAP, &reg32);
509		if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
510			dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device.  You can enable it with 'pcie_aspm=force'\n");
511			return -EINVAL;
512		}
513	}
514	return 0;
515}
516
517static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
518{
519	struct pcie_link_state *link;
520
521	link = kzalloc(sizeof(*link), GFP_KERNEL);
522	if (!link)
523		return NULL;
524	INIT_LIST_HEAD(&link->sibling);
525	INIT_LIST_HEAD(&link->children);
526	INIT_LIST_HEAD(&link->link);
527	link->pdev = pdev;
528	if (pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM) {
529		struct pcie_link_state *parent;
530		parent = pdev->bus->parent->self->link_state;
531		if (!parent) {
532			kfree(link);
533			return NULL;
534		}
535		link->parent = parent;
536		list_add(&link->link, &parent->children);
537	}
538	/* Setup a pointer to the root port link */
539	if (!link->parent)
540		link->root = link;
541	else
542		link->root = link->parent->root;
543
544	list_add(&link->sibling, &link_list);
545	pdev->link_state = link;
546	return link;
547}
548
549/*
550 * pcie_aspm_init_link_state: Initiate PCI express link state.
551 * It is called after the pcie and its children devices are scanned.
552 * @pdev: the root port or switch downstream port
553 */
554void pcie_aspm_init_link_state(struct pci_dev *pdev)
555{
556	struct pcie_link_state *link;
557	int blacklist = !!pcie_aspm_sanity_check(pdev);
558
559	if (!aspm_support_enabled)
560		return;
561
562	if (!pci_is_pcie(pdev) || pdev->link_state)
563		return;
564	if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT &&
565	    pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM)
566		return;
567
568	/* VIA has a strange chipset, root port is under a bridge */
569	if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT &&
570	    pdev->bus->self)
571		return;
572
573	down_read(&pci_bus_sem);
574	if (list_empty(&pdev->subordinate->devices))
575		goto out;
576
577	mutex_lock(&aspm_lock);
578	link = alloc_pcie_link_state(pdev);
579	if (!link)
580		goto unlock;
581	/*
582	 * Setup initial ASPM state. Note that we need to configure
583	 * upstream links also because capable state of them can be
584	 * update through pcie_aspm_cap_init().
585	 */
586	pcie_aspm_cap_init(link, blacklist);
587
588	/* Setup initial Clock PM state */
589	pcie_clkpm_cap_init(link, blacklist);
590
591	/*
592	 * At this stage drivers haven't had an opportunity to change the
593	 * link policy setting. Enabling ASPM on broken hardware can cripple
594	 * it even before the driver has had a chance to disable ASPM, so
595	 * default to a safe level right now. If we're enabling ASPM beyond
596	 * the BIOS's expectation, we'll do so once pci_enable_device() is
597	 * called.
598	 */
599	if (aspm_policy != POLICY_POWERSAVE) {
600		pcie_config_aspm_path(link);
601		pcie_set_clkpm(link, policy_to_clkpm_state(link));
602	}
603
604unlock:
605	mutex_unlock(&aspm_lock);
606out:
607	up_read(&pci_bus_sem);
608}
609
610/* Recheck latencies and update aspm_capable for links under the root */
611static void pcie_update_aspm_capable(struct pcie_link_state *root)
612{
613	struct pcie_link_state *link;
614	BUG_ON(root->parent);
615	list_for_each_entry(link, &link_list, sibling) {
616		if (link->root != root)
617			continue;
618		link->aspm_capable = link->aspm_support;
619	}
620	list_for_each_entry(link, &link_list, sibling) {
621		struct pci_dev *child;
622		struct pci_bus *linkbus = link->pdev->subordinate;
623		if (link->root != root)
624			continue;
625		list_for_each_entry(child, &linkbus->devices, bus_list) {
626			if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) &&
627			    (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END))
628				continue;
629			pcie_aspm_check_latency(child);
630		}
631	}
632}
633
634/* @pdev: the endpoint device */
635void pcie_aspm_exit_link_state(struct pci_dev *pdev)
636{
637	struct pci_dev *parent = pdev->bus->self;
638	struct pcie_link_state *link, *root, *parent_link;
639
640	if (!parent || !parent->link_state)
641		return;
642
643	down_read(&pci_bus_sem);
644	mutex_lock(&aspm_lock);
645	/*
646	 * All PCIe functions are in one slot, remove one function will remove
647	 * the whole slot, so just wait until we are the last function left.
648	 */
649	if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
650		goto out;
651
652	link = parent->link_state;
653	root = link->root;
654	parent_link = link->parent;
655
656	/* All functions are removed, so just disable ASPM for the link */
657	pcie_config_aspm_link(link, 0);
658	list_del(&link->sibling);
659	list_del(&link->link);
660	/* Clock PM is for endpoint device */
661	free_link_state(link);
662
663	/* Recheck latencies and configure upstream links */
664	if (parent_link) {
665		pcie_update_aspm_capable(root);
666		pcie_config_aspm_path(parent_link);
667	}
668out:
669	mutex_unlock(&aspm_lock);
670	up_read(&pci_bus_sem);
671}
672
673/* @pdev: the root port or switch downstream port */
674void pcie_aspm_pm_state_change(struct pci_dev *pdev)
675{
676	struct pcie_link_state *link = pdev->link_state;
677
678	if (aspm_disabled || !pci_is_pcie(pdev) || !link)
679		return;
680	if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) &&
681	    (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM))
682		return;
683	/*
684	 * Devices changed PM state, we should recheck if latency
685	 * meets all functions' requirement
686	 */
687	down_read(&pci_bus_sem);
688	mutex_lock(&aspm_lock);
689	pcie_update_aspm_capable(link->root);
690	pcie_config_aspm_path(link);
691	mutex_unlock(&aspm_lock);
692	up_read(&pci_bus_sem);
693}
694
695void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
696{
697	struct pcie_link_state *link = pdev->link_state;
698
699	if (aspm_disabled || !pci_is_pcie(pdev) || !link)
700		return;
701
702	if (aspm_policy != POLICY_POWERSAVE)
703		return;
704
705	if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) &&
706	    (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM))
707		return;
708
709	down_read(&pci_bus_sem);
710	mutex_lock(&aspm_lock);
711	pcie_config_aspm_path(link);
712	pcie_set_clkpm(link, policy_to_clkpm_state(link));
713	mutex_unlock(&aspm_lock);
714	up_read(&pci_bus_sem);
715}
716
717static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem,
718				     bool force)
719{
720	struct pci_dev *parent = pdev->bus->self;
721	struct pcie_link_state *link;
722
723	if (!pci_is_pcie(pdev))
724		return;
725
726	if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT ||
727	    pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM)
728		parent = pdev;
729	if (!parent || !parent->link_state)
730		return;
731
732	/*
733	 * A driver requested that ASPM be disabled on this device, but
734	 * if we don't have permission to manage ASPM (e.g., on ACPI
735	 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
736	 * the _OSC method), we can't honor that request.  Windows has
737	 * a similar mechanism using "PciASPMOptOut", which is also
738	 * ignored in this situation.
739	 */
740	if (aspm_disabled && !force) {
741		dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n");
742		return;
743	}
744
745	if (sem)
746		down_read(&pci_bus_sem);
747	mutex_lock(&aspm_lock);
748	link = parent->link_state;
749	if (state & PCIE_LINK_STATE_L0S)
750		link->aspm_disable |= ASPM_STATE_L0S;
751	if (state & PCIE_LINK_STATE_L1)
752		link->aspm_disable |= ASPM_STATE_L1;
753	pcie_config_aspm_link(link, policy_to_aspm_state(link));
754
755	if (state & PCIE_LINK_STATE_CLKPM) {
756		link->clkpm_capable = 0;
757		pcie_set_clkpm(link, 0);
758	}
759	mutex_unlock(&aspm_lock);
760	if (sem)
761		up_read(&pci_bus_sem);
762}
763
764void pci_disable_link_state_locked(struct pci_dev *pdev, int state)
765{
766	__pci_disable_link_state(pdev, state, false, false);
767}
768EXPORT_SYMBOL(pci_disable_link_state_locked);
769
770/**
771 * pci_disable_link_state - Disable device's link state, so the link will
772 * never enter specific states.  Note that if the BIOS didn't grant ASPM
773 * control to the OS, this does nothing because we can't touch the LNKCTL
774 * register.
775 *
776 * @pdev: PCI device
777 * @state: ASPM link state to disable
778 */
779void pci_disable_link_state(struct pci_dev *pdev, int state)
780{
781	__pci_disable_link_state(pdev, state, true, false);
782}
783EXPORT_SYMBOL(pci_disable_link_state);
784
785void pcie_clear_aspm(struct pci_bus *bus)
786{
787	struct pci_dev *child;
788
789	if (aspm_force)
790		return;
791
792	/*
793	 * Clear any ASPM setup that the firmware has carried out on this bus
794	 */
795	list_for_each_entry(child, &bus->devices, bus_list) {
796		__pci_disable_link_state(child, PCIE_LINK_STATE_L0S |
797					 PCIE_LINK_STATE_L1 |
798					 PCIE_LINK_STATE_CLKPM,
799					 false, true);
800	}
801}
802
803static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
804{
805	int i;
806	struct pcie_link_state *link;
807
808	if (aspm_disabled)
809		return -EPERM;
810	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
811		if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
812			break;
813	if (i >= ARRAY_SIZE(policy_str))
814		return -EINVAL;
815	if (i == aspm_policy)
816		return 0;
817
818	down_read(&pci_bus_sem);
819	mutex_lock(&aspm_lock);
820	aspm_policy = i;
821	list_for_each_entry(link, &link_list, sibling) {
822		pcie_config_aspm_link(link, policy_to_aspm_state(link));
823		pcie_set_clkpm(link, policy_to_clkpm_state(link));
824	}
825	mutex_unlock(&aspm_lock);
826	up_read(&pci_bus_sem);
827	return 0;
828}
829
830static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
831{
832	int i, cnt = 0;
833	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
834		if (i == aspm_policy)
835			cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
836		else
837			cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
838	return cnt;
839}
840
841module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
842	NULL, 0644);
843
844#ifdef CONFIG_PCIEASPM_DEBUG
845static ssize_t link_state_show(struct device *dev,
846		struct device_attribute *attr,
847		char *buf)
848{
849	struct pci_dev *pci_device = to_pci_dev(dev);
850	struct pcie_link_state *link_state = pci_device->link_state;
851
852	return sprintf(buf, "%d\n", link_state->aspm_enabled);
853}
854
855static ssize_t link_state_store(struct device *dev,
856		struct device_attribute *attr,
857		const char *buf,
858		size_t n)
859{
860	struct pci_dev *pdev = to_pci_dev(dev);
861	struct pcie_link_state *link, *root = pdev->link_state->root;
862	u32 val = buf[0] - '0', state = 0;
863
864	if (aspm_disabled)
865		return -EPERM;
866	if (n < 1 || val > 3)
867		return -EINVAL;
868
869	/* Convert requested state to ASPM state */
870	if (val & PCIE_LINK_STATE_L0S)
871		state |= ASPM_STATE_L0S;
872	if (val & PCIE_LINK_STATE_L1)
873		state |= ASPM_STATE_L1;
874
875	down_read(&pci_bus_sem);
876	mutex_lock(&aspm_lock);
877	list_for_each_entry(link, &link_list, sibling) {
878		if (link->root != root)
879			continue;
880		pcie_config_aspm_link(link, state);
881	}
882	mutex_unlock(&aspm_lock);
883	up_read(&pci_bus_sem);
884	return n;
885}
886
887static ssize_t clk_ctl_show(struct device *dev,
888		struct device_attribute *attr,
889		char *buf)
890{
891	struct pci_dev *pci_device = to_pci_dev(dev);
892	struct pcie_link_state *link_state = pci_device->link_state;
893
894	return sprintf(buf, "%d\n", link_state->clkpm_enabled);
895}
896
897static ssize_t clk_ctl_store(struct device *dev,
898		struct device_attribute *attr,
899		const char *buf,
900		size_t n)
901{
902	struct pci_dev *pdev = to_pci_dev(dev);
903	int state;
904
905	if (n < 1)
906		return -EINVAL;
907	state = buf[0]-'0';
908
909	down_read(&pci_bus_sem);
910	mutex_lock(&aspm_lock);
911	pcie_set_clkpm_nocheck(pdev->link_state, !!state);
912	mutex_unlock(&aspm_lock);
913	up_read(&pci_bus_sem);
914
915	return n;
916}
917
918static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
919static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
920
921static char power_group[] = "power";
922void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
923{
924	struct pcie_link_state *link_state = pdev->link_state;
925
926	if (!pci_is_pcie(pdev) ||
927	    (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT &&
928	     pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
929		return;
930
931	if (link_state->aspm_support)
932		sysfs_add_file_to_group(&pdev->dev.kobj,
933			&dev_attr_link_state.attr, power_group);
934	if (link_state->clkpm_capable)
935		sysfs_add_file_to_group(&pdev->dev.kobj,
936			&dev_attr_clk_ctl.attr, power_group);
937}
938
939void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
940{
941	struct pcie_link_state *link_state = pdev->link_state;
942
943	if (!pci_is_pcie(pdev) ||
944	    (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT &&
945	     pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
946		return;
947
948	if (link_state->aspm_support)
949		sysfs_remove_file_from_group(&pdev->dev.kobj,
950			&dev_attr_link_state.attr, power_group);
951	if (link_state->clkpm_capable)
952		sysfs_remove_file_from_group(&pdev->dev.kobj,
953			&dev_attr_clk_ctl.attr, power_group);
954}
955#endif
956
957static int __init pcie_aspm_disable(char *str)
958{
959	if (!strcmp(str, "off")) {
960		aspm_policy = POLICY_DEFAULT;
961		aspm_disabled = 1;
962		aspm_support_enabled = false;
963		printk(KERN_INFO "PCIe ASPM is disabled\n");
964	} else if (!strcmp(str, "force")) {
965		aspm_force = 1;
966		printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
967	}
968	return 1;
969}
970
971__setup("pcie_aspm=", pcie_aspm_disable);
972
973void pcie_no_aspm(void)
974{
975	/*
976	 * Disabling ASPM is intended to prevent the kernel from modifying
977	 * existing hardware state, not to clear existing state. To that end:
978	 * (a) set policy to POLICY_DEFAULT in order to avoid changing state
979	 * (b) prevent userspace from changing policy
980	 */
981	if (!aspm_force) {
982		aspm_policy = POLICY_DEFAULT;
983		aspm_disabled = 1;
984	}
985}
986
987bool pcie_aspm_support_enabled(void)
988{
989	return aspm_support_enabled;
990}
991EXPORT_SYMBOL(pcie_aspm_support_enabled);