Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Enable PCIe link L0s/L1 state and Clock Power Management
4 *
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
8 */
9
10#include <linux/bitfield.h>
11#include <linux/kernel.h>
12#include <linux/limits.h>
13#include <linux/math.h>
14#include <linux/module.h>
15#include <linux/moduleparam.h>
16#include <linux/pci.h>
17#include <linux/pci_regs.h>
18#include <linux/errno.h>
19#include <linux/pm.h>
20#include <linux/init.h>
21#include <linux/printk.h>
22#include <linux/slab.h>
23#include <linux/time.h>
24
25#include "../pci.h"
26
27#ifdef MODULE_PARAM_PREFIX
28#undef MODULE_PARAM_PREFIX
29#endif
30#define MODULE_PARAM_PREFIX "pcie_aspm."
31
32/* Note: those are not register definitions */
33#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
34#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
35#define ASPM_STATE_L1 (4) /* L1 state */
36#define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */
37#define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */
38#define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */
39#define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */
40#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
41#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
42#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
43 ASPM_STATE_L1_2_MASK)
44#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
45#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
46 ASPM_STATE_L1SS)
47
48struct pcie_link_state {
49 struct pci_dev *pdev; /* Upstream component of the Link */
50 struct pci_dev *downstream; /* Downstream component, function 0 */
51 struct pcie_link_state *root; /* pointer to the root port link */
52 struct pcie_link_state *parent; /* pointer to the parent Link state */
53 struct list_head sibling; /* node in link_list */
54
55 /* ASPM state */
56 u32 aspm_support:7; /* Supported ASPM state */
57 u32 aspm_enabled:7; /* Enabled ASPM state */
58 u32 aspm_capable:7; /* Capable ASPM state with latency */
59 u32 aspm_default:7; /* Default ASPM state by BIOS */
60 u32 aspm_disable:7; /* Disabled ASPM state */
61
62 /* Clock PM state */
63 u32 clkpm_capable:1; /* Clock PM capable? */
64 u32 clkpm_enabled:1; /* Current Clock PM state */
65 u32 clkpm_default:1; /* Default Clock PM state by BIOS */
66 u32 clkpm_disable:1; /* Clock PM disabled */
67};
68
69static int aspm_disabled, aspm_force;
70static bool aspm_support_enabled = true;
71static DEFINE_MUTEX(aspm_lock);
72static LIST_HEAD(link_list);
73
74#define POLICY_DEFAULT 0 /* BIOS default setting */
75#define POLICY_PERFORMANCE 1 /* high performance */
76#define POLICY_POWERSAVE 2 /* high power saving */
77#define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */
78
79#ifdef CONFIG_PCIEASPM_PERFORMANCE
80static int aspm_policy = POLICY_PERFORMANCE;
81#elif defined CONFIG_PCIEASPM_POWERSAVE
82static int aspm_policy = POLICY_POWERSAVE;
83#elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE
84static int aspm_policy = POLICY_POWER_SUPERSAVE;
85#else
86static int aspm_policy;
87#endif
88
89static const char *policy_str[] = {
90 [POLICY_DEFAULT] = "default",
91 [POLICY_PERFORMANCE] = "performance",
92 [POLICY_POWERSAVE] = "powersave",
93 [POLICY_POWER_SUPERSAVE] = "powersupersave"
94};
95
96/*
97 * The L1 PM substate capability is only implemented in function 0 in a
98 * multi function device.
99 */
100static struct pci_dev *pci_function_0(struct pci_bus *linkbus)
101{
102 struct pci_dev *child;
103
104 list_for_each_entry(child, &linkbus->devices, bus_list)
105 if (PCI_FUNC(child->devfn) == 0)
106 return child;
107 return NULL;
108}
109
110static int policy_to_aspm_state(struct pcie_link_state *link)
111{
112 switch (aspm_policy) {
113 case POLICY_PERFORMANCE:
114 /* Disable ASPM and Clock PM */
115 return 0;
116 case POLICY_POWERSAVE:
117 /* Enable ASPM L0s/L1 */
118 return (ASPM_STATE_L0S | ASPM_STATE_L1);
119 case POLICY_POWER_SUPERSAVE:
120 /* Enable Everything */
121 return ASPM_STATE_ALL;
122 case POLICY_DEFAULT:
123 return link->aspm_default;
124 }
125 return 0;
126}
127
128static int policy_to_clkpm_state(struct pcie_link_state *link)
129{
130 switch (aspm_policy) {
131 case POLICY_PERFORMANCE:
132 /* Disable ASPM and Clock PM */
133 return 0;
134 case POLICY_POWERSAVE:
135 case POLICY_POWER_SUPERSAVE:
136 /* Enable Clock PM */
137 return 1;
138 case POLICY_DEFAULT:
139 return link->clkpm_default;
140 }
141 return 0;
142}
143
144static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
145{
146 struct pci_dev *child;
147 struct pci_bus *linkbus = link->pdev->subordinate;
148 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0;
149
150 list_for_each_entry(child, &linkbus->devices, bus_list)
151 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
152 PCI_EXP_LNKCTL_CLKREQ_EN,
153 val);
154 link->clkpm_enabled = !!enable;
155}
156
157static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
158{
159 /*
160 * Don't enable Clock PM if the link is not Clock PM capable
161 * or Clock PM is disabled
162 */
163 if (!link->clkpm_capable || link->clkpm_disable)
164 enable = 0;
165 /* Need nothing if the specified equals to current state */
166 if (link->clkpm_enabled == enable)
167 return;
168 pcie_set_clkpm_nocheck(link, enable);
169}
170
171static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
172{
173 int capable = 1, enabled = 1;
174 u32 reg32;
175 u16 reg16;
176 struct pci_dev *child;
177 struct pci_bus *linkbus = link->pdev->subordinate;
178
179 /* All functions should have the same cap and state, take the worst */
180 list_for_each_entry(child, &linkbus->devices, bus_list) {
181 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32);
182 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
183 capable = 0;
184 enabled = 0;
185 break;
186 }
187 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
188 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
189 enabled = 0;
190 }
191 link->clkpm_enabled = enabled;
192 link->clkpm_default = enabled;
193 link->clkpm_capable = capable;
194 link->clkpm_disable = blacklist ? 1 : 0;
195}
196
197/*
198 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
199 * could use common clock. If they are, configure them to use the
200 * common clock. That will reduce the ASPM state exit latency.
201 */
202static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
203{
204 int same_clock = 1;
205 u16 reg16, ccc, parent_old_ccc, child_old_ccc[8];
206 struct pci_dev *child, *parent = link->pdev;
207 struct pci_bus *linkbus = parent->subordinate;
208 /*
209 * All functions of a slot should have the same Slot Clock
210 * Configuration, so just check one function
211 */
212 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
213 BUG_ON(!pci_is_pcie(child));
214
215 /* Check downstream component if bit Slot Clock Configuration is 1 */
216 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16);
217 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
218 same_clock = 0;
219
220 /* Check upstream component if bit Slot Clock Configuration is 1 */
221 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
222 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
223 same_clock = 0;
224
225 /* Port might be already in common clock mode */
226 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
227 parent_old_ccc = reg16 & PCI_EXP_LNKCTL_CCC;
228 if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) {
229 bool consistent = true;
230
231 list_for_each_entry(child, &linkbus->devices, bus_list) {
232 pcie_capability_read_word(child, PCI_EXP_LNKCTL,
233 ®16);
234 if (!(reg16 & PCI_EXP_LNKCTL_CCC)) {
235 consistent = false;
236 break;
237 }
238 }
239 if (consistent)
240 return;
241 pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n");
242 }
243
244 ccc = same_clock ? PCI_EXP_LNKCTL_CCC : 0;
245 /* Configure downstream component, all functions */
246 list_for_each_entry(child, &linkbus->devices, bus_list) {
247 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
248 child_old_ccc[PCI_FUNC(child->devfn)] = reg16 & PCI_EXP_LNKCTL_CCC;
249 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
250 PCI_EXP_LNKCTL_CCC, ccc);
251 }
252
253 /* Configure upstream component */
254 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
255 PCI_EXP_LNKCTL_CCC, ccc);
256
257 if (pcie_retrain_link(link->pdev, true)) {
258
259 /* Training failed. Restore common clock configurations */
260 pci_err(parent, "ASPM: Could not configure common clock\n");
261 list_for_each_entry(child, &linkbus->devices, bus_list)
262 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
263 PCI_EXP_LNKCTL_CCC,
264 child_old_ccc[PCI_FUNC(child->devfn)]);
265 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
266 PCI_EXP_LNKCTL_CCC, parent_old_ccc);
267 }
268}
269
270/* Convert L0s latency encoding to ns */
271static u32 calc_l0s_latency(u32 lnkcap)
272{
273 u32 encoding = FIELD_GET(PCI_EXP_LNKCAP_L0SEL, lnkcap);
274
275 if (encoding == 0x7)
276 return 5 * NSEC_PER_USEC; /* > 4us */
277 return (64 << encoding);
278}
279
280/* Convert L0s acceptable latency encoding to ns */
281static u32 calc_l0s_acceptable(u32 encoding)
282{
283 if (encoding == 0x7)
284 return U32_MAX;
285 return (64 << encoding);
286}
287
288/* Convert L1 latency encoding to ns */
289static u32 calc_l1_latency(u32 lnkcap)
290{
291 u32 encoding = FIELD_GET(PCI_EXP_LNKCAP_L1EL, lnkcap);
292
293 if (encoding == 0x7)
294 return 65 * NSEC_PER_USEC; /* > 64us */
295 return NSEC_PER_USEC << encoding;
296}
297
298/* Convert L1 acceptable latency encoding to ns */
299static u32 calc_l1_acceptable(u32 encoding)
300{
301 if (encoding == 0x7)
302 return U32_MAX;
303 return NSEC_PER_USEC << encoding;
304}
305
306/* Convert L1SS T_pwr encoding to usec */
307static u32 calc_l12_pwron(struct pci_dev *pdev, u32 scale, u32 val)
308{
309 switch (scale) {
310 case 0:
311 return val * 2;
312 case 1:
313 return val * 10;
314 case 2:
315 return val * 100;
316 }
317 pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale);
318 return 0;
319}
320
321/*
322 * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1
323 * register. Ports enter L1.2 when the most recent LTR value is greater
324 * than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we
325 * don't enter L1.2 too aggressively.
326 *
327 * See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3.
328 */
329static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)
330{
331 u64 threshold_ns = (u64)threshold_us * NSEC_PER_USEC;
332
333 /*
334 * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max
335 * value of 0x3ff.
336 */
337 if (threshold_ns <= 1 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
338 *scale = 0; /* Value times 1ns */
339 *value = threshold_ns;
340 } else if (threshold_ns <= 32 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
341 *scale = 1; /* Value times 32ns */
342 *value = roundup(threshold_ns, 32) / 32;
343 } else if (threshold_ns <= 1024 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
344 *scale = 2; /* Value times 1024ns */
345 *value = roundup(threshold_ns, 1024) / 1024;
346 } else if (threshold_ns <= 32768 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
347 *scale = 3; /* Value times 32768ns */
348 *value = roundup(threshold_ns, 32768) / 32768;
349 } else if (threshold_ns <= 1048576 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
350 *scale = 4; /* Value times 1048576ns */
351 *value = roundup(threshold_ns, 1048576) / 1048576;
352 } else if (threshold_ns <= (u64)33554432 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
353 *scale = 5; /* Value times 33554432ns */
354 *value = roundup(threshold_ns, 33554432) / 33554432;
355 } else {
356 *scale = 5;
357 *value = FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE);
358 }
359}
360
361static void pcie_aspm_check_latency(struct pci_dev *endpoint)
362{
363 u32 latency, encoding, lnkcap_up, lnkcap_dw;
364 u32 l1_switch_latency = 0, latency_up_l0s;
365 u32 latency_up_l1, latency_dw_l0s, latency_dw_l1;
366 u32 acceptable_l0s, acceptable_l1;
367 struct pcie_link_state *link;
368
369 /* Device not in D0 doesn't need latency check */
370 if ((endpoint->current_state != PCI_D0) &&
371 (endpoint->current_state != PCI_UNKNOWN))
372 return;
373
374 link = endpoint->bus->self->link_state;
375
376 /* Calculate endpoint L0s acceptable latency */
377 encoding = FIELD_GET(PCI_EXP_DEVCAP_L0S, endpoint->devcap);
378 acceptable_l0s = calc_l0s_acceptable(encoding);
379
380 /* Calculate endpoint L1 acceptable latency */
381 encoding = FIELD_GET(PCI_EXP_DEVCAP_L1, endpoint->devcap);
382 acceptable_l1 = calc_l1_acceptable(encoding);
383
384 while (link) {
385 struct pci_dev *dev = pci_function_0(link->pdev->subordinate);
386
387 /* Read direction exit latencies */
388 pcie_capability_read_dword(link->pdev, PCI_EXP_LNKCAP,
389 &lnkcap_up);
390 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP,
391 &lnkcap_dw);
392 latency_up_l0s = calc_l0s_latency(lnkcap_up);
393 latency_up_l1 = calc_l1_latency(lnkcap_up);
394 latency_dw_l0s = calc_l0s_latency(lnkcap_dw);
395 latency_dw_l1 = calc_l1_latency(lnkcap_dw);
396
397 /* Check upstream direction L0s latency */
398 if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
399 (latency_up_l0s > acceptable_l0s))
400 link->aspm_capable &= ~ASPM_STATE_L0S_UP;
401
402 /* Check downstream direction L0s latency */
403 if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
404 (latency_dw_l0s > acceptable_l0s))
405 link->aspm_capable &= ~ASPM_STATE_L0S_DW;
406 /*
407 * Check L1 latency.
408 * Every switch on the path to root complex need 1
409 * more microsecond for L1. Spec doesn't mention L0s.
410 *
411 * The exit latencies for L1 substates are not advertised
412 * by a device. Since the spec also doesn't mention a way
413 * to determine max latencies introduced by enabling L1
414 * substates on the components, it is not clear how to do
415 * a L1 substate exit latency check. We assume that the
416 * L1 exit latencies advertised by a device include L1
417 * substate latencies (and hence do not do any check).
418 */
419 latency = max_t(u32, latency_up_l1, latency_dw_l1);
420 if ((link->aspm_capable & ASPM_STATE_L1) &&
421 (latency + l1_switch_latency > acceptable_l1))
422 link->aspm_capable &= ~ASPM_STATE_L1;
423 l1_switch_latency += NSEC_PER_USEC;
424
425 link = link->parent;
426 }
427}
428
429/* Calculate L1.2 PM substate timing parameters */
430static void aspm_calc_l12_info(struct pcie_link_state *link,
431 u32 parent_l1ss_cap, u32 child_l1ss_cap)
432{
433 struct pci_dev *child = link->downstream, *parent = link->pdev;
434 u32 val1, val2, scale1, scale2;
435 u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
436 u32 ctl1 = 0, ctl2 = 0;
437 u32 pctl1, pctl2, cctl1, cctl2;
438 u32 pl1_2_enables, cl1_2_enables;
439
440 /* Choose the greater of the two Port Common_Mode_Restore_Times */
441 val1 = FIELD_GET(PCI_L1SS_CAP_CM_RESTORE_TIME, parent_l1ss_cap);
442 val2 = FIELD_GET(PCI_L1SS_CAP_CM_RESTORE_TIME, child_l1ss_cap);
443 t_common_mode = max(val1, val2);
444
445 /* Choose the greater of the two Port T_POWER_ON times */
446 val1 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_VALUE, parent_l1ss_cap);
447 scale1 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_SCALE, parent_l1ss_cap);
448 val2 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_VALUE, child_l1ss_cap);
449 scale2 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_SCALE, child_l1ss_cap);
450
451 if (calc_l12_pwron(parent, scale1, val1) >
452 calc_l12_pwron(child, scale2, val2)) {
453 ctl2 |= FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_SCALE, scale1) |
454 FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_VALUE, val1);
455 t_power_on = calc_l12_pwron(parent, scale1, val1);
456 } else {
457 ctl2 |= FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_SCALE, scale2) |
458 FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_VALUE, val2);
459 t_power_on = calc_l12_pwron(child, scale2, val2);
460 }
461
462 /*
463 * Set LTR_L1.2_THRESHOLD to the time required to transition the
464 * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if
465 * downstream devices report (via LTR) that they can tolerate at
466 * least that much latency.
467 *
468 * Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and
469 * Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at
470 * least 4us.
471 */
472 l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;
473 encode_l12_threshold(l1_2_threshold, &scale, &value);
474 ctl1 |= FIELD_PREP(PCI_L1SS_CTL1_CM_RESTORE_TIME, t_common_mode) |
475 FIELD_PREP(PCI_L1SS_CTL1_LTR_L12_TH_VALUE, value) |
476 FIELD_PREP(PCI_L1SS_CTL1_LTR_L12_TH_SCALE, scale);
477
478 /* Some broken devices only support dword access to L1 SS */
479 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
480 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
481 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
482 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2);
483
484 if (ctl1 == pctl1 && ctl1 == cctl1 &&
485 ctl2 == pctl2 && ctl2 == cctl2)
486 return;
487
488 /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
489 pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
490 cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
491
492 if (pl1_2_enables || cl1_2_enables) {
493 pci_clear_and_set_config_dword(child,
494 child->l1ss + PCI_L1SS_CTL1,
495 PCI_L1SS_CTL1_L1_2_MASK, 0);
496 pci_clear_and_set_config_dword(parent,
497 parent->l1ss + PCI_L1SS_CTL1,
498 PCI_L1SS_CTL1_L1_2_MASK, 0);
499 }
500
501 /* Program T_POWER_ON times in both ports */
502 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
503 pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
504
505 /* Program Common_Mode_Restore_Time in upstream device */
506 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
507 PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
508
509 /* Program LTR_L1.2_THRESHOLD time in both ports */
510 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
511 PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
512 PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
513 ctl1);
514 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
515 PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
516 PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
517 ctl1);
518
519 if (pl1_2_enables || cl1_2_enables) {
520 pci_clear_and_set_config_dword(parent,
521 parent->l1ss + PCI_L1SS_CTL1, 0,
522 pl1_2_enables);
523 pci_clear_and_set_config_dword(child,
524 child->l1ss + PCI_L1SS_CTL1, 0,
525 cl1_2_enables);
526 }
527}
528
529static void aspm_l1ss_init(struct pcie_link_state *link)
530{
531 struct pci_dev *child = link->downstream, *parent = link->pdev;
532 u32 parent_l1ss_cap, child_l1ss_cap;
533 u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
534
535 if (!parent->l1ss || !child->l1ss)
536 return;
537
538 /* Setup L1 substate */
539 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
540 &parent_l1ss_cap);
541 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
542 &child_l1ss_cap);
543
544 if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
545 parent_l1ss_cap = 0;
546 if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
547 child_l1ss_cap = 0;
548
549 /*
550 * If we don't have LTR for the entire path from the Root Complex
551 * to this device, we can't use ASPM L1.2 because it relies on the
552 * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
553 */
554 if (!child->ltr_path)
555 child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
556
557 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
558 link->aspm_support |= ASPM_STATE_L1_1;
559 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
560 link->aspm_support |= ASPM_STATE_L1_2;
561 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
562 link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
563 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
564 link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
565
566 if (parent_l1ss_cap)
567 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
568 &parent_l1ss_ctl1);
569 if (child_l1ss_cap)
570 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
571 &child_l1ss_ctl1);
572
573 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
574 link->aspm_enabled |= ASPM_STATE_L1_1;
575 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
576 link->aspm_enabled |= ASPM_STATE_L1_2;
577 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
578 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
579 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
580 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
581
582 if (link->aspm_support & ASPM_STATE_L1_2_MASK)
583 aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap);
584}
585
586static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
587{
588 struct pci_dev *child = link->downstream, *parent = link->pdev;
589 u32 parent_lnkcap, child_lnkcap;
590 u16 parent_lnkctl, child_lnkctl;
591 struct pci_bus *linkbus = parent->subordinate;
592
593 if (blacklist) {
594 /* Set enabled/disable so that we will disable ASPM later */
595 link->aspm_enabled = ASPM_STATE_ALL;
596 link->aspm_disable = ASPM_STATE_ALL;
597 return;
598 }
599
600 /*
601 * If ASPM not supported, don't mess with the clocks and link,
602 * bail out now.
603 */
604 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
605 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
606 if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS))
607 return;
608
609 /* Configure common clock before checking latencies */
610 pcie_aspm_configure_common_clock(link);
611
612 /*
613 * Re-read upstream/downstream components' register state after
614 * clock configuration. L0s & L1 exit latencies in the otherwise
615 * read-only Link Capabilities may change depending on common clock
616 * configuration (PCIe r5.0, sec 7.5.3.6).
617 */
618 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
619 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
620 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl);
621 pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl);
622
623 /*
624 * Setup L0s state
625 *
626 * Note that we must not enable L0s in either direction on a
627 * given link unless components on both sides of the link each
628 * support L0s.
629 */
630 if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S)
631 link->aspm_support |= ASPM_STATE_L0S;
632
633 if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
634 link->aspm_enabled |= ASPM_STATE_L0S_UP;
635 if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
636 link->aspm_enabled |= ASPM_STATE_L0S_DW;
637
638 /* Setup L1 state */
639 if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1)
640 link->aspm_support |= ASPM_STATE_L1;
641
642 if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)
643 link->aspm_enabled |= ASPM_STATE_L1;
644
645 aspm_l1ss_init(link);
646
647 /* Save default state */
648 link->aspm_default = link->aspm_enabled;
649
650 /* Setup initial capable state. Will be updated later */
651 link->aspm_capable = link->aspm_support;
652
653 /* Get and check endpoint acceptable latencies */
654 list_for_each_entry(child, &linkbus->devices, bus_list) {
655 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT &&
656 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)
657 continue;
658
659 pcie_aspm_check_latency(child);
660 }
661}
662
663/* Configure the ASPM L1 substates */
664static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
665{
666 u32 val, enable_req;
667 struct pci_dev *child = link->downstream, *parent = link->pdev;
668
669 enable_req = (link->aspm_enabled ^ state) & state;
670
671 /*
672 * Here are the rules specified in the PCIe spec for enabling L1SS:
673 * - When enabling L1.x, enable bit at parent first, then at child
674 * - When disabling L1.x, disable bit at child first, then at parent
675 * - When enabling ASPM L1.x, need to disable L1
676 * (at child followed by parent).
677 * - The ASPM/PCIPM L1.2 must be disabled while programming timing
678 * parameters
679 *
680 * To keep it simple, disable all L1SS bits first, and later enable
681 * what is needed.
682 */
683
684 /* Disable all L1 substates */
685 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
686 PCI_L1SS_CTL1_L1SS_MASK, 0);
687 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
688 PCI_L1SS_CTL1_L1SS_MASK, 0);
689 /*
690 * If needed, disable L1, and it gets enabled later
691 * in pcie_config_aspm_link().
692 */
693 if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) {
694 pcie_capability_clear_word(child, PCI_EXP_LNKCTL,
695 PCI_EXP_LNKCTL_ASPM_L1);
696 pcie_capability_clear_word(parent, PCI_EXP_LNKCTL,
697 PCI_EXP_LNKCTL_ASPM_L1);
698 }
699
700 val = 0;
701 if (state & ASPM_STATE_L1_1)
702 val |= PCI_L1SS_CTL1_ASPM_L1_1;
703 if (state & ASPM_STATE_L1_2)
704 val |= PCI_L1SS_CTL1_ASPM_L1_2;
705 if (state & ASPM_STATE_L1_1_PCIPM)
706 val |= PCI_L1SS_CTL1_PCIPM_L1_1;
707 if (state & ASPM_STATE_L1_2_PCIPM)
708 val |= PCI_L1SS_CTL1_PCIPM_L1_2;
709
710 /* Enable what we need to enable */
711 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
712 PCI_L1SS_CTL1_L1SS_MASK, val);
713 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
714 PCI_L1SS_CTL1_L1SS_MASK, val);
715}
716
717static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
718{
719 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,
720 PCI_EXP_LNKCTL_ASPMC, val);
721}
722
723static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
724{
725 u32 upstream = 0, dwstream = 0;
726 struct pci_dev *child = link->downstream, *parent = link->pdev;
727 struct pci_bus *linkbus = parent->subordinate;
728
729 /* Enable only the states that were not explicitly disabled */
730 state &= (link->aspm_capable & ~link->aspm_disable);
731
732 /* Can't enable any substates if L1 is not enabled */
733 if (!(state & ASPM_STATE_L1))
734 state &= ~ASPM_STATE_L1SS;
735
736 /* Spec says both ports must be in D0 before enabling PCI PM substates*/
737 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) {
738 state &= ~ASPM_STATE_L1_SS_PCIPM;
739 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);
740 }
741
742 /* Nothing to do if the link is already in the requested state */
743 if (link->aspm_enabled == state)
744 return;
745 /* Convert ASPM state to upstream/downstream ASPM register state */
746 if (state & ASPM_STATE_L0S_UP)
747 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;
748 if (state & ASPM_STATE_L0S_DW)
749 upstream |= PCI_EXP_LNKCTL_ASPM_L0S;
750 if (state & ASPM_STATE_L1) {
751 upstream |= PCI_EXP_LNKCTL_ASPM_L1;
752 dwstream |= PCI_EXP_LNKCTL_ASPM_L1;
753 }
754
755 if (link->aspm_capable & ASPM_STATE_L1SS)
756 pcie_config_aspm_l1ss(link, state);
757
758 /*
759 * Spec 2.0 suggests all functions should be configured the
760 * same setting for ASPM. Enabling ASPM L1 should be done in
761 * upstream component first and then downstream, and vice
762 * versa for disabling ASPM L1. Spec doesn't mention L0S.
763 */
764 if (state & ASPM_STATE_L1)
765 pcie_config_aspm_dev(parent, upstream);
766 list_for_each_entry(child, &linkbus->devices, bus_list)
767 pcie_config_aspm_dev(child, dwstream);
768 if (!(state & ASPM_STATE_L1))
769 pcie_config_aspm_dev(parent, upstream);
770
771 link->aspm_enabled = state;
772}
773
774static void pcie_config_aspm_path(struct pcie_link_state *link)
775{
776 while (link) {
777 pcie_config_aspm_link(link, policy_to_aspm_state(link));
778 link = link->parent;
779 }
780}
781
782static void free_link_state(struct pcie_link_state *link)
783{
784 link->pdev->link_state = NULL;
785 kfree(link);
786}
787
788static int pcie_aspm_sanity_check(struct pci_dev *pdev)
789{
790 struct pci_dev *child;
791 u32 reg32;
792
793 /*
794 * Some functions in a slot might not all be PCIe functions,
795 * very strange. Disable ASPM for the whole slot
796 */
797 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
798 if (!pci_is_pcie(child))
799 return -EINVAL;
800
801 /*
802 * If ASPM is disabled then we're not going to change
803 * the BIOS state. It's safe to continue even if it's a
804 * pre-1.1 device
805 */
806
807 if (aspm_disabled)
808 continue;
809
810 /*
811 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
812 * RBER bit to determine if a function is 1.1 version device
813 */
814 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
815 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
816 pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");
817 return -EINVAL;
818 }
819 }
820 return 0;
821}
822
823static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
824{
825 struct pcie_link_state *link;
826
827 link = kzalloc(sizeof(*link), GFP_KERNEL);
828 if (!link)
829 return NULL;
830
831 INIT_LIST_HEAD(&link->sibling);
832 link->pdev = pdev;
833 link->downstream = pci_function_0(pdev->subordinate);
834
835 /*
836 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe
837 * hierarchies. Note that some PCIe host implementations omit
838 * the root ports entirely, in which case a downstream port on
839 * a switch may become the root of the link state chain for all
840 * its subordinate endpoints.
841 */
842 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT ||
843 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE ||
844 !pdev->bus->parent->self) {
845 link->root = link;
846 } else {
847 struct pcie_link_state *parent;
848
849 parent = pdev->bus->parent->self->link_state;
850 if (!parent) {
851 kfree(link);
852 return NULL;
853 }
854
855 link->parent = parent;
856 link->root = link->parent->root;
857 }
858
859 list_add(&link->sibling, &link_list);
860 pdev->link_state = link;
861 return link;
862}
863
864static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev)
865{
866 struct pci_dev *child;
867
868 list_for_each_entry(child, &pdev->subordinate->devices, bus_list)
869 sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group);
870}
871
872/*
873 * pcie_aspm_init_link_state: Initiate PCI express link state.
874 * It is called after the pcie and its children devices are scanned.
875 * @pdev: the root port or switch downstream port
876 */
877void pcie_aspm_init_link_state(struct pci_dev *pdev)
878{
879 struct pcie_link_state *link;
880 int blacklist = !!pcie_aspm_sanity_check(pdev);
881
882 if (!aspm_support_enabled)
883 return;
884
885 if (pdev->link_state)
886 return;
887
888 /*
889 * We allocate pcie_link_state for the component on the upstream
890 * end of a Link, so there's nothing to do unless this device is
891 * downstream port.
892 */
893 if (!pcie_downstream_port(pdev))
894 return;
895
896 /* VIA has a strange chipset, root port is under a bridge */
897 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT &&
898 pdev->bus->self)
899 return;
900
901 down_read(&pci_bus_sem);
902 if (list_empty(&pdev->subordinate->devices))
903 goto out;
904
905 mutex_lock(&aspm_lock);
906 link = alloc_pcie_link_state(pdev);
907 if (!link)
908 goto unlock;
909 /*
910 * Setup initial ASPM state. Note that we need to configure
911 * upstream links also because capable state of them can be
912 * update through pcie_aspm_cap_init().
913 */
914 pcie_aspm_cap_init(link, blacklist);
915
916 /* Setup initial Clock PM state */
917 pcie_clkpm_cap_init(link, blacklist);
918
919 /*
920 * At this stage drivers haven't had an opportunity to change the
921 * link policy setting. Enabling ASPM on broken hardware can cripple
922 * it even before the driver has had a chance to disable ASPM, so
923 * default to a safe level right now. If we're enabling ASPM beyond
924 * the BIOS's expectation, we'll do so once pci_enable_device() is
925 * called.
926 */
927 if (aspm_policy != POLICY_POWERSAVE &&
928 aspm_policy != POLICY_POWER_SUPERSAVE) {
929 pcie_config_aspm_path(link);
930 pcie_set_clkpm(link, policy_to_clkpm_state(link));
931 }
932
933 pcie_aspm_update_sysfs_visibility(pdev);
934
935unlock:
936 mutex_unlock(&aspm_lock);
937out:
938 up_read(&pci_bus_sem);
939}
940
941/* Recheck latencies and update aspm_capable for links under the root */
942static void pcie_update_aspm_capable(struct pcie_link_state *root)
943{
944 struct pcie_link_state *link;
945 BUG_ON(root->parent);
946 list_for_each_entry(link, &link_list, sibling) {
947 if (link->root != root)
948 continue;
949 link->aspm_capable = link->aspm_support;
950 }
951 list_for_each_entry(link, &link_list, sibling) {
952 struct pci_dev *child;
953 struct pci_bus *linkbus = link->pdev->subordinate;
954 if (link->root != root)
955 continue;
956 list_for_each_entry(child, &linkbus->devices, bus_list) {
957 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) &&
958 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END))
959 continue;
960 pcie_aspm_check_latency(child);
961 }
962 }
963}
964
965/* @pdev: the endpoint device */
966void pcie_aspm_exit_link_state(struct pci_dev *pdev)
967{
968 struct pci_dev *parent = pdev->bus->self;
969 struct pcie_link_state *link, *root, *parent_link;
970
971 if (!parent || !parent->link_state)
972 return;
973
974 down_read(&pci_bus_sem);
975 mutex_lock(&aspm_lock);
976
977 link = parent->link_state;
978 root = link->root;
979 parent_link = link->parent;
980
981 /*
982 * link->downstream is a pointer to the pci_dev of function 0. If
983 * we remove that function, the pci_dev is about to be deallocated,
984 * so we can't use link->downstream again. Free the link state to
985 * avoid this.
986 *
987 * If we're removing a non-0 function, it's possible we could
988 * retain the link state, but PCIe r6.0, sec 7.5.3.7, recommends
989 * programming the same ASPM Control value for all functions of
990 * multi-function devices, so disable ASPM for all of them.
991 */
992 pcie_config_aspm_link(link, 0);
993 list_del(&link->sibling);
994 free_link_state(link);
995
996 /* Recheck latencies and configure upstream links */
997 if (parent_link) {
998 pcie_update_aspm_capable(root);
999 pcie_config_aspm_path(parent_link);
1000 }
1001
1002 mutex_unlock(&aspm_lock);
1003 up_read(&pci_bus_sem);
1004}
1005
1006/*
1007 * @pdev: the root port or switch downstream port
1008 * @locked: whether pci_bus_sem is held
1009 */
1010void pcie_aspm_pm_state_change(struct pci_dev *pdev, bool locked)
1011{
1012 struct pcie_link_state *link = pdev->link_state;
1013
1014 if (aspm_disabled || !link)
1015 return;
1016 /*
1017 * Devices changed PM state, we should recheck if latency
1018 * meets all functions' requirement
1019 */
1020 if (!locked)
1021 down_read(&pci_bus_sem);
1022 mutex_lock(&aspm_lock);
1023 pcie_update_aspm_capable(link->root);
1024 pcie_config_aspm_path(link);
1025 mutex_unlock(&aspm_lock);
1026 if (!locked)
1027 up_read(&pci_bus_sem);
1028}
1029
1030void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
1031{
1032 struct pcie_link_state *link = pdev->link_state;
1033
1034 if (aspm_disabled || !link)
1035 return;
1036
1037 if (aspm_policy != POLICY_POWERSAVE &&
1038 aspm_policy != POLICY_POWER_SUPERSAVE)
1039 return;
1040
1041 down_read(&pci_bus_sem);
1042 mutex_lock(&aspm_lock);
1043 pcie_config_aspm_path(link);
1044 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1045 mutex_unlock(&aspm_lock);
1046 up_read(&pci_bus_sem);
1047}
1048
1049static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
1050{
1051 struct pci_dev *bridge;
1052
1053 if (!pci_is_pcie(pdev))
1054 return NULL;
1055
1056 bridge = pci_upstream_bridge(pdev);
1057 if (!bridge || !pci_is_pcie(bridge))
1058 return NULL;
1059
1060 return bridge->link_state;
1061}
1062
1063static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked)
1064{
1065 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1066
1067 if (!link)
1068 return -EINVAL;
1069 /*
1070 * A driver requested that ASPM be disabled on this device, but
1071 * if we don't have permission to manage ASPM (e.g., on ACPI
1072 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
1073 * the _OSC method), we can't honor that request. Windows has
1074 * a similar mechanism using "PciASPMOptOut", which is also
1075 * ignored in this situation.
1076 */
1077 if (aspm_disabled) {
1078 pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
1079 return -EPERM;
1080 }
1081
1082 if (!locked)
1083 down_read(&pci_bus_sem);
1084 mutex_lock(&aspm_lock);
1085 if (state & PCIE_LINK_STATE_L0S)
1086 link->aspm_disable |= ASPM_STATE_L0S;
1087 if (state & PCIE_LINK_STATE_L1)
1088 /* L1 PM substates require L1 */
1089 link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;
1090 if (state & PCIE_LINK_STATE_L1_1)
1091 link->aspm_disable |= ASPM_STATE_L1_1;
1092 if (state & PCIE_LINK_STATE_L1_2)
1093 link->aspm_disable |= ASPM_STATE_L1_2;
1094 if (state & PCIE_LINK_STATE_L1_1_PCIPM)
1095 link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;
1096 if (state & PCIE_LINK_STATE_L1_2_PCIPM)
1097 link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;
1098 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1099
1100 if (state & PCIE_LINK_STATE_CLKPM)
1101 link->clkpm_disable = 1;
1102 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1103 mutex_unlock(&aspm_lock);
1104 if (!locked)
1105 up_read(&pci_bus_sem);
1106
1107 return 0;
1108}
1109
1110int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
1111{
1112 lockdep_assert_held_read(&pci_bus_sem);
1113
1114 return __pci_disable_link_state(pdev, state, true);
1115}
1116EXPORT_SYMBOL(pci_disable_link_state_locked);
1117
1118/**
1119 * pci_disable_link_state - Disable device's link state, so the link will
1120 * never enter specific states. Note that if the BIOS didn't grant ASPM
1121 * control to the OS, this does nothing because we can't touch the LNKCTL
1122 * register. Returns 0 or a negative errno.
1123 *
1124 * @pdev: PCI device
1125 * @state: ASPM link state to disable
1126 */
1127int pci_disable_link_state(struct pci_dev *pdev, int state)
1128{
1129 return __pci_disable_link_state(pdev, state, false);
1130}
1131EXPORT_SYMBOL(pci_disable_link_state);
1132
1133static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked)
1134{
1135 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1136
1137 if (!link)
1138 return -EINVAL;
1139 /*
1140 * A driver requested that ASPM be enabled on this device, but
1141 * if we don't have permission to manage ASPM (e.g., on ACPI
1142 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
1143 * the _OSC method), we can't honor that request.
1144 */
1145 if (aspm_disabled) {
1146 pci_warn(pdev, "can't override BIOS ASPM; OS doesn't have ASPM control\n");
1147 return -EPERM;
1148 }
1149
1150 if (!locked)
1151 down_read(&pci_bus_sem);
1152 mutex_lock(&aspm_lock);
1153 link->aspm_default = 0;
1154 if (state & PCIE_LINK_STATE_L0S)
1155 link->aspm_default |= ASPM_STATE_L0S;
1156 if (state & PCIE_LINK_STATE_L1)
1157 link->aspm_default |= ASPM_STATE_L1;
1158 /* L1 PM substates require L1 */
1159 if (state & PCIE_LINK_STATE_L1_1)
1160 link->aspm_default |= ASPM_STATE_L1_1 | ASPM_STATE_L1;
1161 if (state & PCIE_LINK_STATE_L1_2)
1162 link->aspm_default |= ASPM_STATE_L1_2 | ASPM_STATE_L1;
1163 if (state & PCIE_LINK_STATE_L1_1_PCIPM)
1164 link->aspm_default |= ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1;
1165 if (state & PCIE_LINK_STATE_L1_2_PCIPM)
1166 link->aspm_default |= ASPM_STATE_L1_2_PCIPM | ASPM_STATE_L1;
1167 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1168
1169 link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0;
1170 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1171 mutex_unlock(&aspm_lock);
1172 if (!locked)
1173 up_read(&pci_bus_sem);
1174
1175 return 0;
1176}
1177
1178/**
1179 * pci_enable_link_state - Clear and set the default device link state so that
1180 * the link may be allowed to enter the specified states. Note that if the
1181 * BIOS didn't grant ASPM control to the OS, this does nothing because we can't
1182 * touch the LNKCTL register. Also note that this does not enable states
1183 * disabled by pci_disable_link_state(). Return 0 or a negative errno.
1184 *
1185 * @pdev: PCI device
1186 * @state: Mask of ASPM link states to enable
1187 */
1188int pci_enable_link_state(struct pci_dev *pdev, int state)
1189{
1190 return __pci_enable_link_state(pdev, state, false);
1191}
1192EXPORT_SYMBOL(pci_enable_link_state);
1193
1194/**
1195 * pci_enable_link_state_locked - Clear and set the default device link state
1196 * so that the link may be allowed to enter the specified states. Note that if
1197 * the BIOS didn't grant ASPM control to the OS, this does nothing because we
1198 * can't touch the LNKCTL register. Also note that this does not enable states
1199 * disabled by pci_disable_link_state(). Return 0 or a negative errno.
1200 *
1201 * @pdev: PCI device
1202 * @state: Mask of ASPM link states to enable
1203 *
1204 * Context: Caller holds pci_bus_sem read lock.
1205 */
1206int pci_enable_link_state_locked(struct pci_dev *pdev, int state)
1207{
1208 lockdep_assert_held_read(&pci_bus_sem);
1209
1210 return __pci_enable_link_state(pdev, state, true);
1211}
1212EXPORT_SYMBOL(pci_enable_link_state_locked);
1213
1214static int pcie_aspm_set_policy(const char *val,
1215 const struct kernel_param *kp)
1216{
1217 int i;
1218 struct pcie_link_state *link;
1219
1220 if (aspm_disabled)
1221 return -EPERM;
1222 i = sysfs_match_string(policy_str, val);
1223 if (i < 0)
1224 return i;
1225 if (i == aspm_policy)
1226 return 0;
1227
1228 down_read(&pci_bus_sem);
1229 mutex_lock(&aspm_lock);
1230 aspm_policy = i;
1231 list_for_each_entry(link, &link_list, sibling) {
1232 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1233 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1234 }
1235 mutex_unlock(&aspm_lock);
1236 up_read(&pci_bus_sem);
1237 return 0;
1238}
1239
1240static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)
1241{
1242 int i, cnt = 0;
1243 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
1244 if (i == aspm_policy)
1245 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
1246 else
1247 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
1248 cnt += sprintf(buffer + cnt, "\n");
1249 return cnt;
1250}
1251
1252module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
1253 NULL, 0644);
1254
1255/**
1256 * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
1257 * @pdev: Target device.
1258 *
1259 * Relies on the upstream bridge's link_state being valid. The link_state
1260 * is deallocated only when the last child of the bridge (i.e., @pdev or a
1261 * sibling) is removed, and the caller should be holding a reference to
1262 * @pdev, so this should be safe.
1263 */
1264bool pcie_aspm_enabled(struct pci_dev *pdev)
1265{
1266 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1267
1268 if (!link)
1269 return false;
1270
1271 return link->aspm_enabled;
1272}
1273EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
1274
1275static ssize_t aspm_attr_show_common(struct device *dev,
1276 struct device_attribute *attr,
1277 char *buf, u8 state)
1278{
1279 struct pci_dev *pdev = to_pci_dev(dev);
1280 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1281
1282 return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);
1283}
1284
1285static ssize_t aspm_attr_store_common(struct device *dev,
1286 struct device_attribute *attr,
1287 const char *buf, size_t len, u8 state)
1288{
1289 struct pci_dev *pdev = to_pci_dev(dev);
1290 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1291 bool state_enable;
1292
1293 if (kstrtobool(buf, &state_enable) < 0)
1294 return -EINVAL;
1295
1296 down_read(&pci_bus_sem);
1297 mutex_lock(&aspm_lock);
1298
1299 if (state_enable) {
1300 link->aspm_disable &= ~state;
1301 /* need to enable L1 for substates */
1302 if (state & ASPM_STATE_L1SS)
1303 link->aspm_disable &= ~ASPM_STATE_L1;
1304 } else {
1305 link->aspm_disable |= state;
1306 if (state & ASPM_STATE_L1)
1307 link->aspm_disable |= ASPM_STATE_L1SS;
1308 }
1309
1310 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1311
1312 mutex_unlock(&aspm_lock);
1313 up_read(&pci_bus_sem);
1314
1315 return len;
1316}
1317
1318#define ASPM_ATTR(_f, _s) \
1319static ssize_t _f##_show(struct device *dev, \
1320 struct device_attribute *attr, char *buf) \
1321{ return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); } \
1322 \
1323static ssize_t _f##_store(struct device *dev, \
1324 struct device_attribute *attr, \
1325 const char *buf, size_t len) \
1326{ return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); }
1327
1328ASPM_ATTR(l0s_aspm, L0S)
1329ASPM_ATTR(l1_aspm, L1)
1330ASPM_ATTR(l1_1_aspm, L1_1)
1331ASPM_ATTR(l1_2_aspm, L1_2)
1332ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM)
1333ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM)
1334
1335static ssize_t clkpm_show(struct device *dev,
1336 struct device_attribute *attr, char *buf)
1337{
1338 struct pci_dev *pdev = to_pci_dev(dev);
1339 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1340
1341 return sysfs_emit(buf, "%d\n", link->clkpm_enabled);
1342}
1343
1344static ssize_t clkpm_store(struct device *dev,
1345 struct device_attribute *attr,
1346 const char *buf, size_t len)
1347{
1348 struct pci_dev *pdev = to_pci_dev(dev);
1349 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1350 bool state_enable;
1351
1352 if (kstrtobool(buf, &state_enable) < 0)
1353 return -EINVAL;
1354
1355 down_read(&pci_bus_sem);
1356 mutex_lock(&aspm_lock);
1357
1358 link->clkpm_disable = !state_enable;
1359 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1360
1361 mutex_unlock(&aspm_lock);
1362 up_read(&pci_bus_sem);
1363
1364 return len;
1365}
1366
1367static DEVICE_ATTR_RW(clkpm);
1368static DEVICE_ATTR_RW(l0s_aspm);
1369static DEVICE_ATTR_RW(l1_aspm);
1370static DEVICE_ATTR_RW(l1_1_aspm);
1371static DEVICE_ATTR_RW(l1_2_aspm);
1372static DEVICE_ATTR_RW(l1_1_pcipm);
1373static DEVICE_ATTR_RW(l1_2_pcipm);
1374
1375static struct attribute *aspm_ctrl_attrs[] = {
1376 &dev_attr_clkpm.attr,
1377 &dev_attr_l0s_aspm.attr,
1378 &dev_attr_l1_aspm.attr,
1379 &dev_attr_l1_1_aspm.attr,
1380 &dev_attr_l1_2_aspm.attr,
1381 &dev_attr_l1_1_pcipm.attr,
1382 &dev_attr_l1_2_pcipm.attr,
1383 NULL
1384};
1385
1386static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,
1387 struct attribute *a, int n)
1388{
1389 struct device *dev = kobj_to_dev(kobj);
1390 struct pci_dev *pdev = to_pci_dev(dev);
1391 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1392 static const u8 aspm_state_map[] = {
1393 ASPM_STATE_L0S,
1394 ASPM_STATE_L1,
1395 ASPM_STATE_L1_1,
1396 ASPM_STATE_L1_2,
1397 ASPM_STATE_L1_1_PCIPM,
1398 ASPM_STATE_L1_2_PCIPM,
1399 };
1400
1401 if (aspm_disabled || !link)
1402 return 0;
1403
1404 if (n == 0)
1405 return link->clkpm_capable ? a->mode : 0;
1406
1407 return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;
1408}
1409
1410const struct attribute_group aspm_ctrl_attr_group = {
1411 .name = "link",
1412 .attrs = aspm_ctrl_attrs,
1413 .is_visible = aspm_ctrl_attrs_are_visible,
1414};
1415
1416static int __init pcie_aspm_disable(char *str)
1417{
1418 if (!strcmp(str, "off")) {
1419 aspm_policy = POLICY_DEFAULT;
1420 aspm_disabled = 1;
1421 aspm_support_enabled = false;
1422 pr_info("PCIe ASPM is disabled\n");
1423 } else if (!strcmp(str, "force")) {
1424 aspm_force = 1;
1425 pr_info("PCIe ASPM is forcibly enabled\n");
1426 }
1427 return 1;
1428}
1429
1430__setup("pcie_aspm=", pcie_aspm_disable);
1431
1432void pcie_no_aspm(void)
1433{
1434 /*
1435 * Disabling ASPM is intended to prevent the kernel from modifying
1436 * existing hardware state, not to clear existing state. To that end:
1437 * (a) set policy to POLICY_DEFAULT in order to avoid changing state
1438 * (b) prevent userspace from changing policy
1439 */
1440 if (!aspm_force) {
1441 aspm_policy = POLICY_DEFAULT;
1442 aspm_disabled = 1;
1443 }
1444}
1445
1446bool pcie_aspm_support_enabled(void)
1447{
1448 return aspm_support_enabled;
1449}
1/*
2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIe link L0s/L1 state and Clock Power Management
4 *
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
8 */
9
10#include <linux/kernel.h>
11#include <linux/module.h>
12#include <linux/moduleparam.h>
13#include <linux/pci.h>
14#include <linux/pci_regs.h>
15#include <linux/errno.h>
16#include <linux/pm.h>
17#include <linux/init.h>
18#include <linux/slab.h>
19#include <linux/jiffies.h>
20#include <linux/delay.h>
21#include <linux/pci-aspm.h>
22#include "../pci.h"
23
24#ifdef MODULE_PARAM_PREFIX
25#undef MODULE_PARAM_PREFIX
26#endif
27#define MODULE_PARAM_PREFIX "pcie_aspm."
28
29/* Note: those are not register definitions */
30#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
31#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
32#define ASPM_STATE_L1 (4) /* L1 state */
33#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
34#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1)
35
36struct aspm_latency {
37 u32 l0s; /* L0s latency (nsec) */
38 u32 l1; /* L1 latency (nsec) */
39};
40
41struct pcie_link_state {
42 struct pci_dev *pdev; /* Upstream component of the Link */
43 struct pcie_link_state *root; /* pointer to the root port link */
44 struct pcie_link_state *parent; /* pointer to the parent Link state */
45 struct list_head sibling; /* node in link_list */
46 struct list_head children; /* list of child link states */
47 struct list_head link; /* node in parent's children list */
48
49 /* ASPM state */
50 u32 aspm_support:3; /* Supported ASPM state */
51 u32 aspm_enabled:3; /* Enabled ASPM state */
52 u32 aspm_capable:3; /* Capable ASPM state with latency */
53 u32 aspm_default:3; /* Default ASPM state by BIOS */
54 u32 aspm_disable:3; /* Disabled ASPM state */
55
56 /* Clock PM state */
57 u32 clkpm_capable:1; /* Clock PM capable? */
58 u32 clkpm_enabled:1; /* Current Clock PM state */
59 u32 clkpm_default:1; /* Default Clock PM state by BIOS */
60
61 /* Exit latencies */
62 struct aspm_latency latency_up; /* Upstream direction exit latency */
63 struct aspm_latency latency_dw; /* Downstream direction exit latency */
64 /*
65 * Endpoint acceptable latencies. A pcie downstream port only
66 * has one slot under it, so at most there are 8 functions.
67 */
68 struct aspm_latency acceptable[8];
69};
70
71static int aspm_disabled, aspm_force, aspm_clear_state;
72static bool aspm_support_enabled = true;
73static DEFINE_MUTEX(aspm_lock);
74static LIST_HEAD(link_list);
75
76#define POLICY_DEFAULT 0 /* BIOS default setting */
77#define POLICY_PERFORMANCE 1 /* high performance */
78#define POLICY_POWERSAVE 2 /* high power saving */
79static int aspm_policy;
80static const char *policy_str[] = {
81 [POLICY_DEFAULT] = "default",
82 [POLICY_PERFORMANCE] = "performance",
83 [POLICY_POWERSAVE] = "powersave"
84};
85
86#define LINK_RETRAIN_TIMEOUT HZ
87
88static int policy_to_aspm_state(struct pcie_link_state *link)
89{
90 switch (aspm_policy) {
91 case POLICY_PERFORMANCE:
92 /* Disable ASPM and Clock PM */
93 return 0;
94 case POLICY_POWERSAVE:
95 /* Enable ASPM L0s/L1 */
96 return ASPM_STATE_ALL;
97 case POLICY_DEFAULT:
98 return link->aspm_default;
99 }
100 return 0;
101}
102
103static int policy_to_clkpm_state(struct pcie_link_state *link)
104{
105 switch (aspm_policy) {
106 case POLICY_PERFORMANCE:
107 /* Disable ASPM and Clock PM */
108 return 0;
109 case POLICY_POWERSAVE:
110 /* Disable Clock PM */
111 return 1;
112 case POLICY_DEFAULT:
113 return link->clkpm_default;
114 }
115 return 0;
116}
117
118static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
119{
120 int pos;
121 u16 reg16;
122 struct pci_dev *child;
123 struct pci_bus *linkbus = link->pdev->subordinate;
124
125 list_for_each_entry(child, &linkbus->devices, bus_list) {
126 pos = pci_pcie_cap(child);
127 if (!pos)
128 return;
129 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
130 if (enable)
131 reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
132 else
133 reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
134 pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
135 }
136 link->clkpm_enabled = !!enable;
137}
138
139static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
140{
141 /* Don't enable Clock PM if the link is not Clock PM capable */
142 if (!link->clkpm_capable && enable)
143 enable = 0;
144 /* Need nothing if the specified equals to current state */
145 if (link->clkpm_enabled == enable)
146 return;
147 pcie_set_clkpm_nocheck(link, enable);
148}
149
150static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
151{
152 int pos, capable = 1, enabled = 1;
153 u32 reg32;
154 u16 reg16;
155 struct pci_dev *child;
156 struct pci_bus *linkbus = link->pdev->subordinate;
157
158 /* All functions should have the same cap and state, take the worst */
159 list_for_each_entry(child, &linkbus->devices, bus_list) {
160 pos = pci_pcie_cap(child);
161 if (!pos)
162 return;
163 pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, ®32);
164 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
165 capable = 0;
166 enabled = 0;
167 break;
168 }
169 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
170 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
171 enabled = 0;
172 }
173 link->clkpm_enabled = enabled;
174 link->clkpm_default = enabled;
175 link->clkpm_capable = (blacklist) ? 0 : capable;
176}
177
178/*
179 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
180 * could use common clock. If they are, configure them to use the
181 * common clock. That will reduce the ASPM state exit latency.
182 */
183static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
184{
185 int ppos, cpos, same_clock = 1;
186 u16 reg16, parent_reg, child_reg[8];
187 unsigned long start_jiffies;
188 struct pci_dev *child, *parent = link->pdev;
189 struct pci_bus *linkbus = parent->subordinate;
190 /*
191 * All functions of a slot should have the same Slot Clock
192 * Configuration, so just check one function
193 */
194 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
195 BUG_ON(!pci_is_pcie(child));
196
197 /* Check downstream component if bit Slot Clock Configuration is 1 */
198 cpos = pci_pcie_cap(child);
199 pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, ®16);
200 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
201 same_clock = 0;
202
203 /* Check upstream component if bit Slot Clock Configuration is 1 */
204 ppos = pci_pcie_cap(parent);
205 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
206 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
207 same_clock = 0;
208
209 /* Configure downstream component, all functions */
210 list_for_each_entry(child, &linkbus->devices, bus_list) {
211 cpos = pci_pcie_cap(child);
212 pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, ®16);
213 child_reg[PCI_FUNC(child->devfn)] = reg16;
214 if (same_clock)
215 reg16 |= PCI_EXP_LNKCTL_CCC;
216 else
217 reg16 &= ~PCI_EXP_LNKCTL_CCC;
218 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
219 }
220
221 /* Configure upstream component */
222 pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, ®16);
223 parent_reg = reg16;
224 if (same_clock)
225 reg16 |= PCI_EXP_LNKCTL_CCC;
226 else
227 reg16 &= ~PCI_EXP_LNKCTL_CCC;
228 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
229
230 /* Retrain link */
231 reg16 |= PCI_EXP_LNKCTL_RL;
232 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
233
234 /* Wait for link training end. Break out after waiting for timeout */
235 start_jiffies = jiffies;
236 for (;;) {
237 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
238 if (!(reg16 & PCI_EXP_LNKSTA_LT))
239 break;
240 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
241 break;
242 msleep(1);
243 }
244 if (!(reg16 & PCI_EXP_LNKSTA_LT))
245 return;
246
247 /* Training failed. Restore common clock configurations */
248 dev_printk(KERN_ERR, &parent->dev,
249 "ASPM: Could not configure common clock\n");
250 list_for_each_entry(child, &linkbus->devices, bus_list) {
251 cpos = pci_pcie_cap(child);
252 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
253 child_reg[PCI_FUNC(child->devfn)]);
254 }
255 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
256}
257
258/* Convert L0s latency encoding to ns */
259static u32 calc_l0s_latency(u32 encoding)
260{
261 if (encoding == 0x7)
262 return (5 * 1000); /* > 4us */
263 return (64 << encoding);
264}
265
266/* Convert L0s acceptable latency encoding to ns */
267static u32 calc_l0s_acceptable(u32 encoding)
268{
269 if (encoding == 0x7)
270 return -1U;
271 return (64 << encoding);
272}
273
274/* Convert L1 latency encoding to ns */
275static u32 calc_l1_latency(u32 encoding)
276{
277 if (encoding == 0x7)
278 return (65 * 1000); /* > 64us */
279 return (1000 << encoding);
280}
281
282/* Convert L1 acceptable latency encoding to ns */
283static u32 calc_l1_acceptable(u32 encoding)
284{
285 if (encoding == 0x7)
286 return -1U;
287 return (1000 << encoding);
288}
289
290struct aspm_register_info {
291 u32 support:2;
292 u32 enabled:2;
293 u32 latency_encoding_l0s;
294 u32 latency_encoding_l1;
295};
296
297static void pcie_get_aspm_reg(struct pci_dev *pdev,
298 struct aspm_register_info *info)
299{
300 int pos;
301 u16 reg16;
302 u32 reg32;
303
304 pos = pci_pcie_cap(pdev);
305 pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
306 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
307 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
308 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
309 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
310 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
311}
312
313static void pcie_aspm_check_latency(struct pci_dev *endpoint)
314{
315 u32 latency, l1_switch_latency = 0;
316 struct aspm_latency *acceptable;
317 struct pcie_link_state *link;
318
319 /* Device not in D0 doesn't need latency check */
320 if ((endpoint->current_state != PCI_D0) &&
321 (endpoint->current_state != PCI_UNKNOWN))
322 return;
323
324 link = endpoint->bus->self->link_state;
325 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
326
327 while (link) {
328 /* Check upstream direction L0s latency */
329 if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
330 (link->latency_up.l0s > acceptable->l0s))
331 link->aspm_capable &= ~ASPM_STATE_L0S_UP;
332
333 /* Check downstream direction L0s latency */
334 if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
335 (link->latency_dw.l0s > acceptable->l0s))
336 link->aspm_capable &= ~ASPM_STATE_L0S_DW;
337 /*
338 * Check L1 latency.
339 * Every switch on the path to root complex need 1
340 * more microsecond for L1. Spec doesn't mention L0s.
341 */
342 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
343 if ((link->aspm_capable & ASPM_STATE_L1) &&
344 (latency + l1_switch_latency > acceptable->l1))
345 link->aspm_capable &= ~ASPM_STATE_L1;
346 l1_switch_latency += 1000;
347
348 link = link->parent;
349 }
350}
351
352static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
353{
354 struct pci_dev *child, *parent = link->pdev;
355 struct pci_bus *linkbus = parent->subordinate;
356 struct aspm_register_info upreg, dwreg;
357
358 if (blacklist) {
359 /* Set enabled/disable so that we will disable ASPM later */
360 link->aspm_enabled = ASPM_STATE_ALL;
361 link->aspm_disable = ASPM_STATE_ALL;
362 return;
363 }
364
365 /* Configure common clock before checking latencies */
366 pcie_aspm_configure_common_clock(link);
367
368 /* Get upstream/downstream components' register state */
369 pcie_get_aspm_reg(parent, &upreg);
370 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
371 pcie_get_aspm_reg(child, &dwreg);
372
373 /*
374 * Setup L0s state
375 *
376 * Note that we must not enable L0s in either direction on a
377 * given link unless components on both sides of the link each
378 * support L0s.
379 */
380 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
381 link->aspm_support |= ASPM_STATE_L0S;
382 if (dwreg.enabled & PCIE_LINK_STATE_L0S)
383 link->aspm_enabled |= ASPM_STATE_L0S_UP;
384 if (upreg.enabled & PCIE_LINK_STATE_L0S)
385 link->aspm_enabled |= ASPM_STATE_L0S_DW;
386 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
387 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
388
389 /* Setup L1 state */
390 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
391 link->aspm_support |= ASPM_STATE_L1;
392 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
393 link->aspm_enabled |= ASPM_STATE_L1;
394 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
395 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
396
397 /* Save default state */
398 link->aspm_default = link->aspm_enabled;
399
400 /* Setup initial capable state. Will be updated later */
401 link->aspm_capable = link->aspm_support;
402 /*
403 * If the downstream component has pci bridge function, don't
404 * do ASPM for now.
405 */
406 list_for_each_entry(child, &linkbus->devices, bus_list) {
407 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
408 link->aspm_disable = ASPM_STATE_ALL;
409 break;
410 }
411 }
412
413 /* Get and check endpoint acceptable latencies */
414 list_for_each_entry(child, &linkbus->devices, bus_list) {
415 int pos;
416 u32 reg32, encoding;
417 struct aspm_latency *acceptable =
418 &link->acceptable[PCI_FUNC(child->devfn)];
419
420 if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
421 child->pcie_type != PCI_EXP_TYPE_LEG_END)
422 continue;
423
424 pos = pci_pcie_cap(child);
425 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32);
426 /* Calculate endpoint L0s acceptable latency */
427 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
428 acceptable->l0s = calc_l0s_acceptable(encoding);
429 /* Calculate endpoint L1 acceptable latency */
430 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
431 acceptable->l1 = calc_l1_acceptable(encoding);
432
433 pcie_aspm_check_latency(child);
434 }
435}
436
437static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
438{
439 u16 reg16;
440 int pos = pci_pcie_cap(pdev);
441
442 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
443 reg16 &= ~0x3;
444 reg16 |= val;
445 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
446}
447
448static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
449{
450 u32 upstream = 0, dwstream = 0;
451 struct pci_dev *child, *parent = link->pdev;
452 struct pci_bus *linkbus = parent->subordinate;
453
454 /* Nothing to do if the link is already in the requested state */
455 state &= (link->aspm_capable & ~link->aspm_disable);
456 if (link->aspm_enabled == state)
457 return;
458 /* Convert ASPM state to upstream/downstream ASPM register state */
459 if (state & ASPM_STATE_L0S_UP)
460 dwstream |= PCIE_LINK_STATE_L0S;
461 if (state & ASPM_STATE_L0S_DW)
462 upstream |= PCIE_LINK_STATE_L0S;
463 if (state & ASPM_STATE_L1) {
464 upstream |= PCIE_LINK_STATE_L1;
465 dwstream |= PCIE_LINK_STATE_L1;
466 }
467 /*
468 * Spec 2.0 suggests all functions should be configured the
469 * same setting for ASPM. Enabling ASPM L1 should be done in
470 * upstream component first and then downstream, and vice
471 * versa for disabling ASPM L1. Spec doesn't mention L0S.
472 */
473 if (state & ASPM_STATE_L1)
474 pcie_config_aspm_dev(parent, upstream);
475 list_for_each_entry(child, &linkbus->devices, bus_list)
476 pcie_config_aspm_dev(child, dwstream);
477 if (!(state & ASPM_STATE_L1))
478 pcie_config_aspm_dev(parent, upstream);
479
480 link->aspm_enabled = state;
481}
482
483static void pcie_config_aspm_path(struct pcie_link_state *link)
484{
485 while (link) {
486 pcie_config_aspm_link(link, policy_to_aspm_state(link));
487 link = link->parent;
488 }
489}
490
491static void free_link_state(struct pcie_link_state *link)
492{
493 link->pdev->link_state = NULL;
494 kfree(link);
495}
496
497static int pcie_aspm_sanity_check(struct pci_dev *pdev)
498{
499 struct pci_dev *child;
500 int pos;
501 u32 reg32;
502
503 if (aspm_clear_state)
504 return -EINVAL;
505
506 /*
507 * Some functions in a slot might not all be PCIe functions,
508 * very strange. Disable ASPM for the whole slot
509 */
510 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
511 pos = pci_pcie_cap(child);
512 if (!pos)
513 return -EINVAL;
514 /*
515 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
516 * RBER bit to determine if a function is 1.1 version device
517 */
518 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32);
519 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
520 dev_printk(KERN_INFO, &child->dev, "disabling ASPM"
521 " on pre-1.1 PCIe device. You can enable it"
522 " with 'pcie_aspm=force'\n");
523 return -EINVAL;
524 }
525 }
526 return 0;
527}
528
529static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
530{
531 struct pcie_link_state *link;
532
533 link = kzalloc(sizeof(*link), GFP_KERNEL);
534 if (!link)
535 return NULL;
536 INIT_LIST_HEAD(&link->sibling);
537 INIT_LIST_HEAD(&link->children);
538 INIT_LIST_HEAD(&link->link);
539 link->pdev = pdev;
540 if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
541 struct pcie_link_state *parent;
542 parent = pdev->bus->parent->self->link_state;
543 if (!parent) {
544 kfree(link);
545 return NULL;
546 }
547 link->parent = parent;
548 list_add(&link->link, &parent->children);
549 }
550 /* Setup a pointer to the root port link */
551 if (!link->parent)
552 link->root = link;
553 else
554 link->root = link->parent->root;
555
556 list_add(&link->sibling, &link_list);
557 pdev->link_state = link;
558 return link;
559}
560
561/*
562 * pcie_aspm_init_link_state: Initiate PCI express link state.
563 * It is called after the pcie and its children devices are scaned.
564 * @pdev: the root port or switch downstream port
565 */
566void pcie_aspm_init_link_state(struct pci_dev *pdev)
567{
568 struct pcie_link_state *link;
569 int blacklist = !!pcie_aspm_sanity_check(pdev);
570
571 if (!pci_is_pcie(pdev) || pdev->link_state)
572 return;
573 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
574 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
575 return;
576
577 if (aspm_disabled && !aspm_clear_state)
578 return;
579
580 /* VIA has a strange chipset, root port is under a bridge */
581 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
582 pdev->bus->self)
583 return;
584
585 down_read(&pci_bus_sem);
586 if (list_empty(&pdev->subordinate->devices))
587 goto out;
588
589 mutex_lock(&aspm_lock);
590 link = alloc_pcie_link_state(pdev);
591 if (!link)
592 goto unlock;
593 /*
594 * Setup initial ASPM state. Note that we need to configure
595 * upstream links also because capable state of them can be
596 * update through pcie_aspm_cap_init().
597 */
598 pcie_aspm_cap_init(link, blacklist);
599
600 /* Setup initial Clock PM state */
601 pcie_clkpm_cap_init(link, blacklist);
602
603 /*
604 * At this stage drivers haven't had an opportunity to change the
605 * link policy setting. Enabling ASPM on broken hardware can cripple
606 * it even before the driver has had a chance to disable ASPM, so
607 * default to a safe level right now. If we're enabling ASPM beyond
608 * the BIOS's expectation, we'll do so once pci_enable_device() is
609 * called.
610 */
611 if (aspm_policy != POLICY_POWERSAVE || aspm_clear_state) {
612 pcie_config_aspm_path(link);
613 pcie_set_clkpm(link, policy_to_clkpm_state(link));
614 }
615
616unlock:
617 mutex_unlock(&aspm_lock);
618out:
619 up_read(&pci_bus_sem);
620}
621
622/* Recheck latencies and update aspm_capable for links under the root */
623static void pcie_update_aspm_capable(struct pcie_link_state *root)
624{
625 struct pcie_link_state *link;
626 BUG_ON(root->parent);
627 list_for_each_entry(link, &link_list, sibling) {
628 if (link->root != root)
629 continue;
630 link->aspm_capable = link->aspm_support;
631 }
632 list_for_each_entry(link, &link_list, sibling) {
633 struct pci_dev *child;
634 struct pci_bus *linkbus = link->pdev->subordinate;
635 if (link->root != root)
636 continue;
637 list_for_each_entry(child, &linkbus->devices, bus_list) {
638 if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT) &&
639 (child->pcie_type != PCI_EXP_TYPE_LEG_END))
640 continue;
641 pcie_aspm_check_latency(child);
642 }
643 }
644}
645
646/* @pdev: the endpoint device */
647void pcie_aspm_exit_link_state(struct pci_dev *pdev)
648{
649 struct pci_dev *parent = pdev->bus->self;
650 struct pcie_link_state *link, *root, *parent_link;
651
652 if ((aspm_disabled && !aspm_clear_state) || !pci_is_pcie(pdev) ||
653 !parent || !parent->link_state)
654 return;
655 if ((parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
656 (parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
657 return;
658
659 down_read(&pci_bus_sem);
660 mutex_lock(&aspm_lock);
661 /*
662 * All PCIe functions are in one slot, remove one function will remove
663 * the whole slot, so just wait until we are the last function left.
664 */
665 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
666 goto out;
667
668 link = parent->link_state;
669 root = link->root;
670 parent_link = link->parent;
671
672 /* All functions are removed, so just disable ASPM for the link */
673 pcie_config_aspm_link(link, 0);
674 list_del(&link->sibling);
675 list_del(&link->link);
676 /* Clock PM is for endpoint device */
677 free_link_state(link);
678
679 /* Recheck latencies and configure upstream links */
680 if (parent_link) {
681 pcie_update_aspm_capable(root);
682 pcie_config_aspm_path(parent_link);
683 }
684out:
685 mutex_unlock(&aspm_lock);
686 up_read(&pci_bus_sem);
687}
688
689/* @pdev: the root port or switch downstream port */
690void pcie_aspm_pm_state_change(struct pci_dev *pdev)
691{
692 struct pcie_link_state *link = pdev->link_state;
693
694 if (aspm_disabled || !pci_is_pcie(pdev) || !link)
695 return;
696 if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
697 (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
698 return;
699 /*
700 * Devices changed PM state, we should recheck if latency
701 * meets all functions' requirement
702 */
703 down_read(&pci_bus_sem);
704 mutex_lock(&aspm_lock);
705 pcie_update_aspm_capable(link->root);
706 pcie_config_aspm_path(link);
707 mutex_unlock(&aspm_lock);
708 up_read(&pci_bus_sem);
709}
710
711void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
712{
713 struct pcie_link_state *link = pdev->link_state;
714
715 if (aspm_disabled || !pci_is_pcie(pdev) || !link)
716 return;
717
718 if (aspm_policy != POLICY_POWERSAVE)
719 return;
720
721 if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
722 (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
723 return;
724
725 down_read(&pci_bus_sem);
726 mutex_lock(&aspm_lock);
727 pcie_config_aspm_path(link);
728 pcie_set_clkpm(link, policy_to_clkpm_state(link));
729 mutex_unlock(&aspm_lock);
730 up_read(&pci_bus_sem);
731}
732
733/*
734 * pci_disable_link_state - disable pci device's link state, so the link will
735 * never enter specific states
736 */
737static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
738{
739 struct pci_dev *parent = pdev->bus->self;
740 struct pcie_link_state *link;
741
742 if (aspm_disabled || !pci_is_pcie(pdev))
743 return;
744 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
745 pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
746 parent = pdev;
747 if (!parent || !parent->link_state)
748 return;
749
750 if (sem)
751 down_read(&pci_bus_sem);
752 mutex_lock(&aspm_lock);
753 link = parent->link_state;
754 if (state & PCIE_LINK_STATE_L0S)
755 link->aspm_disable |= ASPM_STATE_L0S;
756 if (state & PCIE_LINK_STATE_L1)
757 link->aspm_disable |= ASPM_STATE_L1;
758 pcie_config_aspm_link(link, policy_to_aspm_state(link));
759
760 if (state & PCIE_LINK_STATE_CLKPM) {
761 link->clkpm_capable = 0;
762 pcie_set_clkpm(link, 0);
763 }
764 mutex_unlock(&aspm_lock);
765 if (sem)
766 up_read(&pci_bus_sem);
767}
768
769void pci_disable_link_state_locked(struct pci_dev *pdev, int state)
770{
771 __pci_disable_link_state(pdev, state, false);
772}
773EXPORT_SYMBOL(pci_disable_link_state_locked);
774
775void pci_disable_link_state(struct pci_dev *pdev, int state)
776{
777 __pci_disable_link_state(pdev, state, true);
778}
779EXPORT_SYMBOL(pci_disable_link_state);
780
781static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
782{
783 int i;
784 struct pcie_link_state *link;
785
786 if (aspm_disabled)
787 return -EPERM;
788 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
789 if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
790 break;
791 if (i >= ARRAY_SIZE(policy_str))
792 return -EINVAL;
793 if (i == aspm_policy)
794 return 0;
795
796 down_read(&pci_bus_sem);
797 mutex_lock(&aspm_lock);
798 aspm_policy = i;
799 list_for_each_entry(link, &link_list, sibling) {
800 pcie_config_aspm_link(link, policy_to_aspm_state(link));
801 pcie_set_clkpm(link, policy_to_clkpm_state(link));
802 }
803 mutex_unlock(&aspm_lock);
804 up_read(&pci_bus_sem);
805 return 0;
806}
807
808static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
809{
810 int i, cnt = 0;
811 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
812 if (i == aspm_policy)
813 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
814 else
815 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
816 return cnt;
817}
818
819module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
820 NULL, 0644);
821
822#ifdef CONFIG_PCIEASPM_DEBUG
823static ssize_t link_state_show(struct device *dev,
824 struct device_attribute *attr,
825 char *buf)
826{
827 struct pci_dev *pci_device = to_pci_dev(dev);
828 struct pcie_link_state *link_state = pci_device->link_state;
829
830 return sprintf(buf, "%d\n", link_state->aspm_enabled);
831}
832
833static ssize_t link_state_store(struct device *dev,
834 struct device_attribute *attr,
835 const char *buf,
836 size_t n)
837{
838 struct pci_dev *pdev = to_pci_dev(dev);
839 struct pcie_link_state *link, *root = pdev->link_state->root;
840 u32 val = buf[0] - '0', state = 0;
841
842 if (aspm_disabled)
843 return -EPERM;
844 if (n < 1 || val > 3)
845 return -EINVAL;
846
847 /* Convert requested state to ASPM state */
848 if (val & PCIE_LINK_STATE_L0S)
849 state |= ASPM_STATE_L0S;
850 if (val & PCIE_LINK_STATE_L1)
851 state |= ASPM_STATE_L1;
852
853 down_read(&pci_bus_sem);
854 mutex_lock(&aspm_lock);
855 list_for_each_entry(link, &link_list, sibling) {
856 if (link->root != root)
857 continue;
858 pcie_config_aspm_link(link, state);
859 }
860 mutex_unlock(&aspm_lock);
861 up_read(&pci_bus_sem);
862 return n;
863}
864
865static ssize_t clk_ctl_show(struct device *dev,
866 struct device_attribute *attr,
867 char *buf)
868{
869 struct pci_dev *pci_device = to_pci_dev(dev);
870 struct pcie_link_state *link_state = pci_device->link_state;
871
872 return sprintf(buf, "%d\n", link_state->clkpm_enabled);
873}
874
875static ssize_t clk_ctl_store(struct device *dev,
876 struct device_attribute *attr,
877 const char *buf,
878 size_t n)
879{
880 struct pci_dev *pdev = to_pci_dev(dev);
881 int state;
882
883 if (n < 1)
884 return -EINVAL;
885 state = buf[0]-'0';
886
887 down_read(&pci_bus_sem);
888 mutex_lock(&aspm_lock);
889 pcie_set_clkpm_nocheck(pdev->link_state, !!state);
890 mutex_unlock(&aspm_lock);
891 up_read(&pci_bus_sem);
892
893 return n;
894}
895
896static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
897static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
898
899static char power_group[] = "power";
900void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
901{
902 struct pcie_link_state *link_state = pdev->link_state;
903
904 if (!pci_is_pcie(pdev) ||
905 (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
906 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
907 return;
908
909 if (link_state->aspm_support)
910 sysfs_add_file_to_group(&pdev->dev.kobj,
911 &dev_attr_link_state.attr, power_group);
912 if (link_state->clkpm_capable)
913 sysfs_add_file_to_group(&pdev->dev.kobj,
914 &dev_attr_clk_ctl.attr, power_group);
915}
916
917void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
918{
919 struct pcie_link_state *link_state = pdev->link_state;
920
921 if (!pci_is_pcie(pdev) ||
922 (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
923 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
924 return;
925
926 if (link_state->aspm_support)
927 sysfs_remove_file_from_group(&pdev->dev.kobj,
928 &dev_attr_link_state.attr, power_group);
929 if (link_state->clkpm_capable)
930 sysfs_remove_file_from_group(&pdev->dev.kobj,
931 &dev_attr_clk_ctl.attr, power_group);
932}
933#endif
934
935static int __init pcie_aspm_disable(char *str)
936{
937 if (!strcmp(str, "off")) {
938 aspm_disabled = 1;
939 aspm_support_enabled = false;
940 printk(KERN_INFO "PCIe ASPM is disabled\n");
941 } else if (!strcmp(str, "force")) {
942 aspm_force = 1;
943 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
944 }
945 return 1;
946}
947
948__setup("pcie_aspm=", pcie_aspm_disable);
949
950void pcie_clear_aspm(void)
951{
952 if (!aspm_force)
953 aspm_clear_state = 1;
954}
955
956void pcie_no_aspm(void)
957{
958 if (!aspm_force)
959 aspm_disabled = 1;
960}
961
962/**
963 * pcie_aspm_enabled - is PCIe ASPM enabled?
964 *
965 * Returns true if ASPM has not been disabled by the command-line option
966 * pcie_aspm=off.
967 **/
968int pcie_aspm_enabled(void)
969{
970 return !aspm_disabled;
971}
972EXPORT_SYMBOL(pcie_aspm_enabled);
973
974bool pcie_aspm_support_enabled(void)
975{
976 return aspm_support_enabled;
977}
978EXPORT_SYMBOL(pcie_aspm_support_enabled);