Loading...
1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2/*
3 * Copyright(c) 2019 Intel Corporation.
4 *
5 */
6
7#include "aspm.h"
8
9/* Time after which the timer interrupt will re-enable ASPM */
10#define ASPM_TIMER_MS 1000
11/* Time for which interrupts are ignored after a timer has been scheduled */
12#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
13/* Two interrupts within this time trigger ASPM disable */
14#define ASPM_TRIGGER_MS 1
15#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
16#define ASPM_L1_SUPPORTED(reg) \
17 ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
18
19uint aspm_mode = ASPM_MODE_DISABLED;
20module_param_named(aspm, aspm_mode, uint, 0444);
21MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");
22
23static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
24{
25 struct pci_dev *parent = dd->pcidev->bus->self;
26 u32 up, dn;
27
28 /*
29 * If the driver does not have access to the upstream component,
30 * it cannot support ASPM L1 at all.
31 */
32 if (!parent)
33 return false;
34
35 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
36 dn = ASPM_L1_SUPPORTED(dn);
37
38 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
39 up = ASPM_L1_SUPPORTED(up);
40
41 /* ASPM works on A-step but is reported as not supported */
42 return (!!dn || is_ax(dd)) && !!up;
43}
44
45/* Set L1 entrance latency for slower entry to L1 */
46static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
47{
48 u32 l1_ent_lat = 0x4u;
49 u32 reg32;
50
51 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32);
52 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
53 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
54 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
55}
56
57static void aspm_hw_enable_l1(struct hfi1_devdata *dd)
58{
59 struct pci_dev *parent = dd->pcidev->bus->self;
60
61 /*
62 * If the driver does not have access to the upstream component,
63 * it cannot support ASPM L1 at all.
64 */
65 if (!parent)
66 return;
67
68 /* Enable ASPM L1 first in upstream component and then downstream */
69 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
70 PCI_EXP_LNKCTL_ASPMC,
71 PCI_EXP_LNKCTL_ASPM_L1);
72 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
73 PCI_EXP_LNKCTL_ASPMC,
74 PCI_EXP_LNKCTL_ASPM_L1);
75}
76
77void aspm_hw_disable_l1(struct hfi1_devdata *dd)
78{
79 struct pci_dev *parent = dd->pcidev->bus->self;
80
81 /* Disable ASPM L1 first in downstream component and then upstream */
82 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
83 PCI_EXP_LNKCTL_ASPMC, 0x0);
84 if (parent)
85 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
86 PCI_EXP_LNKCTL_ASPMC, 0x0);
87}
88
89static void aspm_enable(struct hfi1_devdata *dd)
90{
91 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
92 !dd->aspm_supported)
93 return;
94
95 aspm_hw_enable_l1(dd);
96 dd->aspm_enabled = true;
97}
98
99static void aspm_disable(struct hfi1_devdata *dd)
100{
101 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
102 return;
103
104 aspm_hw_disable_l1(dd);
105 dd->aspm_enabled = false;
106}
107
108static void aspm_disable_inc(struct hfi1_devdata *dd)
109{
110 unsigned long flags;
111
112 spin_lock_irqsave(&dd->aspm_lock, flags);
113 aspm_disable(dd);
114 atomic_inc(&dd->aspm_disabled_cnt);
115 spin_unlock_irqrestore(&dd->aspm_lock, flags);
116}
117
118static void aspm_enable_dec(struct hfi1_devdata *dd)
119{
120 unsigned long flags;
121
122 spin_lock_irqsave(&dd->aspm_lock, flags);
123 if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
124 aspm_enable(dd);
125 spin_unlock_irqrestore(&dd->aspm_lock, flags);
126}
127
128/* ASPM processing for each receive context interrupt */
129void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
130{
131 bool restart_timer;
132 bool close_interrupts;
133 unsigned long flags;
134 ktime_t now, prev;
135
136 spin_lock_irqsave(&rcd->aspm_lock, flags);
137 /* PSM contexts are open */
138 if (!rcd->aspm_intr_enable)
139 goto unlock;
140
141 prev = rcd->aspm_ts_last_intr;
142 now = ktime_get();
143 rcd->aspm_ts_last_intr = now;
144
145 /* An interrupt pair close together in time */
146 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
147
148 /* Don't push out our timer till this much time has elapsed */
149 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
150 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
151 restart_timer = restart_timer && close_interrupts;
152
153 /* Disable ASPM and schedule timer */
154 if (rcd->aspm_enabled && close_interrupts) {
155 aspm_disable_inc(rcd->dd);
156 rcd->aspm_enabled = false;
157 restart_timer = true;
158 }
159
160 if (restart_timer) {
161 mod_timer(&rcd->aspm_timer,
162 jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
163 rcd->aspm_ts_timer_sched = now;
164 }
165unlock:
166 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
167}
168
169/* Timer function for re-enabling ASPM in the absence of interrupt activity */
170static void aspm_ctx_timer_function(struct timer_list *t)
171{
172 struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);
173 unsigned long flags;
174
175 spin_lock_irqsave(&rcd->aspm_lock, flags);
176 aspm_enable_dec(rcd->dd);
177 rcd->aspm_enabled = true;
178 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
179}
180
181/*
182 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
183 * are open.
184 */
185void aspm_disable_all(struct hfi1_devdata *dd)
186{
187 struct hfi1_ctxtdata *rcd;
188 unsigned long flags;
189 u16 i;
190
191 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
192 rcd = hfi1_rcd_get_by_index(dd, i);
193 if (rcd) {
194 del_timer_sync(&rcd->aspm_timer);
195 spin_lock_irqsave(&rcd->aspm_lock, flags);
196 rcd->aspm_intr_enable = false;
197 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
198 hfi1_rcd_put(rcd);
199 }
200 }
201
202 aspm_disable(dd);
203 atomic_set(&dd->aspm_disabled_cnt, 0);
204}
205
206/* Re-enable interrupt processing for verbs contexts */
207void aspm_enable_all(struct hfi1_devdata *dd)
208{
209 struct hfi1_ctxtdata *rcd;
210 unsigned long flags;
211 u16 i;
212
213 aspm_enable(dd);
214
215 if (aspm_mode != ASPM_MODE_DYNAMIC)
216 return;
217
218 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
219 rcd = hfi1_rcd_get_by_index(dd, i);
220 if (rcd) {
221 spin_lock_irqsave(&rcd->aspm_lock, flags);
222 rcd->aspm_intr_enable = true;
223 rcd->aspm_enabled = true;
224 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
225 hfi1_rcd_put(rcd);
226 }
227 }
228}
229
230static void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
231{
232 spin_lock_init(&rcd->aspm_lock);
233 timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);
234 rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
235 aspm_mode == ASPM_MODE_DYNAMIC &&
236 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;
237}
238
239void aspm_init(struct hfi1_devdata *dd)
240{
241 struct hfi1_ctxtdata *rcd;
242 u16 i;
243
244 spin_lock_init(&dd->aspm_lock);
245 dd->aspm_supported = aspm_hw_l1_supported(dd);
246
247 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
248 rcd = hfi1_rcd_get_by_index(dd, i);
249 if (rcd)
250 aspm_ctx_init(rcd);
251 hfi1_rcd_put(rcd);
252 }
253
254 /* Start with ASPM disabled */
255 aspm_hw_set_l1_ent_latency(dd);
256 dd->aspm_enabled = false;
257 aspm_hw_disable_l1(dd);
258
259 /* Now turn on ASPM if configured */
260 aspm_enable_all(dd);
261}
262
263void aspm_exit(struct hfi1_devdata *dd)
264{
265 aspm_disable_all(dd);
266
267 /* Turn on ASPM on exit to conserve power */
268 aspm_enable(dd);
269}
270
1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2/*
3 * Copyright(c) 2019 Intel Corporation.
4 *
5 */
6
7#include "aspm.h"
8
9/* Time after which the timer interrupt will re-enable ASPM */
10#define ASPM_TIMER_MS 1000
11/* Time for which interrupts are ignored after a timer has been scheduled */
12#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
13/* Two interrupts within this time trigger ASPM disable */
14#define ASPM_TRIGGER_MS 1
15#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
16#define ASPM_L1_SUPPORTED(reg) \
17 ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
18
19uint aspm_mode = ASPM_MODE_DISABLED;
20module_param_named(aspm, aspm_mode, uint, 0444);
21MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");
22
23static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
24{
25 struct pci_dev *parent = dd->pcidev->bus->self;
26 u32 up, dn;
27
28 /*
29 * If the driver does not have access to the upstream component,
30 * it cannot support ASPM L1 at all.
31 */
32 if (!parent)
33 return false;
34
35 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
36 dn = ASPM_L1_SUPPORTED(dn);
37
38 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
39 up = ASPM_L1_SUPPORTED(up);
40
41 /* ASPM works on A-step but is reported as not supported */
42 return (!!dn || is_ax(dd)) && !!up;
43}
44
45/* Set L1 entrance latency for slower entry to L1 */
46static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
47{
48 u32 l1_ent_lat = 0x4u;
49 u32 reg32;
50
51 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32);
52 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
53 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
54 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
55}
56
57static void aspm_hw_enable_l1(struct hfi1_devdata *dd)
58{
59 struct pci_dev *parent = dd->pcidev->bus->self;
60
61 /*
62 * If the driver does not have access to the upstream component,
63 * it cannot support ASPM L1 at all.
64 */
65 if (!parent)
66 return;
67
68 /* Enable ASPM L1 first in upstream component and then downstream */
69 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
70 PCI_EXP_LNKCTL_ASPMC,
71 PCI_EXP_LNKCTL_ASPM_L1);
72 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
73 PCI_EXP_LNKCTL_ASPMC,
74 PCI_EXP_LNKCTL_ASPM_L1);
75}
76
77void aspm_hw_disable_l1(struct hfi1_devdata *dd)
78{
79 struct pci_dev *parent = dd->pcidev->bus->self;
80
81 /* Disable ASPM L1 first in downstream component and then upstream */
82 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
83 PCI_EXP_LNKCTL_ASPMC, 0x0);
84 if (parent)
85 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
86 PCI_EXP_LNKCTL_ASPMC, 0x0);
87}
88
89static void aspm_enable(struct hfi1_devdata *dd)
90{
91 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
92 !dd->aspm_supported)
93 return;
94
95 aspm_hw_enable_l1(dd);
96 dd->aspm_enabled = true;
97}
98
99static void aspm_disable(struct hfi1_devdata *dd)
100{
101 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
102 return;
103
104 aspm_hw_disable_l1(dd);
105 dd->aspm_enabled = false;
106}
107
108static void aspm_disable_inc(struct hfi1_devdata *dd)
109{
110 unsigned long flags;
111
112 spin_lock_irqsave(&dd->aspm_lock, flags);
113 aspm_disable(dd);
114 atomic_inc(&dd->aspm_disabled_cnt);
115 spin_unlock_irqrestore(&dd->aspm_lock, flags);
116}
117
118static void aspm_enable_dec(struct hfi1_devdata *dd)
119{
120 unsigned long flags;
121
122 spin_lock_irqsave(&dd->aspm_lock, flags);
123 if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
124 aspm_enable(dd);
125 spin_unlock_irqrestore(&dd->aspm_lock, flags);
126}
127
128/* ASPM processing for each receive context interrupt */
129void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
130{
131 bool restart_timer;
132 bool close_interrupts;
133 unsigned long flags;
134 ktime_t now, prev;
135
136 spin_lock_irqsave(&rcd->aspm_lock, flags);
137 /* PSM contexts are open */
138 if (!rcd->aspm_intr_enable)
139 goto unlock;
140
141 prev = rcd->aspm_ts_last_intr;
142 now = ktime_get();
143 rcd->aspm_ts_last_intr = now;
144
145 /* An interrupt pair close together in time */
146 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
147
148 /* Don't push out our timer till this much time has elapsed */
149 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
150 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
151 restart_timer = restart_timer && close_interrupts;
152
153 /* Disable ASPM and schedule timer */
154 if (rcd->aspm_enabled && close_interrupts) {
155 aspm_disable_inc(rcd->dd);
156 rcd->aspm_enabled = false;
157 restart_timer = true;
158 }
159
160 if (restart_timer) {
161 mod_timer(&rcd->aspm_timer,
162 jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
163 rcd->aspm_ts_timer_sched = now;
164 }
165unlock:
166 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
167}
168
169/* Timer function for re-enabling ASPM in the absence of interrupt activity */
170static void aspm_ctx_timer_function(struct timer_list *t)
171{
172 struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);
173 unsigned long flags;
174
175 spin_lock_irqsave(&rcd->aspm_lock, flags);
176 aspm_enable_dec(rcd->dd);
177 rcd->aspm_enabled = true;
178 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
179}
180
181/*
182 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
183 * are open.
184 */
185void aspm_disable_all(struct hfi1_devdata *dd)
186{
187 struct hfi1_ctxtdata *rcd;
188 unsigned long flags;
189 u16 i;
190
191 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
192 rcd = hfi1_rcd_get_by_index(dd, i);
193 if (rcd) {
194 del_timer_sync(&rcd->aspm_timer);
195 spin_lock_irqsave(&rcd->aspm_lock, flags);
196 rcd->aspm_intr_enable = false;
197 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
198 hfi1_rcd_put(rcd);
199 }
200 }
201
202 aspm_disable(dd);
203 atomic_set(&dd->aspm_disabled_cnt, 0);
204}
205
206/* Re-enable interrupt processing for verbs contexts */
207void aspm_enable_all(struct hfi1_devdata *dd)
208{
209 struct hfi1_ctxtdata *rcd;
210 unsigned long flags;
211 u16 i;
212
213 aspm_enable(dd);
214
215 if (aspm_mode != ASPM_MODE_DYNAMIC)
216 return;
217
218 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
219 rcd = hfi1_rcd_get_by_index(dd, i);
220 if (rcd) {
221 spin_lock_irqsave(&rcd->aspm_lock, flags);
222 rcd->aspm_intr_enable = true;
223 rcd->aspm_enabled = true;
224 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
225 hfi1_rcd_put(rcd);
226 }
227 }
228}
229
230static void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
231{
232 spin_lock_init(&rcd->aspm_lock);
233 timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);
234 rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
235 aspm_mode == ASPM_MODE_DYNAMIC &&
236 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;
237}
238
239void aspm_init(struct hfi1_devdata *dd)
240{
241 struct hfi1_ctxtdata *rcd;
242 u16 i;
243
244 spin_lock_init(&dd->aspm_lock);
245 dd->aspm_supported = aspm_hw_l1_supported(dd);
246
247 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
248 rcd = hfi1_rcd_get_by_index(dd, i);
249 if (rcd)
250 aspm_ctx_init(rcd);
251 hfi1_rcd_put(rcd);
252 }
253
254 /* Start with ASPM disabled */
255 aspm_hw_set_l1_ent_latency(dd);
256 dd->aspm_enabled = false;
257 aspm_hw_disable_l1(dd);
258
259 /* Now turn on ASPM if configured */
260 aspm_enable_all(dd);
261}
262
263void aspm_exit(struct hfi1_devdata *dd)
264{
265 aspm_disable_all(dd);
266
267 /* Turn on ASPM on exit to conserve power */
268 aspm_enable(dd);
269}
270