Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Synopsys DesignWare PCIe PMU driver
4 *
5 * Copyright (C) 2021-2023 Alibaba Inc.
6 */
7
8#include <linux/bitfield.h>
9#include <linux/bitops.h>
10#include <linux/cpuhotplug.h>
11#include <linux/cpumask.h>
12#include <linux/device.h>
13#include <linux/errno.h>
14#include <linux/kernel.h>
15#include <linux/list.h>
16#include <linux/perf_event.h>
17#include <linux/pci.h>
18#include <linux/platform_device.h>
19#include <linux/smp.h>
20#include <linux/sysfs.h>
21#include <linux/types.h>
22
23#define DWC_PCIE_VSEC_RAS_DES_ID 0x02
24#define DWC_PCIE_EVENT_CNT_CTL 0x8
25
26/*
27 * Event Counter Data Select includes two parts:
28 * - 27-24: Group number(4-bit: 0..0x7)
29 * - 23-16: Event number(8-bit: 0..0x13) within the Group
30 *
31 * Put them together as in TRM.
32 */
33#define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16)
34#define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8)
35#define DWC_PCIE_CNT_STATUS BIT(7)
36#define DWC_PCIE_CNT_ENABLE GENMASK(4, 2)
37#define DWC_PCIE_PER_EVENT_OFF 0x1
38#define DWC_PCIE_PER_EVENT_ON 0x3
39#define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0)
40#define DWC_PCIE_EVENT_PER_CLEAR 0x1
41
42#define DWC_PCIE_EVENT_CNT_DATA 0xC
43
44#define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10
45#define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24)
46#define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8)
47#define DWC_PCIE_DURATION_MANUAL_CTL 0x0
48#define DWC_PCIE_DURATION_1MS 0x1
49#define DWC_PCIE_DURATION_10MS 0x2
50#define DWC_PCIE_DURATION_100MS 0x3
51#define DWC_PCIE_DURATION_1S 0x4
52#define DWC_PCIE_DURATION_2S 0x5
53#define DWC_PCIE_DURATION_4S 0x6
54#define DWC_PCIE_DURATION_4US 0xFF
55#define DWC_PCIE_TIME_BASED_TIMER_START BIT(0)
56#define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1
57
58#define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14
59#define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18
60
61/* Event attributes */
62#define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0)
63#define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16)
64#define DWC_PCIE_CONFIG_LANE GENMASK(27, 20)
65
66#define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config)
67#define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config)
68#define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config)
69
70enum dwc_pcie_event_type {
71 DWC_PCIE_TIME_BASE_EVENT,
72 DWC_PCIE_LANE_EVENT,
73 DWC_PCIE_EVENT_TYPE_MAX,
74};
75
76#define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0)
77#define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0)
78
79struct dwc_pcie_pmu {
80 struct pmu pmu;
81 struct pci_dev *pdev; /* Root Port device */
82 u16 ras_des_offset;
83 u32 nr_lanes;
84
85 struct hlist_node cpuhp_node;
86 struct perf_event *event[DWC_PCIE_EVENT_TYPE_MAX];
87 int on_cpu;
88};
89
90#define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu))
91
92static int dwc_pcie_pmu_hp_state;
93static struct list_head dwc_pcie_dev_info_head =
94 LIST_HEAD_INIT(dwc_pcie_dev_info_head);
95static bool notify;
96
97struct dwc_pcie_dev_info {
98 struct platform_device *plat_dev;
99 struct pci_dev *pdev;
100 struct list_head dev_node;
101};
102
103struct dwc_pcie_vendor_id {
104 int vendor_id;
105};
106
107static const struct dwc_pcie_vendor_id dwc_pcie_vendor_ids[] = {
108 {.vendor_id = PCI_VENDOR_ID_ALIBABA },
109 {.vendor_id = PCI_VENDOR_ID_AMPERE },
110 {.vendor_id = PCI_VENDOR_ID_QCOM },
111 {} /* terminator */
112};
113
114static ssize_t cpumask_show(struct device *dev,
115 struct device_attribute *attr,
116 char *buf)
117{
118 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev));
119
120 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
121}
122static DEVICE_ATTR_RO(cpumask);
123
124static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = {
125 &dev_attr_cpumask.attr,
126 NULL
127};
128
129static struct attribute_group dwc_pcie_cpumask_attr_group = {
130 .attrs = dwc_pcie_pmu_cpumask_attrs,
131};
132
133struct dwc_pcie_format_attr {
134 struct device_attribute attr;
135 u64 field;
136 int config;
137};
138
139PMU_FORMAT_ATTR(eventid, "config:0-15");
140PMU_FORMAT_ATTR(type, "config:16-19");
141PMU_FORMAT_ATTR(lane, "config:20-27");
142
143static struct attribute *dwc_pcie_format_attrs[] = {
144 &format_attr_type.attr,
145 &format_attr_eventid.attr,
146 &format_attr_lane.attr,
147 NULL,
148};
149
150static struct attribute_group dwc_pcie_format_attrs_group = {
151 .name = "format",
152 .attrs = dwc_pcie_format_attrs,
153};
154
155struct dwc_pcie_event_attr {
156 struct device_attribute attr;
157 enum dwc_pcie_event_type type;
158 u16 eventid;
159 u8 lane;
160};
161
162static ssize_t dwc_pcie_event_show(struct device *dev,
163 struct device_attribute *attr, char *buf)
164{
165 struct dwc_pcie_event_attr *eattr;
166
167 eattr = container_of(attr, typeof(*eattr), attr);
168
169 if (eattr->type == DWC_PCIE_LANE_EVENT)
170 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n",
171 eattr->eventid, eattr->type);
172 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT)
173 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n",
174 eattr->eventid, eattr->type);
175
176 return 0;
177}
178
179#define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \
180 (&((struct dwc_pcie_event_attr[]) {{ \
181 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \
182 .type = _type, \
183 .eventid = _eventid, \
184 .lane = _lane, \
185 }})[0].attr.attr)
186
187#define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \
188 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0)
189#define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \
190 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0)
191
192static struct attribute *dwc_pcie_pmu_time_event_attrs[] = {
193 /* Group #0 */
194 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00),
195 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01),
196 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02),
197 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03),
198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04),
199 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05),
200 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06),
201 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07),
202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x08),
203 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x09),
204
205 /* Group #1 */
206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20),
207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21),
208 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22),
209 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23),
210
211 /*
212 * Leave it to the user to specify the lane ID to avoid generating
213 * a list of hundreds of events.
214 */
215 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600),
216 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601),
217 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602),
218 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603),
219 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604),
220 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605),
221 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606),
222 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700),
223 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701),
224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702),
225 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703),
226 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704),
227 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705),
228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706),
229 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707),
230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708),
231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709),
232 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A),
233 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B),
234 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C),
235 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F),
236 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710),
237 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711),
238 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712),
239 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713),
240 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714),
241 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715),
242 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716),
243 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717),
244 NULL
245};
246
247static const struct attribute_group dwc_pcie_event_attrs_group = {
248 .name = "events",
249 .attrs = dwc_pcie_pmu_time_event_attrs,
250};
251
252static const struct attribute_group *dwc_pcie_attr_groups[] = {
253 &dwc_pcie_event_attrs_group,
254 &dwc_pcie_format_attrs_group,
255 &dwc_pcie_cpumask_attr_group,
256 NULL
257};
258
259static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu,
260 bool enable)
261{
262 struct pci_dev *pdev = pcie_pmu->pdev;
263 u16 ras_des_offset = pcie_pmu->ras_des_offset;
264
265 if (enable)
266 pci_clear_and_set_config_dword(pdev,
267 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
268 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
269 else
270 pci_clear_and_set_config_dword(pdev,
271 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
272 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);
273}
274
275static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu,
276 bool enable)
277{
278 struct pci_dev *pdev = pcie_pmu->pdev;
279 u16 ras_des_offset = pcie_pmu->ras_des_offset;
280
281 pci_clear_and_set_config_dword(pdev,
282 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL,
283 DWC_PCIE_TIME_BASED_TIMER_START, enable);
284}
285
286static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event)
287{
288 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
289 struct pci_dev *pdev = pcie_pmu->pdev;
290 u16 ras_des_offset = pcie_pmu->ras_des_offset;
291 u32 val;
292
293 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val);
294
295 return val;
296}
297
298static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event)
299{
300 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
301 struct pci_dev *pdev = pcie_pmu->pdev;
302 int event_id = DWC_PCIE_EVENT_ID(event);
303 u16 ras_des_offset = pcie_pmu->ras_des_offset;
304 u32 lo, hi, ss;
305 u64 val;
306
307 /*
308 * The 64-bit value of the data counter is spread across two
309 * registers that are not synchronized. In order to read them
310 * atomically, ensure that the high 32 bits match before and after
311 * reading the low 32 bits.
312 */
313 pci_read_config_dword(pdev,
314 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi);
315 do {
316 /* snapshot the high 32 bits */
317 ss = hi;
318
319 pci_read_config_dword(
320 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW,
321 &lo);
322 pci_read_config_dword(
323 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH,
324 &hi);
325 } while (hi != ss);
326
327 val = ((u64)hi << 32) | lo;
328 /*
329 * The Group#1 event measures the amount of data processed in 16-byte
330 * units. Simplify the end-user interface by multiplying the counter
331 * at the point of read.
332 */
333 if (event_id >= 0x20 && event_id <= 0x23)
334 val *= 16;
335
336 return val;
337}
338
339static void dwc_pcie_pmu_event_update(struct perf_event *event)
340{
341 struct hw_perf_event *hwc = &event->hw;
342 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
343 u64 delta, prev, now = 0;
344
345 do {
346 prev = local64_read(&hwc->prev_count);
347
348 if (type == DWC_PCIE_LANE_EVENT)
349 now = dwc_pcie_pmu_read_lane_event_counter(event);
350 else if (type == DWC_PCIE_TIME_BASE_EVENT)
351 now = dwc_pcie_pmu_read_time_based_counter(event);
352
353 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
354
355 delta = (now - prev) & DWC_PCIE_MAX_PERIOD;
356 /* 32-bit counter for Lane Event Counting */
357 if (type == DWC_PCIE_LANE_EVENT)
358 delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD;
359
360 local64_add(delta, &event->count);
361}
362
363static int dwc_pcie_pmu_event_init(struct perf_event *event)
364{
365 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
366 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
367 struct perf_event *sibling;
368 u32 lane;
369
370 if (event->attr.type != event->pmu->type)
371 return -ENOENT;
372
373 /* We don't support sampling */
374 if (is_sampling_event(event))
375 return -EINVAL;
376
377 /* We cannot support task bound events */
378 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)
379 return -EINVAL;
380
381 if (event->group_leader != event &&
382 !is_software_event(event->group_leader))
383 return -EINVAL;
384
385 for_each_sibling_event(sibling, event->group_leader) {
386 if (sibling->pmu != event->pmu && !is_software_event(sibling))
387 return -EINVAL;
388 }
389
390 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX)
391 return -EINVAL;
392
393 if (type == DWC_PCIE_LANE_EVENT) {
394 lane = DWC_PCIE_EVENT_LANE(event);
395 if (lane < 0 || lane >= pcie_pmu->nr_lanes)
396 return -EINVAL;
397 }
398
399 event->cpu = pcie_pmu->on_cpu;
400
401 return 0;
402}
403
404static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags)
405{
406 struct hw_perf_event *hwc = &event->hw;
407 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
408 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
409
410 hwc->state = 0;
411 local64_set(&hwc->prev_count, 0);
412
413 if (type == DWC_PCIE_LANE_EVENT)
414 dwc_pcie_pmu_lane_event_enable(pcie_pmu, true);
415 else if (type == DWC_PCIE_TIME_BASE_EVENT)
416 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);
417}
418
419static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags)
420{
421 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
422 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
423 struct hw_perf_event *hwc = &event->hw;
424
425 if (event->hw.state & PERF_HES_STOPPED)
426 return;
427
428 if (type == DWC_PCIE_LANE_EVENT)
429 dwc_pcie_pmu_lane_event_enable(pcie_pmu, false);
430 else if (type == DWC_PCIE_TIME_BASE_EVENT)
431 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);
432
433 dwc_pcie_pmu_event_update(event);
434 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
435}
436
437static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags)
438{
439 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
440 struct pci_dev *pdev = pcie_pmu->pdev;
441 struct hw_perf_event *hwc = &event->hw;
442 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
443 int event_id = DWC_PCIE_EVENT_ID(event);
444 int lane = DWC_PCIE_EVENT_LANE(event);
445 u16 ras_des_offset = pcie_pmu->ras_des_offset;
446 u32 ctrl;
447
448 /* one counter for each type and it is in use */
449 if (pcie_pmu->event[type])
450 return -ENOSPC;
451
452 pcie_pmu->event[type] = event;
453 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
454
455 if (type == DWC_PCIE_LANE_EVENT) {
456 /* EVENT_COUNTER_DATA_REG needs clear manually */
457 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
458 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
459 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) |
460 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
461 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
462 ctrl);
463 } else if (type == DWC_PCIE_TIME_BASE_EVENT) {
464 /*
465 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely
466 * use it with any manually controlled duration. And it is
467 * cleared when next measurement starts.
468 */
469 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) |
470 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL,
471 DWC_PCIE_DURATION_MANUAL_CTL) |
472 DWC_PCIE_TIME_BASED_CNT_ENABLE;
473 pci_write_config_dword(
474 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl);
475 }
476
477 if (flags & PERF_EF_START)
478 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD);
479
480 perf_event_update_userpage(event);
481
482 return 0;
483}
484
485static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags)
486{
487 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
488 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
489
490 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE);
491 perf_event_update_userpage(event);
492 pcie_pmu->event[type] = NULL;
493}
494
495static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node)
496{
497 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node);
498}
499
500/*
501 * Find the binded DES capability device info of a PCI device.
502 * @pdev: The PCI device.
503 */
504static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev)
505{
506 struct dwc_pcie_dev_info *dev_info;
507
508 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node)
509 if (dev_info->pdev == pdev)
510 return dev_info;
511
512 return NULL;
513}
514
515static void dwc_pcie_unregister_pmu(void *data)
516{
517 struct dwc_pcie_pmu *pcie_pmu = data;
518
519 perf_pmu_unregister(&pcie_pmu->pmu);
520}
521
522static bool dwc_pcie_match_des_cap(struct pci_dev *pdev)
523{
524 const struct dwc_pcie_vendor_id *vid;
525 u16 vsec = 0;
526 u32 val;
527
528 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT))
529 return false;
530
531 for (vid = dwc_pcie_vendor_ids; vid->vendor_id; vid++) {
532 vsec = pci_find_vsec_capability(pdev, vid->vendor_id,
533 DWC_PCIE_VSEC_RAS_DES_ID);
534 if (vsec)
535 break;
536 }
537 if (!vsec)
538 return false;
539
540 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val);
541 if (PCI_VNDR_HEADER_REV(val) != 0x04)
542 return false;
543
544 pci_dbg(pdev,
545 "Detected PCIe Vendor-Specific Extended Capability RAS DES\n");
546 return true;
547}
548
549static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info)
550{
551 platform_device_unregister(dev_info->plat_dev);
552 list_del(&dev_info->dev_node);
553 kfree(dev_info);
554}
555
556static int dwc_pcie_register_dev(struct pci_dev *pdev)
557{
558 struct platform_device *plat_dev;
559 struct dwc_pcie_dev_info *dev_info;
560 u32 sbdf;
561
562 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn);
563 plat_dev = platform_device_register_data(NULL, "dwc_pcie_pmu", sbdf,
564 pdev, sizeof(*pdev));
565
566 if (IS_ERR(plat_dev))
567 return PTR_ERR(plat_dev);
568
569 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL);
570 if (!dev_info)
571 return -ENOMEM;
572
573 /* Cache platform device to handle pci device hotplug */
574 dev_info->plat_dev = plat_dev;
575 dev_info->pdev = pdev;
576 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head);
577
578 return 0;
579}
580
581static int dwc_pcie_pmu_notifier(struct notifier_block *nb,
582 unsigned long action, void *data)
583{
584 struct device *dev = data;
585 struct pci_dev *pdev = to_pci_dev(dev);
586 struct dwc_pcie_dev_info *dev_info;
587
588 switch (action) {
589 case BUS_NOTIFY_ADD_DEVICE:
590 if (!dwc_pcie_match_des_cap(pdev))
591 return NOTIFY_DONE;
592 if (dwc_pcie_register_dev(pdev))
593 return NOTIFY_BAD;
594 break;
595 case BUS_NOTIFY_DEL_DEVICE:
596 dev_info = dwc_pcie_find_dev_info(pdev);
597 if (!dev_info)
598 return NOTIFY_DONE;
599 dwc_pcie_unregister_dev(dev_info);
600 break;
601 }
602
603 return NOTIFY_OK;
604}
605
606static struct notifier_block dwc_pcie_pmu_nb = {
607 .notifier_call = dwc_pcie_pmu_notifier,
608};
609
610static int dwc_pcie_pmu_probe(struct platform_device *plat_dev)
611{
612 struct pci_dev *pdev = plat_dev->dev.platform_data;
613 struct dwc_pcie_pmu *pcie_pmu;
614 char *name;
615 u32 sbdf, val;
616 u16 vsec;
617 int ret;
618
619 vsec = pci_find_vsec_capability(pdev, pdev->vendor,
620 DWC_PCIE_VSEC_RAS_DES_ID);
621 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val);
622 sbdf = plat_dev->id;
623 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf);
624 if (!name)
625 return -ENOMEM;
626
627 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL);
628 if (!pcie_pmu)
629 return -ENOMEM;
630
631 pcie_pmu->pdev = pdev;
632 pcie_pmu->ras_des_offset = vsec;
633 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev);
634 pcie_pmu->on_cpu = -1;
635 pcie_pmu->pmu = (struct pmu){
636 .name = name,
637 .parent = &pdev->dev,
638 .module = THIS_MODULE,
639 .attr_groups = dwc_pcie_attr_groups,
640 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
641 .task_ctx_nr = perf_invalid_context,
642 .event_init = dwc_pcie_pmu_event_init,
643 .add = dwc_pcie_pmu_event_add,
644 .del = dwc_pcie_pmu_event_del,
645 .start = dwc_pcie_pmu_event_start,
646 .stop = dwc_pcie_pmu_event_stop,
647 .read = dwc_pcie_pmu_event_update,
648 };
649
650 /* Add this instance to the list used by the offline callback */
651 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state,
652 &pcie_pmu->cpuhp_node);
653 if (ret) {
654 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf);
655 return ret;
656 }
657
658 /* Unwind when platform driver removes */
659 ret = devm_add_action_or_reset(&plat_dev->dev,
660 dwc_pcie_pmu_remove_cpuhp_instance,
661 &pcie_pmu->cpuhp_node);
662 if (ret)
663 return ret;
664
665 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1);
666 if (ret) {
667 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf);
668 return ret;
669 }
670 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu,
671 pcie_pmu);
672 if (ret)
673 return ret;
674
675 return 0;
676}
677
678static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
679{
680 struct dwc_pcie_pmu *pcie_pmu;
681
682 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
683 if (pcie_pmu->on_cpu == -1)
684 pcie_pmu->on_cpu = cpumask_local_spread(
685 0, dev_to_node(&pcie_pmu->pdev->dev));
686
687 return 0;
688}
689
690static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
691{
692 struct dwc_pcie_pmu *pcie_pmu;
693 struct pci_dev *pdev;
694 unsigned int target;
695 int node;
696
697 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
698 /* Nothing to do if this CPU doesn't own the PMU */
699 if (cpu != pcie_pmu->on_cpu)
700 return 0;
701
702 pcie_pmu->on_cpu = -1;
703 pdev = pcie_pmu->pdev;
704 node = dev_to_node(&pdev->dev);
705
706 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
707 if (target >= nr_cpu_ids)
708 target = cpumask_any_but(cpu_online_mask, cpu);
709
710 if (target >= nr_cpu_ids) {
711 pci_err(pdev, "There is no CPU to set\n");
712 return 0;
713 }
714
715 /* This PMU does NOT support interrupt, just migrate context. */
716 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);
717 pcie_pmu->on_cpu = target;
718
719 return 0;
720}
721
722static struct platform_driver dwc_pcie_pmu_driver = {
723 .probe = dwc_pcie_pmu_probe,
724 .driver = {.name = "dwc_pcie_pmu",},
725};
726
727static int __init dwc_pcie_pmu_init(void)
728{
729 struct pci_dev *pdev = NULL;
730 int ret;
731
732 for_each_pci_dev(pdev) {
733 if (!dwc_pcie_match_des_cap(pdev))
734 continue;
735
736 ret = dwc_pcie_register_dev(pdev);
737 if (ret) {
738 pci_dev_put(pdev);
739 return ret;
740 }
741 }
742
743 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
744 "perf/dwc_pcie_pmu:online",
745 dwc_pcie_pmu_online_cpu,
746 dwc_pcie_pmu_offline_cpu);
747 if (ret < 0)
748 return ret;
749
750 dwc_pcie_pmu_hp_state = ret;
751
752 ret = platform_driver_register(&dwc_pcie_pmu_driver);
753 if (ret)
754 goto platform_driver_register_err;
755
756 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
757 if (ret)
758 goto platform_driver_register_err;
759 notify = true;
760
761 return 0;
762
763platform_driver_register_err:
764 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
765
766 return ret;
767}
768
769static void __exit dwc_pcie_pmu_exit(void)
770{
771 struct dwc_pcie_dev_info *dev_info, *tmp;
772
773 if (notify)
774 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
775 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node)
776 dwc_pcie_unregister_dev(dev_info);
777 platform_driver_unregister(&dwc_pcie_pmu_driver);
778 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
779}
780
781module_init(dwc_pcie_pmu_init);
782module_exit(dwc_pcie_pmu_exit);
783
784MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller");
785MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
786MODULE_LICENSE("GPL v2");
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Synopsys DesignWare PCIe PMU driver
4 *
5 * Copyright (C) 2021-2023 Alibaba Inc.
6 */
7
8#include <linux/bitfield.h>
9#include <linux/bitops.h>
10#include <linux/cpuhotplug.h>
11#include <linux/cpumask.h>
12#include <linux/device.h>
13#include <linux/errno.h>
14#include <linux/kernel.h>
15#include <linux/list.h>
16#include <linux/perf_event.h>
17#include <linux/pci.h>
18#include <linux/platform_device.h>
19#include <linux/smp.h>
20#include <linux/sysfs.h>
21#include <linux/types.h>
22
23#define DWC_PCIE_VSEC_RAS_DES_ID 0x02
24#define DWC_PCIE_EVENT_CNT_CTL 0x8
25
26/*
27 * Event Counter Data Select includes two parts:
28 * - 27-24: Group number(4-bit: 0..0x7)
29 * - 23-16: Event number(8-bit: 0..0x13) within the Group
30 *
31 * Put them together as in TRM.
32 */
33#define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16)
34#define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8)
35#define DWC_PCIE_CNT_STATUS BIT(7)
36#define DWC_PCIE_CNT_ENABLE GENMASK(4, 2)
37#define DWC_PCIE_PER_EVENT_OFF 0x1
38#define DWC_PCIE_PER_EVENT_ON 0x3
39#define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0)
40#define DWC_PCIE_EVENT_PER_CLEAR 0x1
41
42#define DWC_PCIE_EVENT_CNT_DATA 0xC
43
44#define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10
45#define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24)
46#define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8)
47#define DWC_PCIE_DURATION_MANUAL_CTL 0x0
48#define DWC_PCIE_DURATION_1MS 0x1
49#define DWC_PCIE_DURATION_10MS 0x2
50#define DWC_PCIE_DURATION_100MS 0x3
51#define DWC_PCIE_DURATION_1S 0x4
52#define DWC_PCIE_DURATION_2S 0x5
53#define DWC_PCIE_DURATION_4S 0x6
54#define DWC_PCIE_DURATION_4US 0xFF
55#define DWC_PCIE_TIME_BASED_TIMER_START BIT(0)
56#define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1
57
58#define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14
59#define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18
60
61/* Event attributes */
62#define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0)
63#define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16)
64#define DWC_PCIE_CONFIG_LANE GENMASK(27, 20)
65
66#define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config)
67#define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config)
68#define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config)
69
70enum dwc_pcie_event_type {
71 DWC_PCIE_TIME_BASE_EVENT,
72 DWC_PCIE_LANE_EVENT,
73 DWC_PCIE_EVENT_TYPE_MAX,
74};
75
76#define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0)
77#define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0)
78
79struct dwc_pcie_pmu {
80 struct pmu pmu;
81 struct pci_dev *pdev; /* Root Port device */
82 u16 ras_des_offset;
83 u32 nr_lanes;
84
85 struct list_head pmu_node;
86 struct hlist_node cpuhp_node;
87 struct perf_event *event[DWC_PCIE_EVENT_TYPE_MAX];
88 int on_cpu;
89};
90
91#define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu))
92
93static int dwc_pcie_pmu_hp_state;
94static struct list_head dwc_pcie_dev_info_head =
95 LIST_HEAD_INIT(dwc_pcie_dev_info_head);
96static bool notify;
97
98struct dwc_pcie_dev_info {
99 struct platform_device *plat_dev;
100 struct pci_dev *pdev;
101 struct list_head dev_node;
102};
103
104struct dwc_pcie_vendor_id {
105 int vendor_id;
106};
107
108static const struct dwc_pcie_vendor_id dwc_pcie_vendor_ids[] = {
109 {.vendor_id = PCI_VENDOR_ID_ALIBABA },
110 {} /* terminator */
111};
112
113static ssize_t cpumask_show(struct device *dev,
114 struct device_attribute *attr,
115 char *buf)
116{
117 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev));
118
119 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
120}
121static DEVICE_ATTR_RO(cpumask);
122
123static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = {
124 &dev_attr_cpumask.attr,
125 NULL
126};
127
128static struct attribute_group dwc_pcie_cpumask_attr_group = {
129 .attrs = dwc_pcie_pmu_cpumask_attrs,
130};
131
132struct dwc_pcie_format_attr {
133 struct device_attribute attr;
134 u64 field;
135 int config;
136};
137
138PMU_FORMAT_ATTR(eventid, "config:0-15");
139PMU_FORMAT_ATTR(type, "config:16-19");
140PMU_FORMAT_ATTR(lane, "config:20-27");
141
142static struct attribute *dwc_pcie_format_attrs[] = {
143 &format_attr_type.attr,
144 &format_attr_eventid.attr,
145 &format_attr_lane.attr,
146 NULL,
147};
148
149static struct attribute_group dwc_pcie_format_attrs_group = {
150 .name = "format",
151 .attrs = dwc_pcie_format_attrs,
152};
153
154struct dwc_pcie_event_attr {
155 struct device_attribute attr;
156 enum dwc_pcie_event_type type;
157 u16 eventid;
158 u8 lane;
159};
160
161static ssize_t dwc_pcie_event_show(struct device *dev,
162 struct device_attribute *attr, char *buf)
163{
164 struct dwc_pcie_event_attr *eattr;
165
166 eattr = container_of(attr, typeof(*eattr), attr);
167
168 if (eattr->type == DWC_PCIE_LANE_EVENT)
169 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n",
170 eattr->eventid, eattr->type);
171 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT)
172 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n",
173 eattr->eventid, eattr->type);
174
175 return 0;
176}
177
178#define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \
179 (&((struct dwc_pcie_event_attr[]) {{ \
180 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \
181 .type = _type, \
182 .eventid = _eventid, \
183 .lane = _lane, \
184 }})[0].attr.attr)
185
186#define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \
187 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0)
188#define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \
189 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0)
190
191static struct attribute *dwc_pcie_pmu_time_event_attrs[] = {
192 /* Group #0 */
193 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00),
194 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01),
195 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02),
196 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03),
197 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04),
198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05),
199 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06),
200 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07),
201 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x08),
202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x09),
203
204 /* Group #1 */
205 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Tx_PCIe_TLP_Data_Payload, 0x20),
206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Rx_PCIe_TLP_Data_Payload, 0x21),
207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Tx_CCIX_TLP_Data_Payload, 0x22),
208 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Rx_CCIX_TLP_Data_Payload, 0x23),
209
210 /*
211 * Leave it to the user to specify the lane ID to avoid generating
212 * a list of hundreds of events.
213 */
214 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600),
215 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601),
216 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602),
217 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603),
218 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nulified_tlp, 0x604),
219 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nulified_tlp, 0x605),
220 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tl, 0x606),
221 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700),
222 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701),
223 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702),
224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703),
225 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704),
226 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705),
227 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706),
228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707),
229 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708),
230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709),
231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A),
232 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B),
233 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C),
234 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F),
235 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710),
236 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711),
237 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712),
238 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713),
239 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714),
240 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715),
241 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716),
242 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717),
243 NULL
244};
245
246static const struct attribute_group dwc_pcie_event_attrs_group = {
247 .name = "events",
248 .attrs = dwc_pcie_pmu_time_event_attrs,
249};
250
251static const struct attribute_group *dwc_pcie_attr_groups[] = {
252 &dwc_pcie_event_attrs_group,
253 &dwc_pcie_format_attrs_group,
254 &dwc_pcie_cpumask_attr_group,
255 NULL
256};
257
258static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu,
259 bool enable)
260{
261 struct pci_dev *pdev = pcie_pmu->pdev;
262 u16 ras_des_offset = pcie_pmu->ras_des_offset;
263
264 if (enable)
265 pci_clear_and_set_config_dword(pdev,
266 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
267 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
268 else
269 pci_clear_and_set_config_dword(pdev,
270 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
271 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);
272}
273
274static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu,
275 bool enable)
276{
277 struct pci_dev *pdev = pcie_pmu->pdev;
278 u16 ras_des_offset = pcie_pmu->ras_des_offset;
279
280 pci_clear_and_set_config_dword(pdev,
281 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL,
282 DWC_PCIE_TIME_BASED_TIMER_START, enable);
283}
284
285static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event)
286{
287 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
288 struct pci_dev *pdev = pcie_pmu->pdev;
289 u16 ras_des_offset = pcie_pmu->ras_des_offset;
290 u32 val;
291
292 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val);
293
294 return val;
295}
296
297static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event)
298{
299 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
300 struct pci_dev *pdev = pcie_pmu->pdev;
301 int event_id = DWC_PCIE_EVENT_ID(event);
302 u16 ras_des_offset = pcie_pmu->ras_des_offset;
303 u32 lo, hi, ss;
304 u64 val;
305
306 /*
307 * The 64-bit value of the data counter is spread across two
308 * registers that are not synchronized. In order to read them
309 * atomically, ensure that the high 32 bits match before and after
310 * reading the low 32 bits.
311 */
312 pci_read_config_dword(pdev,
313 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi);
314 do {
315 /* snapshot the high 32 bits */
316 ss = hi;
317
318 pci_read_config_dword(
319 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW,
320 &lo);
321 pci_read_config_dword(
322 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH,
323 &hi);
324 } while (hi != ss);
325
326 val = ((u64)hi << 32) | lo;
327 /*
328 * The Group#1 event measures the amount of data processed in 16-byte
329 * units. Simplify the end-user interface by multiplying the counter
330 * at the point of read.
331 */
332 if (event_id >= 0x20 && event_id <= 0x23)
333 val *= 16;
334
335 return val;
336}
337
338static void dwc_pcie_pmu_event_update(struct perf_event *event)
339{
340 struct hw_perf_event *hwc = &event->hw;
341 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
342 u64 delta, prev, now = 0;
343
344 do {
345 prev = local64_read(&hwc->prev_count);
346
347 if (type == DWC_PCIE_LANE_EVENT)
348 now = dwc_pcie_pmu_read_lane_event_counter(event);
349 else if (type == DWC_PCIE_TIME_BASE_EVENT)
350 now = dwc_pcie_pmu_read_time_based_counter(event);
351
352 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
353
354 delta = (now - prev) & DWC_PCIE_MAX_PERIOD;
355 /* 32-bit counter for Lane Event Counting */
356 if (type == DWC_PCIE_LANE_EVENT)
357 delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD;
358
359 local64_add(delta, &event->count);
360}
361
362static int dwc_pcie_pmu_event_init(struct perf_event *event)
363{
364 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
365 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
366 struct perf_event *sibling;
367 u32 lane;
368
369 if (event->attr.type != event->pmu->type)
370 return -ENOENT;
371
372 /* We don't support sampling */
373 if (is_sampling_event(event))
374 return -EINVAL;
375
376 /* We cannot support task bound events */
377 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)
378 return -EINVAL;
379
380 if (event->group_leader != event &&
381 !is_software_event(event->group_leader))
382 return -EINVAL;
383
384 for_each_sibling_event(sibling, event->group_leader) {
385 if (sibling->pmu != event->pmu && !is_software_event(sibling))
386 return -EINVAL;
387 }
388
389 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX)
390 return -EINVAL;
391
392 if (type == DWC_PCIE_LANE_EVENT) {
393 lane = DWC_PCIE_EVENT_LANE(event);
394 if (lane < 0 || lane >= pcie_pmu->nr_lanes)
395 return -EINVAL;
396 }
397
398 event->cpu = pcie_pmu->on_cpu;
399
400 return 0;
401}
402
403static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags)
404{
405 struct hw_perf_event *hwc = &event->hw;
406 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
407 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
408
409 hwc->state = 0;
410 local64_set(&hwc->prev_count, 0);
411
412 if (type == DWC_PCIE_LANE_EVENT)
413 dwc_pcie_pmu_lane_event_enable(pcie_pmu, true);
414 else if (type == DWC_PCIE_TIME_BASE_EVENT)
415 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);
416}
417
418static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags)
419{
420 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
421 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
422 struct hw_perf_event *hwc = &event->hw;
423
424 if (event->hw.state & PERF_HES_STOPPED)
425 return;
426
427 if (type == DWC_PCIE_LANE_EVENT)
428 dwc_pcie_pmu_lane_event_enable(pcie_pmu, false);
429 else if (type == DWC_PCIE_TIME_BASE_EVENT)
430 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);
431
432 dwc_pcie_pmu_event_update(event);
433 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
434}
435
436static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags)
437{
438 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
439 struct pci_dev *pdev = pcie_pmu->pdev;
440 struct hw_perf_event *hwc = &event->hw;
441 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
442 int event_id = DWC_PCIE_EVENT_ID(event);
443 int lane = DWC_PCIE_EVENT_LANE(event);
444 u16 ras_des_offset = pcie_pmu->ras_des_offset;
445 u32 ctrl;
446
447 /* one counter for each type and it is in use */
448 if (pcie_pmu->event[type])
449 return -ENOSPC;
450
451 pcie_pmu->event[type] = event;
452 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
453
454 if (type == DWC_PCIE_LANE_EVENT) {
455 /* EVENT_COUNTER_DATA_REG needs clear manually */
456 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
457 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
458 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) |
459 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
460 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
461 ctrl);
462 } else if (type == DWC_PCIE_TIME_BASE_EVENT) {
463 /*
464 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely
465 * use it with any manually controlled duration. And it is
466 * cleared when next measurement starts.
467 */
468 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) |
469 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL,
470 DWC_PCIE_DURATION_MANUAL_CTL) |
471 DWC_PCIE_TIME_BASED_CNT_ENABLE;
472 pci_write_config_dword(
473 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl);
474 }
475
476 if (flags & PERF_EF_START)
477 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD);
478
479 perf_event_update_userpage(event);
480
481 return 0;
482}
483
484static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags)
485{
486 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
487 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
488
489 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE);
490 perf_event_update_userpage(event);
491 pcie_pmu->event[type] = NULL;
492}
493
494static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node)
495{
496 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node);
497}
498
499/*
500 * Find the binded DES capability device info of a PCI device.
501 * @pdev: The PCI device.
502 */
503static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev)
504{
505 struct dwc_pcie_dev_info *dev_info;
506
507 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node)
508 if (dev_info->pdev == pdev)
509 return dev_info;
510
511 return NULL;
512}
513
514static void dwc_pcie_unregister_pmu(void *data)
515{
516 struct dwc_pcie_pmu *pcie_pmu = data;
517
518 perf_pmu_unregister(&pcie_pmu->pmu);
519}
520
521static bool dwc_pcie_match_des_cap(struct pci_dev *pdev)
522{
523 const struct dwc_pcie_vendor_id *vid;
524 u16 vsec = 0;
525 u32 val;
526
527 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT))
528 return false;
529
530 for (vid = dwc_pcie_vendor_ids; vid->vendor_id; vid++) {
531 vsec = pci_find_vsec_capability(pdev, vid->vendor_id,
532 DWC_PCIE_VSEC_RAS_DES_ID);
533 if (vsec)
534 break;
535 }
536 if (!vsec)
537 return false;
538
539 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val);
540 if (PCI_VNDR_HEADER_REV(val) != 0x04)
541 return false;
542
543 pci_dbg(pdev,
544 "Detected PCIe Vendor-Specific Extended Capability RAS DES\n");
545 return true;
546}
547
548static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info)
549{
550 platform_device_unregister(dev_info->plat_dev);
551 list_del(&dev_info->dev_node);
552 kfree(dev_info);
553}
554
555static int dwc_pcie_register_dev(struct pci_dev *pdev)
556{
557 struct platform_device *plat_dev;
558 struct dwc_pcie_dev_info *dev_info;
559 u32 bdf;
560
561 bdf = PCI_DEVID(pdev->bus->number, pdev->devfn);
562 plat_dev = platform_device_register_data(NULL, "dwc_pcie_pmu", bdf,
563 pdev, sizeof(*pdev));
564
565 if (IS_ERR(plat_dev))
566 return PTR_ERR(plat_dev);
567
568 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL);
569 if (!dev_info)
570 return -ENOMEM;
571
572 /* Cache platform device to handle pci device hotplug */
573 dev_info->plat_dev = plat_dev;
574 dev_info->pdev = pdev;
575 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head);
576
577 return 0;
578}
579
580static int dwc_pcie_pmu_notifier(struct notifier_block *nb,
581 unsigned long action, void *data)
582{
583 struct device *dev = data;
584 struct pci_dev *pdev = to_pci_dev(dev);
585 struct dwc_pcie_dev_info *dev_info;
586
587 switch (action) {
588 case BUS_NOTIFY_ADD_DEVICE:
589 if (!dwc_pcie_match_des_cap(pdev))
590 return NOTIFY_DONE;
591 if (dwc_pcie_register_dev(pdev))
592 return NOTIFY_BAD;
593 break;
594 case BUS_NOTIFY_DEL_DEVICE:
595 dev_info = dwc_pcie_find_dev_info(pdev);
596 if (!dev_info)
597 return NOTIFY_DONE;
598 dwc_pcie_unregister_dev(dev_info);
599 break;
600 }
601
602 return NOTIFY_OK;
603}
604
605static struct notifier_block dwc_pcie_pmu_nb = {
606 .notifier_call = dwc_pcie_pmu_notifier,
607};
608
609static int dwc_pcie_pmu_probe(struct platform_device *plat_dev)
610{
611 struct pci_dev *pdev = plat_dev->dev.platform_data;
612 struct dwc_pcie_pmu *pcie_pmu;
613 char *name;
614 u32 bdf, val;
615 u16 vsec;
616 int ret;
617
618 vsec = pci_find_vsec_capability(pdev, pdev->vendor,
619 DWC_PCIE_VSEC_RAS_DES_ID);
620 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val);
621 bdf = PCI_DEVID(pdev->bus->number, pdev->devfn);
622 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", bdf);
623 if (!name)
624 return -ENOMEM;
625
626 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL);
627 if (!pcie_pmu)
628 return -ENOMEM;
629
630 pcie_pmu->pdev = pdev;
631 pcie_pmu->ras_des_offset = vsec;
632 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev);
633 pcie_pmu->on_cpu = -1;
634 pcie_pmu->pmu = (struct pmu){
635 .name = name,
636 .parent = &pdev->dev,
637 .module = THIS_MODULE,
638 .attr_groups = dwc_pcie_attr_groups,
639 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
640 .task_ctx_nr = perf_invalid_context,
641 .event_init = dwc_pcie_pmu_event_init,
642 .add = dwc_pcie_pmu_event_add,
643 .del = dwc_pcie_pmu_event_del,
644 .start = dwc_pcie_pmu_event_start,
645 .stop = dwc_pcie_pmu_event_stop,
646 .read = dwc_pcie_pmu_event_update,
647 };
648
649 /* Add this instance to the list used by the offline callback */
650 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state,
651 &pcie_pmu->cpuhp_node);
652 if (ret) {
653 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, bdf);
654 return ret;
655 }
656
657 /* Unwind when platform driver removes */
658 ret = devm_add_action_or_reset(&plat_dev->dev,
659 dwc_pcie_pmu_remove_cpuhp_instance,
660 &pcie_pmu->cpuhp_node);
661 if (ret)
662 return ret;
663
664 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1);
665 if (ret) {
666 pci_err(pdev, "Error %d registering PMU @%x\n", ret, bdf);
667 return ret;
668 }
669 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu,
670 pcie_pmu);
671 if (ret)
672 return ret;
673
674 return 0;
675}
676
677static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
678{
679 struct dwc_pcie_pmu *pcie_pmu;
680
681 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
682 if (pcie_pmu->on_cpu == -1)
683 pcie_pmu->on_cpu = cpumask_local_spread(
684 0, dev_to_node(&pcie_pmu->pdev->dev));
685
686 return 0;
687}
688
689static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
690{
691 struct dwc_pcie_pmu *pcie_pmu;
692 struct pci_dev *pdev;
693 int node;
694 cpumask_t mask;
695 unsigned int target;
696
697 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
698 /* Nothing to do if this CPU doesn't own the PMU */
699 if (cpu != pcie_pmu->on_cpu)
700 return 0;
701
702 pcie_pmu->on_cpu = -1;
703 pdev = pcie_pmu->pdev;
704 node = dev_to_node(&pdev->dev);
705 if (cpumask_and(&mask, cpumask_of_node(node), cpu_online_mask) &&
706 cpumask_andnot(&mask, &mask, cpumask_of(cpu)))
707 target = cpumask_any(&mask);
708 else
709 target = cpumask_any_but(cpu_online_mask, cpu);
710
711 if (target >= nr_cpu_ids) {
712 pci_err(pdev, "There is no CPU to set\n");
713 return 0;
714 }
715
716 /* This PMU does NOT support interrupt, just migrate context. */
717 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);
718 pcie_pmu->on_cpu = target;
719
720 return 0;
721}
722
723static struct platform_driver dwc_pcie_pmu_driver = {
724 .probe = dwc_pcie_pmu_probe,
725 .driver = {.name = "dwc_pcie_pmu",},
726};
727
728static int __init dwc_pcie_pmu_init(void)
729{
730 struct pci_dev *pdev = NULL;
731 bool found = false;
732 int ret;
733
734 for_each_pci_dev(pdev) {
735 if (!dwc_pcie_match_des_cap(pdev))
736 continue;
737
738 ret = dwc_pcie_register_dev(pdev);
739 if (ret) {
740 pci_dev_put(pdev);
741 return ret;
742 }
743
744 found = true;
745 }
746 if (!found)
747 return -ENODEV;
748
749 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
750 "perf/dwc_pcie_pmu:online",
751 dwc_pcie_pmu_online_cpu,
752 dwc_pcie_pmu_offline_cpu);
753 if (ret < 0)
754 return ret;
755
756 dwc_pcie_pmu_hp_state = ret;
757
758 ret = platform_driver_register(&dwc_pcie_pmu_driver);
759 if (ret)
760 goto platform_driver_register_err;
761
762 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
763 if (ret)
764 goto platform_driver_register_err;
765 notify = true;
766
767 return 0;
768
769platform_driver_register_err:
770 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
771
772 return ret;
773}
774
775static void __exit dwc_pcie_pmu_exit(void)
776{
777 struct dwc_pcie_dev_info *dev_info, *tmp;
778
779 if (notify)
780 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
781 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node)
782 dwc_pcie_unregister_dev(dev_info);
783 platform_driver_unregister(&dwc_pcie_pmu_driver);
784 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
785}
786
787module_init(dwc_pcie_pmu_init);
788module_exit(dwc_pcie_pmu_exit);
789
790MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller");
791MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
792MODULE_LICENSE("GPL v2");