Loading...
Note: File does not exist in v3.1.
1/*
2 * L220/L310 cache controller support
3 *
4 * Copyright (C) 2016 ARM Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19#include <linux/errno.h>
20#include <linux/hrtimer.h>
21#include <linux/io.h>
22#include <linux/list.h>
23#include <linux/perf_event.h>
24#include <linux/printk.h>
25#include <linux/slab.h>
26#include <linux/types.h>
27
28#include <asm/hardware/cache-l2x0.h>
29
30#define PMU_NR_COUNTERS 2
31
32static void __iomem *l2x0_base;
33static struct pmu *l2x0_pmu;
34static cpumask_t pmu_cpu;
35
36static const char *l2x0_name;
37
38static ktime_t l2x0_pmu_poll_period;
39static struct hrtimer l2x0_pmu_hrtimer;
40
41/*
42 * The L220/PL310 PMU has two equivalent counters, Counter1 and Counter0.
43 * Registers controlling these are laid out in pairs, in descending order, i.e.
44 * the register for Counter1 comes first, followed by the register for
45 * Counter0.
46 * We ensure that idx 0 -> Counter0, and idx1 -> Counter1.
47 */
48static struct perf_event *events[PMU_NR_COUNTERS];
49
50/* Find an unused counter */
51static int l2x0_pmu_find_idx(void)
52{
53 int i;
54
55 for (i = 0; i < PMU_NR_COUNTERS; i++) {
56 if (!events[i])
57 return i;
58 }
59
60 return -1;
61}
62
63/* How many counters are allocated? */
64static int l2x0_pmu_num_active_counters(void)
65{
66 int i, cnt = 0;
67
68 for (i = 0; i < PMU_NR_COUNTERS; i++) {
69 if (events[i])
70 cnt++;
71 }
72
73 return cnt;
74}
75
76static void l2x0_pmu_counter_config_write(int idx, u32 val)
77{
78 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_CFG - 4 * idx);
79}
80
81static u32 l2x0_pmu_counter_read(int idx)
82{
83 return readl_relaxed(l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
84}
85
86static void l2x0_pmu_counter_write(int idx, u32 val)
87{
88 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
89}
90
91static void __l2x0_pmu_enable(void)
92{
93 u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
94 val |= L2X0_EVENT_CNT_CTRL_ENABLE;
95 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
96}
97
98static void __l2x0_pmu_disable(void)
99{
100 u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
101 val &= ~L2X0_EVENT_CNT_CTRL_ENABLE;
102 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
103}
104
105static void l2x0_pmu_enable(struct pmu *pmu)
106{
107 if (l2x0_pmu_num_active_counters() == 0)
108 return;
109
110 __l2x0_pmu_enable();
111}
112
113static void l2x0_pmu_disable(struct pmu *pmu)
114{
115 if (l2x0_pmu_num_active_counters() == 0)
116 return;
117
118 __l2x0_pmu_disable();
119}
120
121static void warn_if_saturated(u32 count)
122{
123 if (count != 0xffffffff)
124 return;
125
126 pr_warn_ratelimited("L2X0 counter saturated. Poll period too long\n");
127}
128
129static void l2x0_pmu_event_read(struct perf_event *event)
130{
131 struct hw_perf_event *hw = &event->hw;
132 u64 prev_count, new_count, mask;
133
134 do {
135 prev_count = local64_read(&hw->prev_count);
136 new_count = l2x0_pmu_counter_read(hw->idx);
137 } while (local64_xchg(&hw->prev_count, new_count) != prev_count);
138
139 mask = GENMASK_ULL(31, 0);
140 local64_add((new_count - prev_count) & mask, &event->count);
141
142 warn_if_saturated(new_count);
143}
144
145static void l2x0_pmu_event_configure(struct perf_event *event)
146{
147 struct hw_perf_event *hw = &event->hw;
148
149 /*
150 * The L2X0 counters saturate at 0xffffffff rather than wrapping, so we
151 * will *always* lose some number of events when a counter saturates,
152 * and have no way of detecting how many were lost.
153 *
154 * To minimize the impact of this, we try to maximize the period by
155 * always starting counters at zero. To ensure that group ratios are
156 * representative, we poll periodically to avoid counters saturating.
157 * See l2x0_pmu_poll().
158 */
159 local64_set(&hw->prev_count, 0);
160 l2x0_pmu_counter_write(hw->idx, 0);
161}
162
163static enum hrtimer_restart l2x0_pmu_poll(struct hrtimer *hrtimer)
164{
165 unsigned long flags;
166 int i;
167
168 local_irq_save(flags);
169 __l2x0_pmu_disable();
170
171 for (i = 0; i < PMU_NR_COUNTERS; i++) {
172 struct perf_event *event = events[i];
173
174 if (!event)
175 continue;
176
177 l2x0_pmu_event_read(event);
178 l2x0_pmu_event_configure(event);
179 }
180
181 __l2x0_pmu_enable();
182 local_irq_restore(flags);
183
184 hrtimer_forward_now(hrtimer, l2x0_pmu_poll_period);
185 return HRTIMER_RESTART;
186}
187
188
189static void __l2x0_pmu_event_enable(int idx, u32 event)
190{
191 u32 val;
192
193 val = event << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
194 val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
195 l2x0_pmu_counter_config_write(idx, val);
196}
197
198static void l2x0_pmu_event_start(struct perf_event *event, int flags)
199{
200 struct hw_perf_event *hw = &event->hw;
201
202 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
203 return;
204
205 if (flags & PERF_EF_RELOAD) {
206 WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE));
207 l2x0_pmu_event_configure(event);
208 }
209
210 hw->state = 0;
211
212 __l2x0_pmu_event_enable(hw->idx, hw->config_base);
213}
214
215static void __l2x0_pmu_event_disable(int idx)
216{
217 u32 val;
218
219 val = L2X0_EVENT_CNT_CFG_SRC_DISABLED << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
220 val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
221 l2x0_pmu_counter_config_write(idx, val);
222}
223
224static void l2x0_pmu_event_stop(struct perf_event *event, int flags)
225{
226 struct hw_perf_event *hw = &event->hw;
227
228 if (WARN_ON_ONCE(event->hw.state & PERF_HES_STOPPED))
229 return;
230
231 __l2x0_pmu_event_disable(hw->idx);
232
233 hw->state |= PERF_HES_STOPPED;
234
235 if (flags & PERF_EF_UPDATE) {
236 l2x0_pmu_event_read(event);
237 hw->state |= PERF_HES_UPTODATE;
238 }
239}
240
241static int l2x0_pmu_event_add(struct perf_event *event, int flags)
242{
243 struct hw_perf_event *hw = &event->hw;
244 int idx = l2x0_pmu_find_idx();
245
246 if (idx == -1)
247 return -EAGAIN;
248
249 /*
250 * Pin the timer, so that the overflows are handled by the chosen
251 * event->cpu (this is the same one as presented in "cpumask"
252 * attribute).
253 */
254 if (l2x0_pmu_num_active_counters() == 0)
255 hrtimer_start(&l2x0_pmu_hrtimer, l2x0_pmu_poll_period,
256 HRTIMER_MODE_REL_PINNED);
257
258 events[idx] = event;
259 hw->idx = idx;
260
261 l2x0_pmu_event_configure(event);
262
263 hw->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
264
265 if (flags & PERF_EF_START)
266 l2x0_pmu_event_start(event, 0);
267
268 return 0;
269}
270
271static void l2x0_pmu_event_del(struct perf_event *event, int flags)
272{
273 struct hw_perf_event *hw = &event->hw;
274
275 l2x0_pmu_event_stop(event, PERF_EF_UPDATE);
276
277 events[hw->idx] = NULL;
278 hw->idx = -1;
279
280 if (l2x0_pmu_num_active_counters() == 0)
281 hrtimer_cancel(&l2x0_pmu_hrtimer);
282}
283
284static bool l2x0_pmu_group_is_valid(struct perf_event *event)
285{
286 struct pmu *pmu = event->pmu;
287 struct perf_event *leader = event->group_leader;
288 struct perf_event *sibling;
289 int num_hw = 0;
290
291 if (leader->pmu == pmu)
292 num_hw++;
293 else if (!is_software_event(leader))
294 return false;
295
296 for_each_sibling_event(sibling, leader) {
297 if (sibling->pmu == pmu)
298 num_hw++;
299 else if (!is_software_event(sibling))
300 return false;
301 }
302
303 return num_hw <= PMU_NR_COUNTERS;
304}
305
306static int l2x0_pmu_event_init(struct perf_event *event)
307{
308 struct hw_perf_event *hw = &event->hw;
309
310 if (event->attr.type != l2x0_pmu->type)
311 return -ENOENT;
312
313 if (is_sampling_event(event) ||
314 event->attach_state & PERF_ATTACH_TASK)
315 return -EINVAL;
316
317 if (event->attr.exclude_user ||
318 event->attr.exclude_kernel ||
319 event->attr.exclude_hv ||
320 event->attr.exclude_idle ||
321 event->attr.exclude_host ||
322 event->attr.exclude_guest)
323 return -EINVAL;
324
325 if (event->cpu < 0)
326 return -EINVAL;
327
328 if (event->attr.config & ~L2X0_EVENT_CNT_CFG_SRC_MASK)
329 return -EINVAL;
330
331 hw->config_base = event->attr.config;
332
333 if (!l2x0_pmu_group_is_valid(event))
334 return -EINVAL;
335
336 event->cpu = cpumask_first(&pmu_cpu);
337
338 return 0;
339}
340
341struct l2x0_event_attribute {
342 struct device_attribute attr;
343 unsigned int config;
344 bool pl310_only;
345};
346
347#define L2X0_EVENT_ATTR(_name, _config, _pl310_only) \
348 (&((struct l2x0_event_attribute[]) {{ \
349 .attr = __ATTR(_name, S_IRUGO, l2x0_pmu_event_show, NULL), \
350 .config = _config, \
351 .pl310_only = _pl310_only, \
352 }})[0].attr.attr)
353
354#define L220_PLUS_EVENT_ATTR(_name, _config) \
355 L2X0_EVENT_ATTR(_name, _config, false)
356
357#define PL310_EVENT_ATTR(_name, _config) \
358 L2X0_EVENT_ATTR(_name, _config, true)
359
360static ssize_t l2x0_pmu_event_show(struct device *dev,
361 struct device_attribute *attr, char *buf)
362{
363 struct l2x0_event_attribute *lattr;
364
365 lattr = container_of(attr, typeof(*lattr), attr);
366 return snprintf(buf, PAGE_SIZE, "config=0x%x\n", lattr->config);
367}
368
369static umode_t l2x0_pmu_event_attr_is_visible(struct kobject *kobj,
370 struct attribute *attr,
371 int unused)
372{
373 struct device *dev = kobj_to_dev(kobj);
374 struct pmu *pmu = dev_get_drvdata(dev);
375 struct l2x0_event_attribute *lattr;
376
377 lattr = container_of(attr, typeof(*lattr), attr.attr);
378
379 if (!lattr->pl310_only || strcmp("l2c_310", pmu->name) == 0)
380 return attr->mode;
381
382 return 0;
383}
384
385static struct attribute *l2x0_pmu_event_attrs[] = {
386 L220_PLUS_EVENT_ATTR(co, 0x1),
387 L220_PLUS_EVENT_ATTR(drhit, 0x2),
388 L220_PLUS_EVENT_ATTR(drreq, 0x3),
389 L220_PLUS_EVENT_ATTR(dwhit, 0x4),
390 L220_PLUS_EVENT_ATTR(dwreq, 0x5),
391 L220_PLUS_EVENT_ATTR(dwtreq, 0x6),
392 L220_PLUS_EVENT_ATTR(irhit, 0x7),
393 L220_PLUS_EVENT_ATTR(irreq, 0x8),
394 L220_PLUS_EVENT_ATTR(wa, 0x9),
395 PL310_EVENT_ATTR(ipfalloc, 0xa),
396 PL310_EVENT_ATTR(epfhit, 0xb),
397 PL310_EVENT_ATTR(epfalloc, 0xc),
398 PL310_EVENT_ATTR(srrcvd, 0xd),
399 PL310_EVENT_ATTR(srconf, 0xe),
400 PL310_EVENT_ATTR(epfrcvd, 0xf),
401 NULL
402};
403
404static struct attribute_group l2x0_pmu_event_attrs_group = {
405 .name = "events",
406 .attrs = l2x0_pmu_event_attrs,
407 .is_visible = l2x0_pmu_event_attr_is_visible,
408};
409
410static ssize_t l2x0_pmu_cpumask_show(struct device *dev,
411 struct device_attribute *attr, char *buf)
412{
413 return cpumap_print_to_pagebuf(true, buf, &pmu_cpu);
414}
415
416static struct device_attribute l2x0_pmu_cpumask_attr =
417 __ATTR(cpumask, S_IRUGO, l2x0_pmu_cpumask_show, NULL);
418
419static struct attribute *l2x0_pmu_cpumask_attrs[] = {
420 &l2x0_pmu_cpumask_attr.attr,
421 NULL,
422};
423
424static struct attribute_group l2x0_pmu_cpumask_attr_group = {
425 .attrs = l2x0_pmu_cpumask_attrs,
426};
427
428static const struct attribute_group *l2x0_pmu_attr_groups[] = {
429 &l2x0_pmu_event_attrs_group,
430 &l2x0_pmu_cpumask_attr_group,
431 NULL,
432};
433
434static void l2x0_pmu_reset(void)
435{
436 int i;
437
438 __l2x0_pmu_disable();
439
440 for (i = 0; i < PMU_NR_COUNTERS; i++)
441 __l2x0_pmu_event_disable(i);
442}
443
444static int l2x0_pmu_offline_cpu(unsigned int cpu)
445{
446 unsigned int target;
447
448 if (!cpumask_test_and_clear_cpu(cpu, &pmu_cpu))
449 return 0;
450
451 target = cpumask_any_but(cpu_online_mask, cpu);
452 if (target >= nr_cpu_ids)
453 return 0;
454
455 perf_pmu_migrate_context(l2x0_pmu, cpu, target);
456 cpumask_set_cpu(target, &pmu_cpu);
457
458 return 0;
459}
460
461void l2x0_pmu_suspend(void)
462{
463 int i;
464
465 if (!l2x0_pmu)
466 return;
467
468 l2x0_pmu_disable(l2x0_pmu);
469
470 for (i = 0; i < PMU_NR_COUNTERS; i++) {
471 if (events[i])
472 l2x0_pmu_event_stop(events[i], PERF_EF_UPDATE);
473 }
474
475}
476
477void l2x0_pmu_resume(void)
478{
479 int i;
480
481 if (!l2x0_pmu)
482 return;
483
484 l2x0_pmu_reset();
485
486 for (i = 0; i < PMU_NR_COUNTERS; i++) {
487 if (events[i])
488 l2x0_pmu_event_start(events[i], PERF_EF_RELOAD);
489 }
490
491 l2x0_pmu_enable(l2x0_pmu);
492}
493
494void __init l2x0_pmu_register(void __iomem *base, u32 part)
495{
496 /*
497 * Determine whether we support the PMU, and choose the name for sysfs.
498 * This is also used by l2x0_pmu_event_attr_is_visible to determine
499 * which events to display, as the PL310 PMU supports a superset of
500 * L220 events.
501 *
502 * The L210 PMU has a different programmer's interface, and is not
503 * supported by this driver.
504 *
505 * We must defer registering the PMU until the perf subsystem is up and
506 * running, so just stash the name and base, and leave that to another
507 * initcall.
508 */
509 switch (part & L2X0_CACHE_ID_PART_MASK) {
510 case L2X0_CACHE_ID_PART_L220:
511 l2x0_name = "l2c_220";
512 break;
513 case L2X0_CACHE_ID_PART_L310:
514 l2x0_name = "l2c_310";
515 break;
516 default:
517 return;
518 }
519
520 l2x0_base = base;
521}
522
523static __init int l2x0_pmu_init(void)
524{
525 int ret;
526
527 if (!l2x0_base)
528 return 0;
529
530 l2x0_pmu = kzalloc(sizeof(*l2x0_pmu), GFP_KERNEL);
531 if (!l2x0_pmu) {
532 pr_warn("Unable to allocate L2x0 PMU\n");
533 return -ENOMEM;
534 }
535
536 *l2x0_pmu = (struct pmu) {
537 .task_ctx_nr = perf_invalid_context,
538 .pmu_enable = l2x0_pmu_enable,
539 .pmu_disable = l2x0_pmu_disable,
540 .read = l2x0_pmu_event_read,
541 .start = l2x0_pmu_event_start,
542 .stop = l2x0_pmu_event_stop,
543 .add = l2x0_pmu_event_add,
544 .del = l2x0_pmu_event_del,
545 .event_init = l2x0_pmu_event_init,
546 .attr_groups = l2x0_pmu_attr_groups,
547 };
548
549 l2x0_pmu_reset();
550
551 /*
552 * We always use a hrtimer rather than an interrupt.
553 * See comments in l2x0_pmu_event_configure and l2x0_pmu_poll.
554 *
555 * Polling once a second allows the counters to fill up to 1/128th on a
556 * quad-core test chip with cores clocked at 400MHz. Hopefully this
557 * leaves sufficient headroom to avoid overflow on production silicon
558 * at higher frequencies.
559 */
560 l2x0_pmu_poll_period = ms_to_ktime(1000);
561 hrtimer_init(&l2x0_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
562 l2x0_pmu_hrtimer.function = l2x0_pmu_poll;
563
564 cpumask_set_cpu(0, &pmu_cpu);
565 ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE,
566 "perf/arm/l2x0:online", NULL,
567 l2x0_pmu_offline_cpu);
568 if (ret)
569 goto out_pmu;
570
571 ret = perf_pmu_register(l2x0_pmu, l2x0_name, -1);
572 if (ret)
573 goto out_cpuhp;
574
575 return 0;
576
577out_cpuhp:
578 cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE);
579out_pmu:
580 kfree(l2x0_pmu);
581 l2x0_pmu = NULL;
582 return ret;
583}
584device_initcall(l2x0_pmu_init);