Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
4 * sink device could then pair with an appropriate per-cpu coresight source
5 * device (ETE) thus generating required trace data. Trace can be enabled
6 * via the perf framework.
7 *
8 * The AUX buffer handling is inspired from Arm SPE PMU driver.
9 *
10 * Copyright (C) 2020 ARM Ltd.
11 *
12 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
13 */
14#define DRVNAME "arm_trbe"
15
16#define pr_fmt(fmt) DRVNAME ": " fmt
17
18#include <asm/barrier.h>
19#include <asm/cpufeature.h>
20
21#include "coresight-self-hosted-trace.h"
22#include "coresight-trbe.h"
23
24#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
25
26/*
27 * A padding packet that will help the user space tools
28 * in skipping relevant sections in the captured trace
29 * data which could not be decoded. TRBE doesn't support
30 * formatting the trace data, unlike the legacy CoreSight
31 * sinks and thus we use ETE trace packets to pad the
32 * sections of the buffer.
33 */
34#define ETE_IGNORE_PACKET 0x70
35
36/*
37 * Minimum amount of meaningful trace will contain:
38 * A-Sync, Trace Info, Trace On, Address, Atom.
39 * This is about 44bytes of ETE trace. To be on
40 * the safer side, we assume 64bytes is the minimum
41 * space required for a meaningful session, before
42 * we hit a "WRAP" event.
43 */
44#define TRBE_TRACE_MIN_BUF_SIZE 64
45
46enum trbe_fault_action {
47 TRBE_FAULT_ACT_WRAP,
48 TRBE_FAULT_ACT_SPURIOUS,
49 TRBE_FAULT_ACT_FATAL,
50};
51
52struct trbe_buf {
53 /*
54 * Even though trbe_base represents vmap()
55 * mapped allocated buffer's start address,
56 * it's being as unsigned long for various
57 * arithmetic and comparision operations &
58 * also to be consistent with trbe_write &
59 * trbe_limit sibling pointers.
60 */
61 unsigned long trbe_base;
62 /* The base programmed into the TRBE */
63 unsigned long trbe_hw_base;
64 unsigned long trbe_limit;
65 unsigned long trbe_write;
66 int nr_pages;
67 void **pages;
68 bool snapshot;
69 struct trbe_cpudata *cpudata;
70};
71
72/*
73 * TRBE erratum list
74 *
75 * The errata are defined in arm64 generic cpu_errata framework.
76 * Since the errata work arounds could be applied individually
77 * to the affected CPUs inside the TRBE driver, we need to know if
78 * a given CPU is affected by the erratum. Unlike the other erratum
79 * work arounds, TRBE driver needs to check multiple times during
80 * a trace session. Thus we need a quicker access to per-CPU
81 * errata and not issue costly this_cpu_has_cap() everytime.
82 * We keep a set of the affected errata in trbe_cpudata, per TRBE.
83 *
84 * We rely on the corresponding cpucaps to be defined for a given
85 * TRBE erratum. We map the given cpucap into a TRBE internal number
86 * to make the tracking of the errata lean.
87 *
88 * This helps in :
89 * - Not duplicating the detection logic
90 * - Streamlined detection of erratum across the system
91 */
92#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 0
93#define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE 1
94#define TRBE_NEEDS_DRAIN_AFTER_DISABLE 2
95#define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE 3
96#define TRBE_IS_BROKEN 4
97
98static int trbe_errata_cpucaps[] = {
99 [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
100 [TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
101 [TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,
102 [TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,
103 [TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,
104 -1, /* Sentinel, must be the last entry */
105};
106
107/* The total number of listed errata in trbe_errata_cpucaps */
108#define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1)
109
110/*
111 * Safe limit for the number of bytes that may be overwritten
112 * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.
113 */
114#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256
115
116/*
117 * struct trbe_cpudata: TRBE instance specific data
118 * @trbe_flag - TRBE dirty/access flag support
119 * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1.
120 * @trbe_align - Software alignment used for the TRBPTR_EL1.
121 * @cpu - CPU this TRBE belongs to.
122 * @mode - Mode of current operation. (perf/disabled)
123 * @drvdata - TRBE specific drvdata
124 * @errata - Bit map for the errata on this TRBE.
125 */
126struct trbe_cpudata {
127 bool trbe_flag;
128 u64 trbe_hw_align;
129 u64 trbe_align;
130 int cpu;
131 enum cs_mode mode;
132 struct trbe_buf *buf;
133 struct trbe_drvdata *drvdata;
134 DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
135};
136
137struct trbe_drvdata {
138 struct trbe_cpudata __percpu *cpudata;
139 struct perf_output_handle * __percpu *handle;
140 struct hlist_node hotplug_node;
141 int irq;
142 cpumask_t supported_cpus;
143 enum cpuhp_state trbe_online;
144 struct platform_device *pdev;
145};
146
147static void trbe_check_errata(struct trbe_cpudata *cpudata)
148{
149 int i;
150
151 for (i = 0; i < TRBE_ERRATA_MAX; i++) {
152 int cap = trbe_errata_cpucaps[i];
153
154 if (WARN_ON_ONCE(cap < 0))
155 return;
156 if (this_cpu_has_cap(cap))
157 set_bit(i, cpudata->errata);
158 }
159}
160
161static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
162{
163 return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
164}
165
166static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)
167{
168 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);
169}
170
171static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)
172{
173 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);
174}
175
176static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)
177{
178 /*
179 * Errata affected TRBE implementation will need TSB CSYNC and
180 * DSB in order to prevent subsequent writes into certain TRBE
181 * system registers from being ignored and not effected.
182 */
183 return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);
184}
185
186static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)
187{
188 /*
189 * Errata affected TRBE implementation will need an additional
190 * context synchronization in order to prevent an inconsistent
191 * TRBE prohibited region view on the CPU which could possibly
192 * corrupt the TRBE buffer or the TRBE state.
193 */
194 return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);
195}
196
197static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)
198{
199 return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);
200}
201
202static int trbe_alloc_node(struct perf_event *event)
203{
204 if (event->cpu == -1)
205 return NUMA_NO_NODE;
206 return cpu_to_node(event->cpu);
207}
208
209static inline void trbe_drain_buffer(void)
210{
211 tsb_csync();
212 dsb(nsh);
213}
214
215static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)
216{
217 /*
218 * Enable the TRBE without clearing LIMITPTR which
219 * might be required for fetching the buffer limits.
220 */
221 trblimitr |= TRBLIMITR_ENABLE;
222 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
223
224 /* Synchronize the TRBE enable event */
225 isb();
226
227 if (trbe_needs_ctxt_sync_after_enable(cpudata))
228 isb();
229}
230
231static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)
232{
233 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
234
235 /*
236 * Disable the TRBE without clearing LIMITPTR which
237 * might be required for fetching the buffer limits.
238 */
239 trblimitr &= ~TRBLIMITR_ENABLE;
240 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
241
242 if (trbe_needs_drain_after_disable(cpudata))
243 trbe_drain_buffer();
244 isb();
245}
246
247static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)
248{
249 trbe_drain_buffer();
250 set_trbe_disabled(cpudata);
251}
252
253static void trbe_reset_local(struct trbe_cpudata *cpudata)
254{
255 trbe_drain_and_disable_local(cpudata);
256 write_sysreg_s(0, SYS_TRBLIMITR_EL1);
257 write_sysreg_s(0, SYS_TRBPTR_EL1);
258 write_sysreg_s(0, SYS_TRBBASER_EL1);
259 write_sysreg_s(0, SYS_TRBSR_EL1);
260}
261
262static void trbe_report_wrap_event(struct perf_output_handle *handle)
263{
264 /*
265 * Mark the buffer to indicate that there was a WRAP event by
266 * setting the COLLISION flag. This indicates to the user that
267 * the TRBE trace collection was stopped without stopping the
268 * ETE and thus there might be some amount of trace that was
269 * lost between the time the WRAP was detected and the IRQ
270 * was consumed by the CPU.
271 *
272 * Setting the TRUNCATED flag would move the event to STOPPED
273 * state unnecessarily, even when there is space left in the
274 * ring buffer. Using the COLLISION flag doesn't have this side
275 * effect. We only set TRUNCATED flag when there is no space
276 * left in the ring buffer.
277 */
278 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
279}
280
281static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
282{
283 struct trbe_buf *buf = etm_perf_sink_config(handle);
284
285 /*
286 * We cannot proceed with the buffer collection and we
287 * do not have any data for the current session. The
288 * etm_perf driver expects to close out the aux_buffer
289 * at event_stop(). So disable the TRBE here and leave
290 * the update_buffer() to return a 0 size.
291 */
292 trbe_drain_and_disable_local(buf->cpudata);
293 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
294 perf_aux_output_end(handle, 0);
295 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
296}
297
298/*
299 * TRBE Buffer Management
300 *
301 * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
302 * it starts writing trace data from the write pointer onward till the limit pointer.
303 * When the write pointer reaches the address just before the limit pointer, it gets
304 * wrapped around again to the base pointer. This is called a TRBE wrap event, which
305 * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
306 * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
307 * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
308 * LIMIT pointers.
309 *
310 * Wrap around with an IRQ
311 * ------ < ------ < ------- < ----- < -----
312 * | |
313 * ------ > ------ > ------- > ----- > -----
314 *
315 * +---------------+-----------------------+
316 * | | |
317 * +---------------+-----------------------+
318 * Base Pointer Write Pointer Limit Pointer
319 *
320 * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
321 * pointer can be aligned to the implementation defined TRBE trace buffer alignment
322 * as captured in trbe_cpudata->trbe_align.
323 *
324 *
325 * head tail wakeup
326 * +---------------------------------------+----- ~ ~ ------
327 * |$$$$$$$|################|$$$$$$$$$$$$$$| |
328 * +---------------------------------------+----- ~ ~ ------
329 * Base Pointer Write Pointer Limit Pointer
330 *
331 * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
332 * values which tracks all the driver writes and user reads from the perf auxiliary
333 * buffer. Generally [head..tail] is the area where the driver can write into unless
334 * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
335 * configured depending on the perf_output_handle indices, so that the driver does
336 * not override into areas in the perf auxiliary buffer which is being or yet to be
337 * consumed from the user space. The enabled TRBE buffer area is a moving subset of
338 * the allocated perf auxiliary buffer.
339 */
340
341static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
342{
343 memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
344}
345
346static void trbe_pad_buf(struct perf_output_handle *handle, int len)
347{
348 struct trbe_buf *buf = etm_perf_sink_config(handle);
349 u64 head = PERF_IDX2OFF(handle->head, buf);
350
351 __trbe_pad_buf(buf, head, len);
352 if (!buf->snapshot)
353 perf_aux_output_skip(handle, len);
354}
355
356static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
357{
358 struct trbe_buf *buf = etm_perf_sink_config(handle);
359
360 /*
361 * The ETE trace has alignment synchronization packets allowing
362 * the decoder to reset in case of an overflow or corruption.
363 * So we can use the entire buffer for the snapshot mode.
364 */
365 return buf->nr_pages * PAGE_SIZE;
366}
367
368static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)
369{
370 u64 size = TRBE_TRACE_MIN_BUF_SIZE;
371 struct trbe_buf *buf = etm_perf_sink_config(handle);
372 struct trbe_cpudata *cpudata = buf->cpudata;
373
374 /*
375 * When the TRBE is affected by an erratum that could make it
376 * write to the next "virtually addressed" page beyond the LIMIT.
377 * We need to make sure there is always a PAGE after the LIMIT,
378 * within the buffer. Thus we ensure there is at least an extra
379 * page than normal. With this we could then adjust the LIMIT
380 * pointer down by a PAGE later.
381 */
382 if (trbe_may_write_out_of_range(cpudata))
383 size += PAGE_SIZE;
384 return size;
385}
386
387/*
388 * TRBE Limit Calculation
389 *
390 * The following markers are used to illustrate various TRBE buffer situations.
391 *
392 * $$$$ - Data area, unconsumed captured trace data, not to be overridden
393 * #### - Free area, enabled, trace will be written
394 * %%%% - Free area, disabled, trace will not be written
395 * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
396 */
397static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
398{
399 struct trbe_buf *buf = etm_perf_sink_config(handle);
400 struct trbe_cpudata *cpudata = buf->cpudata;
401 const u64 bufsize = buf->nr_pages * PAGE_SIZE;
402 u64 limit = bufsize;
403 u64 head, tail, wakeup;
404
405 head = PERF_IDX2OFF(handle->head, buf);
406
407 /*
408 * head
409 * ------->|
410 * |
411 * head TRBE align tail
412 * +----|-------|---------------|-------+
413 * |$$$$|=======|###############|$$$$$$$|
414 * +----|-------|---------------|-------+
415 * trbe_base trbe_base + nr_pages
416 *
417 * Perf aux buffer output head position can be misaligned depending on
418 * various factors including user space reads. In case misaligned, head
419 * needs to be aligned before TRBE can be configured. Pad the alignment
420 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
421 * and skip this section thus advancing the head.
422 */
423 if (!IS_ALIGNED(head, cpudata->trbe_align)) {
424 unsigned long delta = roundup(head, cpudata->trbe_align) - head;
425
426 delta = min(delta, handle->size);
427 trbe_pad_buf(handle, delta);
428 head = PERF_IDX2OFF(handle->head, buf);
429 }
430
431 /*
432 * head = tail (size = 0)
433 * +----|-------------------------------+
434 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ |
435 * +----|-------------------------------+
436 * trbe_base trbe_base + nr_pages
437 *
438 * Perf aux buffer does not have any space for the driver to write into.
439 */
440 if (!handle->size)
441 return 0;
442
443 /* Compute the tail and wakeup indices now that we've aligned head */
444 tail = PERF_IDX2OFF(handle->head + handle->size, buf);
445 wakeup = PERF_IDX2OFF(handle->wakeup, buf);
446
447 /*
448 * Lets calculate the buffer area which TRBE could write into. There
449 * are three possible scenarios here. Limit needs to be aligned with
450 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
451 * unconsumed data.
452 *
453 * 1) head < tail
454 *
455 * head tail
456 * +----|-----------------------|-------+
457 * |$$$$|#######################|$$$$$$$|
458 * +----|-----------------------|-------+
459 * trbe_base limit trbe_base + nr_pages
460 *
461 * TRBE could write into [head..tail] area. Unless the tail is right at
462 * the end of the buffer, neither an wrap around nor an IRQ is expected
463 * while being enabled.
464 *
465 * 2) head == tail
466 *
467 * head = tail (size > 0)
468 * +----|-------------------------------+
469 * |%%%%|###############################|
470 * +----|-------------------------------+
471 * trbe_base limit = trbe_base + nr_pages
472 *
473 * TRBE should just write into [head..base + nr_pages] area even though
474 * the entire buffer is empty. Reason being, when the trace reaches the
475 * end of the buffer, it will just wrap around with an IRQ giving an
476 * opportunity to reconfigure the buffer.
477 *
478 * 3) tail < head
479 *
480 * tail head
481 * +----|-----------------------|-------+
482 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
483 * +----|-----------------------|-------+
484 * trbe_base limit = trbe_base + nr_pages
485 *
486 * TRBE should just write into [head..base + nr_pages] area even though
487 * the [trbe_base..tail] is also empty. Reason being, when the trace
488 * reaches the end of the buffer, it will just wrap around with an IRQ
489 * giving an opportunity to reconfigure the buffer.
490 */
491 if (head < tail)
492 limit = round_down(tail, PAGE_SIZE);
493
494 /*
495 * Wakeup may be arbitrarily far into the future. If it's not in the
496 * current generation, either we'll wrap before hitting it, or it's
497 * in the past and has been handled already.
498 *
499 * If there's a wakeup before we wrap, arrange to be woken up by the
500 * page boundary following it. Keep the tail boundary if that's lower.
501 *
502 * head wakeup tail
503 * +----|---------------|-------|-------+
504 * |$$$$|###############|%%%%%%%|$$$$$$$|
505 * +----|---------------|-------|-------+
506 * trbe_base limit trbe_base + nr_pages
507 */
508 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
509 limit = min(limit, round_up(wakeup, PAGE_SIZE));
510
511 /*
512 * There are two situation when this can happen i.e limit is before
513 * the head and hence TRBE cannot be configured.
514 *
515 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
516 * within the same PAGE size range.
517 *
518 * PAGE_SIZE
519 * |----------------------|
520 *
521 * limit head tail
522 * +------------|------|--------|-------+
523 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
524 * +------------|------|--------|-------+
525 * trbe_base trbe_base + nr_pages
526 *
527 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
528 * head and wakeup are within same PAGE size range.
529 *
530 * PAGE_SIZE
531 * |----------------------|
532 *
533 * limit head wakeup tail
534 * +----|------|-------|--------|-------+
535 * |$$$$$$$$$$$|=======|========|$$$$$$$|
536 * +----|------|-------|--------|-------+
537 * trbe_base trbe_base + nr_pages
538 */
539 if (limit > head)
540 return limit;
541
542 trbe_pad_buf(handle, handle->size);
543 return 0;
544}
545
546static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
547{
548 struct trbe_buf *buf = etm_perf_sink_config(handle);
549 u64 limit = __trbe_normal_offset(handle);
550 u64 head = PERF_IDX2OFF(handle->head, buf);
551
552 /*
553 * If the head is too close to the limit and we don't
554 * have space for a meaningful run, we rather pad it
555 * and start fresh.
556 *
557 * We might have to do this more than once to make sure
558 * we have enough required space.
559 */
560 while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {
561 trbe_pad_buf(handle, limit - head);
562 limit = __trbe_normal_offset(handle);
563 head = PERF_IDX2OFF(handle->head, buf);
564 }
565 return limit;
566}
567
568static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
569{
570 struct trbe_buf *buf = etm_perf_sink_config(handle);
571 unsigned long offset;
572
573 if (buf->snapshot)
574 offset = trbe_snapshot_offset(handle);
575 else
576 offset = trbe_normal_offset(handle);
577 return buf->trbe_base + offset;
578}
579
580static void clr_trbe_status(void)
581{
582 u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
583
584 WARN_ON(is_trbe_enabled());
585 trbsr &= ~TRBSR_IRQ;
586 trbsr &= ~TRBSR_TRG;
587 trbsr &= ~TRBSR_WRAP;
588 trbsr &= ~(TRBSR_EC_MASK << TRBSR_EC_SHIFT);
589 trbsr &= ~(TRBSR_BSC_MASK << TRBSR_BSC_SHIFT);
590 trbsr &= ~TRBSR_STOP;
591 write_sysreg_s(trbsr, SYS_TRBSR_EL1);
592}
593
594static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
595{
596 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
597 unsigned long addr = buf->trbe_limit;
598
599 WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_LIMIT_SHIFT)));
600 WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
601
602 trblimitr &= ~TRBLIMITR_NVM;
603 trblimitr &= ~(TRBLIMITR_FILL_MODE_MASK << TRBLIMITR_FILL_MODE_SHIFT);
604 trblimitr &= ~(TRBLIMITR_TRIG_MODE_MASK << TRBLIMITR_TRIG_MODE_SHIFT);
605 trblimitr &= ~(TRBLIMITR_LIMIT_MASK << TRBLIMITR_LIMIT_SHIFT);
606
607 /*
608 * Fill trace buffer mode is used here while configuring the
609 * TRBE for trace capture. In this particular mode, the trace
610 * collection is stopped and a maintenance interrupt is raised
611 * when the current write pointer wraps. This pause in trace
612 * collection gives the software an opportunity to capture the
613 * trace data in the interrupt handler, before reconfiguring
614 * the TRBE.
615 */
616 trblimitr |= (TRBE_FILL_MODE_FILL & TRBLIMITR_FILL_MODE_MASK) << TRBLIMITR_FILL_MODE_SHIFT;
617
618 /*
619 * Trigger mode is not used here while configuring the TRBE for
620 * the trace capture. Hence just keep this in the ignore mode.
621 */
622 trblimitr |= (TRBE_TRIG_MODE_IGNORE & TRBLIMITR_TRIG_MODE_MASK) <<
623 TRBLIMITR_TRIG_MODE_SHIFT;
624 trblimitr |= (addr & PAGE_MASK);
625 set_trbe_enabled(buf->cpudata, trblimitr);
626}
627
628static void trbe_enable_hw(struct trbe_buf *buf)
629{
630 WARN_ON(buf->trbe_hw_base < buf->trbe_base);
631 WARN_ON(buf->trbe_write < buf->trbe_hw_base);
632 WARN_ON(buf->trbe_write >= buf->trbe_limit);
633 set_trbe_disabled(buf->cpudata);
634 clr_trbe_status();
635 set_trbe_base_pointer(buf->trbe_hw_base);
636 set_trbe_write_pointer(buf->trbe_write);
637
638 /*
639 * Synchronize all the register updates
640 * till now before enabling the TRBE.
641 */
642 isb();
643 set_trbe_limit_pointer_enabled(buf);
644}
645
646static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,
647 u64 trbsr)
648{
649 int ec = get_trbe_ec(trbsr);
650 int bsc = get_trbe_bsc(trbsr);
651 struct trbe_buf *buf = etm_perf_sink_config(handle);
652 struct trbe_cpudata *cpudata = buf->cpudata;
653
654 WARN_ON(is_trbe_running(trbsr));
655 if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
656 return TRBE_FAULT_ACT_FATAL;
657
658 if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
659 return TRBE_FAULT_ACT_FATAL;
660
661 /*
662 * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
663 * it might write data after a WRAP event in the fill mode.
664 * Thus the check TRBPTR == TRBBASER will not be honored.
665 */
666 if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&
667 (trbe_may_overwrite_in_fill_mode(cpudata) ||
668 get_trbe_write_pointer() == get_trbe_base_pointer()))
669 return TRBE_FAULT_ACT_WRAP;
670
671 return TRBE_FAULT_ACT_SPURIOUS;
672}
673
674static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
675 struct trbe_buf *buf, bool wrap)
676{
677 u64 write;
678 u64 start_off, end_off;
679 u64 size;
680 u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
681
682 /*
683 * If the TRBE has wrapped around the write pointer has
684 * wrapped and should be treated as limit.
685 *
686 * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,
687 * it may write upto 64bytes beyond the "LIMIT". The driver already
688 * keeps a valid page next to the LIMIT and we could potentially
689 * consume the trace data that may have been collected there. But we
690 * cannot be really sure it is available, and the TRBPTR may not
691 * indicate the same. Also, affected cores are also affected by another
692 * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus
693 * could potentially pad an entire PAGE_SIZE - 64bytes, to get those
694 * 64bytes. Thus we ignore the potential triggering of the erratum
695 * on WRAP and limit the data to LIMIT.
696 */
697 if (wrap)
698 write = get_trbe_limit_pointer();
699 else
700 write = get_trbe_write_pointer();
701
702 /*
703 * TRBE may use a different base address than the base
704 * of the ring buffer. Thus use the beginning of the ring
705 * buffer to compute the offsets.
706 */
707 end_off = write - buf->trbe_base;
708 start_off = PERF_IDX2OFF(handle->head, buf);
709
710 if (WARN_ON_ONCE(end_off < start_off))
711 return 0;
712
713 size = end_off - start_off;
714 /*
715 * If the TRBE is affected by the following erratum, we must fill
716 * the space we skipped with IGNORE packets. And we are always
717 * guaranteed to have at least a PAGE_SIZE space in the buffer.
718 */
719 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&
720 !WARN_ON(size < overwrite_skip))
721 __trbe_pad_buf(buf, start_off, overwrite_skip);
722
723 return size;
724}
725
726static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
727 struct perf_event *event, void **pages,
728 int nr_pages, bool snapshot)
729{
730 struct trbe_buf *buf;
731 struct page **pglist;
732 int i;
733
734 /*
735 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
736 * just a single page, there would not be any room left while writing
737 * into a partially filled TRBE buffer after the page size alignment.
738 * Hence restrict the minimum buffer size as two pages.
739 */
740 if (nr_pages < 2)
741 return NULL;
742
743 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));
744 if (!buf)
745 return ERR_PTR(-ENOMEM);
746
747 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
748 if (!pglist) {
749 kfree(buf);
750 return ERR_PTR(-ENOMEM);
751 }
752
753 for (i = 0; i < nr_pages; i++)
754 pglist[i] = virt_to_page(pages[i]);
755
756 buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
757 if (!buf->trbe_base) {
758 kfree(pglist);
759 kfree(buf);
760 return ERR_PTR(-ENOMEM);
761 }
762 buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
763 buf->trbe_write = buf->trbe_base;
764 buf->snapshot = snapshot;
765 buf->nr_pages = nr_pages;
766 buf->pages = pages;
767 kfree(pglist);
768 return buf;
769}
770
771static void arm_trbe_free_buffer(void *config)
772{
773 struct trbe_buf *buf = config;
774
775 vunmap((void *)buf->trbe_base);
776 kfree(buf);
777}
778
779static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
780 struct perf_output_handle *handle,
781 void *config)
782{
783 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
784 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
785 struct trbe_buf *buf = config;
786 enum trbe_fault_action act;
787 unsigned long size, status;
788 unsigned long flags;
789 bool wrap = false;
790
791 WARN_ON(buf->cpudata != cpudata);
792 WARN_ON(cpudata->cpu != smp_processor_id());
793 WARN_ON(cpudata->drvdata != drvdata);
794 if (cpudata->mode != CS_MODE_PERF)
795 return 0;
796
797 /*
798 * We are about to disable the TRBE. And this could in turn
799 * fill up the buffer triggering, an IRQ. This could be consumed
800 * by the PE asynchronously, causing a race here against
801 * the IRQ handler in closing out the handle. So, let us
802 * make sure the IRQ can't trigger while we are collecting
803 * the buffer. We also make sure that a WRAP event is handled
804 * accordingly.
805 */
806 local_irq_save(flags);
807
808 /*
809 * If the TRBE was disabled due to lack of space in the AUX buffer or a
810 * spurious fault, the driver leaves it disabled, truncating the buffer.
811 * Since the etm_perf driver expects to close out the AUX buffer, the
812 * driver skips it. Thus, just pass in 0 size here to indicate that the
813 * buffer was truncated.
814 */
815 if (!is_trbe_enabled()) {
816 size = 0;
817 goto done;
818 }
819 /*
820 * perf handle structure needs to be shared with the TRBE IRQ handler for
821 * capturing trace data and restarting the handle. There is a probability
822 * of an undefined reference based crash when etm event is being stopped
823 * while a TRBE IRQ also getting processed. This happens due the release
824 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
825 * the TRBE here will ensure that no IRQ could be generated when the perf
826 * handle gets freed in etm_event_stop().
827 */
828 trbe_drain_and_disable_local(cpudata);
829
830 /* Check if there is a pending interrupt and handle it here */
831 status = read_sysreg_s(SYS_TRBSR_EL1);
832 if (is_trbe_irq(status)) {
833
834 /*
835 * Now that we are handling the IRQ here, clear the IRQ
836 * from the status, to let the irq handler know that it
837 * is taken care of.
838 */
839 clr_trbe_irq();
840 isb();
841
842 act = trbe_get_fault_act(handle, status);
843 /*
844 * If this was not due to a WRAP event, we have some
845 * errors and as such buffer is empty.
846 */
847 if (act != TRBE_FAULT_ACT_WRAP) {
848 size = 0;
849 goto done;
850 }
851
852 trbe_report_wrap_event(handle);
853 wrap = true;
854 }
855
856 size = trbe_get_trace_size(handle, buf, wrap);
857
858done:
859 local_irq_restore(flags);
860
861 if (buf->snapshot)
862 handle->head += size;
863 return size;
864}
865
866
867static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)
868{
869 /*
870 * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache
871 * line size from the "TRBBASER_EL1" in the event of a "FILL".
872 * Thus, we could loose some amount of the trace at the base.
873 *
874 * Before Fix:
875 *
876 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
877 * | \/ /
878 * -------------------------------------------------------------
879 * | Pg0 | Pg1 | | | PgN |
880 * -------------------------------------------------------------
881 *
882 * In the normal course of action, we would set the TRBBASER to the
883 * beginning of the ring-buffer (normal-BASE). But with the erratum,
884 * the TRBE could overwrite the contents at the "normal-BASE", after
885 * hitting the "normal-LIMIT", since it doesn't stop as expected. And
886 * this is wrong. This could result in overwriting trace collected in
887 * one of the previous runs, being consumed by the user. So we must
888 * always make sure that the TRBBASER is within the region
889 * [head, head+size]. Note that TRBBASER must be PAGE aligned,
890 *
891 * After moving the BASE:
892 *
893 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
894 * | \/ /
895 * -------------------------------------------------------------
896 * | | |xyzdef. |.. tuvw| |
897 * -------------------------------------------------------------
898 * /
899 * New-BASER
900 *
901 * Also, we would set the TRBPTR to head (after adjusting for
902 * alignment) at normal-PTR. This would mean that the last few bytes
903 * of the trace (say, "xyz") might overwrite the first few bytes of
904 * trace written ("abc"). More importantly they will appear in what
905 * userspace sees as the beginning of the trace, which is wrong. We may
906 * not always have space to move the latest trace "xyz" to the correct
907 * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).
908 * Thus it is easier to ignore those bytes than to complicate the
909 * driver to move it, assuming that the erratum was triggered and
910 * doing additional checks to see if there is indeed allowed space at
911 * TRBLIMITR.LIMIT.
912 *
913 * Thus the full workaround will move the BASE and the PTR and would
914 * look like (after padding at the skipped bytes at the end of
915 * session) :
916 *
917 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
918 * | \/ /
919 * -------------------------------------------------------------
920 * | | |///abc.. |.. rst| |
921 * -------------------------------------------------------------
922 * / |
923 * New-BASER New-TRBPTR
924 *
925 * To summarize, with the work around:
926 *
927 * - We always align the offset for the next session to PAGE_SIZE
928 * (This is to ensure we can program the TRBBASER to this offset
929 * within the region [head...head+size]).
930 *
931 * - At TRBE enable:
932 * - Set the TRBBASER to the page aligned offset of the current
933 * proposed write offset. (which is guaranteed to be aligned
934 * as above)
935 * - Move the TRBPTR to skip first 256bytes (that might be
936 * overwritten with the erratum). This ensures that the trace
937 * generated in the session is not re-written.
938 *
939 * - At trace collection:
940 * - Pad the 256bytes skipped above again with IGNORE packets.
941 */
942 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {
943 if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))
944 return -EINVAL;
945 buf->trbe_hw_base = buf->trbe_write;
946 buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
947 }
948
949 /*
950 * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to
951 * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"
952 * may be:
953 * - The page beyond the ring buffer. This could mean, TRBE could
954 * corrupt another entity (kernel / user)
955 * - A portion of the "ring buffer" consumed by the userspace.
956 * i.e, a page outisde [head, head + size].
957 *
958 * We work around this by:
959 * - Making sure that we have at least an extra space of PAGE left
960 * in the ring buffer [head, head + size], than we normally do
961 * without the erratum. See trbe_min_trace_buf_size().
962 *
963 * - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside
964 * the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).
965 */
966 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {
967 s64 space = buf->trbe_limit - buf->trbe_write;
968 /*
969 * We must have more than a PAGE_SIZE worth space in the proposed
970 * range for the TRBE.
971 */
972 if (WARN_ON(space <= PAGE_SIZE ||
973 !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))
974 return -EINVAL;
975 buf->trbe_limit -= PAGE_SIZE;
976 }
977
978 return 0;
979}
980
981static int __arm_trbe_enable(struct trbe_buf *buf,
982 struct perf_output_handle *handle)
983{
984 int ret = 0;
985
986 perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
987 buf->trbe_limit = compute_trbe_buffer_limit(handle);
988 buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
989 if (buf->trbe_limit == buf->trbe_base) {
990 ret = -ENOSPC;
991 goto err;
992 }
993 /* Set the base of the TRBE to the buffer base */
994 buf->trbe_hw_base = buf->trbe_base;
995
996 ret = trbe_apply_work_around_before_enable(buf);
997 if (ret)
998 goto err;
999
1000 *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
1001 trbe_enable_hw(buf);
1002 return 0;
1003err:
1004 trbe_stop_and_truncate_event(handle);
1005 return ret;
1006}
1007
1008static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data)
1009{
1010 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1011 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1012 struct perf_output_handle *handle = data;
1013 struct trbe_buf *buf = etm_perf_sink_config(handle);
1014
1015 WARN_ON(cpudata->cpu != smp_processor_id());
1016 WARN_ON(cpudata->drvdata != drvdata);
1017 if (mode != CS_MODE_PERF)
1018 return -EINVAL;
1019
1020 cpudata->buf = buf;
1021 cpudata->mode = mode;
1022 buf->cpudata = cpudata;
1023
1024 return __arm_trbe_enable(buf, handle);
1025}
1026
1027static int arm_trbe_disable(struct coresight_device *csdev)
1028{
1029 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1030 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1031 struct trbe_buf *buf = cpudata->buf;
1032
1033 WARN_ON(buf->cpudata != cpudata);
1034 WARN_ON(cpudata->cpu != smp_processor_id());
1035 WARN_ON(cpudata->drvdata != drvdata);
1036 if (cpudata->mode != CS_MODE_PERF)
1037 return -EINVAL;
1038
1039 trbe_drain_and_disable_local(cpudata);
1040 buf->cpudata = NULL;
1041 cpudata->buf = NULL;
1042 cpudata->mode = CS_MODE_DISABLED;
1043 return 0;
1044}
1045
1046static void trbe_handle_spurious(struct perf_output_handle *handle)
1047{
1048 struct trbe_buf *buf = etm_perf_sink_config(handle);
1049 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
1050
1051 /*
1052 * If the IRQ was spurious, simply re-enable the TRBE
1053 * back without modifying the buffer parameters to
1054 * retain the trace collected so far.
1055 */
1056 set_trbe_enabled(buf->cpudata, trblimitr);
1057}
1058
1059static int trbe_handle_overflow(struct perf_output_handle *handle)
1060{
1061 struct perf_event *event = handle->event;
1062 struct trbe_buf *buf = etm_perf_sink_config(handle);
1063 unsigned long size;
1064 struct etm_event_data *event_data;
1065
1066 size = trbe_get_trace_size(handle, buf, true);
1067 if (buf->snapshot)
1068 handle->head += size;
1069
1070 trbe_report_wrap_event(handle);
1071 perf_aux_output_end(handle, size);
1072 event_data = perf_aux_output_begin(handle, event);
1073 if (!event_data) {
1074 /*
1075 * We are unable to restart the trace collection,
1076 * thus leave the TRBE disabled. The etm-perf driver
1077 * is able to detect this with a disconnected handle
1078 * (handle->event = NULL).
1079 */
1080 trbe_drain_and_disable_local(buf->cpudata);
1081 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
1082 return -EINVAL;
1083 }
1084
1085 return __arm_trbe_enable(buf, handle);
1086}
1087
1088static bool is_perf_trbe(struct perf_output_handle *handle)
1089{
1090 struct trbe_buf *buf = etm_perf_sink_config(handle);
1091 struct trbe_cpudata *cpudata = buf->cpudata;
1092 struct trbe_drvdata *drvdata = cpudata->drvdata;
1093 int cpu = smp_processor_id();
1094
1095 WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
1096 WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
1097
1098 if (cpudata->mode != CS_MODE_PERF)
1099 return false;
1100
1101 if (cpudata->cpu != cpu)
1102 return false;
1103
1104 if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1105 return false;
1106
1107 return true;
1108}
1109
1110static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
1111{
1112 struct perf_output_handle **handle_ptr = dev;
1113 struct perf_output_handle *handle = *handle_ptr;
1114 struct trbe_buf *buf = etm_perf_sink_config(handle);
1115 enum trbe_fault_action act;
1116 u64 status;
1117 bool truncated = false;
1118 u64 trfcr;
1119
1120 /* Reads to TRBSR_EL1 is fine when TRBE is active */
1121 status = read_sysreg_s(SYS_TRBSR_EL1);
1122 /*
1123 * If the pending IRQ was handled by update_buffer callback
1124 * we have nothing to do here.
1125 */
1126 if (!is_trbe_irq(status))
1127 return IRQ_NONE;
1128
1129 /* Prohibit the CPU from tracing before we disable the TRBE */
1130 trfcr = cpu_prohibit_trace();
1131 /*
1132 * Ensure the trace is visible to the CPUs and
1133 * any external aborts have been resolved.
1134 */
1135 trbe_drain_and_disable_local(buf->cpudata);
1136 clr_trbe_irq();
1137 isb();
1138
1139 if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
1140 return IRQ_NONE;
1141
1142 if (!is_perf_trbe(handle))
1143 return IRQ_NONE;
1144
1145 act = trbe_get_fault_act(handle, status);
1146 switch (act) {
1147 case TRBE_FAULT_ACT_WRAP:
1148 truncated = !!trbe_handle_overflow(handle);
1149 break;
1150 case TRBE_FAULT_ACT_SPURIOUS:
1151 trbe_handle_spurious(handle);
1152 break;
1153 case TRBE_FAULT_ACT_FATAL:
1154 trbe_stop_and_truncate_event(handle);
1155 truncated = true;
1156 break;
1157 }
1158
1159 /*
1160 * If the buffer was truncated, ensure perf callbacks
1161 * have completed, which will disable the event.
1162 *
1163 * Otherwise, restore the trace filter controls to
1164 * allow the tracing.
1165 */
1166 if (truncated)
1167 irq_work_run();
1168 else
1169 write_trfcr(trfcr);
1170
1171 return IRQ_HANDLED;
1172}
1173
1174static const struct coresight_ops_sink arm_trbe_sink_ops = {
1175 .enable = arm_trbe_enable,
1176 .disable = arm_trbe_disable,
1177 .alloc_buffer = arm_trbe_alloc_buffer,
1178 .free_buffer = arm_trbe_free_buffer,
1179 .update_buffer = arm_trbe_update_buffer,
1180};
1181
1182static const struct coresight_ops arm_trbe_cs_ops = {
1183 .sink_ops = &arm_trbe_sink_ops,
1184};
1185
1186static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
1187{
1188 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1189
1190 return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);
1191}
1192static DEVICE_ATTR_RO(align);
1193
1194static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
1195{
1196 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1197
1198 return sprintf(buf, "%d\n", cpudata->trbe_flag);
1199}
1200static DEVICE_ATTR_RO(flag);
1201
1202static struct attribute *arm_trbe_attrs[] = {
1203 &dev_attr_align.attr,
1204 &dev_attr_flag.attr,
1205 NULL,
1206};
1207
1208static const struct attribute_group arm_trbe_group = {
1209 .attrs = arm_trbe_attrs,
1210};
1211
1212static const struct attribute_group *arm_trbe_groups[] = {
1213 &arm_trbe_group,
1214 NULL,
1215};
1216
1217static void arm_trbe_enable_cpu(void *info)
1218{
1219 struct trbe_drvdata *drvdata = info;
1220 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1221
1222 trbe_reset_local(cpudata);
1223 enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);
1224}
1225
1226static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1227{
1228 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1229 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1230 struct coresight_desc desc = { 0 };
1231 struct device *dev;
1232
1233 if (WARN_ON(trbe_csdev))
1234 return;
1235
1236 /* If the TRBE was not probed on the CPU, we shouldn't be here */
1237 if (WARN_ON(!cpudata->drvdata))
1238 return;
1239
1240 dev = &cpudata->drvdata->pdev->dev;
1241 desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
1242 if (!desc.name)
1243 goto cpu_clear;
1244
1245 desc.type = CORESIGHT_DEV_TYPE_SINK;
1246 desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
1247 desc.ops = &arm_trbe_cs_ops;
1248 desc.pdata = dev_get_platdata(dev);
1249 desc.groups = arm_trbe_groups;
1250 desc.dev = dev;
1251 trbe_csdev = coresight_register(&desc);
1252 if (IS_ERR(trbe_csdev))
1253 goto cpu_clear;
1254
1255 dev_set_drvdata(&trbe_csdev->dev, cpudata);
1256 coresight_set_percpu_sink(cpu, trbe_csdev);
1257 return;
1258cpu_clear:
1259 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1260}
1261
1262/*
1263 * Must be called with preemption disabled, for trbe_check_errata().
1264 */
1265static void arm_trbe_probe_cpu(void *info)
1266{
1267 struct trbe_drvdata *drvdata = info;
1268 int cpu = smp_processor_id();
1269 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1270 u64 trbidr;
1271
1272 if (WARN_ON(!cpudata))
1273 goto cpu_clear;
1274
1275 if (!is_trbe_available()) {
1276 pr_err("TRBE is not implemented on cpu %d\n", cpu);
1277 goto cpu_clear;
1278 }
1279
1280 trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
1281 if (!is_trbe_programmable(trbidr)) {
1282 pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
1283 goto cpu_clear;
1284 }
1285
1286 cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
1287 if (cpudata->trbe_hw_align > SZ_2K) {
1288 pr_err("Unsupported alignment on cpu %d\n", cpu);
1289 goto cpu_clear;
1290 }
1291
1292 /*
1293 * Run the TRBE erratum checks, now that we know
1294 * this instance is about to be registered.
1295 */
1296 trbe_check_errata(cpudata);
1297
1298 if (trbe_is_broken(cpudata)) {
1299 pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);
1300 goto cpu_clear;
1301 }
1302
1303 /*
1304 * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
1305 * we must always program the TBRPTR_EL1, 256bytes from a page
1306 * boundary, with TRBBASER_EL1 set to the page, to prevent
1307 * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.
1308 *
1309 * Thus make sure we always align our write pointer to a PAGE_SIZE,
1310 * which also guarantees that we have at least a PAGE_SIZE space in
1311 * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip
1312 * the required bytes at the base.
1313 */
1314 if (trbe_may_overwrite_in_fill_mode(cpudata))
1315 cpudata->trbe_align = PAGE_SIZE;
1316 else
1317 cpudata->trbe_align = cpudata->trbe_hw_align;
1318
1319 cpudata->trbe_flag = get_trbe_flag_update(trbidr);
1320 cpudata->cpu = cpu;
1321 cpudata->drvdata = drvdata;
1322 return;
1323cpu_clear:
1324 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1325}
1326
1327static void arm_trbe_remove_coresight_cpu(void *info)
1328{
1329 int cpu = smp_processor_id();
1330 struct trbe_drvdata *drvdata = info;
1331 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1332 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1333
1334 disable_percpu_irq(drvdata->irq);
1335 trbe_reset_local(cpudata);
1336 if (trbe_csdev) {
1337 coresight_unregister(trbe_csdev);
1338 cpudata->drvdata = NULL;
1339 coresight_set_percpu_sink(cpu, NULL);
1340 }
1341}
1342
1343static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
1344{
1345 int cpu;
1346
1347 drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
1348 if (!drvdata->cpudata)
1349 return -ENOMEM;
1350
1351 for_each_cpu(cpu, &drvdata->supported_cpus) {
1352 /* If we fail to probe the CPU, let us defer it to hotplug callbacks */
1353 if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
1354 continue;
1355 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1356 arm_trbe_register_coresight_cpu(drvdata, cpu);
1357 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1358 smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);
1359 }
1360 return 0;
1361}
1362
1363static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
1364{
1365 int cpu;
1366
1367 for_each_cpu(cpu, &drvdata->supported_cpus)
1368 smp_call_function_single(cpu, arm_trbe_remove_coresight_cpu, drvdata, 1);
1369 free_percpu(drvdata->cpudata);
1370 return 0;
1371}
1372
1373static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
1374{
1375 preempt_disable();
1376 arm_trbe_probe_cpu(drvdata);
1377 preempt_enable();
1378}
1379
1380static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
1381{
1382 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1383
1384 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1385
1386 /*
1387 * If this CPU was not probed for TRBE,
1388 * initialize it now.
1389 */
1390 if (!coresight_get_percpu_sink(cpu)) {
1391 arm_trbe_probe_hotplugged_cpu(drvdata);
1392 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1393 arm_trbe_register_coresight_cpu(drvdata, cpu);
1394 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1395 arm_trbe_enable_cpu(drvdata);
1396 } else {
1397 arm_trbe_enable_cpu(drvdata);
1398 }
1399 }
1400 return 0;
1401}
1402
1403static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1404{
1405 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1406
1407 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1408 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1409
1410 disable_percpu_irq(drvdata->irq);
1411 trbe_reset_local(cpudata);
1412 }
1413 return 0;
1414}
1415
1416static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
1417{
1418 enum cpuhp_state trbe_online;
1419 int ret;
1420
1421 trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
1422 arm_trbe_cpu_startup, arm_trbe_cpu_teardown);
1423 if (trbe_online < 0)
1424 return trbe_online;
1425
1426 ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);
1427 if (ret) {
1428 cpuhp_remove_multi_state(trbe_online);
1429 return ret;
1430 }
1431 drvdata->trbe_online = trbe_online;
1432 return 0;
1433}
1434
1435static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
1436{
1437 cpuhp_state_remove_instance(drvdata->trbe_online, &drvdata->hotplug_node);
1438 cpuhp_remove_multi_state(drvdata->trbe_online);
1439}
1440
1441static int arm_trbe_probe_irq(struct platform_device *pdev,
1442 struct trbe_drvdata *drvdata)
1443{
1444 int ret;
1445
1446 drvdata->irq = platform_get_irq(pdev, 0);
1447 if (drvdata->irq < 0) {
1448 pr_err("IRQ not found for the platform device\n");
1449 return drvdata->irq;
1450 }
1451
1452 if (!irq_is_percpu(drvdata->irq)) {
1453 pr_err("IRQ is not a PPI\n");
1454 return -EINVAL;
1455 }
1456
1457 if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))
1458 return -EINVAL;
1459
1460 drvdata->handle = alloc_percpu(struct perf_output_handle *);
1461 if (!drvdata->handle)
1462 return -ENOMEM;
1463
1464 ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);
1465 if (ret) {
1466 free_percpu(drvdata->handle);
1467 return ret;
1468 }
1469 return 0;
1470}
1471
1472static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
1473{
1474 free_percpu_irq(drvdata->irq, drvdata->handle);
1475 free_percpu(drvdata->handle);
1476}
1477
1478static int arm_trbe_device_probe(struct platform_device *pdev)
1479{
1480 struct coresight_platform_data *pdata;
1481 struct trbe_drvdata *drvdata;
1482 struct device *dev = &pdev->dev;
1483 int ret;
1484
1485 /* Trace capture is not possible with kernel page table isolation */
1486 if (arm64_kernel_unmapped_at_el0()) {
1487 pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
1488 return -EOPNOTSUPP;
1489 }
1490
1491 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1492 if (!drvdata)
1493 return -ENOMEM;
1494
1495 pdata = coresight_get_platform_data(dev);
1496 if (IS_ERR(pdata))
1497 return PTR_ERR(pdata);
1498
1499 dev_set_drvdata(dev, drvdata);
1500 dev->platform_data = pdata;
1501 drvdata->pdev = pdev;
1502 ret = arm_trbe_probe_irq(pdev, drvdata);
1503 if (ret)
1504 return ret;
1505
1506 ret = arm_trbe_probe_coresight(drvdata);
1507 if (ret)
1508 goto probe_failed;
1509
1510 ret = arm_trbe_probe_cpuhp(drvdata);
1511 if (ret)
1512 goto cpuhp_failed;
1513
1514 return 0;
1515cpuhp_failed:
1516 arm_trbe_remove_coresight(drvdata);
1517probe_failed:
1518 arm_trbe_remove_irq(drvdata);
1519 return ret;
1520}
1521
1522static int arm_trbe_device_remove(struct platform_device *pdev)
1523{
1524 struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
1525
1526 arm_trbe_remove_cpuhp(drvdata);
1527 arm_trbe_remove_coresight(drvdata);
1528 arm_trbe_remove_irq(drvdata);
1529 return 0;
1530}
1531
1532static const struct of_device_id arm_trbe_of_match[] = {
1533 { .compatible = "arm,trace-buffer-extension"},
1534 {},
1535};
1536MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
1537
1538static struct platform_driver arm_trbe_driver = {
1539 .driver = {
1540 .name = DRVNAME,
1541 .of_match_table = of_match_ptr(arm_trbe_of_match),
1542 .suppress_bind_attrs = true,
1543 },
1544 .probe = arm_trbe_device_probe,
1545 .remove = arm_trbe_device_remove,
1546};
1547
1548static int __init arm_trbe_init(void)
1549{
1550 int ret;
1551
1552 ret = platform_driver_register(&arm_trbe_driver);
1553 if (!ret)
1554 return 0;
1555
1556 pr_err("Error registering %s platform driver\n", DRVNAME);
1557 return ret;
1558}
1559
1560static void __exit arm_trbe_exit(void)
1561{
1562 platform_driver_unregister(&arm_trbe_driver);
1563}
1564module_init(arm_trbe_init);
1565module_exit(arm_trbe_exit);
1566
1567MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
1568MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
1569MODULE_LICENSE("GPL v2");
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
4 * sink device could then pair with an appropriate per-cpu coresight source
5 * device (ETE) thus generating required trace data. Trace can be enabled
6 * via the perf framework.
7 *
8 * The AUX buffer handling is inspired from Arm SPE PMU driver.
9 *
10 * Copyright (C) 2020 ARM Ltd.
11 *
12 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
13 */
14#define DRVNAME "arm_trbe"
15
16#define pr_fmt(fmt) DRVNAME ": " fmt
17
18#include <asm/barrier.h>
19#include <asm/cpufeature.h>
20#include <linux/vmalloc.h>
21
22#include "coresight-self-hosted-trace.h"
23#include "coresight-trbe.h"
24
25#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
26
27/*
28 * A padding packet that will help the user space tools
29 * in skipping relevant sections in the captured trace
30 * data which could not be decoded. TRBE doesn't support
31 * formatting the trace data, unlike the legacy CoreSight
32 * sinks and thus we use ETE trace packets to pad the
33 * sections of the buffer.
34 */
35#define ETE_IGNORE_PACKET 0x70
36
37/*
38 * Minimum amount of meaningful trace will contain:
39 * A-Sync, Trace Info, Trace On, Address, Atom.
40 * This is about 44bytes of ETE trace. To be on
41 * the safer side, we assume 64bytes is the minimum
42 * space required for a meaningful session, before
43 * we hit a "WRAP" event.
44 */
45#define TRBE_TRACE_MIN_BUF_SIZE 64
46
47enum trbe_fault_action {
48 TRBE_FAULT_ACT_WRAP,
49 TRBE_FAULT_ACT_SPURIOUS,
50 TRBE_FAULT_ACT_FATAL,
51};
52
53struct trbe_buf {
54 /*
55 * Even though trbe_base represents vmap()
56 * mapped allocated buffer's start address,
57 * it's being as unsigned long for various
58 * arithmetic and comparision operations &
59 * also to be consistent with trbe_write &
60 * trbe_limit sibling pointers.
61 */
62 unsigned long trbe_base;
63 /* The base programmed into the TRBE */
64 unsigned long trbe_hw_base;
65 unsigned long trbe_limit;
66 unsigned long trbe_write;
67 int nr_pages;
68 void **pages;
69 bool snapshot;
70 struct trbe_cpudata *cpudata;
71};
72
73/*
74 * TRBE erratum list
75 *
76 * The errata are defined in arm64 generic cpu_errata framework.
77 * Since the errata work arounds could be applied individually
78 * to the affected CPUs inside the TRBE driver, we need to know if
79 * a given CPU is affected by the erratum. Unlike the other erratum
80 * work arounds, TRBE driver needs to check multiple times during
81 * a trace session. Thus we need a quicker access to per-CPU
82 * errata and not issue costly this_cpu_has_cap() everytime.
83 * We keep a set of the affected errata in trbe_cpudata, per TRBE.
84 *
85 * We rely on the corresponding cpucaps to be defined for a given
86 * TRBE erratum. We map the given cpucap into a TRBE internal number
87 * to make the tracking of the errata lean.
88 *
89 * This helps in :
90 * - Not duplicating the detection logic
91 * - Streamlined detection of erratum across the system
92 */
93#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 0
94#define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE 1
95#define TRBE_NEEDS_DRAIN_AFTER_DISABLE 2
96#define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE 3
97#define TRBE_IS_BROKEN 4
98
99static int trbe_errata_cpucaps[] = {
100 [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
101 [TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
102 [TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,
103 [TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,
104 [TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,
105 -1, /* Sentinel, must be the last entry */
106};
107
108/* The total number of listed errata in trbe_errata_cpucaps */
109#define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1)
110
111/*
112 * Safe limit for the number of bytes that may be overwritten
113 * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.
114 */
115#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256
116
117/*
118 * struct trbe_cpudata: TRBE instance specific data
119 * @trbe_flag - TRBE dirty/access flag support
120 * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1.
121 * @trbe_align - Software alignment used for the TRBPTR_EL1.
122 * @cpu - CPU this TRBE belongs to.
123 * @mode - Mode of current operation. (perf/disabled)
124 * @drvdata - TRBE specific drvdata
125 * @errata - Bit map for the errata on this TRBE.
126 */
127struct trbe_cpudata {
128 bool trbe_flag;
129 u64 trbe_hw_align;
130 u64 trbe_align;
131 int cpu;
132 enum cs_mode mode;
133 struct trbe_buf *buf;
134 struct trbe_drvdata *drvdata;
135 DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
136};
137
138struct trbe_drvdata {
139 struct trbe_cpudata __percpu *cpudata;
140 struct perf_output_handle * __percpu *handle;
141 struct hlist_node hotplug_node;
142 int irq;
143 cpumask_t supported_cpus;
144 enum cpuhp_state trbe_online;
145 struct platform_device *pdev;
146};
147
148static void trbe_check_errata(struct trbe_cpudata *cpudata)
149{
150 int i;
151
152 for (i = 0; i < TRBE_ERRATA_MAX; i++) {
153 int cap = trbe_errata_cpucaps[i];
154
155 if (WARN_ON_ONCE(cap < 0))
156 return;
157 if (this_cpu_has_cap(cap))
158 set_bit(i, cpudata->errata);
159 }
160}
161
162static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
163{
164 return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
165}
166
167static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)
168{
169 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);
170}
171
172static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)
173{
174 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);
175}
176
177static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)
178{
179 /*
180 * Errata affected TRBE implementation will need TSB CSYNC and
181 * DSB in order to prevent subsequent writes into certain TRBE
182 * system registers from being ignored and not effected.
183 */
184 return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);
185}
186
187static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)
188{
189 /*
190 * Errata affected TRBE implementation will need an additional
191 * context synchronization in order to prevent an inconsistent
192 * TRBE prohibited region view on the CPU which could possibly
193 * corrupt the TRBE buffer or the TRBE state.
194 */
195 return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);
196}
197
198static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)
199{
200 return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);
201}
202
203static int trbe_alloc_node(struct perf_event *event)
204{
205 if (event->cpu == -1)
206 return NUMA_NO_NODE;
207 return cpu_to_node(event->cpu);
208}
209
210static inline void trbe_drain_buffer(void)
211{
212 tsb_csync();
213 dsb(nsh);
214}
215
216static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)
217{
218 /*
219 * Enable the TRBE without clearing LIMITPTR which
220 * might be required for fetching the buffer limits.
221 */
222 trblimitr |= TRBLIMITR_EL1_E;
223 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
224
225 /* Synchronize the TRBE enable event */
226 isb();
227
228 if (trbe_needs_ctxt_sync_after_enable(cpudata))
229 isb();
230}
231
232static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)
233{
234 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
235
236 /*
237 * Disable the TRBE without clearing LIMITPTR which
238 * might be required for fetching the buffer limits.
239 */
240 trblimitr &= ~TRBLIMITR_EL1_E;
241 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
242
243 if (trbe_needs_drain_after_disable(cpudata))
244 trbe_drain_buffer();
245 isb();
246}
247
248static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)
249{
250 trbe_drain_buffer();
251 set_trbe_disabled(cpudata);
252}
253
254static void trbe_reset_local(struct trbe_cpudata *cpudata)
255{
256 trbe_drain_and_disable_local(cpudata);
257 write_sysreg_s(0, SYS_TRBLIMITR_EL1);
258 write_sysreg_s(0, SYS_TRBPTR_EL1);
259 write_sysreg_s(0, SYS_TRBBASER_EL1);
260 write_sysreg_s(0, SYS_TRBSR_EL1);
261}
262
263static void trbe_report_wrap_event(struct perf_output_handle *handle)
264{
265 /*
266 * Mark the buffer to indicate that there was a WRAP event by
267 * setting the COLLISION flag. This indicates to the user that
268 * the TRBE trace collection was stopped without stopping the
269 * ETE and thus there might be some amount of trace that was
270 * lost between the time the WRAP was detected and the IRQ
271 * was consumed by the CPU.
272 *
273 * Setting the TRUNCATED flag would move the event to STOPPED
274 * state unnecessarily, even when there is space left in the
275 * ring buffer. Using the COLLISION flag doesn't have this side
276 * effect. We only set TRUNCATED flag when there is no space
277 * left in the ring buffer.
278 */
279 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
280}
281
282static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
283{
284 struct trbe_buf *buf = etm_perf_sink_config(handle);
285
286 /*
287 * We cannot proceed with the buffer collection and we
288 * do not have any data for the current session. The
289 * etm_perf driver expects to close out the aux_buffer
290 * at event_stop(). So disable the TRBE here and leave
291 * the update_buffer() to return a 0 size.
292 */
293 trbe_drain_and_disable_local(buf->cpudata);
294 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
295 perf_aux_output_end(handle, 0);
296 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
297}
298
299/*
300 * TRBE Buffer Management
301 *
302 * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
303 * it starts writing trace data from the write pointer onward till the limit pointer.
304 * When the write pointer reaches the address just before the limit pointer, it gets
305 * wrapped around again to the base pointer. This is called a TRBE wrap event, which
306 * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
307 * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
308 * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
309 * LIMIT pointers.
310 *
311 * Wrap around with an IRQ
312 * ------ < ------ < ------- < ----- < -----
313 * | |
314 * ------ > ------ > ------- > ----- > -----
315 *
316 * +---------------+-----------------------+
317 * | | |
318 * +---------------+-----------------------+
319 * Base Pointer Write Pointer Limit Pointer
320 *
321 * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
322 * pointer can be aligned to the implementation defined TRBE trace buffer alignment
323 * as captured in trbe_cpudata->trbe_align.
324 *
325 *
326 * head tail wakeup
327 * +---------------------------------------+----- ~ ~ ------
328 * |$$$$$$$|################|$$$$$$$$$$$$$$| |
329 * +---------------------------------------+----- ~ ~ ------
330 * Base Pointer Write Pointer Limit Pointer
331 *
332 * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
333 * values which tracks all the driver writes and user reads from the perf auxiliary
334 * buffer. Generally [head..tail] is the area where the driver can write into unless
335 * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
336 * configured depending on the perf_output_handle indices, so that the driver does
337 * not override into areas in the perf auxiliary buffer which is being or yet to be
338 * consumed from the user space. The enabled TRBE buffer area is a moving subset of
339 * the allocated perf auxiliary buffer.
340 */
341
342static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
343{
344 memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
345}
346
347static void trbe_pad_buf(struct perf_output_handle *handle, int len)
348{
349 struct trbe_buf *buf = etm_perf_sink_config(handle);
350 u64 head = PERF_IDX2OFF(handle->head, buf);
351
352 __trbe_pad_buf(buf, head, len);
353 if (!buf->snapshot)
354 perf_aux_output_skip(handle, len);
355}
356
357static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
358{
359 struct trbe_buf *buf = etm_perf_sink_config(handle);
360
361 /*
362 * The ETE trace has alignment synchronization packets allowing
363 * the decoder to reset in case of an overflow or corruption.
364 * So we can use the entire buffer for the snapshot mode.
365 */
366 return buf->nr_pages * PAGE_SIZE;
367}
368
369static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)
370{
371 u64 size = TRBE_TRACE_MIN_BUF_SIZE;
372 struct trbe_buf *buf = etm_perf_sink_config(handle);
373 struct trbe_cpudata *cpudata = buf->cpudata;
374
375 /*
376 * When the TRBE is affected by an erratum that could make it
377 * write to the next "virtually addressed" page beyond the LIMIT.
378 * We need to make sure there is always a PAGE after the LIMIT,
379 * within the buffer. Thus we ensure there is at least an extra
380 * page than normal. With this we could then adjust the LIMIT
381 * pointer down by a PAGE later.
382 */
383 if (trbe_may_write_out_of_range(cpudata))
384 size += PAGE_SIZE;
385 return size;
386}
387
388/*
389 * TRBE Limit Calculation
390 *
391 * The following markers are used to illustrate various TRBE buffer situations.
392 *
393 * $$$$ - Data area, unconsumed captured trace data, not to be overridden
394 * #### - Free area, enabled, trace will be written
395 * %%%% - Free area, disabled, trace will not be written
396 * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
397 */
398static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
399{
400 struct trbe_buf *buf = etm_perf_sink_config(handle);
401 struct trbe_cpudata *cpudata = buf->cpudata;
402 const u64 bufsize = buf->nr_pages * PAGE_SIZE;
403 u64 limit = bufsize;
404 u64 head, tail, wakeup;
405
406 head = PERF_IDX2OFF(handle->head, buf);
407
408 /*
409 * head
410 * ------->|
411 * |
412 * head TRBE align tail
413 * +----|-------|---------------|-------+
414 * |$$$$|=======|###############|$$$$$$$|
415 * +----|-------|---------------|-------+
416 * trbe_base trbe_base + nr_pages
417 *
418 * Perf aux buffer output head position can be misaligned depending on
419 * various factors including user space reads. In case misaligned, head
420 * needs to be aligned before TRBE can be configured. Pad the alignment
421 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
422 * and skip this section thus advancing the head.
423 */
424 if (!IS_ALIGNED(head, cpudata->trbe_align)) {
425 unsigned long delta = roundup(head, cpudata->trbe_align) - head;
426
427 delta = min(delta, handle->size);
428 trbe_pad_buf(handle, delta);
429 head = PERF_IDX2OFF(handle->head, buf);
430 }
431
432 /*
433 * head = tail (size = 0)
434 * +----|-------------------------------+
435 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ |
436 * +----|-------------------------------+
437 * trbe_base trbe_base + nr_pages
438 *
439 * Perf aux buffer does not have any space for the driver to write into.
440 */
441 if (!handle->size)
442 return 0;
443
444 /* Compute the tail and wakeup indices now that we've aligned head */
445 tail = PERF_IDX2OFF(handle->head + handle->size, buf);
446 wakeup = PERF_IDX2OFF(handle->wakeup, buf);
447
448 /*
449 * Lets calculate the buffer area which TRBE could write into. There
450 * are three possible scenarios here. Limit needs to be aligned with
451 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
452 * unconsumed data.
453 *
454 * 1) head < tail
455 *
456 * head tail
457 * +----|-----------------------|-------+
458 * |$$$$|#######################|$$$$$$$|
459 * +----|-----------------------|-------+
460 * trbe_base limit trbe_base + nr_pages
461 *
462 * TRBE could write into [head..tail] area. Unless the tail is right at
463 * the end of the buffer, neither an wrap around nor an IRQ is expected
464 * while being enabled.
465 *
466 * 2) head == tail
467 *
468 * head = tail (size > 0)
469 * +----|-------------------------------+
470 * |%%%%|###############################|
471 * +----|-------------------------------+
472 * trbe_base limit = trbe_base + nr_pages
473 *
474 * TRBE should just write into [head..base + nr_pages] area even though
475 * the entire buffer is empty. Reason being, when the trace reaches the
476 * end of the buffer, it will just wrap around with an IRQ giving an
477 * opportunity to reconfigure the buffer.
478 *
479 * 3) tail < head
480 *
481 * tail head
482 * +----|-----------------------|-------+
483 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
484 * +----|-----------------------|-------+
485 * trbe_base limit = trbe_base + nr_pages
486 *
487 * TRBE should just write into [head..base + nr_pages] area even though
488 * the [trbe_base..tail] is also empty. Reason being, when the trace
489 * reaches the end of the buffer, it will just wrap around with an IRQ
490 * giving an opportunity to reconfigure the buffer.
491 */
492 if (head < tail)
493 limit = round_down(tail, PAGE_SIZE);
494
495 /*
496 * Wakeup may be arbitrarily far into the future. If it's not in the
497 * current generation, either we'll wrap before hitting it, or it's
498 * in the past and has been handled already.
499 *
500 * If there's a wakeup before we wrap, arrange to be woken up by the
501 * page boundary following it. Keep the tail boundary if that's lower.
502 *
503 * head wakeup tail
504 * +----|---------------|-------|-------+
505 * |$$$$|###############|%%%%%%%|$$$$$$$|
506 * +----|---------------|-------|-------+
507 * trbe_base limit trbe_base + nr_pages
508 */
509 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
510 limit = min(limit, round_up(wakeup, PAGE_SIZE));
511
512 /*
513 * There are two situation when this can happen i.e limit is before
514 * the head and hence TRBE cannot be configured.
515 *
516 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
517 * within the same PAGE size range.
518 *
519 * PAGE_SIZE
520 * |----------------------|
521 *
522 * limit head tail
523 * +------------|------|--------|-------+
524 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
525 * +------------|------|--------|-------+
526 * trbe_base trbe_base + nr_pages
527 *
528 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
529 * head and wakeup are within same PAGE size range.
530 *
531 * PAGE_SIZE
532 * |----------------------|
533 *
534 * limit head wakeup tail
535 * +----|------|-------|--------|-------+
536 * |$$$$$$$$$$$|=======|========|$$$$$$$|
537 * +----|------|-------|--------|-------+
538 * trbe_base trbe_base + nr_pages
539 */
540 if (limit > head)
541 return limit;
542
543 trbe_pad_buf(handle, handle->size);
544 return 0;
545}
546
547static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
548{
549 struct trbe_buf *buf = etm_perf_sink_config(handle);
550 u64 limit = __trbe_normal_offset(handle);
551 u64 head = PERF_IDX2OFF(handle->head, buf);
552
553 /*
554 * If the head is too close to the limit and we don't
555 * have space for a meaningful run, we rather pad it
556 * and start fresh.
557 *
558 * We might have to do this more than once to make sure
559 * we have enough required space.
560 */
561 while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {
562 trbe_pad_buf(handle, limit - head);
563 limit = __trbe_normal_offset(handle);
564 head = PERF_IDX2OFF(handle->head, buf);
565 }
566 return limit;
567}
568
569static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
570{
571 struct trbe_buf *buf = etm_perf_sink_config(handle);
572 unsigned long offset;
573
574 if (buf->snapshot)
575 offset = trbe_snapshot_offset(handle);
576 else
577 offset = trbe_normal_offset(handle);
578 return buf->trbe_base + offset;
579}
580
581static void clr_trbe_status(void)
582{
583 u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
584
585 WARN_ON(is_trbe_enabled());
586 trbsr &= ~TRBSR_EL1_IRQ;
587 trbsr &= ~TRBSR_EL1_TRG;
588 trbsr &= ~TRBSR_EL1_WRAP;
589 trbsr &= ~TRBSR_EL1_EC_MASK;
590 trbsr &= ~TRBSR_EL1_BSC_MASK;
591 trbsr &= ~TRBSR_EL1_S;
592 write_sysreg_s(trbsr, SYS_TRBSR_EL1);
593}
594
595static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
596{
597 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
598 unsigned long addr = buf->trbe_limit;
599
600 WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT)));
601 WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
602
603 trblimitr &= ~TRBLIMITR_EL1_nVM;
604 trblimitr &= ~TRBLIMITR_EL1_FM_MASK;
605 trblimitr &= ~TRBLIMITR_EL1_TM_MASK;
606 trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;
607
608 /*
609 * Fill trace buffer mode is used here while configuring the
610 * TRBE for trace capture. In this particular mode, the trace
611 * collection is stopped and a maintenance interrupt is raised
612 * when the current write pointer wraps. This pause in trace
613 * collection gives the software an opportunity to capture the
614 * trace data in the interrupt handler, before reconfiguring
615 * the TRBE.
616 */
617 trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) &
618 TRBLIMITR_EL1_FM_MASK;
619
620 /*
621 * Trigger mode is not used here while configuring the TRBE for
622 * the trace capture. Hence just keep this in the ignore mode.
623 */
624 trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) &
625 TRBLIMITR_EL1_TM_MASK;
626 trblimitr |= (addr & PAGE_MASK);
627 set_trbe_enabled(buf->cpudata, trblimitr);
628}
629
630static void trbe_enable_hw(struct trbe_buf *buf)
631{
632 WARN_ON(buf->trbe_hw_base < buf->trbe_base);
633 WARN_ON(buf->trbe_write < buf->trbe_hw_base);
634 WARN_ON(buf->trbe_write >= buf->trbe_limit);
635 set_trbe_disabled(buf->cpudata);
636 clr_trbe_status();
637 set_trbe_base_pointer(buf->trbe_hw_base);
638 set_trbe_write_pointer(buf->trbe_write);
639
640 /*
641 * Synchronize all the register updates
642 * till now before enabling the TRBE.
643 */
644 isb();
645 set_trbe_limit_pointer_enabled(buf);
646}
647
648static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,
649 u64 trbsr)
650{
651 int ec = get_trbe_ec(trbsr);
652 int bsc = get_trbe_bsc(trbsr);
653 struct trbe_buf *buf = etm_perf_sink_config(handle);
654 struct trbe_cpudata *cpudata = buf->cpudata;
655
656 WARN_ON(is_trbe_running(trbsr));
657 if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
658 return TRBE_FAULT_ACT_FATAL;
659
660 if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
661 return TRBE_FAULT_ACT_FATAL;
662
663 /*
664 * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
665 * it might write data after a WRAP event in the fill mode.
666 * Thus the check TRBPTR == TRBBASER will not be honored.
667 */
668 if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&
669 (trbe_may_overwrite_in_fill_mode(cpudata) ||
670 get_trbe_write_pointer() == get_trbe_base_pointer()))
671 return TRBE_FAULT_ACT_WRAP;
672
673 return TRBE_FAULT_ACT_SPURIOUS;
674}
675
676static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
677 struct trbe_buf *buf, bool wrap)
678{
679 u64 write;
680 u64 start_off, end_off;
681 u64 size;
682 u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
683
684 /*
685 * If the TRBE has wrapped around the write pointer has
686 * wrapped and should be treated as limit.
687 *
688 * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,
689 * it may write upto 64bytes beyond the "LIMIT". The driver already
690 * keeps a valid page next to the LIMIT and we could potentially
691 * consume the trace data that may have been collected there. But we
692 * cannot be really sure it is available, and the TRBPTR may not
693 * indicate the same. Also, affected cores are also affected by another
694 * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus
695 * could potentially pad an entire PAGE_SIZE - 64bytes, to get those
696 * 64bytes. Thus we ignore the potential triggering of the erratum
697 * on WRAP and limit the data to LIMIT.
698 */
699 if (wrap)
700 write = get_trbe_limit_pointer();
701 else
702 write = get_trbe_write_pointer();
703
704 /*
705 * TRBE may use a different base address than the base
706 * of the ring buffer. Thus use the beginning of the ring
707 * buffer to compute the offsets.
708 */
709 end_off = write - buf->trbe_base;
710 start_off = PERF_IDX2OFF(handle->head, buf);
711
712 if (WARN_ON_ONCE(end_off < start_off))
713 return 0;
714
715 size = end_off - start_off;
716 /*
717 * If the TRBE is affected by the following erratum, we must fill
718 * the space we skipped with IGNORE packets. And we are always
719 * guaranteed to have at least a PAGE_SIZE space in the buffer.
720 */
721 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&
722 !WARN_ON(size < overwrite_skip))
723 __trbe_pad_buf(buf, start_off, overwrite_skip);
724
725 return size;
726}
727
728static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
729 struct perf_event *event, void **pages,
730 int nr_pages, bool snapshot)
731{
732 struct trbe_buf *buf;
733 struct page **pglist;
734 int i;
735
736 /*
737 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
738 * just a single page, there would not be any room left while writing
739 * into a partially filled TRBE buffer after the page size alignment.
740 * Hence restrict the minimum buffer size as two pages.
741 */
742 if (nr_pages < 2)
743 return NULL;
744
745 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));
746 if (!buf)
747 return ERR_PTR(-ENOMEM);
748
749 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
750 if (!pglist) {
751 kfree(buf);
752 return ERR_PTR(-ENOMEM);
753 }
754
755 for (i = 0; i < nr_pages; i++)
756 pglist[i] = virt_to_page(pages[i]);
757
758 buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
759 if (!buf->trbe_base) {
760 kfree(pglist);
761 kfree(buf);
762 return ERR_PTR(-ENOMEM);
763 }
764 buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
765 buf->trbe_write = buf->trbe_base;
766 buf->snapshot = snapshot;
767 buf->nr_pages = nr_pages;
768 buf->pages = pages;
769 kfree(pglist);
770 return buf;
771}
772
773static void arm_trbe_free_buffer(void *config)
774{
775 struct trbe_buf *buf = config;
776
777 vunmap((void *)buf->trbe_base);
778 kfree(buf);
779}
780
781static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
782 struct perf_output_handle *handle,
783 void *config)
784{
785 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
786 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
787 struct trbe_buf *buf = config;
788 enum trbe_fault_action act;
789 unsigned long size, status;
790 unsigned long flags;
791 bool wrap = false;
792
793 WARN_ON(buf->cpudata != cpudata);
794 WARN_ON(cpudata->cpu != smp_processor_id());
795 WARN_ON(cpudata->drvdata != drvdata);
796 if (cpudata->mode != CS_MODE_PERF)
797 return 0;
798
799 /*
800 * We are about to disable the TRBE. And this could in turn
801 * fill up the buffer triggering, an IRQ. This could be consumed
802 * by the PE asynchronously, causing a race here against
803 * the IRQ handler in closing out the handle. So, let us
804 * make sure the IRQ can't trigger while we are collecting
805 * the buffer. We also make sure that a WRAP event is handled
806 * accordingly.
807 */
808 local_irq_save(flags);
809
810 /*
811 * If the TRBE was disabled due to lack of space in the AUX buffer or a
812 * spurious fault, the driver leaves it disabled, truncating the buffer.
813 * Since the etm_perf driver expects to close out the AUX buffer, the
814 * driver skips it. Thus, just pass in 0 size here to indicate that the
815 * buffer was truncated.
816 */
817 if (!is_trbe_enabled()) {
818 size = 0;
819 goto done;
820 }
821 /*
822 * perf handle structure needs to be shared with the TRBE IRQ handler for
823 * capturing trace data and restarting the handle. There is a probability
824 * of an undefined reference based crash when etm event is being stopped
825 * while a TRBE IRQ also getting processed. This happens due the release
826 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
827 * the TRBE here will ensure that no IRQ could be generated when the perf
828 * handle gets freed in etm_event_stop().
829 */
830 trbe_drain_and_disable_local(cpudata);
831
832 /* Check if there is a pending interrupt and handle it here */
833 status = read_sysreg_s(SYS_TRBSR_EL1);
834 if (is_trbe_irq(status)) {
835
836 /*
837 * Now that we are handling the IRQ here, clear the IRQ
838 * from the status, to let the irq handler know that it
839 * is taken care of.
840 */
841 clr_trbe_irq();
842 isb();
843
844 act = trbe_get_fault_act(handle, status);
845 /*
846 * If this was not due to a WRAP event, we have some
847 * errors and as such buffer is empty.
848 */
849 if (act != TRBE_FAULT_ACT_WRAP) {
850 size = 0;
851 goto done;
852 }
853
854 trbe_report_wrap_event(handle);
855 wrap = true;
856 }
857
858 size = trbe_get_trace_size(handle, buf, wrap);
859
860done:
861 local_irq_restore(flags);
862
863 if (buf->snapshot)
864 handle->head += size;
865 return size;
866}
867
868
869static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)
870{
871 /*
872 * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache
873 * line size from the "TRBBASER_EL1" in the event of a "FILL".
874 * Thus, we could loose some amount of the trace at the base.
875 *
876 * Before Fix:
877 *
878 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
879 * | \/ /
880 * -------------------------------------------------------------
881 * | Pg0 | Pg1 | | | PgN |
882 * -------------------------------------------------------------
883 *
884 * In the normal course of action, we would set the TRBBASER to the
885 * beginning of the ring-buffer (normal-BASE). But with the erratum,
886 * the TRBE could overwrite the contents at the "normal-BASE", after
887 * hitting the "normal-LIMIT", since it doesn't stop as expected. And
888 * this is wrong. This could result in overwriting trace collected in
889 * one of the previous runs, being consumed by the user. So we must
890 * always make sure that the TRBBASER is within the region
891 * [head, head+size]. Note that TRBBASER must be PAGE aligned,
892 *
893 * After moving the BASE:
894 *
895 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
896 * | \/ /
897 * -------------------------------------------------------------
898 * | | |xyzdef. |.. tuvw| |
899 * -------------------------------------------------------------
900 * /
901 * New-BASER
902 *
903 * Also, we would set the TRBPTR to head (after adjusting for
904 * alignment) at normal-PTR. This would mean that the last few bytes
905 * of the trace (say, "xyz") might overwrite the first few bytes of
906 * trace written ("abc"). More importantly they will appear in what
907 * userspace sees as the beginning of the trace, which is wrong. We may
908 * not always have space to move the latest trace "xyz" to the correct
909 * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).
910 * Thus it is easier to ignore those bytes than to complicate the
911 * driver to move it, assuming that the erratum was triggered and
912 * doing additional checks to see if there is indeed allowed space at
913 * TRBLIMITR.LIMIT.
914 *
915 * Thus the full workaround will move the BASE and the PTR and would
916 * look like (after padding at the skipped bytes at the end of
917 * session) :
918 *
919 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
920 * | \/ /
921 * -------------------------------------------------------------
922 * | | |///abc.. |.. rst| |
923 * -------------------------------------------------------------
924 * / |
925 * New-BASER New-TRBPTR
926 *
927 * To summarize, with the work around:
928 *
929 * - We always align the offset for the next session to PAGE_SIZE
930 * (This is to ensure we can program the TRBBASER to this offset
931 * within the region [head...head+size]).
932 *
933 * - At TRBE enable:
934 * - Set the TRBBASER to the page aligned offset of the current
935 * proposed write offset. (which is guaranteed to be aligned
936 * as above)
937 * - Move the TRBPTR to skip first 256bytes (that might be
938 * overwritten with the erratum). This ensures that the trace
939 * generated in the session is not re-written.
940 *
941 * - At trace collection:
942 * - Pad the 256bytes skipped above again with IGNORE packets.
943 */
944 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {
945 if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))
946 return -EINVAL;
947 buf->trbe_hw_base = buf->trbe_write;
948 buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
949 }
950
951 /*
952 * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to
953 * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"
954 * may be:
955 * - The page beyond the ring buffer. This could mean, TRBE could
956 * corrupt another entity (kernel / user)
957 * - A portion of the "ring buffer" consumed by the userspace.
958 * i.e, a page outisde [head, head + size].
959 *
960 * We work around this by:
961 * - Making sure that we have at least an extra space of PAGE left
962 * in the ring buffer [head, head + size], than we normally do
963 * without the erratum. See trbe_min_trace_buf_size().
964 *
965 * - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside
966 * the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).
967 */
968 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {
969 s64 space = buf->trbe_limit - buf->trbe_write;
970 /*
971 * We must have more than a PAGE_SIZE worth space in the proposed
972 * range for the TRBE.
973 */
974 if (WARN_ON(space <= PAGE_SIZE ||
975 !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))
976 return -EINVAL;
977 buf->trbe_limit -= PAGE_SIZE;
978 }
979
980 return 0;
981}
982
983static int __arm_trbe_enable(struct trbe_buf *buf,
984 struct perf_output_handle *handle)
985{
986 int ret = 0;
987
988 perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
989 buf->trbe_limit = compute_trbe_buffer_limit(handle);
990 buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
991 if (buf->trbe_limit == buf->trbe_base) {
992 ret = -ENOSPC;
993 goto err;
994 }
995 /* Set the base of the TRBE to the buffer base */
996 buf->trbe_hw_base = buf->trbe_base;
997
998 ret = trbe_apply_work_around_before_enable(buf);
999 if (ret)
1000 goto err;
1001
1002 *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
1003 trbe_enable_hw(buf);
1004 return 0;
1005err:
1006 trbe_stop_and_truncate_event(handle);
1007 return ret;
1008}
1009
1010static int arm_trbe_enable(struct coresight_device *csdev, enum cs_mode mode,
1011 void *data)
1012{
1013 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1014 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1015 struct perf_output_handle *handle = data;
1016 struct trbe_buf *buf = etm_perf_sink_config(handle);
1017
1018 WARN_ON(cpudata->cpu != smp_processor_id());
1019 WARN_ON(cpudata->drvdata != drvdata);
1020 if (mode != CS_MODE_PERF)
1021 return -EINVAL;
1022
1023 cpudata->buf = buf;
1024 cpudata->mode = mode;
1025 buf->cpudata = cpudata;
1026
1027 return __arm_trbe_enable(buf, handle);
1028}
1029
1030static int arm_trbe_disable(struct coresight_device *csdev)
1031{
1032 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1033 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1034 struct trbe_buf *buf = cpudata->buf;
1035
1036 WARN_ON(buf->cpudata != cpudata);
1037 WARN_ON(cpudata->cpu != smp_processor_id());
1038 WARN_ON(cpudata->drvdata != drvdata);
1039 if (cpudata->mode != CS_MODE_PERF)
1040 return -EINVAL;
1041
1042 trbe_drain_and_disable_local(cpudata);
1043 buf->cpudata = NULL;
1044 cpudata->buf = NULL;
1045 cpudata->mode = CS_MODE_DISABLED;
1046 return 0;
1047}
1048
1049static void trbe_handle_spurious(struct perf_output_handle *handle)
1050{
1051 struct trbe_buf *buf = etm_perf_sink_config(handle);
1052 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
1053
1054 /*
1055 * If the IRQ was spurious, simply re-enable the TRBE
1056 * back without modifying the buffer parameters to
1057 * retain the trace collected so far.
1058 */
1059 set_trbe_enabled(buf->cpudata, trblimitr);
1060}
1061
1062static int trbe_handle_overflow(struct perf_output_handle *handle)
1063{
1064 struct perf_event *event = handle->event;
1065 struct trbe_buf *buf = etm_perf_sink_config(handle);
1066 unsigned long size;
1067 struct etm_event_data *event_data;
1068
1069 size = trbe_get_trace_size(handle, buf, true);
1070 if (buf->snapshot)
1071 handle->head += size;
1072
1073 trbe_report_wrap_event(handle);
1074 perf_aux_output_end(handle, size);
1075 event_data = perf_aux_output_begin(handle, event);
1076 if (!event_data) {
1077 /*
1078 * We are unable to restart the trace collection,
1079 * thus leave the TRBE disabled. The etm-perf driver
1080 * is able to detect this with a disconnected handle
1081 * (handle->event = NULL).
1082 */
1083 trbe_drain_and_disable_local(buf->cpudata);
1084 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
1085 return -EINVAL;
1086 }
1087
1088 return __arm_trbe_enable(buf, handle);
1089}
1090
1091static bool is_perf_trbe(struct perf_output_handle *handle)
1092{
1093 struct trbe_buf *buf = etm_perf_sink_config(handle);
1094 struct trbe_cpudata *cpudata = buf->cpudata;
1095 struct trbe_drvdata *drvdata = cpudata->drvdata;
1096 int cpu = smp_processor_id();
1097
1098 WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
1099 WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
1100
1101 if (cpudata->mode != CS_MODE_PERF)
1102 return false;
1103
1104 if (cpudata->cpu != cpu)
1105 return false;
1106
1107 if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1108 return false;
1109
1110 return true;
1111}
1112
1113static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
1114{
1115 struct perf_output_handle **handle_ptr = dev;
1116 struct perf_output_handle *handle = *handle_ptr;
1117 struct trbe_buf *buf = etm_perf_sink_config(handle);
1118 enum trbe_fault_action act;
1119 u64 status;
1120 bool truncated = false;
1121 u64 trfcr;
1122
1123 /* Reads to TRBSR_EL1 is fine when TRBE is active */
1124 status = read_sysreg_s(SYS_TRBSR_EL1);
1125 /*
1126 * If the pending IRQ was handled by update_buffer callback
1127 * we have nothing to do here.
1128 */
1129 if (!is_trbe_irq(status))
1130 return IRQ_NONE;
1131
1132 /* Prohibit the CPU from tracing before we disable the TRBE */
1133 trfcr = cpu_prohibit_trace();
1134 /*
1135 * Ensure the trace is visible to the CPUs and
1136 * any external aborts have been resolved.
1137 */
1138 trbe_drain_and_disable_local(buf->cpudata);
1139 clr_trbe_irq();
1140 isb();
1141
1142 if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
1143 return IRQ_NONE;
1144
1145 if (!is_perf_trbe(handle))
1146 return IRQ_NONE;
1147
1148 act = trbe_get_fault_act(handle, status);
1149 switch (act) {
1150 case TRBE_FAULT_ACT_WRAP:
1151 truncated = !!trbe_handle_overflow(handle);
1152 break;
1153 case TRBE_FAULT_ACT_SPURIOUS:
1154 trbe_handle_spurious(handle);
1155 break;
1156 case TRBE_FAULT_ACT_FATAL:
1157 trbe_stop_and_truncate_event(handle);
1158 truncated = true;
1159 break;
1160 }
1161
1162 /*
1163 * If the buffer was truncated, ensure perf callbacks
1164 * have completed, which will disable the event.
1165 *
1166 * Otherwise, restore the trace filter controls to
1167 * allow the tracing.
1168 */
1169 if (truncated)
1170 irq_work_run();
1171 else
1172 write_trfcr(trfcr);
1173
1174 return IRQ_HANDLED;
1175}
1176
1177static const struct coresight_ops_sink arm_trbe_sink_ops = {
1178 .enable = arm_trbe_enable,
1179 .disable = arm_trbe_disable,
1180 .alloc_buffer = arm_trbe_alloc_buffer,
1181 .free_buffer = arm_trbe_free_buffer,
1182 .update_buffer = arm_trbe_update_buffer,
1183};
1184
1185static const struct coresight_ops arm_trbe_cs_ops = {
1186 .sink_ops = &arm_trbe_sink_ops,
1187};
1188
1189static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
1190{
1191 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1192
1193 return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);
1194}
1195static DEVICE_ATTR_RO(align);
1196
1197static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
1198{
1199 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1200
1201 return sprintf(buf, "%d\n", cpudata->trbe_flag);
1202}
1203static DEVICE_ATTR_RO(flag);
1204
1205static struct attribute *arm_trbe_attrs[] = {
1206 &dev_attr_align.attr,
1207 &dev_attr_flag.attr,
1208 NULL,
1209};
1210
1211static const struct attribute_group arm_trbe_group = {
1212 .attrs = arm_trbe_attrs,
1213};
1214
1215static const struct attribute_group *arm_trbe_groups[] = {
1216 &arm_trbe_group,
1217 NULL,
1218};
1219
1220static void arm_trbe_enable_cpu(void *info)
1221{
1222 struct trbe_drvdata *drvdata = info;
1223 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1224
1225 trbe_reset_local(cpudata);
1226 enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);
1227}
1228
1229static void arm_trbe_disable_cpu(void *info)
1230{
1231 struct trbe_drvdata *drvdata = info;
1232 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1233
1234 disable_percpu_irq(drvdata->irq);
1235 trbe_reset_local(cpudata);
1236}
1237
1238
1239static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1240{
1241 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1242 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1243 struct coresight_desc desc = { 0 };
1244 struct device *dev;
1245
1246 if (WARN_ON(trbe_csdev))
1247 return;
1248
1249 /* If the TRBE was not probed on the CPU, we shouldn't be here */
1250 if (WARN_ON(!cpudata->drvdata))
1251 return;
1252
1253 dev = &cpudata->drvdata->pdev->dev;
1254 desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
1255 if (!desc.name)
1256 goto cpu_clear;
1257 /*
1258 * TRBE coresight devices do not need regular connections
1259 * information, as the paths get built between all percpu
1260 * source and their respective percpu sink devices. Though
1261 * coresight_register() expect device connections via the
1262 * platform_data, which TRBE devices do not have. As they
1263 * are not real ACPI devices, coresight_get_platform_data()
1264 * ends up failing. Instead let's allocate a dummy zeroed
1265 * coresight_platform_data structure and assign that back
1266 * into the device for that purpose.
1267 */
1268 desc.pdata = devm_kzalloc(dev, sizeof(*desc.pdata), GFP_KERNEL);
1269 if (IS_ERR(desc.pdata))
1270 goto cpu_clear;
1271
1272 desc.type = CORESIGHT_DEV_TYPE_SINK;
1273 desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
1274 desc.ops = &arm_trbe_cs_ops;
1275 desc.groups = arm_trbe_groups;
1276 desc.dev = dev;
1277 trbe_csdev = coresight_register(&desc);
1278 if (IS_ERR(trbe_csdev))
1279 goto cpu_clear;
1280
1281 dev_set_drvdata(&trbe_csdev->dev, cpudata);
1282 coresight_set_percpu_sink(cpu, trbe_csdev);
1283 return;
1284cpu_clear:
1285 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1286}
1287
1288/*
1289 * Must be called with preemption disabled, for trbe_check_errata().
1290 */
1291static void arm_trbe_probe_cpu(void *info)
1292{
1293 struct trbe_drvdata *drvdata = info;
1294 int cpu = smp_processor_id();
1295 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1296 u64 trbidr;
1297
1298 if (WARN_ON(!cpudata))
1299 goto cpu_clear;
1300
1301 if (!is_trbe_available()) {
1302 pr_err("TRBE is not implemented on cpu %d\n", cpu);
1303 goto cpu_clear;
1304 }
1305
1306 trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
1307 if (!is_trbe_programmable(trbidr)) {
1308 pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
1309 goto cpu_clear;
1310 }
1311
1312 cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
1313 if (cpudata->trbe_hw_align > SZ_2K) {
1314 pr_err("Unsupported alignment on cpu %d\n", cpu);
1315 goto cpu_clear;
1316 }
1317
1318 /*
1319 * Run the TRBE erratum checks, now that we know
1320 * this instance is about to be registered.
1321 */
1322 trbe_check_errata(cpudata);
1323
1324 if (trbe_is_broken(cpudata)) {
1325 pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);
1326 goto cpu_clear;
1327 }
1328
1329 /*
1330 * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
1331 * we must always program the TBRPTR_EL1, 256bytes from a page
1332 * boundary, with TRBBASER_EL1 set to the page, to prevent
1333 * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.
1334 *
1335 * Thus make sure we always align our write pointer to a PAGE_SIZE,
1336 * which also guarantees that we have at least a PAGE_SIZE space in
1337 * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip
1338 * the required bytes at the base.
1339 */
1340 if (trbe_may_overwrite_in_fill_mode(cpudata))
1341 cpudata->trbe_align = PAGE_SIZE;
1342 else
1343 cpudata->trbe_align = cpudata->trbe_hw_align;
1344
1345 cpudata->trbe_flag = get_trbe_flag_update(trbidr);
1346 cpudata->cpu = cpu;
1347 cpudata->drvdata = drvdata;
1348 return;
1349cpu_clear:
1350 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1351}
1352
1353static void arm_trbe_remove_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1354{
1355 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1356
1357 if (trbe_csdev) {
1358 coresight_unregister(trbe_csdev);
1359 coresight_set_percpu_sink(cpu, NULL);
1360 }
1361}
1362
1363static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
1364{
1365 int cpu;
1366
1367 drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
1368 if (!drvdata->cpudata)
1369 return -ENOMEM;
1370
1371 for_each_cpu(cpu, &drvdata->supported_cpus) {
1372 /* If we fail to probe the CPU, let us defer it to hotplug callbacks */
1373 if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
1374 continue;
1375 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1376 arm_trbe_register_coresight_cpu(drvdata, cpu);
1377 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1378 smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);
1379 }
1380 return 0;
1381}
1382
1383static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
1384{
1385 int cpu;
1386
1387 for_each_cpu(cpu, &drvdata->supported_cpus) {
1388 smp_call_function_single(cpu, arm_trbe_disable_cpu, drvdata, 1);
1389 arm_trbe_remove_coresight_cpu(drvdata, cpu);
1390 }
1391 free_percpu(drvdata->cpudata);
1392 return 0;
1393}
1394
1395static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
1396{
1397 preempt_disable();
1398 arm_trbe_probe_cpu(drvdata);
1399 preempt_enable();
1400}
1401
1402static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
1403{
1404 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1405
1406 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1407
1408 /*
1409 * If this CPU was not probed for TRBE,
1410 * initialize it now.
1411 */
1412 if (!coresight_get_percpu_sink(cpu)) {
1413 arm_trbe_probe_hotplugged_cpu(drvdata);
1414 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1415 arm_trbe_register_coresight_cpu(drvdata, cpu);
1416 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1417 arm_trbe_enable_cpu(drvdata);
1418 } else {
1419 arm_trbe_enable_cpu(drvdata);
1420 }
1421 }
1422 return 0;
1423}
1424
1425static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1426{
1427 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1428
1429 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1430 arm_trbe_disable_cpu(drvdata);
1431 return 0;
1432}
1433
1434static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
1435{
1436 enum cpuhp_state trbe_online;
1437 int ret;
1438
1439 trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
1440 arm_trbe_cpu_startup, arm_trbe_cpu_teardown);
1441 if (trbe_online < 0)
1442 return trbe_online;
1443
1444 ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);
1445 if (ret) {
1446 cpuhp_remove_multi_state(trbe_online);
1447 return ret;
1448 }
1449 drvdata->trbe_online = trbe_online;
1450 return 0;
1451}
1452
1453static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
1454{
1455 cpuhp_state_remove_instance(drvdata->trbe_online, &drvdata->hotplug_node);
1456 cpuhp_remove_multi_state(drvdata->trbe_online);
1457}
1458
1459static int arm_trbe_probe_irq(struct platform_device *pdev,
1460 struct trbe_drvdata *drvdata)
1461{
1462 int ret;
1463
1464 drvdata->irq = platform_get_irq(pdev, 0);
1465 if (drvdata->irq < 0) {
1466 pr_err("IRQ not found for the platform device\n");
1467 return drvdata->irq;
1468 }
1469
1470 if (!irq_is_percpu(drvdata->irq)) {
1471 pr_err("IRQ is not a PPI\n");
1472 return -EINVAL;
1473 }
1474
1475 if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))
1476 return -EINVAL;
1477
1478 drvdata->handle = alloc_percpu(struct perf_output_handle *);
1479 if (!drvdata->handle)
1480 return -ENOMEM;
1481
1482 ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);
1483 if (ret) {
1484 free_percpu(drvdata->handle);
1485 return ret;
1486 }
1487 return 0;
1488}
1489
1490static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
1491{
1492 free_percpu_irq(drvdata->irq, drvdata->handle);
1493 free_percpu(drvdata->handle);
1494}
1495
1496static int arm_trbe_device_probe(struct platform_device *pdev)
1497{
1498 struct trbe_drvdata *drvdata;
1499 struct device *dev = &pdev->dev;
1500 int ret;
1501
1502 /* Trace capture is not possible with kernel page table isolation */
1503 if (arm64_kernel_unmapped_at_el0()) {
1504 pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
1505 return -EOPNOTSUPP;
1506 }
1507
1508 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1509 if (!drvdata)
1510 return -ENOMEM;
1511
1512 dev_set_drvdata(dev, drvdata);
1513 drvdata->pdev = pdev;
1514 ret = arm_trbe_probe_irq(pdev, drvdata);
1515 if (ret)
1516 return ret;
1517
1518 ret = arm_trbe_probe_coresight(drvdata);
1519 if (ret)
1520 goto probe_failed;
1521
1522 ret = arm_trbe_probe_cpuhp(drvdata);
1523 if (ret)
1524 goto cpuhp_failed;
1525
1526 return 0;
1527cpuhp_failed:
1528 arm_trbe_remove_coresight(drvdata);
1529probe_failed:
1530 arm_trbe_remove_irq(drvdata);
1531 return ret;
1532}
1533
1534static void arm_trbe_device_remove(struct platform_device *pdev)
1535{
1536 struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
1537
1538 arm_trbe_remove_cpuhp(drvdata);
1539 arm_trbe_remove_coresight(drvdata);
1540 arm_trbe_remove_irq(drvdata);
1541}
1542
1543static const struct of_device_id arm_trbe_of_match[] = {
1544 { .compatible = "arm,trace-buffer-extension"},
1545 {},
1546};
1547MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
1548
1549#ifdef CONFIG_ACPI
1550static const struct platform_device_id arm_trbe_acpi_match[] = {
1551 { ARMV8_TRBE_PDEV_NAME, 0 },
1552 { }
1553};
1554MODULE_DEVICE_TABLE(platform, arm_trbe_acpi_match);
1555#endif
1556
1557static struct platform_driver arm_trbe_driver = {
1558 .id_table = ACPI_PTR(arm_trbe_acpi_match),
1559 .driver = {
1560 .name = DRVNAME,
1561 .of_match_table = of_match_ptr(arm_trbe_of_match),
1562 .suppress_bind_attrs = true,
1563 },
1564 .probe = arm_trbe_device_probe,
1565 .remove = arm_trbe_device_remove,
1566};
1567
1568static int __init arm_trbe_init(void)
1569{
1570 int ret;
1571
1572 ret = platform_driver_register(&arm_trbe_driver);
1573 if (!ret)
1574 return 0;
1575
1576 pr_err("Error registering %s platform driver\n", DRVNAME);
1577 return ret;
1578}
1579
1580static void __exit arm_trbe_exit(void)
1581{
1582 platform_driver_unregister(&arm_trbe_driver);
1583}
1584module_init(arm_trbe_init);
1585module_exit(arm_trbe_exit);
1586
1587MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
1588MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
1589MODULE_LICENSE("GPL v2");