Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
4 * sink device could then pair with an appropriate per-cpu coresight source
5 * device (ETE) thus generating required trace data. Trace can be enabled
6 * via the perf framework.
7 *
8 * The AUX buffer handling is inspired from Arm SPE PMU driver.
9 *
10 * Copyright (C) 2020 ARM Ltd.
11 *
12 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
13 */
14#define DRVNAME "arm_trbe"
15
16#define pr_fmt(fmt) DRVNAME ": " fmt
17
18#include <asm/barrier.h>
19#include <asm/cpufeature.h>
20#include <linux/vmalloc.h>
21
22#include "coresight-self-hosted-trace.h"
23#include "coresight-trbe.h"
24
25#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
26
27/*
28 * A padding packet that will help the user space tools
29 * in skipping relevant sections in the captured trace
30 * data which could not be decoded. TRBE doesn't support
31 * formatting the trace data, unlike the legacy CoreSight
32 * sinks and thus we use ETE trace packets to pad the
33 * sections of the buffer.
34 */
35#define ETE_IGNORE_PACKET 0x70
36
37/*
38 * Minimum amount of meaningful trace will contain:
39 * A-Sync, Trace Info, Trace On, Address, Atom.
40 * This is about 44bytes of ETE trace. To be on
41 * the safer side, we assume 64bytes is the minimum
42 * space required for a meaningful session, before
43 * we hit a "WRAP" event.
44 */
45#define TRBE_TRACE_MIN_BUF_SIZE 64
46
47enum trbe_fault_action {
48 TRBE_FAULT_ACT_WRAP,
49 TRBE_FAULT_ACT_SPURIOUS,
50 TRBE_FAULT_ACT_FATAL,
51};
52
53struct trbe_buf {
54 /*
55 * Even though trbe_base represents vmap()
56 * mapped allocated buffer's start address,
57 * it's being as unsigned long for various
58 * arithmetic and comparision operations &
59 * also to be consistent with trbe_write &
60 * trbe_limit sibling pointers.
61 */
62 unsigned long trbe_base;
63 /* The base programmed into the TRBE */
64 unsigned long trbe_hw_base;
65 unsigned long trbe_limit;
66 unsigned long trbe_write;
67 int nr_pages;
68 void **pages;
69 bool snapshot;
70 struct trbe_cpudata *cpudata;
71};
72
73/*
74 * TRBE erratum list
75 *
76 * The errata are defined in arm64 generic cpu_errata framework.
77 * Since the errata work arounds could be applied individually
78 * to the affected CPUs inside the TRBE driver, we need to know if
79 * a given CPU is affected by the erratum. Unlike the other erratum
80 * work arounds, TRBE driver needs to check multiple times during
81 * a trace session. Thus we need a quicker access to per-CPU
82 * errata and not issue costly this_cpu_has_cap() everytime.
83 * We keep a set of the affected errata in trbe_cpudata, per TRBE.
84 *
85 * We rely on the corresponding cpucaps to be defined for a given
86 * TRBE erratum. We map the given cpucap into a TRBE internal number
87 * to make the tracking of the errata lean.
88 *
89 * This helps in :
90 * - Not duplicating the detection logic
91 * - Streamlined detection of erratum across the system
92 */
93#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 0
94#define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE 1
95#define TRBE_NEEDS_DRAIN_AFTER_DISABLE 2
96#define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE 3
97#define TRBE_IS_BROKEN 4
98
99static int trbe_errata_cpucaps[] = {
100 [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
101 [TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
102 [TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,
103 [TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,
104 [TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,
105 -1, /* Sentinel, must be the last entry */
106};
107
108/* The total number of listed errata in trbe_errata_cpucaps */
109#define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1)
110
111/*
112 * Safe limit for the number of bytes that may be overwritten
113 * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.
114 */
115#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256
116
117/*
118 * struct trbe_cpudata: TRBE instance specific data
119 * @trbe_flag - TRBE dirty/access flag support
120 * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1.
121 * @trbe_align - Software alignment used for the TRBPTR_EL1.
122 * @cpu - CPU this TRBE belongs to.
123 * @mode - Mode of current operation. (perf/disabled)
124 * @drvdata - TRBE specific drvdata
125 * @errata - Bit map for the errata on this TRBE.
126 */
127struct trbe_cpudata {
128 bool trbe_flag;
129 u64 trbe_hw_align;
130 u64 trbe_align;
131 int cpu;
132 enum cs_mode mode;
133 struct trbe_buf *buf;
134 struct trbe_drvdata *drvdata;
135 DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
136};
137
138struct trbe_drvdata {
139 struct trbe_cpudata __percpu *cpudata;
140 struct perf_output_handle * __percpu *handle;
141 struct hlist_node hotplug_node;
142 int irq;
143 cpumask_t supported_cpus;
144 enum cpuhp_state trbe_online;
145 struct platform_device *pdev;
146};
147
148static void trbe_check_errata(struct trbe_cpudata *cpudata)
149{
150 int i;
151
152 for (i = 0; i < TRBE_ERRATA_MAX; i++) {
153 int cap = trbe_errata_cpucaps[i];
154
155 if (WARN_ON_ONCE(cap < 0))
156 return;
157 if (this_cpu_has_cap(cap))
158 set_bit(i, cpudata->errata);
159 }
160}
161
162static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
163{
164 return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
165}
166
167static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)
168{
169 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);
170}
171
172static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)
173{
174 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);
175}
176
177static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)
178{
179 /*
180 * Errata affected TRBE implementation will need TSB CSYNC and
181 * DSB in order to prevent subsequent writes into certain TRBE
182 * system registers from being ignored and not effected.
183 */
184 return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);
185}
186
187static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)
188{
189 /*
190 * Errata affected TRBE implementation will need an additional
191 * context synchronization in order to prevent an inconsistent
192 * TRBE prohibited region view on the CPU which could possibly
193 * corrupt the TRBE buffer or the TRBE state.
194 */
195 return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);
196}
197
198static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)
199{
200 return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);
201}
202
203static int trbe_alloc_node(struct perf_event *event)
204{
205 if (event->cpu == -1)
206 return NUMA_NO_NODE;
207 return cpu_to_node(event->cpu);
208}
209
210static inline void trbe_drain_buffer(void)
211{
212 tsb_csync();
213 dsb(nsh);
214}
215
216static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)
217{
218 /*
219 * Enable the TRBE without clearing LIMITPTR which
220 * might be required for fetching the buffer limits.
221 */
222 trblimitr |= TRBLIMITR_EL1_E;
223 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
224
225 /* Synchronize the TRBE enable event */
226 isb();
227
228 if (trbe_needs_ctxt_sync_after_enable(cpudata))
229 isb();
230}
231
232static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)
233{
234 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
235
236 /*
237 * Disable the TRBE without clearing LIMITPTR which
238 * might be required for fetching the buffer limits.
239 */
240 trblimitr &= ~TRBLIMITR_EL1_E;
241 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
242
243 if (trbe_needs_drain_after_disable(cpudata))
244 trbe_drain_buffer();
245 isb();
246}
247
248static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)
249{
250 trbe_drain_buffer();
251 set_trbe_disabled(cpudata);
252}
253
254static void trbe_reset_local(struct trbe_cpudata *cpudata)
255{
256 trbe_drain_and_disable_local(cpudata);
257 write_sysreg_s(0, SYS_TRBLIMITR_EL1);
258 write_sysreg_s(0, SYS_TRBPTR_EL1);
259 write_sysreg_s(0, SYS_TRBBASER_EL1);
260 write_sysreg_s(0, SYS_TRBSR_EL1);
261}
262
263static void trbe_report_wrap_event(struct perf_output_handle *handle)
264{
265 /*
266 * Mark the buffer to indicate that there was a WRAP event by
267 * setting the COLLISION flag. This indicates to the user that
268 * the TRBE trace collection was stopped without stopping the
269 * ETE and thus there might be some amount of trace that was
270 * lost between the time the WRAP was detected and the IRQ
271 * was consumed by the CPU.
272 *
273 * Setting the TRUNCATED flag would move the event to STOPPED
274 * state unnecessarily, even when there is space left in the
275 * ring buffer. Using the COLLISION flag doesn't have this side
276 * effect. We only set TRUNCATED flag when there is no space
277 * left in the ring buffer.
278 */
279 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
280}
281
282static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
283{
284 struct trbe_buf *buf = etm_perf_sink_config(handle);
285
286 /*
287 * We cannot proceed with the buffer collection and we
288 * do not have any data for the current session. The
289 * etm_perf driver expects to close out the aux_buffer
290 * at event_stop(). So disable the TRBE here and leave
291 * the update_buffer() to return a 0 size.
292 */
293 trbe_drain_and_disable_local(buf->cpudata);
294 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
295 perf_aux_output_end(handle, 0);
296 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
297}
298
299/*
300 * TRBE Buffer Management
301 *
302 * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
303 * it starts writing trace data from the write pointer onward till the limit pointer.
304 * When the write pointer reaches the address just before the limit pointer, it gets
305 * wrapped around again to the base pointer. This is called a TRBE wrap event, which
306 * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
307 * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
308 * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
309 * LIMIT pointers.
310 *
311 * Wrap around with an IRQ
312 * ------ < ------ < ------- < ----- < -----
313 * | |
314 * ------ > ------ > ------- > ----- > -----
315 *
316 * +---------------+-----------------------+
317 * | | |
318 * +---------------+-----------------------+
319 * Base Pointer Write Pointer Limit Pointer
320 *
321 * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
322 * pointer can be aligned to the implementation defined TRBE trace buffer alignment
323 * as captured in trbe_cpudata->trbe_align.
324 *
325 *
326 * head tail wakeup
327 * +---------------------------------------+----- ~ ~ ------
328 * |$$$$$$$|################|$$$$$$$$$$$$$$| |
329 * +---------------------------------------+----- ~ ~ ------
330 * Base Pointer Write Pointer Limit Pointer
331 *
332 * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
333 * values which tracks all the driver writes and user reads from the perf auxiliary
334 * buffer. Generally [head..tail] is the area where the driver can write into unless
335 * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
336 * configured depending on the perf_output_handle indices, so that the driver does
337 * not override into areas in the perf auxiliary buffer which is being or yet to be
338 * consumed from the user space. The enabled TRBE buffer area is a moving subset of
339 * the allocated perf auxiliary buffer.
340 */
341
342static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
343{
344 memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
345}
346
347static void trbe_pad_buf(struct perf_output_handle *handle, int len)
348{
349 struct trbe_buf *buf = etm_perf_sink_config(handle);
350 u64 head = PERF_IDX2OFF(handle->head, buf);
351
352 __trbe_pad_buf(buf, head, len);
353 if (!buf->snapshot)
354 perf_aux_output_skip(handle, len);
355}
356
357static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
358{
359 struct trbe_buf *buf = etm_perf_sink_config(handle);
360
361 /*
362 * The ETE trace has alignment synchronization packets allowing
363 * the decoder to reset in case of an overflow or corruption.
364 * So we can use the entire buffer for the snapshot mode.
365 */
366 return buf->nr_pages * PAGE_SIZE;
367}
368
369static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)
370{
371 u64 size = TRBE_TRACE_MIN_BUF_SIZE;
372 struct trbe_buf *buf = etm_perf_sink_config(handle);
373 struct trbe_cpudata *cpudata = buf->cpudata;
374
375 /*
376 * When the TRBE is affected by an erratum that could make it
377 * write to the next "virtually addressed" page beyond the LIMIT.
378 * We need to make sure there is always a PAGE after the LIMIT,
379 * within the buffer. Thus we ensure there is at least an extra
380 * page than normal. With this we could then adjust the LIMIT
381 * pointer down by a PAGE later.
382 */
383 if (trbe_may_write_out_of_range(cpudata))
384 size += PAGE_SIZE;
385 return size;
386}
387
388/*
389 * TRBE Limit Calculation
390 *
391 * The following markers are used to illustrate various TRBE buffer situations.
392 *
393 * $$$$ - Data area, unconsumed captured trace data, not to be overridden
394 * #### - Free area, enabled, trace will be written
395 * %%%% - Free area, disabled, trace will not be written
396 * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
397 */
398static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
399{
400 struct trbe_buf *buf = etm_perf_sink_config(handle);
401 struct trbe_cpudata *cpudata = buf->cpudata;
402 const u64 bufsize = buf->nr_pages * PAGE_SIZE;
403 u64 limit = bufsize;
404 u64 head, tail, wakeup;
405
406 head = PERF_IDX2OFF(handle->head, buf);
407
408 /*
409 * head
410 * ------->|
411 * |
412 * head TRBE align tail
413 * +----|-------|---------------|-------+
414 * |$$$$|=======|###############|$$$$$$$|
415 * +----|-------|---------------|-------+
416 * trbe_base trbe_base + nr_pages
417 *
418 * Perf aux buffer output head position can be misaligned depending on
419 * various factors including user space reads. In case misaligned, head
420 * needs to be aligned before TRBE can be configured. Pad the alignment
421 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
422 * and skip this section thus advancing the head.
423 */
424 if (!IS_ALIGNED(head, cpudata->trbe_align)) {
425 unsigned long delta = roundup(head, cpudata->trbe_align) - head;
426
427 delta = min(delta, handle->size);
428 trbe_pad_buf(handle, delta);
429 head = PERF_IDX2OFF(handle->head, buf);
430 }
431
432 /*
433 * head = tail (size = 0)
434 * +----|-------------------------------+
435 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ |
436 * +----|-------------------------------+
437 * trbe_base trbe_base + nr_pages
438 *
439 * Perf aux buffer does not have any space for the driver to write into.
440 */
441 if (!handle->size)
442 return 0;
443
444 /* Compute the tail and wakeup indices now that we've aligned head */
445 tail = PERF_IDX2OFF(handle->head + handle->size, buf);
446 wakeup = PERF_IDX2OFF(handle->wakeup, buf);
447
448 /*
449 * Lets calculate the buffer area which TRBE could write into. There
450 * are three possible scenarios here. Limit needs to be aligned with
451 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
452 * unconsumed data.
453 *
454 * 1) head < tail
455 *
456 * head tail
457 * +----|-----------------------|-------+
458 * |$$$$|#######################|$$$$$$$|
459 * +----|-----------------------|-------+
460 * trbe_base limit trbe_base + nr_pages
461 *
462 * TRBE could write into [head..tail] area. Unless the tail is right at
463 * the end of the buffer, neither an wrap around nor an IRQ is expected
464 * while being enabled.
465 *
466 * 2) head == tail
467 *
468 * head = tail (size > 0)
469 * +----|-------------------------------+
470 * |%%%%|###############################|
471 * +----|-------------------------------+
472 * trbe_base limit = trbe_base + nr_pages
473 *
474 * TRBE should just write into [head..base + nr_pages] area even though
475 * the entire buffer is empty. Reason being, when the trace reaches the
476 * end of the buffer, it will just wrap around with an IRQ giving an
477 * opportunity to reconfigure the buffer.
478 *
479 * 3) tail < head
480 *
481 * tail head
482 * +----|-----------------------|-------+
483 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
484 * +----|-----------------------|-------+
485 * trbe_base limit = trbe_base + nr_pages
486 *
487 * TRBE should just write into [head..base + nr_pages] area even though
488 * the [trbe_base..tail] is also empty. Reason being, when the trace
489 * reaches the end of the buffer, it will just wrap around with an IRQ
490 * giving an opportunity to reconfigure the buffer.
491 */
492 if (head < tail)
493 limit = round_down(tail, PAGE_SIZE);
494
495 /*
496 * Wakeup may be arbitrarily far into the future. If it's not in the
497 * current generation, either we'll wrap before hitting it, or it's
498 * in the past and has been handled already.
499 *
500 * If there's a wakeup before we wrap, arrange to be woken up by the
501 * page boundary following it. Keep the tail boundary if that's lower.
502 *
503 * head wakeup tail
504 * +----|---------------|-------|-------+
505 * |$$$$|###############|%%%%%%%|$$$$$$$|
506 * +----|---------------|-------|-------+
507 * trbe_base limit trbe_base + nr_pages
508 */
509 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
510 limit = min(limit, round_up(wakeup, PAGE_SIZE));
511
512 /*
513 * There are two situation when this can happen i.e limit is before
514 * the head and hence TRBE cannot be configured.
515 *
516 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
517 * within the same PAGE size range.
518 *
519 * PAGE_SIZE
520 * |----------------------|
521 *
522 * limit head tail
523 * +------------|------|--------|-------+
524 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
525 * +------------|------|--------|-------+
526 * trbe_base trbe_base + nr_pages
527 *
528 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
529 * head and wakeup are within same PAGE size range.
530 *
531 * PAGE_SIZE
532 * |----------------------|
533 *
534 * limit head wakeup tail
535 * +----|------|-------|--------|-------+
536 * |$$$$$$$$$$$|=======|========|$$$$$$$|
537 * +----|------|-------|--------|-------+
538 * trbe_base trbe_base + nr_pages
539 */
540 if (limit > head)
541 return limit;
542
543 trbe_pad_buf(handle, handle->size);
544 return 0;
545}
546
547static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
548{
549 struct trbe_buf *buf = etm_perf_sink_config(handle);
550 u64 limit = __trbe_normal_offset(handle);
551 u64 head = PERF_IDX2OFF(handle->head, buf);
552
553 /*
554 * If the head is too close to the limit and we don't
555 * have space for a meaningful run, we rather pad it
556 * and start fresh.
557 *
558 * We might have to do this more than once to make sure
559 * we have enough required space.
560 */
561 while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {
562 trbe_pad_buf(handle, limit - head);
563 limit = __trbe_normal_offset(handle);
564 head = PERF_IDX2OFF(handle->head, buf);
565 }
566 return limit;
567}
568
569static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
570{
571 struct trbe_buf *buf = etm_perf_sink_config(handle);
572 unsigned long offset;
573
574 if (buf->snapshot)
575 offset = trbe_snapshot_offset(handle);
576 else
577 offset = trbe_normal_offset(handle);
578 return buf->trbe_base + offset;
579}
580
581static void clr_trbe_status(void)
582{
583 u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
584
585 WARN_ON(is_trbe_enabled());
586 trbsr &= ~TRBSR_EL1_IRQ;
587 trbsr &= ~TRBSR_EL1_TRG;
588 trbsr &= ~TRBSR_EL1_WRAP;
589 trbsr &= ~TRBSR_EL1_EC_MASK;
590 trbsr &= ~TRBSR_EL1_BSC_MASK;
591 trbsr &= ~TRBSR_EL1_S;
592 write_sysreg_s(trbsr, SYS_TRBSR_EL1);
593}
594
595static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
596{
597 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
598 unsigned long addr = buf->trbe_limit;
599
600 WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT)));
601 WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
602
603 trblimitr &= ~TRBLIMITR_EL1_nVM;
604 trblimitr &= ~TRBLIMITR_EL1_FM_MASK;
605 trblimitr &= ~TRBLIMITR_EL1_TM_MASK;
606 trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;
607
608 /*
609 * Fill trace buffer mode is used here while configuring the
610 * TRBE for trace capture. In this particular mode, the trace
611 * collection is stopped and a maintenance interrupt is raised
612 * when the current write pointer wraps. This pause in trace
613 * collection gives the software an opportunity to capture the
614 * trace data in the interrupt handler, before reconfiguring
615 * the TRBE.
616 */
617 trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) &
618 TRBLIMITR_EL1_FM_MASK;
619
620 /*
621 * Trigger mode is not used here while configuring the TRBE for
622 * the trace capture. Hence just keep this in the ignore mode.
623 */
624 trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) &
625 TRBLIMITR_EL1_TM_MASK;
626 trblimitr |= (addr & PAGE_MASK);
627 set_trbe_enabled(buf->cpudata, trblimitr);
628}
629
630static void trbe_enable_hw(struct trbe_buf *buf)
631{
632 WARN_ON(buf->trbe_hw_base < buf->trbe_base);
633 WARN_ON(buf->trbe_write < buf->trbe_hw_base);
634 WARN_ON(buf->trbe_write >= buf->trbe_limit);
635 set_trbe_disabled(buf->cpudata);
636 clr_trbe_status();
637 set_trbe_base_pointer(buf->trbe_hw_base);
638 set_trbe_write_pointer(buf->trbe_write);
639
640 /*
641 * Synchronize all the register updates
642 * till now before enabling the TRBE.
643 */
644 isb();
645 set_trbe_limit_pointer_enabled(buf);
646}
647
648static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,
649 u64 trbsr)
650{
651 int ec = get_trbe_ec(trbsr);
652 int bsc = get_trbe_bsc(trbsr);
653 struct trbe_buf *buf = etm_perf_sink_config(handle);
654 struct trbe_cpudata *cpudata = buf->cpudata;
655
656 WARN_ON(is_trbe_running(trbsr));
657 if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
658 return TRBE_FAULT_ACT_FATAL;
659
660 if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
661 return TRBE_FAULT_ACT_FATAL;
662
663 /*
664 * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
665 * it might write data after a WRAP event in the fill mode.
666 * Thus the check TRBPTR == TRBBASER will not be honored.
667 */
668 if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&
669 (trbe_may_overwrite_in_fill_mode(cpudata) ||
670 get_trbe_write_pointer() == get_trbe_base_pointer()))
671 return TRBE_FAULT_ACT_WRAP;
672
673 return TRBE_FAULT_ACT_SPURIOUS;
674}
675
676static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
677 struct trbe_buf *buf, bool wrap)
678{
679 u64 write;
680 u64 start_off, end_off;
681 u64 size;
682 u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
683
684 /*
685 * If the TRBE has wrapped around the write pointer has
686 * wrapped and should be treated as limit.
687 *
688 * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,
689 * it may write upto 64bytes beyond the "LIMIT". The driver already
690 * keeps a valid page next to the LIMIT and we could potentially
691 * consume the trace data that may have been collected there. But we
692 * cannot be really sure it is available, and the TRBPTR may not
693 * indicate the same. Also, affected cores are also affected by another
694 * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus
695 * could potentially pad an entire PAGE_SIZE - 64bytes, to get those
696 * 64bytes. Thus we ignore the potential triggering of the erratum
697 * on WRAP and limit the data to LIMIT.
698 */
699 if (wrap)
700 write = get_trbe_limit_pointer();
701 else
702 write = get_trbe_write_pointer();
703
704 /*
705 * TRBE may use a different base address than the base
706 * of the ring buffer. Thus use the beginning of the ring
707 * buffer to compute the offsets.
708 */
709 end_off = write - buf->trbe_base;
710 start_off = PERF_IDX2OFF(handle->head, buf);
711
712 if (WARN_ON_ONCE(end_off < start_off))
713 return 0;
714
715 size = end_off - start_off;
716 /*
717 * If the TRBE is affected by the following erratum, we must fill
718 * the space we skipped with IGNORE packets. And we are always
719 * guaranteed to have at least a PAGE_SIZE space in the buffer.
720 */
721 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&
722 !WARN_ON(size < overwrite_skip))
723 __trbe_pad_buf(buf, start_off, overwrite_skip);
724
725 return size;
726}
727
728static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
729 struct perf_event *event, void **pages,
730 int nr_pages, bool snapshot)
731{
732 struct trbe_buf *buf;
733 struct page **pglist;
734 int i;
735
736 /*
737 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
738 * just a single page, there would not be any room left while writing
739 * into a partially filled TRBE buffer after the page size alignment.
740 * Hence restrict the minimum buffer size as two pages.
741 */
742 if (nr_pages < 2)
743 return NULL;
744
745 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));
746 if (!buf)
747 return ERR_PTR(-ENOMEM);
748
749 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
750 if (!pglist) {
751 kfree(buf);
752 return ERR_PTR(-ENOMEM);
753 }
754
755 for (i = 0; i < nr_pages; i++)
756 pglist[i] = virt_to_page(pages[i]);
757
758 buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
759 if (!buf->trbe_base) {
760 kfree(pglist);
761 kfree(buf);
762 return ERR_PTR(-ENOMEM);
763 }
764 buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
765 buf->trbe_write = buf->trbe_base;
766 buf->snapshot = snapshot;
767 buf->nr_pages = nr_pages;
768 buf->pages = pages;
769 kfree(pglist);
770 return buf;
771}
772
773static void arm_trbe_free_buffer(void *config)
774{
775 struct trbe_buf *buf = config;
776
777 vunmap((void *)buf->trbe_base);
778 kfree(buf);
779}
780
781static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
782 struct perf_output_handle *handle,
783 void *config)
784{
785 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
786 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
787 struct trbe_buf *buf = config;
788 enum trbe_fault_action act;
789 unsigned long size, status;
790 unsigned long flags;
791 bool wrap = false;
792
793 WARN_ON(buf->cpudata != cpudata);
794 WARN_ON(cpudata->cpu != smp_processor_id());
795 WARN_ON(cpudata->drvdata != drvdata);
796 if (cpudata->mode != CS_MODE_PERF)
797 return 0;
798
799 /*
800 * We are about to disable the TRBE. And this could in turn
801 * fill up the buffer triggering, an IRQ. This could be consumed
802 * by the PE asynchronously, causing a race here against
803 * the IRQ handler in closing out the handle. So, let us
804 * make sure the IRQ can't trigger while we are collecting
805 * the buffer. We also make sure that a WRAP event is handled
806 * accordingly.
807 */
808 local_irq_save(flags);
809
810 /*
811 * If the TRBE was disabled due to lack of space in the AUX buffer or a
812 * spurious fault, the driver leaves it disabled, truncating the buffer.
813 * Since the etm_perf driver expects to close out the AUX buffer, the
814 * driver skips it. Thus, just pass in 0 size here to indicate that the
815 * buffer was truncated.
816 */
817 if (!is_trbe_enabled()) {
818 size = 0;
819 goto done;
820 }
821 /*
822 * perf handle structure needs to be shared with the TRBE IRQ handler for
823 * capturing trace data and restarting the handle. There is a probability
824 * of an undefined reference based crash when etm event is being stopped
825 * while a TRBE IRQ also getting processed. This happens due the release
826 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
827 * the TRBE here will ensure that no IRQ could be generated when the perf
828 * handle gets freed in etm_event_stop().
829 */
830 trbe_drain_and_disable_local(cpudata);
831
832 /* Check if there is a pending interrupt and handle it here */
833 status = read_sysreg_s(SYS_TRBSR_EL1);
834 if (is_trbe_irq(status)) {
835
836 /*
837 * Now that we are handling the IRQ here, clear the IRQ
838 * from the status, to let the irq handler know that it
839 * is taken care of.
840 */
841 clr_trbe_irq();
842 isb();
843
844 act = trbe_get_fault_act(handle, status);
845 /*
846 * If this was not due to a WRAP event, we have some
847 * errors and as such buffer is empty.
848 */
849 if (act != TRBE_FAULT_ACT_WRAP) {
850 size = 0;
851 goto done;
852 }
853
854 trbe_report_wrap_event(handle);
855 wrap = true;
856 }
857
858 size = trbe_get_trace_size(handle, buf, wrap);
859
860done:
861 local_irq_restore(flags);
862
863 if (buf->snapshot)
864 handle->head += size;
865 return size;
866}
867
868
869static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)
870{
871 /*
872 * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache
873 * line size from the "TRBBASER_EL1" in the event of a "FILL".
874 * Thus, we could loose some amount of the trace at the base.
875 *
876 * Before Fix:
877 *
878 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
879 * | \/ /
880 * -------------------------------------------------------------
881 * | Pg0 | Pg1 | | | PgN |
882 * -------------------------------------------------------------
883 *
884 * In the normal course of action, we would set the TRBBASER to the
885 * beginning of the ring-buffer (normal-BASE). But with the erratum,
886 * the TRBE could overwrite the contents at the "normal-BASE", after
887 * hitting the "normal-LIMIT", since it doesn't stop as expected. And
888 * this is wrong. This could result in overwriting trace collected in
889 * one of the previous runs, being consumed by the user. So we must
890 * always make sure that the TRBBASER is within the region
891 * [head, head+size]. Note that TRBBASER must be PAGE aligned,
892 *
893 * After moving the BASE:
894 *
895 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
896 * | \/ /
897 * -------------------------------------------------------------
898 * | | |xyzdef. |.. tuvw| |
899 * -------------------------------------------------------------
900 * /
901 * New-BASER
902 *
903 * Also, we would set the TRBPTR to head (after adjusting for
904 * alignment) at normal-PTR. This would mean that the last few bytes
905 * of the trace (say, "xyz") might overwrite the first few bytes of
906 * trace written ("abc"). More importantly they will appear in what
907 * userspace sees as the beginning of the trace, which is wrong. We may
908 * not always have space to move the latest trace "xyz" to the correct
909 * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).
910 * Thus it is easier to ignore those bytes than to complicate the
911 * driver to move it, assuming that the erratum was triggered and
912 * doing additional checks to see if there is indeed allowed space at
913 * TRBLIMITR.LIMIT.
914 *
915 * Thus the full workaround will move the BASE and the PTR and would
916 * look like (after padding at the skipped bytes at the end of
917 * session) :
918 *
919 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
920 * | \/ /
921 * -------------------------------------------------------------
922 * | | |///abc.. |.. rst| |
923 * -------------------------------------------------------------
924 * / |
925 * New-BASER New-TRBPTR
926 *
927 * To summarize, with the work around:
928 *
929 * - We always align the offset for the next session to PAGE_SIZE
930 * (This is to ensure we can program the TRBBASER to this offset
931 * within the region [head...head+size]).
932 *
933 * - At TRBE enable:
934 * - Set the TRBBASER to the page aligned offset of the current
935 * proposed write offset. (which is guaranteed to be aligned
936 * as above)
937 * - Move the TRBPTR to skip first 256bytes (that might be
938 * overwritten with the erratum). This ensures that the trace
939 * generated in the session is not re-written.
940 *
941 * - At trace collection:
942 * - Pad the 256bytes skipped above again with IGNORE packets.
943 */
944 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {
945 if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))
946 return -EINVAL;
947 buf->trbe_hw_base = buf->trbe_write;
948 buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
949 }
950
951 /*
952 * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to
953 * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"
954 * may be:
955 * - The page beyond the ring buffer. This could mean, TRBE could
956 * corrupt another entity (kernel / user)
957 * - A portion of the "ring buffer" consumed by the userspace.
958 * i.e, a page outisde [head, head + size].
959 *
960 * We work around this by:
961 * - Making sure that we have at least an extra space of PAGE left
962 * in the ring buffer [head, head + size], than we normally do
963 * without the erratum. See trbe_min_trace_buf_size().
964 *
965 * - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside
966 * the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).
967 */
968 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {
969 s64 space = buf->trbe_limit - buf->trbe_write;
970 /*
971 * We must have more than a PAGE_SIZE worth space in the proposed
972 * range for the TRBE.
973 */
974 if (WARN_ON(space <= PAGE_SIZE ||
975 !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))
976 return -EINVAL;
977 buf->trbe_limit -= PAGE_SIZE;
978 }
979
980 return 0;
981}
982
983static int __arm_trbe_enable(struct trbe_buf *buf,
984 struct perf_output_handle *handle)
985{
986 int ret = 0;
987
988 perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
989 buf->trbe_limit = compute_trbe_buffer_limit(handle);
990 buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
991 if (buf->trbe_limit == buf->trbe_base) {
992 ret = -ENOSPC;
993 goto err;
994 }
995 /* Set the base of the TRBE to the buffer base */
996 buf->trbe_hw_base = buf->trbe_base;
997
998 ret = trbe_apply_work_around_before_enable(buf);
999 if (ret)
1000 goto err;
1001
1002 *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
1003 trbe_enable_hw(buf);
1004 return 0;
1005err:
1006 trbe_stop_and_truncate_event(handle);
1007 return ret;
1008}
1009
1010static int arm_trbe_enable(struct coresight_device *csdev, enum cs_mode mode,
1011 void *data)
1012{
1013 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1014 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1015 struct perf_output_handle *handle = data;
1016 struct trbe_buf *buf = etm_perf_sink_config(handle);
1017
1018 WARN_ON(cpudata->cpu != smp_processor_id());
1019 WARN_ON(cpudata->drvdata != drvdata);
1020 if (mode != CS_MODE_PERF)
1021 return -EINVAL;
1022
1023 cpudata->buf = buf;
1024 cpudata->mode = mode;
1025 buf->cpudata = cpudata;
1026
1027 return __arm_trbe_enable(buf, handle);
1028}
1029
1030static int arm_trbe_disable(struct coresight_device *csdev)
1031{
1032 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1033 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1034 struct trbe_buf *buf = cpudata->buf;
1035
1036 WARN_ON(buf->cpudata != cpudata);
1037 WARN_ON(cpudata->cpu != smp_processor_id());
1038 WARN_ON(cpudata->drvdata != drvdata);
1039 if (cpudata->mode != CS_MODE_PERF)
1040 return -EINVAL;
1041
1042 trbe_drain_and_disable_local(cpudata);
1043 buf->cpudata = NULL;
1044 cpudata->buf = NULL;
1045 cpudata->mode = CS_MODE_DISABLED;
1046 return 0;
1047}
1048
1049static void trbe_handle_spurious(struct perf_output_handle *handle)
1050{
1051 struct trbe_buf *buf = etm_perf_sink_config(handle);
1052 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
1053
1054 /*
1055 * If the IRQ was spurious, simply re-enable the TRBE
1056 * back without modifying the buffer parameters to
1057 * retain the trace collected so far.
1058 */
1059 set_trbe_enabled(buf->cpudata, trblimitr);
1060}
1061
1062static int trbe_handle_overflow(struct perf_output_handle *handle)
1063{
1064 struct perf_event *event = handle->event;
1065 struct trbe_buf *buf = etm_perf_sink_config(handle);
1066 unsigned long size;
1067 struct etm_event_data *event_data;
1068
1069 size = trbe_get_trace_size(handle, buf, true);
1070 if (buf->snapshot)
1071 handle->head += size;
1072
1073 trbe_report_wrap_event(handle);
1074 perf_aux_output_end(handle, size);
1075 event_data = perf_aux_output_begin(handle, event);
1076 if (!event_data) {
1077 /*
1078 * We are unable to restart the trace collection,
1079 * thus leave the TRBE disabled. The etm-perf driver
1080 * is able to detect this with a disconnected handle
1081 * (handle->event = NULL).
1082 */
1083 trbe_drain_and_disable_local(buf->cpudata);
1084 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
1085 return -EINVAL;
1086 }
1087
1088 return __arm_trbe_enable(buf, handle);
1089}
1090
1091static bool is_perf_trbe(struct perf_output_handle *handle)
1092{
1093 struct trbe_buf *buf = etm_perf_sink_config(handle);
1094 struct trbe_cpudata *cpudata = buf->cpudata;
1095 struct trbe_drvdata *drvdata = cpudata->drvdata;
1096 int cpu = smp_processor_id();
1097
1098 WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
1099 WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
1100
1101 if (cpudata->mode != CS_MODE_PERF)
1102 return false;
1103
1104 if (cpudata->cpu != cpu)
1105 return false;
1106
1107 if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1108 return false;
1109
1110 return true;
1111}
1112
1113static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
1114{
1115 struct perf_output_handle **handle_ptr = dev;
1116 struct perf_output_handle *handle = *handle_ptr;
1117 struct trbe_buf *buf = etm_perf_sink_config(handle);
1118 enum trbe_fault_action act;
1119 u64 status;
1120 bool truncated = false;
1121 u64 trfcr;
1122
1123 /* Reads to TRBSR_EL1 is fine when TRBE is active */
1124 status = read_sysreg_s(SYS_TRBSR_EL1);
1125 /*
1126 * If the pending IRQ was handled by update_buffer callback
1127 * we have nothing to do here.
1128 */
1129 if (!is_trbe_irq(status))
1130 return IRQ_NONE;
1131
1132 /* Prohibit the CPU from tracing before we disable the TRBE */
1133 trfcr = cpu_prohibit_trace();
1134 /*
1135 * Ensure the trace is visible to the CPUs and
1136 * any external aborts have been resolved.
1137 */
1138 trbe_drain_and_disable_local(buf->cpudata);
1139 clr_trbe_irq();
1140 isb();
1141
1142 if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
1143 return IRQ_NONE;
1144
1145 if (!is_perf_trbe(handle))
1146 return IRQ_NONE;
1147
1148 act = trbe_get_fault_act(handle, status);
1149 switch (act) {
1150 case TRBE_FAULT_ACT_WRAP:
1151 truncated = !!trbe_handle_overflow(handle);
1152 break;
1153 case TRBE_FAULT_ACT_SPURIOUS:
1154 trbe_handle_spurious(handle);
1155 break;
1156 case TRBE_FAULT_ACT_FATAL:
1157 trbe_stop_and_truncate_event(handle);
1158 truncated = true;
1159 break;
1160 }
1161
1162 /*
1163 * If the buffer was truncated, ensure perf callbacks
1164 * have completed, which will disable the event.
1165 *
1166 * Otherwise, restore the trace filter controls to
1167 * allow the tracing.
1168 */
1169 if (truncated)
1170 irq_work_run();
1171 else
1172 write_trfcr(trfcr);
1173
1174 return IRQ_HANDLED;
1175}
1176
1177static const struct coresight_ops_sink arm_trbe_sink_ops = {
1178 .enable = arm_trbe_enable,
1179 .disable = arm_trbe_disable,
1180 .alloc_buffer = arm_trbe_alloc_buffer,
1181 .free_buffer = arm_trbe_free_buffer,
1182 .update_buffer = arm_trbe_update_buffer,
1183};
1184
1185static const struct coresight_ops arm_trbe_cs_ops = {
1186 .sink_ops = &arm_trbe_sink_ops,
1187};
1188
1189static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
1190{
1191 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1192
1193 return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);
1194}
1195static DEVICE_ATTR_RO(align);
1196
1197static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
1198{
1199 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1200
1201 return sprintf(buf, "%d\n", cpudata->trbe_flag);
1202}
1203static DEVICE_ATTR_RO(flag);
1204
1205static struct attribute *arm_trbe_attrs[] = {
1206 &dev_attr_align.attr,
1207 &dev_attr_flag.attr,
1208 NULL,
1209};
1210
1211static const struct attribute_group arm_trbe_group = {
1212 .attrs = arm_trbe_attrs,
1213};
1214
1215static const struct attribute_group *arm_trbe_groups[] = {
1216 &arm_trbe_group,
1217 NULL,
1218};
1219
1220static void arm_trbe_enable_cpu(void *info)
1221{
1222 struct trbe_drvdata *drvdata = info;
1223 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1224
1225 trbe_reset_local(cpudata);
1226 enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);
1227}
1228
1229static void arm_trbe_disable_cpu(void *info)
1230{
1231 struct trbe_drvdata *drvdata = info;
1232 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1233
1234 disable_percpu_irq(drvdata->irq);
1235 trbe_reset_local(cpudata);
1236}
1237
1238
1239static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1240{
1241 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1242 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1243 struct coresight_desc desc = { 0 };
1244 struct device *dev;
1245
1246 if (WARN_ON(trbe_csdev))
1247 return;
1248
1249 /* If the TRBE was not probed on the CPU, we shouldn't be here */
1250 if (WARN_ON(!cpudata->drvdata))
1251 return;
1252
1253 dev = &cpudata->drvdata->pdev->dev;
1254 desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
1255 if (!desc.name)
1256 goto cpu_clear;
1257 /*
1258 * TRBE coresight devices do not need regular connections
1259 * information, as the paths get built between all percpu
1260 * source and their respective percpu sink devices. Though
1261 * coresight_register() expect device connections via the
1262 * platform_data, which TRBE devices do not have. As they
1263 * are not real ACPI devices, coresight_get_platform_data()
1264 * ends up failing. Instead let's allocate a dummy zeroed
1265 * coresight_platform_data structure and assign that back
1266 * into the device for that purpose.
1267 */
1268 desc.pdata = devm_kzalloc(dev, sizeof(*desc.pdata), GFP_KERNEL);
1269 if (IS_ERR(desc.pdata))
1270 goto cpu_clear;
1271
1272 desc.type = CORESIGHT_DEV_TYPE_SINK;
1273 desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
1274 desc.ops = &arm_trbe_cs_ops;
1275 desc.groups = arm_trbe_groups;
1276 desc.dev = dev;
1277 trbe_csdev = coresight_register(&desc);
1278 if (IS_ERR(trbe_csdev))
1279 goto cpu_clear;
1280
1281 dev_set_drvdata(&trbe_csdev->dev, cpudata);
1282 coresight_set_percpu_sink(cpu, trbe_csdev);
1283 return;
1284cpu_clear:
1285 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1286}
1287
1288/*
1289 * Must be called with preemption disabled, for trbe_check_errata().
1290 */
1291static void arm_trbe_probe_cpu(void *info)
1292{
1293 struct trbe_drvdata *drvdata = info;
1294 int cpu = smp_processor_id();
1295 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1296 u64 trbidr;
1297
1298 if (WARN_ON(!cpudata))
1299 goto cpu_clear;
1300
1301 if (!is_trbe_available()) {
1302 pr_err("TRBE is not implemented on cpu %d\n", cpu);
1303 goto cpu_clear;
1304 }
1305
1306 trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
1307 if (!is_trbe_programmable(trbidr)) {
1308 pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
1309 goto cpu_clear;
1310 }
1311
1312 cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
1313 if (cpudata->trbe_hw_align > SZ_2K) {
1314 pr_err("Unsupported alignment on cpu %d\n", cpu);
1315 goto cpu_clear;
1316 }
1317
1318 /*
1319 * Run the TRBE erratum checks, now that we know
1320 * this instance is about to be registered.
1321 */
1322 trbe_check_errata(cpudata);
1323
1324 if (trbe_is_broken(cpudata)) {
1325 pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);
1326 goto cpu_clear;
1327 }
1328
1329 /*
1330 * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
1331 * we must always program the TBRPTR_EL1, 256bytes from a page
1332 * boundary, with TRBBASER_EL1 set to the page, to prevent
1333 * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.
1334 *
1335 * Thus make sure we always align our write pointer to a PAGE_SIZE,
1336 * which also guarantees that we have at least a PAGE_SIZE space in
1337 * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip
1338 * the required bytes at the base.
1339 */
1340 if (trbe_may_overwrite_in_fill_mode(cpudata))
1341 cpudata->trbe_align = PAGE_SIZE;
1342 else
1343 cpudata->trbe_align = cpudata->trbe_hw_align;
1344
1345 cpudata->trbe_flag = get_trbe_flag_update(trbidr);
1346 cpudata->cpu = cpu;
1347 cpudata->drvdata = drvdata;
1348 return;
1349cpu_clear:
1350 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1351}
1352
1353static void arm_trbe_remove_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1354{
1355 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1356
1357 if (trbe_csdev) {
1358 coresight_unregister(trbe_csdev);
1359 coresight_set_percpu_sink(cpu, NULL);
1360 }
1361}
1362
1363static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
1364{
1365 int cpu;
1366
1367 drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
1368 if (!drvdata->cpudata)
1369 return -ENOMEM;
1370
1371 for_each_cpu(cpu, &drvdata->supported_cpus) {
1372 /* If we fail to probe the CPU, let us defer it to hotplug callbacks */
1373 if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
1374 continue;
1375 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1376 arm_trbe_register_coresight_cpu(drvdata, cpu);
1377 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1378 smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);
1379 }
1380 return 0;
1381}
1382
1383static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
1384{
1385 int cpu;
1386
1387 for_each_cpu(cpu, &drvdata->supported_cpus) {
1388 smp_call_function_single(cpu, arm_trbe_disable_cpu, drvdata, 1);
1389 arm_trbe_remove_coresight_cpu(drvdata, cpu);
1390 }
1391 free_percpu(drvdata->cpudata);
1392 return 0;
1393}
1394
1395static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
1396{
1397 preempt_disable();
1398 arm_trbe_probe_cpu(drvdata);
1399 preempt_enable();
1400}
1401
1402static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
1403{
1404 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1405
1406 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1407
1408 /*
1409 * If this CPU was not probed for TRBE,
1410 * initialize it now.
1411 */
1412 if (!coresight_get_percpu_sink(cpu)) {
1413 arm_trbe_probe_hotplugged_cpu(drvdata);
1414 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1415 arm_trbe_register_coresight_cpu(drvdata, cpu);
1416 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1417 arm_trbe_enable_cpu(drvdata);
1418 } else {
1419 arm_trbe_enable_cpu(drvdata);
1420 }
1421 }
1422 return 0;
1423}
1424
1425static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1426{
1427 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1428
1429 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1430 arm_trbe_disable_cpu(drvdata);
1431 return 0;
1432}
1433
1434static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
1435{
1436 enum cpuhp_state trbe_online;
1437 int ret;
1438
1439 trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
1440 arm_trbe_cpu_startup, arm_trbe_cpu_teardown);
1441 if (trbe_online < 0)
1442 return trbe_online;
1443
1444 ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);
1445 if (ret) {
1446 cpuhp_remove_multi_state(trbe_online);
1447 return ret;
1448 }
1449 drvdata->trbe_online = trbe_online;
1450 return 0;
1451}
1452
1453static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
1454{
1455 cpuhp_state_remove_instance(drvdata->trbe_online, &drvdata->hotplug_node);
1456 cpuhp_remove_multi_state(drvdata->trbe_online);
1457}
1458
1459static int arm_trbe_probe_irq(struct platform_device *pdev,
1460 struct trbe_drvdata *drvdata)
1461{
1462 int ret;
1463
1464 drvdata->irq = platform_get_irq(pdev, 0);
1465 if (drvdata->irq < 0) {
1466 pr_err("IRQ not found for the platform device\n");
1467 return drvdata->irq;
1468 }
1469
1470 if (!irq_is_percpu(drvdata->irq)) {
1471 pr_err("IRQ is not a PPI\n");
1472 return -EINVAL;
1473 }
1474
1475 if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))
1476 return -EINVAL;
1477
1478 drvdata->handle = alloc_percpu(struct perf_output_handle *);
1479 if (!drvdata->handle)
1480 return -ENOMEM;
1481
1482 ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);
1483 if (ret) {
1484 free_percpu(drvdata->handle);
1485 return ret;
1486 }
1487 return 0;
1488}
1489
1490static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
1491{
1492 free_percpu_irq(drvdata->irq, drvdata->handle);
1493 free_percpu(drvdata->handle);
1494}
1495
1496static int arm_trbe_device_probe(struct platform_device *pdev)
1497{
1498 struct trbe_drvdata *drvdata;
1499 struct device *dev = &pdev->dev;
1500 int ret;
1501
1502 /* Trace capture is not possible with kernel page table isolation */
1503 if (arm64_kernel_unmapped_at_el0()) {
1504 pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
1505 return -EOPNOTSUPP;
1506 }
1507
1508 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1509 if (!drvdata)
1510 return -ENOMEM;
1511
1512 dev_set_drvdata(dev, drvdata);
1513 drvdata->pdev = pdev;
1514 ret = arm_trbe_probe_irq(pdev, drvdata);
1515 if (ret)
1516 return ret;
1517
1518 ret = arm_trbe_probe_coresight(drvdata);
1519 if (ret)
1520 goto probe_failed;
1521
1522 ret = arm_trbe_probe_cpuhp(drvdata);
1523 if (ret)
1524 goto cpuhp_failed;
1525
1526 return 0;
1527cpuhp_failed:
1528 arm_trbe_remove_coresight(drvdata);
1529probe_failed:
1530 arm_trbe_remove_irq(drvdata);
1531 return ret;
1532}
1533
1534static void arm_trbe_device_remove(struct platform_device *pdev)
1535{
1536 struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
1537
1538 arm_trbe_remove_cpuhp(drvdata);
1539 arm_trbe_remove_coresight(drvdata);
1540 arm_trbe_remove_irq(drvdata);
1541}
1542
1543static const struct of_device_id arm_trbe_of_match[] = {
1544 { .compatible = "arm,trace-buffer-extension"},
1545 {},
1546};
1547MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
1548
1549#ifdef CONFIG_ACPI
1550static const struct platform_device_id arm_trbe_acpi_match[] = {
1551 { ARMV8_TRBE_PDEV_NAME, 0 },
1552 { }
1553};
1554MODULE_DEVICE_TABLE(platform, arm_trbe_acpi_match);
1555#endif
1556
1557static struct platform_driver arm_trbe_driver = {
1558 .id_table = ACPI_PTR(arm_trbe_acpi_match),
1559 .driver = {
1560 .name = DRVNAME,
1561 .of_match_table = of_match_ptr(arm_trbe_of_match),
1562 .suppress_bind_attrs = true,
1563 },
1564 .probe = arm_trbe_device_probe,
1565 .remove = arm_trbe_device_remove,
1566};
1567
1568static int __init arm_trbe_init(void)
1569{
1570 int ret;
1571
1572 ret = platform_driver_register(&arm_trbe_driver);
1573 if (!ret)
1574 return 0;
1575
1576 pr_err("Error registering %s platform driver\n", DRVNAME);
1577 return ret;
1578}
1579
1580static void __exit arm_trbe_exit(void)
1581{
1582 platform_driver_unregister(&arm_trbe_driver);
1583}
1584module_init(arm_trbe_init);
1585module_exit(arm_trbe_exit);
1586
1587MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
1588MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
1589MODULE_LICENSE("GPL v2");