1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
   4 * sink device could then pair with an appropriate per-cpu coresight source
   5 * device (ETE) thus generating required trace data. Trace can be enabled
   6 * via the perf framework.
   7 *
   8 * The AUX buffer handling is inspired from Arm SPE PMU driver.
   9 *
  10 * Copyright (C) 2020 ARM Ltd.
  11 *
  12 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
  13 */
  14#define DRVNAME "arm_trbe"
  15
  16#define pr_fmt(fmt) DRVNAME ": " fmt
  17
  18#include <asm/barrier.h>
  19#include <asm/cpufeature.h>
  20
  21#include "coresight-self-hosted-trace.h"
  22#include "coresight-trbe.h"
  23
  24#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
  25
  26/*
  27 * A padding packet that will help the user space tools
  28 * in skipping relevant sections in the captured trace
  29 * data which could not be decoded. TRBE doesn't support
  30 * formatting the trace data, unlike the legacy CoreSight
  31 * sinks and thus we use ETE trace packets to pad the
  32 * sections of the buffer.
  33 */
  34#define ETE_IGNORE_PACKET		0x70
  35
  36/*
  37 * Minimum amount of meaningful trace will contain:
  38 * A-Sync, Trace Info, Trace On, Address, Atom.
  39 * This is about 44bytes of ETE trace. To be on
  40 * the safer side, we assume 64bytes is the minimum
  41 * space required for a meaningful session, before
  42 * we hit a "WRAP" event.
  43 */
  44#define TRBE_TRACE_MIN_BUF_SIZE		64
  45
  46enum trbe_fault_action {
  47	TRBE_FAULT_ACT_WRAP,
  48	TRBE_FAULT_ACT_SPURIOUS,
  49	TRBE_FAULT_ACT_FATAL,
  50};
  51
  52struct trbe_buf {
  53	/*
  54	 * Even though trbe_base represents vmap()
  55	 * mapped allocated buffer's start address,
  56	 * it's being as unsigned long for various
  57	 * arithmetic and comparision operations &
  58	 * also to be consistent with trbe_write &
  59	 * trbe_limit sibling pointers.
  60	 */
  61	unsigned long trbe_base;
  62	/* The base programmed into the TRBE */
  63	unsigned long trbe_hw_base;
  64	unsigned long trbe_limit;
  65	unsigned long trbe_write;
  66	int nr_pages;
  67	void **pages;
  68	bool snapshot;
  69	struct trbe_cpudata *cpudata;
  70};
  71
  72/*
  73 * TRBE erratum list
  74 *
  75 * The errata are defined in arm64 generic cpu_errata framework.
  76 * Since the errata work arounds could be applied individually
  77 * to the affected CPUs inside the TRBE driver, we need to know if
  78 * a given CPU is affected by the erratum. Unlike the other erratum
  79 * work arounds, TRBE driver needs to check multiple times during
  80 * a trace session. Thus we need a quicker access to per-CPU
  81 * errata and not issue costly this_cpu_has_cap() everytime.
  82 * We keep a set of the affected errata in trbe_cpudata, per TRBE.
  83 *
  84 * We rely on the corresponding cpucaps to be defined for a given
  85 * TRBE erratum. We map the given cpucap into a TRBE internal number
  86 * to make the tracking of the errata lean.
  87 *
  88 * This helps in :
  89 *   - Not duplicating the detection logic
  90 *   - Streamlined detection of erratum across the system
  91 */
  92#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE	0
  93#define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE	1
  94#define TRBE_NEEDS_DRAIN_AFTER_DISABLE		2
  95#define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE	3
  96#define TRBE_IS_BROKEN				4
  97
  98static int trbe_errata_cpucaps[] = {
  99	[TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
 100	[TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
 101	[TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,
 102	[TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,
 103	[TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,
 104	-1,		/* Sentinel, must be the last entry */
 105};
 106
 107/* The total number of listed errata in trbe_errata_cpucaps */
 108#define TRBE_ERRATA_MAX			(ARRAY_SIZE(trbe_errata_cpucaps) - 1)
 109
 110/*
 111 * Safe limit for the number of bytes that may be overwritten
 112 * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.
 113 */
 114#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES	256
 115
 116/*
 117 * struct trbe_cpudata: TRBE instance specific data
 118 * @trbe_flag		- TRBE dirty/access flag support
 119 * @trbe_hw_align	- Actual TRBE alignment required for TRBPTR_EL1.
 120 * @trbe_align		- Software alignment used for the TRBPTR_EL1.
 121 * @cpu			- CPU this TRBE belongs to.
 122 * @mode		- Mode of current operation. (perf/disabled)
 123 * @drvdata		- TRBE specific drvdata
 124 * @errata		- Bit map for the errata on this TRBE.
 125 */
 126struct trbe_cpudata {
 127	bool trbe_flag;
 128	u64 trbe_hw_align;
 129	u64 trbe_align;
 130	int cpu;
 131	enum cs_mode mode;
 132	struct trbe_buf *buf;
 133	struct trbe_drvdata *drvdata;
 134	DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
 135};
 136
 137struct trbe_drvdata {
 138	struct trbe_cpudata __percpu *cpudata;
 139	struct perf_output_handle * __percpu *handle;
 140	struct hlist_node hotplug_node;
 141	int irq;
 142	cpumask_t supported_cpus;
 143	enum cpuhp_state trbe_online;
 144	struct platform_device *pdev;
 145};
 146
 147static void trbe_check_errata(struct trbe_cpudata *cpudata)
 148{
 149	int i;
 150
 151	for (i = 0; i < TRBE_ERRATA_MAX; i++) {
 152		int cap = trbe_errata_cpucaps[i];
 153
 154		if (WARN_ON_ONCE(cap < 0))
 155			return;
 156		if (this_cpu_has_cap(cap))
 157			set_bit(i, cpudata->errata);
 158	}
 159}
 160
 161static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
 162{
 163	return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
 164}
 165
 166static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)
 167{
 168	return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);
 169}
 170
 171static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)
 172{
 173	return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);
 174}
 175
 176static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)
 177{
 178	/*
 179	 * Errata affected TRBE implementation will need TSB CSYNC and
 180	 * DSB in order to prevent subsequent writes into certain TRBE
 181	 * system registers from being ignored and not effected.
 182	 */
 183	return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);
 184}
 185
 186static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)
 187{
 188	/*
 189	 * Errata affected TRBE implementation will need an additional
 190	 * context synchronization in order to prevent an inconsistent
 191	 * TRBE prohibited region view on the CPU which could possibly
 192	 * corrupt the TRBE buffer or the TRBE state.
 193	 */
 194	return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);
 195}
 196
 197static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)
 198{
 199	return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);
 200}
 201
 202static int trbe_alloc_node(struct perf_event *event)
 203{
 204	if (event->cpu == -1)
 205		return NUMA_NO_NODE;
 206	return cpu_to_node(event->cpu);
 207}
 208
 209static inline void trbe_drain_buffer(void)
 210{
 211	tsb_csync();
 212	dsb(nsh);
 213}
 214
 215static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)
 216{
 217	/*
 218	 * Enable the TRBE without clearing LIMITPTR which
 219	 * might be required for fetching the buffer limits.
 220	 */
 221	trblimitr |= TRBLIMITR_EL1_E;
 222	write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
 223
 224	/* Synchronize the TRBE enable event */
 225	isb();
 226
 227	if (trbe_needs_ctxt_sync_after_enable(cpudata))
 228		isb();
 229}
 230
 231static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)
 232{
 233	u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
 234
 235	/*
 236	 * Disable the TRBE without clearing LIMITPTR which
 237	 * might be required for fetching the buffer limits.
 238	 */
 239	trblimitr &= ~TRBLIMITR_EL1_E;
 240	write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
 241
 242	if (trbe_needs_drain_after_disable(cpudata))
 243		trbe_drain_buffer();
 244	isb();
 245}
 246
 247static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)
 248{
 249	trbe_drain_buffer();
 250	set_trbe_disabled(cpudata);
 251}
 252
 253static void trbe_reset_local(struct trbe_cpudata *cpudata)
 254{
 255	trbe_drain_and_disable_local(cpudata);
 256	write_sysreg_s(0, SYS_TRBLIMITR_EL1);
 257	write_sysreg_s(0, SYS_TRBPTR_EL1);
 258	write_sysreg_s(0, SYS_TRBBASER_EL1);
 259	write_sysreg_s(0, SYS_TRBSR_EL1);
 260}
 261
 262static void trbe_report_wrap_event(struct perf_output_handle *handle)
 263{
 264	/*
 265	 * Mark the buffer to indicate that there was a WRAP event by
 266	 * setting the COLLISION flag. This indicates to the user that
 267	 * the TRBE trace collection was stopped without stopping the
 268	 * ETE and thus there might be some amount of trace that was
 269	 * lost between the time the WRAP was detected and the IRQ
 270	 * was consumed by the CPU.
 271	 *
 272	 * Setting the TRUNCATED flag would move the event to STOPPED
 273	 * state unnecessarily, even when there is space left in the
 274	 * ring buffer. Using the COLLISION flag doesn't have this side
 275	 * effect. We only set TRUNCATED flag when there is no space
 276	 * left in the ring buffer.
 277	 */
 278	perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
 279}
 280
 281static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
 282{
 283	struct trbe_buf *buf = etm_perf_sink_config(handle);
 284
 285	/*
 286	 * We cannot proceed with the buffer collection and we
 287	 * do not have any data for the current session. The
 288	 * etm_perf driver expects to close out the aux_buffer
 289	 * at event_stop(). So disable the TRBE here and leave
 290	 * the update_buffer() to return a 0 size.
 291	 */
 292	trbe_drain_and_disable_local(buf->cpudata);
 293	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 294	perf_aux_output_end(handle, 0);
 295	*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
 296}
 297
 298/*
 299 * TRBE Buffer Management
 300 *
 301 * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
 302 * it starts writing trace data from the write pointer onward till the limit pointer.
 303 * When the write pointer reaches the address just before the limit pointer, it gets
 304 * wrapped around again to the base pointer. This is called a TRBE wrap event, which
 305 * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
 306 * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
 307 * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
 308 * LIMIT pointers.
 309 *
 310 *	Wrap around with an IRQ
 311 *	------ < ------ < ------- < ----- < -----
 312 *	|					|
 313 *	------ > ------ > ------- > ----- > -----
 314 *
 315 *	+---------------+-----------------------+
 316 *	|		|			|
 317 *	+---------------+-----------------------+
 318 *	Base Pointer	Write Pointer		Limit Pointer
 319 *
 320 * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
 321 * pointer can be aligned to the implementation defined TRBE trace buffer alignment
 322 * as captured in trbe_cpudata->trbe_align.
 323 *
 324 *
 325 *		head		tail		wakeup
 326 *	+---------------------------------------+----- ~ ~ ------
 327 *	|$$$$$$$|################|$$$$$$$$$$$$$$|		|
 328 *	+---------------------------------------+----- ~ ~ ------
 329 *	Base Pointer	Write Pointer		Limit Pointer
 330 *
 331 * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
 332 * values which tracks all the driver writes and user reads from the perf auxiliary
 333 * buffer. Generally [head..tail] is the area where the driver can write into unless
 334 * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
 335 * configured depending on the perf_output_handle indices, so that the driver does
 336 * not override into areas in the perf auxiliary buffer which is being or yet to be
 337 * consumed from the user space. The enabled TRBE buffer area is a moving subset of
 338 * the allocated perf auxiliary buffer.
 339 */
 340
 341static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
 342{
 343	memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
 344}
 345
 346static void trbe_pad_buf(struct perf_output_handle *handle, int len)
 347{
 348	struct trbe_buf *buf = etm_perf_sink_config(handle);
 349	u64 head = PERF_IDX2OFF(handle->head, buf);
 350
 351	__trbe_pad_buf(buf, head, len);
 352	if (!buf->snapshot)
 353		perf_aux_output_skip(handle, len);
 354}
 355
 356static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
 357{
 358	struct trbe_buf *buf = etm_perf_sink_config(handle);
 359
 360	/*
 361	 * The ETE trace has alignment synchronization packets allowing
 362	 * the decoder to reset in case of an overflow or corruption.
 363	 * So we can use the entire buffer for the snapshot mode.
 364	 */
 365	return buf->nr_pages * PAGE_SIZE;
 366}
 367
 368static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)
 369{
 370	u64 size = TRBE_TRACE_MIN_BUF_SIZE;
 371	struct trbe_buf *buf = etm_perf_sink_config(handle);
 372	struct trbe_cpudata *cpudata = buf->cpudata;
 373
 374	/*
 375	 * When the TRBE is affected by an erratum that could make it
 376	 * write to the next "virtually addressed" page beyond the LIMIT.
 377	 * We need to make sure there is always a PAGE after the LIMIT,
 378	 * within the buffer. Thus we ensure there is at least an extra
 379	 * page than normal. With this we could then adjust the LIMIT
 380	 * pointer down by a PAGE later.
 381	 */
 382	if (trbe_may_write_out_of_range(cpudata))
 383		size += PAGE_SIZE;
 384	return size;
 385}
 386
 387/*
 388 * TRBE Limit Calculation
 389 *
 390 * The following markers are used to illustrate various TRBE buffer situations.
 391 *
 392 * $$$$ - Data area, unconsumed captured trace data, not to be overridden
 393 * #### - Free area, enabled, trace will be written
 394 * %%%% - Free area, disabled, trace will not be written
 395 * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
 396 */
 397static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
 398{
 399	struct trbe_buf *buf = etm_perf_sink_config(handle);
 400	struct trbe_cpudata *cpudata = buf->cpudata;
 401	const u64 bufsize = buf->nr_pages * PAGE_SIZE;
 402	u64 limit = bufsize;
 403	u64 head, tail, wakeup;
 404
 405	head = PERF_IDX2OFF(handle->head, buf);
 406
 407	/*
 408	 *		head
 409	 *	------->|
 410	 *	|
 411	 *	head	TRBE align	tail
 412	 * +----|-------|---------------|-------+
 413	 * |$$$$|=======|###############|$$$$$$$|
 414	 * +----|-------|---------------|-------+
 415	 * trbe_base				trbe_base + nr_pages
 416	 *
 417	 * Perf aux buffer output head position can be misaligned depending on
 418	 * various factors including user space reads. In case misaligned, head
 419	 * needs to be aligned before TRBE can be configured. Pad the alignment
 420	 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
 421	 * and skip this section thus advancing the head.
 422	 */
 423	if (!IS_ALIGNED(head, cpudata->trbe_align)) {
 424		unsigned long delta = roundup(head, cpudata->trbe_align) - head;
 425
 426		delta = min(delta, handle->size);
 427		trbe_pad_buf(handle, delta);
 428		head = PERF_IDX2OFF(handle->head, buf);
 429	}
 430
 431	/*
 432	 *	head = tail (size = 0)
 433	 * +----|-------------------------------+
 434	 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$	|
 435	 * +----|-------------------------------+
 436	 * trbe_base				trbe_base + nr_pages
 437	 *
 438	 * Perf aux buffer does not have any space for the driver to write into.
 439	 */
 440	if (!handle->size)
 441		return 0;
 442
 443	/* Compute the tail and wakeup indices now that we've aligned head */
 444	tail = PERF_IDX2OFF(handle->head + handle->size, buf);
 445	wakeup = PERF_IDX2OFF(handle->wakeup, buf);
 446
 447	/*
 448	 * Lets calculate the buffer area which TRBE could write into. There
 449	 * are three possible scenarios here. Limit needs to be aligned with
 450	 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
 451	 * unconsumed data.
 452	 *
 453	 * 1) head < tail
 454	 *
 455	 *	head			tail
 456	 * +----|-----------------------|-------+
 457	 * |$$$$|#######################|$$$$$$$|
 458	 * +----|-----------------------|-------+
 459	 * trbe_base			limit	trbe_base + nr_pages
 460	 *
 461	 * TRBE could write into [head..tail] area. Unless the tail is right at
 462	 * the end of the buffer, neither an wrap around nor an IRQ is expected
 463	 * while being enabled.
 464	 *
 465	 * 2) head == tail
 466	 *
 467	 *	head = tail (size > 0)
 468	 * +----|-------------------------------+
 469	 * |%%%%|###############################|
 470	 * +----|-------------------------------+
 471	 * trbe_base				limit = trbe_base + nr_pages
 472	 *
 473	 * TRBE should just write into [head..base + nr_pages] area even though
 474	 * the entire buffer is empty. Reason being, when the trace reaches the
 475	 * end of the buffer, it will just wrap around with an IRQ giving an
 476	 * opportunity to reconfigure the buffer.
 477	 *
 478	 * 3) tail < head
 479	 *
 480	 *	tail			head
 481	 * +----|-----------------------|-------+
 482	 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
 483	 * +----|-----------------------|-------+
 484	 * trbe_base				limit = trbe_base + nr_pages
 485	 *
 486	 * TRBE should just write into [head..base + nr_pages] area even though
 487	 * the [trbe_base..tail] is also empty. Reason being, when the trace
 488	 * reaches the end of the buffer, it will just wrap around with an IRQ
 489	 * giving an opportunity to reconfigure the buffer.
 490	 */
 491	if (head < tail)
 492		limit = round_down(tail, PAGE_SIZE);
 493
 494	/*
 495	 * Wakeup may be arbitrarily far into the future. If it's not in the
 496	 * current generation, either we'll wrap before hitting it, or it's
 497	 * in the past and has been handled already.
 498	 *
 499	 * If there's a wakeup before we wrap, arrange to be woken up by the
 500	 * page boundary following it. Keep the tail boundary if that's lower.
 501	 *
 502	 *	head		wakeup	tail
 503	 * +----|---------------|-------|-------+
 504	 * |$$$$|###############|%%%%%%%|$$$$$$$|
 505	 * +----|---------------|-------|-------+
 506	 * trbe_base		limit		trbe_base + nr_pages
 507	 */
 508	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
 509		limit = min(limit, round_up(wakeup, PAGE_SIZE));
 510
 511	/*
 512	 * There are two situation when this can happen i.e limit is before
 513	 * the head and hence TRBE cannot be configured.
 514	 *
 515	 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
 516	 * within the same PAGE size range.
 517	 *
 518	 *			PAGE_SIZE
 519	 *		|----------------------|
 520	 *
 521	 *		limit	head	tail
 522	 * +------------|------|--------|-------+
 523	 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
 524	 * +------------|------|--------|-------+
 525	 * trbe_base				trbe_base + nr_pages
 526	 *
 527	 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
 528	 * head and wakeup are within same PAGE size range.
 529	 *
 530	 *		PAGE_SIZE
 531	 *	|----------------------|
 532	 *
 533	 *	limit	head	wakeup  tail
 534	 * +----|------|-------|--------|-------+
 535	 * |$$$$$$$$$$$|=======|========|$$$$$$$|
 536	 * +----|------|-------|--------|-------+
 537	 * trbe_base				trbe_base + nr_pages
 538	 */
 539	if (limit > head)
 540		return limit;
 541
 542	trbe_pad_buf(handle, handle->size);
 543	return 0;
 544}
 545
 546static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
 547{
 548	struct trbe_buf *buf = etm_perf_sink_config(handle);
 549	u64 limit = __trbe_normal_offset(handle);
 550	u64 head = PERF_IDX2OFF(handle->head, buf);
 551
 552	/*
 553	 * If the head is too close to the limit and we don't
 554	 * have space for a meaningful run, we rather pad it
 555	 * and start fresh.
 556	 *
 557	 * We might have to do this more than once to make sure
 558	 * we have enough required space.
 559	 */
 560	while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {
 561		trbe_pad_buf(handle, limit - head);
 562		limit = __trbe_normal_offset(handle);
 563		head = PERF_IDX2OFF(handle->head, buf);
 564	}
 565	return limit;
 566}
 567
 568static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
 569{
 570	struct trbe_buf *buf = etm_perf_sink_config(handle);
 571	unsigned long offset;
 572
 573	if (buf->snapshot)
 574		offset = trbe_snapshot_offset(handle);
 575	else
 576		offset = trbe_normal_offset(handle);
 577	return buf->trbe_base + offset;
 578}
 579
 580static void clr_trbe_status(void)
 581{
 582	u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
 583
 584	WARN_ON(is_trbe_enabled());
 585	trbsr &= ~TRBSR_EL1_IRQ;
 586	trbsr &= ~TRBSR_EL1_TRG;
 587	trbsr &= ~TRBSR_EL1_WRAP;
 588	trbsr &= ~TRBSR_EL1_EC_MASK;
 589	trbsr &= ~TRBSR_EL1_BSC_MASK;
 590	trbsr &= ~TRBSR_EL1_S;
 591	write_sysreg_s(trbsr, SYS_TRBSR_EL1);
 592}
 593
 594static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
 595{
 596	u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
 597	unsigned long addr = buf->trbe_limit;
 598
 599	WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT)));
 600	WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
 601
 602	trblimitr &= ~TRBLIMITR_EL1_nVM;
 603	trblimitr &= ~TRBLIMITR_EL1_FM_MASK;
 604	trblimitr &= ~TRBLIMITR_EL1_TM_MASK;
 605	trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;
 606
 607	/*
 608	 * Fill trace buffer mode is used here while configuring the
 609	 * TRBE for trace capture. In this particular mode, the trace
 610	 * collection is stopped and a maintenance interrupt is raised
 611	 * when the current write pointer wraps. This pause in trace
 612	 * collection gives the software an opportunity to capture the
 613	 * trace data in the interrupt handler, before reconfiguring
 614	 * the TRBE.
 615	 */
 616	trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) &
 617		     TRBLIMITR_EL1_FM_MASK;
 618
 619	/*
 620	 * Trigger mode is not used here while configuring the TRBE for
 621	 * the trace capture. Hence just keep this in the ignore mode.
 622	 */
 623	trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) &
 624		     TRBLIMITR_EL1_TM_MASK;
 625	trblimitr |= (addr & PAGE_MASK);
 626	set_trbe_enabled(buf->cpudata, trblimitr);
 627}
 628
 629static void trbe_enable_hw(struct trbe_buf *buf)
 630{
 631	WARN_ON(buf->trbe_hw_base < buf->trbe_base);
 632	WARN_ON(buf->trbe_write < buf->trbe_hw_base);
 633	WARN_ON(buf->trbe_write >= buf->trbe_limit);
 634	set_trbe_disabled(buf->cpudata);
 635	clr_trbe_status();
 636	set_trbe_base_pointer(buf->trbe_hw_base);
 637	set_trbe_write_pointer(buf->trbe_write);
 638
 639	/*
 640	 * Synchronize all the register updates
 641	 * till now before enabling the TRBE.
 642	 */
 643	isb();
 644	set_trbe_limit_pointer_enabled(buf);
 645}
 646
 647static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,
 648						 u64 trbsr)
 649{
 650	int ec = get_trbe_ec(trbsr);
 651	int bsc = get_trbe_bsc(trbsr);
 652	struct trbe_buf *buf = etm_perf_sink_config(handle);
 653	struct trbe_cpudata *cpudata = buf->cpudata;
 654
 655	WARN_ON(is_trbe_running(trbsr));
 656	if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
 657		return TRBE_FAULT_ACT_FATAL;
 658
 659	if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
 660		return TRBE_FAULT_ACT_FATAL;
 661
 662	/*
 663	 * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
 664	 * it might write data after a WRAP event in the fill mode.
 665	 * Thus the check TRBPTR == TRBBASER will not be honored.
 666	 */
 667	if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&
 668	    (trbe_may_overwrite_in_fill_mode(cpudata) ||
 669	     get_trbe_write_pointer() == get_trbe_base_pointer()))
 670		return TRBE_FAULT_ACT_WRAP;
 671
 672	return TRBE_FAULT_ACT_SPURIOUS;
 673}
 674
 675static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
 676					 struct trbe_buf *buf, bool wrap)
 677{
 678	u64 write;
 679	u64 start_off, end_off;
 680	u64 size;
 681	u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
 682
 683	/*
 684	 * If the TRBE has wrapped around the write pointer has
 685	 * wrapped and should be treated as limit.
 686	 *
 687	 * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,
 688	 * it may write upto 64bytes beyond the "LIMIT". The driver already
 689	 * keeps a valid page next to the LIMIT and we could potentially
 690	 * consume the trace data that may have been collected there. But we
 691	 * cannot be really sure it is available, and the TRBPTR may not
 692	 * indicate the same. Also, affected cores are also affected by another
 693	 * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus
 694	 * could potentially pad an entire PAGE_SIZE - 64bytes, to get those
 695	 * 64bytes. Thus we ignore the potential triggering of the erratum
 696	 * on WRAP and limit the data to LIMIT.
 697	 */
 698	if (wrap)
 699		write = get_trbe_limit_pointer();
 700	else
 701		write = get_trbe_write_pointer();
 702
 703	/*
 704	 * TRBE may use a different base address than the base
 705	 * of the ring buffer. Thus use the beginning of the ring
 706	 * buffer to compute the offsets.
 707	 */
 708	end_off = write - buf->trbe_base;
 709	start_off = PERF_IDX2OFF(handle->head, buf);
 710
 711	if (WARN_ON_ONCE(end_off < start_off))
 712		return 0;
 713
 714	size = end_off - start_off;
 715	/*
 716	 * If the TRBE is affected by the following erratum, we must fill
 717	 * the space we skipped with IGNORE packets. And we are always
 718	 * guaranteed to have at least a PAGE_SIZE space in the buffer.
 719	 */
 720	if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&
 721	    !WARN_ON(size < overwrite_skip))
 722		__trbe_pad_buf(buf, start_off, overwrite_skip);
 723
 724	return size;
 725}
 726
 727static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
 728				   struct perf_event *event, void **pages,
 729				   int nr_pages, bool snapshot)
 730{
 731	struct trbe_buf *buf;
 732	struct page **pglist;
 733	int i;
 734
 735	/*
 736	 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
 737	 * just a single page, there would not be any room left while writing
 738	 * into a partially filled TRBE buffer after the page size alignment.
 739	 * Hence restrict the minimum buffer size as two pages.
 740	 */
 741	if (nr_pages < 2)
 742		return NULL;
 743
 744	buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));
 745	if (!buf)
 746		return ERR_PTR(-ENOMEM);
 747
 748	pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
 749	if (!pglist) {
 750		kfree(buf);
 751		return ERR_PTR(-ENOMEM);
 752	}
 753
 754	for (i = 0; i < nr_pages; i++)
 755		pglist[i] = virt_to_page(pages[i]);
 756
 757	buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
 758	if (!buf->trbe_base) {
 759		kfree(pglist);
 760		kfree(buf);
 761		return ERR_PTR(-ENOMEM);
 762	}
 763	buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
 764	buf->trbe_write = buf->trbe_base;
 765	buf->snapshot = snapshot;
 766	buf->nr_pages = nr_pages;
 767	buf->pages = pages;
 768	kfree(pglist);
 769	return buf;
 770}
 771
 772static void arm_trbe_free_buffer(void *config)
 773{
 774	struct trbe_buf *buf = config;
 775
 776	vunmap((void *)buf->trbe_base);
 777	kfree(buf);
 778}
 779
 780static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
 781					    struct perf_output_handle *handle,
 782					    void *config)
 783{
 784	struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 785	struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
 786	struct trbe_buf *buf = config;
 787	enum trbe_fault_action act;
 788	unsigned long size, status;
 789	unsigned long flags;
 790	bool wrap = false;
 791
 792	WARN_ON(buf->cpudata != cpudata);
 793	WARN_ON(cpudata->cpu != smp_processor_id());
 794	WARN_ON(cpudata->drvdata != drvdata);
 795	if (cpudata->mode != CS_MODE_PERF)
 796		return 0;
 797
 798	/*
 799	 * We are about to disable the TRBE. And this could in turn
 800	 * fill up the buffer triggering, an IRQ. This could be consumed
 801	 * by the PE asynchronously, causing a race here against
 802	 * the IRQ handler in closing out the handle. So, let us
 803	 * make sure the IRQ can't trigger while we are collecting
 804	 * the buffer. We also make sure that a WRAP event is handled
 805	 * accordingly.
 806	 */
 807	local_irq_save(flags);
 808
 809	/*
 810	 * If the TRBE was disabled due to lack of space in the AUX buffer or a
 811	 * spurious fault, the driver leaves it disabled, truncating the buffer.
 812	 * Since the etm_perf driver expects to close out the AUX buffer, the
 813	 * driver skips it. Thus, just pass in 0 size here to indicate that the
 814	 * buffer was truncated.
 815	 */
 816	if (!is_trbe_enabled()) {
 817		size = 0;
 818		goto done;
 819	}
 820	/*
 821	 * perf handle structure needs to be shared with the TRBE IRQ handler for
 822	 * capturing trace data and restarting the handle. There is a probability
 823	 * of an undefined reference based crash when etm event is being stopped
 824	 * while a TRBE IRQ also getting processed. This happens due the release
 825	 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
 826	 * the TRBE here will ensure that no IRQ could be generated when the perf
 827	 * handle gets freed in etm_event_stop().
 828	 */
 829	trbe_drain_and_disable_local(cpudata);
 830
 831	/* Check if there is a pending interrupt and handle it here */
 832	status = read_sysreg_s(SYS_TRBSR_EL1);
 833	if (is_trbe_irq(status)) {
 834
 835		/*
 836		 * Now that we are handling the IRQ here, clear the IRQ
 837		 * from the status, to let the irq handler know that it
 838		 * is taken care of.
 839		 */
 840		clr_trbe_irq();
 841		isb();
 842
 843		act = trbe_get_fault_act(handle, status);
 844		/*
 845		 * If this was not due to a WRAP event, we have some
 846		 * errors and as such buffer is empty.
 847		 */
 848		if (act != TRBE_FAULT_ACT_WRAP) {
 849			size = 0;
 850			goto done;
 851		}
 852
 853		trbe_report_wrap_event(handle);
 854		wrap = true;
 855	}
 856
 857	size = trbe_get_trace_size(handle, buf, wrap);
 858
 859done:
 860	local_irq_restore(flags);
 861
 862	if (buf->snapshot)
 863		handle->head += size;
 864	return size;
 865}
 866
 867
 868static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)
 869{
 870	/*
 871	 * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache
 872	 * line size from the "TRBBASER_EL1" in the event of a "FILL".
 873	 * Thus, we could loose some amount of the trace at the base.
 874	 *
 875	 * Before Fix:
 876	 *
 877	 *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
 878	 *  |                   \/                       /
 879	 *   -------------------------------------------------------------
 880	 *  |   Pg0      |   Pg1       |           |          |  PgN     |
 881	 *   -------------------------------------------------------------
 882	 *
 883	 * In the normal course of action, we would set the TRBBASER to the
 884	 * beginning of the ring-buffer (normal-BASE). But with the erratum,
 885	 * the TRBE could overwrite the contents at the "normal-BASE", after
 886	 * hitting the "normal-LIMIT", since it doesn't stop as expected. And
 887	 * this is wrong. This could result in overwriting trace collected in
 888	 * one of the previous runs, being consumed by the user. So we must
 889	 * always make sure that the TRBBASER is within the region
 890	 * [head, head+size]. Note that TRBBASER must be PAGE aligned,
 891	 *
 892	 *  After moving the BASE:
 893	 *
 894	 *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
 895	 *  |                   \/                       /
 896	 *   -------------------------------------------------------------
 897	 *  |         |          |xyzdef.     |..   tuvw|                |
 898	 *   -------------------------------------------------------------
 899	 *                      /
 900	 *              New-BASER
 901	 *
 902	 * Also, we would set the TRBPTR to head (after adjusting for
 903	 * alignment) at normal-PTR. This would mean that the last few bytes
 904	 * of the trace (say, "xyz") might overwrite the first few bytes of
 905	 * trace written ("abc"). More importantly they will appear in what
 906	 * userspace sees as the beginning of the trace, which is wrong. We may
 907	 * not always have space to move the latest trace "xyz" to the correct
 908	 * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).
 909	 * Thus it is easier to ignore those bytes than to complicate the
 910	 * driver to move it, assuming that the erratum was triggered and
 911	 * doing additional checks to see if there is indeed allowed space at
 912	 * TRBLIMITR.LIMIT.
 913	 *
 914	 *  Thus the full workaround will move the BASE and the PTR and would
 915	 *  look like (after padding at the skipped bytes at the end of
 916	 *  session) :
 917	 *
 918	 *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
 919	 *  |                   \/                       /
 920	 *   -------------------------------------------------------------
 921	 *  |         |          |///abc..     |..  rst|                |
 922	 *   -------------------------------------------------------------
 923	 *                      /    |
 924	 *              New-BASER    New-TRBPTR
 925	 *
 926	 * To summarize, with the work around:
 927	 *
 928	 *  - We always align the offset for the next session to PAGE_SIZE
 929	 *    (This is to ensure we can program the TRBBASER to this offset
 930	 *    within the region [head...head+size]).
 931	 *
 932	 *  - At TRBE enable:
 933	 *     - Set the TRBBASER to the page aligned offset of the current
 934	 *       proposed write offset. (which is guaranteed to be aligned
 935	 *       as above)
 936	 *     - Move the TRBPTR to skip first 256bytes (that might be
 937	 *       overwritten with the erratum). This ensures that the trace
 938	 *       generated in the session is not re-written.
 939	 *
 940	 *  - At trace collection:
 941	 *     - Pad the 256bytes skipped above again with IGNORE packets.
 942	 */
 943	if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {
 944		if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))
 945			return -EINVAL;
 946		buf->trbe_hw_base = buf->trbe_write;
 947		buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
 948	}
 949
 950	/*
 951	 * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to
 952	 * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"
 953	 * may be:
 954	 *     - The page beyond the ring buffer. This could mean, TRBE could
 955	 *       corrupt another entity (kernel / user)
 956	 *     - A portion of the "ring buffer" consumed by the userspace.
 957	 *       i.e, a page outisde [head, head + size].
 958	 *
 959	 * We work around this by:
 960	 *     - Making sure that we have at least an extra space of PAGE left
 961	 *       in the ring buffer [head, head + size], than we normally do
 962	 *       without the erratum. See trbe_min_trace_buf_size().
 963	 *
 964	 *     - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside
 965	 *       the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).
 966	 */
 967	if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {
 968		s64 space = buf->trbe_limit - buf->trbe_write;
 969		/*
 970		 * We must have more than a PAGE_SIZE worth space in the proposed
 971		 * range for the TRBE.
 972		 */
 973		if (WARN_ON(space <= PAGE_SIZE ||
 974			    !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))
 975			return -EINVAL;
 976		buf->trbe_limit -= PAGE_SIZE;
 977	}
 978
 979	return 0;
 980}
 981
 982static int __arm_trbe_enable(struct trbe_buf *buf,
 983			     struct perf_output_handle *handle)
 984{
 985	int ret = 0;
 986
 987	perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
 988	buf->trbe_limit = compute_trbe_buffer_limit(handle);
 989	buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
 990	if (buf->trbe_limit == buf->trbe_base) {
 991		ret = -ENOSPC;
 992		goto err;
 993	}
 994	/* Set the base of the TRBE to the buffer base */
 995	buf->trbe_hw_base = buf->trbe_base;
 996
 997	ret = trbe_apply_work_around_before_enable(buf);
 998	if (ret)
 999		goto err;
1000
1001	*this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
1002	trbe_enable_hw(buf);
1003	return 0;
1004err:
1005	trbe_stop_and_truncate_event(handle);
1006	return ret;
1007}
1008
1009static int arm_trbe_enable(struct coresight_device *csdev, enum cs_mode mode,
1010			   void *data)
1011{
1012	struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1013	struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1014	struct perf_output_handle *handle = data;
1015	struct trbe_buf *buf = etm_perf_sink_config(handle);
1016
1017	WARN_ON(cpudata->cpu != smp_processor_id());
1018	WARN_ON(cpudata->drvdata != drvdata);
1019	if (mode != CS_MODE_PERF)
1020		return -EINVAL;
1021
1022	cpudata->buf = buf;
1023	cpudata->mode = mode;
1024	buf->cpudata = cpudata;
1025
1026	return __arm_trbe_enable(buf, handle);
1027}
1028
1029static int arm_trbe_disable(struct coresight_device *csdev)
1030{
1031	struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1032	struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1033	struct trbe_buf *buf = cpudata->buf;
1034
1035	WARN_ON(buf->cpudata != cpudata);
1036	WARN_ON(cpudata->cpu != smp_processor_id());
1037	WARN_ON(cpudata->drvdata != drvdata);
1038	if (cpudata->mode != CS_MODE_PERF)
1039		return -EINVAL;
1040
1041	trbe_drain_and_disable_local(cpudata);
1042	buf->cpudata = NULL;
1043	cpudata->buf = NULL;
1044	cpudata->mode = CS_MODE_DISABLED;
1045	return 0;
1046}
1047
1048static void trbe_handle_spurious(struct perf_output_handle *handle)
1049{
1050	struct trbe_buf *buf = etm_perf_sink_config(handle);
1051	u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
1052
1053	/*
1054	 * If the IRQ was spurious, simply re-enable the TRBE
1055	 * back without modifying the buffer parameters to
1056	 * retain the trace collected so far.
1057	 */
1058	set_trbe_enabled(buf->cpudata, trblimitr);
1059}
1060
1061static int trbe_handle_overflow(struct perf_output_handle *handle)
1062{
1063	struct perf_event *event = handle->event;
1064	struct trbe_buf *buf = etm_perf_sink_config(handle);
1065	unsigned long size;
1066	struct etm_event_data *event_data;
1067
1068	size = trbe_get_trace_size(handle, buf, true);
1069	if (buf->snapshot)
1070		handle->head += size;
1071
1072	trbe_report_wrap_event(handle);
1073	perf_aux_output_end(handle, size);
1074	event_data = perf_aux_output_begin(handle, event);
1075	if (!event_data) {
1076		/*
1077		 * We are unable to restart the trace collection,
1078		 * thus leave the TRBE disabled. The etm-perf driver
1079		 * is able to detect this with a disconnected handle
1080		 * (handle->event = NULL).
1081		 */
1082		trbe_drain_and_disable_local(buf->cpudata);
1083		*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
1084		return -EINVAL;
1085	}
1086
1087	return __arm_trbe_enable(buf, handle);
1088}
1089
1090static bool is_perf_trbe(struct perf_output_handle *handle)
1091{
1092	struct trbe_buf *buf = etm_perf_sink_config(handle);
1093	struct trbe_cpudata *cpudata = buf->cpudata;
1094	struct trbe_drvdata *drvdata = cpudata->drvdata;
1095	int cpu = smp_processor_id();
1096
1097	WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
1098	WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
1099
1100	if (cpudata->mode != CS_MODE_PERF)
1101		return false;
1102
1103	if (cpudata->cpu != cpu)
1104		return false;
1105
1106	if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1107		return false;
1108
1109	return true;
1110}
1111
1112static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
1113{
1114	struct perf_output_handle **handle_ptr = dev;
1115	struct perf_output_handle *handle = *handle_ptr;
1116	struct trbe_buf *buf = etm_perf_sink_config(handle);
1117	enum trbe_fault_action act;
1118	u64 status;
1119	bool truncated = false;
1120	u64 trfcr;
1121
1122	/* Reads to TRBSR_EL1 is fine when TRBE is active */
1123	status = read_sysreg_s(SYS_TRBSR_EL1);
1124	/*
1125	 * If the pending IRQ was handled by update_buffer callback
1126	 * we have nothing to do here.
1127	 */
1128	if (!is_trbe_irq(status))
1129		return IRQ_NONE;
1130
1131	/* Prohibit the CPU from tracing before we disable the TRBE */
1132	trfcr = cpu_prohibit_trace();
1133	/*
1134	 * Ensure the trace is visible to the CPUs and
1135	 * any external aborts have been resolved.
1136	 */
1137	trbe_drain_and_disable_local(buf->cpudata);
1138	clr_trbe_irq();
1139	isb();
1140
1141	if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
1142		return IRQ_NONE;
1143
1144	if (!is_perf_trbe(handle))
1145		return IRQ_NONE;
1146
1147	act = trbe_get_fault_act(handle, status);
1148	switch (act) {
1149	case TRBE_FAULT_ACT_WRAP:
1150		truncated = !!trbe_handle_overflow(handle);
1151		break;
1152	case TRBE_FAULT_ACT_SPURIOUS:
1153		trbe_handle_spurious(handle);
1154		break;
1155	case TRBE_FAULT_ACT_FATAL:
1156		trbe_stop_and_truncate_event(handle);
1157		truncated = true;
1158		break;
1159	}
1160
1161	/*
1162	 * If the buffer was truncated, ensure perf callbacks
1163	 * have completed, which will disable the event.
1164	 *
1165	 * Otherwise, restore the trace filter controls to
1166	 * allow the tracing.
1167	 */
1168	if (truncated)
1169		irq_work_run();
1170	else
1171		write_trfcr(trfcr);
1172
1173	return IRQ_HANDLED;
1174}
1175
1176static const struct coresight_ops_sink arm_trbe_sink_ops = {
1177	.enable		= arm_trbe_enable,
1178	.disable	= arm_trbe_disable,
1179	.alloc_buffer	= arm_trbe_alloc_buffer,
1180	.free_buffer	= arm_trbe_free_buffer,
1181	.update_buffer	= arm_trbe_update_buffer,
1182};
1183
1184static const struct coresight_ops arm_trbe_cs_ops = {
1185	.sink_ops	= &arm_trbe_sink_ops,
1186};
1187
1188static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
1189{
1190	struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1191
1192	return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);
1193}
1194static DEVICE_ATTR_RO(align);
1195
1196static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
1197{
1198	struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1199
1200	return sprintf(buf, "%d\n", cpudata->trbe_flag);
1201}
1202static DEVICE_ATTR_RO(flag);
1203
1204static struct attribute *arm_trbe_attrs[] = {
1205	&dev_attr_align.attr,
1206	&dev_attr_flag.attr,
1207	NULL,
1208};
1209
1210static const struct attribute_group arm_trbe_group = {
1211	.attrs = arm_trbe_attrs,
1212};
1213
1214static const struct attribute_group *arm_trbe_groups[] = {
1215	&arm_trbe_group,
1216	NULL,
1217};
1218
1219static void arm_trbe_enable_cpu(void *info)
1220{
1221	struct trbe_drvdata *drvdata = info;
1222	struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1223
1224	trbe_reset_local(cpudata);
1225	enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);
1226}
1227
1228static void arm_trbe_disable_cpu(void *info)
1229{
1230	struct trbe_drvdata *drvdata = info;
1231	struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1232
1233	disable_percpu_irq(drvdata->irq);
1234	trbe_reset_local(cpudata);
1235}
1236
1237
1238static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1239{
1240	struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1241	struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1242	struct coresight_desc desc = { 0 };
1243	struct device *dev;
1244
1245	if (WARN_ON(trbe_csdev))
1246		return;
1247
1248	/* If the TRBE was not probed on the CPU, we shouldn't be here */
1249	if (WARN_ON(!cpudata->drvdata))
1250		return;
1251
1252	dev = &cpudata->drvdata->pdev->dev;
1253	desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
1254	if (!desc.name)
1255		goto cpu_clear;
1256	/*
1257	 * TRBE coresight devices do not need regular connections
1258	 * information, as the paths get built between all percpu
1259	 * source and their respective percpu sink devices. Though
1260	 * coresight_register() expect device connections via the
1261	 * platform_data, which TRBE devices do not have. As they
1262	 * are not real ACPI devices, coresight_get_platform_data()
1263	 * ends up failing. Instead let's allocate a dummy zeroed
1264	 * coresight_platform_data structure and assign that back
1265	 * into the device for that purpose.
1266	 */
1267	desc.pdata = devm_kzalloc(dev, sizeof(*desc.pdata), GFP_KERNEL);
1268	if (IS_ERR(desc.pdata))
1269		goto cpu_clear;
1270
1271	desc.type = CORESIGHT_DEV_TYPE_SINK;
1272	desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
1273	desc.ops = &arm_trbe_cs_ops;
 
1274	desc.groups = arm_trbe_groups;
1275	desc.dev = dev;
1276	trbe_csdev = coresight_register(&desc);
1277	if (IS_ERR(trbe_csdev))
1278		goto cpu_clear;
1279
1280	dev_set_drvdata(&trbe_csdev->dev, cpudata);
1281	coresight_set_percpu_sink(cpu, trbe_csdev);
1282	return;
1283cpu_clear:
1284	cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1285}
1286
1287/*
1288 * Must be called with preemption disabled, for trbe_check_errata().
1289 */
1290static void arm_trbe_probe_cpu(void *info)
1291{
1292	struct trbe_drvdata *drvdata = info;
1293	int cpu = smp_processor_id();
1294	struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1295	u64 trbidr;
1296
1297	if (WARN_ON(!cpudata))
1298		goto cpu_clear;
1299
1300	if (!is_trbe_available()) {
1301		pr_err("TRBE is not implemented on cpu %d\n", cpu);
1302		goto cpu_clear;
1303	}
1304
1305	trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
1306	if (!is_trbe_programmable(trbidr)) {
1307		pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
1308		goto cpu_clear;
1309	}
1310
1311	cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
1312	if (cpudata->trbe_hw_align > SZ_2K) {
1313		pr_err("Unsupported alignment on cpu %d\n", cpu);
1314		goto cpu_clear;
1315	}
1316
1317	/*
1318	 * Run the TRBE erratum checks, now that we know
1319	 * this instance is about to be registered.
1320	 */
1321	trbe_check_errata(cpudata);
1322
1323	if (trbe_is_broken(cpudata)) {
1324		pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);
1325		goto cpu_clear;
1326	}
1327
1328	/*
1329	 * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
1330	 * we must always program the TBRPTR_EL1, 256bytes from a page
1331	 * boundary, with TRBBASER_EL1 set to the page, to prevent
1332	 * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.
1333	 *
1334	 * Thus make sure we always align our write pointer to a PAGE_SIZE,
1335	 * which also guarantees that we have at least a PAGE_SIZE space in
1336	 * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip
1337	 * the required bytes at the base.
1338	 */
1339	if (trbe_may_overwrite_in_fill_mode(cpudata))
1340		cpudata->trbe_align = PAGE_SIZE;
1341	else
1342		cpudata->trbe_align = cpudata->trbe_hw_align;
1343
1344	cpudata->trbe_flag = get_trbe_flag_update(trbidr);
1345	cpudata->cpu = cpu;
1346	cpudata->drvdata = drvdata;
1347	return;
1348cpu_clear:
1349	cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1350}
1351
1352static void arm_trbe_remove_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1353{
 
 
 
1354	struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1355
 
 
1356	if (trbe_csdev) {
1357		coresight_unregister(trbe_csdev);
 
1358		coresight_set_percpu_sink(cpu, NULL);
1359	}
1360}
1361
1362static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
1363{
1364	int cpu;
1365
1366	drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
1367	if (!drvdata->cpudata)
1368		return -ENOMEM;
1369
1370	for_each_cpu(cpu, &drvdata->supported_cpus) {
1371		/* If we fail to probe the CPU, let us defer it to hotplug callbacks */
1372		if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
1373			continue;
1374		if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1375			arm_trbe_register_coresight_cpu(drvdata, cpu);
1376		if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1377			smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);
1378	}
1379	return 0;
1380}
1381
1382static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
1383{
1384	int cpu;
1385
1386	for_each_cpu(cpu, &drvdata->supported_cpus) {
1387		smp_call_function_single(cpu, arm_trbe_disable_cpu, drvdata, 1);
1388		arm_trbe_remove_coresight_cpu(drvdata, cpu);
1389	}
1390	free_percpu(drvdata->cpudata);
1391	return 0;
1392}
1393
1394static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
1395{
1396	preempt_disable();
1397	arm_trbe_probe_cpu(drvdata);
1398	preempt_enable();
1399}
1400
1401static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
1402{
1403	struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1404
1405	if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1406
1407		/*
1408		 * If this CPU was not probed for TRBE,
1409		 * initialize it now.
1410		 */
1411		if (!coresight_get_percpu_sink(cpu)) {
1412			arm_trbe_probe_hotplugged_cpu(drvdata);
1413			if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1414				arm_trbe_register_coresight_cpu(drvdata, cpu);
1415			if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1416				arm_trbe_enable_cpu(drvdata);
1417		} else {
1418			arm_trbe_enable_cpu(drvdata);
1419		}
1420	}
1421	return 0;
1422}
1423
1424static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1425{
1426	struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1427
1428	if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1429		arm_trbe_disable_cpu(drvdata);
 
 
 
 
1430	return 0;
1431}
1432
1433static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
1434{
1435	enum cpuhp_state trbe_online;
1436	int ret;
1437
1438	trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
1439					      arm_trbe_cpu_startup, arm_trbe_cpu_teardown);
1440	if (trbe_online < 0)
1441		return trbe_online;
1442
1443	ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);
1444	if (ret) {
1445		cpuhp_remove_multi_state(trbe_online);
1446		return ret;
1447	}
1448	drvdata->trbe_online = trbe_online;
1449	return 0;
1450}
1451
1452static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
1453{
1454	cpuhp_state_remove_instance(drvdata->trbe_online, &drvdata->hotplug_node);
1455	cpuhp_remove_multi_state(drvdata->trbe_online);
1456}
1457
1458static int arm_trbe_probe_irq(struct platform_device *pdev,
1459			      struct trbe_drvdata *drvdata)
1460{
1461	int ret;
1462
1463	drvdata->irq = platform_get_irq(pdev, 0);
1464	if (drvdata->irq < 0) {
1465		pr_err("IRQ not found for the platform device\n");
1466		return drvdata->irq;
1467	}
1468
1469	if (!irq_is_percpu(drvdata->irq)) {
1470		pr_err("IRQ is not a PPI\n");
1471		return -EINVAL;
1472	}
1473
1474	if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))
1475		return -EINVAL;
1476
1477	drvdata->handle = alloc_percpu(struct perf_output_handle *);
1478	if (!drvdata->handle)
1479		return -ENOMEM;
1480
1481	ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);
1482	if (ret) {
1483		free_percpu(drvdata->handle);
1484		return ret;
1485	}
1486	return 0;
1487}
1488
1489static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
1490{
1491	free_percpu_irq(drvdata->irq, drvdata->handle);
1492	free_percpu(drvdata->handle);
1493}
1494
1495static int arm_trbe_device_probe(struct platform_device *pdev)
1496{
 
1497	struct trbe_drvdata *drvdata;
1498	struct device *dev = &pdev->dev;
1499	int ret;
1500
1501	/* Trace capture is not possible with kernel page table isolation */
1502	if (arm64_kernel_unmapped_at_el0()) {
1503		pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
1504		return -EOPNOTSUPP;
1505	}
1506
1507	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1508	if (!drvdata)
1509		return -ENOMEM;
1510
 
 
 
 
1511	dev_set_drvdata(dev, drvdata);
 
1512	drvdata->pdev = pdev;
1513	ret = arm_trbe_probe_irq(pdev, drvdata);
1514	if (ret)
1515		return ret;
1516
1517	ret = arm_trbe_probe_coresight(drvdata);
1518	if (ret)
1519		goto probe_failed;
1520
1521	ret = arm_trbe_probe_cpuhp(drvdata);
1522	if (ret)
1523		goto cpuhp_failed;
1524
1525	return 0;
1526cpuhp_failed:
1527	arm_trbe_remove_coresight(drvdata);
1528probe_failed:
1529	arm_trbe_remove_irq(drvdata);
1530	return ret;
1531}
1532
1533static void arm_trbe_device_remove(struct platform_device *pdev)
1534{
1535	struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
1536
1537	arm_trbe_remove_cpuhp(drvdata);
1538	arm_trbe_remove_coresight(drvdata);
1539	arm_trbe_remove_irq(drvdata);
 
1540}
1541
1542static const struct of_device_id arm_trbe_of_match[] = {
1543	{ .compatible = "arm,trace-buffer-extension"},
1544	{},
1545};
1546MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
1547
1548#ifdef CONFIG_ACPI
1549static const struct platform_device_id arm_trbe_acpi_match[] = {
1550	{ ARMV8_TRBE_PDEV_NAME, 0 },
1551	{ }
1552};
1553MODULE_DEVICE_TABLE(platform, arm_trbe_acpi_match);
1554#endif
1555
1556static struct platform_driver arm_trbe_driver = {
1557	.id_table = ACPI_PTR(arm_trbe_acpi_match),
1558	.driver	= {
1559		.name = DRVNAME,
1560		.of_match_table = of_match_ptr(arm_trbe_of_match),
1561		.suppress_bind_attrs = true,
1562	},
1563	.probe	= arm_trbe_device_probe,
1564	.remove_new = arm_trbe_device_remove,
1565};
1566
1567static int __init arm_trbe_init(void)
1568{
1569	int ret;
1570
1571	ret = platform_driver_register(&arm_trbe_driver);
1572	if (!ret)
1573		return 0;
1574
1575	pr_err("Error registering %s platform driver\n", DRVNAME);
1576	return ret;
1577}
1578
1579static void __exit arm_trbe_exit(void)
1580{
1581	platform_driver_unregister(&arm_trbe_driver);
1582}
1583module_init(arm_trbe_init);
1584module_exit(arm_trbe_exit);
1585
1586MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
1587MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
1588MODULE_LICENSE("GPL v2");