Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5 */
6
7#include <linux/atomic.h>
8#include <linux/coresight.h>
9#include <linux/dma-mapping.h>
10#include <linux/iommu.h>
11#include <linux/idr.h>
12#include <linux/mutex.h>
13#include <linux/refcount.h>
14#include <linux/slab.h>
15#include <linux/types.h>
16#include <linux/vmalloc.h>
17#include "coresight-catu.h"
18#include "coresight-etm-perf.h"
19#include "coresight-priv.h"
20#include "coresight-tmc.h"
21
22struct etr_flat_buf {
23 struct device *dev;
24 dma_addr_t daddr;
25 void *vaddr;
26 size_t size;
27};
28
29struct etr_buf_hw {
30 bool has_iommu;
31 bool has_etr_sg;
32 bool has_catu;
33};
34
35/*
36 * etr_perf_buffer - Perf buffer used for ETR
37 * @drvdata - The ETR drvdaga this buffer has been allocated for.
38 * @etr_buf - Actual buffer used by the ETR
39 * @pid - The PID of the session owner that etr_perf_buffer
40 * belongs to.
41 * @snaphost - Perf session mode
42 * @nr_pages - Number of pages in the ring buffer.
43 * @pages - Array of Pages in the ring buffer.
44 */
45struct etr_perf_buffer {
46 struct tmc_drvdata *drvdata;
47 struct etr_buf *etr_buf;
48 pid_t pid;
49 bool snapshot;
50 int nr_pages;
51 void **pages;
52};
53
54/* Convert the perf index to an offset within the ETR buffer */
55#define PERF_IDX2OFF(idx, buf) \
56 ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
57
58/* Lower limit for ETR hardware buffer */
59#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
60
61/*
62 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
63 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
64 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
65 * contain more than one SG buffer and tables.
66 *
67 * A table entry has the following format:
68 *
69 * ---Bit31------------Bit4-------Bit1-----Bit0--
70 * | Address[39:12] | SBZ | Entry Type |
71 * ----------------------------------------------
72 *
73 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
74 * always zero.
75 *
76 * Entry type:
77 * b00 - Reserved.
78 * b01 - Last entry in the tables, points to 4K page buffer.
79 * b10 - Normal entry, points to 4K page buffer.
80 * b11 - Link. The address points to the base of next table.
81 */
82
83typedef u32 sgte_t;
84
85#define ETR_SG_PAGE_SHIFT 12
86#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
87#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
88#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
89#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
90
91#define ETR_SG_ET_MASK 0x3
92#define ETR_SG_ET_LAST 0x1
93#define ETR_SG_ET_NORMAL 0x2
94#define ETR_SG_ET_LINK 0x3
95
96#define ETR_SG_ADDR_SHIFT 4
97
98#define ETR_SG_ENTRY(addr, type) \
99 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
100 (type & ETR_SG_ET_MASK))
101
102#define ETR_SG_ADDR(entry) \
103 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
104#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
105
106/*
107 * struct etr_sg_table : ETR SG Table
108 * @sg_table: Generic SG Table holding the data/table pages.
109 * @hwaddr: hwaddress used by the TMC, which is the base
110 * address of the table.
111 */
112struct etr_sg_table {
113 struct tmc_sg_table *sg_table;
114 dma_addr_t hwaddr;
115};
116
117/*
118 * tmc_etr_sg_table_entries: Total number of table entries required to map
119 * @nr_pages system pages.
120 *
121 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
122 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
123 * with the last entry pointing to another page of table entries.
124 * If we spill over to a new page for mapping 1 entry, we could as
125 * well replace the link entry of the previous page with the last entry.
126 */
127static inline unsigned long __attribute_const__
128tmc_etr_sg_table_entries(int nr_pages)
129{
130 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
131 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
132 /*
133 * If we spill over to a new page for 1 entry, we could as well
134 * make it the LAST entry in the previous page, skipping the Link
135 * address.
136 */
137 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
138 nr_sglinks--;
139 return nr_sgpages + nr_sglinks;
140}
141
142/*
143 * tmc_pages_get_offset: Go through all the pages in the tmc_pages
144 * and map the device address @addr to an offset within the virtual
145 * contiguous buffer.
146 */
147static long
148tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
149{
150 int i;
151 dma_addr_t page_start;
152
153 for (i = 0; i < tmc_pages->nr_pages; i++) {
154 page_start = tmc_pages->daddrs[i];
155 if (addr >= page_start && addr < (page_start + PAGE_SIZE))
156 return i * PAGE_SIZE + (addr - page_start);
157 }
158
159 return -EINVAL;
160}
161
162/*
163 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
164 * If the pages were not allocated in tmc_pages_alloc(), we would
165 * simply drop the refcount.
166 */
167static void tmc_pages_free(struct tmc_pages *tmc_pages,
168 struct device *dev, enum dma_data_direction dir)
169{
170 int i;
171 struct device *real_dev = dev->parent;
172
173 for (i = 0; i < tmc_pages->nr_pages; i++) {
174 if (tmc_pages->daddrs && tmc_pages->daddrs[i])
175 dma_unmap_page(real_dev, tmc_pages->daddrs[i],
176 PAGE_SIZE, dir);
177 if (tmc_pages->pages && tmc_pages->pages[i])
178 __free_page(tmc_pages->pages[i]);
179 }
180
181 kfree(tmc_pages->pages);
182 kfree(tmc_pages->daddrs);
183 tmc_pages->pages = NULL;
184 tmc_pages->daddrs = NULL;
185 tmc_pages->nr_pages = 0;
186}
187
188/*
189 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
190 * If @pages is not NULL, the list of page virtual addresses are
191 * used as the data pages. The pages are then dma_map'ed for @dev
192 * with dma_direction @dir.
193 *
194 * Returns 0 upon success, else the error number.
195 */
196static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
197 struct device *dev, int node,
198 enum dma_data_direction dir, void **pages)
199{
200 int i, nr_pages;
201 dma_addr_t paddr;
202 struct page *page;
203 struct device *real_dev = dev->parent;
204
205 nr_pages = tmc_pages->nr_pages;
206 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
207 GFP_KERNEL);
208 if (!tmc_pages->daddrs)
209 return -ENOMEM;
210 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
211 GFP_KERNEL);
212 if (!tmc_pages->pages) {
213 kfree(tmc_pages->daddrs);
214 tmc_pages->daddrs = NULL;
215 return -ENOMEM;
216 }
217
218 for (i = 0; i < nr_pages; i++) {
219 if (pages && pages[i]) {
220 page = virt_to_page(pages[i]);
221 /* Hold a refcount on the page */
222 get_page(page);
223 } else {
224 page = alloc_pages_node(node,
225 GFP_KERNEL | __GFP_ZERO, 0);
226 if (!page)
227 goto err;
228 }
229 paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
230 if (dma_mapping_error(real_dev, paddr))
231 goto err;
232 tmc_pages->daddrs[i] = paddr;
233 tmc_pages->pages[i] = page;
234 }
235 return 0;
236err:
237 tmc_pages_free(tmc_pages, dev, dir);
238 return -ENOMEM;
239}
240
241static inline long
242tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
243{
244 return tmc_pages_get_offset(&sg_table->data_pages, addr);
245}
246
247static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
248{
249 if (sg_table->table_vaddr)
250 vunmap(sg_table->table_vaddr);
251 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
252}
253
254static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
255{
256 if (sg_table->data_vaddr)
257 vunmap(sg_table->data_vaddr);
258 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
259}
260
261void tmc_free_sg_table(struct tmc_sg_table *sg_table)
262{
263 tmc_free_table_pages(sg_table);
264 tmc_free_data_pages(sg_table);
265 kfree(sg_table);
266}
267EXPORT_SYMBOL_GPL(tmc_free_sg_table);
268
269/*
270 * Alloc pages for the table. Since this will be used by the device,
271 * allocate the pages closer to the device (i.e, dev_to_node(dev)
272 * rather than the CPU node).
273 */
274static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
275{
276 int rc;
277 struct tmc_pages *table_pages = &sg_table->table_pages;
278
279 rc = tmc_pages_alloc(table_pages, sg_table->dev,
280 dev_to_node(sg_table->dev),
281 DMA_TO_DEVICE, NULL);
282 if (rc)
283 return rc;
284 sg_table->table_vaddr = vmap(table_pages->pages,
285 table_pages->nr_pages,
286 VM_MAP,
287 PAGE_KERNEL);
288 if (!sg_table->table_vaddr)
289 rc = -ENOMEM;
290 else
291 sg_table->table_daddr = table_pages->daddrs[0];
292 return rc;
293}
294
295static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
296{
297 int rc;
298
299 /* Allocate data pages on the node requested by the caller */
300 rc = tmc_pages_alloc(&sg_table->data_pages,
301 sg_table->dev, sg_table->node,
302 DMA_FROM_DEVICE, pages);
303 if (!rc) {
304 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
305 sg_table->data_pages.nr_pages,
306 VM_MAP,
307 PAGE_KERNEL);
308 if (!sg_table->data_vaddr)
309 rc = -ENOMEM;
310 }
311 return rc;
312}
313
314/*
315 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
316 * and data buffers. TMC writes to the data buffers and reads from the SG
317 * Table pages.
318 *
319 * @dev - Coresight device to which page should be DMA mapped.
320 * @node - Numa node for mem allocations
321 * @nr_tpages - Number of pages for the table entries.
322 * @nr_dpages - Number of pages for Data buffer.
323 * @pages - Optional list of virtual address of pages.
324 */
325struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
326 int node,
327 int nr_tpages,
328 int nr_dpages,
329 void **pages)
330{
331 long rc;
332 struct tmc_sg_table *sg_table;
333
334 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
335 if (!sg_table)
336 return ERR_PTR(-ENOMEM);
337 sg_table->data_pages.nr_pages = nr_dpages;
338 sg_table->table_pages.nr_pages = nr_tpages;
339 sg_table->node = node;
340 sg_table->dev = dev;
341
342 rc = tmc_alloc_data_pages(sg_table, pages);
343 if (!rc)
344 rc = tmc_alloc_table_pages(sg_table);
345 if (rc) {
346 tmc_free_sg_table(sg_table);
347 return ERR_PTR(rc);
348 }
349
350 return sg_table;
351}
352EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
353
354/*
355 * tmc_sg_table_sync_data_range: Sync the data buffer written
356 * by the device from @offset upto a @size bytes.
357 */
358void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
359 u64 offset, u64 size)
360{
361 int i, index, start;
362 int npages = DIV_ROUND_UP(size, PAGE_SIZE);
363 struct device *real_dev = table->dev->parent;
364 struct tmc_pages *data = &table->data_pages;
365
366 start = offset >> PAGE_SHIFT;
367 for (i = start; i < (start + npages); i++) {
368 index = i % data->nr_pages;
369 dma_sync_single_for_cpu(real_dev, data->daddrs[index],
370 PAGE_SIZE, DMA_FROM_DEVICE);
371 }
372}
373EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
374
375/* tmc_sg_sync_table: Sync the page table */
376void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
377{
378 int i;
379 struct device *real_dev = sg_table->dev->parent;
380 struct tmc_pages *table_pages = &sg_table->table_pages;
381
382 for (i = 0; i < table_pages->nr_pages; i++)
383 dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
384 PAGE_SIZE, DMA_TO_DEVICE);
385}
386EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
387
388/*
389 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
390 * in the SG buffer. The @bufpp is updated to point to the buffer.
391 * Returns :
392 * the length of linear data available at @offset.
393 * or
394 * <= 0 if no data is available.
395 */
396ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
397 u64 offset, size_t len, char **bufpp)
398{
399 size_t size;
400 int pg_idx = offset >> PAGE_SHIFT;
401 int pg_offset = offset & (PAGE_SIZE - 1);
402 struct tmc_pages *data_pages = &sg_table->data_pages;
403
404 size = tmc_sg_table_buf_size(sg_table);
405 if (offset >= size)
406 return -EINVAL;
407
408 /* Make sure we don't go beyond the end */
409 len = (len < (size - offset)) ? len : size - offset;
410 /* Respect the page boundaries */
411 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
412 if (len > 0)
413 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
414 return len;
415}
416EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
417
418#ifdef ETR_SG_DEBUG
419/* Map a dma address to virtual address */
420static unsigned long
421tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
422 dma_addr_t addr, bool table)
423{
424 long offset;
425 unsigned long base;
426 struct tmc_pages *tmc_pages;
427
428 if (table) {
429 tmc_pages = &sg_table->table_pages;
430 base = (unsigned long)sg_table->table_vaddr;
431 } else {
432 tmc_pages = &sg_table->data_pages;
433 base = (unsigned long)sg_table->data_vaddr;
434 }
435
436 offset = tmc_pages_get_offset(tmc_pages, addr);
437 if (offset < 0)
438 return 0;
439 return base + offset;
440}
441
442/* Dump the given sg_table */
443static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
444{
445 sgte_t *ptr;
446 int i = 0;
447 dma_addr_t addr;
448 struct tmc_sg_table *sg_table = etr_table->sg_table;
449
450 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
451 etr_table->hwaddr, true);
452 while (ptr) {
453 addr = ETR_SG_ADDR(*ptr);
454 switch (ETR_SG_ET(*ptr)) {
455 case ETR_SG_ET_NORMAL:
456 dev_dbg(sg_table->dev,
457 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
458 ptr++;
459 break;
460 case ETR_SG_ET_LINK:
461 dev_dbg(sg_table->dev,
462 "%05d: *** %p\t:{L} 0x%llx ***\n",
463 i, ptr, addr);
464 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
465 addr, true);
466 break;
467 case ETR_SG_ET_LAST:
468 dev_dbg(sg_table->dev,
469 "%05d: ### %p\t:[L] 0x%llx ###\n",
470 i, ptr, addr);
471 return;
472 default:
473 dev_dbg(sg_table->dev,
474 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
475 i, ptr, addr);
476 return;
477 }
478 i++;
479 }
480 dev_dbg(sg_table->dev, "******* End of Table *****\n");
481}
482#else
483static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
484#endif
485
486/*
487 * Populate the SG Table page table entries from table/data
488 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
489 * So does a Table page. So we keep track of indices of the tables
490 * in each system page and move the pointers accordingly.
491 */
492#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
493static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
494{
495 dma_addr_t paddr;
496 int i, type, nr_entries;
497 int tpidx = 0; /* index to the current system table_page */
498 int sgtidx = 0; /* index to the sg_table within the current syspage */
499 int sgtentry = 0; /* the entry within the sg_table */
500 int dpidx = 0; /* index to the current system data_page */
501 int spidx = 0; /* index to the SG page within the current data page */
502 sgte_t *ptr; /* pointer to the table entry to fill */
503 struct tmc_sg_table *sg_table = etr_table->sg_table;
504 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
505 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
506
507 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
508 /*
509 * Use the contiguous virtual address of the table to update entries.
510 */
511 ptr = sg_table->table_vaddr;
512 /*
513 * Fill all the entries, except the last entry to avoid special
514 * checks within the loop.
515 */
516 for (i = 0; i < nr_entries - 1; i++) {
517 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
518 /*
519 * Last entry in a sg_table page is a link address to
520 * the next table page. If this sg_table is the last
521 * one in the system page, it links to the first
522 * sg_table in the next system page. Otherwise, it
523 * links to the next sg_table page within the system
524 * page.
525 */
526 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
527 paddr = table_daddrs[tpidx + 1];
528 } else {
529 paddr = table_daddrs[tpidx] +
530 (ETR_SG_PAGE_SIZE * (sgtidx + 1));
531 }
532 type = ETR_SG_ET_LINK;
533 } else {
534 /*
535 * Update the indices to the data_pages to point to the
536 * next sg_page in the data buffer.
537 */
538 type = ETR_SG_ET_NORMAL;
539 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
540 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
541 dpidx++;
542 }
543 *ptr++ = ETR_SG_ENTRY(paddr, type);
544 /*
545 * Move to the next table pointer, moving the table page index
546 * if necessary
547 */
548 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
549 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
550 tpidx++;
551 }
552 }
553
554 /* Set up the last entry, which is always a data pointer */
555 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
556 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
557}
558
559/*
560 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
561 * populate the table.
562 *
563 * @dev - Device pointer for the TMC
564 * @node - NUMA node where the memory should be allocated
565 * @size - Total size of the data buffer
566 * @pages - Optional list of page virtual address
567 */
568static struct etr_sg_table *
569tmc_init_etr_sg_table(struct device *dev, int node,
570 unsigned long size, void **pages)
571{
572 int nr_entries, nr_tpages;
573 int nr_dpages = size >> PAGE_SHIFT;
574 struct tmc_sg_table *sg_table;
575 struct etr_sg_table *etr_table;
576
577 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
578 if (!etr_table)
579 return ERR_PTR(-ENOMEM);
580 nr_entries = tmc_etr_sg_table_entries(nr_dpages);
581 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
582
583 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
584 if (IS_ERR(sg_table)) {
585 kfree(etr_table);
586 return ERR_CAST(sg_table);
587 }
588
589 etr_table->sg_table = sg_table;
590 /* TMC should use table base address for DBA */
591 etr_table->hwaddr = sg_table->table_daddr;
592 tmc_etr_sg_table_populate(etr_table);
593 /* Sync the table pages for the HW */
594 tmc_sg_table_sync_table(sg_table);
595 tmc_etr_sg_table_dump(etr_table);
596
597 return etr_table;
598}
599
600/*
601 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
602 */
603static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
604 struct etr_buf *etr_buf, int node,
605 void **pages)
606{
607 struct etr_flat_buf *flat_buf;
608 struct device *real_dev = drvdata->csdev->dev.parent;
609
610 /* We cannot reuse existing pages for flat buf */
611 if (pages)
612 return -EINVAL;
613
614 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
615 if (!flat_buf)
616 return -ENOMEM;
617
618 flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,
619 &flat_buf->daddr,
620 DMA_FROM_DEVICE,
621 GFP_KERNEL | __GFP_NOWARN);
622 if (!flat_buf->vaddr) {
623 kfree(flat_buf);
624 return -ENOMEM;
625 }
626
627 flat_buf->size = etr_buf->size;
628 flat_buf->dev = &drvdata->csdev->dev;
629 etr_buf->hwaddr = flat_buf->daddr;
630 etr_buf->mode = ETR_MODE_FLAT;
631 etr_buf->private = flat_buf;
632 return 0;
633}
634
635static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
636{
637 struct etr_flat_buf *flat_buf = etr_buf->private;
638
639 if (flat_buf && flat_buf->daddr) {
640 struct device *real_dev = flat_buf->dev->parent;
641
642 dma_free_noncoherent(real_dev, etr_buf->size,
643 flat_buf->vaddr, flat_buf->daddr,
644 DMA_FROM_DEVICE);
645 }
646 kfree(flat_buf);
647}
648
649static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
650{
651 struct etr_flat_buf *flat_buf = etr_buf->private;
652 struct device *real_dev = flat_buf->dev->parent;
653
654 /*
655 * Adjust the buffer to point to the beginning of the trace data
656 * and update the available trace data.
657 */
658 etr_buf->offset = rrp - etr_buf->hwaddr;
659 if (etr_buf->full)
660 etr_buf->len = etr_buf->size;
661 else
662 etr_buf->len = rwp - rrp;
663
664 /*
665 * The driver always starts tracing at the beginning of the buffer,
666 * the only reason why we would get a wrap around is when the buffer
667 * is full. Sync the entire buffer in one go for this case.
668 */
669 if (etr_buf->offset + etr_buf->len > etr_buf->size)
670 dma_sync_single_for_cpu(real_dev, flat_buf->daddr,
671 etr_buf->size, DMA_FROM_DEVICE);
672 else
673 dma_sync_single_for_cpu(real_dev,
674 flat_buf->daddr + etr_buf->offset,
675 etr_buf->len, DMA_FROM_DEVICE);
676}
677
678static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
679 u64 offset, size_t len, char **bufpp)
680{
681 struct etr_flat_buf *flat_buf = etr_buf->private;
682
683 *bufpp = (char *)flat_buf->vaddr + offset;
684 /*
685 * tmc_etr_buf_get_data already adjusts the length to handle
686 * buffer wrapping around.
687 */
688 return len;
689}
690
691static const struct etr_buf_operations etr_flat_buf_ops = {
692 .alloc = tmc_etr_alloc_flat_buf,
693 .free = tmc_etr_free_flat_buf,
694 .sync = tmc_etr_sync_flat_buf,
695 .get_data = tmc_etr_get_data_flat_buf,
696};
697
698/*
699 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
700 * appropriately.
701 */
702static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
703 struct etr_buf *etr_buf, int node,
704 void **pages)
705{
706 struct etr_sg_table *etr_table;
707 struct device *dev = &drvdata->csdev->dev;
708
709 etr_table = tmc_init_etr_sg_table(dev, node,
710 etr_buf->size, pages);
711 if (IS_ERR(etr_table))
712 return -ENOMEM;
713 etr_buf->hwaddr = etr_table->hwaddr;
714 etr_buf->mode = ETR_MODE_ETR_SG;
715 etr_buf->private = etr_table;
716 return 0;
717}
718
719static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
720{
721 struct etr_sg_table *etr_table = etr_buf->private;
722
723 if (etr_table) {
724 tmc_free_sg_table(etr_table->sg_table);
725 kfree(etr_table);
726 }
727}
728
729static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
730 size_t len, char **bufpp)
731{
732 struct etr_sg_table *etr_table = etr_buf->private;
733
734 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
735}
736
737static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
738{
739 long r_offset, w_offset;
740 struct etr_sg_table *etr_table = etr_buf->private;
741 struct tmc_sg_table *table = etr_table->sg_table;
742
743 /* Convert hw address to offset in the buffer */
744 r_offset = tmc_sg_get_data_page_offset(table, rrp);
745 if (r_offset < 0) {
746 dev_warn(table->dev,
747 "Unable to map RRP %llx to offset\n", rrp);
748 etr_buf->len = 0;
749 return;
750 }
751
752 w_offset = tmc_sg_get_data_page_offset(table, rwp);
753 if (w_offset < 0) {
754 dev_warn(table->dev,
755 "Unable to map RWP %llx to offset\n", rwp);
756 etr_buf->len = 0;
757 return;
758 }
759
760 etr_buf->offset = r_offset;
761 if (etr_buf->full)
762 etr_buf->len = etr_buf->size;
763 else
764 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
765 w_offset - r_offset;
766 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
767}
768
769static const struct etr_buf_operations etr_sg_buf_ops = {
770 .alloc = tmc_etr_alloc_sg_buf,
771 .free = tmc_etr_free_sg_buf,
772 .sync = tmc_etr_sync_sg_buf,
773 .get_data = tmc_etr_get_data_sg_buf,
774};
775
776/*
777 * TMC ETR could be connected to a CATU device, which can provide address
778 * translation service. This is represented by the Output port of the TMC
779 * (ETR) connected to the input port of the CATU.
780 *
781 * Returns : coresight_device ptr for the CATU device if a CATU is found.
782 * : NULL otherwise.
783 */
784struct coresight_device *
785tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
786{
787 struct coresight_device *etr = drvdata->csdev;
788 union coresight_dev_subtype catu_subtype = {
789 .helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU
790 };
791
792 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
793 return NULL;
794
795 return coresight_find_output_type(etr->pdata, CORESIGHT_DEV_TYPE_HELPER,
796 catu_subtype);
797}
798EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
799
800static const struct etr_buf_operations *etr_buf_ops[] = {
801 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
802 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
803 [ETR_MODE_CATU] = NULL,
804};
805
806void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
807{
808 etr_buf_ops[ETR_MODE_CATU] = catu;
809}
810EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
811
812void tmc_etr_remove_catu_ops(void)
813{
814 etr_buf_ops[ETR_MODE_CATU] = NULL;
815}
816EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
817
818static inline int tmc_etr_mode_alloc_buf(int mode,
819 struct tmc_drvdata *drvdata,
820 struct etr_buf *etr_buf, int node,
821 void **pages)
822{
823 int rc = -EINVAL;
824
825 switch (mode) {
826 case ETR_MODE_FLAT:
827 case ETR_MODE_ETR_SG:
828 case ETR_MODE_CATU:
829 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
830 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
831 node, pages);
832 if (!rc)
833 etr_buf->ops = etr_buf_ops[mode];
834 return rc;
835 default:
836 return -EINVAL;
837 }
838}
839
840static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)
841{
842 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
843
844 buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent);
845 buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
846 buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);
847}
848
849static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)
850{
851 bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;
852
853 return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;
854}
855
856/*
857 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
858 * @drvdata : ETR device details.
859 * @size : size of the requested buffer.
860 * @flags : Required properties for the buffer.
861 * @node : Node for memory allocations.
862 * @pages : An optional list of pages.
863 */
864static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
865 ssize_t size, int flags,
866 int node, void **pages)
867{
868 int rc = -ENOMEM;
869 struct etr_buf *etr_buf;
870 struct etr_buf_hw buf_hw;
871 struct device *dev = &drvdata->csdev->dev;
872
873 get_etr_buf_hw(dev, &buf_hw);
874 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
875 if (!etr_buf)
876 return ERR_PTR(-ENOMEM);
877
878 etr_buf->size = size;
879
880 /* If there is user directive for buffer mode, try that first */
881 if (drvdata->etr_mode != ETR_MODE_AUTO)
882 rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata,
883 etr_buf, node, pages);
884
885 /*
886 * If we have to use an existing list of pages, we cannot reliably
887 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
888 * we use the contiguous DMA memory if at least one of the following
889 * conditions is true:
890 * a) The ETR cannot use Scatter-Gather.
891 * b) we have a backing IOMMU
892 * c) The requested memory size is smaller (< 1M).
893 *
894 * Fallback to available mechanisms.
895 *
896 */
897 if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size))
898 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
899 etr_buf, node, pages);
900 if (rc && buf_hw.has_etr_sg)
901 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
902 etr_buf, node, pages);
903 if (rc && buf_hw.has_catu)
904 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
905 etr_buf, node, pages);
906 if (rc) {
907 kfree(etr_buf);
908 return ERR_PTR(rc);
909 }
910
911 refcount_set(&etr_buf->refcount, 1);
912 dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
913 (unsigned long)size >> 10, etr_buf->mode);
914 return etr_buf;
915}
916
917static void tmc_free_etr_buf(struct etr_buf *etr_buf)
918{
919 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
920 etr_buf->ops->free(etr_buf);
921 kfree(etr_buf);
922}
923
924/*
925 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
926 * with a maximum of @len bytes.
927 * Returns: The size of the linear data available @pos, with *bufpp
928 * updated to point to the buffer.
929 */
930static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
931 u64 offset, size_t len, char **bufpp)
932{
933 /* Adjust the length to limit this transaction to end of buffer */
934 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
935
936 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
937}
938
939static inline s64
940tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
941{
942 ssize_t len;
943 char *bufp;
944
945 len = tmc_etr_buf_get_data(etr_buf, offset,
946 CORESIGHT_BARRIER_PKT_SIZE, &bufp);
947 if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))
948 return -EINVAL;
949 coresight_insert_barrier_packet(bufp);
950 return offset + CORESIGHT_BARRIER_PKT_SIZE;
951}
952
953/*
954 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
955 * Makes sure the trace data is synced to the memory for consumption.
956 * @etr_buf->offset will hold the offset to the beginning of the trace data
957 * within the buffer, with @etr_buf->len bytes to consume.
958 */
959static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
960{
961 struct etr_buf *etr_buf = drvdata->etr_buf;
962 u64 rrp, rwp;
963 u32 status;
964
965 rrp = tmc_read_rrp(drvdata);
966 rwp = tmc_read_rwp(drvdata);
967 status = readl_relaxed(drvdata->base + TMC_STS);
968
969 /*
970 * If there were memory errors in the session, truncate the
971 * buffer.
972 */
973 if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
974 dev_dbg(&drvdata->csdev->dev,
975 "tmc memory error detected, truncating buffer\n");
976 etr_buf->len = 0;
977 etr_buf->full = false;
978 return;
979 }
980
981 etr_buf->full = !!(status & TMC_STS_FULL);
982
983 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
984
985 etr_buf->ops->sync(etr_buf, rrp, rwp);
986}
987
988static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
989{
990 u32 axictl, sts;
991 struct etr_buf *etr_buf = drvdata->etr_buf;
992 int rc = 0;
993
994 CS_UNLOCK(drvdata->base);
995
996 /* Wait for TMCSReady bit to be set */
997 rc = tmc_wait_for_tmcready(drvdata);
998 if (rc) {
999 dev_err(&drvdata->csdev->dev,
1000 "Failed to enable : TMC not ready\n");
1001 CS_LOCK(drvdata->base);
1002 return rc;
1003 }
1004
1005 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
1006 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
1007
1008 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
1009 axictl &= ~TMC_AXICTL_CLEAR_MASK;
1010 axictl |= TMC_AXICTL_PROT_CTL_B1;
1011 axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
1012 axictl |= TMC_AXICTL_AXCACHE_OS;
1013
1014 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
1015 axictl &= ~TMC_AXICTL_ARCACHE_MASK;
1016 axictl |= TMC_AXICTL_ARCACHE_OS;
1017 }
1018
1019 if (etr_buf->mode == ETR_MODE_ETR_SG)
1020 axictl |= TMC_AXICTL_SCT_GAT_MODE;
1021
1022 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1023 tmc_write_dba(drvdata, etr_buf->hwaddr);
1024 /*
1025 * If the TMC pointers must be programmed before the session,
1026 * we have to set it properly (i.e, RRP/RWP to base address and
1027 * STS to "not full").
1028 */
1029 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1030 tmc_write_rrp(drvdata, etr_buf->hwaddr);
1031 tmc_write_rwp(drvdata, etr_buf->hwaddr);
1032 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1033 writel_relaxed(sts, drvdata->base + TMC_STS);
1034 }
1035
1036 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1037 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1038 TMC_FFCR_TRIGON_TRIGIN,
1039 drvdata->base + TMC_FFCR);
1040 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1041 tmc_enable_hw(drvdata);
1042
1043 CS_LOCK(drvdata->base);
1044 return rc;
1045}
1046
1047static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1048 struct etr_buf *etr_buf)
1049{
1050 int rc;
1051
1052 /* Callers should provide an appropriate buffer for use */
1053 if (WARN_ON(!etr_buf))
1054 return -EINVAL;
1055
1056 if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1057 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1058 return -EINVAL;
1059
1060 if (WARN_ON(drvdata->etr_buf))
1061 return -EBUSY;
1062
1063 rc = coresight_claim_device(drvdata->csdev);
1064 if (!rc) {
1065 drvdata->etr_buf = etr_buf;
1066 rc = __tmc_etr_enable_hw(drvdata);
1067 if (rc) {
1068 drvdata->etr_buf = NULL;
1069 coresight_disclaim_device(drvdata->csdev);
1070 }
1071 }
1072
1073 return rc;
1074}
1075
1076/*
1077 * Return the available trace data in the buffer (starts at etr_buf->offset,
1078 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1079 * also updating the @bufpp on where to find it. Since the trace data
1080 * starts at anywhere in the buffer, depending on the RRP, we adjust the
1081 * @len returned to handle buffer wrapping around.
1082 *
1083 * We are protected here by drvdata->reading != 0, which ensures the
1084 * sysfs_buf stays alive.
1085 */
1086ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1087 loff_t pos, size_t len, char **bufpp)
1088{
1089 s64 offset;
1090 ssize_t actual = len;
1091 struct etr_buf *etr_buf = drvdata->sysfs_buf;
1092
1093 if (pos + actual > etr_buf->len)
1094 actual = etr_buf->len - pos;
1095 if (actual <= 0)
1096 return actual;
1097
1098 /* Compute the offset from which we read the data */
1099 offset = etr_buf->offset + pos;
1100 if (offset >= etr_buf->size)
1101 offset -= etr_buf->size;
1102 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1103}
1104
1105static struct etr_buf *
1106tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1107{
1108 return tmc_alloc_etr_buf(drvdata, drvdata->size,
1109 0, cpu_to_node(0), NULL);
1110}
1111
1112static void
1113tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1114{
1115 if (buf)
1116 tmc_free_etr_buf(buf);
1117}
1118
1119static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1120{
1121 struct etr_buf *etr_buf = drvdata->etr_buf;
1122
1123 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1124 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1125 drvdata->sysfs_buf = NULL;
1126 } else {
1127 tmc_sync_etr_buf(drvdata);
1128 /*
1129 * Insert barrier packets at the beginning, if there was
1130 * an overflow.
1131 */
1132 if (etr_buf->full)
1133 tmc_etr_buf_insert_barrier_packet(etr_buf,
1134 etr_buf->offset);
1135 }
1136}
1137
1138static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1139{
1140 CS_UNLOCK(drvdata->base);
1141
1142 tmc_flush_and_stop(drvdata);
1143 /*
1144 * When operating in sysFS mode the content of the buffer needs to be
1145 * read before the TMC is disabled.
1146 */
1147 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
1148 tmc_etr_sync_sysfs_buf(drvdata);
1149
1150 tmc_disable_hw(drvdata);
1151
1152 CS_LOCK(drvdata->base);
1153
1154}
1155
1156void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1157{
1158 __tmc_etr_disable_hw(drvdata);
1159 coresight_disclaim_device(drvdata->csdev);
1160 /* Reset the ETR buf used by hardware */
1161 drvdata->etr_buf = NULL;
1162}
1163
1164static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1165{
1166 int ret = 0;
1167 unsigned long flags;
1168 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1169 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1170
1171 /*
1172 * If we are enabling the ETR from disabled state, we need to make
1173 * sure we have a buffer with the right size. The etr_buf is not reset
1174 * immediately after we stop the tracing in SYSFS mode as we wait for
1175 * the user to collect the data. We may be able to reuse the existing
1176 * buffer, provided the size matches. Any allocation has to be done
1177 * with the lock released.
1178 */
1179 spin_lock_irqsave(&drvdata->spinlock, flags);
1180 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1181 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1182 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1183
1184 /* Allocate memory with the locks released */
1185 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1186 if (IS_ERR(new_buf))
1187 return new_buf;
1188
1189 /* Let's try again */
1190 spin_lock_irqsave(&drvdata->spinlock, flags);
1191 }
1192
1193 if (drvdata->reading || coresight_get_mode(csdev) == CS_MODE_PERF) {
1194 ret = -EBUSY;
1195 goto out;
1196 }
1197
1198 /*
1199 * If we don't have a buffer or it doesn't match the requested size,
1200 * use the buffer allocated above. Otherwise reuse the existing buffer.
1201 */
1202 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1203 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1204 free_buf = sysfs_buf;
1205 drvdata->sysfs_buf = new_buf;
1206 }
1207
1208out:
1209 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1210
1211 /* Free memory outside the spinlock if need be */
1212 if (free_buf)
1213 tmc_etr_free_sysfs_buf(free_buf);
1214 return ret ? ERR_PTR(ret) : drvdata->sysfs_buf;
1215}
1216
1217static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1218{
1219 int ret = 0;
1220 unsigned long flags;
1221 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1222 struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);
1223
1224 if (IS_ERR(sysfs_buf))
1225 return PTR_ERR(sysfs_buf);
1226
1227 spin_lock_irqsave(&drvdata->spinlock, flags);
1228
1229 /*
1230 * In sysFS mode we can have multiple writers per sink. Since this
1231 * sink is already enabled no memory is needed and the HW need not be
1232 * touched, even if the buffer size has changed.
1233 */
1234 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1235 csdev->refcnt++;
1236 goto out;
1237 }
1238
1239 ret = tmc_etr_enable_hw(drvdata, sysfs_buf);
1240 if (!ret) {
1241 coresight_set_mode(csdev, CS_MODE_SYSFS);
1242 csdev->refcnt++;
1243 }
1244
1245out:
1246 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1247
1248 if (!ret)
1249 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1250
1251 return ret;
1252}
1253
1254struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
1255 enum cs_mode mode, void *data)
1256{
1257 struct perf_output_handle *handle = data;
1258 struct etr_perf_buffer *etr_perf;
1259
1260 switch (mode) {
1261 case CS_MODE_SYSFS:
1262 return tmc_etr_get_sysfs_buffer(csdev);
1263 case CS_MODE_PERF:
1264 etr_perf = etm_perf_sink_config(handle);
1265 if (WARN_ON(!etr_perf || !etr_perf->etr_buf))
1266 return ERR_PTR(-EINVAL);
1267 return etr_perf->etr_buf;
1268 default:
1269 return ERR_PTR(-EINVAL);
1270 }
1271}
1272EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);
1273
1274/*
1275 * alloc_etr_buf: Allocate ETR buffer for use by perf.
1276 * The size of the hardware buffer is dependent on the size configured
1277 * via sysfs and the perf ring buffer size. We prefer to allocate the
1278 * largest possible size, scaling down the size by half until it
1279 * reaches a minimum limit (1M), beyond which we give up.
1280 */
1281static struct etr_buf *
1282alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1283 int nr_pages, void **pages, bool snapshot)
1284{
1285 int node;
1286 struct etr_buf *etr_buf;
1287 unsigned long size;
1288
1289 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1290 /*
1291 * Try to match the perf ring buffer size if it is larger
1292 * than the size requested via sysfs.
1293 */
1294 if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1295 etr_buf = tmc_alloc_etr_buf(drvdata, ((ssize_t)nr_pages << PAGE_SHIFT),
1296 0, node, NULL);
1297 if (!IS_ERR(etr_buf))
1298 goto done;
1299 }
1300
1301 /*
1302 * Else switch to configured size for this ETR
1303 * and scale down until we hit the minimum limit.
1304 */
1305 size = drvdata->size;
1306 do {
1307 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1308 if (!IS_ERR(etr_buf))
1309 goto done;
1310 size /= 2;
1311 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1312
1313 return ERR_PTR(-ENOMEM);
1314
1315done:
1316 return etr_buf;
1317}
1318
1319static struct etr_buf *
1320get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1321 struct perf_event *event, int nr_pages,
1322 void **pages, bool snapshot)
1323{
1324 int ret;
1325 pid_t pid = task_pid_nr(event->owner);
1326 struct etr_buf *etr_buf;
1327
1328retry:
1329 /*
1330 * An etr_perf_buffer is associated with an event and holds a reference
1331 * to the AUX ring buffer that was created for that event. In CPU-wide
1332 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1333 * buffer, share a sink. As such an etr_perf_buffer is created for each
1334 * event but a single etr_buf associated with the ETR is shared between
1335 * them. The last event in a trace session will copy the content of the
1336 * etr_buf to its AUX ring buffer. Ring buffer associated to other
1337 * events are simply not used an freed as events are destoyed. We still
1338 * need to allocate a ring buffer for each event since we don't know
1339 * which event will be last.
1340 */
1341
1342 /*
1343 * The first thing to do here is check if an etr_buf has already been
1344 * allocated for this session. If so it is shared with this event,
1345 * otherwise it is created.
1346 */
1347 mutex_lock(&drvdata->idr_mutex);
1348 etr_buf = idr_find(&drvdata->idr, pid);
1349 if (etr_buf) {
1350 refcount_inc(&etr_buf->refcount);
1351 mutex_unlock(&drvdata->idr_mutex);
1352 return etr_buf;
1353 }
1354
1355 /* If we made it here no buffer has been allocated, do so now. */
1356 mutex_unlock(&drvdata->idr_mutex);
1357
1358 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1359 if (IS_ERR(etr_buf))
1360 return etr_buf;
1361
1362 /* Now that we have a buffer, add it to the IDR. */
1363 mutex_lock(&drvdata->idr_mutex);
1364 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1365 mutex_unlock(&drvdata->idr_mutex);
1366
1367 /* Another event with this session ID has allocated this buffer. */
1368 if (ret == -ENOSPC) {
1369 tmc_free_etr_buf(etr_buf);
1370 goto retry;
1371 }
1372
1373 /* The IDR can't allocate room for a new session, abandon ship. */
1374 if (ret == -ENOMEM) {
1375 tmc_free_etr_buf(etr_buf);
1376 return ERR_PTR(ret);
1377 }
1378
1379
1380 return etr_buf;
1381}
1382
1383static struct etr_buf *
1384get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1385 struct perf_event *event, int nr_pages,
1386 void **pages, bool snapshot)
1387{
1388 /*
1389 * In per-thread mode the etr_buf isn't shared, so just go ahead
1390 * with memory allocation.
1391 */
1392 return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1393}
1394
1395static struct etr_buf *
1396get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1397 int nr_pages, void **pages, bool snapshot)
1398{
1399 if (event->cpu == -1)
1400 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1401 pages, snapshot);
1402
1403 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1404 pages, snapshot);
1405}
1406
1407static struct etr_perf_buffer *
1408tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1409 int nr_pages, void **pages, bool snapshot)
1410{
1411 int node;
1412 struct etr_buf *etr_buf;
1413 struct etr_perf_buffer *etr_perf;
1414
1415 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1416
1417 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1418 if (!etr_perf)
1419 return ERR_PTR(-ENOMEM);
1420
1421 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1422 if (!IS_ERR(etr_buf))
1423 goto done;
1424
1425 kfree(etr_perf);
1426 return ERR_PTR(-ENOMEM);
1427
1428done:
1429 /*
1430 * Keep a reference to the ETR this buffer has been allocated for
1431 * in order to have access to the IDR in tmc_free_etr_buffer().
1432 */
1433 etr_perf->drvdata = drvdata;
1434 etr_perf->etr_buf = etr_buf;
1435
1436 return etr_perf;
1437}
1438
1439
1440static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1441 struct perf_event *event, void **pages,
1442 int nr_pages, bool snapshot)
1443{
1444 struct etr_perf_buffer *etr_perf;
1445 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1446
1447 etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1448 nr_pages, pages, snapshot);
1449 if (IS_ERR(etr_perf)) {
1450 dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1451 return NULL;
1452 }
1453
1454 etr_perf->pid = task_pid_nr(event->owner);
1455 etr_perf->snapshot = snapshot;
1456 etr_perf->nr_pages = nr_pages;
1457 etr_perf->pages = pages;
1458
1459 return etr_perf;
1460}
1461
1462static void tmc_free_etr_buffer(void *config)
1463{
1464 struct etr_perf_buffer *etr_perf = config;
1465 struct tmc_drvdata *drvdata = etr_perf->drvdata;
1466 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1467
1468 if (!etr_buf)
1469 goto free_etr_perf_buffer;
1470
1471 mutex_lock(&drvdata->idr_mutex);
1472 /* If we are not the last one to use the buffer, don't touch it. */
1473 if (!refcount_dec_and_test(&etr_buf->refcount)) {
1474 mutex_unlock(&drvdata->idr_mutex);
1475 goto free_etr_perf_buffer;
1476 }
1477
1478 /* We are the last one, remove from the IDR and free the buffer. */
1479 buf = idr_remove(&drvdata->idr, etr_perf->pid);
1480 mutex_unlock(&drvdata->idr_mutex);
1481
1482 /*
1483 * Something went very wrong if the buffer associated with this ID
1484 * is not the same in the IDR. Leak to avoid use after free.
1485 */
1486 if (buf && WARN_ON(buf != etr_buf))
1487 goto free_etr_perf_buffer;
1488
1489 tmc_free_etr_buf(etr_perf->etr_buf);
1490
1491free_etr_perf_buffer:
1492 kfree(etr_perf);
1493}
1494
1495/*
1496 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1497 * buffer to the perf ring buffer.
1498 */
1499static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1500 unsigned long head,
1501 unsigned long src_offset,
1502 unsigned long to_copy)
1503{
1504 long bytes;
1505 long pg_idx, pg_offset;
1506 char **dst_pages, *src_buf;
1507 struct etr_buf *etr_buf = etr_perf->etr_buf;
1508
1509 head = PERF_IDX2OFF(head, etr_perf);
1510 pg_idx = head >> PAGE_SHIFT;
1511 pg_offset = head & (PAGE_SIZE - 1);
1512 dst_pages = (char **)etr_perf->pages;
1513
1514 while (to_copy > 0) {
1515 /*
1516 * In one iteration, we can copy minimum of :
1517 * 1) what is available in the source buffer,
1518 * 2) what is available in the source buffer, before it
1519 * wraps around.
1520 * 3) what is available in the destination page.
1521 * in one iteration.
1522 */
1523 if (src_offset >= etr_buf->size)
1524 src_offset -= etr_buf->size;
1525 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1526 &src_buf);
1527 if (WARN_ON_ONCE(bytes <= 0))
1528 break;
1529 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1530
1531 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1532
1533 to_copy -= bytes;
1534
1535 /* Move destination pointers */
1536 pg_offset += bytes;
1537 if (pg_offset == PAGE_SIZE) {
1538 pg_offset = 0;
1539 if (++pg_idx == etr_perf->nr_pages)
1540 pg_idx = 0;
1541 }
1542
1543 /* Move source pointers */
1544 src_offset += bytes;
1545 }
1546}
1547
1548/*
1549 * tmc_update_etr_buffer : Update the perf ring buffer with the
1550 * available trace data. We use software double buffering at the moment.
1551 *
1552 * TODO: Add support for reusing the perf ring buffer.
1553 */
1554static unsigned long
1555tmc_update_etr_buffer(struct coresight_device *csdev,
1556 struct perf_output_handle *handle,
1557 void *config)
1558{
1559 bool lost = false;
1560 unsigned long flags, offset, size = 0;
1561 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1562 struct etr_perf_buffer *etr_perf = config;
1563 struct etr_buf *etr_buf = etr_perf->etr_buf;
1564
1565 spin_lock_irqsave(&drvdata->spinlock, flags);
1566
1567 /* Don't do anything if another tracer is using this sink */
1568 if (csdev->refcnt != 1) {
1569 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1570 goto out;
1571 }
1572
1573 if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1574 lost = true;
1575 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1576 goto out;
1577 }
1578
1579 CS_UNLOCK(drvdata->base);
1580
1581 tmc_flush_and_stop(drvdata);
1582 tmc_sync_etr_buf(drvdata);
1583
1584 CS_LOCK(drvdata->base);
1585 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1586
1587 lost = etr_buf->full;
1588 offset = etr_buf->offset;
1589 size = etr_buf->len;
1590
1591 /*
1592 * The ETR buffer may be bigger than the space available in the
1593 * perf ring buffer (handle->size). If so advance the offset so that we
1594 * get the latest trace data. In snapshot mode none of that matters
1595 * since we are expected to clobber stale data in favour of the latest
1596 * traces.
1597 */
1598 if (!etr_perf->snapshot && size > handle->size) {
1599 u32 mask = tmc_get_memwidth_mask(drvdata);
1600
1601 /*
1602 * Make sure the new size is aligned in accordance with the
1603 * requirement explained in function tmc_get_memwidth_mask().
1604 */
1605 size = handle->size & mask;
1606 offset = etr_buf->offset + etr_buf->len - size;
1607
1608 if (offset >= etr_buf->size)
1609 offset -= etr_buf->size;
1610 lost = true;
1611 }
1612
1613 /* Insert barrier packets at the beginning, if there was an overflow */
1614 if (lost)
1615 tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1616 tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);
1617
1618 /*
1619 * In snapshot mode we simply increment the head by the number of byte
1620 * that were written. User space will figure out how many bytes to get
1621 * from the AUX buffer based on the position of the head.
1622 */
1623 if (etr_perf->snapshot)
1624 handle->head += size;
1625
1626 /*
1627 * Ensure that the AUX trace data is visible before the aux_head
1628 * is updated via perf_aux_output_end(), as expected by the
1629 * perf ring buffer.
1630 */
1631 smp_wmb();
1632
1633out:
1634 /*
1635 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1636 * captured buffer is expected to be truncated and 2) a full buffer
1637 * prevents the event from being re-enabled by the perf core,
1638 * resulting in stale data being send to user space.
1639 */
1640 if (!etr_perf->snapshot && lost)
1641 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1642 return size;
1643}
1644
1645static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1646{
1647 int rc = 0;
1648 pid_t pid;
1649 unsigned long flags;
1650 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1651 struct perf_output_handle *handle = data;
1652 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1653
1654 spin_lock_irqsave(&drvdata->spinlock, flags);
1655 /* Don't use this sink if it is already claimed by sysFS */
1656 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1657 rc = -EBUSY;
1658 goto unlock_out;
1659 }
1660
1661 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1662 rc = -EINVAL;
1663 goto unlock_out;
1664 }
1665
1666 /* Get a handle on the pid of the session owner */
1667 pid = etr_perf->pid;
1668
1669 /* Do not proceed if this device is associated with another session */
1670 if (drvdata->pid != -1 && drvdata->pid != pid) {
1671 rc = -EBUSY;
1672 goto unlock_out;
1673 }
1674
1675 /*
1676 * No HW configuration is needed if the sink is already in
1677 * use for this session.
1678 */
1679 if (drvdata->pid == pid) {
1680 csdev->refcnt++;
1681 goto unlock_out;
1682 }
1683
1684 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1685 if (!rc) {
1686 /* Associate with monitored process. */
1687 drvdata->pid = pid;
1688 coresight_set_mode(csdev, CS_MODE_PERF);
1689 drvdata->perf_buf = etr_perf->etr_buf;
1690 csdev->refcnt++;
1691 }
1692
1693unlock_out:
1694 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1695 return rc;
1696}
1697
1698static int tmc_enable_etr_sink(struct coresight_device *csdev,
1699 enum cs_mode mode, void *data)
1700{
1701 switch (mode) {
1702 case CS_MODE_SYSFS:
1703 return tmc_enable_etr_sink_sysfs(csdev);
1704 case CS_MODE_PERF:
1705 return tmc_enable_etr_sink_perf(csdev, data);
1706 default:
1707 return -EINVAL;
1708 }
1709}
1710
1711static int tmc_disable_etr_sink(struct coresight_device *csdev)
1712{
1713 unsigned long flags;
1714 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1715
1716 spin_lock_irqsave(&drvdata->spinlock, flags);
1717
1718 if (drvdata->reading) {
1719 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1720 return -EBUSY;
1721 }
1722
1723 csdev->refcnt--;
1724 if (csdev->refcnt) {
1725 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1726 return -EBUSY;
1727 }
1728
1729 /* Complain if we (somehow) got out of sync */
1730 WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);
1731 tmc_etr_disable_hw(drvdata);
1732 /* Dissociate from monitored process. */
1733 drvdata->pid = -1;
1734 coresight_set_mode(csdev, CS_MODE_DISABLED);
1735 /* Reset perf specific data */
1736 drvdata->perf_buf = NULL;
1737
1738 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1739
1740 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1741 return 0;
1742}
1743
1744static const struct coresight_ops_sink tmc_etr_sink_ops = {
1745 .enable = tmc_enable_etr_sink,
1746 .disable = tmc_disable_etr_sink,
1747 .alloc_buffer = tmc_alloc_etr_buffer,
1748 .update_buffer = tmc_update_etr_buffer,
1749 .free_buffer = tmc_free_etr_buffer,
1750};
1751
1752const struct coresight_ops tmc_etr_cs_ops = {
1753 .sink_ops = &tmc_etr_sink_ops,
1754};
1755
1756int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1757{
1758 int ret = 0;
1759 unsigned long flags;
1760
1761 /* config types are set a boot time and never change */
1762 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1763 return -EINVAL;
1764
1765 spin_lock_irqsave(&drvdata->spinlock, flags);
1766 if (drvdata->reading) {
1767 ret = -EBUSY;
1768 goto out;
1769 }
1770
1771 /*
1772 * We can safely allow reads even if the ETR is operating in PERF mode,
1773 * since the sysfs session is captured in mode specific data.
1774 * If drvdata::sysfs_data is NULL the trace data has been read already.
1775 */
1776 if (!drvdata->sysfs_buf) {
1777 ret = -EINVAL;
1778 goto out;
1779 }
1780
1781 /* Disable the TMC if we are trying to read from a running session. */
1782 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
1783 __tmc_etr_disable_hw(drvdata);
1784
1785 drvdata->reading = true;
1786out:
1787 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1788
1789 return ret;
1790}
1791
1792int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1793{
1794 unsigned long flags;
1795 struct etr_buf *sysfs_buf = NULL;
1796
1797 /* config types are set a boot time and never change */
1798 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1799 return -EINVAL;
1800
1801 spin_lock_irqsave(&drvdata->spinlock, flags);
1802
1803 /* RE-enable the TMC if need be */
1804 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
1805 /*
1806 * The trace run will continue with the same allocated trace
1807 * buffer. Since the tracer is still enabled drvdata::buf can't
1808 * be NULL.
1809 */
1810 __tmc_etr_enable_hw(drvdata);
1811 } else {
1812 /*
1813 * The ETR is not tracing and the buffer was just read.
1814 * As such prepare to free the trace buffer.
1815 */
1816 sysfs_buf = drvdata->sysfs_buf;
1817 drvdata->sysfs_buf = NULL;
1818 }
1819
1820 drvdata->reading = false;
1821 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1822
1823 /* Free allocated memory out side of the spinlock */
1824 if (sysfs_buf)
1825 tmc_etr_free_sysfs_buf(sysfs_buf);
1826
1827 return 0;
1828}
1829
1830static const char *const buf_modes_str[] = {
1831 [ETR_MODE_FLAT] = "flat",
1832 [ETR_MODE_ETR_SG] = "tmc-sg",
1833 [ETR_MODE_CATU] = "catu",
1834 [ETR_MODE_AUTO] = "auto",
1835};
1836
1837static ssize_t buf_modes_available_show(struct device *dev,
1838 struct device_attribute *attr, char *buf)
1839{
1840 struct etr_buf_hw buf_hw;
1841 ssize_t size = 0;
1842
1843 get_etr_buf_hw(dev, &buf_hw);
1844 size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]);
1845 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]);
1846 if (buf_hw.has_etr_sg)
1847 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]);
1848
1849 if (buf_hw.has_catu)
1850 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]);
1851
1852 size += sysfs_emit_at(buf, size, "\n");
1853 return size;
1854}
1855static DEVICE_ATTR_RO(buf_modes_available);
1856
1857static ssize_t buf_mode_preferred_show(struct device *dev,
1858 struct device_attribute *attr, char *buf)
1859{
1860 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
1861
1862 return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]);
1863}
1864
1865static ssize_t buf_mode_preferred_store(struct device *dev,
1866 struct device_attribute *attr,
1867 const char *buf, size_t size)
1868{
1869 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
1870 struct etr_buf_hw buf_hw;
1871
1872 get_etr_buf_hw(dev, &buf_hw);
1873 if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT]))
1874 drvdata->etr_mode = ETR_MODE_FLAT;
1875 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)
1876 drvdata->etr_mode = ETR_MODE_ETR_SG;
1877 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)
1878 drvdata->etr_mode = ETR_MODE_CATU;
1879 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO]))
1880 drvdata->etr_mode = ETR_MODE_AUTO;
1881 else
1882 return -EINVAL;
1883 return size;
1884}
1885static DEVICE_ATTR_RW(buf_mode_preferred);
1886
1887static struct attribute *coresight_etr_attrs[] = {
1888 &dev_attr_buf_modes_available.attr,
1889 &dev_attr_buf_mode_preferred.attr,
1890 NULL,
1891};
1892
1893const struct attribute_group coresight_etr_group = {
1894 .attrs = coresight_etr_attrs,
1895};