1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
   4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   5 */
   6
   7#include <linux/atomic.h>
   8#include <linux/coresight.h>
   9#include <linux/dma-mapping.h>
  10#include <linux/iommu.h>
  11#include <linux/idr.h>
  12#include <linux/mutex.h>
  13#include <linux/refcount.h>
  14#include <linux/slab.h>
  15#include <linux/types.h>
  16#include <linux/vmalloc.h>
  17#include "coresight-catu.h"
  18#include "coresight-etm-perf.h"
  19#include "coresight-priv.h"
  20#include "coresight-tmc.h"
  21
  22struct etr_flat_buf {
  23	struct device	*dev;
  24	dma_addr_t	daddr;
  25	void		*vaddr;
  26	size_t		size;
  27};
  28
  29struct etr_buf_hw {
  30	bool	has_iommu;
  31	bool	has_etr_sg;
  32	bool	has_catu;
  33};
  34
  35/*
  36 * etr_perf_buffer - Perf buffer used for ETR
  37 * @drvdata		- The ETR drvdaga this buffer has been allocated for.
  38 * @etr_buf		- Actual buffer used by the ETR
  39 * @pid			- The PID of the session owner that etr_perf_buffer
  40 *			  belongs to.
  41 * @snaphost		- Perf session mode
 
  42 * @nr_pages		- Number of pages in the ring buffer.
  43 * @pages		- Array of Pages in the ring buffer.
  44 */
  45struct etr_perf_buffer {
  46	struct tmc_drvdata	*drvdata;
  47	struct etr_buf		*etr_buf;
  48	pid_t			pid;
  49	bool			snapshot;
 
  50	int			nr_pages;
  51	void			**pages;
  52};
  53
  54/* Convert the perf index to an offset within the ETR buffer */
  55#define PERF_IDX2OFF(idx, buf)		\
  56		((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
  57
  58/* Lower limit for ETR hardware buffer */
  59#define TMC_ETR_PERF_MIN_BUF_SIZE	SZ_1M
  60
  61/*
  62 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
  63 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
  64 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
  65 * contain more than one SG buffer and tables.
  66 *
  67 * A table entry has the following format:
  68 *
  69 * ---Bit31------------Bit4-------Bit1-----Bit0--
  70 * |     Address[39:12]    | SBZ |  Entry Type  |
  71 * ----------------------------------------------
  72 *
  73 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
  74 *	    always zero.
  75 *
  76 * Entry type:
  77 *	b00 - Reserved.
  78 *	b01 - Last entry in the tables, points to 4K page buffer.
  79 *	b10 - Normal entry, points to 4K page buffer.
  80 *	b11 - Link. The address points to the base of next table.
  81 */
  82
  83typedef u32 sgte_t;
  84
  85#define ETR_SG_PAGE_SHIFT		12
  86#define ETR_SG_PAGE_SIZE		(1UL << ETR_SG_PAGE_SHIFT)
  87#define ETR_SG_PAGES_PER_SYSPAGE	(PAGE_SIZE / ETR_SG_PAGE_SIZE)
  88#define ETR_SG_PTRS_PER_PAGE		(ETR_SG_PAGE_SIZE / sizeof(sgte_t))
  89#define ETR_SG_PTRS_PER_SYSPAGE		(PAGE_SIZE / sizeof(sgte_t))
  90
  91#define ETR_SG_ET_MASK			0x3
  92#define ETR_SG_ET_LAST			0x1
  93#define ETR_SG_ET_NORMAL		0x2
  94#define ETR_SG_ET_LINK			0x3
  95
  96#define ETR_SG_ADDR_SHIFT		4
  97
  98#define ETR_SG_ENTRY(addr, type) \
  99	(sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
 100		 (type & ETR_SG_ET_MASK))
 101
 102#define ETR_SG_ADDR(entry) \
 103	(((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
 104#define ETR_SG_ET(entry)		((entry) & ETR_SG_ET_MASK)
 105
 106/*
 107 * struct etr_sg_table : ETR SG Table
 108 * @sg_table:		Generic SG Table holding the data/table pages.
 109 * @hwaddr:		hwaddress used by the TMC, which is the base
 110 *			address of the table.
 111 */
 112struct etr_sg_table {
 113	struct tmc_sg_table	*sg_table;
 114	dma_addr_t		hwaddr;
 115};
 116
 117/*
 118 * tmc_etr_sg_table_entries: Total number of table entries required to map
 119 * @nr_pages system pages.
 120 *
 121 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
 122 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
 123 * with the last entry pointing to another page of table entries.
 124 * If we spill over to a new page for mapping 1 entry, we could as
 125 * well replace the link entry of the previous page with the last entry.
 126 */
 127static inline unsigned long __attribute_const__
 128tmc_etr_sg_table_entries(int nr_pages)
 129{
 130	unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
 131	unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
 132	/*
 133	 * If we spill over to a new page for 1 entry, we could as well
 134	 * make it the LAST entry in the previous page, skipping the Link
 135	 * address.
 136	 */
 137	if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
 138		nr_sglinks--;
 139	return nr_sgpages + nr_sglinks;
 140}
 141
 142/*
 143 * tmc_pages_get_offset:  Go through all the pages in the tmc_pages
 144 * and map the device address @addr to an offset within the virtual
 145 * contiguous buffer.
 146 */
 147static long
 148tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
 149{
 150	int i;
 151	dma_addr_t page_start;
 152
 153	for (i = 0; i < tmc_pages->nr_pages; i++) {
 154		page_start = tmc_pages->daddrs[i];
 155		if (addr >= page_start && addr < (page_start + PAGE_SIZE))
 156			return i * PAGE_SIZE + (addr - page_start);
 157	}
 158
 159	return -EINVAL;
 160}
 161
 162/*
 163 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
 164 * If the pages were not allocated in tmc_pages_alloc(), we would
 165 * simply drop the refcount.
 166 */
 167static void tmc_pages_free(struct tmc_pages *tmc_pages,
 168			   struct device *dev, enum dma_data_direction dir)
 169{
 170	int i;
 171	struct device *real_dev = dev->parent;
 172
 173	for (i = 0; i < tmc_pages->nr_pages; i++) {
 174		if (tmc_pages->daddrs && tmc_pages->daddrs[i])
 175			dma_unmap_page(real_dev, tmc_pages->daddrs[i],
 176					 PAGE_SIZE, dir);
 177		if (tmc_pages->pages && tmc_pages->pages[i])
 178			__free_page(tmc_pages->pages[i]);
 179	}
 180
 181	kfree(tmc_pages->pages);
 182	kfree(tmc_pages->daddrs);
 183	tmc_pages->pages = NULL;
 184	tmc_pages->daddrs = NULL;
 185	tmc_pages->nr_pages = 0;
 186}
 187
 188/*
 189 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
 190 * If @pages is not NULL, the list of page virtual addresses are
 191 * used as the data pages. The pages are then dma_map'ed for @dev
 192 * with dma_direction @dir.
 193 *
 194 * Returns 0 upon success, else the error number.
 195 */
 196static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
 197			   struct device *dev, int node,
 198			   enum dma_data_direction dir, void **pages)
 199{
 200	int i, nr_pages;
 201	dma_addr_t paddr;
 202	struct page *page;
 203	struct device *real_dev = dev->parent;
 204
 205	nr_pages = tmc_pages->nr_pages;
 206	tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
 207					 GFP_KERNEL);
 208	if (!tmc_pages->daddrs)
 209		return -ENOMEM;
 210	tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
 211					 GFP_KERNEL);
 212	if (!tmc_pages->pages) {
 213		kfree(tmc_pages->daddrs);
 214		tmc_pages->daddrs = NULL;
 215		return -ENOMEM;
 216	}
 217
 218	for (i = 0; i < nr_pages; i++) {
 219		if (pages && pages[i]) {
 220			page = virt_to_page(pages[i]);
 221			/* Hold a refcount on the page */
 222			get_page(page);
 223		} else {
 224			page = alloc_pages_node(node,
 225						GFP_KERNEL | __GFP_ZERO, 0);
 226			if (!page)
 227				goto err;
 228		}
 229		paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
 230		if (dma_mapping_error(real_dev, paddr))
 231			goto err;
 232		tmc_pages->daddrs[i] = paddr;
 233		tmc_pages->pages[i] = page;
 234	}
 235	return 0;
 236err:
 237	tmc_pages_free(tmc_pages, dev, dir);
 238	return -ENOMEM;
 239}
 240
 241static inline long
 242tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
 243{
 244	return tmc_pages_get_offset(&sg_table->data_pages, addr);
 245}
 246
 247static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
 248{
 249	if (sg_table->table_vaddr)
 250		vunmap(sg_table->table_vaddr);
 251	tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
 252}
 253
 254static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
 255{
 256	if (sg_table->data_vaddr)
 257		vunmap(sg_table->data_vaddr);
 258	tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
 259}
 260
 261void tmc_free_sg_table(struct tmc_sg_table *sg_table)
 262{
 263	tmc_free_table_pages(sg_table);
 264	tmc_free_data_pages(sg_table);
 265	kfree(sg_table);
 266}
 267EXPORT_SYMBOL_GPL(tmc_free_sg_table);
 268
 269/*
 270 * Alloc pages for the table. Since this will be used by the device,
 271 * allocate the pages closer to the device (i.e, dev_to_node(dev)
 272 * rather than the CPU node).
 273 */
 274static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
 275{
 276	int rc;
 277	struct tmc_pages *table_pages = &sg_table->table_pages;
 278
 279	rc = tmc_pages_alloc(table_pages, sg_table->dev,
 280			     dev_to_node(sg_table->dev),
 281			     DMA_TO_DEVICE, NULL);
 282	if (rc)
 283		return rc;
 284	sg_table->table_vaddr = vmap(table_pages->pages,
 285				     table_pages->nr_pages,
 286				     VM_MAP,
 287				     PAGE_KERNEL);
 288	if (!sg_table->table_vaddr)
 289		rc = -ENOMEM;
 290	else
 291		sg_table->table_daddr = table_pages->daddrs[0];
 292	return rc;
 293}
 294
 295static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
 296{
 297	int rc;
 298
 299	/* Allocate data pages on the node requested by the caller */
 300	rc = tmc_pages_alloc(&sg_table->data_pages,
 301			     sg_table->dev, sg_table->node,
 302			     DMA_FROM_DEVICE, pages);
 303	if (!rc) {
 304		sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
 305					    sg_table->data_pages.nr_pages,
 306					    VM_MAP,
 307					    PAGE_KERNEL);
 308		if (!sg_table->data_vaddr)
 309			rc = -ENOMEM;
 310	}
 311	return rc;
 312}
 313
 314/*
 315 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
 316 * and data buffers. TMC writes to the data buffers and reads from the SG
 317 * Table pages.
 318 *
 319 * @dev		- Coresight device to which page should be DMA mapped.
 320 * @node	- Numa node for mem allocations
 321 * @nr_tpages	- Number of pages for the table entries.
 322 * @nr_dpages	- Number of pages for Data buffer.
 323 * @pages	- Optional list of virtual address of pages.
 324 */
 325struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
 326					int node,
 327					int nr_tpages,
 328					int nr_dpages,
 329					void **pages)
 330{
 331	long rc;
 332	struct tmc_sg_table *sg_table;
 333
 334	sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
 335	if (!sg_table)
 336		return ERR_PTR(-ENOMEM);
 337	sg_table->data_pages.nr_pages = nr_dpages;
 338	sg_table->table_pages.nr_pages = nr_tpages;
 339	sg_table->node = node;
 340	sg_table->dev = dev;
 341
 342	rc  = tmc_alloc_data_pages(sg_table, pages);
 343	if (!rc)
 344		rc = tmc_alloc_table_pages(sg_table);
 345	if (rc) {
 346		tmc_free_sg_table(sg_table);
 
 347		return ERR_PTR(rc);
 348	}
 349
 350	return sg_table;
 351}
 352EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
 353
 354/*
 355 * tmc_sg_table_sync_data_range: Sync the data buffer written
 356 * by the device from @offset upto a @size bytes.
 357 */
 358void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
 359				  u64 offset, u64 size)
 360{
 361	int i, index, start;
 362	int npages = DIV_ROUND_UP(size, PAGE_SIZE);
 363	struct device *real_dev = table->dev->parent;
 364	struct tmc_pages *data = &table->data_pages;
 365
 366	start = offset >> PAGE_SHIFT;
 367	for (i = start; i < (start + npages); i++) {
 368		index = i % data->nr_pages;
 369		dma_sync_single_for_cpu(real_dev, data->daddrs[index],
 370					PAGE_SIZE, DMA_FROM_DEVICE);
 371	}
 372}
 373EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
 374
 375/* tmc_sg_sync_table: Sync the page table */
 376void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
 377{
 378	int i;
 379	struct device *real_dev = sg_table->dev->parent;
 380	struct tmc_pages *table_pages = &sg_table->table_pages;
 381
 382	for (i = 0; i < table_pages->nr_pages; i++)
 383		dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
 384					   PAGE_SIZE, DMA_TO_DEVICE);
 385}
 386EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
 387
 388/*
 389 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
 390 * in the SG buffer. The @bufpp is updated to point to the buffer.
 391 * Returns :
 392 *	the length of linear data available at @offset.
 393 *	or
 394 *	<= 0 if no data is available.
 395 */
 396ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
 397			      u64 offset, size_t len, char **bufpp)
 398{
 399	size_t size;
 400	int pg_idx = offset >> PAGE_SHIFT;
 401	int pg_offset = offset & (PAGE_SIZE - 1);
 402	struct tmc_pages *data_pages = &sg_table->data_pages;
 403
 404	size = tmc_sg_table_buf_size(sg_table);
 405	if (offset >= size)
 406		return -EINVAL;
 407
 408	/* Make sure we don't go beyond the end */
 409	len = (len < (size - offset)) ? len : size - offset;
 410	/* Respect the page boundaries */
 411	len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
 412	if (len > 0)
 413		*bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
 414	return len;
 415}
 416EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
 417
 418#ifdef ETR_SG_DEBUG
 419/* Map a dma address to virtual address */
 420static unsigned long
 421tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
 422		      dma_addr_t addr, bool table)
 423{
 424	long offset;
 425	unsigned long base;
 426	struct tmc_pages *tmc_pages;
 427
 428	if (table) {
 429		tmc_pages = &sg_table->table_pages;
 430		base = (unsigned long)sg_table->table_vaddr;
 431	} else {
 432		tmc_pages = &sg_table->data_pages;
 433		base = (unsigned long)sg_table->data_vaddr;
 434	}
 435
 436	offset = tmc_pages_get_offset(tmc_pages, addr);
 437	if (offset < 0)
 438		return 0;
 439	return base + offset;
 440}
 441
 442/* Dump the given sg_table */
 443static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
 444{
 445	sgte_t *ptr;
 446	int i = 0;
 447	dma_addr_t addr;
 448	struct tmc_sg_table *sg_table = etr_table->sg_table;
 449
 450	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
 451					      etr_table->hwaddr, true);
 452	while (ptr) {
 453		addr = ETR_SG_ADDR(*ptr);
 454		switch (ETR_SG_ET(*ptr)) {
 455		case ETR_SG_ET_NORMAL:
 456			dev_dbg(sg_table->dev,
 457				"%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
 458			ptr++;
 459			break;
 460		case ETR_SG_ET_LINK:
 461			dev_dbg(sg_table->dev,
 462				"%05d: *** %p\t:{L} 0x%llx ***\n",
 463				 i, ptr, addr);
 464			ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
 465							      addr, true);
 466			break;
 467		case ETR_SG_ET_LAST:
 468			dev_dbg(sg_table->dev,
 469				"%05d: ### %p\t:[L] 0x%llx ###\n",
 470				 i, ptr, addr);
 471			return;
 472		default:
 473			dev_dbg(sg_table->dev,
 474				"%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
 475				 i, ptr, addr);
 476			return;
 477		}
 478		i++;
 479	}
 480	dev_dbg(sg_table->dev, "******* End of Table *****\n");
 481}
 482#else
 483static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
 484#endif
 485
 486/*
 487 * Populate the SG Table page table entries from table/data
 488 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
 489 * So does a Table page. So we keep track of indices of the tables
 490 * in each system page and move the pointers accordingly.
 491 */
 492#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
 493static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
 494{
 495	dma_addr_t paddr;
 496	int i, type, nr_entries;
 497	int tpidx = 0; /* index to the current system table_page */
 498	int sgtidx = 0;	/* index to the sg_table within the current syspage */
 499	int sgtentry = 0; /* the entry within the sg_table */
 500	int dpidx = 0; /* index to the current system data_page */
 501	int spidx = 0; /* index to the SG page within the current data page */
 502	sgte_t *ptr; /* pointer to the table entry to fill */
 503	struct tmc_sg_table *sg_table = etr_table->sg_table;
 504	dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
 505	dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
 506
 507	nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
 508	/*
 509	 * Use the contiguous virtual address of the table to update entries.
 510	 */
 511	ptr = sg_table->table_vaddr;
 512	/*
 513	 * Fill all the entries, except the last entry to avoid special
 514	 * checks within the loop.
 515	 */
 516	for (i = 0; i < nr_entries - 1; i++) {
 517		if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
 518			/*
 519			 * Last entry in a sg_table page is a link address to
 520			 * the next table page. If this sg_table is the last
 521			 * one in the system page, it links to the first
 522			 * sg_table in the next system page. Otherwise, it
 523			 * links to the next sg_table page within the system
 524			 * page.
 525			 */
 526			if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
 527				paddr = table_daddrs[tpidx + 1];
 528			} else {
 529				paddr = table_daddrs[tpidx] +
 530					(ETR_SG_PAGE_SIZE * (sgtidx + 1));
 531			}
 532			type = ETR_SG_ET_LINK;
 533		} else {
 534			/*
 535			 * Update the indices to the data_pages to point to the
 536			 * next sg_page in the data buffer.
 537			 */
 538			type = ETR_SG_ET_NORMAL;
 539			paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
 540			if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
 541				dpidx++;
 542		}
 543		*ptr++ = ETR_SG_ENTRY(paddr, type);
 544		/*
 545		 * Move to the next table pointer, moving the table page index
 546		 * if necessary
 547		 */
 548		if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
 549			if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
 550				tpidx++;
 551		}
 552	}
 553
 554	/* Set up the last entry, which is always a data pointer */
 555	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
 556	*ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
 557}
 558
 559/*
 560 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
 561 * populate the table.
 562 *
 563 * @dev		- Device pointer for the TMC
 564 * @node	- NUMA node where the memory should be allocated
 565 * @size	- Total size of the data buffer
 566 * @pages	- Optional list of page virtual address
 567 */
 568static struct etr_sg_table *
 569tmc_init_etr_sg_table(struct device *dev, int node,
 570		      unsigned long size, void **pages)
 571{
 572	int nr_entries, nr_tpages;
 573	int nr_dpages = size >> PAGE_SHIFT;
 574	struct tmc_sg_table *sg_table;
 575	struct etr_sg_table *etr_table;
 576
 577	etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
 578	if (!etr_table)
 579		return ERR_PTR(-ENOMEM);
 580	nr_entries = tmc_etr_sg_table_entries(nr_dpages);
 581	nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
 582
 583	sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
 584	if (IS_ERR(sg_table)) {
 585		kfree(etr_table);
 586		return ERR_CAST(sg_table);
 587	}
 588
 589	etr_table->sg_table = sg_table;
 590	/* TMC should use table base address for DBA */
 591	etr_table->hwaddr = sg_table->table_daddr;
 592	tmc_etr_sg_table_populate(etr_table);
 593	/* Sync the table pages for the HW */
 594	tmc_sg_table_sync_table(sg_table);
 595	tmc_etr_sg_table_dump(etr_table);
 596
 597	return etr_table;
 598}
 599
 600/*
 601 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
 602 */
 603static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
 604				  struct etr_buf *etr_buf, int node,
 605				  void **pages)
 606{
 607	struct etr_flat_buf *flat_buf;
 608	struct device *real_dev = drvdata->csdev->dev.parent;
 609
 610	/* We cannot reuse existing pages for flat buf */
 611	if (pages)
 612		return -EINVAL;
 613
 614	flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
 615	if (!flat_buf)
 616		return -ENOMEM;
 617
 618	flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,
 619						&flat_buf->daddr,
 620						DMA_FROM_DEVICE,
 621						GFP_KERNEL | __GFP_NOWARN);
 622	if (!flat_buf->vaddr) {
 623		kfree(flat_buf);
 624		return -ENOMEM;
 625	}
 626
 627	flat_buf->size = etr_buf->size;
 628	flat_buf->dev = &drvdata->csdev->dev;
 629	etr_buf->hwaddr = flat_buf->daddr;
 630	etr_buf->mode = ETR_MODE_FLAT;
 631	etr_buf->private = flat_buf;
 632	return 0;
 633}
 634
 635static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
 636{
 637	struct etr_flat_buf *flat_buf = etr_buf->private;
 638
 639	if (flat_buf && flat_buf->daddr) {
 640		struct device *real_dev = flat_buf->dev->parent;
 641
 642		dma_free_noncoherent(real_dev, etr_buf->size,
 643				     flat_buf->vaddr, flat_buf->daddr,
 644				     DMA_FROM_DEVICE);
 645	}
 646	kfree(flat_buf);
 647}
 648
 649static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
 650{
 651	struct etr_flat_buf *flat_buf = etr_buf->private;
 652	struct device *real_dev = flat_buf->dev->parent;
 653
 654	/*
 655	 * Adjust the buffer to point to the beginning of the trace data
 656	 * and update the available trace data.
 657	 */
 658	etr_buf->offset = rrp - etr_buf->hwaddr;
 659	if (etr_buf->full)
 660		etr_buf->len = etr_buf->size;
 661	else
 662		etr_buf->len = rwp - rrp;
 663
 664	/*
 665	 * The driver always starts tracing at the beginning of the buffer,
 666	 * the only reason why we would get a wrap around is when the buffer
 667	 * is full.  Sync the entire buffer in one go for this case.
 668	 */
 669	if (etr_buf->offset + etr_buf->len > etr_buf->size)
 670		dma_sync_single_for_cpu(real_dev, flat_buf->daddr,
 671					etr_buf->size, DMA_FROM_DEVICE);
 672	else
 673		dma_sync_single_for_cpu(real_dev,
 674					flat_buf->daddr + etr_buf->offset,
 675					etr_buf->len, DMA_FROM_DEVICE);
 676}
 677
 678static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
 679					 u64 offset, size_t len, char **bufpp)
 680{
 681	struct etr_flat_buf *flat_buf = etr_buf->private;
 682
 683	*bufpp = (char *)flat_buf->vaddr + offset;
 684	/*
 685	 * tmc_etr_buf_get_data already adjusts the length to handle
 686	 * buffer wrapping around.
 687	 */
 688	return len;
 689}
 690
 691static const struct etr_buf_operations etr_flat_buf_ops = {
 692	.alloc = tmc_etr_alloc_flat_buf,
 693	.free = tmc_etr_free_flat_buf,
 694	.sync = tmc_etr_sync_flat_buf,
 695	.get_data = tmc_etr_get_data_flat_buf,
 696};
 697
 698/*
 699 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
 700 * appropriately.
 701 */
 702static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
 703				struct etr_buf *etr_buf, int node,
 704				void **pages)
 705{
 706	struct etr_sg_table *etr_table;
 707	struct device *dev = &drvdata->csdev->dev;
 708
 709	etr_table = tmc_init_etr_sg_table(dev, node,
 710					  etr_buf->size, pages);
 711	if (IS_ERR(etr_table))
 712		return -ENOMEM;
 713	etr_buf->hwaddr = etr_table->hwaddr;
 714	etr_buf->mode = ETR_MODE_ETR_SG;
 715	etr_buf->private = etr_table;
 716	return 0;
 717}
 718
 719static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
 720{
 721	struct etr_sg_table *etr_table = etr_buf->private;
 722
 723	if (etr_table) {
 724		tmc_free_sg_table(etr_table->sg_table);
 725		kfree(etr_table);
 726	}
 727}
 728
 729static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
 730				       size_t len, char **bufpp)
 731{
 732	struct etr_sg_table *etr_table = etr_buf->private;
 733
 734	return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
 735}
 736
 737static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
 738{
 739	long r_offset, w_offset;
 740	struct etr_sg_table *etr_table = etr_buf->private;
 741	struct tmc_sg_table *table = etr_table->sg_table;
 742
 743	/* Convert hw address to offset in the buffer */
 744	r_offset = tmc_sg_get_data_page_offset(table, rrp);
 745	if (r_offset < 0) {
 746		dev_warn(table->dev,
 747			 "Unable to map RRP %llx to offset\n", rrp);
 748		etr_buf->len = 0;
 749		return;
 750	}
 751
 752	w_offset = tmc_sg_get_data_page_offset(table, rwp);
 753	if (w_offset < 0) {
 754		dev_warn(table->dev,
 755			 "Unable to map RWP %llx to offset\n", rwp);
 756		etr_buf->len = 0;
 757		return;
 758	}
 759
 760	etr_buf->offset = r_offset;
 761	if (etr_buf->full)
 762		etr_buf->len = etr_buf->size;
 763	else
 764		etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
 765				w_offset - r_offset;
 766	tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
 767}
 768
 769static const struct etr_buf_operations etr_sg_buf_ops = {
 770	.alloc = tmc_etr_alloc_sg_buf,
 771	.free = tmc_etr_free_sg_buf,
 772	.sync = tmc_etr_sync_sg_buf,
 773	.get_data = tmc_etr_get_data_sg_buf,
 774};
 775
 776/*
 777 * TMC ETR could be connected to a CATU device, which can provide address
 778 * translation service. This is represented by the Output port of the TMC
 779 * (ETR) connected to the input port of the CATU.
 780 *
 781 * Returns	: coresight_device ptr for the CATU device if a CATU is found.
 782 *		: NULL otherwise.
 783 */
 784struct coresight_device *
 785tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
 786{
 787	struct coresight_device *etr = drvdata->csdev;
 788	union coresight_dev_subtype catu_subtype = {
 789		.helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU
 790	};
 791
 792	if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
 793		return NULL;
 794
 795	return coresight_find_output_type(etr->pdata, CORESIGHT_DEV_TYPE_HELPER,
 796					  catu_subtype);
 797}
 798EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
 
 799
 800static const struct etr_buf_operations *etr_buf_ops[] = {
 801	[ETR_MODE_FLAT] = &etr_flat_buf_ops,
 802	[ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
 803	[ETR_MODE_CATU] = NULL,
 804};
 805
 806void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
 
 807{
 808	etr_buf_ops[ETR_MODE_CATU] = catu;
 
 
 
 
 809}
 810EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
 811
 812void tmc_etr_remove_catu_ops(void)
 813{
 814	etr_buf_ops[ETR_MODE_CATU] = NULL;
 
 
 
 815}
 816EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
 
 
 
 
 
 
 817
 818static inline int tmc_etr_mode_alloc_buf(int mode,
 819					 struct tmc_drvdata *drvdata,
 820					 struct etr_buf *etr_buf, int node,
 821					 void **pages)
 822{
 823	int rc = -EINVAL;
 824
 825	switch (mode) {
 826	case ETR_MODE_FLAT:
 827	case ETR_MODE_ETR_SG:
 828	case ETR_MODE_CATU:
 829		if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
 830			rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
 831						      node, pages);
 832		if (!rc)
 833			etr_buf->ops = etr_buf_ops[mode];
 834		return rc;
 835	default:
 836		return -EINVAL;
 837	}
 838}
 839
 840static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)
 841{
 842	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
 843
 844	buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent);
 845	buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
 846	buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);
 847}
 848
 849static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)
 850{
 851	bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;
 852
 853	return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;
 854}
 855
 856/*
 857 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
 858 * @drvdata	: ETR device details.
 859 * @size	: size of the requested buffer.
 860 * @flags	: Required properties for the buffer.
 861 * @node	: Node for memory allocations.
 862 * @pages	: An optional list of pages.
 863 */
 864static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
 865					 ssize_t size, int flags,
 866					 int node, void **pages)
 867{
 868	int rc = -ENOMEM;
 
 
 869	struct etr_buf *etr_buf;
 870	struct etr_buf_hw buf_hw;
 871	struct device *dev = &drvdata->csdev->dev;
 872
 873	get_etr_buf_hw(dev, &buf_hw);
 
 
 
 
 
 874	etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
 875	if (!etr_buf)
 876		return ERR_PTR(-ENOMEM);
 877
 878	etr_buf->size = size;
 879
 880	/* If there is user directive for buffer mode, try that first */
 881	if (drvdata->etr_mode != ETR_MODE_AUTO)
 882		rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata,
 883					    etr_buf, node, pages);
 884
 885	/*
 886	 * If we have to use an existing list of pages, we cannot reliably
 887	 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
 888	 * we use the contiguous DMA memory if at least one of the following
 889	 * conditions is true:
 890	 *  a) The ETR cannot use Scatter-Gather.
 891	 *  b) we have a backing IOMMU
 892	 *  c) The requested memory size is smaller (< 1M).
 893	 *
 894	 * Fallback to available mechanisms.
 895	 *
 896	 */
 897	if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size))
 
 898		rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
 899					    etr_buf, node, pages);
 900	if (rc && buf_hw.has_etr_sg)
 901		rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
 902					    etr_buf, node, pages);
 903	if (rc && buf_hw.has_catu)
 904		rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
 905					    etr_buf, node, pages);
 906	if (rc) {
 907		kfree(etr_buf);
 908		return ERR_PTR(rc);
 909	}
 910
 911	refcount_set(&etr_buf->refcount, 1);
 912	dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
 913		(unsigned long)size >> 10, etr_buf->mode);
 914	return etr_buf;
 915}
 916
 917static void tmc_free_etr_buf(struct etr_buf *etr_buf)
 918{
 919	WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
 920	etr_buf->ops->free(etr_buf);
 921	kfree(etr_buf);
 922}
 923
 924/*
 925 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
 926 * with a maximum of @len bytes.
 927 * Returns: The size of the linear data available @pos, with *bufpp
 928 * updated to point to the buffer.
 929 */
 930static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
 931				    u64 offset, size_t len, char **bufpp)
 932{
 933	/* Adjust the length to limit this transaction to end of buffer */
 934	len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
 935
 936	return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
 937}
 938
 939static inline s64
 940tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
 941{
 942	ssize_t len;
 943	char *bufp;
 944
 945	len = tmc_etr_buf_get_data(etr_buf, offset,
 946				   CORESIGHT_BARRIER_PKT_SIZE, &bufp);
 947	if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))
 948		return -EINVAL;
 949	coresight_insert_barrier_packet(bufp);
 950	return offset + CORESIGHT_BARRIER_PKT_SIZE;
 951}
 952
 953/*
 954 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
 955 * Makes sure the trace data is synced to the memory for consumption.
 956 * @etr_buf->offset will hold the offset to the beginning of the trace data
 957 * within the buffer, with @etr_buf->len bytes to consume.
 958 */
 959static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
 960{
 961	struct etr_buf *etr_buf = drvdata->etr_buf;
 962	u64 rrp, rwp;
 963	u32 status;
 964
 965	rrp = tmc_read_rrp(drvdata);
 966	rwp = tmc_read_rwp(drvdata);
 967	status = readl_relaxed(drvdata->base + TMC_STS);
 968
 969	/*
 970	 * If there were memory errors in the session, truncate the
 971	 * buffer.
 972	 */
 973	if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
 974		dev_dbg(&drvdata->csdev->dev,
 975			"tmc memory error detected, truncating buffer\n");
 976		etr_buf->len = 0;
 977		etr_buf->full = false;
 978		return;
 979	}
 980
 981	etr_buf->full = !!(status & TMC_STS_FULL);
 982
 983	WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
 984
 985	etr_buf->ops->sync(etr_buf, rrp, rwp);
 986}
 987
 988static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
 989{
 990	u32 axictl, sts;
 991	struct etr_buf *etr_buf = drvdata->etr_buf;
 992	int rc = 0;
 993
 994	CS_UNLOCK(drvdata->base);
 995
 996	/* Wait for TMCSReady bit to be set */
 997	rc = tmc_wait_for_tmcready(drvdata);
 998	if (rc) {
 999		dev_err(&drvdata->csdev->dev,
1000			"Failed to enable : TMC not ready\n");
1001		CS_LOCK(drvdata->base);
1002		return rc;
1003	}
1004
1005	writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
1006	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
1007
1008	axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
1009	axictl &= ~TMC_AXICTL_CLEAR_MASK;
1010	axictl |= TMC_AXICTL_PROT_CTL_B1;
1011	axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
1012	axictl |= TMC_AXICTL_AXCACHE_OS;
1013
1014	if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
1015		axictl &= ~TMC_AXICTL_ARCACHE_MASK;
1016		axictl |= TMC_AXICTL_ARCACHE_OS;
1017	}
1018
1019	if (etr_buf->mode == ETR_MODE_ETR_SG)
1020		axictl |= TMC_AXICTL_SCT_GAT_MODE;
1021
1022	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1023	tmc_write_dba(drvdata, etr_buf->hwaddr);
1024	/*
1025	 * If the TMC pointers must be programmed before the session,
1026	 * we have to set it properly (i.e, RRP/RWP to base address and
1027	 * STS to "not full").
1028	 */
1029	if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1030		tmc_write_rrp(drvdata, etr_buf->hwaddr);
1031		tmc_write_rwp(drvdata, etr_buf->hwaddr);
1032		sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1033		writel_relaxed(sts, drvdata->base + TMC_STS);
1034	}
1035
1036	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1037		       TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1038		       TMC_FFCR_TRIGON_TRIGIN,
1039		       drvdata->base + TMC_FFCR);
1040	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1041	tmc_enable_hw(drvdata);
1042
1043	CS_LOCK(drvdata->base);
1044	return rc;
1045}
1046
1047static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1048			     struct etr_buf *etr_buf)
1049{
1050	int rc;
1051
1052	/* Callers should provide an appropriate buffer for use */
1053	if (WARN_ON(!etr_buf))
1054		return -EINVAL;
1055
1056	if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1057	    WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1058		return -EINVAL;
1059
1060	if (WARN_ON(drvdata->etr_buf))
1061		return -EBUSY;
1062
1063	rc = coresight_claim_device(drvdata->csdev);
 
 
 
 
 
 
 
1064	if (!rc) {
1065		drvdata->etr_buf = etr_buf;
1066		rc = __tmc_etr_enable_hw(drvdata);
1067		if (rc) {
1068			drvdata->etr_buf = NULL;
1069			coresight_disclaim_device(drvdata->csdev);
1070		}
1071	}
1072
1073	return rc;
1074}
1075
1076/*
1077 * Return the available trace data in the buffer (starts at etr_buf->offset,
1078 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1079 * also updating the @bufpp on where to find it. Since the trace data
1080 * starts at anywhere in the buffer, depending on the RRP, we adjust the
1081 * @len returned to handle buffer wrapping around.
1082 *
1083 * We are protected here by drvdata->reading != 0, which ensures the
1084 * sysfs_buf stays alive.
1085 */
1086ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1087				loff_t pos, size_t len, char **bufpp)
1088{
1089	s64 offset;
1090	ssize_t actual = len;
1091	struct etr_buf *etr_buf = drvdata->sysfs_buf;
1092
1093	if (pos + actual > etr_buf->len)
1094		actual = etr_buf->len - pos;
1095	if (actual <= 0)
1096		return actual;
1097
1098	/* Compute the offset from which we read the data */
1099	offset = etr_buf->offset + pos;
1100	if (offset >= etr_buf->size)
1101		offset -= etr_buf->size;
1102	return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1103}
1104
1105static struct etr_buf *
1106tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1107{
1108	return tmc_alloc_etr_buf(drvdata, drvdata->size,
1109				 0, cpu_to_node(0), NULL);
1110}
1111
1112static void
1113tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1114{
1115	if (buf)
1116		tmc_free_etr_buf(buf);
1117}
1118
1119static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1120{
1121	struct etr_buf *etr_buf = drvdata->etr_buf;
1122
1123	if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1124		tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1125		drvdata->sysfs_buf = NULL;
1126	} else {
1127		tmc_sync_etr_buf(drvdata);
1128		/*
1129		 * Insert barrier packets at the beginning, if there was
1130		 * an overflow.
1131		 */
1132		if (etr_buf->full)
1133			tmc_etr_buf_insert_barrier_packet(etr_buf,
1134							  etr_buf->offset);
1135	}
1136}
1137
1138static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1139{
1140	CS_UNLOCK(drvdata->base);
1141
1142	tmc_flush_and_stop(drvdata);
1143	/*
1144	 * When operating in sysFS mode the content of the buffer needs to be
1145	 * read before the TMC is disabled.
1146	 */
1147	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
1148		tmc_etr_sync_sysfs_buf(drvdata);
1149
1150	tmc_disable_hw(drvdata);
1151
1152	CS_LOCK(drvdata->base);
1153
1154}
1155
1156void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1157{
1158	__tmc_etr_disable_hw(drvdata);
1159	coresight_disclaim_device(drvdata->csdev);
 
 
1160	/* Reset the ETR buf used by hardware */
1161	drvdata->etr_buf = NULL;
1162}
1163
1164static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1165{
1166	int ret = 0;
1167	unsigned long flags;
1168	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1169	struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1170
1171	/*
1172	 * If we are enabling the ETR from disabled state, we need to make
1173	 * sure we have a buffer with the right size. The etr_buf is not reset
1174	 * immediately after we stop the tracing in SYSFS mode as we wait for
1175	 * the user to collect the data. We may be able to reuse the existing
1176	 * buffer, provided the size matches. Any allocation has to be done
1177	 * with the lock released.
1178	 */
1179	spin_lock_irqsave(&drvdata->spinlock, flags);
1180	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1181	if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1182		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1183
1184		/* Allocate memory with the locks released */
1185		free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1186		if (IS_ERR(new_buf))
1187			return new_buf;
1188
1189		/* Let's try again */
1190		spin_lock_irqsave(&drvdata->spinlock, flags);
1191	}
1192
1193	if (drvdata->reading || coresight_get_mode(csdev) == CS_MODE_PERF) {
1194		ret = -EBUSY;
1195		goto out;
1196	}
1197
1198	/*
 
 
 
 
 
 
 
 
 
 
1199	 * If we don't have a buffer or it doesn't match the requested size,
1200	 * use the buffer allocated above. Otherwise reuse the existing buffer.
1201	 */
1202	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1203	if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1204		free_buf = sysfs_buf;
1205		drvdata->sysfs_buf = new_buf;
1206	}
1207
 
 
 
 
 
1208out:
1209	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1210
1211	/* Free memory outside the spinlock if need be */
1212	if (free_buf)
1213		tmc_etr_free_sysfs_buf(free_buf);
1214	return ret ? ERR_PTR(ret) : drvdata->sysfs_buf;
1215}
1216
1217static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1218{
1219	int ret = 0;
1220	unsigned long flags;
1221	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1222	struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);
1223
1224	if (IS_ERR(sysfs_buf))
1225		return PTR_ERR(sysfs_buf);
1226
1227	spin_lock_irqsave(&drvdata->spinlock, flags);
1228
1229	/*
1230	 * In sysFS mode we can have multiple writers per sink.  Since this
1231	 * sink is already enabled no memory is needed and the HW need not be
1232	 * touched, even if the buffer size has changed.
1233	 */
1234	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1235		csdev->refcnt++;
1236		goto out;
1237	}
1238
1239	ret = tmc_etr_enable_hw(drvdata, sysfs_buf);
1240	if (!ret) {
1241		coresight_set_mode(csdev, CS_MODE_SYSFS);
1242		csdev->refcnt++;
1243	}
1244
1245out:
1246	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1247
1248	if (!ret)
1249		dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1250
1251	return ret;
1252}
1253
1254struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
1255				   enum cs_mode mode, void *data)
1256{
1257	struct perf_output_handle *handle = data;
1258	struct etr_perf_buffer *etr_perf;
1259
1260	switch (mode) {
1261	case CS_MODE_SYSFS:
1262		return tmc_etr_get_sysfs_buffer(csdev);
1263	case CS_MODE_PERF:
1264		etr_perf = etm_perf_sink_config(handle);
1265		if (WARN_ON(!etr_perf || !etr_perf->etr_buf))
1266			return ERR_PTR(-EINVAL);
1267		return etr_perf->etr_buf;
1268	default:
1269		return ERR_PTR(-EINVAL);
1270	}
1271}
1272EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);
1273
1274/*
1275 * alloc_etr_buf: Allocate ETR buffer for use by perf.
1276 * The size of the hardware buffer is dependent on the size configured
1277 * via sysfs and the perf ring buffer size. We prefer to allocate the
1278 * largest possible size, scaling down the size by half until it
1279 * reaches a minimum limit (1M), beyond which we give up.
1280 */
1281static struct etr_buf *
1282alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1283	      int nr_pages, void **pages, bool snapshot)
1284{
1285	int node;
1286	struct etr_buf *etr_buf;
1287	unsigned long size;
1288
1289	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1290	/*
1291	 * Try to match the perf ring buffer size if it is larger
1292	 * than the size requested via sysfs.
1293	 */
1294	if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1295		etr_buf = tmc_alloc_etr_buf(drvdata, ((ssize_t)nr_pages << PAGE_SHIFT),
1296					    0, node, NULL);
1297		if (!IS_ERR(etr_buf))
1298			goto done;
1299	}
1300
1301	/*
1302	 * Else switch to configured size for this ETR
1303	 * and scale down until we hit the minimum limit.
1304	 */
1305	size = drvdata->size;
1306	do {
1307		etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1308		if (!IS_ERR(etr_buf))
1309			goto done;
1310		size /= 2;
1311	} while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1312
1313	return ERR_PTR(-ENOMEM);
1314
1315done:
1316	return etr_buf;
1317}
1318
1319static struct etr_buf *
1320get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1321			  struct perf_event *event, int nr_pages,
1322			  void **pages, bool snapshot)
1323{
1324	int ret;
1325	pid_t pid = task_pid_nr(event->owner);
1326	struct etr_buf *etr_buf;
1327
1328retry:
1329	/*
1330	 * An etr_perf_buffer is associated with an event and holds a reference
1331	 * to the AUX ring buffer that was created for that event.  In CPU-wide
1332	 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1333	 * buffer, share a sink.  As such an etr_perf_buffer is created for each
1334	 * event but a single etr_buf associated with the ETR is shared between
1335	 * them.  The last event in a trace session will copy the content of the
1336	 * etr_buf to its AUX ring buffer.  Ring buffer associated to other
1337	 * events are simply not used an freed as events are destoyed.  We still
1338	 * need to allocate a ring buffer for each event since we don't know
1339	 * which event will be last.
1340	 */
1341
1342	/*
1343	 * The first thing to do here is check if an etr_buf has already been
1344	 * allocated for this session.  If so it is shared with this event,
1345	 * otherwise it is created.
1346	 */
1347	mutex_lock(&drvdata->idr_mutex);
1348	etr_buf = idr_find(&drvdata->idr, pid);
1349	if (etr_buf) {
1350		refcount_inc(&etr_buf->refcount);
1351		mutex_unlock(&drvdata->idr_mutex);
1352		return etr_buf;
1353	}
1354
1355	/* If we made it here no buffer has been allocated, do so now. */
1356	mutex_unlock(&drvdata->idr_mutex);
1357
1358	etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1359	if (IS_ERR(etr_buf))
1360		return etr_buf;
1361
1362	/* Now that we have a buffer, add it to the IDR. */
1363	mutex_lock(&drvdata->idr_mutex);
1364	ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1365	mutex_unlock(&drvdata->idr_mutex);
1366
1367	/* Another event with this session ID has allocated this buffer. */
1368	if (ret == -ENOSPC) {
1369		tmc_free_etr_buf(etr_buf);
1370		goto retry;
1371	}
1372
1373	/* The IDR can't allocate room for a new session, abandon ship. */
1374	if (ret == -ENOMEM) {
1375		tmc_free_etr_buf(etr_buf);
1376		return ERR_PTR(ret);
1377	}
1378
1379
1380	return etr_buf;
1381}
1382
1383static struct etr_buf *
1384get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1385			    struct perf_event *event, int nr_pages,
1386			    void **pages, bool snapshot)
1387{
1388	/*
1389	 * In per-thread mode the etr_buf isn't shared, so just go ahead
1390	 * with memory allocation.
1391	 */
1392	return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1393}
1394
1395static struct etr_buf *
1396get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1397		 int nr_pages, void **pages, bool snapshot)
1398{
1399	if (event->cpu == -1)
1400		return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1401						   pages, snapshot);
1402
1403	return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1404					 pages, snapshot);
1405}
1406
1407static struct etr_perf_buffer *
1408tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1409		       int nr_pages, void **pages, bool snapshot)
1410{
1411	int node;
1412	struct etr_buf *etr_buf;
1413	struct etr_perf_buffer *etr_perf;
1414
1415	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1416
1417	etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1418	if (!etr_perf)
1419		return ERR_PTR(-ENOMEM);
1420
1421	etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1422	if (!IS_ERR(etr_buf))
1423		goto done;
1424
1425	kfree(etr_perf);
1426	return ERR_PTR(-ENOMEM);
1427
1428done:
1429	/*
1430	 * Keep a reference to the ETR this buffer has been allocated for
1431	 * in order to have access to the IDR in tmc_free_etr_buffer().
1432	 */
1433	etr_perf->drvdata = drvdata;
1434	etr_perf->etr_buf = etr_buf;
1435
1436	return etr_perf;
1437}
1438
1439
1440static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1441				  struct perf_event *event, void **pages,
1442				  int nr_pages, bool snapshot)
1443{
1444	struct etr_perf_buffer *etr_perf;
1445	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1446
1447	etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1448					  nr_pages, pages, snapshot);
1449	if (IS_ERR(etr_perf)) {
1450		dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1451		return NULL;
1452	}
1453
1454	etr_perf->pid = task_pid_nr(event->owner);
1455	etr_perf->snapshot = snapshot;
1456	etr_perf->nr_pages = nr_pages;
1457	etr_perf->pages = pages;
1458
1459	return etr_perf;
1460}
1461
1462static void tmc_free_etr_buffer(void *config)
1463{
1464	struct etr_perf_buffer *etr_perf = config;
1465	struct tmc_drvdata *drvdata = etr_perf->drvdata;
1466	struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1467
1468	if (!etr_buf)
1469		goto free_etr_perf_buffer;
1470
1471	mutex_lock(&drvdata->idr_mutex);
1472	/* If we are not the last one to use the buffer, don't touch it. */
1473	if (!refcount_dec_and_test(&etr_buf->refcount)) {
1474		mutex_unlock(&drvdata->idr_mutex);
1475		goto free_etr_perf_buffer;
1476	}
1477
1478	/* We are the last one, remove from the IDR and free the buffer. */
1479	buf = idr_remove(&drvdata->idr, etr_perf->pid);
1480	mutex_unlock(&drvdata->idr_mutex);
1481
1482	/*
1483	 * Something went very wrong if the buffer associated with this ID
1484	 * is not the same in the IDR.  Leak to avoid use after free.
1485	 */
1486	if (buf && WARN_ON(buf != etr_buf))
1487		goto free_etr_perf_buffer;
1488
1489	tmc_free_etr_buf(etr_perf->etr_buf);
1490
1491free_etr_perf_buffer:
1492	kfree(etr_perf);
1493}
1494
1495/*
1496 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1497 * buffer to the perf ring buffer.
1498 */
1499static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1500				     unsigned long head,
1501				     unsigned long src_offset,
1502				     unsigned long to_copy)
1503{
1504	long bytes;
1505	long pg_idx, pg_offset;
 
1506	char **dst_pages, *src_buf;
1507	struct etr_buf *etr_buf = etr_perf->etr_buf;
1508
1509	head = PERF_IDX2OFF(head, etr_perf);
1510	pg_idx = head >> PAGE_SHIFT;
1511	pg_offset = head & (PAGE_SIZE - 1);
1512	dst_pages = (char **)etr_perf->pages;
1513
1514	while (to_copy > 0) {
1515		/*
1516		 * In one iteration, we can copy minimum of :
1517		 *  1) what is available in the source buffer,
1518		 *  2) what is available in the source buffer, before it
1519		 *     wraps around.
1520		 *  3) what is available in the destination page.
1521		 * in one iteration.
1522		 */
1523		if (src_offset >= etr_buf->size)
1524			src_offset -= etr_buf->size;
1525		bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1526					     &src_buf);
1527		if (WARN_ON_ONCE(bytes <= 0))
1528			break;
1529		bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1530
1531		memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1532
1533		to_copy -= bytes;
1534
1535		/* Move destination pointers */
1536		pg_offset += bytes;
1537		if (pg_offset == PAGE_SIZE) {
1538			pg_offset = 0;
1539			if (++pg_idx == etr_perf->nr_pages)
1540				pg_idx = 0;
1541		}
1542
1543		/* Move source pointers */
1544		src_offset += bytes;
1545	}
1546}
1547
1548/*
1549 * tmc_update_etr_buffer : Update the perf ring buffer with the
1550 * available trace data. We use software double buffering at the moment.
1551 *
1552 * TODO: Add support for reusing the perf ring buffer.
1553 */
1554static unsigned long
1555tmc_update_etr_buffer(struct coresight_device *csdev,
1556		      struct perf_output_handle *handle,
1557		      void *config)
1558{
1559	bool lost = false;
1560	unsigned long flags, offset, size = 0;
1561	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1562	struct etr_perf_buffer *etr_perf = config;
1563	struct etr_buf *etr_buf = etr_perf->etr_buf;
1564
1565	spin_lock_irqsave(&drvdata->spinlock, flags);
1566
1567	/* Don't do anything if another tracer is using this sink */
1568	if (csdev->refcnt != 1) {
1569		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1570		goto out;
1571	}
1572
1573	if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1574		lost = true;
1575		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1576		goto out;
1577	}
1578
1579	CS_UNLOCK(drvdata->base);
1580
1581	tmc_flush_and_stop(drvdata);
1582	tmc_sync_etr_buf(drvdata);
1583
1584	CS_LOCK(drvdata->base);
1585	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1586
1587	lost = etr_buf->full;
1588	offset = etr_buf->offset;
1589	size = etr_buf->len;
1590
1591	/*
1592	 * The ETR buffer may be bigger than the space available in the
1593	 * perf ring buffer (handle->size).  If so advance the offset so that we
1594	 * get the latest trace data.  In snapshot mode none of that matters
1595	 * since we are expected to clobber stale data in favour of the latest
1596	 * traces.
1597	 */
1598	if (!etr_perf->snapshot && size > handle->size) {
1599		u32 mask = tmc_get_memwidth_mask(drvdata);
1600
1601		/*
1602		 * Make sure the new size is aligned in accordance with the
1603		 * requirement explained in function tmc_get_memwidth_mask().
1604		 */
1605		size = handle->size & mask;
1606		offset = etr_buf->offset + etr_buf->len - size;
1607
1608		if (offset >= etr_buf->size)
1609			offset -= etr_buf->size;
1610		lost = true;
1611	}
1612
1613	/* Insert barrier packets at the beginning, if there was an overflow */
1614	if (lost)
1615		tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1616	tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);
1617
1618	/*
1619	 * In snapshot mode we simply increment the head by the number of byte
1620	 * that were written.  User space will figure out how many bytes to get
1621	 * from the AUX buffer based on the position of the head.
 
1622	 */
1623	if (etr_perf->snapshot)
1624		handle->head += size;
1625
1626	/*
1627	 * Ensure that the AUX trace data is visible before the aux_head
1628	 * is updated via perf_aux_output_end(), as expected by the
1629	 * perf ring buffer.
1630	 */
1631	smp_wmb();
1632
1633out:
1634	/*
1635	 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1636	 * captured buffer is expected to be truncated and 2) a full buffer
1637	 * prevents the event from being re-enabled by the perf core,
1638	 * resulting in stale data being send to user space.
1639	 */
1640	if (!etr_perf->snapshot && lost)
1641		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1642	return size;
1643}
1644
1645static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1646{
1647	int rc = 0;
1648	pid_t pid;
1649	unsigned long flags;
1650	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1651	struct perf_output_handle *handle = data;
1652	struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1653
1654	spin_lock_irqsave(&drvdata->spinlock, flags);
1655	 /* Don't use this sink if it is already claimed by sysFS */
1656	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1657		rc = -EBUSY;
1658		goto unlock_out;
1659	}
1660
1661	if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1662		rc = -EINVAL;
1663		goto unlock_out;
1664	}
1665
1666	/* Get a handle on the pid of the session owner */
1667	pid = etr_perf->pid;
1668
1669	/* Do not proceed if this device is associated with another session */
1670	if (drvdata->pid != -1 && drvdata->pid != pid) {
1671		rc = -EBUSY;
1672		goto unlock_out;
1673	}
1674
 
 
1675	/*
1676	 * No HW configuration is needed if the sink is already in
1677	 * use for this session.
1678	 */
1679	if (drvdata->pid == pid) {
1680		csdev->refcnt++;
1681		goto unlock_out;
1682	}
1683
1684	rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1685	if (!rc) {
1686		/* Associate with monitored process. */
1687		drvdata->pid = pid;
1688		coresight_set_mode(csdev, CS_MODE_PERF);
1689		drvdata->perf_buf = etr_perf->etr_buf;
1690		csdev->refcnt++;
1691	}
1692
1693unlock_out:
1694	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1695	return rc;
1696}
1697
1698static int tmc_enable_etr_sink(struct coresight_device *csdev,
1699			       enum cs_mode mode, void *data)
1700{
1701	switch (mode) {
1702	case CS_MODE_SYSFS:
1703		return tmc_enable_etr_sink_sysfs(csdev);
1704	case CS_MODE_PERF:
1705		return tmc_enable_etr_sink_perf(csdev, data);
1706	default:
1707		return -EINVAL;
1708	}
 
 
 
1709}
1710
1711static int tmc_disable_etr_sink(struct coresight_device *csdev)
1712{
1713	unsigned long flags;
1714	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1715
1716	spin_lock_irqsave(&drvdata->spinlock, flags);
1717
1718	if (drvdata->reading) {
1719		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1720		return -EBUSY;
1721	}
1722
1723	csdev->refcnt--;
1724	if (csdev->refcnt) {
1725		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1726		return -EBUSY;
1727	}
1728
1729	/* Complain if we (somehow) got out of sync */
1730	WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);
1731	tmc_etr_disable_hw(drvdata);
1732	/* Dissociate from monitored process. */
1733	drvdata->pid = -1;
1734	coresight_set_mode(csdev, CS_MODE_DISABLED);
1735	/* Reset perf specific data */
1736	drvdata->perf_buf = NULL;
1737
1738	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1739
1740	dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1741	return 0;
1742}
1743
1744static const struct coresight_ops_sink tmc_etr_sink_ops = {
1745	.enable		= tmc_enable_etr_sink,
1746	.disable	= tmc_disable_etr_sink,
1747	.alloc_buffer	= tmc_alloc_etr_buffer,
1748	.update_buffer	= tmc_update_etr_buffer,
1749	.free_buffer	= tmc_free_etr_buffer,
1750};
1751
1752const struct coresight_ops tmc_etr_cs_ops = {
1753	.sink_ops	= &tmc_etr_sink_ops,
1754};
1755
1756int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1757{
1758	int ret = 0;
1759	unsigned long flags;
1760
1761	/* config types are set a boot time and never change */
1762	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1763		return -EINVAL;
1764
1765	spin_lock_irqsave(&drvdata->spinlock, flags);
1766	if (drvdata->reading) {
1767		ret = -EBUSY;
1768		goto out;
1769	}
1770
1771	/*
1772	 * We can safely allow reads even if the ETR is operating in PERF mode,
1773	 * since the sysfs session is captured in mode specific data.
1774	 * If drvdata::sysfs_data is NULL the trace data has been read already.
1775	 */
1776	if (!drvdata->sysfs_buf) {
1777		ret = -EINVAL;
1778		goto out;
1779	}
1780
1781	/* Disable the TMC if we are trying to read from a running session. */
1782	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
1783		__tmc_etr_disable_hw(drvdata);
1784
1785	drvdata->reading = true;
1786out:
1787	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1788
1789	return ret;
1790}
1791
1792int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1793{
1794	unsigned long flags;
1795	struct etr_buf *sysfs_buf = NULL;
1796
1797	/* config types are set a boot time and never change */
1798	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1799		return -EINVAL;
1800
1801	spin_lock_irqsave(&drvdata->spinlock, flags);
1802
1803	/* RE-enable the TMC if need be */
1804	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
1805		/*
1806		 * The trace run will continue with the same allocated trace
1807		 * buffer. Since the tracer is still enabled drvdata::buf can't
1808		 * be NULL.
1809		 */
1810		__tmc_etr_enable_hw(drvdata);
1811	} else {
1812		/*
1813		 * The ETR is not tracing and the buffer was just read.
1814		 * As such prepare to free the trace buffer.
1815		 */
1816		sysfs_buf = drvdata->sysfs_buf;
1817		drvdata->sysfs_buf = NULL;
1818	}
1819
1820	drvdata->reading = false;
1821	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1822
1823	/* Free allocated memory out side of the spinlock */
1824	if (sysfs_buf)
1825		tmc_etr_free_sysfs_buf(sysfs_buf);
1826
1827	return 0;
1828}
1829
1830static const char *const buf_modes_str[] = {
1831	[ETR_MODE_FLAT]		= "flat",
1832	[ETR_MODE_ETR_SG]	= "tmc-sg",
1833	[ETR_MODE_CATU]		= "catu",
1834	[ETR_MODE_AUTO]		= "auto",
1835};
1836
1837static ssize_t buf_modes_available_show(struct device *dev,
1838					    struct device_attribute *attr, char *buf)
1839{
1840	struct etr_buf_hw buf_hw;
1841	ssize_t size = 0;
1842
1843	get_etr_buf_hw(dev, &buf_hw);
1844	size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]);
1845	size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]);
1846	if (buf_hw.has_etr_sg)
1847		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]);
1848
1849	if (buf_hw.has_catu)
1850		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]);
1851
1852	size += sysfs_emit_at(buf, size, "\n");
1853	return size;
1854}
1855static DEVICE_ATTR_RO(buf_modes_available);
1856
1857static ssize_t buf_mode_preferred_show(struct device *dev,
1858					 struct device_attribute *attr, char *buf)
1859{
1860	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
1861
1862	return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]);
1863}
1864
1865static ssize_t buf_mode_preferred_store(struct device *dev,
1866					  struct device_attribute *attr,
1867					  const char *buf, size_t size)
1868{
1869	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
1870	struct etr_buf_hw buf_hw;
1871
1872	get_etr_buf_hw(dev, &buf_hw);
1873	if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT]))
1874		drvdata->etr_mode = ETR_MODE_FLAT;
1875	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)
1876		drvdata->etr_mode = ETR_MODE_ETR_SG;
1877	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)
1878		drvdata->etr_mode = ETR_MODE_CATU;
1879	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO]))
1880		drvdata->etr_mode = ETR_MODE_AUTO;
1881	else
1882		return -EINVAL;
1883	return size;
1884}
1885static DEVICE_ATTR_RW(buf_mode_preferred);
1886
1887static struct attribute *coresight_etr_attrs[] = {
1888	&dev_attr_buf_modes_available.attr,
1889	&dev_attr_buf_mode_preferred.attr,
1890	NULL,
1891};
1892
1893const struct attribute_group coresight_etr_group = {
1894	.attrs = coresight_etr_attrs,
1895};