1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright(C) 2016 Linaro Limited. All rights reserved.
4  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5  */
6 
7 #include <linux/atomic.h>
8 #include <linux/coresight.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/iommu.h>
11 #include <linux/idr.h>
12 #include <linux/mutex.h>
13 #include <linux/refcount.h>
14 #include <linux/slab.h>
15 #include <linux/types.h>
16 #include <linux/vmalloc.h>
17 #include "coresight-catu.h"
18 #include "coresight-etm-perf.h"
19 #include "coresight-priv.h"
20 #include "coresight-tmc.h"
21 
22 struct etr_flat_buf {
23 	struct device	*dev;
24 	dma_addr_t	daddr;
25 	void		*vaddr;
26 	size_t		size;
27 };
28 
29 /*
30  * etr_perf_buffer - Perf buffer used for ETR
31  * @drvdata		- The ETR drvdaga this buffer has been allocated for.
32  * @etr_buf		- Actual buffer used by the ETR
33  * @pid			- The PID this etr_perf_buffer belongs to.
34  * @snaphost		- Perf session mode
35  * @nr_pages		- Number of pages in the ring buffer.
36  * @pages		- Array of Pages in the ring buffer.
37  */
38 struct etr_perf_buffer {
39 	struct tmc_drvdata	*drvdata;
40 	struct etr_buf		*etr_buf;
41 	pid_t			pid;
42 	bool			snapshot;
43 	int			nr_pages;
44 	void			**pages;
45 };
46 
47 /* Convert the perf index to an offset within the ETR buffer */
48 #define PERF_IDX2OFF(idx, buf)		\
49 		((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
50 
51 /* Lower limit for ETR hardware buffer */
52 #define TMC_ETR_PERF_MIN_BUF_SIZE	SZ_1M
53 
54 /*
55  * The TMC ETR SG has a page size of 4K. The SG table contains pointers
56  * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
57  * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
58  * contain more than one SG buffer and tables.
59  *
60  * A table entry has the following format:
61  *
62  * ---Bit31------------Bit4-------Bit1-----Bit0--
63  * |     Address[39:12]    | SBZ |  Entry Type  |
64  * ----------------------------------------------
65  *
66  * Address: Bits [39:12] of a physical page address. Bits [11:0] are
67  *	    always zero.
68  *
69  * Entry type:
70  *	b00 - Reserved.
71  *	b01 - Last entry in the tables, points to 4K page buffer.
72  *	b10 - Normal entry, points to 4K page buffer.
73  *	b11 - Link. The address points to the base of next table.
74  */
75 
76 typedef u32 sgte_t;
77 
78 #define ETR_SG_PAGE_SHIFT		12
79 #define ETR_SG_PAGE_SIZE		(1UL << ETR_SG_PAGE_SHIFT)
80 #define ETR_SG_PAGES_PER_SYSPAGE	(PAGE_SIZE / ETR_SG_PAGE_SIZE)
81 #define ETR_SG_PTRS_PER_PAGE		(ETR_SG_PAGE_SIZE / sizeof(sgte_t))
82 #define ETR_SG_PTRS_PER_SYSPAGE		(PAGE_SIZE / sizeof(sgte_t))
83 
84 #define ETR_SG_ET_MASK			0x3
85 #define ETR_SG_ET_LAST			0x1
86 #define ETR_SG_ET_NORMAL		0x2
87 #define ETR_SG_ET_LINK			0x3
88 
89 #define ETR_SG_ADDR_SHIFT		4
90 
91 #define ETR_SG_ENTRY(addr, type) \
92 	(sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
93 		 (type & ETR_SG_ET_MASK))
94 
95 #define ETR_SG_ADDR(entry) \
96 	(((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
97 #define ETR_SG_ET(entry)		((entry) & ETR_SG_ET_MASK)
98 
99 /*
100  * struct etr_sg_table : ETR SG Table
101  * @sg_table:		Generic SG Table holding the data/table pages.
102  * @hwaddr:		hwaddress used by the TMC, which is the base
103  *			address of the table.
104  */
105 struct etr_sg_table {
106 	struct tmc_sg_table	*sg_table;
107 	dma_addr_t		hwaddr;
108 };
109 
110 /*
111  * tmc_etr_sg_table_entries: Total number of table entries required to map
112  * @nr_pages system pages.
113  *
114  * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
115  * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
116  * with the last entry pointing to another page of table entries.
117  * If we spill over to a new page for mapping 1 entry, we could as
118  * well replace the link entry of the previous page with the last entry.
119  */
120 static inline unsigned long __attribute_const__
121 tmc_etr_sg_table_entries(int nr_pages)
122 {
123 	unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
124 	unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
125 	/*
126 	 * If we spill over to a new page for 1 entry, we could as well
127 	 * make it the LAST entry in the previous page, skipping the Link
128 	 * address.
129 	 */
130 	if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
131 		nr_sglinks--;
132 	return nr_sgpages + nr_sglinks;
133 }
134 
135 /*
136  * tmc_pages_get_offset:  Go through all the pages in the tmc_pages
137  * and map the device address @addr to an offset within the virtual
138  * contiguous buffer.
139  */
140 static long
141 tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
142 {
143 	int i;
144 	dma_addr_t page_start;
145 
146 	for (i = 0; i < tmc_pages->nr_pages; i++) {
147 		page_start = tmc_pages->daddrs[i];
148 		if (addr >= page_start && addr < (page_start + PAGE_SIZE))
149 			return i * PAGE_SIZE + (addr - page_start);
150 	}
151 
152 	return -EINVAL;
153 }
154 
155 /*
156  * tmc_pages_free : Unmap and free the pages used by tmc_pages.
157  * If the pages were not allocated in tmc_pages_alloc(), we would
158  * simply drop the refcount.
159  */
160 static void tmc_pages_free(struct tmc_pages *tmc_pages,
161 			   struct device *dev, enum dma_data_direction dir)
162 {
163 	int i;
164 	struct device *real_dev = dev->parent;
165 
166 	for (i = 0; i < tmc_pages->nr_pages; i++) {
167 		if (tmc_pages->daddrs && tmc_pages->daddrs[i])
168 			dma_unmap_page(real_dev, tmc_pages->daddrs[i],
169 					 PAGE_SIZE, dir);
170 		if (tmc_pages->pages && tmc_pages->pages[i])
171 			__free_page(tmc_pages->pages[i]);
172 	}
173 
174 	kfree(tmc_pages->pages);
175 	kfree(tmc_pages->daddrs);
176 	tmc_pages->pages = NULL;
177 	tmc_pages->daddrs = NULL;
178 	tmc_pages->nr_pages = 0;
179 }
180 
181 /*
182  * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
183  * If @pages is not NULL, the list of page virtual addresses are
184  * used as the data pages. The pages are then dma_map'ed for @dev
185  * with dma_direction @dir.
186  *
187  * Returns 0 upon success, else the error number.
188  */
189 static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
190 			   struct device *dev, int node,
191 			   enum dma_data_direction dir, void **pages)
192 {
193 	int i, nr_pages;
194 	dma_addr_t paddr;
195 	struct page *page;
196 	struct device *real_dev = dev->parent;
197 
198 	nr_pages = tmc_pages->nr_pages;
199 	tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
200 					 GFP_KERNEL);
201 	if (!tmc_pages->daddrs)
202 		return -ENOMEM;
203 	tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
204 					 GFP_KERNEL);
205 	if (!tmc_pages->pages) {
206 		kfree(tmc_pages->daddrs);
207 		tmc_pages->daddrs = NULL;
208 		return -ENOMEM;
209 	}
210 
211 	for (i = 0; i < nr_pages; i++) {
212 		if (pages && pages[i]) {
213 			page = virt_to_page(pages[i]);
214 			/* Hold a refcount on the page */
215 			get_page(page);
216 		} else {
217 			page = alloc_pages_node(node,
218 						GFP_KERNEL | __GFP_ZERO, 0);
219 			if (!page)
220 				goto err;
221 		}
222 		paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
223 		if (dma_mapping_error(real_dev, paddr))
224 			goto err;
225 		tmc_pages->daddrs[i] = paddr;
226 		tmc_pages->pages[i] = page;
227 	}
228 	return 0;
229 err:
230 	tmc_pages_free(tmc_pages, dev, dir);
231 	return -ENOMEM;
232 }
233 
234 static inline long
235 tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
236 {
237 	return tmc_pages_get_offset(&sg_table->data_pages, addr);
238 }
239 
240 static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
241 {
242 	if (sg_table->table_vaddr)
243 		vunmap(sg_table->table_vaddr);
244 	tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
245 }
246 
247 static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
248 {
249 	if (sg_table->data_vaddr)
250 		vunmap(sg_table->data_vaddr);
251 	tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
252 }
253 
254 void tmc_free_sg_table(struct tmc_sg_table *sg_table)
255 {
256 	tmc_free_table_pages(sg_table);
257 	tmc_free_data_pages(sg_table);
258 }
259 EXPORT_SYMBOL_GPL(tmc_free_sg_table);
260 
261 /*
262  * Alloc pages for the table. Since this will be used by the device,
263  * allocate the pages closer to the device (i.e, dev_to_node(dev)
264  * rather than the CPU node).
265  */
266 static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
267 {
268 	int rc;
269 	struct tmc_pages *table_pages = &sg_table->table_pages;
270 
271 	rc = tmc_pages_alloc(table_pages, sg_table->dev,
272 			     dev_to_node(sg_table->dev),
273 			     DMA_TO_DEVICE, NULL);
274 	if (rc)
275 		return rc;
276 	sg_table->table_vaddr = vmap(table_pages->pages,
277 				     table_pages->nr_pages,
278 				     VM_MAP,
279 				     PAGE_KERNEL);
280 	if (!sg_table->table_vaddr)
281 		rc = -ENOMEM;
282 	else
283 		sg_table->table_daddr = table_pages->daddrs[0];
284 	return rc;
285 }
286 
287 static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
288 {
289 	int rc;
290 
291 	/* Allocate data pages on the node requested by the caller */
292 	rc = tmc_pages_alloc(&sg_table->data_pages,
293 			     sg_table->dev, sg_table->node,
294 			     DMA_FROM_DEVICE, pages);
295 	if (!rc) {
296 		sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
297 					    sg_table->data_pages.nr_pages,
298 					    VM_MAP,
299 					    PAGE_KERNEL);
300 		if (!sg_table->data_vaddr)
301 			rc = -ENOMEM;
302 	}
303 	return rc;
304 }
305 
306 /*
307  * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
308  * and data buffers. TMC writes to the data buffers and reads from the SG
309  * Table pages.
310  *
311  * @dev		- Coresight device to which page should be DMA mapped.
312  * @node	- Numa node for mem allocations
313  * @nr_tpages	- Number of pages for the table entries.
314  * @nr_dpages	- Number of pages for Data buffer.
315  * @pages	- Optional list of virtual address of pages.
316  */
317 struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
318 					int node,
319 					int nr_tpages,
320 					int nr_dpages,
321 					void **pages)
322 {
323 	long rc;
324 	struct tmc_sg_table *sg_table;
325 
326 	sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
327 	if (!sg_table)
328 		return ERR_PTR(-ENOMEM);
329 	sg_table->data_pages.nr_pages = nr_dpages;
330 	sg_table->table_pages.nr_pages = nr_tpages;
331 	sg_table->node = node;
332 	sg_table->dev = dev;
333 
334 	rc  = tmc_alloc_data_pages(sg_table, pages);
335 	if (!rc)
336 		rc = tmc_alloc_table_pages(sg_table);
337 	if (rc) {
338 		tmc_free_sg_table(sg_table);
339 		kfree(sg_table);
340 		return ERR_PTR(rc);
341 	}
342 
343 	return sg_table;
344 }
345 EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
346 
347 /*
348  * tmc_sg_table_sync_data_range: Sync the data buffer written
349  * by the device from @offset upto a @size bytes.
350  */
351 void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
352 				  u64 offset, u64 size)
353 {
354 	int i, index, start;
355 	int npages = DIV_ROUND_UP(size, PAGE_SIZE);
356 	struct device *real_dev = table->dev->parent;
357 	struct tmc_pages *data = &table->data_pages;
358 
359 	start = offset >> PAGE_SHIFT;
360 	for (i = start; i < (start + npages); i++) {
361 		index = i % data->nr_pages;
362 		dma_sync_single_for_cpu(real_dev, data->daddrs[index],
363 					PAGE_SIZE, DMA_FROM_DEVICE);
364 	}
365 }
366 EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
367 
368 /* tmc_sg_sync_table: Sync the page table */
369 void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
370 {
371 	int i;
372 	struct device *real_dev = sg_table->dev->parent;
373 	struct tmc_pages *table_pages = &sg_table->table_pages;
374 
375 	for (i = 0; i < table_pages->nr_pages; i++)
376 		dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
377 					   PAGE_SIZE, DMA_TO_DEVICE);
378 }
379 EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
380 
381 /*
382  * tmc_sg_table_get_data: Get the buffer pointer for data @offset
383  * in the SG buffer. The @bufpp is updated to point to the buffer.
384  * Returns :
385  *	the length of linear data available at @offset.
386  *	or
387  *	<= 0 if no data is available.
388  */
389 ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
390 			      u64 offset, size_t len, char **bufpp)
391 {
392 	size_t size;
393 	int pg_idx = offset >> PAGE_SHIFT;
394 	int pg_offset = offset & (PAGE_SIZE - 1);
395 	struct tmc_pages *data_pages = &sg_table->data_pages;
396 
397 	size = tmc_sg_table_buf_size(sg_table);
398 	if (offset >= size)
399 		return -EINVAL;
400 
401 	/* Make sure we don't go beyond the end */
402 	len = (len < (size - offset)) ? len : size - offset;
403 	/* Respect the page boundaries */
404 	len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
405 	if (len > 0)
406 		*bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
407 	return len;
408 }
409 EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
410 
411 #ifdef ETR_SG_DEBUG
412 /* Map a dma address to virtual address */
413 static unsigned long
414 tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
415 		      dma_addr_t addr, bool table)
416 {
417 	long offset;
418 	unsigned long base;
419 	struct tmc_pages *tmc_pages;
420 
421 	if (table) {
422 		tmc_pages = &sg_table->table_pages;
423 		base = (unsigned long)sg_table->table_vaddr;
424 	} else {
425 		tmc_pages = &sg_table->data_pages;
426 		base = (unsigned long)sg_table->data_vaddr;
427 	}
428 
429 	offset = tmc_pages_get_offset(tmc_pages, addr);
430 	if (offset < 0)
431 		return 0;
432 	return base + offset;
433 }
434 
435 /* Dump the given sg_table */
436 static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
437 {
438 	sgte_t *ptr;
439 	int i = 0;
440 	dma_addr_t addr;
441 	struct tmc_sg_table *sg_table = etr_table->sg_table;
442 
443 	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
444 					      etr_table->hwaddr, true);
445 	while (ptr) {
446 		addr = ETR_SG_ADDR(*ptr);
447 		switch (ETR_SG_ET(*ptr)) {
448 		case ETR_SG_ET_NORMAL:
449 			dev_dbg(sg_table->dev,
450 				"%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
451 			ptr++;
452 			break;
453 		case ETR_SG_ET_LINK:
454 			dev_dbg(sg_table->dev,
455 				"%05d: *** %p\t:{L} 0x%llx ***\n",
456 				 i, ptr, addr);
457 			ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
458 							      addr, true);
459 			break;
460 		case ETR_SG_ET_LAST:
461 			dev_dbg(sg_table->dev,
462 				"%05d: ### %p\t:[L] 0x%llx ###\n",
463 				 i, ptr, addr);
464 			return;
465 		default:
466 			dev_dbg(sg_table->dev,
467 				"%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
468 				 i, ptr, addr);
469 			return;
470 		}
471 		i++;
472 	}
473 	dev_dbg(sg_table->dev, "******* End of Table *****\n");
474 }
475 #else
476 static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
477 #endif
478 
479 /*
480  * Populate the SG Table page table entries from table/data
481  * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
482  * So does a Table page. So we keep track of indices of the tables
483  * in each system page and move the pointers accordingly.
484  */
485 #define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
486 static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
487 {
488 	dma_addr_t paddr;
489 	int i, type, nr_entries;
490 	int tpidx = 0; /* index to the current system table_page */
491 	int sgtidx = 0;	/* index to the sg_table within the current syspage */
492 	int sgtentry = 0; /* the entry within the sg_table */
493 	int dpidx = 0; /* index to the current system data_page */
494 	int spidx = 0; /* index to the SG page within the current data page */
495 	sgte_t *ptr; /* pointer to the table entry to fill */
496 	struct tmc_sg_table *sg_table = etr_table->sg_table;
497 	dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
498 	dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
499 
500 	nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
501 	/*
502 	 * Use the contiguous virtual address of the table to update entries.
503 	 */
504 	ptr = sg_table->table_vaddr;
505 	/*
506 	 * Fill all the entries, except the last entry to avoid special
507 	 * checks within the loop.
508 	 */
509 	for (i = 0; i < nr_entries - 1; i++) {
510 		if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
511 			/*
512 			 * Last entry in a sg_table page is a link address to
513 			 * the next table page. If this sg_table is the last
514 			 * one in the system page, it links to the first
515 			 * sg_table in the next system page. Otherwise, it
516 			 * links to the next sg_table page within the system
517 			 * page.
518 			 */
519 			if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
520 				paddr = table_daddrs[tpidx + 1];
521 			} else {
522 				paddr = table_daddrs[tpidx] +
523 					(ETR_SG_PAGE_SIZE * (sgtidx + 1));
524 			}
525 			type = ETR_SG_ET_LINK;
526 		} else {
527 			/*
528 			 * Update the indices to the data_pages to point to the
529 			 * next sg_page in the data buffer.
530 			 */
531 			type = ETR_SG_ET_NORMAL;
532 			paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
533 			if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
534 				dpidx++;
535 		}
536 		*ptr++ = ETR_SG_ENTRY(paddr, type);
537 		/*
538 		 * Move to the next table pointer, moving the table page index
539 		 * if necessary
540 		 */
541 		if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
542 			if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
543 				tpidx++;
544 		}
545 	}
546 
547 	/* Set up the last entry, which is always a data pointer */
548 	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
549 	*ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
550 }
551 
552 /*
553  * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
554  * populate the table.
555  *
556  * @dev		- Device pointer for the TMC
557  * @node	- NUMA node where the memory should be allocated
558  * @size	- Total size of the data buffer
559  * @pages	- Optional list of page virtual address
560  */
561 static struct etr_sg_table *
562 tmc_init_etr_sg_table(struct device *dev, int node,
563 		      unsigned long size, void **pages)
564 {
565 	int nr_entries, nr_tpages;
566 	int nr_dpages = size >> PAGE_SHIFT;
567 	struct tmc_sg_table *sg_table;
568 	struct etr_sg_table *etr_table;
569 
570 	etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
571 	if (!etr_table)
572 		return ERR_PTR(-ENOMEM);
573 	nr_entries = tmc_etr_sg_table_entries(nr_dpages);
574 	nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
575 
576 	sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
577 	if (IS_ERR(sg_table)) {
578 		kfree(etr_table);
579 		return ERR_CAST(sg_table);
580 	}
581 
582 	etr_table->sg_table = sg_table;
583 	/* TMC should use table base address for DBA */
584 	etr_table->hwaddr = sg_table->table_daddr;
585 	tmc_etr_sg_table_populate(etr_table);
586 	/* Sync the table pages for the HW */
587 	tmc_sg_table_sync_table(sg_table);
588 	tmc_etr_sg_table_dump(etr_table);
589 
590 	return etr_table;
591 }
592 
593 /*
594  * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
595  */
596 static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
597 				  struct etr_buf *etr_buf, int node,
598 				  void **pages)
599 {
600 	struct etr_flat_buf *flat_buf;
601 	struct device *real_dev = drvdata->csdev->dev.parent;
602 
603 	/* We cannot reuse existing pages for flat buf */
604 	if (pages)
605 		return -EINVAL;
606 
607 	flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
608 	if (!flat_buf)
609 		return -ENOMEM;
610 
611 	flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,
612 						&flat_buf->daddr,
613 						DMA_FROM_DEVICE,
614 						GFP_KERNEL | __GFP_NOWARN);
615 	if (!flat_buf->vaddr) {
616 		kfree(flat_buf);
617 		return -ENOMEM;
618 	}
619 
620 	flat_buf->size = etr_buf->size;
621 	flat_buf->dev = &drvdata->csdev->dev;
622 	etr_buf->hwaddr = flat_buf->daddr;
623 	etr_buf->mode = ETR_MODE_FLAT;
624 	etr_buf->private = flat_buf;
625 	return 0;
626 }
627 
628 static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
629 {
630 	struct etr_flat_buf *flat_buf = etr_buf->private;
631 
632 	if (flat_buf && flat_buf->daddr) {
633 		struct device *real_dev = flat_buf->dev->parent;
634 
635 		dma_free_noncoherent(real_dev, etr_buf->size,
636 				     flat_buf->vaddr, flat_buf->daddr,
637 				     DMA_FROM_DEVICE);
638 	}
639 	kfree(flat_buf);
640 }
641 
642 static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
643 {
644 	struct etr_flat_buf *flat_buf = etr_buf->private;
645 	struct device *real_dev = flat_buf->dev->parent;
646 
647 	/*
648 	 * Adjust the buffer to point to the beginning of the trace data
649 	 * and update the available trace data.
650 	 */
651 	etr_buf->offset = rrp - etr_buf->hwaddr;
652 	if (etr_buf->full)
653 		etr_buf->len = etr_buf->size;
654 	else
655 		etr_buf->len = rwp - rrp;
656 
657 	/*
658 	 * The driver always starts tracing at the beginning of the buffer,
659 	 * the only reason why we would get a wrap around is when the buffer
660 	 * is full.  Sync the entire buffer in one go for this case.
661 	 */
662 	if (etr_buf->offset + etr_buf->len > etr_buf->size)
663 		dma_sync_single_for_cpu(real_dev, flat_buf->daddr,
664 					etr_buf->size, DMA_FROM_DEVICE);
665 	else
666 		dma_sync_single_for_cpu(real_dev,
667 					flat_buf->daddr + etr_buf->offset,
668 					etr_buf->len, DMA_FROM_DEVICE);
669 }
670 
671 static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
672 					 u64 offset, size_t len, char **bufpp)
673 {
674 	struct etr_flat_buf *flat_buf = etr_buf->private;
675 
676 	*bufpp = (char *)flat_buf->vaddr + offset;
677 	/*
678 	 * tmc_etr_buf_get_data already adjusts the length to handle
679 	 * buffer wrapping around.
680 	 */
681 	return len;
682 }
683 
684 static const struct etr_buf_operations etr_flat_buf_ops = {
685 	.alloc = tmc_etr_alloc_flat_buf,
686 	.free = tmc_etr_free_flat_buf,
687 	.sync = tmc_etr_sync_flat_buf,
688 	.get_data = tmc_etr_get_data_flat_buf,
689 };
690 
691 /*
692  * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
693  * appropriately.
694  */
695 static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
696 				struct etr_buf *etr_buf, int node,
697 				void **pages)
698 {
699 	struct etr_sg_table *etr_table;
700 	struct device *dev = &drvdata->csdev->dev;
701 
702 	etr_table = tmc_init_etr_sg_table(dev, node,
703 					  etr_buf->size, pages);
704 	if (IS_ERR(etr_table))
705 		return -ENOMEM;
706 	etr_buf->hwaddr = etr_table->hwaddr;
707 	etr_buf->mode = ETR_MODE_ETR_SG;
708 	etr_buf->private = etr_table;
709 	return 0;
710 }
711 
712 static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
713 {
714 	struct etr_sg_table *etr_table = etr_buf->private;
715 
716 	if (etr_table) {
717 		tmc_free_sg_table(etr_table->sg_table);
718 		kfree(etr_table);
719 	}
720 }
721 
722 static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
723 				       size_t len, char **bufpp)
724 {
725 	struct etr_sg_table *etr_table = etr_buf->private;
726 
727 	return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
728 }
729 
730 static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
731 {
732 	long r_offset, w_offset;
733 	struct etr_sg_table *etr_table = etr_buf->private;
734 	struct tmc_sg_table *table = etr_table->sg_table;
735 
736 	/* Convert hw address to offset in the buffer */
737 	r_offset = tmc_sg_get_data_page_offset(table, rrp);
738 	if (r_offset < 0) {
739 		dev_warn(table->dev,
740 			 "Unable to map RRP %llx to offset\n", rrp);
741 		etr_buf->len = 0;
742 		return;
743 	}
744 
745 	w_offset = tmc_sg_get_data_page_offset(table, rwp);
746 	if (w_offset < 0) {
747 		dev_warn(table->dev,
748 			 "Unable to map RWP %llx to offset\n", rwp);
749 		etr_buf->len = 0;
750 		return;
751 	}
752 
753 	etr_buf->offset = r_offset;
754 	if (etr_buf->full)
755 		etr_buf->len = etr_buf->size;
756 	else
757 		etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
758 				w_offset - r_offset;
759 	tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
760 }
761 
762 static const struct etr_buf_operations etr_sg_buf_ops = {
763 	.alloc = tmc_etr_alloc_sg_buf,
764 	.free = tmc_etr_free_sg_buf,
765 	.sync = tmc_etr_sync_sg_buf,
766 	.get_data = tmc_etr_get_data_sg_buf,
767 };
768 
769 /*
770  * TMC ETR could be connected to a CATU device, which can provide address
771  * translation service. This is represented by the Output port of the TMC
772  * (ETR) connected to the input port of the CATU.
773  *
774  * Returns	: coresight_device ptr for the CATU device if a CATU is found.
775  *		: NULL otherwise.
776  */
777 struct coresight_device *
778 tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
779 {
780 	struct coresight_device *etr = drvdata->csdev;
781 	union coresight_dev_subtype catu_subtype = {
782 		.helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU
783 	};
784 
785 	if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
786 		return NULL;
787 
788 	return coresight_find_output_type(etr->pdata, CORESIGHT_DEV_TYPE_HELPER,
789 					  catu_subtype);
790 }
791 EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
792 
793 static const struct etr_buf_operations *etr_buf_ops[] = {
794 	[ETR_MODE_FLAT] = &etr_flat_buf_ops,
795 	[ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
796 	[ETR_MODE_CATU] = NULL,
797 };
798 
799 void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
800 {
801 	etr_buf_ops[ETR_MODE_CATU] = catu;
802 }
803 EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
804 
805 void tmc_etr_remove_catu_ops(void)
806 {
807 	etr_buf_ops[ETR_MODE_CATU] = NULL;
808 }
809 EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
810 
811 static inline int tmc_etr_mode_alloc_buf(int mode,
812 					 struct tmc_drvdata *drvdata,
813 					 struct etr_buf *etr_buf, int node,
814 					 void **pages)
815 {
816 	int rc = -EINVAL;
817 
818 	switch (mode) {
819 	case ETR_MODE_FLAT:
820 	case ETR_MODE_ETR_SG:
821 	case ETR_MODE_CATU:
822 		if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
823 			rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
824 						      node, pages);
825 		if (!rc)
826 			etr_buf->ops = etr_buf_ops[mode];
827 		return rc;
828 	default:
829 		return -EINVAL;
830 	}
831 }
832 
833 /*
834  * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
835  * @drvdata	: ETR device details.
836  * @size	: size of the requested buffer.
837  * @flags	: Required properties for the buffer.
838  * @node	: Node for memory allocations.
839  * @pages	: An optional list of pages.
840  */
841 static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
842 					 ssize_t size, int flags,
843 					 int node, void **pages)
844 {
845 	int rc = -ENOMEM;
846 	bool has_etr_sg, has_iommu;
847 	bool has_sg, has_catu;
848 	struct etr_buf *etr_buf;
849 	struct device *dev = &drvdata->csdev->dev;
850 
851 	has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
852 	has_iommu = iommu_get_domain_for_dev(dev->parent);
853 	has_catu = !!tmc_etr_get_catu_device(drvdata);
854 
855 	has_sg = has_catu || has_etr_sg;
856 
857 	etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
858 	if (!etr_buf)
859 		return ERR_PTR(-ENOMEM);
860 
861 	etr_buf->size = size;
862 
863 	/*
864 	 * If we have to use an existing list of pages, we cannot reliably
865 	 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
866 	 * we use the contiguous DMA memory if at least one of the following
867 	 * conditions is true:
868 	 *  a) The ETR cannot use Scatter-Gather.
869 	 *  b) we have a backing IOMMU
870 	 *  c) The requested memory size is smaller (< 1M).
871 	 *
872 	 * Fallback to available mechanisms.
873 	 *
874 	 */
875 	if (!pages &&
876 	    (!has_sg || has_iommu || size < SZ_1M))
877 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
878 					    etr_buf, node, pages);
879 	if (rc && has_etr_sg)
880 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
881 					    etr_buf, node, pages);
882 	if (rc && has_catu)
883 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
884 					    etr_buf, node, pages);
885 	if (rc) {
886 		kfree(etr_buf);
887 		return ERR_PTR(rc);
888 	}
889 
890 	refcount_set(&etr_buf->refcount, 1);
891 	dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
892 		(unsigned long)size >> 10, etr_buf->mode);
893 	return etr_buf;
894 }
895 
896 static void tmc_free_etr_buf(struct etr_buf *etr_buf)
897 {
898 	WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
899 	etr_buf->ops->free(etr_buf);
900 	kfree(etr_buf);
901 }
902 
903 /*
904  * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
905  * with a maximum of @len bytes.
906  * Returns: The size of the linear data available @pos, with *bufpp
907  * updated to point to the buffer.
908  */
909 static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
910 				    u64 offset, size_t len, char **bufpp)
911 {
912 	/* Adjust the length to limit this transaction to end of buffer */
913 	len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
914 
915 	return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
916 }
917 
918 static inline s64
919 tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
920 {
921 	ssize_t len;
922 	char *bufp;
923 
924 	len = tmc_etr_buf_get_data(etr_buf, offset,
925 				   CORESIGHT_BARRIER_PKT_SIZE, &bufp);
926 	if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))
927 		return -EINVAL;
928 	coresight_insert_barrier_packet(bufp);
929 	return offset + CORESIGHT_BARRIER_PKT_SIZE;
930 }
931 
932 /*
933  * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
934  * Makes sure the trace data is synced to the memory for consumption.
935  * @etr_buf->offset will hold the offset to the beginning of the trace data
936  * within the buffer, with @etr_buf->len bytes to consume.
937  */
938 static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
939 {
940 	struct etr_buf *etr_buf = drvdata->etr_buf;
941 	u64 rrp, rwp;
942 	u32 status;
943 
944 	rrp = tmc_read_rrp(drvdata);
945 	rwp = tmc_read_rwp(drvdata);
946 	status = readl_relaxed(drvdata->base + TMC_STS);
947 
948 	/*
949 	 * If there were memory errors in the session, truncate the
950 	 * buffer.
951 	 */
952 	if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
953 		dev_dbg(&drvdata->csdev->dev,
954 			"tmc memory error detected, truncating buffer\n");
955 		etr_buf->len = 0;
956 		etr_buf->full = false;
957 		return;
958 	}
959 
960 	etr_buf->full = !!(status & TMC_STS_FULL);
961 
962 	WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
963 
964 	etr_buf->ops->sync(etr_buf, rrp, rwp);
965 }
966 
967 static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
968 {
969 	u32 axictl, sts;
970 	struct etr_buf *etr_buf = drvdata->etr_buf;
971 	int rc = 0;
972 
973 	CS_UNLOCK(drvdata->base);
974 
975 	/* Wait for TMCSReady bit to be set */
976 	rc = tmc_wait_for_tmcready(drvdata);
977 	if (rc) {
978 		dev_err(&drvdata->csdev->dev,
979 			"Failed to enable : TMC not ready\n");
980 		CS_LOCK(drvdata->base);
981 		return rc;
982 	}
983 
984 	writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
985 	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
986 
987 	axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
988 	axictl &= ~TMC_AXICTL_CLEAR_MASK;
989 	axictl |= TMC_AXICTL_PROT_CTL_B1;
990 	axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
991 	axictl |= TMC_AXICTL_AXCACHE_OS;
992 
993 	if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
994 		axictl &= ~TMC_AXICTL_ARCACHE_MASK;
995 		axictl |= TMC_AXICTL_ARCACHE_OS;
996 	}
997 
998 	if (etr_buf->mode == ETR_MODE_ETR_SG)
999 		axictl |= TMC_AXICTL_SCT_GAT_MODE;
1000 
1001 	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1002 	tmc_write_dba(drvdata, etr_buf->hwaddr);
1003 	/*
1004 	 * If the TMC pointers must be programmed before the session,
1005 	 * we have to set it properly (i.e, RRP/RWP to base address and
1006 	 * STS to "not full").
1007 	 */
1008 	if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1009 		tmc_write_rrp(drvdata, etr_buf->hwaddr);
1010 		tmc_write_rwp(drvdata, etr_buf->hwaddr);
1011 		sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1012 		writel_relaxed(sts, drvdata->base + TMC_STS);
1013 	}
1014 
1015 	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1016 		       TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1017 		       TMC_FFCR_TRIGON_TRIGIN,
1018 		       drvdata->base + TMC_FFCR);
1019 	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1020 	tmc_enable_hw(drvdata);
1021 
1022 	CS_LOCK(drvdata->base);
1023 	return rc;
1024 }
1025 
1026 static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1027 			     struct etr_buf *etr_buf)
1028 {
1029 	int rc;
1030 
1031 	/* Callers should provide an appropriate buffer for use */
1032 	if (WARN_ON(!etr_buf))
1033 		return -EINVAL;
1034 
1035 	if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1036 	    WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1037 		return -EINVAL;
1038 
1039 	if (WARN_ON(drvdata->etr_buf))
1040 		return -EBUSY;
1041 
1042 	rc = coresight_claim_device(drvdata->csdev);
1043 	if (!rc) {
1044 		drvdata->etr_buf = etr_buf;
1045 		rc = __tmc_etr_enable_hw(drvdata);
1046 		if (rc) {
1047 			drvdata->etr_buf = NULL;
1048 			coresight_disclaim_device(drvdata->csdev);
1049 		}
1050 	}
1051 
1052 	return rc;
1053 }
1054 
1055 /*
1056  * Return the available trace data in the buffer (starts at etr_buf->offset,
1057  * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1058  * also updating the @bufpp on where to find it. Since the trace data
1059  * starts at anywhere in the buffer, depending on the RRP, we adjust the
1060  * @len returned to handle buffer wrapping around.
1061  *
1062  * We are protected here by drvdata->reading != 0, which ensures the
1063  * sysfs_buf stays alive.
1064  */
1065 ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1066 				loff_t pos, size_t len, char **bufpp)
1067 {
1068 	s64 offset;
1069 	ssize_t actual = len;
1070 	struct etr_buf *etr_buf = drvdata->sysfs_buf;
1071 
1072 	if (pos + actual > etr_buf->len)
1073 		actual = etr_buf->len - pos;
1074 	if (actual <= 0)
1075 		return actual;
1076 
1077 	/* Compute the offset from which we read the data */
1078 	offset = etr_buf->offset + pos;
1079 	if (offset >= etr_buf->size)
1080 		offset -= etr_buf->size;
1081 	return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1082 }
1083 
1084 static struct etr_buf *
1085 tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1086 {
1087 	return tmc_alloc_etr_buf(drvdata, drvdata->size,
1088 				 0, cpu_to_node(0), NULL);
1089 }
1090 
1091 static void
1092 tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1093 {
1094 	if (buf)
1095 		tmc_free_etr_buf(buf);
1096 }
1097 
1098 static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1099 {
1100 	struct etr_buf *etr_buf = drvdata->etr_buf;
1101 
1102 	if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1103 		tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1104 		drvdata->sysfs_buf = NULL;
1105 	} else {
1106 		tmc_sync_etr_buf(drvdata);
1107 		/*
1108 		 * Insert barrier packets at the beginning, if there was
1109 		 * an overflow.
1110 		 */
1111 		if (etr_buf->full)
1112 			tmc_etr_buf_insert_barrier_packet(etr_buf,
1113 							  etr_buf->offset);
1114 	}
1115 }
1116 
1117 static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1118 {
1119 	CS_UNLOCK(drvdata->base);
1120 
1121 	tmc_flush_and_stop(drvdata);
1122 	/*
1123 	 * When operating in sysFS mode the content of the buffer needs to be
1124 	 * read before the TMC is disabled.
1125 	 */
1126 	if (drvdata->mode == CS_MODE_SYSFS)
1127 		tmc_etr_sync_sysfs_buf(drvdata);
1128 
1129 	tmc_disable_hw(drvdata);
1130 
1131 	CS_LOCK(drvdata->base);
1132 
1133 }
1134 
1135 void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1136 {
1137 	__tmc_etr_disable_hw(drvdata);
1138 	coresight_disclaim_device(drvdata->csdev);
1139 	/* Reset the ETR buf used by hardware */
1140 	drvdata->etr_buf = NULL;
1141 }
1142 
1143 static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1144 {
1145 	int ret = 0;
1146 	unsigned long flags;
1147 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1148 	struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1149 
1150 	/*
1151 	 * If we are enabling the ETR from disabled state, we need to make
1152 	 * sure we have a buffer with the right size. The etr_buf is not reset
1153 	 * immediately after we stop the tracing in SYSFS mode as we wait for
1154 	 * the user to collect the data. We may be able to reuse the existing
1155 	 * buffer, provided the size matches. Any allocation has to be done
1156 	 * with the lock released.
1157 	 */
1158 	spin_lock_irqsave(&drvdata->spinlock, flags);
1159 	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1160 	if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1161 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1162 
1163 		/* Allocate memory with the locks released */
1164 		free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1165 		if (IS_ERR(new_buf))
1166 			return new_buf;
1167 
1168 		/* Let's try again */
1169 		spin_lock_irqsave(&drvdata->spinlock, flags);
1170 	}
1171 
1172 	if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1173 		ret = -EBUSY;
1174 		goto out;
1175 	}
1176 
1177 	/*
1178 	 * If we don't have a buffer or it doesn't match the requested size,
1179 	 * use the buffer allocated above. Otherwise reuse the existing buffer.
1180 	 */
1181 	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1182 	if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1183 		free_buf = sysfs_buf;
1184 		drvdata->sysfs_buf = new_buf;
1185 	}
1186 
1187 out:
1188 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1189 
1190 	/* Free memory outside the spinlock if need be */
1191 	if (free_buf)
1192 		tmc_etr_free_sysfs_buf(free_buf);
1193 	return ret ? ERR_PTR(ret) : drvdata->sysfs_buf;
1194 }
1195 
1196 static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1197 {
1198 	int ret = 0;
1199 	unsigned long flags;
1200 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1201 	struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);
1202 
1203 	if (IS_ERR(sysfs_buf))
1204 		return PTR_ERR(sysfs_buf);
1205 
1206 	spin_lock_irqsave(&drvdata->spinlock, flags);
1207 
1208 	/*
1209 	 * In sysFS mode we can have multiple writers per sink.  Since this
1210 	 * sink is already enabled no memory is needed and the HW need not be
1211 	 * touched, even if the buffer size has changed.
1212 	 */
1213 	if (drvdata->mode == CS_MODE_SYSFS) {
1214 		atomic_inc(&csdev->refcnt);
1215 		goto out;
1216 	}
1217 
1218 	ret = tmc_etr_enable_hw(drvdata, sysfs_buf);
1219 	if (!ret) {
1220 		drvdata->mode = CS_MODE_SYSFS;
1221 		atomic_inc(&csdev->refcnt);
1222 	}
1223 
1224 out:
1225 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1226 
1227 	if (!ret)
1228 		dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1229 
1230 	return ret;
1231 }
1232 
1233 struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
1234 				   enum cs_mode mode, void *data)
1235 {
1236 	struct perf_output_handle *handle = data;
1237 	struct etr_perf_buffer *etr_perf;
1238 
1239 	switch (mode) {
1240 	case CS_MODE_SYSFS:
1241 		return tmc_etr_get_sysfs_buffer(csdev);
1242 	case CS_MODE_PERF:
1243 		etr_perf = etm_perf_sink_config(handle);
1244 		if (WARN_ON(!etr_perf || !etr_perf->etr_buf))
1245 			return ERR_PTR(-EINVAL);
1246 		return etr_perf->etr_buf;
1247 	default:
1248 		return ERR_PTR(-EINVAL);
1249 	}
1250 }
1251 EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);
1252 
1253 /*
1254  * alloc_etr_buf: Allocate ETR buffer for use by perf.
1255  * The size of the hardware buffer is dependent on the size configured
1256  * via sysfs and the perf ring buffer size. We prefer to allocate the
1257  * largest possible size, scaling down the size by half until it
1258  * reaches a minimum limit (1M), beyond which we give up.
1259  */
1260 static struct etr_buf *
1261 alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1262 	      int nr_pages, void **pages, bool snapshot)
1263 {
1264 	int node;
1265 	struct etr_buf *etr_buf;
1266 	unsigned long size;
1267 
1268 	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1269 	/*
1270 	 * Try to match the perf ring buffer size if it is larger
1271 	 * than the size requested via sysfs.
1272 	 */
1273 	if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1274 		etr_buf = tmc_alloc_etr_buf(drvdata, ((ssize_t)nr_pages << PAGE_SHIFT),
1275 					    0, node, NULL);
1276 		if (!IS_ERR(etr_buf))
1277 			goto done;
1278 	}
1279 
1280 	/*
1281 	 * Else switch to configured size for this ETR
1282 	 * and scale down until we hit the minimum limit.
1283 	 */
1284 	size = drvdata->size;
1285 	do {
1286 		etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1287 		if (!IS_ERR(etr_buf))
1288 			goto done;
1289 		size /= 2;
1290 	} while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1291 
1292 	return ERR_PTR(-ENOMEM);
1293 
1294 done:
1295 	return etr_buf;
1296 }
1297 
1298 static struct etr_buf *
1299 get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1300 			  struct perf_event *event, int nr_pages,
1301 			  void **pages, bool snapshot)
1302 {
1303 	int ret;
1304 	pid_t pid = task_pid_nr(event->owner);
1305 	struct etr_buf *etr_buf;
1306 
1307 retry:
1308 	/*
1309 	 * An etr_perf_buffer is associated with an event and holds a reference
1310 	 * to the AUX ring buffer that was created for that event.  In CPU-wide
1311 	 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1312 	 * buffer, share a sink.  As such an etr_perf_buffer is created for each
1313 	 * event but a single etr_buf associated with the ETR is shared between
1314 	 * them.  The last event in a trace session will copy the content of the
1315 	 * etr_buf to its AUX ring buffer.  Ring buffer associated to other
1316 	 * events are simply not used an freed as events are destoyed.  We still
1317 	 * need to allocate a ring buffer for each event since we don't know
1318 	 * which event will be last.
1319 	 */
1320 
1321 	/*
1322 	 * The first thing to do here is check if an etr_buf has already been
1323 	 * allocated for this session.  If so it is shared with this event,
1324 	 * otherwise it is created.
1325 	 */
1326 	mutex_lock(&drvdata->idr_mutex);
1327 	etr_buf = idr_find(&drvdata->idr, pid);
1328 	if (etr_buf) {
1329 		refcount_inc(&etr_buf->refcount);
1330 		mutex_unlock(&drvdata->idr_mutex);
1331 		return etr_buf;
1332 	}
1333 
1334 	/* If we made it here no buffer has been allocated, do so now. */
1335 	mutex_unlock(&drvdata->idr_mutex);
1336 
1337 	etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1338 	if (IS_ERR(etr_buf))
1339 		return etr_buf;
1340 
1341 	/* Now that we have a buffer, add it to the IDR. */
1342 	mutex_lock(&drvdata->idr_mutex);
1343 	ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1344 	mutex_unlock(&drvdata->idr_mutex);
1345 
1346 	/* Another event with this session ID has allocated this buffer. */
1347 	if (ret == -ENOSPC) {
1348 		tmc_free_etr_buf(etr_buf);
1349 		goto retry;
1350 	}
1351 
1352 	/* The IDR can't allocate room for a new session, abandon ship. */
1353 	if (ret == -ENOMEM) {
1354 		tmc_free_etr_buf(etr_buf);
1355 		return ERR_PTR(ret);
1356 	}
1357 
1358 
1359 	return etr_buf;
1360 }
1361 
1362 static struct etr_buf *
1363 get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1364 			    struct perf_event *event, int nr_pages,
1365 			    void **pages, bool snapshot)
1366 {
1367 	/*
1368 	 * In per-thread mode the etr_buf isn't shared, so just go ahead
1369 	 * with memory allocation.
1370 	 */
1371 	return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1372 }
1373 
1374 static struct etr_buf *
1375 get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1376 		 int nr_pages, void **pages, bool snapshot)
1377 {
1378 	if (event->cpu == -1)
1379 		return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1380 						   pages, snapshot);
1381 
1382 	return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1383 					 pages, snapshot);
1384 }
1385 
1386 static struct etr_perf_buffer *
1387 tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1388 		       int nr_pages, void **pages, bool snapshot)
1389 {
1390 	int node;
1391 	struct etr_buf *etr_buf;
1392 	struct etr_perf_buffer *etr_perf;
1393 
1394 	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1395 
1396 	etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1397 	if (!etr_perf)
1398 		return ERR_PTR(-ENOMEM);
1399 
1400 	etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1401 	if (!IS_ERR(etr_buf))
1402 		goto done;
1403 
1404 	kfree(etr_perf);
1405 	return ERR_PTR(-ENOMEM);
1406 
1407 done:
1408 	/*
1409 	 * Keep a reference to the ETR this buffer has been allocated for
1410 	 * in order to have access to the IDR in tmc_free_etr_buffer().
1411 	 */
1412 	etr_perf->drvdata = drvdata;
1413 	etr_perf->etr_buf = etr_buf;
1414 
1415 	return etr_perf;
1416 }
1417 
1418 
1419 static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1420 				  struct perf_event *event, void **pages,
1421 				  int nr_pages, bool snapshot)
1422 {
1423 	struct etr_perf_buffer *etr_perf;
1424 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1425 
1426 	etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1427 					  nr_pages, pages, snapshot);
1428 	if (IS_ERR(etr_perf)) {
1429 		dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1430 		return NULL;
1431 	}
1432 
1433 	etr_perf->pid = task_pid_nr(event->owner);
1434 	etr_perf->snapshot = snapshot;
1435 	etr_perf->nr_pages = nr_pages;
1436 	etr_perf->pages = pages;
1437 
1438 	return etr_perf;
1439 }
1440 
1441 static void tmc_free_etr_buffer(void *config)
1442 {
1443 	struct etr_perf_buffer *etr_perf = config;
1444 	struct tmc_drvdata *drvdata = etr_perf->drvdata;
1445 	struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1446 
1447 	if (!etr_buf)
1448 		goto free_etr_perf_buffer;
1449 
1450 	mutex_lock(&drvdata->idr_mutex);
1451 	/* If we are not the last one to use the buffer, don't touch it. */
1452 	if (!refcount_dec_and_test(&etr_buf->refcount)) {
1453 		mutex_unlock(&drvdata->idr_mutex);
1454 		goto free_etr_perf_buffer;
1455 	}
1456 
1457 	/* We are the last one, remove from the IDR and free the buffer. */
1458 	buf = idr_remove(&drvdata->idr, etr_perf->pid);
1459 	mutex_unlock(&drvdata->idr_mutex);
1460 
1461 	/*
1462 	 * Something went very wrong if the buffer associated with this ID
1463 	 * is not the same in the IDR.  Leak to avoid use after free.
1464 	 */
1465 	if (buf && WARN_ON(buf != etr_buf))
1466 		goto free_etr_perf_buffer;
1467 
1468 	tmc_free_etr_buf(etr_perf->etr_buf);
1469 
1470 free_etr_perf_buffer:
1471 	kfree(etr_perf);
1472 }
1473 
1474 /*
1475  * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1476  * buffer to the perf ring buffer.
1477  */
1478 static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1479 				     unsigned long head,
1480 				     unsigned long src_offset,
1481 				     unsigned long to_copy)
1482 {
1483 	long bytes;
1484 	long pg_idx, pg_offset;
1485 	char **dst_pages, *src_buf;
1486 	struct etr_buf *etr_buf = etr_perf->etr_buf;
1487 
1488 	head = PERF_IDX2OFF(head, etr_perf);
1489 	pg_idx = head >> PAGE_SHIFT;
1490 	pg_offset = head & (PAGE_SIZE - 1);
1491 	dst_pages = (char **)etr_perf->pages;
1492 
1493 	while (to_copy > 0) {
1494 		/*
1495 		 * In one iteration, we can copy minimum of :
1496 		 *  1) what is available in the source buffer,
1497 		 *  2) what is available in the source buffer, before it
1498 		 *     wraps around.
1499 		 *  3) what is available in the destination page.
1500 		 * in one iteration.
1501 		 */
1502 		if (src_offset >= etr_buf->size)
1503 			src_offset -= etr_buf->size;
1504 		bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1505 					     &src_buf);
1506 		if (WARN_ON_ONCE(bytes <= 0))
1507 			break;
1508 		bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1509 
1510 		memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1511 
1512 		to_copy -= bytes;
1513 
1514 		/* Move destination pointers */
1515 		pg_offset += bytes;
1516 		if (pg_offset == PAGE_SIZE) {
1517 			pg_offset = 0;
1518 			if (++pg_idx == etr_perf->nr_pages)
1519 				pg_idx = 0;
1520 		}
1521 
1522 		/* Move source pointers */
1523 		src_offset += bytes;
1524 	}
1525 }
1526 
1527 /*
1528  * tmc_update_etr_buffer : Update the perf ring buffer with the
1529  * available trace data. We use software double buffering at the moment.
1530  *
1531  * TODO: Add support for reusing the perf ring buffer.
1532  */
1533 static unsigned long
1534 tmc_update_etr_buffer(struct coresight_device *csdev,
1535 		      struct perf_output_handle *handle,
1536 		      void *config)
1537 {
1538 	bool lost = false;
1539 	unsigned long flags, offset, size = 0;
1540 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1541 	struct etr_perf_buffer *etr_perf = config;
1542 	struct etr_buf *etr_buf = etr_perf->etr_buf;
1543 
1544 	spin_lock_irqsave(&drvdata->spinlock, flags);
1545 
1546 	/* Don't do anything if another tracer is using this sink */
1547 	if (atomic_read(&csdev->refcnt) != 1) {
1548 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1549 		goto out;
1550 	}
1551 
1552 	if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1553 		lost = true;
1554 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1555 		goto out;
1556 	}
1557 
1558 	CS_UNLOCK(drvdata->base);
1559 
1560 	tmc_flush_and_stop(drvdata);
1561 	tmc_sync_etr_buf(drvdata);
1562 
1563 	CS_LOCK(drvdata->base);
1564 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1565 
1566 	lost = etr_buf->full;
1567 	offset = etr_buf->offset;
1568 	size = etr_buf->len;
1569 
1570 	/*
1571 	 * The ETR buffer may be bigger than the space available in the
1572 	 * perf ring buffer (handle->size).  If so advance the offset so that we
1573 	 * get the latest trace data.  In snapshot mode none of that matters
1574 	 * since we are expected to clobber stale data in favour of the latest
1575 	 * traces.
1576 	 */
1577 	if (!etr_perf->snapshot && size > handle->size) {
1578 		u32 mask = tmc_get_memwidth_mask(drvdata);
1579 
1580 		/*
1581 		 * Make sure the new size is aligned in accordance with the
1582 		 * requirement explained in function tmc_get_memwidth_mask().
1583 		 */
1584 		size = handle->size & mask;
1585 		offset = etr_buf->offset + etr_buf->len - size;
1586 
1587 		if (offset >= etr_buf->size)
1588 			offset -= etr_buf->size;
1589 		lost = true;
1590 	}
1591 
1592 	/* Insert barrier packets at the beginning, if there was an overflow */
1593 	if (lost)
1594 		tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1595 	tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);
1596 
1597 	/*
1598 	 * In snapshot mode we simply increment the head by the number of byte
1599 	 * that were written.  User space will figure out how many bytes to get
1600 	 * from the AUX buffer based on the position of the head.
1601 	 */
1602 	if (etr_perf->snapshot)
1603 		handle->head += size;
1604 
1605 	/*
1606 	 * Ensure that the AUX trace data is visible before the aux_head
1607 	 * is updated via perf_aux_output_end(), as expected by the
1608 	 * perf ring buffer.
1609 	 */
1610 	smp_wmb();
1611 
1612 out:
1613 	/*
1614 	 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1615 	 * captured buffer is expected to be truncated and 2) a full buffer
1616 	 * prevents the event from being re-enabled by the perf core,
1617 	 * resulting in stale data being send to user space.
1618 	 */
1619 	if (!etr_perf->snapshot && lost)
1620 		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1621 	return size;
1622 }
1623 
1624 static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1625 {
1626 	int rc = 0;
1627 	pid_t pid;
1628 	unsigned long flags;
1629 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1630 	struct perf_output_handle *handle = data;
1631 	struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1632 
1633 	spin_lock_irqsave(&drvdata->spinlock, flags);
1634 	 /* Don't use this sink if it is already claimed by sysFS */
1635 	if (drvdata->mode == CS_MODE_SYSFS) {
1636 		rc = -EBUSY;
1637 		goto unlock_out;
1638 	}
1639 
1640 	if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1641 		rc = -EINVAL;
1642 		goto unlock_out;
1643 	}
1644 
1645 	/* Get a handle on the pid of the process to monitor */
1646 	pid = etr_perf->pid;
1647 
1648 	/* Do not proceed if this device is associated with another session */
1649 	if (drvdata->pid != -1 && drvdata->pid != pid) {
1650 		rc = -EBUSY;
1651 		goto unlock_out;
1652 	}
1653 
1654 	/*
1655 	 * No HW configuration is needed if the sink is already in
1656 	 * use for this session.
1657 	 */
1658 	if (drvdata->pid == pid) {
1659 		atomic_inc(&csdev->refcnt);
1660 		goto unlock_out;
1661 	}
1662 
1663 	rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1664 	if (!rc) {
1665 		/* Associate with monitored process. */
1666 		drvdata->pid = pid;
1667 		drvdata->mode = CS_MODE_PERF;
1668 		drvdata->perf_buf = etr_perf->etr_buf;
1669 		atomic_inc(&csdev->refcnt);
1670 	}
1671 
1672 unlock_out:
1673 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1674 	return rc;
1675 }
1676 
1677 static int tmc_enable_etr_sink(struct coresight_device *csdev,
1678 			       enum cs_mode mode, void *data)
1679 {
1680 	switch (mode) {
1681 	case CS_MODE_SYSFS:
1682 		return tmc_enable_etr_sink_sysfs(csdev);
1683 	case CS_MODE_PERF:
1684 		return tmc_enable_etr_sink_perf(csdev, data);
1685 	default:
1686 		return -EINVAL;
1687 	}
1688 }
1689 
1690 static int tmc_disable_etr_sink(struct coresight_device *csdev)
1691 {
1692 	unsigned long flags;
1693 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1694 
1695 	spin_lock_irqsave(&drvdata->spinlock, flags);
1696 
1697 	if (drvdata->reading) {
1698 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1699 		return -EBUSY;
1700 	}
1701 
1702 	if (atomic_dec_return(&csdev->refcnt)) {
1703 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1704 		return -EBUSY;
1705 	}
1706 
1707 	/* Complain if we (somehow) got out of sync */
1708 	WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1709 	tmc_etr_disable_hw(drvdata);
1710 	/* Dissociate from monitored process. */
1711 	drvdata->pid = -1;
1712 	drvdata->mode = CS_MODE_DISABLED;
1713 	/* Reset perf specific data */
1714 	drvdata->perf_buf = NULL;
1715 
1716 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1717 
1718 	dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1719 	return 0;
1720 }
1721 
1722 static const struct coresight_ops_sink tmc_etr_sink_ops = {
1723 	.enable		= tmc_enable_etr_sink,
1724 	.disable	= tmc_disable_etr_sink,
1725 	.alloc_buffer	= tmc_alloc_etr_buffer,
1726 	.update_buffer	= tmc_update_etr_buffer,
1727 	.free_buffer	= tmc_free_etr_buffer,
1728 };
1729 
1730 const struct coresight_ops tmc_etr_cs_ops = {
1731 	.sink_ops	= &tmc_etr_sink_ops,
1732 };
1733 
1734 int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1735 {
1736 	int ret = 0;
1737 	unsigned long flags;
1738 
1739 	/* config types are set a boot time and never change */
1740 	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1741 		return -EINVAL;
1742 
1743 	spin_lock_irqsave(&drvdata->spinlock, flags);
1744 	if (drvdata->reading) {
1745 		ret = -EBUSY;
1746 		goto out;
1747 	}
1748 
1749 	/*
1750 	 * We can safely allow reads even if the ETR is operating in PERF mode,
1751 	 * since the sysfs session is captured in mode specific data.
1752 	 * If drvdata::sysfs_data is NULL the trace data has been read already.
1753 	 */
1754 	if (!drvdata->sysfs_buf) {
1755 		ret = -EINVAL;
1756 		goto out;
1757 	}
1758 
1759 	/* Disable the TMC if we are trying to read from a running session. */
1760 	if (drvdata->mode == CS_MODE_SYSFS)
1761 		__tmc_etr_disable_hw(drvdata);
1762 
1763 	drvdata->reading = true;
1764 out:
1765 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1766 
1767 	return ret;
1768 }
1769 
1770 int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1771 {
1772 	unsigned long flags;
1773 	struct etr_buf *sysfs_buf = NULL;
1774 
1775 	/* config types are set a boot time and never change */
1776 	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1777 		return -EINVAL;
1778 
1779 	spin_lock_irqsave(&drvdata->spinlock, flags);
1780 
1781 	/* RE-enable the TMC if need be */
1782 	if (drvdata->mode == CS_MODE_SYSFS) {
1783 		/*
1784 		 * The trace run will continue with the same allocated trace
1785 		 * buffer. Since the tracer is still enabled drvdata::buf can't
1786 		 * be NULL.
1787 		 */
1788 		__tmc_etr_enable_hw(drvdata);
1789 	} else {
1790 		/*
1791 		 * The ETR is not tracing and the buffer was just read.
1792 		 * As such prepare to free the trace buffer.
1793 		 */
1794 		sysfs_buf = drvdata->sysfs_buf;
1795 		drvdata->sysfs_buf = NULL;
1796 	}
1797 
1798 	drvdata->reading = false;
1799 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1800 
1801 	/* Free allocated memory out side of the spinlock */
1802 	if (sysfs_buf)
1803 		tmc_etr_free_sysfs_buf(sysfs_buf);
1804 
1805 	return 0;
1806 }
1807