1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright(C) 2016 Linaro Limited. All rights reserved.
4  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5  */
6 
7 #include <linux/atomic.h>
8 #include <linux/coresight.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/iommu.h>
11 #include <linux/idr.h>
12 #include <linux/mutex.h>
13 #include <linux/refcount.h>
14 #include <linux/slab.h>
15 #include <linux/types.h>
16 #include <linux/vmalloc.h>
17 #include "coresight-catu.h"
18 #include "coresight-etm-perf.h"
19 #include "coresight-priv.h"
20 #include "coresight-tmc.h"
21 
22 struct etr_flat_buf {
23 	struct device	*dev;
24 	dma_addr_t	daddr;
25 	void		*vaddr;
26 	size_t		size;
27 };
28 
29 /*
30  * etr_perf_buffer - Perf buffer used for ETR
31  * @drvdata		- The ETR drvdaga this buffer has been allocated for.
32  * @etr_buf		- Actual buffer used by the ETR
33  * @pid			- The PID this etr_perf_buffer belongs to.
34  * @snaphost		- Perf session mode
35  * @head		- handle->head at the beginning of the session.
36  * @nr_pages		- Number of pages in the ring buffer.
37  * @pages		- Array of Pages in the ring buffer.
38  */
39 struct etr_perf_buffer {
40 	struct tmc_drvdata	*drvdata;
41 	struct etr_buf		*etr_buf;
42 	pid_t			pid;
43 	bool			snapshot;
44 	unsigned long		head;
45 	int			nr_pages;
46 	void			**pages;
47 };
48 
49 /* Convert the perf index to an offset within the ETR buffer */
50 #define PERF_IDX2OFF(idx, buf)	((idx) % ((buf)->nr_pages << PAGE_SHIFT))
51 
52 /* Lower limit for ETR hardware buffer */
53 #define TMC_ETR_PERF_MIN_BUF_SIZE	SZ_1M
54 
55 /*
56  * The TMC ETR SG has a page size of 4K. The SG table contains pointers
57  * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
58  * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
59  * contain more than one SG buffer and tables.
60  *
61  * A table entry has the following format:
62  *
63  * ---Bit31------------Bit4-------Bit1-----Bit0--
64  * |     Address[39:12]    | SBZ |  Entry Type  |
65  * ----------------------------------------------
66  *
67  * Address: Bits [39:12] of a physical page address. Bits [11:0] are
68  *	    always zero.
69  *
70  * Entry type:
71  *	b00 - Reserved.
72  *	b01 - Last entry in the tables, points to 4K page buffer.
73  *	b10 - Normal entry, points to 4K page buffer.
74  *	b11 - Link. The address points to the base of next table.
75  */
76 
77 typedef u32 sgte_t;
78 
79 #define ETR_SG_PAGE_SHIFT		12
80 #define ETR_SG_PAGE_SIZE		(1UL << ETR_SG_PAGE_SHIFT)
81 #define ETR_SG_PAGES_PER_SYSPAGE	(PAGE_SIZE / ETR_SG_PAGE_SIZE)
82 #define ETR_SG_PTRS_PER_PAGE		(ETR_SG_PAGE_SIZE / sizeof(sgte_t))
83 #define ETR_SG_PTRS_PER_SYSPAGE		(PAGE_SIZE / sizeof(sgte_t))
84 
85 #define ETR_SG_ET_MASK			0x3
86 #define ETR_SG_ET_LAST			0x1
87 #define ETR_SG_ET_NORMAL		0x2
88 #define ETR_SG_ET_LINK			0x3
89 
90 #define ETR_SG_ADDR_SHIFT		4
91 
92 #define ETR_SG_ENTRY(addr, type) \
93 	(sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
94 		 (type & ETR_SG_ET_MASK))
95 
96 #define ETR_SG_ADDR(entry) \
97 	(((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
98 #define ETR_SG_ET(entry)		((entry) & ETR_SG_ET_MASK)
99 
100 /*
101  * struct etr_sg_table : ETR SG Table
102  * @sg_table:		Generic SG Table holding the data/table pages.
103  * @hwaddr:		hwaddress used by the TMC, which is the base
104  *			address of the table.
105  */
106 struct etr_sg_table {
107 	struct tmc_sg_table	*sg_table;
108 	dma_addr_t		hwaddr;
109 };
110 
111 /*
112  * tmc_etr_sg_table_entries: Total number of table entries required to map
113  * @nr_pages system pages.
114  *
115  * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
116  * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
117  * with the last entry pointing to another page of table entries.
118  * If we spill over to a new page for mapping 1 entry, we could as
119  * well replace the link entry of the previous page with the last entry.
120  */
121 static inline unsigned long __attribute_const__
122 tmc_etr_sg_table_entries(int nr_pages)
123 {
124 	unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
125 	unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
126 	/*
127 	 * If we spill over to a new page for 1 entry, we could as well
128 	 * make it the LAST entry in the previous page, skipping the Link
129 	 * address.
130 	 */
131 	if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
132 		nr_sglinks--;
133 	return nr_sgpages + nr_sglinks;
134 }
135 
136 /*
137  * tmc_pages_get_offset:  Go through all the pages in the tmc_pages
138  * and map the device address @addr to an offset within the virtual
139  * contiguous buffer.
140  */
141 static long
142 tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
143 {
144 	int i;
145 	dma_addr_t page_start;
146 
147 	for (i = 0; i < tmc_pages->nr_pages; i++) {
148 		page_start = tmc_pages->daddrs[i];
149 		if (addr >= page_start && addr < (page_start + PAGE_SIZE))
150 			return i * PAGE_SIZE + (addr - page_start);
151 	}
152 
153 	return -EINVAL;
154 }
155 
156 /*
157  * tmc_pages_free : Unmap and free the pages used by tmc_pages.
158  * If the pages were not allocated in tmc_pages_alloc(), we would
159  * simply drop the refcount.
160  */
161 static void tmc_pages_free(struct tmc_pages *tmc_pages,
162 			   struct device *dev, enum dma_data_direction dir)
163 {
164 	int i;
165 	struct device *real_dev = dev->parent;
166 
167 	for (i = 0; i < tmc_pages->nr_pages; i++) {
168 		if (tmc_pages->daddrs && tmc_pages->daddrs[i])
169 			dma_unmap_page(real_dev, tmc_pages->daddrs[i],
170 					 PAGE_SIZE, dir);
171 		if (tmc_pages->pages && tmc_pages->pages[i])
172 			__free_page(tmc_pages->pages[i]);
173 	}
174 
175 	kfree(tmc_pages->pages);
176 	kfree(tmc_pages->daddrs);
177 	tmc_pages->pages = NULL;
178 	tmc_pages->daddrs = NULL;
179 	tmc_pages->nr_pages = 0;
180 }
181 
182 /*
183  * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
184  * If @pages is not NULL, the list of page virtual addresses are
185  * used as the data pages. The pages are then dma_map'ed for @dev
186  * with dma_direction @dir.
187  *
188  * Returns 0 upon success, else the error number.
189  */
190 static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
191 			   struct device *dev, int node,
192 			   enum dma_data_direction dir, void **pages)
193 {
194 	int i, nr_pages;
195 	dma_addr_t paddr;
196 	struct page *page;
197 	struct device *real_dev = dev->parent;
198 
199 	nr_pages = tmc_pages->nr_pages;
200 	tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
201 					 GFP_KERNEL);
202 	if (!tmc_pages->daddrs)
203 		return -ENOMEM;
204 	tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
205 					 GFP_KERNEL);
206 	if (!tmc_pages->pages) {
207 		kfree(tmc_pages->daddrs);
208 		tmc_pages->daddrs = NULL;
209 		return -ENOMEM;
210 	}
211 
212 	for (i = 0; i < nr_pages; i++) {
213 		if (pages && pages[i]) {
214 			page = virt_to_page(pages[i]);
215 			/* Hold a refcount on the page */
216 			get_page(page);
217 		} else {
218 			page = alloc_pages_node(node,
219 						GFP_KERNEL | __GFP_ZERO, 0);
220 		}
221 		paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
222 		if (dma_mapping_error(real_dev, paddr))
223 			goto err;
224 		tmc_pages->daddrs[i] = paddr;
225 		tmc_pages->pages[i] = page;
226 	}
227 	return 0;
228 err:
229 	tmc_pages_free(tmc_pages, dev, dir);
230 	return -ENOMEM;
231 }
232 
233 static inline long
234 tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
235 {
236 	return tmc_pages_get_offset(&sg_table->data_pages, addr);
237 }
238 
239 static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
240 {
241 	if (sg_table->table_vaddr)
242 		vunmap(sg_table->table_vaddr);
243 	tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
244 }
245 
246 static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
247 {
248 	if (sg_table->data_vaddr)
249 		vunmap(sg_table->data_vaddr);
250 	tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
251 }
252 
253 void tmc_free_sg_table(struct tmc_sg_table *sg_table)
254 {
255 	tmc_free_table_pages(sg_table);
256 	tmc_free_data_pages(sg_table);
257 }
258 
259 /*
260  * Alloc pages for the table. Since this will be used by the device,
261  * allocate the pages closer to the device (i.e, dev_to_node(dev)
262  * rather than the CPU node).
263  */
264 static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
265 {
266 	int rc;
267 	struct tmc_pages *table_pages = &sg_table->table_pages;
268 
269 	rc = tmc_pages_alloc(table_pages, sg_table->dev,
270 			     dev_to_node(sg_table->dev),
271 			     DMA_TO_DEVICE, NULL);
272 	if (rc)
273 		return rc;
274 	sg_table->table_vaddr = vmap(table_pages->pages,
275 				     table_pages->nr_pages,
276 				     VM_MAP,
277 				     PAGE_KERNEL);
278 	if (!sg_table->table_vaddr)
279 		rc = -ENOMEM;
280 	else
281 		sg_table->table_daddr = table_pages->daddrs[0];
282 	return rc;
283 }
284 
285 static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
286 {
287 	int rc;
288 
289 	/* Allocate data pages on the node requested by the caller */
290 	rc = tmc_pages_alloc(&sg_table->data_pages,
291 			     sg_table->dev, sg_table->node,
292 			     DMA_FROM_DEVICE, pages);
293 	if (!rc) {
294 		sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
295 					    sg_table->data_pages.nr_pages,
296 					    VM_MAP,
297 					    PAGE_KERNEL);
298 		if (!sg_table->data_vaddr)
299 			rc = -ENOMEM;
300 	}
301 	return rc;
302 }
303 
304 /*
305  * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
306  * and data buffers. TMC writes to the data buffers and reads from the SG
307  * Table pages.
308  *
309  * @dev		- Coresight device to which page should be DMA mapped.
310  * @node	- Numa node for mem allocations
311  * @nr_tpages	- Number of pages for the table entries.
312  * @nr_dpages	- Number of pages for Data buffer.
313  * @pages	- Optional list of virtual address of pages.
314  */
315 struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
316 					int node,
317 					int nr_tpages,
318 					int nr_dpages,
319 					void **pages)
320 {
321 	long rc;
322 	struct tmc_sg_table *sg_table;
323 
324 	sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
325 	if (!sg_table)
326 		return ERR_PTR(-ENOMEM);
327 	sg_table->data_pages.nr_pages = nr_dpages;
328 	sg_table->table_pages.nr_pages = nr_tpages;
329 	sg_table->node = node;
330 	sg_table->dev = dev;
331 
332 	rc  = tmc_alloc_data_pages(sg_table, pages);
333 	if (!rc)
334 		rc = tmc_alloc_table_pages(sg_table);
335 	if (rc) {
336 		tmc_free_sg_table(sg_table);
337 		kfree(sg_table);
338 		return ERR_PTR(rc);
339 	}
340 
341 	return sg_table;
342 }
343 
344 /*
345  * tmc_sg_table_sync_data_range: Sync the data buffer written
346  * by the device from @offset upto a @size bytes.
347  */
348 void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
349 				  u64 offset, u64 size)
350 {
351 	int i, index, start;
352 	int npages = DIV_ROUND_UP(size, PAGE_SIZE);
353 	struct device *real_dev = table->dev->parent;
354 	struct tmc_pages *data = &table->data_pages;
355 
356 	start = offset >> PAGE_SHIFT;
357 	for (i = start; i < (start + npages); i++) {
358 		index = i % data->nr_pages;
359 		dma_sync_single_for_cpu(real_dev, data->daddrs[index],
360 					PAGE_SIZE, DMA_FROM_DEVICE);
361 	}
362 }
363 
364 /* tmc_sg_sync_table: Sync the page table */
365 void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
366 {
367 	int i;
368 	struct device *real_dev = sg_table->dev->parent;
369 	struct tmc_pages *table_pages = &sg_table->table_pages;
370 
371 	for (i = 0; i < table_pages->nr_pages; i++)
372 		dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
373 					   PAGE_SIZE, DMA_TO_DEVICE);
374 }
375 
376 /*
377  * tmc_sg_table_get_data: Get the buffer pointer for data @offset
378  * in the SG buffer. The @bufpp is updated to point to the buffer.
379  * Returns :
380  *	the length of linear data available at @offset.
381  *	or
382  *	<= 0 if no data is available.
383  */
384 ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
385 			      u64 offset, size_t len, char **bufpp)
386 {
387 	size_t size;
388 	int pg_idx = offset >> PAGE_SHIFT;
389 	int pg_offset = offset & (PAGE_SIZE - 1);
390 	struct tmc_pages *data_pages = &sg_table->data_pages;
391 
392 	size = tmc_sg_table_buf_size(sg_table);
393 	if (offset >= size)
394 		return -EINVAL;
395 
396 	/* Make sure we don't go beyond the end */
397 	len = (len < (size - offset)) ? len : size - offset;
398 	/* Respect the page boundaries */
399 	len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
400 	if (len > 0)
401 		*bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
402 	return len;
403 }
404 
405 #ifdef ETR_SG_DEBUG
406 /* Map a dma address to virtual address */
407 static unsigned long
408 tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
409 		      dma_addr_t addr, bool table)
410 {
411 	long offset;
412 	unsigned long base;
413 	struct tmc_pages *tmc_pages;
414 
415 	if (table) {
416 		tmc_pages = &sg_table->table_pages;
417 		base = (unsigned long)sg_table->table_vaddr;
418 	} else {
419 		tmc_pages = &sg_table->data_pages;
420 		base = (unsigned long)sg_table->data_vaddr;
421 	}
422 
423 	offset = tmc_pages_get_offset(tmc_pages, addr);
424 	if (offset < 0)
425 		return 0;
426 	return base + offset;
427 }
428 
429 /* Dump the given sg_table */
430 static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
431 {
432 	sgte_t *ptr;
433 	int i = 0;
434 	dma_addr_t addr;
435 	struct tmc_sg_table *sg_table = etr_table->sg_table;
436 
437 	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
438 					      etr_table->hwaddr, true);
439 	while (ptr) {
440 		addr = ETR_SG_ADDR(*ptr);
441 		switch (ETR_SG_ET(*ptr)) {
442 		case ETR_SG_ET_NORMAL:
443 			dev_dbg(sg_table->dev,
444 				"%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
445 			ptr++;
446 			break;
447 		case ETR_SG_ET_LINK:
448 			dev_dbg(sg_table->dev,
449 				"%05d: *** %p\t:{L} 0x%llx ***\n",
450 				 i, ptr, addr);
451 			ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
452 							      addr, true);
453 			break;
454 		case ETR_SG_ET_LAST:
455 			dev_dbg(sg_table->dev,
456 				"%05d: ### %p\t:[L] 0x%llx ###\n",
457 				 i, ptr, addr);
458 			return;
459 		default:
460 			dev_dbg(sg_table->dev,
461 				"%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
462 				 i, ptr, addr);
463 			return;
464 		}
465 		i++;
466 	}
467 	dev_dbg(sg_table->dev, "******* End of Table *****\n");
468 }
469 #else
470 static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
471 #endif
472 
473 /*
474  * Populate the SG Table page table entries from table/data
475  * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
476  * So does a Table page. So we keep track of indices of the tables
477  * in each system page and move the pointers accordingly.
478  */
479 #define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
480 static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
481 {
482 	dma_addr_t paddr;
483 	int i, type, nr_entries;
484 	int tpidx = 0; /* index to the current system table_page */
485 	int sgtidx = 0;	/* index to the sg_table within the current syspage */
486 	int sgtentry = 0; /* the entry within the sg_table */
487 	int dpidx = 0; /* index to the current system data_page */
488 	int spidx = 0; /* index to the SG page within the current data page */
489 	sgte_t *ptr; /* pointer to the table entry to fill */
490 	struct tmc_sg_table *sg_table = etr_table->sg_table;
491 	dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
492 	dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
493 
494 	nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
495 	/*
496 	 * Use the contiguous virtual address of the table to update entries.
497 	 */
498 	ptr = sg_table->table_vaddr;
499 	/*
500 	 * Fill all the entries, except the last entry to avoid special
501 	 * checks within the loop.
502 	 */
503 	for (i = 0; i < nr_entries - 1; i++) {
504 		if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
505 			/*
506 			 * Last entry in a sg_table page is a link address to
507 			 * the next table page. If this sg_table is the last
508 			 * one in the system page, it links to the first
509 			 * sg_table in the next system page. Otherwise, it
510 			 * links to the next sg_table page within the system
511 			 * page.
512 			 */
513 			if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
514 				paddr = table_daddrs[tpidx + 1];
515 			} else {
516 				paddr = table_daddrs[tpidx] +
517 					(ETR_SG_PAGE_SIZE * (sgtidx + 1));
518 			}
519 			type = ETR_SG_ET_LINK;
520 		} else {
521 			/*
522 			 * Update the indices to the data_pages to point to the
523 			 * next sg_page in the data buffer.
524 			 */
525 			type = ETR_SG_ET_NORMAL;
526 			paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
527 			if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
528 				dpidx++;
529 		}
530 		*ptr++ = ETR_SG_ENTRY(paddr, type);
531 		/*
532 		 * Move to the next table pointer, moving the table page index
533 		 * if necessary
534 		 */
535 		if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
536 			if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
537 				tpidx++;
538 		}
539 	}
540 
541 	/* Set up the last entry, which is always a data pointer */
542 	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
543 	*ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
544 }
545 
546 /*
547  * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
548  * populate the table.
549  *
550  * @dev		- Device pointer for the TMC
551  * @node	- NUMA node where the memory should be allocated
552  * @size	- Total size of the data buffer
553  * @pages	- Optional list of page virtual address
554  */
555 static struct etr_sg_table *
556 tmc_init_etr_sg_table(struct device *dev, int node,
557 		      unsigned long size, void **pages)
558 {
559 	int nr_entries, nr_tpages;
560 	int nr_dpages = size >> PAGE_SHIFT;
561 	struct tmc_sg_table *sg_table;
562 	struct etr_sg_table *etr_table;
563 
564 	etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
565 	if (!etr_table)
566 		return ERR_PTR(-ENOMEM);
567 	nr_entries = tmc_etr_sg_table_entries(nr_dpages);
568 	nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
569 
570 	sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
571 	if (IS_ERR(sg_table)) {
572 		kfree(etr_table);
573 		return ERR_CAST(sg_table);
574 	}
575 
576 	etr_table->sg_table = sg_table;
577 	/* TMC should use table base address for DBA */
578 	etr_table->hwaddr = sg_table->table_daddr;
579 	tmc_etr_sg_table_populate(etr_table);
580 	/* Sync the table pages for the HW */
581 	tmc_sg_table_sync_table(sg_table);
582 	tmc_etr_sg_table_dump(etr_table);
583 
584 	return etr_table;
585 }
586 
587 /*
588  * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
589  */
590 static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
591 				  struct etr_buf *etr_buf, int node,
592 				  void **pages)
593 {
594 	struct etr_flat_buf *flat_buf;
595 	struct device *real_dev = drvdata->csdev->dev.parent;
596 
597 	/* We cannot reuse existing pages for flat buf */
598 	if (pages)
599 		return -EINVAL;
600 
601 	flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
602 	if (!flat_buf)
603 		return -ENOMEM;
604 
605 	flat_buf->vaddr = dma_alloc_coherent(real_dev, etr_buf->size,
606 					     &flat_buf->daddr, GFP_KERNEL);
607 	if (!flat_buf->vaddr) {
608 		kfree(flat_buf);
609 		return -ENOMEM;
610 	}
611 
612 	flat_buf->size = etr_buf->size;
613 	flat_buf->dev = &drvdata->csdev->dev;
614 	etr_buf->hwaddr = flat_buf->daddr;
615 	etr_buf->mode = ETR_MODE_FLAT;
616 	etr_buf->private = flat_buf;
617 	return 0;
618 }
619 
620 static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
621 {
622 	struct etr_flat_buf *flat_buf = etr_buf->private;
623 
624 	if (flat_buf && flat_buf->daddr) {
625 		struct device *real_dev = flat_buf->dev->parent;
626 
627 		dma_free_coherent(real_dev, flat_buf->size,
628 				  flat_buf->vaddr, flat_buf->daddr);
629 	}
630 	kfree(flat_buf);
631 }
632 
633 static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
634 {
635 	/*
636 	 * Adjust the buffer to point to the beginning of the trace data
637 	 * and update the available trace data.
638 	 */
639 	etr_buf->offset = rrp - etr_buf->hwaddr;
640 	if (etr_buf->full)
641 		etr_buf->len = etr_buf->size;
642 	else
643 		etr_buf->len = rwp - rrp;
644 }
645 
646 static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
647 					 u64 offset, size_t len, char **bufpp)
648 {
649 	struct etr_flat_buf *flat_buf = etr_buf->private;
650 
651 	*bufpp = (char *)flat_buf->vaddr + offset;
652 	/*
653 	 * tmc_etr_buf_get_data already adjusts the length to handle
654 	 * buffer wrapping around.
655 	 */
656 	return len;
657 }
658 
659 static const struct etr_buf_operations etr_flat_buf_ops = {
660 	.alloc = tmc_etr_alloc_flat_buf,
661 	.free = tmc_etr_free_flat_buf,
662 	.sync = tmc_etr_sync_flat_buf,
663 	.get_data = tmc_etr_get_data_flat_buf,
664 };
665 
666 /*
667  * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
668  * appropriately.
669  */
670 static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
671 				struct etr_buf *etr_buf, int node,
672 				void **pages)
673 {
674 	struct etr_sg_table *etr_table;
675 	struct device *dev = &drvdata->csdev->dev;
676 
677 	etr_table = tmc_init_etr_sg_table(dev, node,
678 					  etr_buf->size, pages);
679 	if (IS_ERR(etr_table))
680 		return -ENOMEM;
681 	etr_buf->hwaddr = etr_table->hwaddr;
682 	etr_buf->mode = ETR_MODE_ETR_SG;
683 	etr_buf->private = etr_table;
684 	return 0;
685 }
686 
687 static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
688 {
689 	struct etr_sg_table *etr_table = etr_buf->private;
690 
691 	if (etr_table) {
692 		tmc_free_sg_table(etr_table->sg_table);
693 		kfree(etr_table);
694 	}
695 }
696 
697 static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
698 				       size_t len, char **bufpp)
699 {
700 	struct etr_sg_table *etr_table = etr_buf->private;
701 
702 	return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
703 }
704 
705 static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
706 {
707 	long r_offset, w_offset;
708 	struct etr_sg_table *etr_table = etr_buf->private;
709 	struct tmc_sg_table *table = etr_table->sg_table;
710 
711 	/* Convert hw address to offset in the buffer */
712 	r_offset = tmc_sg_get_data_page_offset(table, rrp);
713 	if (r_offset < 0) {
714 		dev_warn(table->dev,
715 			 "Unable to map RRP %llx to offset\n", rrp);
716 		etr_buf->len = 0;
717 		return;
718 	}
719 
720 	w_offset = tmc_sg_get_data_page_offset(table, rwp);
721 	if (w_offset < 0) {
722 		dev_warn(table->dev,
723 			 "Unable to map RWP %llx to offset\n", rwp);
724 		etr_buf->len = 0;
725 		return;
726 	}
727 
728 	etr_buf->offset = r_offset;
729 	if (etr_buf->full)
730 		etr_buf->len = etr_buf->size;
731 	else
732 		etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
733 				w_offset - r_offset;
734 	tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
735 }
736 
737 static const struct etr_buf_operations etr_sg_buf_ops = {
738 	.alloc = tmc_etr_alloc_sg_buf,
739 	.free = tmc_etr_free_sg_buf,
740 	.sync = tmc_etr_sync_sg_buf,
741 	.get_data = tmc_etr_get_data_sg_buf,
742 };
743 
744 /*
745  * TMC ETR could be connected to a CATU device, which can provide address
746  * translation service. This is represented by the Output port of the TMC
747  * (ETR) connected to the input port of the CATU.
748  *
749  * Returns	: coresight_device ptr for the CATU device if a CATU is found.
750  *		: NULL otherwise.
751  */
752 struct coresight_device *
753 tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
754 {
755 	int i;
756 	struct coresight_device *tmp, *etr = drvdata->csdev;
757 
758 	if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
759 		return NULL;
760 
761 	for (i = 0; i < etr->pdata->nr_outport; i++) {
762 		tmp = etr->pdata->conns[i].child_dev;
763 		if (tmp && coresight_is_catu_device(tmp))
764 			return tmp;
765 	}
766 
767 	return NULL;
768 }
769 
770 static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata,
771 				      struct etr_buf *etr_buf)
772 {
773 	struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
774 
775 	if (catu && helper_ops(catu)->enable)
776 		return helper_ops(catu)->enable(catu, etr_buf);
777 	return 0;
778 }
779 
780 static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata)
781 {
782 	struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
783 
784 	if (catu && helper_ops(catu)->disable)
785 		helper_ops(catu)->disable(catu, drvdata->etr_buf);
786 }
787 
788 static const struct etr_buf_operations *etr_buf_ops[] = {
789 	[ETR_MODE_FLAT] = &etr_flat_buf_ops,
790 	[ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
791 	[ETR_MODE_CATU] = IS_ENABLED(CONFIG_CORESIGHT_CATU)
792 						? &etr_catu_buf_ops : NULL,
793 };
794 
795 static inline int tmc_etr_mode_alloc_buf(int mode,
796 					 struct tmc_drvdata *drvdata,
797 					 struct etr_buf *etr_buf, int node,
798 					 void **pages)
799 {
800 	int rc = -EINVAL;
801 
802 	switch (mode) {
803 	case ETR_MODE_FLAT:
804 	case ETR_MODE_ETR_SG:
805 	case ETR_MODE_CATU:
806 		if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
807 			rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
808 						      node, pages);
809 		if (!rc)
810 			etr_buf->ops = etr_buf_ops[mode];
811 		return rc;
812 	default:
813 		return -EINVAL;
814 	}
815 }
816 
817 /*
818  * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
819  * @drvdata	: ETR device details.
820  * @size	: size of the requested buffer.
821  * @flags	: Required properties for the buffer.
822  * @node	: Node for memory allocations.
823  * @pages	: An optional list of pages.
824  */
825 static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
826 					 ssize_t size, int flags,
827 					 int node, void **pages)
828 {
829 	int rc = -ENOMEM;
830 	bool has_etr_sg, has_iommu;
831 	bool has_sg, has_catu;
832 	struct etr_buf *etr_buf;
833 	struct device *dev = &drvdata->csdev->dev;
834 
835 	has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
836 	has_iommu = iommu_get_domain_for_dev(dev->parent);
837 	has_catu = !!tmc_etr_get_catu_device(drvdata);
838 
839 	has_sg = has_catu || has_etr_sg;
840 
841 	etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
842 	if (!etr_buf)
843 		return ERR_PTR(-ENOMEM);
844 
845 	etr_buf->size = size;
846 
847 	/*
848 	 * If we have to use an existing list of pages, we cannot reliably
849 	 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
850 	 * we use the contiguous DMA memory if at least one of the following
851 	 * conditions is true:
852 	 *  a) The ETR cannot use Scatter-Gather.
853 	 *  b) we have a backing IOMMU
854 	 *  c) The requested memory size is smaller (< 1M).
855 	 *
856 	 * Fallback to available mechanisms.
857 	 *
858 	 */
859 	if (!pages &&
860 	    (!has_sg || has_iommu || size < SZ_1M))
861 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
862 					    etr_buf, node, pages);
863 	if (rc && has_etr_sg)
864 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
865 					    etr_buf, node, pages);
866 	if (rc && has_catu)
867 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
868 					    etr_buf, node, pages);
869 	if (rc) {
870 		kfree(etr_buf);
871 		return ERR_PTR(rc);
872 	}
873 
874 	refcount_set(&etr_buf->refcount, 1);
875 	dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
876 		(unsigned long)size >> 10, etr_buf->mode);
877 	return etr_buf;
878 }
879 
880 static void tmc_free_etr_buf(struct etr_buf *etr_buf)
881 {
882 	WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
883 	etr_buf->ops->free(etr_buf);
884 	kfree(etr_buf);
885 }
886 
887 /*
888  * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
889  * with a maximum of @len bytes.
890  * Returns: The size of the linear data available @pos, with *bufpp
891  * updated to point to the buffer.
892  */
893 static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
894 				    u64 offset, size_t len, char **bufpp)
895 {
896 	/* Adjust the length to limit this transaction to end of buffer */
897 	len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
898 
899 	return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
900 }
901 
902 static inline s64
903 tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
904 {
905 	ssize_t len;
906 	char *bufp;
907 
908 	len = tmc_etr_buf_get_data(etr_buf, offset,
909 				   CORESIGHT_BARRIER_PKT_SIZE, &bufp);
910 	if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE))
911 		return -EINVAL;
912 	coresight_insert_barrier_packet(bufp);
913 	return offset + CORESIGHT_BARRIER_PKT_SIZE;
914 }
915 
916 /*
917  * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
918  * Makes sure the trace data is synced to the memory for consumption.
919  * @etr_buf->offset will hold the offset to the beginning of the trace data
920  * within the buffer, with @etr_buf->len bytes to consume.
921  */
922 static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
923 {
924 	struct etr_buf *etr_buf = drvdata->etr_buf;
925 	u64 rrp, rwp;
926 	u32 status;
927 
928 	rrp = tmc_read_rrp(drvdata);
929 	rwp = tmc_read_rwp(drvdata);
930 	status = readl_relaxed(drvdata->base + TMC_STS);
931 
932 	/*
933 	 * If there were memory errors in the session, truncate the
934 	 * buffer.
935 	 */
936 	if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
937 		dev_dbg(&drvdata->csdev->dev,
938 			"tmc memory error detected, truncating buffer\n");
939 		etr_buf->len = 0;
940 		etr_buf->full = 0;
941 		return;
942 	}
943 
944 	etr_buf->full = status & TMC_STS_FULL;
945 
946 	WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
947 
948 	etr_buf->ops->sync(etr_buf, rrp, rwp);
949 }
950 
951 static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
952 {
953 	u32 axictl, sts;
954 	struct etr_buf *etr_buf = drvdata->etr_buf;
955 
956 	CS_UNLOCK(drvdata->base);
957 
958 	/* Wait for TMCSReady bit to be set */
959 	tmc_wait_for_tmcready(drvdata);
960 
961 	writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
962 	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
963 
964 	axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
965 	axictl &= ~TMC_AXICTL_CLEAR_MASK;
966 	axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16);
967 	axictl |= TMC_AXICTL_AXCACHE_OS;
968 
969 	if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
970 		axictl &= ~TMC_AXICTL_ARCACHE_MASK;
971 		axictl |= TMC_AXICTL_ARCACHE_OS;
972 	}
973 
974 	if (etr_buf->mode == ETR_MODE_ETR_SG)
975 		axictl |= TMC_AXICTL_SCT_GAT_MODE;
976 
977 	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
978 	tmc_write_dba(drvdata, etr_buf->hwaddr);
979 	/*
980 	 * If the TMC pointers must be programmed before the session,
981 	 * we have to set it properly (i.e, RRP/RWP to base address and
982 	 * STS to "not full").
983 	 */
984 	if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
985 		tmc_write_rrp(drvdata, etr_buf->hwaddr);
986 		tmc_write_rwp(drvdata, etr_buf->hwaddr);
987 		sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
988 		writel_relaxed(sts, drvdata->base + TMC_STS);
989 	}
990 
991 	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
992 		       TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
993 		       TMC_FFCR_TRIGON_TRIGIN,
994 		       drvdata->base + TMC_FFCR);
995 	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
996 	tmc_enable_hw(drvdata);
997 
998 	CS_LOCK(drvdata->base);
999 }
1000 
1001 static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1002 			     struct etr_buf *etr_buf)
1003 {
1004 	int rc;
1005 
1006 	/* Callers should provide an appropriate buffer for use */
1007 	if (WARN_ON(!etr_buf))
1008 		return -EINVAL;
1009 
1010 	if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1011 	    WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1012 		return -EINVAL;
1013 
1014 	if (WARN_ON(drvdata->etr_buf))
1015 		return -EBUSY;
1016 
1017 	/*
1018 	 * If this ETR is connected to a CATU, enable it before we turn
1019 	 * this on.
1020 	 */
1021 	rc = tmc_etr_enable_catu(drvdata, etr_buf);
1022 	if (rc)
1023 		return rc;
1024 	rc = coresight_claim_device(drvdata->base);
1025 	if (!rc) {
1026 		drvdata->etr_buf = etr_buf;
1027 		__tmc_etr_enable_hw(drvdata);
1028 	}
1029 
1030 	return rc;
1031 }
1032 
1033 /*
1034  * Return the available trace data in the buffer (starts at etr_buf->offset,
1035  * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1036  * also updating the @bufpp on where to find it. Since the trace data
1037  * starts at anywhere in the buffer, depending on the RRP, we adjust the
1038  * @len returned to handle buffer wrapping around.
1039  *
1040  * We are protected here by drvdata->reading != 0, which ensures the
1041  * sysfs_buf stays alive.
1042  */
1043 ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1044 				loff_t pos, size_t len, char **bufpp)
1045 {
1046 	s64 offset;
1047 	ssize_t actual = len;
1048 	struct etr_buf *etr_buf = drvdata->sysfs_buf;
1049 
1050 	if (pos + actual > etr_buf->len)
1051 		actual = etr_buf->len - pos;
1052 	if (actual <= 0)
1053 		return actual;
1054 
1055 	/* Compute the offset from which we read the data */
1056 	offset = etr_buf->offset + pos;
1057 	if (offset >= etr_buf->size)
1058 		offset -= etr_buf->size;
1059 	return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1060 }
1061 
1062 static struct etr_buf *
1063 tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1064 {
1065 	return tmc_alloc_etr_buf(drvdata, drvdata->size,
1066 				 0, cpu_to_node(0), NULL);
1067 }
1068 
1069 static void
1070 tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1071 {
1072 	if (buf)
1073 		tmc_free_etr_buf(buf);
1074 }
1075 
1076 static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1077 {
1078 	struct etr_buf *etr_buf = drvdata->etr_buf;
1079 
1080 	if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1081 		tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1082 		drvdata->sysfs_buf = NULL;
1083 	} else {
1084 		tmc_sync_etr_buf(drvdata);
1085 		/*
1086 		 * Insert barrier packets at the beginning, if there was
1087 		 * an overflow.
1088 		 */
1089 		if (etr_buf->full)
1090 			tmc_etr_buf_insert_barrier_packet(etr_buf,
1091 							  etr_buf->offset);
1092 	}
1093 }
1094 
1095 static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1096 {
1097 	CS_UNLOCK(drvdata->base);
1098 
1099 	tmc_flush_and_stop(drvdata);
1100 	/*
1101 	 * When operating in sysFS mode the content of the buffer needs to be
1102 	 * read before the TMC is disabled.
1103 	 */
1104 	if (drvdata->mode == CS_MODE_SYSFS)
1105 		tmc_etr_sync_sysfs_buf(drvdata);
1106 
1107 	tmc_disable_hw(drvdata);
1108 
1109 	CS_LOCK(drvdata->base);
1110 
1111 }
1112 
1113 void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1114 {
1115 	__tmc_etr_disable_hw(drvdata);
1116 	/* Disable CATU device if this ETR is connected to one */
1117 	tmc_etr_disable_catu(drvdata);
1118 	coresight_disclaim_device(drvdata->base);
1119 	/* Reset the ETR buf used by hardware */
1120 	drvdata->etr_buf = NULL;
1121 }
1122 
1123 static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1124 {
1125 	int ret = 0;
1126 	unsigned long flags;
1127 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1128 	struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1129 
1130 	/*
1131 	 * If we are enabling the ETR from disabled state, we need to make
1132 	 * sure we have a buffer with the right size. The etr_buf is not reset
1133 	 * immediately after we stop the tracing in SYSFS mode as we wait for
1134 	 * the user to collect the data. We may be able to reuse the existing
1135 	 * buffer, provided the size matches. Any allocation has to be done
1136 	 * with the lock released.
1137 	 */
1138 	spin_lock_irqsave(&drvdata->spinlock, flags);
1139 	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1140 	if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1141 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1142 
1143 		/* Allocate memory with the locks released */
1144 		free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1145 		if (IS_ERR(new_buf))
1146 			return PTR_ERR(new_buf);
1147 
1148 		/* Let's try again */
1149 		spin_lock_irqsave(&drvdata->spinlock, flags);
1150 	}
1151 
1152 	if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1153 		ret = -EBUSY;
1154 		goto out;
1155 	}
1156 
1157 	/*
1158 	 * In sysFS mode we can have multiple writers per sink.  Since this
1159 	 * sink is already enabled no memory is needed and the HW need not be
1160 	 * touched, even if the buffer size has changed.
1161 	 */
1162 	if (drvdata->mode == CS_MODE_SYSFS) {
1163 		atomic_inc(csdev->refcnt);
1164 		goto out;
1165 	}
1166 
1167 	/*
1168 	 * If we don't have a buffer or it doesn't match the requested size,
1169 	 * use the buffer allocated above. Otherwise reuse the existing buffer.
1170 	 */
1171 	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1172 	if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1173 		free_buf = sysfs_buf;
1174 		drvdata->sysfs_buf = new_buf;
1175 	}
1176 
1177 	ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf);
1178 	if (!ret) {
1179 		drvdata->mode = CS_MODE_SYSFS;
1180 		atomic_inc(csdev->refcnt);
1181 	}
1182 out:
1183 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1184 
1185 	/* Free memory outside the spinlock if need be */
1186 	if (free_buf)
1187 		tmc_etr_free_sysfs_buf(free_buf);
1188 
1189 	if (!ret)
1190 		dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1191 
1192 	return ret;
1193 }
1194 
1195 /*
1196  * alloc_etr_buf: Allocate ETR buffer for use by perf.
1197  * The size of the hardware buffer is dependent on the size configured
1198  * via sysfs and the perf ring buffer size. We prefer to allocate the
1199  * largest possible size, scaling down the size by half until it
1200  * reaches a minimum limit (1M), beyond which we give up.
1201  */
1202 static struct etr_buf *
1203 alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1204 	      int nr_pages, void **pages, bool snapshot)
1205 {
1206 	int node;
1207 	struct etr_buf *etr_buf;
1208 	unsigned long size;
1209 
1210 	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1211 	/*
1212 	 * Try to match the perf ring buffer size if it is larger
1213 	 * than the size requested via sysfs.
1214 	 */
1215 	if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1216 		etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT),
1217 					    0, node, NULL);
1218 		if (!IS_ERR(etr_buf))
1219 			goto done;
1220 	}
1221 
1222 	/*
1223 	 * Else switch to configured size for this ETR
1224 	 * and scale down until we hit the minimum limit.
1225 	 */
1226 	size = drvdata->size;
1227 	do {
1228 		etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1229 		if (!IS_ERR(etr_buf))
1230 			goto done;
1231 		size /= 2;
1232 	} while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1233 
1234 	return ERR_PTR(-ENOMEM);
1235 
1236 done:
1237 	return etr_buf;
1238 }
1239 
1240 static struct etr_buf *
1241 get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1242 			  struct perf_event *event, int nr_pages,
1243 			  void **pages, bool snapshot)
1244 {
1245 	int ret;
1246 	pid_t pid = task_pid_nr(event->owner);
1247 	struct etr_buf *etr_buf;
1248 
1249 retry:
1250 	/*
1251 	 * An etr_perf_buffer is associated with an event and holds a reference
1252 	 * to the AUX ring buffer that was created for that event.  In CPU-wide
1253 	 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1254 	 * buffer, share a sink.  As such an etr_perf_buffer is created for each
1255 	 * event but a single etr_buf associated with the ETR is shared between
1256 	 * them.  The last event in a trace session will copy the content of the
1257 	 * etr_buf to its AUX ring buffer.  Ring buffer associated to other
1258 	 * events are simply not used an freed as events are destoyed.  We still
1259 	 * need to allocate a ring buffer for each event since we don't know
1260 	 * which event will be last.
1261 	 */
1262 
1263 	/*
1264 	 * The first thing to do here is check if an etr_buf has already been
1265 	 * allocated for this session.  If so it is shared with this event,
1266 	 * otherwise it is created.
1267 	 */
1268 	mutex_lock(&drvdata->idr_mutex);
1269 	etr_buf = idr_find(&drvdata->idr, pid);
1270 	if (etr_buf) {
1271 		refcount_inc(&etr_buf->refcount);
1272 		mutex_unlock(&drvdata->idr_mutex);
1273 		return etr_buf;
1274 	}
1275 
1276 	/* If we made it here no buffer has been allocated, do so now. */
1277 	mutex_unlock(&drvdata->idr_mutex);
1278 
1279 	etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1280 	if (IS_ERR(etr_buf))
1281 		return etr_buf;
1282 
1283 	/* Now that we have a buffer, add it to the IDR. */
1284 	mutex_lock(&drvdata->idr_mutex);
1285 	ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1286 	mutex_unlock(&drvdata->idr_mutex);
1287 
1288 	/* Another event with this session ID has allocated this buffer. */
1289 	if (ret == -ENOSPC) {
1290 		tmc_free_etr_buf(etr_buf);
1291 		goto retry;
1292 	}
1293 
1294 	/* The IDR can't allocate room for a new session, abandon ship. */
1295 	if (ret == -ENOMEM) {
1296 		tmc_free_etr_buf(etr_buf);
1297 		return ERR_PTR(ret);
1298 	}
1299 
1300 
1301 	return etr_buf;
1302 }
1303 
1304 static struct etr_buf *
1305 get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1306 			    struct perf_event *event, int nr_pages,
1307 			    void **pages, bool snapshot)
1308 {
1309 	/*
1310 	 * In per-thread mode the etr_buf isn't shared, so just go ahead
1311 	 * with memory allocation.
1312 	 */
1313 	return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1314 }
1315 
1316 static struct etr_buf *
1317 get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1318 		 int nr_pages, void **pages, bool snapshot)
1319 {
1320 	if (event->cpu == -1)
1321 		return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1322 						   pages, snapshot);
1323 
1324 	return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1325 					 pages, snapshot);
1326 }
1327 
1328 static struct etr_perf_buffer *
1329 tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1330 		       int nr_pages, void **pages, bool snapshot)
1331 {
1332 	int node;
1333 	struct etr_buf *etr_buf;
1334 	struct etr_perf_buffer *etr_perf;
1335 
1336 	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1337 
1338 	etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1339 	if (!etr_perf)
1340 		return ERR_PTR(-ENOMEM);
1341 
1342 	etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1343 	if (!IS_ERR(etr_buf))
1344 		goto done;
1345 
1346 	kfree(etr_perf);
1347 	return ERR_PTR(-ENOMEM);
1348 
1349 done:
1350 	/*
1351 	 * Keep a reference to the ETR this buffer has been allocated for
1352 	 * in order to have access to the IDR in tmc_free_etr_buffer().
1353 	 */
1354 	etr_perf->drvdata = drvdata;
1355 	etr_perf->etr_buf = etr_buf;
1356 
1357 	return etr_perf;
1358 }
1359 
1360 
1361 static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1362 				  struct perf_event *event, void **pages,
1363 				  int nr_pages, bool snapshot)
1364 {
1365 	struct etr_perf_buffer *etr_perf;
1366 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1367 
1368 	etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1369 					  nr_pages, pages, snapshot);
1370 	if (IS_ERR(etr_perf)) {
1371 		dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1372 		return NULL;
1373 	}
1374 
1375 	etr_perf->pid = task_pid_nr(event->owner);
1376 	etr_perf->snapshot = snapshot;
1377 	etr_perf->nr_pages = nr_pages;
1378 	etr_perf->pages = pages;
1379 
1380 	return etr_perf;
1381 }
1382 
1383 static void tmc_free_etr_buffer(void *config)
1384 {
1385 	struct etr_perf_buffer *etr_perf = config;
1386 	struct tmc_drvdata *drvdata = etr_perf->drvdata;
1387 	struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1388 
1389 	if (!etr_buf)
1390 		goto free_etr_perf_buffer;
1391 
1392 	mutex_lock(&drvdata->idr_mutex);
1393 	/* If we are not the last one to use the buffer, don't touch it. */
1394 	if (!refcount_dec_and_test(&etr_buf->refcount)) {
1395 		mutex_unlock(&drvdata->idr_mutex);
1396 		goto free_etr_perf_buffer;
1397 	}
1398 
1399 	/* We are the last one, remove from the IDR and free the buffer. */
1400 	buf = idr_remove(&drvdata->idr, etr_perf->pid);
1401 	mutex_unlock(&drvdata->idr_mutex);
1402 
1403 	/*
1404 	 * Something went very wrong if the buffer associated with this ID
1405 	 * is not the same in the IDR.  Leak to avoid use after free.
1406 	 */
1407 	if (buf && WARN_ON(buf != etr_buf))
1408 		goto free_etr_perf_buffer;
1409 
1410 	tmc_free_etr_buf(etr_perf->etr_buf);
1411 
1412 free_etr_perf_buffer:
1413 	kfree(etr_perf);
1414 }
1415 
1416 /*
1417  * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1418  * buffer to the perf ring buffer.
1419  */
1420 static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1421 				     unsigned long src_offset,
1422 				     unsigned long to_copy)
1423 {
1424 	long bytes;
1425 	long pg_idx, pg_offset;
1426 	unsigned long head = etr_perf->head;
1427 	char **dst_pages, *src_buf;
1428 	struct etr_buf *etr_buf = etr_perf->etr_buf;
1429 
1430 	head = etr_perf->head;
1431 	pg_idx = head >> PAGE_SHIFT;
1432 	pg_offset = head & (PAGE_SIZE - 1);
1433 	dst_pages = (char **)etr_perf->pages;
1434 
1435 	while (to_copy > 0) {
1436 		/*
1437 		 * In one iteration, we can copy minimum of :
1438 		 *  1) what is available in the source buffer,
1439 		 *  2) what is available in the source buffer, before it
1440 		 *     wraps around.
1441 		 *  3) what is available in the destination page.
1442 		 * in one iteration.
1443 		 */
1444 		if (src_offset >= etr_buf->size)
1445 			src_offset -= etr_buf->size;
1446 		bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1447 					     &src_buf);
1448 		if (WARN_ON_ONCE(bytes <= 0))
1449 			break;
1450 		bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1451 
1452 		memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1453 
1454 		to_copy -= bytes;
1455 
1456 		/* Move destination pointers */
1457 		pg_offset += bytes;
1458 		if (pg_offset == PAGE_SIZE) {
1459 			pg_offset = 0;
1460 			if (++pg_idx == etr_perf->nr_pages)
1461 				pg_idx = 0;
1462 		}
1463 
1464 		/* Move source pointers */
1465 		src_offset += bytes;
1466 	}
1467 }
1468 
1469 /*
1470  * tmc_update_etr_buffer : Update the perf ring buffer with the
1471  * available trace data. We use software double buffering at the moment.
1472  *
1473  * TODO: Add support for reusing the perf ring buffer.
1474  */
1475 static unsigned long
1476 tmc_update_etr_buffer(struct coresight_device *csdev,
1477 		      struct perf_output_handle *handle,
1478 		      void *config)
1479 {
1480 	bool lost = false;
1481 	unsigned long flags, offset, size = 0;
1482 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1483 	struct etr_perf_buffer *etr_perf = config;
1484 	struct etr_buf *etr_buf = etr_perf->etr_buf;
1485 
1486 	spin_lock_irqsave(&drvdata->spinlock, flags);
1487 
1488 	/* Don't do anything if another tracer is using this sink */
1489 	if (atomic_read(csdev->refcnt) != 1) {
1490 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1491 		goto out;
1492 	}
1493 
1494 	if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1495 		lost = true;
1496 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1497 		goto out;
1498 	}
1499 
1500 	CS_UNLOCK(drvdata->base);
1501 
1502 	tmc_flush_and_stop(drvdata);
1503 	tmc_sync_etr_buf(drvdata);
1504 
1505 	CS_LOCK(drvdata->base);
1506 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1507 
1508 	lost = etr_buf->full;
1509 	offset = etr_buf->offset;
1510 	size = etr_buf->len;
1511 
1512 	/*
1513 	 * The ETR buffer may be bigger than the space available in the
1514 	 * perf ring buffer (handle->size).  If so advance the offset so that we
1515 	 * get the latest trace data.  In snapshot mode none of that matters
1516 	 * since we are expected to clobber stale data in favour of the latest
1517 	 * traces.
1518 	 */
1519 	if (!etr_perf->snapshot && size > handle->size) {
1520 		u32 mask = tmc_get_memwidth_mask(drvdata);
1521 
1522 		/*
1523 		 * Make sure the new size is aligned in accordance with the
1524 		 * requirement explained in function tmc_get_memwidth_mask().
1525 		 */
1526 		size = handle->size & mask;
1527 		offset = etr_buf->offset + etr_buf->len - size;
1528 
1529 		if (offset >= etr_buf->size)
1530 			offset -= etr_buf->size;
1531 		lost = true;
1532 	}
1533 
1534 	/* Insert barrier packets at the beginning, if there was an overflow */
1535 	if (lost)
1536 		tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset);
1537 	tmc_etr_sync_perf_buffer(etr_perf, offset, size);
1538 
1539 	/*
1540 	 * In snapshot mode we simply increment the head by the number of byte
1541 	 * that were written.  User space function  cs_etm_find_snapshot() will
1542 	 * figure out how many bytes to get from the AUX buffer based on the
1543 	 * position of the head.
1544 	 */
1545 	if (etr_perf->snapshot)
1546 		handle->head += size;
1547 out:
1548 	/*
1549 	 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1550 	 * captured buffer is expected to be truncated and 2) a full buffer
1551 	 * prevents the event from being re-enabled by the perf core,
1552 	 * resulting in stale data being send to user space.
1553 	 */
1554 	if (!etr_perf->snapshot && lost)
1555 		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1556 	return size;
1557 }
1558 
1559 static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1560 {
1561 	int rc = 0;
1562 	pid_t pid;
1563 	unsigned long flags;
1564 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1565 	struct perf_output_handle *handle = data;
1566 	struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1567 
1568 	spin_lock_irqsave(&drvdata->spinlock, flags);
1569 	 /* Don't use this sink if it is already claimed by sysFS */
1570 	if (drvdata->mode == CS_MODE_SYSFS) {
1571 		rc = -EBUSY;
1572 		goto unlock_out;
1573 	}
1574 
1575 	if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1576 		rc = -EINVAL;
1577 		goto unlock_out;
1578 	}
1579 
1580 	/* Get a handle on the pid of the process to monitor */
1581 	pid = etr_perf->pid;
1582 
1583 	/* Do not proceed if this device is associated with another session */
1584 	if (drvdata->pid != -1 && drvdata->pid != pid) {
1585 		rc = -EBUSY;
1586 		goto unlock_out;
1587 	}
1588 
1589 	etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf);
1590 
1591 	/*
1592 	 * No HW configuration is needed if the sink is already in
1593 	 * use for this session.
1594 	 */
1595 	if (drvdata->pid == pid) {
1596 		atomic_inc(csdev->refcnt);
1597 		goto unlock_out;
1598 	}
1599 
1600 	rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1601 	if (!rc) {
1602 		/* Associate with monitored process. */
1603 		drvdata->pid = pid;
1604 		drvdata->mode = CS_MODE_PERF;
1605 		drvdata->perf_buf = etr_perf->etr_buf;
1606 		atomic_inc(csdev->refcnt);
1607 	}
1608 
1609 unlock_out:
1610 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1611 	return rc;
1612 }
1613 
1614 static int tmc_enable_etr_sink(struct coresight_device *csdev,
1615 			       u32 mode, void *data)
1616 {
1617 	switch (mode) {
1618 	case CS_MODE_SYSFS:
1619 		return tmc_enable_etr_sink_sysfs(csdev);
1620 	case CS_MODE_PERF:
1621 		return tmc_enable_etr_sink_perf(csdev, data);
1622 	}
1623 
1624 	/* We shouldn't be here */
1625 	return -EINVAL;
1626 }
1627 
1628 static int tmc_disable_etr_sink(struct coresight_device *csdev)
1629 {
1630 	unsigned long flags;
1631 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1632 
1633 	spin_lock_irqsave(&drvdata->spinlock, flags);
1634 
1635 	if (drvdata->reading) {
1636 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1637 		return -EBUSY;
1638 	}
1639 
1640 	if (atomic_dec_return(csdev->refcnt)) {
1641 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
1642 		return -EBUSY;
1643 	}
1644 
1645 	/* Complain if we (somehow) got out of sync */
1646 	WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1647 	tmc_etr_disable_hw(drvdata);
1648 	/* Dissociate from monitored process. */
1649 	drvdata->pid = -1;
1650 	drvdata->mode = CS_MODE_DISABLED;
1651 	/* Reset perf specific data */
1652 	drvdata->perf_buf = NULL;
1653 
1654 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1655 
1656 	dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1657 	return 0;
1658 }
1659 
1660 static const struct coresight_ops_sink tmc_etr_sink_ops = {
1661 	.enable		= tmc_enable_etr_sink,
1662 	.disable	= tmc_disable_etr_sink,
1663 	.alloc_buffer	= tmc_alloc_etr_buffer,
1664 	.update_buffer	= tmc_update_etr_buffer,
1665 	.free_buffer	= tmc_free_etr_buffer,
1666 };
1667 
1668 const struct coresight_ops tmc_etr_cs_ops = {
1669 	.sink_ops	= &tmc_etr_sink_ops,
1670 };
1671 
1672 int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1673 {
1674 	int ret = 0;
1675 	unsigned long flags;
1676 
1677 	/* config types are set a boot time and never change */
1678 	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1679 		return -EINVAL;
1680 
1681 	spin_lock_irqsave(&drvdata->spinlock, flags);
1682 	if (drvdata->reading) {
1683 		ret = -EBUSY;
1684 		goto out;
1685 	}
1686 
1687 	/*
1688 	 * We can safely allow reads even if the ETR is operating in PERF mode,
1689 	 * since the sysfs session is captured in mode specific data.
1690 	 * If drvdata::sysfs_data is NULL the trace data has been read already.
1691 	 */
1692 	if (!drvdata->sysfs_buf) {
1693 		ret = -EINVAL;
1694 		goto out;
1695 	}
1696 
1697 	/* Disable the TMC if we are trying to read from a running session. */
1698 	if (drvdata->mode == CS_MODE_SYSFS)
1699 		__tmc_etr_disable_hw(drvdata);
1700 
1701 	drvdata->reading = true;
1702 out:
1703 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1704 
1705 	return ret;
1706 }
1707 
1708 int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1709 {
1710 	unsigned long flags;
1711 	struct etr_buf *sysfs_buf = NULL;
1712 
1713 	/* config types are set a boot time and never change */
1714 	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1715 		return -EINVAL;
1716 
1717 	spin_lock_irqsave(&drvdata->spinlock, flags);
1718 
1719 	/* RE-enable the TMC if need be */
1720 	if (drvdata->mode == CS_MODE_SYSFS) {
1721 		/*
1722 		 * The trace run will continue with the same allocated trace
1723 		 * buffer. Since the tracer is still enabled drvdata::buf can't
1724 		 * be NULL.
1725 		 */
1726 		__tmc_etr_enable_hw(drvdata);
1727 	} else {
1728 		/*
1729 		 * The ETR is not tracing and the buffer was just read.
1730 		 * As such prepare to free the trace buffer.
1731 		 */
1732 		sysfs_buf = drvdata->sysfs_buf;
1733 		drvdata->sysfs_buf = NULL;
1734 	}
1735 
1736 	drvdata->reading = false;
1737 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
1738 
1739 	/* Free allocated memory out side of the spinlock */
1740 	if (sysfs_buf)
1741 		tmc_etr_free_sysfs_buf(sysfs_buf);
1742 
1743 	return 0;
1744 }
1745