1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright(C) 2016 Linaro Limited. All rights reserved.
4  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5  */
6 
7 #include <linux/atomic.h>
8 #include <linux/circ_buf.h>
9 #include <linux/coresight.h>
10 #include <linux/perf_event.h>
11 #include <linux/slab.h>
12 #include "coresight-priv.h"
13 #include "coresight-tmc.h"
14 #include "coresight-etm-perf.h"
15 
16 static int tmc_set_etf_buffer(struct coresight_device *csdev,
17 			      struct perf_output_handle *handle);
18 
19 static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
20 {
21 	CS_UNLOCK(drvdata->base);
22 
23 	/* Wait for TMCSReady bit to be set */
24 	tmc_wait_for_tmcready(drvdata);
25 
26 	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
27 	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
28 		       TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
29 		       TMC_FFCR_TRIGON_TRIGIN,
30 		       drvdata->base + TMC_FFCR);
31 
32 	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
33 	tmc_enable_hw(drvdata);
34 
35 	CS_LOCK(drvdata->base);
36 }
37 
38 static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
39 {
40 	int rc = coresight_claim_device(drvdata->base);
41 
42 	if (rc)
43 		return rc;
44 
45 	__tmc_etb_enable_hw(drvdata);
46 	return 0;
47 }
48 
49 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
50 {
51 	char *bufp;
52 	u32 read_data, lost;
53 
54 	/* Check if the buffer wrapped around. */
55 	lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
56 	bufp = drvdata->buf;
57 	drvdata->len = 0;
58 	while (1) {
59 		read_data = readl_relaxed(drvdata->base + TMC_RRD);
60 		if (read_data == 0xFFFFFFFF)
61 			break;
62 		memcpy(bufp, &read_data, 4);
63 		bufp += 4;
64 		drvdata->len += 4;
65 	}
66 
67 	if (lost)
68 		coresight_insert_barrier_packet(drvdata->buf);
69 	return;
70 }
71 
72 static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
73 {
74 	CS_UNLOCK(drvdata->base);
75 
76 	tmc_flush_and_stop(drvdata);
77 	/*
78 	 * When operating in sysFS mode the content of the buffer needs to be
79 	 * read before the TMC is disabled.
80 	 */
81 	if (drvdata->mode == CS_MODE_SYSFS)
82 		tmc_etb_dump_hw(drvdata);
83 	tmc_disable_hw(drvdata);
84 
85 	CS_LOCK(drvdata->base);
86 }
87 
88 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
89 {
90 	__tmc_etb_disable_hw(drvdata);
91 	coresight_disclaim_device(drvdata->base);
92 }
93 
94 static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
95 {
96 	CS_UNLOCK(drvdata->base);
97 
98 	/* Wait for TMCSReady bit to be set */
99 	tmc_wait_for_tmcready(drvdata);
100 
101 	writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
102 	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
103 		       drvdata->base + TMC_FFCR);
104 	writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
105 	tmc_enable_hw(drvdata);
106 
107 	CS_LOCK(drvdata->base);
108 }
109 
110 static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
111 {
112 	int rc = coresight_claim_device(drvdata->base);
113 
114 	if (rc)
115 		return rc;
116 
117 	__tmc_etf_enable_hw(drvdata);
118 	return 0;
119 }
120 
121 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
122 {
123 	CS_UNLOCK(drvdata->base);
124 
125 	tmc_flush_and_stop(drvdata);
126 	tmc_disable_hw(drvdata);
127 	coresight_disclaim_device_unlocked(drvdata->base);
128 	CS_LOCK(drvdata->base);
129 }
130 
131 /*
132  * Return the available trace data in the buffer from @pos, with
133  * a maximum limit of @len, updating the @bufpp on where to
134  * find it.
135  */
136 ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
137 				loff_t pos, size_t len, char **bufpp)
138 {
139 	ssize_t actual = len;
140 
141 	/* Adjust the len to available size @pos */
142 	if (pos + actual > drvdata->len)
143 		actual = drvdata->len - pos;
144 	if (actual > 0)
145 		*bufpp = drvdata->buf + pos;
146 	return actual;
147 }
148 
149 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
150 {
151 	int ret = 0;
152 	bool used = false;
153 	char *buf = NULL;
154 	unsigned long flags;
155 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
156 
157 	/*
158 	 * If we don't have a buffer release the lock and allocate memory.
159 	 * Otherwise keep the lock and move along.
160 	 */
161 	spin_lock_irqsave(&drvdata->spinlock, flags);
162 	if (!drvdata->buf) {
163 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
164 
165 		/* Allocating the memory here while outside of the spinlock */
166 		buf = kzalloc(drvdata->size, GFP_KERNEL);
167 		if (!buf)
168 			return -ENOMEM;
169 
170 		/* Let's try again */
171 		spin_lock_irqsave(&drvdata->spinlock, flags);
172 	}
173 
174 	if (drvdata->reading) {
175 		ret = -EBUSY;
176 		goto out;
177 	}
178 
179 	/*
180 	 * In sysFS mode we can have multiple writers per sink.  Since this
181 	 * sink is already enabled no memory is needed and the HW need not be
182 	 * touched.
183 	 */
184 	if (drvdata->mode == CS_MODE_SYSFS) {
185 		atomic_inc(csdev->refcnt);
186 		goto out;
187 	}
188 
189 	/*
190 	 * If drvdata::buf isn't NULL, memory was allocated for a previous
191 	 * trace run but wasn't read.  If so simply zero-out the memory.
192 	 * Otherwise use the memory allocated above.
193 	 *
194 	 * The memory is freed when users read the buffer using the
195 	 * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
196 	 * details.
197 	 */
198 	if (drvdata->buf) {
199 		memset(drvdata->buf, 0, drvdata->size);
200 	} else {
201 		used = true;
202 		drvdata->buf = buf;
203 	}
204 
205 	ret = tmc_etb_enable_hw(drvdata);
206 	if (!ret) {
207 		drvdata->mode = CS_MODE_SYSFS;
208 		atomic_inc(csdev->refcnt);
209 	} else {
210 		/* Free up the buffer if we failed to enable */
211 		used = false;
212 	}
213 out:
214 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
215 
216 	/* Free memory outside the spinlock if need be */
217 	if (!used)
218 		kfree(buf);
219 
220 	return ret;
221 }
222 
223 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)
224 {
225 	int ret = 0;
226 	pid_t pid;
227 	unsigned long flags;
228 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
229 	struct perf_output_handle *handle = data;
230 
231 	spin_lock_irqsave(&drvdata->spinlock, flags);
232 	do {
233 		ret = -EINVAL;
234 		if (drvdata->reading)
235 			break;
236 		/*
237 		 * No need to continue if the ETB/ETF is already operated
238 		 * from sysFS.
239 		 */
240 		if (drvdata->mode == CS_MODE_SYSFS) {
241 			ret = -EBUSY;
242 			break;
243 		}
244 
245 		/* Get a handle on the pid of the process to monitor */
246 		pid = task_pid_nr(handle->event->owner);
247 
248 		if (drvdata->pid != -1 && drvdata->pid != pid) {
249 			ret = -EBUSY;
250 			break;
251 		}
252 
253 		ret = tmc_set_etf_buffer(csdev, handle);
254 		if (ret)
255 			break;
256 
257 		/*
258 		 * No HW configuration is needed if the sink is already in
259 		 * use for this session.
260 		 */
261 		if (drvdata->pid == pid) {
262 			atomic_inc(csdev->refcnt);
263 			break;
264 		}
265 
266 		ret  = tmc_etb_enable_hw(drvdata);
267 		if (!ret) {
268 			/* Associate with monitored process. */
269 			drvdata->pid = pid;
270 			drvdata->mode = CS_MODE_PERF;
271 			atomic_inc(csdev->refcnt);
272 		}
273 	} while (0);
274 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
275 
276 	return ret;
277 }
278 
279 static int tmc_enable_etf_sink(struct coresight_device *csdev,
280 			       u32 mode, void *data)
281 {
282 	int ret;
283 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
284 
285 	switch (mode) {
286 	case CS_MODE_SYSFS:
287 		ret = tmc_enable_etf_sink_sysfs(csdev);
288 		break;
289 	case CS_MODE_PERF:
290 		ret = tmc_enable_etf_sink_perf(csdev, data);
291 		break;
292 	/* We shouldn't be here */
293 	default:
294 		ret = -EINVAL;
295 		break;
296 	}
297 
298 	if (ret)
299 		return ret;
300 
301 	dev_dbg(drvdata->dev, "TMC-ETB/ETF enabled\n");
302 	return 0;
303 }
304 
305 static int tmc_disable_etf_sink(struct coresight_device *csdev)
306 {
307 	unsigned long flags;
308 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
309 
310 	spin_lock_irqsave(&drvdata->spinlock, flags);
311 
312 	if (drvdata->reading) {
313 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
314 		return -EBUSY;
315 	}
316 
317 	if (atomic_dec_return(csdev->refcnt)) {
318 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
319 		return -EBUSY;
320 	}
321 
322 	/* Complain if we (somehow) got out of sync */
323 	WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
324 	tmc_etb_disable_hw(drvdata);
325 	/* Dissociate from monitored process. */
326 	drvdata->pid = -1;
327 	drvdata->mode = CS_MODE_DISABLED;
328 
329 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
330 
331 	dev_dbg(drvdata->dev, "TMC-ETB/ETF disabled\n");
332 	return 0;
333 }
334 
335 static int tmc_enable_etf_link(struct coresight_device *csdev,
336 			       int inport, int outport)
337 {
338 	int ret;
339 	unsigned long flags;
340 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
341 
342 	spin_lock_irqsave(&drvdata->spinlock, flags);
343 	if (drvdata->reading) {
344 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
345 		return -EBUSY;
346 	}
347 
348 	ret = tmc_etf_enable_hw(drvdata);
349 	if (!ret)
350 		drvdata->mode = CS_MODE_SYSFS;
351 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
352 
353 	if (!ret)
354 		dev_dbg(drvdata->dev, "TMC-ETF enabled\n");
355 	return ret;
356 }
357 
358 static void tmc_disable_etf_link(struct coresight_device *csdev,
359 				 int inport, int outport)
360 {
361 	unsigned long flags;
362 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
363 
364 	spin_lock_irqsave(&drvdata->spinlock, flags);
365 	if (drvdata->reading) {
366 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
367 		return;
368 	}
369 
370 	tmc_etf_disable_hw(drvdata);
371 	drvdata->mode = CS_MODE_DISABLED;
372 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
373 
374 	dev_dbg(drvdata->dev, "TMC-ETF disabled\n");
375 }
376 
377 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,
378 				  struct perf_event *event, void **pages,
379 				  int nr_pages, bool overwrite)
380 {
381 	int node, cpu = event->cpu;
382 	struct cs_buffers *buf;
383 
384 	if (cpu == -1)
385 		cpu = smp_processor_id();
386 	node = cpu_to_node(cpu);
387 
388 	/* Allocate memory structure for interaction with Perf */
389 	buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
390 	if (!buf)
391 		return NULL;
392 
393 	buf->snapshot = overwrite;
394 	buf->nr_pages = nr_pages;
395 	buf->data_pages = pages;
396 
397 	return buf;
398 }
399 
400 static void tmc_free_etf_buffer(void *config)
401 {
402 	struct cs_buffers *buf = config;
403 
404 	kfree(buf);
405 }
406 
407 static int tmc_set_etf_buffer(struct coresight_device *csdev,
408 			      struct perf_output_handle *handle)
409 {
410 	int ret = 0;
411 	unsigned long head;
412 	struct cs_buffers *buf = etm_perf_sink_config(handle);
413 
414 	if (!buf)
415 		return -EINVAL;
416 
417 	/* wrap head around to the amount of space we have */
418 	head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
419 
420 	/* find the page to write to */
421 	buf->cur = head / PAGE_SIZE;
422 
423 	/* and offset within that page */
424 	buf->offset = head % PAGE_SIZE;
425 
426 	local_set(&buf->data_size, 0);
427 
428 	return ret;
429 }
430 
431 static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
432 				  struct perf_output_handle *handle,
433 				  void *sink_config)
434 {
435 	bool lost = false;
436 	int i, cur;
437 	const u32 *barrier;
438 	u32 *buf_ptr;
439 	u64 read_ptr, write_ptr;
440 	u32 status;
441 	unsigned long offset, to_read = 0, flags;
442 	struct cs_buffers *buf = sink_config;
443 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
444 
445 	if (!buf)
446 		return 0;
447 
448 	/* This shouldn't happen */
449 	if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
450 		return 0;
451 
452 	spin_lock_irqsave(&drvdata->spinlock, flags);
453 
454 	/* Don't do anything if another tracer is using this sink */
455 	if (atomic_read(csdev->refcnt) != 1)
456 		goto out;
457 
458 	CS_UNLOCK(drvdata->base);
459 
460 	tmc_flush_and_stop(drvdata);
461 
462 	read_ptr = tmc_read_rrp(drvdata);
463 	write_ptr = tmc_read_rwp(drvdata);
464 
465 	/*
466 	 * Get a hold of the status register and see if a wrap around
467 	 * has occurred.  If so adjust things accordingly.
468 	 */
469 	status = readl_relaxed(drvdata->base + TMC_STS);
470 	if (status & TMC_STS_FULL) {
471 		lost = true;
472 		to_read = drvdata->size;
473 	} else {
474 		to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
475 	}
476 
477 	/*
478 	 * The TMC RAM buffer may be bigger than the space available in the
479 	 * perf ring buffer (handle->size).  If so advance the RRP so that we
480 	 * get the latest trace data.
481 	 */
482 	if (to_read > handle->size) {
483 		u32 mask = 0;
484 
485 		/*
486 		 * The value written to RRP must be byte-address aligned to
487 		 * the width of the trace memory databus _and_ to a frame
488 		 * boundary (16 byte), whichever is the biggest. For example,
489 		 * for 32-bit, 64-bit and 128-bit wide trace memory, the four
490 		 * LSBs must be 0s. For 256-bit wide trace memory, the five
491 		 * LSBs must be 0s.
492 		 */
493 		switch (drvdata->memwidth) {
494 		case TMC_MEM_INTF_WIDTH_32BITS:
495 		case TMC_MEM_INTF_WIDTH_64BITS:
496 		case TMC_MEM_INTF_WIDTH_128BITS:
497 			mask = GENMASK(31, 4);
498 			break;
499 		case TMC_MEM_INTF_WIDTH_256BITS:
500 			mask = GENMASK(31, 5);
501 			break;
502 		}
503 
504 		/*
505 		 * Make sure the new size is aligned in accordance with the
506 		 * requirement explained above.
507 		 */
508 		to_read = handle->size & mask;
509 		/* Move the RAM read pointer up */
510 		read_ptr = (write_ptr + drvdata->size) - to_read;
511 		/* Make sure we are still within our limits */
512 		if (read_ptr > (drvdata->size - 1))
513 			read_ptr -= drvdata->size;
514 		/* Tell the HW */
515 		tmc_write_rrp(drvdata, read_ptr);
516 		lost = true;
517 	}
518 
519 	if (lost)
520 		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
521 
522 	cur = buf->cur;
523 	offset = buf->offset;
524 	barrier = barrier_pkt;
525 
526 	/* for every byte to read */
527 	for (i = 0; i < to_read; i += 4) {
528 		buf_ptr = buf->data_pages[cur] + offset;
529 		*buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
530 
531 		if (lost && *barrier) {
532 			*buf_ptr = *barrier;
533 			barrier++;
534 		}
535 
536 		offset += 4;
537 		if (offset >= PAGE_SIZE) {
538 			offset = 0;
539 			cur++;
540 			/* wrap around at the end of the buffer */
541 			cur &= buf->nr_pages - 1;
542 		}
543 	}
544 
545 	/* In snapshot mode we have to update the head */
546 	if (buf->snapshot) {
547 		handle->head = (cur * PAGE_SIZE) + offset;
548 		to_read = buf->nr_pages << PAGE_SHIFT;
549 	}
550 	CS_LOCK(drvdata->base);
551 out:
552 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
553 
554 	return to_read;
555 }
556 
557 static const struct coresight_ops_sink tmc_etf_sink_ops = {
558 	.enable		= tmc_enable_etf_sink,
559 	.disable	= tmc_disable_etf_sink,
560 	.alloc_buffer	= tmc_alloc_etf_buffer,
561 	.free_buffer	= tmc_free_etf_buffer,
562 	.update_buffer	= tmc_update_etf_buffer,
563 };
564 
565 static const struct coresight_ops_link tmc_etf_link_ops = {
566 	.enable		= tmc_enable_etf_link,
567 	.disable	= tmc_disable_etf_link,
568 };
569 
570 const struct coresight_ops tmc_etb_cs_ops = {
571 	.sink_ops	= &tmc_etf_sink_ops,
572 };
573 
574 const struct coresight_ops tmc_etf_cs_ops = {
575 	.sink_ops	= &tmc_etf_sink_ops,
576 	.link_ops	= &tmc_etf_link_ops,
577 };
578 
579 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
580 {
581 	enum tmc_mode mode;
582 	int ret = 0;
583 	unsigned long flags;
584 
585 	/* config types are set a boot time and never change */
586 	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
587 			 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
588 		return -EINVAL;
589 
590 	spin_lock_irqsave(&drvdata->spinlock, flags);
591 
592 	if (drvdata->reading) {
593 		ret = -EBUSY;
594 		goto out;
595 	}
596 
597 	/* There is no point in reading a TMC in HW FIFO mode */
598 	mode = readl_relaxed(drvdata->base + TMC_MODE);
599 	if (mode != TMC_MODE_CIRCULAR_BUFFER) {
600 		ret = -EINVAL;
601 		goto out;
602 	}
603 
604 	/* Don't interfere if operated from Perf */
605 	if (drvdata->mode == CS_MODE_PERF) {
606 		ret = -EINVAL;
607 		goto out;
608 	}
609 
610 	/* If drvdata::buf is NULL the trace data has been read already */
611 	if (drvdata->buf == NULL) {
612 		ret = -EINVAL;
613 		goto out;
614 	}
615 
616 	/* Disable the TMC if need be */
617 	if (drvdata->mode == CS_MODE_SYSFS)
618 		__tmc_etb_disable_hw(drvdata);
619 
620 	drvdata->reading = true;
621 out:
622 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
623 
624 	return ret;
625 }
626 
627 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
628 {
629 	char *buf = NULL;
630 	enum tmc_mode mode;
631 	unsigned long flags;
632 
633 	/* config types are set a boot time and never change */
634 	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
635 			 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
636 		return -EINVAL;
637 
638 	spin_lock_irqsave(&drvdata->spinlock, flags);
639 
640 	/* There is no point in reading a TMC in HW FIFO mode */
641 	mode = readl_relaxed(drvdata->base + TMC_MODE);
642 	if (mode != TMC_MODE_CIRCULAR_BUFFER) {
643 		spin_unlock_irqrestore(&drvdata->spinlock, flags);
644 		return -EINVAL;
645 	}
646 
647 	/* Re-enable the TMC if need be */
648 	if (drvdata->mode == CS_MODE_SYSFS) {
649 		/*
650 		 * The trace run will continue with the same allocated trace
651 		 * buffer. As such zero-out the buffer so that we don't end
652 		 * up with stale data.
653 		 *
654 		 * Since the tracer is still enabled drvdata::buf
655 		 * can't be NULL.
656 		 */
657 		memset(drvdata->buf, 0, drvdata->size);
658 		__tmc_etb_enable_hw(drvdata);
659 	} else {
660 		/*
661 		 * The ETB/ETF is not tracing and the buffer was just read.
662 		 * As such prepare to free the trace buffer.
663 		 */
664 		buf = drvdata->buf;
665 		drvdata->buf = NULL;
666 	}
667 
668 	drvdata->reading = false;
669 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
670 
671 	/*
672 	 * Free allocated memory outside of the spinlock.  There is no need
673 	 * to assert the validity of 'buf' since calling kfree(NULL) is safe.
674 	 */
675 	kfree(buf);
676 
677 	return 0;
678 }
679