1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
4  * sink device could then pair with an appropriate per-cpu coresight source
5  * device (ETE) thus generating required trace data. Trace can be enabled
6  * via the perf framework.
7  *
8  * The AUX buffer handling is inspired from Arm SPE PMU driver.
9  *
10  * Copyright (C) 2020 ARM Ltd.
11  *
12  * Author: Anshuman Khandual <anshuman.khandual@arm.com>
13  */
14 #define DRVNAME "arm_trbe"
15 
16 #define pr_fmt(fmt) DRVNAME ": " fmt
17 
18 #include <asm/barrier.h>
19 #include <asm/cpufeature.h>
20 
21 #include "coresight-self-hosted-trace.h"
22 #include "coresight-trbe.h"
23 
24 #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
25 
26 /*
27  * A padding packet that will help the user space tools
28  * in skipping relevant sections in the captured trace
29  * data which could not be decoded. TRBE doesn't support
30  * formatting the trace data, unlike the legacy CoreSight
31  * sinks and thus we use ETE trace packets to pad the
32  * sections of the buffer.
33  */
34 #define ETE_IGNORE_PACKET		0x70
35 
36 /*
37  * Minimum amount of meaningful trace will contain:
38  * A-Sync, Trace Info, Trace On, Address, Atom.
39  * This is about 44bytes of ETE trace. To be on
40  * the safer side, we assume 64bytes is the minimum
41  * space required for a meaningful session, before
42  * we hit a "WRAP" event.
43  */
44 #define TRBE_TRACE_MIN_BUF_SIZE		64
45 
46 enum trbe_fault_action {
47 	TRBE_FAULT_ACT_WRAP,
48 	TRBE_FAULT_ACT_SPURIOUS,
49 	TRBE_FAULT_ACT_FATAL,
50 };
51 
52 struct trbe_buf {
53 	/*
54 	 * Even though trbe_base represents vmap()
55 	 * mapped allocated buffer's start address,
56 	 * it's being as unsigned long for various
57 	 * arithmetic and comparision operations &
58 	 * also to be consistent with trbe_write &
59 	 * trbe_limit sibling pointers.
60 	 */
61 	unsigned long trbe_base;
62 	/* The base programmed into the TRBE */
63 	unsigned long trbe_hw_base;
64 	unsigned long trbe_limit;
65 	unsigned long trbe_write;
66 	int nr_pages;
67 	void **pages;
68 	bool snapshot;
69 	struct trbe_cpudata *cpudata;
70 };
71 
72 /*
73  * TRBE erratum list
74  *
75  * The errata are defined in arm64 generic cpu_errata framework.
76  * Since the errata work arounds could be applied individually
77  * to the affected CPUs inside the TRBE driver, we need to know if
78  * a given CPU is affected by the erratum. Unlike the other erratum
79  * work arounds, TRBE driver needs to check multiple times during
80  * a trace session. Thus we need a quicker access to per-CPU
81  * errata and not issue costly this_cpu_has_cap() everytime.
82  * We keep a set of the affected errata in trbe_cpudata, per TRBE.
83  *
84  * We rely on the corresponding cpucaps to be defined for a given
85  * TRBE erratum. We map the given cpucap into a TRBE internal number
86  * to make the tracking of the errata lean.
87  *
88  * This helps in :
89  *   - Not duplicating the detection logic
90  *   - Streamlined detection of erratum across the system
91  */
92 #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE	0
93 #define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE	1
94 #define TRBE_NEEDS_DRAIN_AFTER_DISABLE		2
95 #define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE	3
96 #define TRBE_IS_BROKEN				4
97 
98 static int trbe_errata_cpucaps[] = {
99 	[TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
100 	[TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
101 	[TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,
102 	[TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,
103 	[TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,
104 	-1,		/* Sentinel, must be the last entry */
105 };
106 
107 /* The total number of listed errata in trbe_errata_cpucaps */
108 #define TRBE_ERRATA_MAX			(ARRAY_SIZE(trbe_errata_cpucaps) - 1)
109 
110 /*
111  * Safe limit for the number of bytes that may be overwritten
112  * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.
113  */
114 #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES	256
115 
116 /*
117  * struct trbe_cpudata: TRBE instance specific data
118  * @trbe_flag		- TRBE dirty/access flag support
119  * @trbe_hw_align	- Actual TRBE alignment required for TRBPTR_EL1.
120  * @trbe_align		- Software alignment used for the TRBPTR_EL1.
121  * @cpu			- CPU this TRBE belongs to.
122  * @mode		- Mode of current operation. (perf/disabled)
123  * @drvdata		- TRBE specific drvdata
124  * @errata		- Bit map for the errata on this TRBE.
125  */
126 struct trbe_cpudata {
127 	bool trbe_flag;
128 	u64 trbe_hw_align;
129 	u64 trbe_align;
130 	int cpu;
131 	enum cs_mode mode;
132 	struct trbe_buf *buf;
133 	struct trbe_drvdata *drvdata;
134 	DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
135 };
136 
137 struct trbe_drvdata {
138 	struct trbe_cpudata __percpu *cpudata;
139 	struct perf_output_handle * __percpu *handle;
140 	struct hlist_node hotplug_node;
141 	int irq;
142 	cpumask_t supported_cpus;
143 	enum cpuhp_state trbe_online;
144 	struct platform_device *pdev;
145 };
146 
147 static void trbe_check_errata(struct trbe_cpudata *cpudata)
148 {
149 	int i;
150 
151 	for (i = 0; i < TRBE_ERRATA_MAX; i++) {
152 		int cap = trbe_errata_cpucaps[i];
153 
154 		if (WARN_ON_ONCE(cap < 0))
155 			return;
156 		if (this_cpu_has_cap(cap))
157 			set_bit(i, cpudata->errata);
158 	}
159 }
160 
161 static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
162 {
163 	return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
164 }
165 
166 static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)
167 {
168 	return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);
169 }
170 
171 static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)
172 {
173 	return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);
174 }
175 
176 static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)
177 {
178 	/*
179 	 * Errata affected TRBE implementation will need TSB CSYNC and
180 	 * DSB in order to prevent subsequent writes into certain TRBE
181 	 * system registers from being ignored and not effected.
182 	 */
183 	return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);
184 }
185 
186 static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)
187 {
188 	/*
189 	 * Errata affected TRBE implementation will need an additional
190 	 * context synchronization in order to prevent an inconsistent
191 	 * TRBE prohibited region view on the CPU which could possibly
192 	 * corrupt the TRBE buffer or the TRBE state.
193 	 */
194 	return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);
195 }
196 
197 static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)
198 {
199 	return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);
200 }
201 
202 static int trbe_alloc_node(struct perf_event *event)
203 {
204 	if (event->cpu == -1)
205 		return NUMA_NO_NODE;
206 	return cpu_to_node(event->cpu);
207 }
208 
209 static inline void trbe_drain_buffer(void)
210 {
211 	tsb_csync();
212 	dsb(nsh);
213 }
214 
215 static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)
216 {
217 	/*
218 	 * Enable the TRBE without clearing LIMITPTR which
219 	 * might be required for fetching the buffer limits.
220 	 */
221 	trblimitr |= TRBLIMITR_EL1_E;
222 	write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
223 
224 	/* Synchronize the TRBE enable event */
225 	isb();
226 
227 	if (trbe_needs_ctxt_sync_after_enable(cpudata))
228 		isb();
229 }
230 
231 static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)
232 {
233 	u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
234 
235 	/*
236 	 * Disable the TRBE without clearing LIMITPTR which
237 	 * might be required for fetching the buffer limits.
238 	 */
239 	trblimitr &= ~TRBLIMITR_EL1_E;
240 	write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
241 
242 	if (trbe_needs_drain_after_disable(cpudata))
243 		trbe_drain_buffer();
244 	isb();
245 }
246 
247 static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)
248 {
249 	trbe_drain_buffer();
250 	set_trbe_disabled(cpudata);
251 }
252 
253 static void trbe_reset_local(struct trbe_cpudata *cpudata)
254 {
255 	trbe_drain_and_disable_local(cpudata);
256 	write_sysreg_s(0, SYS_TRBLIMITR_EL1);
257 	write_sysreg_s(0, SYS_TRBPTR_EL1);
258 	write_sysreg_s(0, SYS_TRBBASER_EL1);
259 	write_sysreg_s(0, SYS_TRBSR_EL1);
260 }
261 
262 static void trbe_report_wrap_event(struct perf_output_handle *handle)
263 {
264 	/*
265 	 * Mark the buffer to indicate that there was a WRAP event by
266 	 * setting the COLLISION flag. This indicates to the user that
267 	 * the TRBE trace collection was stopped without stopping the
268 	 * ETE and thus there might be some amount of trace that was
269 	 * lost between the time the WRAP was detected and the IRQ
270 	 * was consumed by the CPU.
271 	 *
272 	 * Setting the TRUNCATED flag would move the event to STOPPED
273 	 * state unnecessarily, even when there is space left in the
274 	 * ring buffer. Using the COLLISION flag doesn't have this side
275 	 * effect. We only set TRUNCATED flag when there is no space
276 	 * left in the ring buffer.
277 	 */
278 	perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
279 }
280 
281 static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
282 {
283 	struct trbe_buf *buf = etm_perf_sink_config(handle);
284 
285 	/*
286 	 * We cannot proceed with the buffer collection and we
287 	 * do not have any data for the current session. The
288 	 * etm_perf driver expects to close out the aux_buffer
289 	 * at event_stop(). So disable the TRBE here and leave
290 	 * the update_buffer() to return a 0 size.
291 	 */
292 	trbe_drain_and_disable_local(buf->cpudata);
293 	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
294 	perf_aux_output_end(handle, 0);
295 	*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
296 }
297 
298 /*
299  * TRBE Buffer Management
300  *
301  * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
302  * it starts writing trace data from the write pointer onward till the limit pointer.
303  * When the write pointer reaches the address just before the limit pointer, it gets
304  * wrapped around again to the base pointer. This is called a TRBE wrap event, which
305  * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
306  * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
307  * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
308  * LIMIT pointers.
309  *
310  *	Wrap around with an IRQ
311  *	------ < ------ < ------- < ----- < -----
312  *	|					|
313  *	------ > ------ > ------- > ----- > -----
314  *
315  *	+---------------+-----------------------+
316  *	|		|			|
317  *	+---------------+-----------------------+
318  *	Base Pointer	Write Pointer		Limit Pointer
319  *
320  * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
321  * pointer can be aligned to the implementation defined TRBE trace buffer alignment
322  * as captured in trbe_cpudata->trbe_align.
323  *
324  *
325  *		head		tail		wakeup
326  *	+---------------------------------------+----- ~ ~ ------
327  *	|$$$$$$$|################|$$$$$$$$$$$$$$|		|
328  *	+---------------------------------------+----- ~ ~ ------
329  *	Base Pointer	Write Pointer		Limit Pointer
330  *
331  * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
332  * values which tracks all the driver writes and user reads from the perf auxiliary
333  * buffer. Generally [head..tail] is the area where the driver can write into unless
334  * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
335  * configured depending on the perf_output_handle indices, so that the driver does
336  * not override into areas in the perf auxiliary buffer which is being or yet to be
337  * consumed from the user space. The enabled TRBE buffer area is a moving subset of
338  * the allocated perf auxiliary buffer.
339  */
340 
341 static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
342 {
343 	memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
344 }
345 
346 static void trbe_pad_buf(struct perf_output_handle *handle, int len)
347 {
348 	struct trbe_buf *buf = etm_perf_sink_config(handle);
349 	u64 head = PERF_IDX2OFF(handle->head, buf);
350 
351 	__trbe_pad_buf(buf, head, len);
352 	if (!buf->snapshot)
353 		perf_aux_output_skip(handle, len);
354 }
355 
356 static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
357 {
358 	struct trbe_buf *buf = etm_perf_sink_config(handle);
359 
360 	/*
361 	 * The ETE trace has alignment synchronization packets allowing
362 	 * the decoder to reset in case of an overflow or corruption.
363 	 * So we can use the entire buffer for the snapshot mode.
364 	 */
365 	return buf->nr_pages * PAGE_SIZE;
366 }
367 
368 static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)
369 {
370 	u64 size = TRBE_TRACE_MIN_BUF_SIZE;
371 	struct trbe_buf *buf = etm_perf_sink_config(handle);
372 	struct trbe_cpudata *cpudata = buf->cpudata;
373 
374 	/*
375 	 * When the TRBE is affected by an erratum that could make it
376 	 * write to the next "virtually addressed" page beyond the LIMIT.
377 	 * We need to make sure there is always a PAGE after the LIMIT,
378 	 * within the buffer. Thus we ensure there is at least an extra
379 	 * page than normal. With this we could then adjust the LIMIT
380 	 * pointer down by a PAGE later.
381 	 */
382 	if (trbe_may_write_out_of_range(cpudata))
383 		size += PAGE_SIZE;
384 	return size;
385 }
386 
387 /*
388  * TRBE Limit Calculation
389  *
390  * The following markers are used to illustrate various TRBE buffer situations.
391  *
392  * $$$$ - Data area, unconsumed captured trace data, not to be overridden
393  * #### - Free area, enabled, trace will be written
394  * %%%% - Free area, disabled, trace will not be written
395  * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
396  */
397 static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
398 {
399 	struct trbe_buf *buf = etm_perf_sink_config(handle);
400 	struct trbe_cpudata *cpudata = buf->cpudata;
401 	const u64 bufsize = buf->nr_pages * PAGE_SIZE;
402 	u64 limit = bufsize;
403 	u64 head, tail, wakeup;
404 
405 	head = PERF_IDX2OFF(handle->head, buf);
406 
407 	/*
408 	 *		head
409 	 *	------->|
410 	 *	|
411 	 *	head	TRBE align	tail
412 	 * +----|-------|---------------|-------+
413 	 * |$$$$|=======|###############|$$$$$$$|
414 	 * +----|-------|---------------|-------+
415 	 * trbe_base				trbe_base + nr_pages
416 	 *
417 	 * Perf aux buffer output head position can be misaligned depending on
418 	 * various factors including user space reads. In case misaligned, head
419 	 * needs to be aligned before TRBE can be configured. Pad the alignment
420 	 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
421 	 * and skip this section thus advancing the head.
422 	 */
423 	if (!IS_ALIGNED(head, cpudata->trbe_align)) {
424 		unsigned long delta = roundup(head, cpudata->trbe_align) - head;
425 
426 		delta = min(delta, handle->size);
427 		trbe_pad_buf(handle, delta);
428 		head = PERF_IDX2OFF(handle->head, buf);
429 	}
430 
431 	/*
432 	 *	head = tail (size = 0)
433 	 * +----|-------------------------------+
434 	 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$	|
435 	 * +----|-------------------------------+
436 	 * trbe_base				trbe_base + nr_pages
437 	 *
438 	 * Perf aux buffer does not have any space for the driver to write into.
439 	 */
440 	if (!handle->size)
441 		return 0;
442 
443 	/* Compute the tail and wakeup indices now that we've aligned head */
444 	tail = PERF_IDX2OFF(handle->head + handle->size, buf);
445 	wakeup = PERF_IDX2OFF(handle->wakeup, buf);
446 
447 	/*
448 	 * Lets calculate the buffer area which TRBE could write into. There
449 	 * are three possible scenarios here. Limit needs to be aligned with
450 	 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
451 	 * unconsumed data.
452 	 *
453 	 * 1) head < tail
454 	 *
455 	 *	head			tail
456 	 * +----|-----------------------|-------+
457 	 * |$$$$|#######################|$$$$$$$|
458 	 * +----|-----------------------|-------+
459 	 * trbe_base			limit	trbe_base + nr_pages
460 	 *
461 	 * TRBE could write into [head..tail] area. Unless the tail is right at
462 	 * the end of the buffer, neither an wrap around nor an IRQ is expected
463 	 * while being enabled.
464 	 *
465 	 * 2) head == tail
466 	 *
467 	 *	head = tail (size > 0)
468 	 * +----|-------------------------------+
469 	 * |%%%%|###############################|
470 	 * +----|-------------------------------+
471 	 * trbe_base				limit = trbe_base + nr_pages
472 	 *
473 	 * TRBE should just write into [head..base + nr_pages] area even though
474 	 * the entire buffer is empty. Reason being, when the trace reaches the
475 	 * end of the buffer, it will just wrap around with an IRQ giving an
476 	 * opportunity to reconfigure the buffer.
477 	 *
478 	 * 3) tail < head
479 	 *
480 	 *	tail			head
481 	 * +----|-----------------------|-------+
482 	 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
483 	 * +----|-----------------------|-------+
484 	 * trbe_base				limit = trbe_base + nr_pages
485 	 *
486 	 * TRBE should just write into [head..base + nr_pages] area even though
487 	 * the [trbe_base..tail] is also empty. Reason being, when the trace
488 	 * reaches the end of the buffer, it will just wrap around with an IRQ
489 	 * giving an opportunity to reconfigure the buffer.
490 	 */
491 	if (head < tail)
492 		limit = round_down(tail, PAGE_SIZE);
493 
494 	/*
495 	 * Wakeup may be arbitrarily far into the future. If it's not in the
496 	 * current generation, either we'll wrap before hitting it, or it's
497 	 * in the past and has been handled already.
498 	 *
499 	 * If there's a wakeup before we wrap, arrange to be woken up by the
500 	 * page boundary following it. Keep the tail boundary if that's lower.
501 	 *
502 	 *	head		wakeup	tail
503 	 * +----|---------------|-------|-------+
504 	 * |$$$$|###############|%%%%%%%|$$$$$$$|
505 	 * +----|---------------|-------|-------+
506 	 * trbe_base		limit		trbe_base + nr_pages
507 	 */
508 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
509 		limit = min(limit, round_up(wakeup, PAGE_SIZE));
510 
511 	/*
512 	 * There are two situation when this can happen i.e limit is before
513 	 * the head and hence TRBE cannot be configured.
514 	 *
515 	 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
516 	 * within the same PAGE size range.
517 	 *
518 	 *			PAGE_SIZE
519 	 *		|----------------------|
520 	 *
521 	 *		limit	head	tail
522 	 * +------------|------|--------|-------+
523 	 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
524 	 * +------------|------|--------|-------+
525 	 * trbe_base				trbe_base + nr_pages
526 	 *
527 	 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
528 	 * head and wakeup are within same PAGE size range.
529 	 *
530 	 *		PAGE_SIZE
531 	 *	|----------------------|
532 	 *
533 	 *	limit	head	wakeup  tail
534 	 * +----|------|-------|--------|-------+
535 	 * |$$$$$$$$$$$|=======|========|$$$$$$$|
536 	 * +----|------|-------|--------|-------+
537 	 * trbe_base				trbe_base + nr_pages
538 	 */
539 	if (limit > head)
540 		return limit;
541 
542 	trbe_pad_buf(handle, handle->size);
543 	return 0;
544 }
545 
546 static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
547 {
548 	struct trbe_buf *buf = etm_perf_sink_config(handle);
549 	u64 limit = __trbe_normal_offset(handle);
550 	u64 head = PERF_IDX2OFF(handle->head, buf);
551 
552 	/*
553 	 * If the head is too close to the limit and we don't
554 	 * have space for a meaningful run, we rather pad it
555 	 * and start fresh.
556 	 *
557 	 * We might have to do this more than once to make sure
558 	 * we have enough required space.
559 	 */
560 	while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {
561 		trbe_pad_buf(handle, limit - head);
562 		limit = __trbe_normal_offset(handle);
563 		head = PERF_IDX2OFF(handle->head, buf);
564 	}
565 	return limit;
566 }
567 
568 static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
569 {
570 	struct trbe_buf *buf = etm_perf_sink_config(handle);
571 	unsigned long offset;
572 
573 	if (buf->snapshot)
574 		offset = trbe_snapshot_offset(handle);
575 	else
576 		offset = trbe_normal_offset(handle);
577 	return buf->trbe_base + offset;
578 }
579 
580 static void clr_trbe_status(void)
581 {
582 	u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
583 
584 	WARN_ON(is_trbe_enabled());
585 	trbsr &= ~TRBSR_EL1_IRQ;
586 	trbsr &= ~TRBSR_EL1_TRG;
587 	trbsr &= ~TRBSR_EL1_WRAP;
588 	trbsr &= ~TRBSR_EL1_EC_MASK;
589 	trbsr &= ~TRBSR_EL1_BSC_MASK;
590 	trbsr &= ~TRBSR_EL1_S;
591 	write_sysreg_s(trbsr, SYS_TRBSR_EL1);
592 }
593 
594 static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
595 {
596 	u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
597 	unsigned long addr = buf->trbe_limit;
598 
599 	WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT)));
600 	WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
601 
602 	trblimitr &= ~TRBLIMITR_EL1_nVM;
603 	trblimitr &= ~TRBLIMITR_EL1_FM_MASK;
604 	trblimitr &= ~TRBLIMITR_EL1_TM_MASK;
605 	trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;
606 
607 	/*
608 	 * Fill trace buffer mode is used here while configuring the
609 	 * TRBE for trace capture. In this particular mode, the trace
610 	 * collection is stopped and a maintenance interrupt is raised
611 	 * when the current write pointer wraps. This pause in trace
612 	 * collection gives the software an opportunity to capture the
613 	 * trace data in the interrupt handler, before reconfiguring
614 	 * the TRBE.
615 	 */
616 	trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) &
617 		     TRBLIMITR_EL1_FM_MASK;
618 
619 	/*
620 	 * Trigger mode is not used here while configuring the TRBE for
621 	 * the trace capture. Hence just keep this in the ignore mode.
622 	 */
623 	trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) &
624 		     TRBLIMITR_EL1_TM_MASK;
625 	trblimitr |= (addr & PAGE_MASK);
626 	set_trbe_enabled(buf->cpudata, trblimitr);
627 }
628 
629 static void trbe_enable_hw(struct trbe_buf *buf)
630 {
631 	WARN_ON(buf->trbe_hw_base < buf->trbe_base);
632 	WARN_ON(buf->trbe_write < buf->trbe_hw_base);
633 	WARN_ON(buf->trbe_write >= buf->trbe_limit);
634 	set_trbe_disabled(buf->cpudata);
635 	clr_trbe_status();
636 	set_trbe_base_pointer(buf->trbe_hw_base);
637 	set_trbe_write_pointer(buf->trbe_write);
638 
639 	/*
640 	 * Synchronize all the register updates
641 	 * till now before enabling the TRBE.
642 	 */
643 	isb();
644 	set_trbe_limit_pointer_enabled(buf);
645 }
646 
647 static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,
648 						 u64 trbsr)
649 {
650 	int ec = get_trbe_ec(trbsr);
651 	int bsc = get_trbe_bsc(trbsr);
652 	struct trbe_buf *buf = etm_perf_sink_config(handle);
653 	struct trbe_cpudata *cpudata = buf->cpudata;
654 
655 	WARN_ON(is_trbe_running(trbsr));
656 	if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
657 		return TRBE_FAULT_ACT_FATAL;
658 
659 	if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
660 		return TRBE_FAULT_ACT_FATAL;
661 
662 	/*
663 	 * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
664 	 * it might write data after a WRAP event in the fill mode.
665 	 * Thus the check TRBPTR == TRBBASER will not be honored.
666 	 */
667 	if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&
668 	    (trbe_may_overwrite_in_fill_mode(cpudata) ||
669 	     get_trbe_write_pointer() == get_trbe_base_pointer()))
670 		return TRBE_FAULT_ACT_WRAP;
671 
672 	return TRBE_FAULT_ACT_SPURIOUS;
673 }
674 
675 static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
676 					 struct trbe_buf *buf, bool wrap)
677 {
678 	u64 write;
679 	u64 start_off, end_off;
680 	u64 size;
681 	u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
682 
683 	/*
684 	 * If the TRBE has wrapped around the write pointer has
685 	 * wrapped and should be treated as limit.
686 	 *
687 	 * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,
688 	 * it may write upto 64bytes beyond the "LIMIT". The driver already
689 	 * keeps a valid page next to the LIMIT and we could potentially
690 	 * consume the trace data that may have been collected there. But we
691 	 * cannot be really sure it is available, and the TRBPTR may not
692 	 * indicate the same. Also, affected cores are also affected by another
693 	 * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus
694 	 * could potentially pad an entire PAGE_SIZE - 64bytes, to get those
695 	 * 64bytes. Thus we ignore the potential triggering of the erratum
696 	 * on WRAP and limit the data to LIMIT.
697 	 */
698 	if (wrap)
699 		write = get_trbe_limit_pointer();
700 	else
701 		write = get_trbe_write_pointer();
702 
703 	/*
704 	 * TRBE may use a different base address than the base
705 	 * of the ring buffer. Thus use the beginning of the ring
706 	 * buffer to compute the offsets.
707 	 */
708 	end_off = write - buf->trbe_base;
709 	start_off = PERF_IDX2OFF(handle->head, buf);
710 
711 	if (WARN_ON_ONCE(end_off < start_off))
712 		return 0;
713 
714 	size = end_off - start_off;
715 	/*
716 	 * If the TRBE is affected by the following erratum, we must fill
717 	 * the space we skipped with IGNORE packets. And we are always
718 	 * guaranteed to have at least a PAGE_SIZE space in the buffer.
719 	 */
720 	if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&
721 	    !WARN_ON(size < overwrite_skip))
722 		__trbe_pad_buf(buf, start_off, overwrite_skip);
723 
724 	return size;
725 }
726 
727 static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
728 				   struct perf_event *event, void **pages,
729 				   int nr_pages, bool snapshot)
730 {
731 	struct trbe_buf *buf;
732 	struct page **pglist;
733 	int i;
734 
735 	/*
736 	 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
737 	 * just a single page, there would not be any room left while writing
738 	 * into a partially filled TRBE buffer after the page size alignment.
739 	 * Hence restrict the minimum buffer size as two pages.
740 	 */
741 	if (nr_pages < 2)
742 		return NULL;
743 
744 	buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));
745 	if (!buf)
746 		return ERR_PTR(-ENOMEM);
747 
748 	pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
749 	if (!pglist) {
750 		kfree(buf);
751 		return ERR_PTR(-ENOMEM);
752 	}
753 
754 	for (i = 0; i < nr_pages; i++)
755 		pglist[i] = virt_to_page(pages[i]);
756 
757 	buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
758 	if (!buf->trbe_base) {
759 		kfree(pglist);
760 		kfree(buf);
761 		return ERR_PTR(-ENOMEM);
762 	}
763 	buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
764 	buf->trbe_write = buf->trbe_base;
765 	buf->snapshot = snapshot;
766 	buf->nr_pages = nr_pages;
767 	buf->pages = pages;
768 	kfree(pglist);
769 	return buf;
770 }
771 
772 static void arm_trbe_free_buffer(void *config)
773 {
774 	struct trbe_buf *buf = config;
775 
776 	vunmap((void *)buf->trbe_base);
777 	kfree(buf);
778 }
779 
780 static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
781 					    struct perf_output_handle *handle,
782 					    void *config)
783 {
784 	struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
785 	struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
786 	struct trbe_buf *buf = config;
787 	enum trbe_fault_action act;
788 	unsigned long size, status;
789 	unsigned long flags;
790 	bool wrap = false;
791 
792 	WARN_ON(buf->cpudata != cpudata);
793 	WARN_ON(cpudata->cpu != smp_processor_id());
794 	WARN_ON(cpudata->drvdata != drvdata);
795 	if (cpudata->mode != CS_MODE_PERF)
796 		return 0;
797 
798 	/*
799 	 * We are about to disable the TRBE. And this could in turn
800 	 * fill up the buffer triggering, an IRQ. This could be consumed
801 	 * by the PE asynchronously, causing a race here against
802 	 * the IRQ handler in closing out the handle. So, let us
803 	 * make sure the IRQ can't trigger while we are collecting
804 	 * the buffer. We also make sure that a WRAP event is handled
805 	 * accordingly.
806 	 */
807 	local_irq_save(flags);
808 
809 	/*
810 	 * If the TRBE was disabled due to lack of space in the AUX buffer or a
811 	 * spurious fault, the driver leaves it disabled, truncating the buffer.
812 	 * Since the etm_perf driver expects to close out the AUX buffer, the
813 	 * driver skips it. Thus, just pass in 0 size here to indicate that the
814 	 * buffer was truncated.
815 	 */
816 	if (!is_trbe_enabled()) {
817 		size = 0;
818 		goto done;
819 	}
820 	/*
821 	 * perf handle structure needs to be shared with the TRBE IRQ handler for
822 	 * capturing trace data and restarting the handle. There is a probability
823 	 * of an undefined reference based crash when etm event is being stopped
824 	 * while a TRBE IRQ also getting processed. This happens due the release
825 	 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
826 	 * the TRBE here will ensure that no IRQ could be generated when the perf
827 	 * handle gets freed in etm_event_stop().
828 	 */
829 	trbe_drain_and_disable_local(cpudata);
830 
831 	/* Check if there is a pending interrupt and handle it here */
832 	status = read_sysreg_s(SYS_TRBSR_EL1);
833 	if (is_trbe_irq(status)) {
834 
835 		/*
836 		 * Now that we are handling the IRQ here, clear the IRQ
837 		 * from the status, to let the irq handler know that it
838 		 * is taken care of.
839 		 */
840 		clr_trbe_irq();
841 		isb();
842 
843 		act = trbe_get_fault_act(handle, status);
844 		/*
845 		 * If this was not due to a WRAP event, we have some
846 		 * errors and as such buffer is empty.
847 		 */
848 		if (act != TRBE_FAULT_ACT_WRAP) {
849 			size = 0;
850 			goto done;
851 		}
852 
853 		trbe_report_wrap_event(handle);
854 		wrap = true;
855 	}
856 
857 	size = trbe_get_trace_size(handle, buf, wrap);
858 
859 done:
860 	local_irq_restore(flags);
861 
862 	if (buf->snapshot)
863 		handle->head += size;
864 	return size;
865 }
866 
867 
868 static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)
869 {
870 	/*
871 	 * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache
872 	 * line size from the "TRBBASER_EL1" in the event of a "FILL".
873 	 * Thus, we could loose some amount of the trace at the base.
874 	 *
875 	 * Before Fix:
876 	 *
877 	 *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
878 	 *  |                   \/                       /
879 	 *   -------------------------------------------------------------
880 	 *  |   Pg0      |   Pg1       |           |          |  PgN     |
881 	 *   -------------------------------------------------------------
882 	 *
883 	 * In the normal course of action, we would set the TRBBASER to the
884 	 * beginning of the ring-buffer (normal-BASE). But with the erratum,
885 	 * the TRBE could overwrite the contents at the "normal-BASE", after
886 	 * hitting the "normal-LIMIT", since it doesn't stop as expected. And
887 	 * this is wrong. This could result in overwriting trace collected in
888 	 * one of the previous runs, being consumed by the user. So we must
889 	 * always make sure that the TRBBASER is within the region
890 	 * [head, head+size]. Note that TRBBASER must be PAGE aligned,
891 	 *
892 	 *  After moving the BASE:
893 	 *
894 	 *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
895 	 *  |                   \/                       /
896 	 *   -------------------------------------------------------------
897 	 *  |         |          |xyzdef.     |..   tuvw|                |
898 	 *   -------------------------------------------------------------
899 	 *                      /
900 	 *              New-BASER
901 	 *
902 	 * Also, we would set the TRBPTR to head (after adjusting for
903 	 * alignment) at normal-PTR. This would mean that the last few bytes
904 	 * of the trace (say, "xyz") might overwrite the first few bytes of
905 	 * trace written ("abc"). More importantly they will appear in what
906 	 * userspace sees as the beginning of the trace, which is wrong. We may
907 	 * not always have space to move the latest trace "xyz" to the correct
908 	 * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).
909 	 * Thus it is easier to ignore those bytes than to complicate the
910 	 * driver to move it, assuming that the erratum was triggered and
911 	 * doing additional checks to see if there is indeed allowed space at
912 	 * TRBLIMITR.LIMIT.
913 	 *
914 	 *  Thus the full workaround will move the BASE and the PTR and would
915 	 *  look like (after padding at the skipped bytes at the end of
916 	 *  session) :
917 	 *
918 	 *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
919 	 *  |                   \/                       /
920 	 *   -------------------------------------------------------------
921 	 *  |         |          |///abc..     |..  rst|                |
922 	 *   -------------------------------------------------------------
923 	 *                      /    |
924 	 *              New-BASER    New-TRBPTR
925 	 *
926 	 * To summarize, with the work around:
927 	 *
928 	 *  - We always align the offset for the next session to PAGE_SIZE
929 	 *    (This is to ensure we can program the TRBBASER to this offset
930 	 *    within the region [head...head+size]).
931 	 *
932 	 *  - At TRBE enable:
933 	 *     - Set the TRBBASER to the page aligned offset of the current
934 	 *       proposed write offset. (which is guaranteed to be aligned
935 	 *       as above)
936 	 *     - Move the TRBPTR to skip first 256bytes (that might be
937 	 *       overwritten with the erratum). This ensures that the trace
938 	 *       generated in the session is not re-written.
939 	 *
940 	 *  - At trace collection:
941 	 *     - Pad the 256bytes skipped above again with IGNORE packets.
942 	 */
943 	if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {
944 		if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))
945 			return -EINVAL;
946 		buf->trbe_hw_base = buf->trbe_write;
947 		buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
948 	}
949 
950 	/*
951 	 * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to
952 	 * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"
953 	 * may be:
954 	 *     - The page beyond the ring buffer. This could mean, TRBE could
955 	 *       corrupt another entity (kernel / user)
956 	 *     - A portion of the "ring buffer" consumed by the userspace.
957 	 *       i.e, a page outisde [head, head + size].
958 	 *
959 	 * We work around this by:
960 	 *     - Making sure that we have at least an extra space of PAGE left
961 	 *       in the ring buffer [head, head + size], than we normally do
962 	 *       without the erratum. See trbe_min_trace_buf_size().
963 	 *
964 	 *     - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside
965 	 *       the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).
966 	 */
967 	if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {
968 		s64 space = buf->trbe_limit - buf->trbe_write;
969 		/*
970 		 * We must have more than a PAGE_SIZE worth space in the proposed
971 		 * range for the TRBE.
972 		 */
973 		if (WARN_ON(space <= PAGE_SIZE ||
974 			    !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))
975 			return -EINVAL;
976 		buf->trbe_limit -= PAGE_SIZE;
977 	}
978 
979 	return 0;
980 }
981 
982 static int __arm_trbe_enable(struct trbe_buf *buf,
983 			     struct perf_output_handle *handle)
984 {
985 	int ret = 0;
986 
987 	perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
988 	buf->trbe_limit = compute_trbe_buffer_limit(handle);
989 	buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
990 	if (buf->trbe_limit == buf->trbe_base) {
991 		ret = -ENOSPC;
992 		goto err;
993 	}
994 	/* Set the base of the TRBE to the buffer base */
995 	buf->trbe_hw_base = buf->trbe_base;
996 
997 	ret = trbe_apply_work_around_before_enable(buf);
998 	if (ret)
999 		goto err;
1000 
1001 	*this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
1002 	trbe_enable_hw(buf);
1003 	return 0;
1004 err:
1005 	trbe_stop_and_truncate_event(handle);
1006 	return ret;
1007 }
1008 
1009 static int arm_trbe_enable(struct coresight_device *csdev, enum cs_mode mode,
1010 			   void *data)
1011 {
1012 	struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1013 	struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1014 	struct perf_output_handle *handle = data;
1015 	struct trbe_buf *buf = etm_perf_sink_config(handle);
1016 
1017 	WARN_ON(cpudata->cpu != smp_processor_id());
1018 	WARN_ON(cpudata->drvdata != drvdata);
1019 	if (mode != CS_MODE_PERF)
1020 		return -EINVAL;
1021 
1022 	cpudata->buf = buf;
1023 	cpudata->mode = mode;
1024 	buf->cpudata = cpudata;
1025 
1026 	return __arm_trbe_enable(buf, handle);
1027 }
1028 
1029 static int arm_trbe_disable(struct coresight_device *csdev)
1030 {
1031 	struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1032 	struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1033 	struct trbe_buf *buf = cpudata->buf;
1034 
1035 	WARN_ON(buf->cpudata != cpudata);
1036 	WARN_ON(cpudata->cpu != smp_processor_id());
1037 	WARN_ON(cpudata->drvdata != drvdata);
1038 	if (cpudata->mode != CS_MODE_PERF)
1039 		return -EINVAL;
1040 
1041 	trbe_drain_and_disable_local(cpudata);
1042 	buf->cpudata = NULL;
1043 	cpudata->buf = NULL;
1044 	cpudata->mode = CS_MODE_DISABLED;
1045 	return 0;
1046 }
1047 
1048 static void trbe_handle_spurious(struct perf_output_handle *handle)
1049 {
1050 	struct trbe_buf *buf = etm_perf_sink_config(handle);
1051 	u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
1052 
1053 	/*
1054 	 * If the IRQ was spurious, simply re-enable the TRBE
1055 	 * back without modifying the buffer parameters to
1056 	 * retain the trace collected so far.
1057 	 */
1058 	set_trbe_enabled(buf->cpudata, trblimitr);
1059 }
1060 
1061 static int trbe_handle_overflow(struct perf_output_handle *handle)
1062 {
1063 	struct perf_event *event = handle->event;
1064 	struct trbe_buf *buf = etm_perf_sink_config(handle);
1065 	unsigned long size;
1066 	struct etm_event_data *event_data;
1067 
1068 	size = trbe_get_trace_size(handle, buf, true);
1069 	if (buf->snapshot)
1070 		handle->head += size;
1071 
1072 	trbe_report_wrap_event(handle);
1073 	perf_aux_output_end(handle, size);
1074 	event_data = perf_aux_output_begin(handle, event);
1075 	if (!event_data) {
1076 		/*
1077 		 * We are unable to restart the trace collection,
1078 		 * thus leave the TRBE disabled. The etm-perf driver
1079 		 * is able to detect this with a disconnected handle
1080 		 * (handle->event = NULL).
1081 		 */
1082 		trbe_drain_and_disable_local(buf->cpudata);
1083 		*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
1084 		return -EINVAL;
1085 	}
1086 
1087 	return __arm_trbe_enable(buf, handle);
1088 }
1089 
1090 static bool is_perf_trbe(struct perf_output_handle *handle)
1091 {
1092 	struct trbe_buf *buf = etm_perf_sink_config(handle);
1093 	struct trbe_cpudata *cpudata = buf->cpudata;
1094 	struct trbe_drvdata *drvdata = cpudata->drvdata;
1095 	int cpu = smp_processor_id();
1096 
1097 	WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
1098 	WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
1099 
1100 	if (cpudata->mode != CS_MODE_PERF)
1101 		return false;
1102 
1103 	if (cpudata->cpu != cpu)
1104 		return false;
1105 
1106 	if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1107 		return false;
1108 
1109 	return true;
1110 }
1111 
1112 static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
1113 {
1114 	struct perf_output_handle **handle_ptr = dev;
1115 	struct perf_output_handle *handle = *handle_ptr;
1116 	struct trbe_buf *buf = etm_perf_sink_config(handle);
1117 	enum trbe_fault_action act;
1118 	u64 status;
1119 	bool truncated = false;
1120 	u64 trfcr;
1121 
1122 	/* Reads to TRBSR_EL1 is fine when TRBE is active */
1123 	status = read_sysreg_s(SYS_TRBSR_EL1);
1124 	/*
1125 	 * If the pending IRQ was handled by update_buffer callback
1126 	 * we have nothing to do here.
1127 	 */
1128 	if (!is_trbe_irq(status))
1129 		return IRQ_NONE;
1130 
1131 	/* Prohibit the CPU from tracing before we disable the TRBE */
1132 	trfcr = cpu_prohibit_trace();
1133 	/*
1134 	 * Ensure the trace is visible to the CPUs and
1135 	 * any external aborts have been resolved.
1136 	 */
1137 	trbe_drain_and_disable_local(buf->cpudata);
1138 	clr_trbe_irq();
1139 	isb();
1140 
1141 	if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
1142 		return IRQ_NONE;
1143 
1144 	if (!is_perf_trbe(handle))
1145 		return IRQ_NONE;
1146 
1147 	act = trbe_get_fault_act(handle, status);
1148 	switch (act) {
1149 	case TRBE_FAULT_ACT_WRAP:
1150 		truncated = !!trbe_handle_overflow(handle);
1151 		break;
1152 	case TRBE_FAULT_ACT_SPURIOUS:
1153 		trbe_handle_spurious(handle);
1154 		break;
1155 	case TRBE_FAULT_ACT_FATAL:
1156 		trbe_stop_and_truncate_event(handle);
1157 		truncated = true;
1158 		break;
1159 	}
1160 
1161 	/*
1162 	 * If the buffer was truncated, ensure perf callbacks
1163 	 * have completed, which will disable the event.
1164 	 *
1165 	 * Otherwise, restore the trace filter controls to
1166 	 * allow the tracing.
1167 	 */
1168 	if (truncated)
1169 		irq_work_run();
1170 	else
1171 		write_trfcr(trfcr);
1172 
1173 	return IRQ_HANDLED;
1174 }
1175 
1176 static const struct coresight_ops_sink arm_trbe_sink_ops = {
1177 	.enable		= arm_trbe_enable,
1178 	.disable	= arm_trbe_disable,
1179 	.alloc_buffer	= arm_trbe_alloc_buffer,
1180 	.free_buffer	= arm_trbe_free_buffer,
1181 	.update_buffer	= arm_trbe_update_buffer,
1182 };
1183 
1184 static const struct coresight_ops arm_trbe_cs_ops = {
1185 	.sink_ops	= &arm_trbe_sink_ops,
1186 };
1187 
1188 static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
1189 {
1190 	struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1191 
1192 	return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);
1193 }
1194 static DEVICE_ATTR_RO(align);
1195 
1196 static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
1197 {
1198 	struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1199 
1200 	return sprintf(buf, "%d\n", cpudata->trbe_flag);
1201 }
1202 static DEVICE_ATTR_RO(flag);
1203 
1204 static struct attribute *arm_trbe_attrs[] = {
1205 	&dev_attr_align.attr,
1206 	&dev_attr_flag.attr,
1207 	NULL,
1208 };
1209 
1210 static const struct attribute_group arm_trbe_group = {
1211 	.attrs = arm_trbe_attrs,
1212 };
1213 
1214 static const struct attribute_group *arm_trbe_groups[] = {
1215 	&arm_trbe_group,
1216 	NULL,
1217 };
1218 
1219 static void arm_trbe_enable_cpu(void *info)
1220 {
1221 	struct trbe_drvdata *drvdata = info;
1222 	struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1223 
1224 	trbe_reset_local(cpudata);
1225 	enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);
1226 }
1227 
1228 static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1229 {
1230 	struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1231 	struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1232 	struct coresight_desc desc = { 0 };
1233 	struct device *dev;
1234 
1235 	if (WARN_ON(trbe_csdev))
1236 		return;
1237 
1238 	/* If the TRBE was not probed on the CPU, we shouldn't be here */
1239 	if (WARN_ON(!cpudata->drvdata))
1240 		return;
1241 
1242 	dev = &cpudata->drvdata->pdev->dev;
1243 	desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
1244 	if (!desc.name)
1245 		goto cpu_clear;
1246 
1247 	desc.type = CORESIGHT_DEV_TYPE_SINK;
1248 	desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
1249 	desc.ops = &arm_trbe_cs_ops;
1250 	desc.pdata = dev_get_platdata(dev);
1251 	desc.groups = arm_trbe_groups;
1252 	desc.dev = dev;
1253 	trbe_csdev = coresight_register(&desc);
1254 	if (IS_ERR(trbe_csdev))
1255 		goto cpu_clear;
1256 
1257 	dev_set_drvdata(&trbe_csdev->dev, cpudata);
1258 	coresight_set_percpu_sink(cpu, trbe_csdev);
1259 	return;
1260 cpu_clear:
1261 	cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1262 }
1263 
1264 /*
1265  * Must be called with preemption disabled, for trbe_check_errata().
1266  */
1267 static void arm_trbe_probe_cpu(void *info)
1268 {
1269 	struct trbe_drvdata *drvdata = info;
1270 	int cpu = smp_processor_id();
1271 	struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1272 	u64 trbidr;
1273 
1274 	if (WARN_ON(!cpudata))
1275 		goto cpu_clear;
1276 
1277 	if (!is_trbe_available()) {
1278 		pr_err("TRBE is not implemented on cpu %d\n", cpu);
1279 		goto cpu_clear;
1280 	}
1281 
1282 	trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
1283 	if (!is_trbe_programmable(trbidr)) {
1284 		pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
1285 		goto cpu_clear;
1286 	}
1287 
1288 	cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
1289 	if (cpudata->trbe_hw_align > SZ_2K) {
1290 		pr_err("Unsupported alignment on cpu %d\n", cpu);
1291 		goto cpu_clear;
1292 	}
1293 
1294 	/*
1295 	 * Run the TRBE erratum checks, now that we know
1296 	 * this instance is about to be registered.
1297 	 */
1298 	trbe_check_errata(cpudata);
1299 
1300 	if (trbe_is_broken(cpudata)) {
1301 		pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);
1302 		goto cpu_clear;
1303 	}
1304 
1305 	/*
1306 	 * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
1307 	 * we must always program the TBRPTR_EL1, 256bytes from a page
1308 	 * boundary, with TRBBASER_EL1 set to the page, to prevent
1309 	 * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.
1310 	 *
1311 	 * Thus make sure we always align our write pointer to a PAGE_SIZE,
1312 	 * which also guarantees that we have at least a PAGE_SIZE space in
1313 	 * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip
1314 	 * the required bytes at the base.
1315 	 */
1316 	if (trbe_may_overwrite_in_fill_mode(cpudata))
1317 		cpudata->trbe_align = PAGE_SIZE;
1318 	else
1319 		cpudata->trbe_align = cpudata->trbe_hw_align;
1320 
1321 	cpudata->trbe_flag = get_trbe_flag_update(trbidr);
1322 	cpudata->cpu = cpu;
1323 	cpudata->drvdata = drvdata;
1324 	return;
1325 cpu_clear:
1326 	cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1327 }
1328 
1329 static void arm_trbe_remove_coresight_cpu(void *info)
1330 {
1331 	int cpu = smp_processor_id();
1332 	struct trbe_drvdata *drvdata = info;
1333 	struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1334 	struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1335 
1336 	disable_percpu_irq(drvdata->irq);
1337 	trbe_reset_local(cpudata);
1338 	if (trbe_csdev) {
1339 		coresight_unregister(trbe_csdev);
1340 		cpudata->drvdata = NULL;
1341 		coresight_set_percpu_sink(cpu, NULL);
1342 	}
1343 }
1344 
1345 static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
1346 {
1347 	int cpu;
1348 
1349 	drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
1350 	if (!drvdata->cpudata)
1351 		return -ENOMEM;
1352 
1353 	for_each_cpu(cpu, &drvdata->supported_cpus) {
1354 		/* If we fail to probe the CPU, let us defer it to hotplug callbacks */
1355 		if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
1356 			continue;
1357 		if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1358 			arm_trbe_register_coresight_cpu(drvdata, cpu);
1359 		if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1360 			smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);
1361 	}
1362 	return 0;
1363 }
1364 
1365 static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
1366 {
1367 	int cpu;
1368 
1369 	for_each_cpu(cpu, &drvdata->supported_cpus)
1370 		smp_call_function_single(cpu, arm_trbe_remove_coresight_cpu, drvdata, 1);
1371 	free_percpu(drvdata->cpudata);
1372 	return 0;
1373 }
1374 
1375 static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
1376 {
1377 	preempt_disable();
1378 	arm_trbe_probe_cpu(drvdata);
1379 	preempt_enable();
1380 }
1381 
1382 static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
1383 {
1384 	struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1385 
1386 	if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1387 
1388 		/*
1389 		 * If this CPU was not probed for TRBE,
1390 		 * initialize it now.
1391 		 */
1392 		if (!coresight_get_percpu_sink(cpu)) {
1393 			arm_trbe_probe_hotplugged_cpu(drvdata);
1394 			if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1395 				arm_trbe_register_coresight_cpu(drvdata, cpu);
1396 			if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1397 				arm_trbe_enable_cpu(drvdata);
1398 		} else {
1399 			arm_trbe_enable_cpu(drvdata);
1400 		}
1401 	}
1402 	return 0;
1403 }
1404 
1405 static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1406 {
1407 	struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1408 
1409 	if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1410 		struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1411 
1412 		disable_percpu_irq(drvdata->irq);
1413 		trbe_reset_local(cpudata);
1414 	}
1415 	return 0;
1416 }
1417 
1418 static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
1419 {
1420 	enum cpuhp_state trbe_online;
1421 	int ret;
1422 
1423 	trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
1424 					      arm_trbe_cpu_startup, arm_trbe_cpu_teardown);
1425 	if (trbe_online < 0)
1426 		return trbe_online;
1427 
1428 	ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);
1429 	if (ret) {
1430 		cpuhp_remove_multi_state(trbe_online);
1431 		return ret;
1432 	}
1433 	drvdata->trbe_online = trbe_online;
1434 	return 0;
1435 }
1436 
1437 static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
1438 {
1439 	cpuhp_state_remove_instance(drvdata->trbe_online, &drvdata->hotplug_node);
1440 	cpuhp_remove_multi_state(drvdata->trbe_online);
1441 }
1442 
1443 static int arm_trbe_probe_irq(struct platform_device *pdev,
1444 			      struct trbe_drvdata *drvdata)
1445 {
1446 	int ret;
1447 
1448 	drvdata->irq = platform_get_irq(pdev, 0);
1449 	if (drvdata->irq < 0) {
1450 		pr_err("IRQ not found for the platform device\n");
1451 		return drvdata->irq;
1452 	}
1453 
1454 	if (!irq_is_percpu(drvdata->irq)) {
1455 		pr_err("IRQ is not a PPI\n");
1456 		return -EINVAL;
1457 	}
1458 
1459 	if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))
1460 		return -EINVAL;
1461 
1462 	drvdata->handle = alloc_percpu(struct perf_output_handle *);
1463 	if (!drvdata->handle)
1464 		return -ENOMEM;
1465 
1466 	ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);
1467 	if (ret) {
1468 		free_percpu(drvdata->handle);
1469 		return ret;
1470 	}
1471 	return 0;
1472 }
1473 
1474 static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
1475 {
1476 	free_percpu_irq(drvdata->irq, drvdata->handle);
1477 	free_percpu(drvdata->handle);
1478 }
1479 
1480 static int arm_trbe_device_probe(struct platform_device *pdev)
1481 {
1482 	struct coresight_platform_data *pdata;
1483 	struct trbe_drvdata *drvdata;
1484 	struct device *dev = &pdev->dev;
1485 	int ret;
1486 
1487 	/* Trace capture is not possible with kernel page table isolation */
1488 	if (arm64_kernel_unmapped_at_el0()) {
1489 		pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
1490 		return -EOPNOTSUPP;
1491 	}
1492 
1493 	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1494 	if (!drvdata)
1495 		return -ENOMEM;
1496 
1497 	pdata = coresight_get_platform_data(dev);
1498 	if (IS_ERR(pdata))
1499 		return PTR_ERR(pdata);
1500 
1501 	dev_set_drvdata(dev, drvdata);
1502 	dev->platform_data = pdata;
1503 	drvdata->pdev = pdev;
1504 	ret = arm_trbe_probe_irq(pdev, drvdata);
1505 	if (ret)
1506 		return ret;
1507 
1508 	ret = arm_trbe_probe_coresight(drvdata);
1509 	if (ret)
1510 		goto probe_failed;
1511 
1512 	ret = arm_trbe_probe_cpuhp(drvdata);
1513 	if (ret)
1514 		goto cpuhp_failed;
1515 
1516 	return 0;
1517 cpuhp_failed:
1518 	arm_trbe_remove_coresight(drvdata);
1519 probe_failed:
1520 	arm_trbe_remove_irq(drvdata);
1521 	return ret;
1522 }
1523 
1524 static int arm_trbe_device_remove(struct platform_device *pdev)
1525 {
1526 	struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
1527 
1528 	arm_trbe_remove_cpuhp(drvdata);
1529 	arm_trbe_remove_coresight(drvdata);
1530 	arm_trbe_remove_irq(drvdata);
1531 	return 0;
1532 }
1533 
1534 static const struct of_device_id arm_trbe_of_match[] = {
1535 	{ .compatible = "arm,trace-buffer-extension"},
1536 	{},
1537 };
1538 MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
1539 
1540 static struct platform_driver arm_trbe_driver = {
1541 	.driver	= {
1542 		.name = DRVNAME,
1543 		.of_match_table = of_match_ptr(arm_trbe_of_match),
1544 		.suppress_bind_attrs = true,
1545 	},
1546 	.probe	= arm_trbe_device_probe,
1547 	.remove	= arm_trbe_device_remove,
1548 };
1549 
1550 static int __init arm_trbe_init(void)
1551 {
1552 	int ret;
1553 
1554 	ret = platform_driver_register(&arm_trbe_driver);
1555 	if (!ret)
1556 		return 0;
1557 
1558 	pr_err("Error registering %s platform driver\n", DRVNAME);
1559 	return ret;
1560 }
1561 
1562 static void __exit arm_trbe_exit(void)
1563 {
1564 	platform_driver_unregister(&arm_trbe_driver);
1565 }
1566 module_init(arm_trbe_init);
1567 module_exit(arm_trbe_exit);
1568 
1569 MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
1570 MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
1571 MODULE_LICENSE("GPL v2");
1572