139 	struct perf_output_handle * __percpu *handle;  member
262 static void trbe_report_wrap_event(struct perf_output_handle *handle)  in trbe_report_wrap_event()  argument
278 	perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);  in trbe_report_wrap_event()
281 static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)  in trbe_stop_and_truncate_event()  argument
283 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_stop_and_truncate_event()
293 	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);  in trbe_stop_and_truncate_event()
294 	perf_aux_output_end(handle, 0);  in trbe_stop_and_truncate_event()
295 	*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;  in trbe_stop_and_truncate_event()
346 static void trbe_pad_buf(struct perf_output_handle *handle, int len)  in trbe_pad_buf()  argument
348 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_pad_buf()
349 	u64 head = PERF_IDX2OFF(handle->head, buf);  in trbe_pad_buf()
353 		perf_aux_output_skip(handle, len);  in trbe_pad_buf()
356 static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)  in trbe_snapshot_offset()  argument
358 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_snapshot_offset()
368 static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)  in trbe_min_trace_buf_size()  argument
371 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_min_trace_buf_size()
397 static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)  in __trbe_normal_offset()  argument
399 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in __trbe_normal_offset()
405 	head = PERF_IDX2OFF(handle->head, buf);  in __trbe_normal_offset()
426 		delta = min(delta, handle->size);  in __trbe_normal_offset()
427 		trbe_pad_buf(handle, delta);  in __trbe_normal_offset()
428 		head = PERF_IDX2OFF(handle->head, buf);  in __trbe_normal_offset()
440 	if (!handle->size)  in __trbe_normal_offset()
444 	tail = PERF_IDX2OFF(handle->head + handle->size, buf);  in __trbe_normal_offset()
445 	wakeup = PERF_IDX2OFF(handle->wakeup, buf);  in __trbe_normal_offset()
508 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)  in __trbe_normal_offset()
542 	trbe_pad_buf(handle, handle->size);  in __trbe_normal_offset()
546 static unsigned long trbe_normal_offset(struct perf_output_handle *handle)  in trbe_normal_offset()  argument
548 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_normal_offset()
549 	u64 limit = __trbe_normal_offset(handle);  in trbe_normal_offset()
550 	u64 head = PERF_IDX2OFF(handle->head, buf);  in trbe_normal_offset()
560 	while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {  in trbe_normal_offset()
561 		trbe_pad_buf(handle, limit - head);  in trbe_normal_offset()
562 		limit = __trbe_normal_offset(handle);  in trbe_normal_offset()
563 		head = PERF_IDX2OFF(handle->head, buf);  in trbe_normal_offset()
568 static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)  in compute_trbe_buffer_limit()  argument
570 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in compute_trbe_buffer_limit()
574 		offset = trbe_snapshot_offset(handle);  in compute_trbe_buffer_limit()
576 		offset = trbe_normal_offset(handle);  in compute_trbe_buffer_limit()
647 static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,  in trbe_get_fault_act()  argument
652 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_get_fault_act()
675 static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,  in trbe_get_trace_size()  argument
709 	start_off = PERF_IDX2OFF(handle->head, buf);  in trbe_get_trace_size()
781 					    struct perf_output_handle *handle,  in arm_trbe_update_buffer()  argument
843 		act = trbe_get_fault_act(handle, status);  in arm_trbe_update_buffer()
853 		trbe_report_wrap_event(handle);  in arm_trbe_update_buffer()
857 	size = trbe_get_trace_size(handle, buf, wrap);  in arm_trbe_update_buffer()
863 		handle->head += size;  in arm_trbe_update_buffer()
983 			     struct perf_output_handle *handle)  in __arm_trbe_enable()  argument
987 	perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);  in __arm_trbe_enable()
988 	buf->trbe_limit = compute_trbe_buffer_limit(handle);  in __arm_trbe_enable()
989 	buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);  in __arm_trbe_enable()
1001 	*this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;  in __arm_trbe_enable()
1005 	trbe_stop_and_truncate_event(handle);  in __arm_trbe_enable()
1014 	struct perf_output_handle *handle = data;  in arm_trbe_enable()  local
1015 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in arm_trbe_enable()
1026 	return __arm_trbe_enable(buf, handle);  in arm_trbe_enable()
1048 static void trbe_handle_spurious(struct perf_output_handle *handle)  in trbe_handle_spurious()  argument
1050 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_handle_spurious()
1061 static int trbe_handle_overflow(struct perf_output_handle *handle)  in trbe_handle_overflow()  argument
1063 	struct perf_event *event = handle->event;  in trbe_handle_overflow()
1064 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_handle_overflow()
1068 	size = trbe_get_trace_size(handle, buf, true);  in trbe_handle_overflow()
1070 		handle->head += size;  in trbe_handle_overflow()
1072 	trbe_report_wrap_event(handle);  in trbe_handle_overflow()
1073 	perf_aux_output_end(handle, size);  in trbe_handle_overflow()
1074 	event_data = perf_aux_output_begin(handle, event);  in trbe_handle_overflow()
1083 		*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;  in trbe_handle_overflow()
1087 	return __arm_trbe_enable(buf, handle);  in trbe_handle_overflow()
1090 static bool is_perf_trbe(struct perf_output_handle *handle)  in is_perf_trbe()  argument
1092 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in is_perf_trbe()
1115 	struct perf_output_handle *handle = *handle_ptr;  in arm_trbe_irq_handler()  local
1116 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in arm_trbe_irq_handler()
1141 	if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))  in arm_trbe_irq_handler()
1144 	if (!is_perf_trbe(handle))  in arm_trbe_irq_handler()
1147 	act = trbe_get_fault_act(handle, status);  in arm_trbe_irq_handler()
1150 		truncated = !!trbe_handle_overflow(handle);  in arm_trbe_irq_handler()
1153 		trbe_handle_spurious(handle);  in arm_trbe_irq_handler()
1156 		trbe_stop_and_truncate_event(handle);  in arm_trbe_irq_handler()
1467 	drvdata->handle = alloc_percpu(struct perf_output_handle *);  in arm_trbe_probe_irq()
1468 	if (!drvdata->handle)  in arm_trbe_probe_irq()
1471 	ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);  in arm_trbe_probe_irq()
1473 		free_percpu(drvdata->handle);  in arm_trbe_probe_irq()
1481 	free_percpu_irq(drvdata->irq, drvdata->handle);  in arm_trbe_remove_irq()
1482 	free_percpu(drvdata->handle);  in arm_trbe_remove_irq()