93 	struct perf_output_handle __percpu	*handle;  member
393 static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len)  in arm_spe_pmu_pad_buf()  argument
395 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in arm_spe_pmu_pad_buf()
396 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_pad_buf()
400 		perf_aux_output_skip(handle, len);  in arm_spe_pmu_pad_buf()
403 static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle)  in arm_spe_pmu_next_snapshot_off()  argument
405 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in arm_spe_pmu_next_snapshot_off()
406 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_snapshot_off()
407 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_snapshot_off()
424 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_snapshot_off()
425 		handle->head = PERF_IDX2OFF(limit, buf);  in arm_spe_pmu_next_snapshot_off()
426 		limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;  in arm_spe_pmu_next_snapshot_off()
432 static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle)  in __arm_spe_pmu_next_off()  argument
434 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in __arm_spe_pmu_next_off()
435 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in __arm_spe_pmu_next_off()
454 	head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
458 		delta = min(delta, handle->size);  in __arm_spe_pmu_next_off()
459 		arm_spe_pmu_pad_buf(handle, delta);  in __arm_spe_pmu_next_off()
460 		head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
464 	if (!handle->size)  in __arm_spe_pmu_next_off()
468 	tail = PERF_IDX2OFF(handle->head + handle->size, buf);  in __arm_spe_pmu_next_off()
469 	wakeup = PERF_IDX2OFF(handle->wakeup, buf);  in __arm_spe_pmu_next_off()
489 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)  in __arm_spe_pmu_next_off()
495 	arm_spe_pmu_pad_buf(handle, handle->size);  in __arm_spe_pmu_next_off()
497 	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);  in __arm_spe_pmu_next_off()
498 	perf_aux_output_end(handle, 0);  in __arm_spe_pmu_next_off()
502 static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle)  in arm_spe_pmu_next_off()  argument
504 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in arm_spe_pmu_next_off()
505 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_off()
506 	u64 limit = __arm_spe_pmu_next_off(handle);  in arm_spe_pmu_next_off()
507 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_off()
514 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_off()
515 		limit = __arm_spe_pmu_next_off(handle);  in arm_spe_pmu_next_off()
521 static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle,  in arm_spe_perf_aux_output_begin()  argument
528 	buf = perf_aux_output_begin(handle, event);  in arm_spe_perf_aux_output_begin()
539 	limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)  in arm_spe_perf_aux_output_begin()
540 			      : arm_spe_pmu_next_off(handle);  in arm_spe_perf_aux_output_begin()
545 	base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_begin()
552 static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle)  in arm_spe_perf_aux_output_end()  argument
554 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in arm_spe_perf_aux_output_end()
558 	size = offset - PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_end()
561 		handle->head = offset;  in arm_spe_perf_aux_output_end()
563 	perf_aux_output_end(handle, size);  in arm_spe_perf_aux_output_end()
583 arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle)  in arm_spe_pmu_buf_get_fault_act()  argument
609 		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED |  in arm_spe_pmu_buf_get_fault_act()
614 		perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);  in arm_spe_pmu_buf_get_fault_act()
647 	arm_spe_perf_aux_output_end(handle);  in arm_spe_pmu_buf_get_fault_act()
653 	struct perf_output_handle *handle = dev;  in arm_spe_pmu_irq_handler()  local
654 	struct perf_event *event = handle->event;  in arm_spe_pmu_irq_handler()
657 	if (!perf_get_aux(handle))  in arm_spe_pmu_irq_handler()
660 	act = arm_spe_pmu_buf_get_fault_act(handle);  in arm_spe_pmu_irq_handler()
688 		if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {  in arm_spe_pmu_irq_handler()
689 			arm_spe_perf_aux_output_begin(handle, event);  in arm_spe_pmu_irq_handler()
782 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_start()  local
785 	arm_spe_perf_aux_output_begin(handle, event);  in arm_spe_pmu_start()
820 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_stop()  local
835 		if (perf_get_aux(handle)) {  in arm_spe_pmu_stop()
838 			act = arm_spe_pmu_buf_get_fault_act(handle);  in arm_spe_pmu_stop()
840 				arm_spe_perf_aux_output_end(handle);  in arm_spe_pmu_stop()
1183 				 spe_pmu->handle);  in arm_spe_pmu_dev_init()
1195 		free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_init()
1203 	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_teardown()
1260 	spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));  in arm_spe_pmu_device_probe()
1261 	if (!spe_pmu->handle)  in arm_spe_pmu_device_probe()
1284 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_probe()
1294 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_remove()