3  * Arm Statistical Profiling Extensions (SPE) support
33 #include "arm-spe.h"
34 #include "arm-spe-decoder/arm-spe-decoder.h"
35 #include "arm-spe-decoder/arm-spe-pkt-decoder.h"
84 	struct arm_spe			*spe;  member
101 static void arm_spe_dump(struct arm_spe *spe __maybe_unused,  in arm_spe_dump()
111 		      ". ... ARM SPE data: size %#zx bytes\n",  in arm_spe_dump()
140 static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf,  in arm_spe_dump_event()  argument
144 	arm_spe_dump(spe, buf, len);  in arm_spe_dump_event()
154 	queue = &speq->spe->queues.queue_array[speq->queue_nr];  in arm_spe_get_trace()
170 		int fd = perf_data__fd(speq->spe->session->data);  in arm_spe_get_trace()
192 static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe,  in arm_spe__alloc_queue()  argument
206 	speq->spe = spe;  in arm_spe__alloc_queue()
231 static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip)  in arm_spe_cpumode()  argument
233 	return ip >= spe->kernel_start ?  in arm_spe_cpumode()
238 static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe,  in arm_spe_set_pid_tid_cpu()  argument
244 	tid = machine__get_current_tid(spe->machine, speq->cpu);  in arm_spe_set_pid_tid_cpu()
252 		speq->thread = machine__find_thread(spe->machine, -1,  in arm_spe_set_pid_tid_cpu()
265 	struct arm_spe *spe = speq->spe;  in arm_spe_set_tid()  local
266 	int err = machine__set_current_tid(spe->machine, speq->cpu, -1, tid);  in arm_spe_set_tid()
271 	arm_spe_set_pid_tid_cpu(spe, &spe->queues.queue_array[speq->queue_nr]);  in arm_spe_set_tid()
295 static void arm_spe_prep_sample(struct arm_spe *spe,  in arm_spe_prep_sample()  argument
302 	if (!spe->timeless_decoding)  in arm_spe_prep_sample()
303 		sample->time = tsc_to_perf_time(record->timestamp, &spe->tc);  in arm_spe_prep_sample()
306 	sample->cpumode = arm_spe_cpumode(spe, sample->ip);  in arm_spe_prep_sample()
325 arm_spe_deliver_synth_event(struct arm_spe *spe,  in arm_spe_deliver_synth_event()  argument
332 	if (spe->synth_opts.inject) {  in arm_spe_deliver_synth_event()
333 		ret = arm_spe__inject_event(event, sample, spe->sample_type);  in arm_spe_deliver_synth_event()
338 	ret = perf_session__deliver_synth_event(spe->session, event, sample);  in arm_spe_deliver_synth_event()
340 		pr_err("ARM SPE: failed to deliver event, error %d\n", ret);  in arm_spe_deliver_synth_event()
348 	struct arm_spe *spe = speq->spe;  in arm_spe__synth_mem_sample()  local
353 	arm_spe_prep_sample(spe, speq, event, &sample);  in arm_spe__synth_mem_sample()
362 	return arm_spe_deliver_synth_event(spe, speq, event, &sample);  in arm_spe__synth_mem_sample()
368 	struct arm_spe *spe = speq->spe;  in arm_spe__synth_branch_sample()  local
373 	arm_spe_prep_sample(spe, speq, event, &sample);  in arm_spe__synth_branch_sample()
380 	return arm_spe_deliver_synth_event(spe, speq, event, &sample);  in arm_spe__synth_branch_sample()
386 	struct arm_spe *spe = speq->spe;  in arm_spe__synth_instruction_sample()  local
395 	if (speq->period_instructions < spe->instructions_sample_period)  in arm_spe__synth_instruction_sample()
399 	arm_spe_prep_sample(spe, speq, event, &sample);  in arm_spe__synth_instruction_sample()
406 	sample.period = spe->instructions_sample_period;  in arm_spe__synth_instruction_sample()
409 	return arm_spe_deliver_synth_event(spe, speq, event, &sample);  in arm_spe__synth_instruction_sample()
432 	 * Neoverse SPE records.  in arm_spe__synth_data_source_neoverse()
550 	struct arm_spe *spe = speq->spe;  in arm_spe_sample()  local
554 	data_src = arm_spe__synth_data_source(record, spe->midr);  in arm_spe_sample()
556 	if (spe->sample_flc) {  in arm_spe_sample()
558 			err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id,  in arm_spe_sample()
565 			err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id,  in arm_spe_sample()
572 	if (spe->sample_llc) {  in arm_spe_sample()
574 			err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id,  in arm_spe_sample()
581 			err = arm_spe__synth_mem_sample(speq, spe->llc_access_id,  in arm_spe_sample()
588 	if (spe->sample_tlb) {  in arm_spe_sample()
590 			err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id,  in arm_spe_sample()
597 			err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id,  in arm_spe_sample()
604 	if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) {  in arm_spe_sample()
605 		err = arm_spe__synth_branch_sample(speq, spe->branch_miss_id);  in arm_spe_sample()
610 	if (spe->sample_remote_access &&  in arm_spe_sample()
612 		err = arm_spe__synth_mem_sample(speq, spe->remote_access_id,  in arm_spe_sample()
622 	if (spe->sample_memory && data_src) {  in arm_spe_sample()
623 		err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src);  in arm_spe_sample()
628 	if (spe->sample_instructions) {  in arm_spe_sample()
629 		err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src);  in arm_spe_sample()
639 	struct arm_spe *spe = speq->spe;  in arm_spe_run_decoder()  local
643 	if (!spe->kernel_start)  in arm_spe_run_decoder()
644 		spe->kernel_start = machine__kernel_start(spe->machine);  in arm_spe_run_decoder()
663 		 * can correlate samples between Arm SPE trace data and other  in arm_spe_run_decoder()
671 		if (!spe->timeless_decoding && record->context_id != (u64)-1) {  in arm_spe_run_decoder()
676 			spe->use_ctx_pkt_for_pid = true;  in arm_spe_run_decoder()
690 		 * Error is detected when decode SPE trace data, continue to  in arm_spe_run_decoder()
707 		if (!spe->timeless_decoding && speq->timestamp >= *timestamp) {  in arm_spe_run_decoder()
716 static int arm_spe__setup_queue(struct arm_spe *spe,  in arm_spe__setup_queue()  argument
726 	speq = arm_spe__alloc_queue(spe, queue_nr);  in arm_spe__setup_queue()
739 		if (spe->timeless_decoding)  in arm_spe__setup_queue()
754 		ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp);  in arm_spe__setup_queue()
763 static int arm_spe__setup_queues(struct arm_spe *spe)  in arm_spe__setup_queues()  argument
768 	for (i = 0; i < spe->queues.nr_queues; i++) {  in arm_spe__setup_queues()
769 		ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i);  in arm_spe__setup_queues()
777 static int arm_spe__update_queues(struct arm_spe *spe)  in arm_spe__update_queues()  argument
779 	if (spe->queues.new_data) {  in arm_spe__update_queues()
780 		spe->queues.new_data = false;  in arm_spe__update_queues()
781 		return arm_spe__setup_queues(spe);  in arm_spe__update_queues()
787 static bool arm_spe__is_timeless_decoding(struct arm_spe *spe)  in arm_spe__is_timeless_decoding()  argument
790 	struct evlist *evlist = spe->session->evlist;  in arm_spe__is_timeless_decoding()
805 static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp)  in arm_spe_process_queues()  argument
815 		if (!spe->heap.heap_cnt)  in arm_spe_process_queues()
818 		if (spe->heap.heap_array[0].ordinal >= timestamp)  in arm_spe_process_queues()
821 		queue_nr = spe->heap.heap_array[0].queue_nr;  in arm_spe_process_queues()
822 		queue = &spe->queues.queue_array[queue_nr];  in arm_spe_process_queues()
825 		auxtrace_heap__pop(&spe->heap);  in arm_spe_process_queues()
827 		if (spe->heap.heap_cnt) {  in arm_spe_process_queues()
828 			ts = spe->heap.heap_array[0].ordinal + 1;  in arm_spe_process_queues()
837 		 * here we need to update the pid/tid in the thread and SPE queue.  in arm_spe_process_queues()
839 		if (!spe->use_ctx_pkt_for_pid)  in arm_spe_process_queues()
840 			arm_spe_set_pid_tid_cpu(spe, queue);  in arm_spe_process_queues()
844 			auxtrace_heap__add(&spe->heap, queue_nr, ts);  in arm_spe_process_queues()
849 			ret = auxtrace_heap__add(&spe->heap, queue_nr, ts);  in arm_spe_process_queues()
860 static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid,  in arm_spe_process_timeless_queues()  argument
863 	struct auxtrace_queues *queues = &spe->queues;  in arm_spe_process_timeless_queues()
868 		struct auxtrace_queue *queue = &spe->queues.queue_array[i];  in arm_spe_process_timeless_queues()
873 			arm_spe_set_pid_tid_cpu(spe, queue);  in arm_spe_process_timeless_queues()
880 static int arm_spe_context_switch(struct arm_spe *spe, union perf_event *event,  in arm_spe_context_switch()  argument
896 	return machine__set_current_tid(spe->machine, cpu, pid, tid);  in arm_spe_context_switch()
906 	struct arm_spe *spe = container_of(session->auxtrace,  in arm_spe_process_event()  local
913 		pr_err("SPE trace requires ordered events\n");  in arm_spe_process_event()
918 		timestamp = perf_time_to_tsc(sample->time, &spe->tc);  in arm_spe_process_event()
922 	if (timestamp || spe->timeless_decoding) {  in arm_spe_process_event()
923 		err = arm_spe__update_queues(spe);  in arm_spe_process_event()
928 	if (spe->timeless_decoding) {  in arm_spe_process_event()
930 			err = arm_spe_process_timeless_queues(spe,  in arm_spe_process_event()
935 		err = arm_spe_process_queues(spe, timestamp);  in arm_spe_process_event()
939 		if (!spe->use_ctx_pkt_for_pid &&  in arm_spe_process_event()
942 			err = arm_spe_context_switch(spe, event, sample);  in arm_spe_process_event()
952 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,  in arm_spe_process_auxtrace_event()  local
955 	if (!spe->data_queued) {  in arm_spe_process_auxtrace_event()
969 		err = auxtrace_queues__add_event(&spe->queues, session, event,  in arm_spe_process_auxtrace_event()
977 				arm_spe_dump_event(spe, buffer->data,  in arm_spe_process_auxtrace_event()
990 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,  in arm_spe_flush()  local
1000 	ret = arm_spe__update_queues(spe);  in arm_spe_flush()
1004 	if (spe->timeless_decoding)  in arm_spe_flush()
1005 		return arm_spe_process_timeless_queues(spe, -1,  in arm_spe_flush()
1008 	ret = arm_spe_process_queues(spe, MAX_TIMESTAMP);  in arm_spe_flush()
1012 	if (!spe->use_ctx_pkt_for_pid)  in arm_spe_flush()
1013 		ui__warning("Arm SPE CONTEXT packets not found in the traces.\n"  in arm_spe_flush()
1014 			    "Matching of TIDs to SPE events could be inaccurate.\n");  in arm_spe_flush()
1033 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,  in arm_spe_free_events()  local
1035 	struct auxtrace_queues *queues = &spe->queues;  in arm_spe_free_events()
1047 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,  in arm_spe_free()  local
1050 	auxtrace_heap__free(&spe->heap);  in arm_spe_free()
1053 	free(spe);  in arm_spe_free()
1059 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace);  in arm_spe_evsel_is_auxtrace()  local
1061 	return evsel->core.attr.type == spe->pmu_type;  in arm_spe_evsel_is_auxtrace()
1121 arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)  in arm_spe_synth_events()  argument
1131 		if (evsel->core.attr.type == spe->pmu_type) {  in arm_spe_synth_events()
1138 		pr_debug("No selected events with SPE trace data\n");  in arm_spe_synth_events()
1150 	if (spe->timeless_decoding)  in arm_spe_synth_events()
1155 	spe->sample_type = attr.sample_type;  in arm_spe_synth_events()
1171 	if (spe->synth_opts.flc) {  in arm_spe_synth_events()
1172 		spe->sample_flc = true;  in arm_spe_synth_events()
1178 		spe->l1d_miss_id = id;  in arm_spe_synth_events()
1186 		spe->l1d_access_id = id;  in arm_spe_synth_events()
1191 	if (spe->synth_opts.llc) {  in arm_spe_synth_events()
1192 		spe->sample_llc = true;  in arm_spe_synth_events()
1198 		spe->llc_miss_id = id;  in arm_spe_synth_events()
1206 		spe->llc_access_id = id;  in arm_spe_synth_events()
1211 	if (spe->synth_opts.tlb) {  in arm_spe_synth_events()
1212 		spe->sample_tlb = true;  in arm_spe_synth_events()
1218 		spe->tlb_miss_id = id;  in arm_spe_synth_events()
1226 		spe->tlb_access_id = id;  in arm_spe_synth_events()
1231 	if (spe->synth_opts.branches) {  in arm_spe_synth_events()
1232 		spe->sample_branch = true;  in arm_spe_synth_events()
1238 		spe->branch_miss_id = id;  in arm_spe_synth_events()
1243 	if (spe->synth_opts.remote_access) {  in arm_spe_synth_events()
1244 		spe->sample_remote_access = true;  in arm_spe_synth_events()
1250 		spe->remote_access_id = id;  in arm_spe_synth_events()
1255 	if (spe->synth_opts.mem) {  in arm_spe_synth_events()
1256 		spe->sample_memory = true;  in arm_spe_synth_events()
1261 		spe->memory_id = id;  in arm_spe_synth_events()
1266 	if (spe->synth_opts.instructions) {  in arm_spe_synth_events()
1267 		if (spe->synth_opts.period_type != PERF_ITRACE_PERIOD_INSTRUCTIONS) {  in arm_spe_synth_events()
1268 			pr_warning("Only instruction-based sampling period is currently supported by Arm SPE.\n");  in arm_spe_synth_events()
1271 		if (spe->synth_opts.period > 1)  in arm_spe_synth_events()
1272 			pr_warning("Arm SPE has a hardware-based sample period.\n"  in arm_spe_synth_events()
1275 		spe->sample_instructions = true;  in arm_spe_synth_events()
1277 		attr.sample_period = spe->synth_opts.period;  in arm_spe_synth_events()
1278 		spe->instructions_sample_period = attr.sample_period;  in arm_spe_synth_events()
1282 		spe->instructions_id = id;  in arm_spe_synth_events()
1298 	struct arm_spe *spe;  in arm_spe_process_auxtrace_info()  local
1305 	spe = zalloc(sizeof(struct arm_spe));  in arm_spe_process_auxtrace_info()
1306 	if (!spe)  in arm_spe_process_auxtrace_info()
1309 	err = auxtrace_queues__init(&spe->queues);  in arm_spe_process_auxtrace_info()
1313 	spe->session = session;  in arm_spe_process_auxtrace_info()
1314 	spe->machine = &session->machines.host; /* No kvm support */  in arm_spe_process_auxtrace_info()
1315 	spe->auxtrace_type = auxtrace_info->type;  in arm_spe_process_auxtrace_info()
1316 	spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE];  in arm_spe_process_auxtrace_info()
1317 	spe->midr = midr;  in arm_spe_process_auxtrace_info()
1319 	spe->timeless_decoding = arm_spe__is_timeless_decoding(spe);  in arm_spe_process_auxtrace_info()
1325 	 * in "spe->tc", which is used for later conversion between clock  in arm_spe_process_auxtrace_info()
1331 	spe->tc.time_shift = tc->time_shift;  in arm_spe_process_auxtrace_info()
1332 	spe->tc.time_mult = tc->time_mult;  in arm_spe_process_auxtrace_info()
1333 	spe->tc.time_zero = tc->time_zero;  in arm_spe_process_auxtrace_info()
1336 		spe->tc.time_cycles = tc->time_cycles;  in arm_spe_process_auxtrace_info()
1337 		spe->tc.time_mask = tc->time_mask;  in arm_spe_process_auxtrace_info()
1338 		spe->tc.cap_user_time_zero = tc->cap_user_time_zero;  in arm_spe_process_auxtrace_info()
1339 		spe->tc.cap_user_time_short = tc->cap_user_time_short;  in arm_spe_process_auxtrace_info()
1342 	spe->auxtrace.process_event = arm_spe_process_event;  in arm_spe_process_auxtrace_info()
1343 	spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event;  in arm_spe_process_auxtrace_info()
1344 	spe->auxtrace.flush_events = arm_spe_flush;  in arm_spe_process_auxtrace_info()
1345 	spe->auxtrace.free_events = arm_spe_free_events;  in arm_spe_process_auxtrace_info()
1346 	spe->auxtrace.free = arm_spe_free;  in arm_spe_process_auxtrace_info()
1347 	spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace;  in arm_spe_process_auxtrace_info()
1348 	session->auxtrace = &spe->auxtrace;  in arm_spe_process_auxtrace_info()
1356 		spe->synth_opts = *session->itrace_synth_opts;  in arm_spe_process_auxtrace_info()
1358 		itrace_synth_opts__set_default(&spe->synth_opts, false);  in arm_spe_process_auxtrace_info()
1360 	err = arm_spe_synth_events(spe, session);  in arm_spe_process_auxtrace_info()
1364 	err = auxtrace_queues__process_index(&spe->queues, session);  in arm_spe_process_auxtrace_info()
1368 	if (spe->queues.populated)  in arm_spe_process_auxtrace_info()
1369 		spe->data_queued = true;  in arm_spe_process_auxtrace_info()
1374 	auxtrace_queues__free(&spe->queues);  in arm_spe_process_auxtrace_info()
1377 	free(spe);  in arm_spe_process_auxtrace_info()