xref: /openbmc/linux/tools/perf/util/intel-pt.c (revision 4f205687)
1 /*
2  * intel_pt.c: Intel Processor Trace support
3  * Copyright (c) 2013-2015, Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  *
14  */
15 
16 #include <stdio.h>
17 #include <stdbool.h>
18 #include <errno.h>
19 #include <linux/kernel.h>
20 #include <linux/types.h>
21 
22 #include "../perf.h"
23 #include "session.h"
24 #include "machine.h"
25 #include "sort.h"
26 #include "tool.h"
27 #include "event.h"
28 #include "evlist.h"
29 #include "evsel.h"
30 #include "map.h"
31 #include "color.h"
32 #include "util.h"
33 #include "thread.h"
34 #include "thread-stack.h"
35 #include "symbol.h"
36 #include "callchain.h"
37 #include "dso.h"
38 #include "debug.h"
39 #include "auxtrace.h"
40 #include "tsc.h"
41 #include "intel-pt.h"
42 
43 #include "intel-pt-decoder/intel-pt-log.h"
44 #include "intel-pt-decoder/intel-pt-decoder.h"
45 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
46 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
47 
48 #define MAX_TIMESTAMP (~0ULL)
49 
50 struct intel_pt {
51 	struct auxtrace auxtrace;
52 	struct auxtrace_queues queues;
53 	struct auxtrace_heap heap;
54 	u32 auxtrace_type;
55 	struct perf_session *session;
56 	struct machine *machine;
57 	struct perf_evsel *switch_evsel;
58 	struct thread *unknown_thread;
59 	bool timeless_decoding;
60 	bool sampling_mode;
61 	bool snapshot_mode;
62 	bool per_cpu_mmaps;
63 	bool have_tsc;
64 	bool data_queued;
65 	bool est_tsc;
66 	bool sync_switch;
67 	bool mispred_all;
68 	int have_sched_switch;
69 	u32 pmu_type;
70 	u64 kernel_start;
71 	u64 switch_ip;
72 	u64 ptss_ip;
73 
74 	struct perf_tsc_conversion tc;
75 	bool cap_user_time_zero;
76 
77 	struct itrace_synth_opts synth_opts;
78 
79 	bool sample_instructions;
80 	u64 instructions_sample_type;
81 	u64 instructions_sample_period;
82 	u64 instructions_id;
83 
84 	bool sample_branches;
85 	u32 branches_filter;
86 	u64 branches_sample_type;
87 	u64 branches_id;
88 
89 	bool sample_transactions;
90 	u64 transactions_sample_type;
91 	u64 transactions_id;
92 
93 	bool synth_needs_swap;
94 
95 	u64 tsc_bit;
96 	u64 mtc_bit;
97 	u64 mtc_freq_bits;
98 	u32 tsc_ctc_ratio_n;
99 	u32 tsc_ctc_ratio_d;
100 	u64 cyc_bit;
101 	u64 noretcomp_bit;
102 	unsigned max_non_turbo_ratio;
103 
104 	unsigned long num_events;
105 };
106 
107 enum switch_state {
108 	INTEL_PT_SS_NOT_TRACING,
109 	INTEL_PT_SS_UNKNOWN,
110 	INTEL_PT_SS_TRACING,
111 	INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
112 	INTEL_PT_SS_EXPECTING_SWITCH_IP,
113 };
114 
115 struct intel_pt_queue {
116 	struct intel_pt *pt;
117 	unsigned int queue_nr;
118 	struct auxtrace_buffer *buffer;
119 	void *decoder;
120 	const struct intel_pt_state *state;
121 	struct ip_callchain *chain;
122 	struct branch_stack *last_branch;
123 	struct branch_stack *last_branch_rb;
124 	size_t last_branch_pos;
125 	union perf_event *event_buf;
126 	bool on_heap;
127 	bool stop;
128 	bool step_through_buffers;
129 	bool use_buffer_pid_tid;
130 	pid_t pid, tid;
131 	int cpu;
132 	int switch_state;
133 	pid_t next_tid;
134 	struct thread *thread;
135 	bool exclude_kernel;
136 	bool have_sample;
137 	u64 time;
138 	u64 timestamp;
139 	u32 flags;
140 	u16 insn_len;
141 	u64 last_insn_cnt;
142 };
143 
144 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
145 			  unsigned char *buf, size_t len)
146 {
147 	struct intel_pt_pkt packet;
148 	size_t pos = 0;
149 	int ret, pkt_len, i;
150 	char desc[INTEL_PT_PKT_DESC_MAX];
151 	const char *color = PERF_COLOR_BLUE;
152 
153 	color_fprintf(stdout, color,
154 		      ". ... Intel Processor Trace data: size %zu bytes\n",
155 		      len);
156 
157 	while (len) {
158 		ret = intel_pt_get_packet(buf, len, &packet);
159 		if (ret > 0)
160 			pkt_len = ret;
161 		else
162 			pkt_len = 1;
163 		printf(".");
164 		color_fprintf(stdout, color, "  %08x: ", pos);
165 		for (i = 0; i < pkt_len; i++)
166 			color_fprintf(stdout, color, " %02x", buf[i]);
167 		for (; i < 16; i++)
168 			color_fprintf(stdout, color, "   ");
169 		if (ret > 0) {
170 			ret = intel_pt_pkt_desc(&packet, desc,
171 						INTEL_PT_PKT_DESC_MAX);
172 			if (ret > 0)
173 				color_fprintf(stdout, color, " %s\n", desc);
174 		} else {
175 			color_fprintf(stdout, color, " Bad packet!\n");
176 		}
177 		pos += pkt_len;
178 		buf += pkt_len;
179 		len -= pkt_len;
180 	}
181 }
182 
183 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
184 				size_t len)
185 {
186 	printf(".\n");
187 	intel_pt_dump(pt, buf, len);
188 }
189 
190 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
191 				   struct auxtrace_buffer *b)
192 {
193 	void *start;
194 
195 	start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
196 				      pt->have_tsc);
197 	if (!start)
198 		return -EINVAL;
199 	b->use_size = b->data + b->size - start;
200 	b->use_data = start;
201 	return 0;
202 }
203 
204 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
205 					struct auxtrace_queue *queue,
206 					struct auxtrace_buffer *buffer)
207 {
208 	if (queue->cpu == -1 && buffer->cpu != -1)
209 		ptq->cpu = buffer->cpu;
210 
211 	ptq->pid = buffer->pid;
212 	ptq->tid = buffer->tid;
213 
214 	intel_pt_log("queue %u cpu %d pid %d tid %d\n",
215 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
216 
217 	thread__zput(ptq->thread);
218 
219 	if (ptq->tid != -1) {
220 		if (ptq->pid != -1)
221 			ptq->thread = machine__findnew_thread(ptq->pt->machine,
222 							      ptq->pid,
223 							      ptq->tid);
224 		else
225 			ptq->thread = machine__find_thread(ptq->pt->machine, -1,
226 							   ptq->tid);
227 	}
228 }
229 
230 /* This function assumes data is processed sequentially only */
231 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
232 {
233 	struct intel_pt_queue *ptq = data;
234 	struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
235 	struct auxtrace_queue *queue;
236 
237 	if (ptq->stop) {
238 		b->len = 0;
239 		return 0;
240 	}
241 
242 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
243 
244 	buffer = auxtrace_buffer__next(queue, buffer);
245 	if (!buffer) {
246 		if (old_buffer)
247 			auxtrace_buffer__drop_data(old_buffer);
248 		b->len = 0;
249 		return 0;
250 	}
251 
252 	ptq->buffer = buffer;
253 
254 	if (!buffer->data) {
255 		int fd = perf_data_file__fd(ptq->pt->session->file);
256 
257 		buffer->data = auxtrace_buffer__get_data(buffer, fd);
258 		if (!buffer->data)
259 			return -ENOMEM;
260 	}
261 
262 	if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
263 	    intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
264 		return -ENOMEM;
265 
266 	if (old_buffer)
267 		auxtrace_buffer__drop_data(old_buffer);
268 
269 	if (buffer->use_data) {
270 		b->len = buffer->use_size;
271 		b->buf = buffer->use_data;
272 	} else {
273 		b->len = buffer->size;
274 		b->buf = buffer->data;
275 	}
276 	b->ref_timestamp = buffer->reference;
277 
278 	if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
279 						      !buffer->consecutive)) {
280 		b->consecutive = false;
281 		b->trace_nr = buffer->buffer_nr + 1;
282 	} else {
283 		b->consecutive = true;
284 	}
285 
286 	if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
287 					ptq->tid != buffer->tid))
288 		intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
289 
290 	if (ptq->step_through_buffers)
291 		ptq->stop = true;
292 
293 	if (!b->len)
294 		return intel_pt_get_trace(b, data);
295 
296 	return 0;
297 }
298 
299 struct intel_pt_cache_entry {
300 	struct auxtrace_cache_entry	entry;
301 	u64				insn_cnt;
302 	u64				byte_cnt;
303 	enum intel_pt_insn_op		op;
304 	enum intel_pt_insn_branch	branch;
305 	int				length;
306 	int32_t				rel;
307 };
308 
309 static int intel_pt_config_div(const char *var, const char *value, void *data)
310 {
311 	int *d = data;
312 	long val;
313 
314 	if (!strcmp(var, "intel-pt.cache-divisor")) {
315 		val = strtol(value, NULL, 0);
316 		if (val > 0 && val <= INT_MAX)
317 			*d = val;
318 	}
319 
320 	return 0;
321 }
322 
323 static int intel_pt_cache_divisor(void)
324 {
325 	static int d;
326 
327 	if (d)
328 		return d;
329 
330 	perf_config(intel_pt_config_div, &d);
331 
332 	if (!d)
333 		d = 64;
334 
335 	return d;
336 }
337 
338 static unsigned int intel_pt_cache_size(struct dso *dso,
339 					struct machine *machine)
340 {
341 	off_t size;
342 
343 	size = dso__data_size(dso, machine);
344 	size /= intel_pt_cache_divisor();
345 	if (size < 1000)
346 		return 10;
347 	if (size > (1 << 21))
348 		return 21;
349 	return 32 - __builtin_clz(size);
350 }
351 
352 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
353 					     struct machine *machine)
354 {
355 	struct auxtrace_cache *c;
356 	unsigned int bits;
357 
358 	if (dso->auxtrace_cache)
359 		return dso->auxtrace_cache;
360 
361 	bits = intel_pt_cache_size(dso, machine);
362 
363 	/* Ignoring cache creation failure */
364 	c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
365 
366 	dso->auxtrace_cache = c;
367 
368 	return c;
369 }
370 
371 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
372 			      u64 offset, u64 insn_cnt, u64 byte_cnt,
373 			      struct intel_pt_insn *intel_pt_insn)
374 {
375 	struct auxtrace_cache *c = intel_pt_cache(dso, machine);
376 	struct intel_pt_cache_entry *e;
377 	int err;
378 
379 	if (!c)
380 		return -ENOMEM;
381 
382 	e = auxtrace_cache__alloc_entry(c);
383 	if (!e)
384 		return -ENOMEM;
385 
386 	e->insn_cnt = insn_cnt;
387 	e->byte_cnt = byte_cnt;
388 	e->op = intel_pt_insn->op;
389 	e->branch = intel_pt_insn->branch;
390 	e->length = intel_pt_insn->length;
391 	e->rel = intel_pt_insn->rel;
392 
393 	err = auxtrace_cache__add(c, offset, &e->entry);
394 	if (err)
395 		auxtrace_cache__free_entry(c, e);
396 
397 	return err;
398 }
399 
400 static struct intel_pt_cache_entry *
401 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
402 {
403 	struct auxtrace_cache *c = intel_pt_cache(dso, machine);
404 
405 	if (!c)
406 		return NULL;
407 
408 	return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
409 }
410 
411 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
412 				   uint64_t *insn_cnt_ptr, uint64_t *ip,
413 				   uint64_t to_ip, uint64_t max_insn_cnt,
414 				   void *data)
415 {
416 	struct intel_pt_queue *ptq = data;
417 	struct machine *machine = ptq->pt->machine;
418 	struct thread *thread;
419 	struct addr_location al;
420 	unsigned char buf[1024];
421 	size_t bufsz;
422 	ssize_t len;
423 	int x86_64;
424 	u8 cpumode;
425 	u64 offset, start_offset, start_ip;
426 	u64 insn_cnt = 0;
427 	bool one_map = true;
428 
429 	if (to_ip && *ip == to_ip)
430 		goto out_no_cache;
431 
432 	bufsz = intel_pt_insn_max_size();
433 
434 	if (*ip >= ptq->pt->kernel_start)
435 		cpumode = PERF_RECORD_MISC_KERNEL;
436 	else
437 		cpumode = PERF_RECORD_MISC_USER;
438 
439 	thread = ptq->thread;
440 	if (!thread) {
441 		if (cpumode != PERF_RECORD_MISC_KERNEL)
442 			return -EINVAL;
443 		thread = ptq->pt->unknown_thread;
444 	}
445 
446 	while (1) {
447 		thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
448 		if (!al.map || !al.map->dso)
449 			return -EINVAL;
450 
451 		if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
452 		    dso__data_status_seen(al.map->dso,
453 					  DSO_DATA_STATUS_SEEN_ITRACE))
454 			return -ENOENT;
455 
456 		offset = al.map->map_ip(al.map, *ip);
457 
458 		if (!to_ip && one_map) {
459 			struct intel_pt_cache_entry *e;
460 
461 			e = intel_pt_cache_lookup(al.map->dso, machine, offset);
462 			if (e &&
463 			    (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
464 				*insn_cnt_ptr = e->insn_cnt;
465 				*ip += e->byte_cnt;
466 				intel_pt_insn->op = e->op;
467 				intel_pt_insn->branch = e->branch;
468 				intel_pt_insn->length = e->length;
469 				intel_pt_insn->rel = e->rel;
470 				intel_pt_log_insn_no_data(intel_pt_insn, *ip);
471 				return 0;
472 			}
473 		}
474 
475 		start_offset = offset;
476 		start_ip = *ip;
477 
478 		/* Load maps to ensure dso->is_64_bit has been updated */
479 		map__load(al.map, machine->symbol_filter);
480 
481 		x86_64 = al.map->dso->is_64_bit;
482 
483 		while (1) {
484 			len = dso__data_read_offset(al.map->dso, machine,
485 						    offset, buf, bufsz);
486 			if (len <= 0)
487 				return -EINVAL;
488 
489 			if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
490 				return -EINVAL;
491 
492 			intel_pt_log_insn(intel_pt_insn, *ip);
493 
494 			insn_cnt += 1;
495 
496 			if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
497 				goto out;
498 
499 			if (max_insn_cnt && insn_cnt >= max_insn_cnt)
500 				goto out_no_cache;
501 
502 			*ip += intel_pt_insn->length;
503 
504 			if (to_ip && *ip == to_ip)
505 				goto out_no_cache;
506 
507 			if (*ip >= al.map->end)
508 				break;
509 
510 			offset += intel_pt_insn->length;
511 		}
512 		one_map = false;
513 	}
514 out:
515 	*insn_cnt_ptr = insn_cnt;
516 
517 	if (!one_map)
518 		goto out_no_cache;
519 
520 	/*
521 	 * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
522 	 * entries.
523 	 */
524 	if (to_ip) {
525 		struct intel_pt_cache_entry *e;
526 
527 		e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
528 		if (e)
529 			return 0;
530 	}
531 
532 	/* Ignore cache errors */
533 	intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
534 			   *ip - start_ip, intel_pt_insn);
535 
536 	return 0;
537 
538 out_no_cache:
539 	*insn_cnt_ptr = insn_cnt;
540 	return 0;
541 }
542 
543 static bool intel_pt_get_config(struct intel_pt *pt,
544 				struct perf_event_attr *attr, u64 *config)
545 {
546 	if (attr->type == pt->pmu_type) {
547 		if (config)
548 			*config = attr->config;
549 		return true;
550 	}
551 
552 	return false;
553 }
554 
555 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
556 {
557 	struct perf_evsel *evsel;
558 
559 	evlist__for_each(pt->session->evlist, evsel) {
560 		if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
561 		    !evsel->attr.exclude_kernel)
562 			return false;
563 	}
564 	return true;
565 }
566 
567 static bool intel_pt_return_compression(struct intel_pt *pt)
568 {
569 	struct perf_evsel *evsel;
570 	u64 config;
571 
572 	if (!pt->noretcomp_bit)
573 		return true;
574 
575 	evlist__for_each(pt->session->evlist, evsel) {
576 		if (intel_pt_get_config(pt, &evsel->attr, &config) &&
577 		    (config & pt->noretcomp_bit))
578 			return false;
579 	}
580 	return true;
581 }
582 
583 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
584 {
585 	struct perf_evsel *evsel;
586 	unsigned int shift;
587 	u64 config;
588 
589 	if (!pt->mtc_freq_bits)
590 		return 0;
591 
592 	for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
593 		config >>= 1;
594 
595 	evlist__for_each(pt->session->evlist, evsel) {
596 		if (intel_pt_get_config(pt, &evsel->attr, &config))
597 			return (config & pt->mtc_freq_bits) >> shift;
598 	}
599 	return 0;
600 }
601 
602 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
603 {
604 	struct perf_evsel *evsel;
605 	bool timeless_decoding = true;
606 	u64 config;
607 
608 	if (!pt->tsc_bit || !pt->cap_user_time_zero)
609 		return true;
610 
611 	evlist__for_each(pt->session->evlist, evsel) {
612 		if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
613 			return true;
614 		if (intel_pt_get_config(pt, &evsel->attr, &config)) {
615 			if (config & pt->tsc_bit)
616 				timeless_decoding = false;
617 			else
618 				return true;
619 		}
620 	}
621 	return timeless_decoding;
622 }
623 
624 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
625 {
626 	struct perf_evsel *evsel;
627 
628 	evlist__for_each(pt->session->evlist, evsel) {
629 		if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
630 		    !evsel->attr.exclude_kernel)
631 			return true;
632 	}
633 	return false;
634 }
635 
636 static bool intel_pt_have_tsc(struct intel_pt *pt)
637 {
638 	struct perf_evsel *evsel;
639 	bool have_tsc = false;
640 	u64 config;
641 
642 	if (!pt->tsc_bit)
643 		return false;
644 
645 	evlist__for_each(pt->session->evlist, evsel) {
646 		if (intel_pt_get_config(pt, &evsel->attr, &config)) {
647 			if (config & pt->tsc_bit)
648 				have_tsc = true;
649 			else
650 				return false;
651 		}
652 	}
653 	return have_tsc;
654 }
655 
656 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
657 {
658 	u64 quot, rem;
659 
660 	quot = ns / pt->tc.time_mult;
661 	rem  = ns % pt->tc.time_mult;
662 	return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
663 		pt->tc.time_mult;
664 }
665 
666 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
667 						   unsigned int queue_nr)
668 {
669 	struct intel_pt_params params = { .get_trace = 0, };
670 	struct intel_pt_queue *ptq;
671 
672 	ptq = zalloc(sizeof(struct intel_pt_queue));
673 	if (!ptq)
674 		return NULL;
675 
676 	if (pt->synth_opts.callchain) {
677 		size_t sz = sizeof(struct ip_callchain);
678 
679 		sz += pt->synth_opts.callchain_sz * sizeof(u64);
680 		ptq->chain = zalloc(sz);
681 		if (!ptq->chain)
682 			goto out_free;
683 	}
684 
685 	if (pt->synth_opts.last_branch) {
686 		size_t sz = sizeof(struct branch_stack);
687 
688 		sz += pt->synth_opts.last_branch_sz *
689 		      sizeof(struct branch_entry);
690 		ptq->last_branch = zalloc(sz);
691 		if (!ptq->last_branch)
692 			goto out_free;
693 		ptq->last_branch_rb = zalloc(sz);
694 		if (!ptq->last_branch_rb)
695 			goto out_free;
696 	}
697 
698 	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
699 	if (!ptq->event_buf)
700 		goto out_free;
701 
702 	ptq->pt = pt;
703 	ptq->queue_nr = queue_nr;
704 	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
705 	ptq->pid = -1;
706 	ptq->tid = -1;
707 	ptq->cpu = -1;
708 	ptq->next_tid = -1;
709 
710 	params.get_trace = intel_pt_get_trace;
711 	params.walk_insn = intel_pt_walk_next_insn;
712 	params.data = ptq;
713 	params.return_compression = intel_pt_return_compression(pt);
714 	params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
715 	params.mtc_period = intel_pt_mtc_period(pt);
716 	params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
717 	params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
718 
719 	if (pt->synth_opts.instructions) {
720 		if (pt->synth_opts.period) {
721 			switch (pt->synth_opts.period_type) {
722 			case PERF_ITRACE_PERIOD_INSTRUCTIONS:
723 				params.period_type =
724 						INTEL_PT_PERIOD_INSTRUCTIONS;
725 				params.period = pt->synth_opts.period;
726 				break;
727 			case PERF_ITRACE_PERIOD_TICKS:
728 				params.period_type = INTEL_PT_PERIOD_TICKS;
729 				params.period = pt->synth_opts.period;
730 				break;
731 			case PERF_ITRACE_PERIOD_NANOSECS:
732 				params.period_type = INTEL_PT_PERIOD_TICKS;
733 				params.period = intel_pt_ns_to_ticks(pt,
734 							pt->synth_opts.period);
735 				break;
736 			default:
737 				break;
738 			}
739 		}
740 
741 		if (!params.period) {
742 			params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
743 			params.period = 1;
744 		}
745 	}
746 
747 	ptq->decoder = intel_pt_decoder_new(&params);
748 	if (!ptq->decoder)
749 		goto out_free;
750 
751 	return ptq;
752 
753 out_free:
754 	zfree(&ptq->event_buf);
755 	zfree(&ptq->last_branch);
756 	zfree(&ptq->last_branch_rb);
757 	zfree(&ptq->chain);
758 	free(ptq);
759 	return NULL;
760 }
761 
762 static void intel_pt_free_queue(void *priv)
763 {
764 	struct intel_pt_queue *ptq = priv;
765 
766 	if (!ptq)
767 		return;
768 	thread__zput(ptq->thread);
769 	intel_pt_decoder_free(ptq->decoder);
770 	zfree(&ptq->event_buf);
771 	zfree(&ptq->last_branch);
772 	zfree(&ptq->last_branch_rb);
773 	zfree(&ptq->chain);
774 	free(ptq);
775 }
776 
777 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
778 				     struct auxtrace_queue *queue)
779 {
780 	struct intel_pt_queue *ptq = queue->priv;
781 
782 	if (queue->tid == -1 || pt->have_sched_switch) {
783 		ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
784 		thread__zput(ptq->thread);
785 	}
786 
787 	if (!ptq->thread && ptq->tid != -1)
788 		ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
789 
790 	if (ptq->thread) {
791 		ptq->pid = ptq->thread->pid_;
792 		if (queue->cpu == -1)
793 			ptq->cpu = ptq->thread->cpu;
794 	}
795 }
796 
797 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
798 {
799 	if (ptq->state->flags & INTEL_PT_ABORT_TX) {
800 		ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
801 	} else if (ptq->state->flags & INTEL_PT_ASYNC) {
802 		if (ptq->state->to_ip)
803 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
804 				     PERF_IP_FLAG_ASYNC |
805 				     PERF_IP_FLAG_INTERRUPT;
806 		else
807 			ptq->flags = PERF_IP_FLAG_BRANCH |
808 				     PERF_IP_FLAG_TRACE_END;
809 		ptq->insn_len = 0;
810 	} else {
811 		if (ptq->state->from_ip)
812 			ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
813 		else
814 			ptq->flags = PERF_IP_FLAG_BRANCH |
815 				     PERF_IP_FLAG_TRACE_BEGIN;
816 		if (ptq->state->flags & INTEL_PT_IN_TX)
817 			ptq->flags |= PERF_IP_FLAG_IN_TX;
818 		ptq->insn_len = ptq->state->insn_len;
819 	}
820 }
821 
822 static int intel_pt_setup_queue(struct intel_pt *pt,
823 				struct auxtrace_queue *queue,
824 				unsigned int queue_nr)
825 {
826 	struct intel_pt_queue *ptq = queue->priv;
827 
828 	if (list_empty(&queue->head))
829 		return 0;
830 
831 	if (!ptq) {
832 		ptq = intel_pt_alloc_queue(pt, queue_nr);
833 		if (!ptq)
834 			return -ENOMEM;
835 		queue->priv = ptq;
836 
837 		if (queue->cpu != -1)
838 			ptq->cpu = queue->cpu;
839 		ptq->tid = queue->tid;
840 
841 		if (pt->sampling_mode) {
842 			if (pt->timeless_decoding)
843 				ptq->step_through_buffers = true;
844 			if (pt->timeless_decoding || !pt->have_sched_switch)
845 				ptq->use_buffer_pid_tid = true;
846 		}
847 	}
848 
849 	if (!ptq->on_heap &&
850 	    (!pt->sync_switch ||
851 	     ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
852 		const struct intel_pt_state *state;
853 		int ret;
854 
855 		if (pt->timeless_decoding)
856 			return 0;
857 
858 		intel_pt_log("queue %u getting timestamp\n", queue_nr);
859 		intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
860 			     queue_nr, ptq->cpu, ptq->pid, ptq->tid);
861 		while (1) {
862 			state = intel_pt_decode(ptq->decoder);
863 			if (state->err) {
864 				if (state->err == INTEL_PT_ERR_NODATA) {
865 					intel_pt_log("queue %u has no timestamp\n",
866 						     queue_nr);
867 					return 0;
868 				}
869 				continue;
870 			}
871 			if (state->timestamp)
872 				break;
873 		}
874 
875 		ptq->timestamp = state->timestamp;
876 		intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
877 			     queue_nr, ptq->timestamp);
878 		ptq->state = state;
879 		ptq->have_sample = true;
880 		intel_pt_sample_flags(ptq);
881 		ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
882 		if (ret)
883 			return ret;
884 		ptq->on_heap = true;
885 	}
886 
887 	return 0;
888 }
889 
890 static int intel_pt_setup_queues(struct intel_pt *pt)
891 {
892 	unsigned int i;
893 	int ret;
894 
895 	for (i = 0; i < pt->queues.nr_queues; i++) {
896 		ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
897 		if (ret)
898 			return ret;
899 	}
900 	return 0;
901 }
902 
903 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
904 {
905 	struct branch_stack *bs_src = ptq->last_branch_rb;
906 	struct branch_stack *bs_dst = ptq->last_branch;
907 	size_t nr = 0;
908 
909 	bs_dst->nr = bs_src->nr;
910 
911 	if (!bs_src->nr)
912 		return;
913 
914 	nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
915 	memcpy(&bs_dst->entries[0],
916 	       &bs_src->entries[ptq->last_branch_pos],
917 	       sizeof(struct branch_entry) * nr);
918 
919 	if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
920 		memcpy(&bs_dst->entries[nr],
921 		       &bs_src->entries[0],
922 		       sizeof(struct branch_entry) * ptq->last_branch_pos);
923 	}
924 }
925 
926 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
927 {
928 	ptq->last_branch_pos = 0;
929 	ptq->last_branch_rb->nr = 0;
930 }
931 
932 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
933 {
934 	const struct intel_pt_state *state = ptq->state;
935 	struct branch_stack *bs = ptq->last_branch_rb;
936 	struct branch_entry *be;
937 
938 	if (!ptq->last_branch_pos)
939 		ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
940 
941 	ptq->last_branch_pos -= 1;
942 
943 	be              = &bs->entries[ptq->last_branch_pos];
944 	be->from        = state->from_ip;
945 	be->to          = state->to_ip;
946 	be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
947 	be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
948 	/* No support for mispredict */
949 	be->flags.mispred = ptq->pt->mispred_all;
950 
951 	if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
952 		bs->nr += 1;
953 }
954 
955 static int intel_pt_inject_event(union perf_event *event,
956 				 struct perf_sample *sample, u64 type,
957 				 bool swapped)
958 {
959 	event->header.size = perf_event__sample_event_size(sample, type, 0);
960 	return perf_event__synthesize_sample(event, type, 0, sample, swapped);
961 }
962 
963 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
964 {
965 	int ret;
966 	struct intel_pt *pt = ptq->pt;
967 	union perf_event *event = ptq->event_buf;
968 	struct perf_sample sample = { .ip = 0, };
969 	struct dummy_branch_stack {
970 		u64			nr;
971 		struct branch_entry	entries;
972 	} dummy_bs;
973 
974 	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
975 		return 0;
976 
977 	if (pt->synth_opts.initial_skip &&
978 	    pt->num_events++ < pt->synth_opts.initial_skip)
979 		return 0;
980 
981 	event->sample.header.type = PERF_RECORD_SAMPLE;
982 	event->sample.header.misc = PERF_RECORD_MISC_USER;
983 	event->sample.header.size = sizeof(struct perf_event_header);
984 
985 	if (!pt->timeless_decoding)
986 		sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
987 
988 	sample.cpumode = PERF_RECORD_MISC_USER;
989 	sample.ip = ptq->state->from_ip;
990 	sample.pid = ptq->pid;
991 	sample.tid = ptq->tid;
992 	sample.addr = ptq->state->to_ip;
993 	sample.id = ptq->pt->branches_id;
994 	sample.stream_id = ptq->pt->branches_id;
995 	sample.period = 1;
996 	sample.cpu = ptq->cpu;
997 	sample.flags = ptq->flags;
998 	sample.insn_len = ptq->insn_len;
999 
1000 	/*
1001 	 * perf report cannot handle events without a branch stack when using
1002 	 * SORT_MODE__BRANCH so make a dummy one.
1003 	 */
1004 	if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1005 		dummy_bs = (struct dummy_branch_stack){
1006 			.nr = 1,
1007 			.entries = {
1008 				.from = sample.ip,
1009 				.to = sample.addr,
1010 			},
1011 		};
1012 		sample.branch_stack = (struct branch_stack *)&dummy_bs;
1013 	}
1014 
1015 	if (pt->synth_opts.inject) {
1016 		ret = intel_pt_inject_event(event, &sample,
1017 					    pt->branches_sample_type,
1018 					    pt->synth_needs_swap);
1019 		if (ret)
1020 			return ret;
1021 	}
1022 
1023 	ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1024 	if (ret)
1025 		pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1026 		       ret);
1027 
1028 	return ret;
1029 }
1030 
1031 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1032 {
1033 	int ret;
1034 	struct intel_pt *pt = ptq->pt;
1035 	union perf_event *event = ptq->event_buf;
1036 	struct perf_sample sample = { .ip = 0, };
1037 
1038 	if (pt->synth_opts.initial_skip &&
1039 	    pt->num_events++ < pt->synth_opts.initial_skip)
1040 		return 0;
1041 
1042 	event->sample.header.type = PERF_RECORD_SAMPLE;
1043 	event->sample.header.misc = PERF_RECORD_MISC_USER;
1044 	event->sample.header.size = sizeof(struct perf_event_header);
1045 
1046 	if (!pt->timeless_decoding)
1047 		sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1048 
1049 	sample.cpumode = PERF_RECORD_MISC_USER;
1050 	sample.ip = ptq->state->from_ip;
1051 	sample.pid = ptq->pid;
1052 	sample.tid = ptq->tid;
1053 	sample.addr = ptq->state->to_ip;
1054 	sample.id = ptq->pt->instructions_id;
1055 	sample.stream_id = ptq->pt->instructions_id;
1056 	sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1057 	sample.cpu = ptq->cpu;
1058 	sample.flags = ptq->flags;
1059 	sample.insn_len = ptq->insn_len;
1060 
1061 	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1062 
1063 	if (pt->synth_opts.callchain) {
1064 		thread_stack__sample(ptq->thread, ptq->chain,
1065 				     pt->synth_opts.callchain_sz, sample.ip);
1066 		sample.callchain = ptq->chain;
1067 	}
1068 
1069 	if (pt->synth_opts.last_branch) {
1070 		intel_pt_copy_last_branch_rb(ptq);
1071 		sample.branch_stack = ptq->last_branch;
1072 	}
1073 
1074 	if (pt->synth_opts.inject) {
1075 		ret = intel_pt_inject_event(event, &sample,
1076 					    pt->instructions_sample_type,
1077 					    pt->synth_needs_swap);
1078 		if (ret)
1079 			return ret;
1080 	}
1081 
1082 	ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1083 	if (ret)
1084 		pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1085 		       ret);
1086 
1087 	if (pt->synth_opts.last_branch)
1088 		intel_pt_reset_last_branch_rb(ptq);
1089 
1090 	return ret;
1091 }
1092 
1093 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1094 {
1095 	int ret;
1096 	struct intel_pt *pt = ptq->pt;
1097 	union perf_event *event = ptq->event_buf;
1098 	struct perf_sample sample = { .ip = 0, };
1099 
1100 	if (pt->synth_opts.initial_skip &&
1101 	    pt->num_events++ < pt->synth_opts.initial_skip)
1102 		return 0;
1103 
1104 	event->sample.header.type = PERF_RECORD_SAMPLE;
1105 	event->sample.header.misc = PERF_RECORD_MISC_USER;
1106 	event->sample.header.size = sizeof(struct perf_event_header);
1107 
1108 	if (!pt->timeless_decoding)
1109 		sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1110 
1111 	sample.cpumode = PERF_RECORD_MISC_USER;
1112 	sample.ip = ptq->state->from_ip;
1113 	sample.pid = ptq->pid;
1114 	sample.tid = ptq->tid;
1115 	sample.addr = ptq->state->to_ip;
1116 	sample.id = ptq->pt->transactions_id;
1117 	sample.stream_id = ptq->pt->transactions_id;
1118 	sample.period = 1;
1119 	sample.cpu = ptq->cpu;
1120 	sample.flags = ptq->flags;
1121 	sample.insn_len = ptq->insn_len;
1122 
1123 	if (pt->synth_opts.callchain) {
1124 		thread_stack__sample(ptq->thread, ptq->chain,
1125 				     pt->synth_opts.callchain_sz, sample.ip);
1126 		sample.callchain = ptq->chain;
1127 	}
1128 
1129 	if (pt->synth_opts.last_branch) {
1130 		intel_pt_copy_last_branch_rb(ptq);
1131 		sample.branch_stack = ptq->last_branch;
1132 	}
1133 
1134 	if (pt->synth_opts.inject) {
1135 		ret = intel_pt_inject_event(event, &sample,
1136 					    pt->transactions_sample_type,
1137 					    pt->synth_needs_swap);
1138 		if (ret)
1139 			return ret;
1140 	}
1141 
1142 	ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1143 	if (ret)
1144 		pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1145 		       ret);
1146 
1147 	if (pt->synth_opts.last_branch)
1148 		intel_pt_reset_last_branch_rb(ptq);
1149 
1150 	return ret;
1151 }
1152 
1153 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1154 				pid_t pid, pid_t tid, u64 ip)
1155 {
1156 	union perf_event event;
1157 	char msg[MAX_AUXTRACE_ERROR_MSG];
1158 	int err;
1159 
1160 	intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1161 
1162 	auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1163 			     code, cpu, pid, tid, ip, msg);
1164 
1165 	err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1166 	if (err)
1167 		pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1168 		       err);
1169 
1170 	return err;
1171 }
1172 
1173 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1174 {
1175 	struct auxtrace_queue *queue;
1176 	pid_t tid = ptq->next_tid;
1177 	int err;
1178 
1179 	if (tid == -1)
1180 		return 0;
1181 
1182 	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1183 
1184 	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1185 
1186 	queue = &pt->queues.queue_array[ptq->queue_nr];
1187 	intel_pt_set_pid_tid_cpu(pt, queue);
1188 
1189 	ptq->next_tid = -1;
1190 
1191 	return err;
1192 }
1193 
1194 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1195 {
1196 	struct intel_pt *pt = ptq->pt;
1197 
1198 	return ip == pt->switch_ip &&
1199 	       (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1200 	       !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1201 			       PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1202 }
1203 
1204 static int intel_pt_sample(struct intel_pt_queue *ptq)
1205 {
1206 	const struct intel_pt_state *state = ptq->state;
1207 	struct intel_pt *pt = ptq->pt;
1208 	int err;
1209 
1210 	if (!ptq->have_sample)
1211 		return 0;
1212 
1213 	ptq->have_sample = false;
1214 
1215 	if (pt->sample_instructions &&
1216 	    (state->type & INTEL_PT_INSTRUCTION) &&
1217 	    (!pt->synth_opts.initial_skip ||
1218 	     pt->num_events++ >= pt->synth_opts.initial_skip)) {
1219 		err = intel_pt_synth_instruction_sample(ptq);
1220 		if (err)
1221 			return err;
1222 	}
1223 
1224 	if (pt->sample_transactions &&
1225 	    (state->type & INTEL_PT_TRANSACTION) &&
1226 	    (!pt->synth_opts.initial_skip ||
1227 	     pt->num_events++ >= pt->synth_opts.initial_skip)) {
1228 		err = intel_pt_synth_transaction_sample(ptq);
1229 		if (err)
1230 			return err;
1231 	}
1232 
1233 	if (!(state->type & INTEL_PT_BRANCH))
1234 		return 0;
1235 
1236 	if (pt->synth_opts.callchain)
1237 		thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1238 				    state->to_ip, ptq->insn_len,
1239 				    state->trace_nr);
1240 	else
1241 		thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1242 
1243 	if (pt->sample_branches) {
1244 		err = intel_pt_synth_branch_sample(ptq);
1245 		if (err)
1246 			return err;
1247 	}
1248 
1249 	if (pt->synth_opts.last_branch)
1250 		intel_pt_update_last_branch_rb(ptq);
1251 
1252 	if (!pt->sync_switch)
1253 		return 0;
1254 
1255 	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1256 		switch (ptq->switch_state) {
1257 		case INTEL_PT_SS_UNKNOWN:
1258 		case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1259 			err = intel_pt_next_tid(pt, ptq);
1260 			if (err)
1261 				return err;
1262 			ptq->switch_state = INTEL_PT_SS_TRACING;
1263 			break;
1264 		default:
1265 			ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1266 			return 1;
1267 		}
1268 	} else if (!state->to_ip) {
1269 		ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1270 	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1271 		ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1272 	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1273 		   state->to_ip == pt->ptss_ip &&
1274 		   (ptq->flags & PERF_IP_FLAG_CALL)) {
1275 		ptq->switch_state = INTEL_PT_SS_TRACING;
1276 	}
1277 
1278 	return 0;
1279 }
1280 
1281 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1282 {
1283 	struct machine *machine = pt->machine;
1284 	struct map *map;
1285 	struct symbol *sym, *start;
1286 	u64 ip, switch_ip = 0;
1287 	const char *ptss;
1288 
1289 	if (ptss_ip)
1290 		*ptss_ip = 0;
1291 
1292 	map = machine__kernel_map(machine);
1293 	if (!map)
1294 		return 0;
1295 
1296 	if (map__load(map, machine->symbol_filter))
1297 		return 0;
1298 
1299 	start = dso__first_symbol(map->dso, MAP__FUNCTION);
1300 
1301 	for (sym = start; sym; sym = dso__next_symbol(sym)) {
1302 		if (sym->binding == STB_GLOBAL &&
1303 		    !strcmp(sym->name, "__switch_to")) {
1304 			ip = map->unmap_ip(map, sym->start);
1305 			if (ip >= map->start && ip < map->end) {
1306 				switch_ip = ip;
1307 				break;
1308 			}
1309 		}
1310 	}
1311 
1312 	if (!switch_ip || !ptss_ip)
1313 		return 0;
1314 
1315 	if (pt->have_sched_switch == 1)
1316 		ptss = "perf_trace_sched_switch";
1317 	else
1318 		ptss = "__perf_event_task_sched_out";
1319 
1320 	for (sym = start; sym; sym = dso__next_symbol(sym)) {
1321 		if (!strcmp(sym->name, ptss)) {
1322 			ip = map->unmap_ip(map, sym->start);
1323 			if (ip >= map->start && ip < map->end) {
1324 				*ptss_ip = ip;
1325 				break;
1326 			}
1327 		}
1328 	}
1329 
1330 	return switch_ip;
1331 }
1332 
1333 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1334 {
1335 	const struct intel_pt_state *state = ptq->state;
1336 	struct intel_pt *pt = ptq->pt;
1337 	int err;
1338 
1339 	if (!pt->kernel_start) {
1340 		pt->kernel_start = machine__kernel_start(pt->machine);
1341 		if (pt->per_cpu_mmaps &&
1342 		    (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1343 		    !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1344 		    !pt->sampling_mode) {
1345 			pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1346 			if (pt->switch_ip) {
1347 				intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1348 					     pt->switch_ip, pt->ptss_ip);
1349 				pt->sync_switch = true;
1350 			}
1351 		}
1352 	}
1353 
1354 	intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1355 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1356 	while (1) {
1357 		err = intel_pt_sample(ptq);
1358 		if (err)
1359 			return err;
1360 
1361 		state = intel_pt_decode(ptq->decoder);
1362 		if (state->err) {
1363 			if (state->err == INTEL_PT_ERR_NODATA)
1364 				return 1;
1365 			if (pt->sync_switch &&
1366 			    state->from_ip >= pt->kernel_start) {
1367 				pt->sync_switch = false;
1368 				intel_pt_next_tid(pt, ptq);
1369 			}
1370 			if (pt->synth_opts.errors) {
1371 				err = intel_pt_synth_error(pt, state->err,
1372 							   ptq->cpu, ptq->pid,
1373 							   ptq->tid,
1374 							   state->from_ip);
1375 				if (err)
1376 					return err;
1377 			}
1378 			continue;
1379 		}
1380 
1381 		ptq->state = state;
1382 		ptq->have_sample = true;
1383 		intel_pt_sample_flags(ptq);
1384 
1385 		/* Use estimated TSC upon return to user space */
1386 		if (pt->est_tsc &&
1387 		    (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1388 		    state->to_ip && state->to_ip < pt->kernel_start) {
1389 			intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1390 				     state->timestamp, state->est_timestamp);
1391 			ptq->timestamp = state->est_timestamp;
1392 		/* Use estimated TSC in unknown switch state */
1393 		} else if (pt->sync_switch &&
1394 			   ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1395 			   intel_pt_is_switch_ip(ptq, state->to_ip) &&
1396 			   ptq->next_tid == -1) {
1397 			intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1398 				     state->timestamp, state->est_timestamp);
1399 			ptq->timestamp = state->est_timestamp;
1400 		} else if (state->timestamp > ptq->timestamp) {
1401 			ptq->timestamp = state->timestamp;
1402 		}
1403 
1404 		if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1405 			*timestamp = ptq->timestamp;
1406 			return 0;
1407 		}
1408 	}
1409 	return 0;
1410 }
1411 
1412 static inline int intel_pt_update_queues(struct intel_pt *pt)
1413 {
1414 	if (pt->queues.new_data) {
1415 		pt->queues.new_data = false;
1416 		return intel_pt_setup_queues(pt);
1417 	}
1418 	return 0;
1419 }
1420 
1421 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1422 {
1423 	unsigned int queue_nr;
1424 	u64 ts;
1425 	int ret;
1426 
1427 	while (1) {
1428 		struct auxtrace_queue *queue;
1429 		struct intel_pt_queue *ptq;
1430 
1431 		if (!pt->heap.heap_cnt)
1432 			return 0;
1433 
1434 		if (pt->heap.heap_array[0].ordinal >= timestamp)
1435 			return 0;
1436 
1437 		queue_nr = pt->heap.heap_array[0].queue_nr;
1438 		queue = &pt->queues.queue_array[queue_nr];
1439 		ptq = queue->priv;
1440 
1441 		intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1442 			     queue_nr, pt->heap.heap_array[0].ordinal,
1443 			     timestamp);
1444 
1445 		auxtrace_heap__pop(&pt->heap);
1446 
1447 		if (pt->heap.heap_cnt) {
1448 			ts = pt->heap.heap_array[0].ordinal + 1;
1449 			if (ts > timestamp)
1450 				ts = timestamp;
1451 		} else {
1452 			ts = timestamp;
1453 		}
1454 
1455 		intel_pt_set_pid_tid_cpu(pt, queue);
1456 
1457 		ret = intel_pt_run_decoder(ptq, &ts);
1458 
1459 		if (ret < 0) {
1460 			auxtrace_heap__add(&pt->heap, queue_nr, ts);
1461 			return ret;
1462 		}
1463 
1464 		if (!ret) {
1465 			ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1466 			if (ret < 0)
1467 				return ret;
1468 		} else {
1469 			ptq->on_heap = false;
1470 		}
1471 	}
1472 
1473 	return 0;
1474 }
1475 
1476 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1477 					    u64 time_)
1478 {
1479 	struct auxtrace_queues *queues = &pt->queues;
1480 	unsigned int i;
1481 	u64 ts = 0;
1482 
1483 	for (i = 0; i < queues->nr_queues; i++) {
1484 		struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1485 		struct intel_pt_queue *ptq = queue->priv;
1486 
1487 		if (ptq && (tid == -1 || ptq->tid == tid)) {
1488 			ptq->time = time_;
1489 			intel_pt_set_pid_tid_cpu(pt, queue);
1490 			intel_pt_run_decoder(ptq, &ts);
1491 		}
1492 	}
1493 	return 0;
1494 }
1495 
1496 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1497 {
1498 	return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1499 				    sample->pid, sample->tid, 0);
1500 }
1501 
1502 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1503 {
1504 	unsigned i, j;
1505 
1506 	if (cpu < 0 || !pt->queues.nr_queues)
1507 		return NULL;
1508 
1509 	if ((unsigned)cpu >= pt->queues.nr_queues)
1510 		i = pt->queues.nr_queues - 1;
1511 	else
1512 		i = cpu;
1513 
1514 	if (pt->queues.queue_array[i].cpu == cpu)
1515 		return pt->queues.queue_array[i].priv;
1516 
1517 	for (j = 0; i > 0; j++) {
1518 		if (pt->queues.queue_array[--i].cpu == cpu)
1519 			return pt->queues.queue_array[i].priv;
1520 	}
1521 
1522 	for (; j < pt->queues.nr_queues; j++) {
1523 		if (pt->queues.queue_array[j].cpu == cpu)
1524 			return pt->queues.queue_array[j].priv;
1525 	}
1526 
1527 	return NULL;
1528 }
1529 
1530 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1531 				u64 timestamp)
1532 {
1533 	struct intel_pt_queue *ptq;
1534 	int err;
1535 
1536 	if (!pt->sync_switch)
1537 		return 1;
1538 
1539 	ptq = intel_pt_cpu_to_ptq(pt, cpu);
1540 	if (!ptq)
1541 		return 1;
1542 
1543 	switch (ptq->switch_state) {
1544 	case INTEL_PT_SS_NOT_TRACING:
1545 		ptq->next_tid = -1;
1546 		break;
1547 	case INTEL_PT_SS_UNKNOWN:
1548 	case INTEL_PT_SS_TRACING:
1549 		ptq->next_tid = tid;
1550 		ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1551 		return 0;
1552 	case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1553 		if (!ptq->on_heap) {
1554 			ptq->timestamp = perf_time_to_tsc(timestamp,
1555 							  &pt->tc);
1556 			err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1557 						 ptq->timestamp);
1558 			if (err)
1559 				return err;
1560 			ptq->on_heap = true;
1561 		}
1562 		ptq->switch_state = INTEL_PT_SS_TRACING;
1563 		break;
1564 	case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1565 		ptq->next_tid = tid;
1566 		intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1567 		break;
1568 	default:
1569 		break;
1570 	}
1571 
1572 	return 1;
1573 }
1574 
1575 static int intel_pt_process_switch(struct intel_pt *pt,
1576 				   struct perf_sample *sample)
1577 {
1578 	struct perf_evsel *evsel;
1579 	pid_t tid;
1580 	int cpu, ret;
1581 
1582 	evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1583 	if (evsel != pt->switch_evsel)
1584 		return 0;
1585 
1586 	tid = perf_evsel__intval(evsel, sample, "next_pid");
1587 	cpu = sample->cpu;
1588 
1589 	intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1590 		     cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1591 		     &pt->tc));
1592 
1593 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1594 	if (ret <= 0)
1595 		return ret;
1596 
1597 	return machine__set_current_tid(pt->machine, cpu, -1, tid);
1598 }
1599 
1600 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1601 				   struct perf_sample *sample)
1602 {
1603 	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1604 	pid_t pid, tid;
1605 	int cpu, ret;
1606 
1607 	cpu = sample->cpu;
1608 
1609 	if (pt->have_sched_switch == 3) {
1610 		if (!out)
1611 			return 0;
1612 		if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1613 			pr_err("Expecting CPU-wide context switch event\n");
1614 			return -EINVAL;
1615 		}
1616 		pid = event->context_switch.next_prev_pid;
1617 		tid = event->context_switch.next_prev_tid;
1618 	} else {
1619 		if (out)
1620 			return 0;
1621 		pid = sample->pid;
1622 		tid = sample->tid;
1623 	}
1624 
1625 	if (tid == -1) {
1626 		pr_err("context_switch event has no tid\n");
1627 		return -EINVAL;
1628 	}
1629 
1630 	intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1631 		     cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1632 		     &pt->tc));
1633 
1634 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1635 	if (ret <= 0)
1636 		return ret;
1637 
1638 	return machine__set_current_tid(pt->machine, cpu, pid, tid);
1639 }
1640 
1641 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1642 					 union perf_event *event,
1643 					 struct perf_sample *sample)
1644 {
1645 	if (!pt->per_cpu_mmaps)
1646 		return 0;
1647 
1648 	intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1649 		     sample->cpu, event->itrace_start.pid,
1650 		     event->itrace_start.tid, sample->time,
1651 		     perf_time_to_tsc(sample->time, &pt->tc));
1652 
1653 	return machine__set_current_tid(pt->machine, sample->cpu,
1654 					event->itrace_start.pid,
1655 					event->itrace_start.tid);
1656 }
1657 
1658 static int intel_pt_process_event(struct perf_session *session,
1659 				  union perf_event *event,
1660 				  struct perf_sample *sample,
1661 				  struct perf_tool *tool)
1662 {
1663 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1664 					   auxtrace);
1665 	u64 timestamp;
1666 	int err = 0;
1667 
1668 	if (dump_trace)
1669 		return 0;
1670 
1671 	if (!tool->ordered_events) {
1672 		pr_err("Intel Processor Trace requires ordered events\n");
1673 		return -EINVAL;
1674 	}
1675 
1676 	if (sample->time && sample->time != (u64)-1)
1677 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1678 	else
1679 		timestamp = 0;
1680 
1681 	if (timestamp || pt->timeless_decoding) {
1682 		err = intel_pt_update_queues(pt);
1683 		if (err)
1684 			return err;
1685 	}
1686 
1687 	if (pt->timeless_decoding) {
1688 		if (event->header.type == PERF_RECORD_EXIT) {
1689 			err = intel_pt_process_timeless_queues(pt,
1690 							       event->fork.tid,
1691 							       sample->time);
1692 		}
1693 	} else if (timestamp) {
1694 		err = intel_pt_process_queues(pt, timestamp);
1695 	}
1696 	if (err)
1697 		return err;
1698 
1699 	if (event->header.type == PERF_RECORD_AUX &&
1700 	    (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1701 	    pt->synth_opts.errors) {
1702 		err = intel_pt_lost(pt, sample);
1703 		if (err)
1704 			return err;
1705 	}
1706 
1707 	if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1708 		err = intel_pt_process_switch(pt, sample);
1709 	else if (event->header.type == PERF_RECORD_ITRACE_START)
1710 		err = intel_pt_process_itrace_start(pt, event, sample);
1711 	else if (event->header.type == PERF_RECORD_SWITCH ||
1712 		 event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1713 		err = intel_pt_context_switch(pt, event, sample);
1714 
1715 	intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1716 		     perf_event__name(event->header.type), event->header.type,
1717 		     sample->cpu, sample->time, timestamp);
1718 
1719 	return err;
1720 }
1721 
1722 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1723 {
1724 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1725 					   auxtrace);
1726 	int ret;
1727 
1728 	if (dump_trace)
1729 		return 0;
1730 
1731 	if (!tool->ordered_events)
1732 		return -EINVAL;
1733 
1734 	ret = intel_pt_update_queues(pt);
1735 	if (ret < 0)
1736 		return ret;
1737 
1738 	if (pt->timeless_decoding)
1739 		return intel_pt_process_timeless_queues(pt, -1,
1740 							MAX_TIMESTAMP - 1);
1741 
1742 	return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1743 }
1744 
1745 static void intel_pt_free_events(struct perf_session *session)
1746 {
1747 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1748 					   auxtrace);
1749 	struct auxtrace_queues *queues = &pt->queues;
1750 	unsigned int i;
1751 
1752 	for (i = 0; i < queues->nr_queues; i++) {
1753 		intel_pt_free_queue(queues->queue_array[i].priv);
1754 		queues->queue_array[i].priv = NULL;
1755 	}
1756 	intel_pt_log_disable();
1757 	auxtrace_queues__free(queues);
1758 }
1759 
1760 static void intel_pt_free(struct perf_session *session)
1761 {
1762 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1763 					   auxtrace);
1764 
1765 	auxtrace_heap__free(&pt->heap);
1766 	intel_pt_free_events(session);
1767 	session->auxtrace = NULL;
1768 	thread__put(pt->unknown_thread);
1769 	free(pt);
1770 }
1771 
1772 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1773 					   union perf_event *event,
1774 					   struct perf_tool *tool __maybe_unused)
1775 {
1776 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1777 					   auxtrace);
1778 
1779 	if (pt->sampling_mode)
1780 		return 0;
1781 
1782 	if (!pt->data_queued) {
1783 		struct auxtrace_buffer *buffer;
1784 		off_t data_offset;
1785 		int fd = perf_data_file__fd(session->file);
1786 		int err;
1787 
1788 		if (perf_data_file__is_pipe(session->file)) {
1789 			data_offset = 0;
1790 		} else {
1791 			data_offset = lseek(fd, 0, SEEK_CUR);
1792 			if (data_offset == -1)
1793 				return -errno;
1794 		}
1795 
1796 		err = auxtrace_queues__add_event(&pt->queues, session, event,
1797 						 data_offset, &buffer);
1798 		if (err)
1799 			return err;
1800 
1801 		/* Dump here now we have copied a piped trace out of the pipe */
1802 		if (dump_trace) {
1803 			if (auxtrace_buffer__get_data(buffer, fd)) {
1804 				intel_pt_dump_event(pt, buffer->data,
1805 						    buffer->size);
1806 				auxtrace_buffer__put_data(buffer);
1807 			}
1808 		}
1809 	}
1810 
1811 	return 0;
1812 }
1813 
1814 struct intel_pt_synth {
1815 	struct perf_tool dummy_tool;
1816 	struct perf_session *session;
1817 };
1818 
1819 static int intel_pt_event_synth(struct perf_tool *tool,
1820 				union perf_event *event,
1821 				struct perf_sample *sample __maybe_unused,
1822 				struct machine *machine __maybe_unused)
1823 {
1824 	struct intel_pt_synth *intel_pt_synth =
1825 			container_of(tool, struct intel_pt_synth, dummy_tool);
1826 
1827 	return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1828 						 NULL);
1829 }
1830 
1831 static int intel_pt_synth_event(struct perf_session *session,
1832 				struct perf_event_attr *attr, u64 id)
1833 {
1834 	struct intel_pt_synth intel_pt_synth;
1835 
1836 	memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1837 	intel_pt_synth.session = session;
1838 
1839 	return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1840 					   &id, intel_pt_event_synth);
1841 }
1842 
1843 static int intel_pt_synth_events(struct intel_pt *pt,
1844 				 struct perf_session *session)
1845 {
1846 	struct perf_evlist *evlist = session->evlist;
1847 	struct perf_evsel *evsel;
1848 	struct perf_event_attr attr;
1849 	bool found = false;
1850 	u64 id;
1851 	int err;
1852 
1853 	evlist__for_each(evlist, evsel) {
1854 		if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1855 			found = true;
1856 			break;
1857 		}
1858 	}
1859 
1860 	if (!found) {
1861 		pr_debug("There are no selected events with Intel Processor Trace data\n");
1862 		return 0;
1863 	}
1864 
1865 	memset(&attr, 0, sizeof(struct perf_event_attr));
1866 	attr.size = sizeof(struct perf_event_attr);
1867 	attr.type = PERF_TYPE_HARDWARE;
1868 	attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1869 	attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1870 			    PERF_SAMPLE_PERIOD;
1871 	if (pt->timeless_decoding)
1872 		attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1873 	else
1874 		attr.sample_type |= PERF_SAMPLE_TIME;
1875 	if (!pt->per_cpu_mmaps)
1876 		attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1877 	attr.exclude_user = evsel->attr.exclude_user;
1878 	attr.exclude_kernel = evsel->attr.exclude_kernel;
1879 	attr.exclude_hv = evsel->attr.exclude_hv;
1880 	attr.exclude_host = evsel->attr.exclude_host;
1881 	attr.exclude_guest = evsel->attr.exclude_guest;
1882 	attr.sample_id_all = evsel->attr.sample_id_all;
1883 	attr.read_format = evsel->attr.read_format;
1884 
1885 	id = evsel->id[0] + 1000000000;
1886 	if (!id)
1887 		id = 1;
1888 
1889 	if (pt->synth_opts.instructions) {
1890 		attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1891 		if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
1892 			attr.sample_period =
1893 				intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
1894 		else
1895 			attr.sample_period = pt->synth_opts.period;
1896 		pt->instructions_sample_period = attr.sample_period;
1897 		if (pt->synth_opts.callchain)
1898 			attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1899 		if (pt->synth_opts.last_branch)
1900 			attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1901 		pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1902 			 id, (u64)attr.sample_type);
1903 		err = intel_pt_synth_event(session, &attr, id);
1904 		if (err) {
1905 			pr_err("%s: failed to synthesize 'instructions' event type\n",
1906 			       __func__);
1907 			return err;
1908 		}
1909 		pt->sample_instructions = true;
1910 		pt->instructions_sample_type = attr.sample_type;
1911 		pt->instructions_id = id;
1912 		id += 1;
1913 	}
1914 
1915 	if (pt->synth_opts.transactions) {
1916 		attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1917 		attr.sample_period = 1;
1918 		if (pt->synth_opts.callchain)
1919 			attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1920 		if (pt->synth_opts.last_branch)
1921 			attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1922 		pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1923 			 id, (u64)attr.sample_type);
1924 		err = intel_pt_synth_event(session, &attr, id);
1925 		if (err) {
1926 			pr_err("%s: failed to synthesize 'transactions' event type\n",
1927 			       __func__);
1928 			return err;
1929 		}
1930 		pt->sample_transactions = true;
1931 		pt->transactions_id = id;
1932 		id += 1;
1933 		evlist__for_each(evlist, evsel) {
1934 			if (evsel->id && evsel->id[0] == pt->transactions_id) {
1935 				if (evsel->name)
1936 					zfree(&evsel->name);
1937 				evsel->name = strdup("transactions");
1938 				break;
1939 			}
1940 		}
1941 	}
1942 
1943 	if (pt->synth_opts.branches) {
1944 		attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1945 		attr.sample_period = 1;
1946 		attr.sample_type |= PERF_SAMPLE_ADDR;
1947 		attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
1948 		attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
1949 		pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1950 			 id, (u64)attr.sample_type);
1951 		err = intel_pt_synth_event(session, &attr, id);
1952 		if (err) {
1953 			pr_err("%s: failed to synthesize 'branches' event type\n",
1954 			       __func__);
1955 			return err;
1956 		}
1957 		pt->sample_branches = true;
1958 		pt->branches_sample_type = attr.sample_type;
1959 		pt->branches_id = id;
1960 	}
1961 
1962 	pt->synth_needs_swap = evsel->needs_swap;
1963 
1964 	return 0;
1965 }
1966 
1967 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
1968 {
1969 	struct perf_evsel *evsel;
1970 
1971 	evlist__for_each_reverse(evlist, evsel) {
1972 		const char *name = perf_evsel__name(evsel);
1973 
1974 		if (!strcmp(name, "sched:sched_switch"))
1975 			return evsel;
1976 	}
1977 
1978 	return NULL;
1979 }
1980 
1981 static bool intel_pt_find_switch(struct perf_evlist *evlist)
1982 {
1983 	struct perf_evsel *evsel;
1984 
1985 	evlist__for_each(evlist, evsel) {
1986 		if (evsel->attr.context_switch)
1987 			return true;
1988 	}
1989 
1990 	return false;
1991 }
1992 
1993 static int intel_pt_perf_config(const char *var, const char *value, void *data)
1994 {
1995 	struct intel_pt *pt = data;
1996 
1997 	if (!strcmp(var, "intel-pt.mispred-all"))
1998 		pt->mispred_all = perf_config_bool(var, value);
1999 
2000 	return 0;
2001 }
2002 
2003 static const char * const intel_pt_info_fmts[] = {
2004 	[INTEL_PT_PMU_TYPE]		= "  PMU Type            %"PRId64"\n",
2005 	[INTEL_PT_TIME_SHIFT]		= "  Time Shift          %"PRIu64"\n",
2006 	[INTEL_PT_TIME_MULT]		= "  Time Muliplier      %"PRIu64"\n",
2007 	[INTEL_PT_TIME_ZERO]		= "  Time Zero           %"PRIu64"\n",
2008 	[INTEL_PT_CAP_USER_TIME_ZERO]	= "  Cap Time Zero       %"PRId64"\n",
2009 	[INTEL_PT_TSC_BIT]		= "  TSC bit             %#"PRIx64"\n",
2010 	[INTEL_PT_NORETCOMP_BIT]	= "  NoRETComp bit       %#"PRIx64"\n",
2011 	[INTEL_PT_HAVE_SCHED_SWITCH]	= "  Have sched_switch   %"PRId64"\n",
2012 	[INTEL_PT_SNAPSHOT_MODE]	= "  Snapshot mode       %"PRId64"\n",
2013 	[INTEL_PT_PER_CPU_MMAPS]	= "  Per-cpu maps        %"PRId64"\n",
2014 	[INTEL_PT_MTC_BIT]		= "  MTC bit             %#"PRIx64"\n",
2015 	[INTEL_PT_TSC_CTC_N]		= "  TSC:CTC numerator   %"PRIu64"\n",
2016 	[INTEL_PT_TSC_CTC_D]		= "  TSC:CTC denominator %"PRIu64"\n",
2017 	[INTEL_PT_CYC_BIT]		= "  CYC bit             %#"PRIx64"\n",
2018 };
2019 
2020 static void intel_pt_print_info(u64 *arr, int start, int finish)
2021 {
2022 	int i;
2023 
2024 	if (!dump_trace)
2025 		return;
2026 
2027 	for (i = start; i <= finish; i++)
2028 		fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2029 }
2030 
2031 int intel_pt_process_auxtrace_info(union perf_event *event,
2032 				   struct perf_session *session)
2033 {
2034 	struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2035 	size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2036 	struct intel_pt *pt;
2037 	int err;
2038 
2039 	if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2040 					min_sz)
2041 		return -EINVAL;
2042 
2043 	pt = zalloc(sizeof(struct intel_pt));
2044 	if (!pt)
2045 		return -ENOMEM;
2046 
2047 	perf_config(intel_pt_perf_config, pt);
2048 
2049 	err = auxtrace_queues__init(&pt->queues);
2050 	if (err)
2051 		goto err_free;
2052 
2053 	intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2054 
2055 	pt->session = session;
2056 	pt->machine = &session->machines.host; /* No kvm support */
2057 	pt->auxtrace_type = auxtrace_info->type;
2058 	pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2059 	pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2060 	pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2061 	pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2062 	pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2063 	pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2064 	pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2065 	pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2066 	pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2067 	pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2068 	intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2069 			    INTEL_PT_PER_CPU_MMAPS);
2070 
2071 	if (auxtrace_info->header.size >= sizeof(struct auxtrace_info_event) +
2072 					(sizeof(u64) * INTEL_PT_CYC_BIT)) {
2073 		pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2074 		pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2075 		pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2076 		pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2077 		pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2078 		intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2079 				    INTEL_PT_CYC_BIT);
2080 	}
2081 
2082 	pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2083 	pt->have_tsc = intel_pt_have_tsc(pt);
2084 	pt->sampling_mode = false;
2085 	pt->est_tsc = !pt->timeless_decoding;
2086 
2087 	pt->unknown_thread = thread__new(999999999, 999999999);
2088 	if (!pt->unknown_thread) {
2089 		err = -ENOMEM;
2090 		goto err_free_queues;
2091 	}
2092 
2093 	/*
2094 	 * Since this thread will not be kept in any rbtree not in a
2095 	 * list, initialize its list node so that at thread__put() the
2096 	 * current thread lifetime assuption is kept and we don't segfault
2097 	 * at list_del_init().
2098 	 */
2099 	INIT_LIST_HEAD(&pt->unknown_thread->node);
2100 
2101 	err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2102 	if (err)
2103 		goto err_delete_thread;
2104 	if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2105 		err = -ENOMEM;
2106 		goto err_delete_thread;
2107 	}
2108 
2109 	pt->auxtrace.process_event = intel_pt_process_event;
2110 	pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2111 	pt->auxtrace.flush_events = intel_pt_flush;
2112 	pt->auxtrace.free_events = intel_pt_free_events;
2113 	pt->auxtrace.free = intel_pt_free;
2114 	session->auxtrace = &pt->auxtrace;
2115 
2116 	if (dump_trace)
2117 		return 0;
2118 
2119 	if (pt->have_sched_switch == 1) {
2120 		pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2121 		if (!pt->switch_evsel) {
2122 			pr_err("%s: missing sched_switch event\n", __func__);
2123 			goto err_delete_thread;
2124 		}
2125 	} else if (pt->have_sched_switch == 2 &&
2126 		   !intel_pt_find_switch(session->evlist)) {
2127 		pr_err("%s: missing context_switch attribute flag\n", __func__);
2128 		goto err_delete_thread;
2129 	}
2130 
2131 	if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2132 		pt->synth_opts = *session->itrace_synth_opts;
2133 	} else {
2134 		itrace_synth_opts__set_default(&pt->synth_opts);
2135 		if (use_browser != -1) {
2136 			pt->synth_opts.branches = false;
2137 			pt->synth_opts.callchain = true;
2138 		}
2139 	}
2140 
2141 	if (pt->synth_opts.log)
2142 		intel_pt_log_enable();
2143 
2144 	/* Maximum non-turbo ratio is TSC freq / 100 MHz */
2145 	if (pt->tc.time_mult) {
2146 		u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2147 
2148 		pt->max_non_turbo_ratio = (tsc_freq + 50000000) / 100000000;
2149 		intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2150 		intel_pt_log("Maximum non-turbo ratio %u\n",
2151 			     pt->max_non_turbo_ratio);
2152 	}
2153 
2154 	if (pt->synth_opts.calls)
2155 		pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2156 				       PERF_IP_FLAG_TRACE_END;
2157 	if (pt->synth_opts.returns)
2158 		pt->branches_filter |= PERF_IP_FLAG_RETURN |
2159 				       PERF_IP_FLAG_TRACE_BEGIN;
2160 
2161 	if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2162 		symbol_conf.use_callchain = true;
2163 		if (callchain_register_param(&callchain_param) < 0) {
2164 			symbol_conf.use_callchain = false;
2165 			pt->synth_opts.callchain = false;
2166 		}
2167 	}
2168 
2169 	err = intel_pt_synth_events(pt, session);
2170 	if (err)
2171 		goto err_delete_thread;
2172 
2173 	err = auxtrace_queues__process_index(&pt->queues, session);
2174 	if (err)
2175 		goto err_delete_thread;
2176 
2177 	if (pt->queues.populated)
2178 		pt->data_queued = true;
2179 
2180 	if (pt->timeless_decoding)
2181 		pr_debug2("Intel PT decoding without timestamps\n");
2182 
2183 	return 0;
2184 
2185 err_delete_thread:
2186 	thread__zput(pt->unknown_thread);
2187 err_free_queues:
2188 	intel_pt_log_disable();
2189 	auxtrace_queues__free(&pt->queues);
2190 	session->auxtrace = NULL;
2191 err_free:
2192 	free(pt);
2193 	return err;
2194 }
2195