22 #include "intel-pt-insn-decoder.h"
23 #include "intel-pt-pkt-decoder.h"
24 #include "intel-pt-decoder.h"
111 #define INTEL_PT_STATE_ERR1	(decoder->pkt_state)
269 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,  in intel_pt_print_once()  argument
274 	if (decoder->print_once & bit)  in intel_pt_print_once()
276 	decoder->print_once |= bit;  in intel_pt_print_once()
288 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)  in intel_pt_setup_period()  argument
290 	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {  in intel_pt_setup_period()
293 		period = intel_pt_lower_power_of_2(decoder->period);  in intel_pt_setup_period()
294 		decoder->period_mask  = ~(period - 1);  in intel_pt_setup_period()
295 		decoder->period_ticks = period;  in intel_pt_setup_period()
308 	struct intel_pt_decoder *decoder;  in intel_pt_decoder_new()  local
313 	decoder = zalloc(sizeof(struct intel_pt_decoder));  in intel_pt_decoder_new()
314 	if (!decoder)  in intel_pt_decoder_new()
317 	decoder->get_trace          = params->get_trace;  in intel_pt_decoder_new()
318 	decoder->walk_insn          = params->walk_insn;  in intel_pt_decoder_new()
319 	decoder->pgd_ip             = params->pgd_ip;  in intel_pt_decoder_new()
320 	decoder->lookahead          = params->lookahead;  in intel_pt_decoder_new()
321 	decoder->findnew_vmcs_info  = params->findnew_vmcs_info;  in intel_pt_decoder_new()
322 	decoder->data               = params->data;  in intel_pt_decoder_new()
323 	decoder->return_compression = params->return_compression;  in intel_pt_decoder_new()
324 	decoder->branch_enable      = params->branch_enable;  in intel_pt_decoder_new()
325 	decoder->hop                = params->quick >= 1;  in intel_pt_decoder_new()
326 	decoder->leap               = params->quick >= 2;  in intel_pt_decoder_new()
327 	decoder->vm_time_correlation = params->vm_time_correlation;  in intel_pt_decoder_new()
328 	decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;  in intel_pt_decoder_new()
329 	decoder->first_timestamp    = params->first_timestamp;  in intel_pt_decoder_new()
330 	decoder->last_reliable_timestamp = params->first_timestamp;  in intel_pt_decoder_new()
331 	decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;  in intel_pt_decoder_new()
333 	decoder->flags              = params->flags;  in intel_pt_decoder_new()
335 	decoder->ctl                = params->ctl;  in intel_pt_decoder_new()
336 	decoder->period             = params->period;  in intel_pt_decoder_new()
337 	decoder->period_type        = params->period_type;  in intel_pt_decoder_new()
339 	decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
340 	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
342 	decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);  in intel_pt_decoder_new()
344 	intel_pt_setup_period(decoder);  in intel_pt_decoder_new()
346 	decoder->mtc_shift = params->mtc_period;  in intel_pt_decoder_new()
347 	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;  in intel_pt_decoder_new()
349 	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;  in intel_pt_decoder_new()
350 	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
352 	if (!decoder->tsc_ctc_ratio_n)  in intel_pt_decoder_new()
353 		decoder->tsc_ctc_ratio_d = 0;  in intel_pt_decoder_new()
355 	if (decoder->tsc_ctc_ratio_d) {  in intel_pt_decoder_new()
356 		if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))  in intel_pt_decoder_new()
357 			decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /  in intel_pt_decoder_new()
358 						decoder->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
367 	decoder->tsc_slip = 0x10000;  in intel_pt_decoder_new()
369 	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);  in intel_pt_decoder_new()
370 	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);  in intel_pt_decoder_new()
371 	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);  in intel_pt_decoder_new()
372 	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);  in intel_pt_decoder_new()
373 	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);  in intel_pt_decoder_new()
375 	if (decoder->hop)  in intel_pt_decoder_new()
378 	return decoder;  in intel_pt_decoder_new()
381 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,  in intel_pt_set_first_timestamp()  argument
384 	decoder->first_timestamp = first_timestamp;  in intel_pt_set_first_timestamp()
458 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)  in intel_pt_decoder_free()  argument
460 	intel_pt_free_stack(&decoder->stack);  in intel_pt_decoder_free()
461 	free(decoder);  in intel_pt_decoder_free()
548 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_last_ip()  argument
550 	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);  in intel_pt_set_last_ip()
551 	decoder->have_last_ip = true;  in intel_pt_set_last_ip()
554 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_ip()  argument
556 	intel_pt_set_last_ip(decoder);  in intel_pt_set_ip()
557 	decoder->ip = decoder->last_ip;  in intel_pt_set_ip()
560 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)  in intel_pt_decoder_log_packet()  argument
562 	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,  in intel_pt_decoder_log_packet()
563 			    decoder->buf);  in intel_pt_decoder_log_packet()
566 static int intel_pt_bug(struct intel_pt_decoder *decoder)  in intel_pt_bug()  argument
569 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_bug()
573 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)  in intel_pt_clear_tx_flags()  argument
575 	decoder->tx_flags = 0;  in intel_pt_clear_tx_flags()
578 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)  in intel_pt_update_in_tx()  argument
580 	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;  in intel_pt_update_in_tx()
583 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)  in intel_pt_update_pip()  argument
585 	decoder->pip_payload = decoder->packet.payload;  in intel_pt_update_pip()
588 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)  in intel_pt_update_nr()  argument
590 	decoder->next_nr = decoder->pip_payload & 1;  in intel_pt_update_nr()
593 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)  in intel_pt_set_nr()  argument
595 	decoder->nr = decoder->pip_payload & 1;  in intel_pt_set_nr()
596 	decoder->next_nr = decoder->nr;  in intel_pt_set_nr()
599 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)  in intel_pt_set_pip()  argument
601 	intel_pt_update_pip(decoder);  in intel_pt_set_pip()
602 	intel_pt_set_nr(decoder);  in intel_pt_set_pip()
605 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)  in intel_pt_bad_packet()  argument
607 	intel_pt_clear_tx_flags(decoder);  in intel_pt_bad_packet()
608 	decoder->have_tma = false;  in intel_pt_bad_packet()
609 	decoder->pkt_len = 1;  in intel_pt_bad_packet()
610 	decoder->pkt_step = 1;  in intel_pt_bad_packet()
611 	intel_pt_decoder_log_packet(decoder);  in intel_pt_bad_packet()
612 	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {  in intel_pt_bad_packet()
614 		decoder->pkt_state = INTEL_PT_STATE_ERR1;  in intel_pt_bad_packet()
619 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)  in intel_pt_update_sample_time()  argument
621 	decoder->sample_timestamp = decoder->timestamp;  in intel_pt_update_sample_time()
622 	decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;  in intel_pt_update_sample_time()
623 	decoder->state.cycles = decoder->tot_cyc_cnt;  in intel_pt_update_sample_time()
626 static void intel_pt_reposition(struct intel_pt_decoder *decoder)  in intel_pt_reposition()  argument
628 	decoder->ip = 0;  in intel_pt_reposition()
629 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_reposition()
630 	decoder->timestamp = 0;  in intel_pt_reposition()
631 	decoder->have_tma = false;  in intel_pt_reposition()
634 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)  in intel_pt_get_data()  argument
639 	decoder->pkt_step = 0;  in intel_pt_get_data()
642 	ret = decoder->get_trace(&buffer, decoder->data);  in intel_pt_get_data()
645 	decoder->buf = buffer.buf;  in intel_pt_get_data()
646 	decoder->len = buffer.len;  in intel_pt_get_data()
647 	if (!decoder->len) {  in intel_pt_get_data()
651 	decoder->buf_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
653 		intel_pt_reposition(decoder);  in intel_pt_get_data()
654 		decoder->ref_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
655 		decoder->state.trace_nr = buffer.trace_nr;  in intel_pt_get_data()
656 		decoder->vm_tm_corr_same_buf = false;  in intel_pt_get_data()
658 			     decoder->ref_timestamp);  in intel_pt_get_data()
665 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,  in intel_pt_get_next_data()  argument
668 	if (!decoder->next_buf)  in intel_pt_get_next_data()
669 		return intel_pt_get_data(decoder, reposition);  in intel_pt_get_next_data()
671 	decoder->buf = decoder->next_buf;  in intel_pt_get_next_data()
672 	decoder->len = decoder->next_len;  in intel_pt_get_next_data()
673 	decoder->next_buf = 0;  in intel_pt_get_next_data()
674 	decoder->next_len = 0;  in intel_pt_get_next_data()
678 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_split_packet()  argument
680 	unsigned char *buf = decoder->temp_buf;  in intel_pt_get_split_packet()
684 	old_len = decoder->len;  in intel_pt_get_split_packet()
685 	len = decoder->len;  in intel_pt_get_split_packet()
686 	memcpy(buf, decoder->buf, len);  in intel_pt_get_split_packet()
688 	ret = intel_pt_get_data(decoder, false);  in intel_pt_get_split_packet()
690 		decoder->pos += old_len;  in intel_pt_get_split_packet()
695 	if (n > decoder->len)  in intel_pt_get_split_packet()
696 		n = decoder->len;  in intel_pt_get_split_packet()
697 	memcpy(buf + len, decoder->buf, n);  in intel_pt_get_split_packet()
700 	decoder->prev_pkt_ctx = decoder->pkt_ctx;  in intel_pt_get_split_packet()
701 	ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);  in intel_pt_get_split_packet()
703 		decoder->next_buf = decoder->buf;  in intel_pt_get_split_packet()
704 		decoder->next_len = decoder->len;  in intel_pt_get_split_packet()
705 		decoder->buf = buf;  in intel_pt_get_split_packet()
706 		decoder->len = old_len;  in intel_pt_get_split_packet()
707 		return intel_pt_bad_packet(decoder);  in intel_pt_get_split_packet()
710 	decoder->next_buf = decoder->buf + (ret - old_len);  in intel_pt_get_split_packet()
711 	decoder->next_len = decoder->len - (ret - old_len);  in intel_pt_get_split_packet()
713 	decoder->buf = buf;  in intel_pt_get_split_packet()
714 	decoder->len = ret;  in intel_pt_get_split_packet()
720 	struct intel_pt_decoder	  *decoder;  member
731 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,  in intel_pt_pkt_lookahead()  argument
735 	const unsigned char *buf = decoder->buf;  in intel_pt_pkt_lookahead()
736 	enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;  in intel_pt_pkt_lookahead()
737 	size_t len = decoder->len;  in intel_pt_pkt_lookahead()
740 	pkt_info.decoder          = decoder;  in intel_pt_pkt_lookahead()
741 	pkt_info.pos              = decoder->pos;  in intel_pt_pkt_lookahead()
742 	pkt_info.pkt_len          = decoder->pkt_step;  in intel_pt_pkt_lookahead()
743 	pkt_info.last_packet_type = decoder->last_packet_type;  in intel_pt_pkt_lookahead()
809 	struct intel_pt_decoder *decoder = pkt_info->decoder;  in intel_pt_calc_cyc_cb()  local
845 		if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {  in intel_pt_calc_cyc_cb()
847 			intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_cyc_cb()
854 		data->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_cyc_cb()
857 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
859 				data->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
863 					decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
864 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
887 		    data->timestamp - timestamp < decoder->tsc_slip)  in intel_pt_calc_cyc_cb()
904 		if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_cyc_cb()
909 		ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_cyc_cb()
911 		data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_cyc_cb()
914 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
915 			data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
918 				multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
919 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
937 		data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cyc_cb()
953 	if (!data->cbr && decoder->cbr) {  in intel_pt_calc_cyc_cb()
954 		data->cbr = decoder->cbr;  in intel_pt_calc_cyc_cb()
955 		data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_cb()
961 	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;  in intel_pt_calc_cyc_cb()
970 	decoder->calc_cyc_to_tsc = cyc_to_tsc;  in intel_pt_calc_cyc_cb()
971 	decoder->have_calc_cyc_to_tsc = true;  in intel_pt_calc_cyc_cb()
984 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,  in intel_pt_calc_cyc_to_tsc()  argument
990 		.last_mtc       = decoder->last_mtc,  in intel_pt_calc_cyc_to_tsc()
991 		.ctc_timestamp  = decoder->ctc_timestamp,  in intel_pt_calc_cyc_to_tsc()
992 		.ctc_delta      = decoder->ctc_delta,  in intel_pt_calc_cyc_to_tsc()
993 		.tsc_timestamp  = decoder->tsc_timestamp,  in intel_pt_calc_cyc_to_tsc()
994 		.timestamp      = decoder->timestamp,  in intel_pt_calc_cyc_to_tsc()
995 		.have_tma       = decoder->have_tma,  in intel_pt_calc_cyc_to_tsc()
996 		.fixup_last_mtc = decoder->fixup_last_mtc,  in intel_pt_calc_cyc_to_tsc()
1009 	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);  in intel_pt_calc_cyc_to_tsc()
1012 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_next_packet()  argument
1016 	decoder->last_packet_type = decoder->packet.type;  in intel_pt_get_next_packet()
1019 		decoder->pos += decoder->pkt_step;  in intel_pt_get_next_packet()
1020 		decoder->buf += decoder->pkt_step;  in intel_pt_get_next_packet()
1021 		decoder->len -= decoder->pkt_step;  in intel_pt_get_next_packet()
1023 		if (!decoder->len) {  in intel_pt_get_next_packet()
1024 			ret = intel_pt_get_next_data(decoder, false);  in intel_pt_get_next_packet()
1029 		decoder->prev_pkt_ctx = decoder->pkt_ctx;  in intel_pt_get_next_packet()
1030 		ret = intel_pt_get_packet(decoder->buf, decoder->len,  in intel_pt_get_next_packet()
1031 					  &decoder->packet, &decoder->pkt_ctx);  in intel_pt_get_next_packet()
1033 		    decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {  in intel_pt_get_next_packet()
1034 			ret = intel_pt_get_split_packet(decoder);  in intel_pt_get_next_packet()
1039 			return intel_pt_bad_packet(decoder);  in intel_pt_get_next_packet()
1041 		decoder->pkt_len = ret;  in intel_pt_get_next_packet()
1042 		decoder->pkt_step = ret;  in intel_pt_get_next_packet()
1043 		intel_pt_decoder_log_packet(decoder);  in intel_pt_get_next_packet()
1044 	} while (decoder->packet.type == INTEL_PT_PAD);  in intel_pt_get_next_packet()
1049 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)  in intel_pt_next_period()  argument
1053 	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_next_period()
1054 	masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
1055 	if (decoder->continuous_period) {  in intel_pt_next_period()
1056 		if (masked_timestamp > decoder->last_masked_timestamp)  in intel_pt_next_period()
1060 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
1061 		if (masked_timestamp > decoder->last_masked_timestamp) {  in intel_pt_next_period()
1062 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_next_period()
1063 			decoder->continuous_period = true;  in intel_pt_next_period()
1067 	if (masked_timestamp < decoder->last_masked_timestamp)  in intel_pt_next_period()
1068 		return decoder->period_ticks;  in intel_pt_next_period()
1070 	return decoder->period_ticks - (timestamp - masked_timestamp);  in intel_pt_next_period()
1073 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)  in intel_pt_next_sample()  argument
1075 	switch (decoder->period_type) {  in intel_pt_next_sample()
1077 		return decoder->period - decoder->period_insn_cnt;  in intel_pt_next_sample()
1079 		return intel_pt_next_period(decoder);  in intel_pt_next_sample()
1087 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)  in intel_pt_sample_insn()  argument
1091 	switch (decoder->period_type) {  in intel_pt_sample_insn()
1093 		decoder->period_insn_cnt = 0;  in intel_pt_sample_insn()
1096 		timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_sample_insn()
1097 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_sample_insn()
1098 		if (masked_timestamp > decoder->last_masked_timestamp)  in intel_pt_sample_insn()
1099 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_sample_insn()
1101 			decoder->last_masked_timestamp += decoder->period_ticks;  in intel_pt_sample_insn()
1109 	decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_sample_insn()
1116 static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder)  in intel_pt_sample_fup_insn()  argument
1122 	decoder->state.insn_op = INTEL_PT_OP_OTHER;  in intel_pt_sample_fup_insn()
1123 	decoder->state.insn_len = 0;  in intel_pt_sample_fup_insn()
1125 	if (!decoder->branch_enable || !decoder->pge || decoder->hop ||  in intel_pt_sample_fup_insn()
1126 	    decoder->ip != decoder->last_ip)  in intel_pt_sample_fup_insn()
1129 	if (!decoder->mtc_insn)  in intel_pt_sample_fup_insn()
1130 		decoder->mtc_insn = true;  in intel_pt_sample_fup_insn()
1132 	max_insn_cnt = intel_pt_next_sample(decoder);  in intel_pt_sample_fup_insn()
1136 	err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip,  in intel_pt_sample_fup_insn()
1137 				 0, max_insn_cnt, decoder->data);  in intel_pt_sample_fup_insn()
1143 		intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip);  in intel_pt_sample_fup_insn()
1147 	decoder->tot_insn_cnt += insn_cnt;  in intel_pt_sample_fup_insn()
1148 	decoder->timestamp_insn_cnt += insn_cnt;  in intel_pt_sample_fup_insn()
1149 	decoder->sample_insn_cnt += insn_cnt;  in intel_pt_sample_fup_insn()
1150 	decoder->period_insn_cnt += insn_cnt;  in intel_pt_sample_fup_insn()
1152 	intel_pt_sample_insn(decoder);  in intel_pt_sample_fup_insn()
1154 	decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_sample_fup_insn()
1155 	decoder->ip += intel_pt_insn.length;  in intel_pt_sample_fup_insn()
1158 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,  in intel_pt_walk_insn()  argument
1164 	if (!decoder->mtc_insn)  in intel_pt_walk_insn()
1165 		decoder->mtc_insn = true;  in intel_pt_walk_insn()
1167 	max_insn_cnt = intel_pt_next_sample(decoder);  in intel_pt_walk_insn()
1169 	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,  in intel_pt_walk_insn()
1170 				 max_insn_cnt, decoder->data);  in intel_pt_walk_insn()
1172 	decoder->tot_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1173 	decoder->timestamp_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1174 	decoder->sample_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1175 	decoder->period_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1178 		decoder->no_progress = 0;  in intel_pt_walk_insn()
1179 		decoder->pkt_state = INTEL_PT_STATE_ERR2;  in intel_pt_walk_insn()
1181 				decoder->ip);  in intel_pt_walk_insn()
1187 	if (ip && decoder->ip == ip) {  in intel_pt_walk_insn()
1193 		intel_pt_sample_insn(decoder);  in intel_pt_walk_insn()
1196 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_insn()
1197 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1198 		decoder->state.to_ip = 0;  in intel_pt_walk_insn()
1199 		decoder->ip += intel_pt_insn->length;  in intel_pt_walk_insn()
1208 			err = intel_pt_push(&decoder->stack, decoder->ip +  in intel_pt_walk_insn()
1214 		decoder->ret_addr = intel_pt_pop(&decoder->stack);  in intel_pt_walk_insn()
1218 		int cnt = decoder->no_progress++;  in intel_pt_walk_insn()
1220 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1221 		decoder->ip += intel_pt_insn->length +  in intel_pt_walk_insn()
1223 		decoder->state.to_ip = decoder->ip;  in intel_pt_walk_insn()
1228 		 * decoder error results in the decoder erroneously setting the  in intel_pt_walk_insn()
1235 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1236 				decoder->stuck_ip_prd = 1;  in intel_pt_walk_insn()
1237 				decoder->stuck_ip_cnt = 1;  in intel_pt_walk_insn()
1238 			} else if (cnt > decoder->max_loops ||  in intel_pt_walk_insn()
1239 				   decoder->state.to_ip == decoder->stuck_ip) {  in intel_pt_walk_insn()
1241 						decoder->state.to_ip);  in intel_pt_walk_insn()
1242 				decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_insn()
1245 			} else if (!--decoder->stuck_ip_cnt) {  in intel_pt_walk_insn()
1246 				decoder->stuck_ip_prd += 1;  in intel_pt_walk_insn()
1247 				decoder->stuck_ip_cnt = decoder->stuck_ip_prd;  in intel_pt_walk_insn()
1248 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1254 	decoder->no_progress = 0;  in intel_pt_walk_insn()
1256 	decoder->state.insn_op = intel_pt_insn->op;  in intel_pt_walk_insn()
1257 	decoder->state.insn_len = intel_pt_insn->length;  in intel_pt_walk_insn()
1258 	memcpy(decoder->state.insn, intel_pt_insn->buf,  in intel_pt_walk_insn()
1261 	if (decoder->tx_flags & INTEL_PT_IN_TX)  in intel_pt_walk_insn()
1262 		decoder->state.flags |= INTEL_PT_IN_TX;  in intel_pt_walk_insn()
1267 static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder)  in intel_pt_mode_exec_status()  argument
1269 	bool iflag = decoder->packet.count & INTEL_PT_IFLAG;  in intel_pt_mode_exec_status()
1271 	decoder->exec_mode = decoder->packet.payload;  in intel_pt_mode_exec_status()
1272 	decoder->iflag = iflag;  in intel_pt_mode_exec_status()
1273 	decoder->next_iflag = iflag;  in intel_pt_mode_exec_status()
1274 	decoder->state.from_iflag = iflag;  in intel_pt_mode_exec_status()
1275 	decoder->state.to_iflag = iflag;  in intel_pt_mode_exec_status()
1278 static void intel_pt_mode_exec(struct intel_pt_decoder *decoder)  in intel_pt_mode_exec()  argument
1280 	bool iflag = decoder->packet.count & INTEL_PT_IFLAG;  in intel_pt_mode_exec()
1282 	decoder->exec_mode = decoder->packet.payload;  in intel_pt_mode_exec()
1283 	decoder->next_iflag = iflag;  in intel_pt_mode_exec()
1286 static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder)  in intel_pt_sample_iflag()  argument
1288 	decoder->state.type |= INTEL_PT_IFLAG_CHG;  in intel_pt_sample_iflag()
1289 	decoder->state.from_iflag = decoder->iflag;  in intel_pt_sample_iflag()
1290 	decoder->state.to_iflag = decoder->next_iflag;  in intel_pt_sample_iflag()
1291 	decoder->iflag = decoder->next_iflag;  in intel_pt_sample_iflag()
1294 static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder)  in intel_pt_sample_iflag_chg()  argument
1296 	if (decoder->iflag != decoder->next_iflag)  in intel_pt_sample_iflag_chg()
1297 		intel_pt_sample_iflag(decoder);  in intel_pt_sample_iflag_chg()
1300 static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder)  in intel_pt_clear_fup_event()  argument
1302 	decoder->set_fup_tx_flags = false;  in intel_pt_clear_fup_event()
1303 	decoder->set_fup_ptw = false;  in intel_pt_clear_fup_event()
1304 	decoder->set_fup_mwait = false;  in intel_pt_clear_fup_event()
1305 	decoder->set_fup_pwre = false;  in intel_pt_clear_fup_event()
1306 	decoder->set_fup_exstop = false;  in intel_pt_clear_fup_event()
1307 	decoder->set_fup_bep = false;  in intel_pt_clear_fup_event()
1308 	decoder->set_fup_cfe_ip = false;  in intel_pt_clear_fup_event()
1309 	decoder->set_fup_cfe = false;  in intel_pt_clear_fup_event()
1310 	decoder->evd_cnt = 0;  in intel_pt_clear_fup_event()
1311 	decoder->set_fup_mode_exec = false;  in intel_pt_clear_fup_event()
1312 	decoder->iflag = decoder->next_iflag;  in intel_pt_clear_fup_event()
1315 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip)  in intel_pt_fup_event()  argument
1317 	enum intel_pt_sample_type type = decoder->state.type;  in intel_pt_fup_event()
1321 	decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1322 	decoder->state.insn_op = INTEL_PT_OP_OTHER;  in intel_pt_fup_event()
1323 	decoder->state.insn_len = 0;  in intel_pt_fup_event()
1325 	if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) {  in intel_pt_fup_event()
1326 		bool ip = decoder->set_fup_cfe_ip;  in intel_pt_fup_event()
1328 		decoder->set_fup_cfe_ip = false;  in intel_pt_fup_event()
1329 		decoder->set_fup_cfe = false;  in intel_pt_fup_event()
1330 		decoder->state.type |= INTEL_PT_EVT;  in intel_pt_fup_event()
1331 		if (!ip && decoder->pge)  in intel_pt_fup_event()
1332 			decoder->state.type |= INTEL_PT_BRANCH;  in intel_pt_fup_event()
1333 		decoder->state.cfe_type = decoder->fup_cfe_pkt.count;  in intel_pt_fup_event()
1334 		decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload;  in intel_pt_fup_event()
1335 		decoder->state.evd_cnt = decoder->evd_cnt;  in intel_pt_fup_event()
1336 		decoder->state.evd = decoder->evd;  in intel_pt_fup_event()
1337 		decoder->evd_cnt = 0;  in intel_pt_fup_event()
1338 		if (ip || decoder->pge)  in intel_pt_fup_event()
1339 			decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1342 	if (decoder->set_fup_mode_exec) {  in intel_pt_fup_event()
1343 		decoder->set_fup_mode_exec = false;  in intel_pt_fup_event()
1344 		intel_pt_sample_iflag(decoder);  in intel_pt_fup_event()
1348 	if (decoder->set_fup_tx_flags) {  in intel_pt_fup_event()
1349 		decoder->set_fup_tx_flags = false;  in intel_pt_fup_event()
1350 		decoder->tx_flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1351 		decoder->state.type |= INTEL_PT_TRANSACTION;  in intel_pt_fup_event()
1352 		if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)  in intel_pt_fup_event()
1353 			decoder->state.type |= INTEL_PT_BRANCH;  in intel_pt_fup_event()
1354 		decoder->state.flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1357 	if (decoder->set_fup_ptw) {  in intel_pt_fup_event()
1358 		decoder->set_fup_ptw = false;  in intel_pt_fup_event()
1359 		decoder->state.type |= INTEL_PT_PTW;  in intel_pt_fup_event()
1360 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1361 		decoder->state.ptw_payload = decoder->fup_ptw_payload;  in intel_pt_fup_event()
1364 	if (decoder->set_fup_mwait) {  in intel_pt_fup_event()
1365 		decoder->set_fup_mwait = false;  in intel_pt_fup_event()
1366 		decoder->state.type |= INTEL_PT_MWAIT_OP;  in intel_pt_fup_event()
1367 		decoder->state.mwait_payload = decoder->fup_mwait_payload;  in intel_pt_fup_event()
1370 	if (decoder->set_fup_pwre) {  in intel_pt_fup_event()
1371 		decoder->set_fup_pwre = false;  in intel_pt_fup_event()
1372 		decoder->state.type |= INTEL_PT_PWR_ENTRY;  in intel_pt_fup_event()
1373 		decoder->state.pwre_payload = decoder->fup_pwre_payload;  in intel_pt_fup_event()
1376 	if (decoder->set_fup_exstop) {  in intel_pt_fup_event()
1377 		decoder->set_fup_exstop = false;  in intel_pt_fup_event()
1378 		decoder->state.type |= INTEL_PT_EX_STOP;  in intel_pt_fup_event()
1379 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1382 	if (decoder->set_fup_bep) {  in intel_pt_fup_event()
1383 		decoder->set_fup_bep = false;  in intel_pt_fup_event()
1384 		decoder->state.type |= INTEL_PT_BLK_ITEMS;  in intel_pt_fup_event()
1387 	if (decoder->overflow) {  in intel_pt_fup_event()
1388 		decoder->overflow = false;  in intel_pt_fup_event()
1389 		if (!ret && !decoder->pge) {  in intel_pt_fup_event()
1390 			if (decoder->hop) {  in intel_pt_fup_event()
1391 				decoder->state.type = 0;  in intel_pt_fup_event()
1392 				decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_fup_event()
1394 			decoder->pge = true;  in intel_pt_fup_event()
1395 			decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;  in intel_pt_fup_event()
1396 			decoder->state.from_ip = 0;  in intel_pt_fup_event()
1397 			decoder->state.to_ip = decoder->ip;  in intel_pt_fup_event()
1402 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1403 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1405 			intel_pt_sample_fup_insn(decoder);  in intel_pt_fup_event()
1407 		decoder->state.type = type;  in intel_pt_fup_event()
1412 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,  in intel_pt_fup_with_nlip()  argument
1416 	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&  in intel_pt_fup_with_nlip()
1418 	       ip == decoder->ip + intel_pt_insn->length;  in intel_pt_fup_with_nlip()
1421 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup()  argument
1427 	ip = decoder->last_ip;  in intel_pt_walk_fup()
1430 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);  in intel_pt_walk_fup()
1434 		    intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {  in intel_pt_walk_fup()
1435 			bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;  in intel_pt_walk_fup()
1437 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_fup()
1438 			if (intel_pt_fup_event(decoder, no_tip) && no_tip)  in intel_pt_walk_fup()
1442 		decoder->set_fup_tx_flags = false;  in intel_pt_walk_fup()
1448 					decoder->ip);  in intel_pt_walk_fup()
1449 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1455 					decoder->ip);  in intel_pt_walk_fup()
1456 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1460 		intel_pt_bug(decoder);  in intel_pt_walk_fup()
1464 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_tip()  argument
1469 	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tip()
1471 	    decoder->pgd_ip &&  in intel_pt_walk_tip()
1472 	    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1473 	    (decoder->state.type & INTEL_PT_BRANCH) &&  in intel_pt_walk_tip()
1474 	    decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {  in intel_pt_walk_tip()
1476 		decoder->no_progress = 0;  in intel_pt_walk_tip()
1477 		decoder->pge = false;  in intel_pt_walk_tip()
1478 		decoder->continuous_period = false;  in intel_pt_walk_tip()
1479 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1480 		decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1481 		intel_pt_update_nr(decoder);  in intel_pt_walk_tip()
1489 	intel_pt_update_nr(decoder);  in intel_pt_walk_tip()
1490 	intel_pt_sample_iflag_chg(decoder);  in intel_pt_walk_tip()
1493 		if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {  in intel_pt_walk_tip()
1494 			decoder->pge = false;  in intel_pt_walk_tip()
1495 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1496 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1497 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1498 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1499 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1501 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1502 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1504 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1506 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1507 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1508 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1509 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1511 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1512 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1519 		uint64_t to_ip = decoder->ip + intel_pt_insn.length +  in intel_pt_walk_tip()
1522 		if (decoder->pgd_ip &&  in intel_pt_walk_tip()
1523 		    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1524 		    decoder->pgd_ip(to_ip, decoder->data)) {  in intel_pt_walk_tip()
1526 			decoder->pge = false;  in intel_pt_walk_tip()
1527 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1528 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1529 			decoder->ip = to_ip;  in intel_pt_walk_tip()
1530 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1531 			decoder->state.to_ip = to_ip;  in intel_pt_walk_tip()
1532 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1536 				decoder->ip);  in intel_pt_walk_tip()
1537 		decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_tip()
1541 	return intel_pt_bug(decoder);  in intel_pt_walk_tip()
1607 static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder)  in intel_pt_emulated_ptwrite()  argument
1609 	int n = 64 - decoder->tnt.count;  in intel_pt_emulated_ptwrite()
1612 		.payload = decoder->tnt.payload >> n,  in intel_pt_emulated_ptwrite()
1615 	decoder->emulated_ptwrite = false;  in intel_pt_emulated_ptwrite()
1618 	intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data);  in intel_pt_emulated_ptwrite()
1622 	decoder->state.type = INTEL_PT_PTW;  in intel_pt_emulated_ptwrite()
1623 	decoder->state.from_ip = decoder->ip;  in intel_pt_emulated_ptwrite()
1624 	decoder->state.to_ip = 0;  in intel_pt_emulated_ptwrite()
1625 	decoder->state.ptw_payload = data.payload;  in intel_pt_emulated_ptwrite()
1629 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)  in intel_pt_walk_tnt()  argument
1635 		if (decoder->emulated_ptwrite)  in intel_pt_walk_tnt()
1636 			return intel_pt_emulated_ptwrite(decoder);  in intel_pt_walk_tnt()
1637 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tnt()
1639 			decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite;  in intel_pt_walk_tnt()
1643 			decoder->emulated_ptwrite = false;  in intel_pt_walk_tnt()
1648 			if (!decoder->return_compression) {  in intel_pt_walk_tnt()
1650 						decoder->ip);  in intel_pt_walk_tnt()
1651 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1654 			if (!decoder->ret_addr) {  in intel_pt_walk_tnt()
1656 						decoder->ip);  in intel_pt_walk_tnt()
1657 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1660 			if (!(decoder->tnt.payload & BIT63)) {  in intel_pt_walk_tnt()
1662 						decoder->ip);  in intel_pt_walk_tnt()
1663 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1666 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1667 			if (decoder->tnt.count)  in intel_pt_walk_tnt()
1668 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;  in intel_pt_walk_tnt()
1670 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1671 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1672 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1673 			decoder->ip = decoder->ret_addr;  in intel_pt_walk_tnt()
1674 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1680 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_tnt()
1683 			if (decoder->packet.type != INTEL_PT_TIP ||  in intel_pt_walk_tnt()
1684 			    decoder->packet.count == 0) {  in intel_pt_walk_tnt()
1686 						decoder->ip);  in intel_pt_walk_tnt()
1687 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1688 				decoder->pkt_step = 0;  in intel_pt_walk_tnt()
1691 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_tnt()
1692 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1693 			decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tnt()
1694 			decoder->ip = decoder->last_ip;  in intel_pt_walk_tnt()
1695 			intel_pt_update_nr(decoder);  in intel_pt_walk_tnt()
1696 			intel_pt_sample_iflag_chg(decoder);  in intel_pt_walk_tnt()
1701 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1702 			if (decoder->tnt.count)  in intel_pt_walk_tnt()
1703 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;  in intel_pt_walk_tnt()
1705 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1706 			if (decoder->tnt.payload & BIT63) {  in intel_pt_walk_tnt()
1707 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1708 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1709 				decoder->ip += intel_pt_insn.length +  in intel_pt_walk_tnt()
1711 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1715 			if (decoder->state.type & INTEL_PT_INSTRUCTION) {  in intel_pt_walk_tnt()
1716 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1717 				decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_tnt()
1718 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1719 				decoder->state.to_ip = 0;  in intel_pt_walk_tnt()
1720 				decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1723 			decoder->sample_cyc = false;  in intel_pt_walk_tnt()
1724 			decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1725 			if (!decoder->tnt.count) {  in intel_pt_walk_tnt()
1726 				intel_pt_update_sample_time(decoder);  in intel_pt_walk_tnt()
1729 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1733 		return intel_pt_bug(decoder);  in intel_pt_walk_tnt()
1737 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)  in intel_pt_mode_tsx()  argument
1742 	fup_tx_flags = decoder->packet.payload &  in intel_pt_mode_tsx()
1744 	err = intel_pt_get_next_packet(decoder);  in intel_pt_mode_tsx()
1747 	if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_mode_tsx()
1748 		decoder->fup_tx_flags = fup_tx_flags;  in intel_pt_mode_tsx()
1749 		decoder->set_fup_tx_flags = true;  in intel_pt_mode_tsx()
1750 		if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))  in intel_pt_mode_tsx()
1754 				decoder->pos);  in intel_pt_mode_tsx()
1755 		intel_pt_update_in_tx(decoder);  in intel_pt_mode_tsx()
1760 static int intel_pt_evd(struct intel_pt_decoder *decoder)  in intel_pt_evd()  argument
1762 	if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) {  in intel_pt_evd()
1763 		intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos);  in intel_pt_evd()
1766 	decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){  in intel_pt_evd()
1767 		.type = decoder->packet.count,  in intel_pt_evd()
1768 		.payload = decoder->packet.payload,  in intel_pt_evd()
1788 /* For use only when decoder->vm_time_correlation is true */
1789 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,  in intel_pt_time_in_range()  argument
1792 	uint64_t max_timestamp = decoder->buf_timestamp;  in intel_pt_time_in_range()
1795 		max_timestamp = decoder->last_reliable_timestamp +  in intel_pt_time_in_range()
1798 	return timestamp >= decoder->last_reliable_timestamp &&  in intel_pt_time_in_range()
1799 	       timestamp < decoder->buf_timestamp;  in intel_pt_time_in_range()
1802 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_tsc_timestamp()  argument
1807 	decoder->have_tma = false;  in intel_pt_calc_tsc_timestamp()
1809 	if (decoder->ref_timestamp) {  in intel_pt_calc_tsc_timestamp()
1810 		timestamp = intel_pt_8b_tsc(decoder->packet.payload,  in intel_pt_calc_tsc_timestamp()
1811 					    decoder->ref_timestamp);  in intel_pt_calc_tsc_timestamp()
1812 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1813 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1814 		decoder->ref_timestamp = 0;  in intel_pt_calc_tsc_timestamp()
1815 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1816 	} else if (decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1817 		timestamp = decoder->packet.payload |  in intel_pt_calc_tsc_timestamp()
1818 			    (decoder->timestamp & (0xffULL << 56));  in intel_pt_calc_tsc_timestamp()
1819 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1820 		if (timestamp < decoder->timestamp &&  in intel_pt_calc_tsc_timestamp()
1821 		    decoder->timestamp - timestamp < decoder->tsc_slip) {  in intel_pt_calc_tsc_timestamp()
1824 			timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1826 		if (timestamp < decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1827 			if (!decoder->buf_timestamp ||  in intel_pt_calc_tsc_timestamp()
1828 			    (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {  in intel_pt_calc_tsc_timestamp()
1831 				decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1834 				timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1838 		if (decoder->vm_time_correlation &&  in intel_pt_calc_tsc_timestamp()
1839 		    (bad || !intel_pt_time_in_range(decoder, timestamp)) &&  in intel_pt_calc_tsc_timestamp()
1840 		    intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))  in intel_pt_calc_tsc_timestamp()
1842 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1843 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1846 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_tsc_timestamp()
1847 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1848 		decoder->cycle_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1849 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_tsc_timestamp()
1850 		intel_pt_calc_cyc_to_tsc(decoder, false);  in intel_pt_calc_tsc_timestamp()
1853 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_tsc_timestamp()
1856 static int intel_pt_overflow(struct intel_pt_decoder *decoder)  in intel_pt_overflow()  argument
1859 	intel_pt_clear_tx_flags(decoder);  in intel_pt_overflow()
1860 	intel_pt_set_nr(decoder);  in intel_pt_overflow()
1861 	decoder->timestamp_insn_cnt = 0;  in intel_pt_overflow()
1862 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_overflow()
1863 	decoder->state.from_ip = decoder->ip;  in intel_pt_overflow()
1864 	decoder->ip = 0;  in intel_pt_overflow()
1865 	decoder->pge = false;  in intel_pt_overflow()
1866 	intel_pt_clear_fup_event(decoder);  in intel_pt_overflow()
1867 	decoder->overflow = true;  in intel_pt_overflow()
1871 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_pge()  argument
1873 	if (decoder->have_cyc)  in intel_pt_mtc_cyc_cnt_pge()
1876 	decoder->cyc_cnt_timestamp = decoder->timestamp;  in intel_pt_mtc_cyc_cnt_pge()
1877 	decoder->base_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_mtc_cyc_cnt_pge()
1880 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_cbr()  argument
1882 	decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;  in intel_pt_mtc_cyc_cnt_cbr()
1884 	if (decoder->pge)  in intel_pt_mtc_cyc_cnt_cbr()
1885 		intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_mtc_cyc_cnt_cbr()
1888 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_upd()  argument
1892 	if (decoder->have_cyc)  in intel_pt_mtc_cyc_cnt_upd()
1895 	decoder->sample_cyc = true;  in intel_pt_mtc_cyc_cnt_upd()
1897 	if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)  in intel_pt_mtc_cyc_cnt_upd()
1900 	tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;  in intel_pt_mtc_cyc_cnt_upd()
1901 	tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;  in intel_pt_mtc_cyc_cnt_upd()
1903 	if (tot_cyc_cnt > decoder->tot_cyc_cnt)  in intel_pt_mtc_cyc_cnt_upd()
1904 		decoder->tot_cyc_cnt = tot_cyc_cnt;  in intel_pt_mtc_cyc_cnt_upd()
1907 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)  in intel_pt_calc_tma()  argument
1909 	uint32_t ctc = decoder->packet.payload;  in intel_pt_calc_tma()
1910 	uint32_t fc = decoder->packet.count;  in intel_pt_calc_tma()
1911 	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_tma()
1913 	if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_tma()
1916 	if (decoder->pge && !decoder->in_psb)  in intel_pt_calc_tma()
1917 		intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_calc_tma()
1919 		intel_pt_mtc_cyc_cnt_upd(decoder);  in intel_pt_calc_tma()
1921 	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_tma()
1922 	decoder->last_ctc = ctc - ctc_rem;  in intel_pt_calc_tma()
1923 	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;  in intel_pt_calc_tma()
1924 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_tma()
1925 		decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_tma()
1927 		decoder->ctc_timestamp -= multdiv(ctc_rem,  in intel_pt_calc_tma()
1928 						  decoder->tsc_ctc_ratio_n,  in intel_pt_calc_tma()
1929 						  decoder->tsc_ctc_ratio_d);  in intel_pt_calc_tma()
1931 	decoder->ctc_delta = 0;  in intel_pt_calc_tma()
1932 	decoder->have_tma = true;  in intel_pt_calc_tma()
1933 	decoder->fixup_last_mtc = true;  in intel_pt_calc_tma()
1935 		     decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);  in intel_pt_calc_tma()
1938 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_mtc_timestamp()  argument
1943 	if (!decoder->have_tma)  in intel_pt_calc_mtc_timestamp()
1946 	mtc = decoder->packet.payload;  in intel_pt_calc_mtc_timestamp()
1948 	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {  in intel_pt_calc_mtc_timestamp()
1949 		decoder->fixup_last_mtc = false;  in intel_pt_calc_mtc_timestamp()
1950 		intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_mtc_timestamp()
1951 					&decoder->last_mtc);  in intel_pt_calc_mtc_timestamp()
1954 	if (mtc > decoder->last_mtc)  in intel_pt_calc_mtc_timestamp()
1955 		mtc_delta = mtc - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1957 		mtc_delta = mtc + 256 - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1959 	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_mtc_timestamp()
1961 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_mtc_timestamp()
1962 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1963 			    decoder->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_mtc_timestamp()
1965 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1966 			    multdiv(decoder->ctc_delta,  in intel_pt_calc_mtc_timestamp()
1967 				    decoder->tsc_ctc_ratio_n,  in intel_pt_calc_mtc_timestamp()
1968 				    decoder->tsc_ctc_ratio_d);  in intel_pt_calc_mtc_timestamp()
1971 	if (timestamp < decoder->timestamp)  in intel_pt_calc_mtc_timestamp()
1973 			     timestamp, decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1975 		decoder->timestamp = timestamp;  in intel_pt_calc_mtc_timestamp()
1977 	intel_pt_mtc_cyc_cnt_upd(decoder);  in intel_pt_calc_mtc_timestamp()
1979 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1980 	decoder->last_mtc = mtc;  in intel_pt_calc_mtc_timestamp()
1982 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_mtc_timestamp()
1983 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_mtc_timestamp()
1984 		decoder->cycle_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1985 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_mtc_timestamp()
1986 		intel_pt_calc_cyc_to_tsc(decoder, true);  in intel_pt_calc_mtc_timestamp()
1989 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1992 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)  in intel_pt_calc_cbr()  argument
1994 	unsigned int cbr = decoder->packet.payload & 0xff;  in intel_pt_calc_cbr()
1996 	decoder->cbr_payload = decoder->packet.payload;  in intel_pt_calc_cbr()
1998 	if (decoder->cbr == cbr)  in intel_pt_calc_cbr()
2001 	decoder->cbr = cbr;  in intel_pt_calc_cbr()
2002 	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cbr()
2003 	decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_cbr()
2004 	decoder->cycle_cnt = 0;  in intel_pt_calc_cbr()
2006 	intel_pt_mtc_cyc_cnt_cbr(decoder);  in intel_pt_calc_cbr()
2009 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_cyc_timestamp()  argument
2011 	uint64_t timestamp = decoder->cyc_ref_timestamp;  in intel_pt_calc_cyc_timestamp()
2013 	decoder->have_cyc = true;  in intel_pt_calc_cyc_timestamp()
2015 	decoder->cycle_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
2016 	if (decoder->pge)  in intel_pt_calc_cyc_timestamp()
2017 		decoder->tot_cyc_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
2018 	decoder->sample_cyc = true;  in intel_pt_calc_cyc_timestamp()
2020 	if (!decoder->cyc_ref_timestamp)  in intel_pt_calc_cyc_timestamp()
2023 	if (decoder->have_calc_cyc_to_tsc)  in intel_pt_calc_cyc_timestamp()
2024 		timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
2025 	else if (decoder->cbr)  in intel_pt_calc_cyc_timestamp()
2026 		timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
2030 	if (timestamp < decoder->timestamp)  in intel_pt_calc_cyc_timestamp()
2032 			     timestamp, decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
2034 		decoder->timestamp = timestamp;  in intel_pt_calc_cyc_timestamp()
2036 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_cyc_timestamp()
2038 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
2041 static void intel_pt_bbp(struct intel_pt_decoder *decoder)  in intel_pt_bbp()  argument
2043 	if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {  in intel_pt_bbp()
2044 		memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));  in intel_pt_bbp()
2045 		decoder->state.items.is_32_bit = false;  in intel_pt_bbp()
2047 	decoder->blk_type = decoder->packet.payload;  in intel_pt_bbp()
2048 	decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);  in intel_pt_bbp()
2049 	if (decoder->blk_type == INTEL_PT_GP_REGS)  in intel_pt_bbp()
2050 		decoder->state.items.is_32_bit = decoder->packet.count;  in intel_pt_bbp()
2051 	if (decoder->blk_type_pos < 0) {  in intel_pt_bbp()
2053 			     decoder->blk_type);  in intel_pt_bbp()
2054 	} else if (decoder->state.items.mask[decoder->blk_type_pos]) {  in intel_pt_bbp()
2056 			     decoder->blk_type);  in intel_pt_bbp()
2060 static void intel_pt_bip(struct intel_pt_decoder *decoder)  in intel_pt_bip()  argument
2062 	uint32_t id = decoder->packet.count;  in intel_pt_bip()
2064 	int pos = decoder->blk_type_pos;  in intel_pt_bip()
2068 			     id, decoder->blk_type);  in intel_pt_bip()
2072 	if (decoder->state.items.mask[pos] & bit) {  in intel_pt_bip()
2074 			     id, decoder->blk_type);  in intel_pt_bip()
2077 	decoder->state.items.mask[pos] |= bit;  in intel_pt_bip()
2078 	decoder->state.items.val[pos][id] = decoder->packet.payload;  in intel_pt_bip()
2082 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)  in intel_pt_walk_psbend()  argument
2086 	decoder->in_psb = true;  in intel_pt_walk_psbend()
2089 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psbend()
2093 		switch (decoder->packet.type) {  in intel_pt_walk_psbend()
2119 			decoder->have_tma = false;  in intel_pt_walk_psbend()
2125 			err = intel_pt_overflow(decoder);  in intel_pt_walk_psbend()
2129 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psbend()
2133 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psbend()
2137 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psbend()
2141 			intel_pt_mode_exec_status(decoder);  in intel_pt_walk_psbend()
2145 			intel_pt_set_pip(decoder);  in intel_pt_walk_psbend()
2149 			decoder->pge = true;  in intel_pt_walk_psbend()
2150 			if (decoder->packet.count) {  in intel_pt_walk_psbend()
2151 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_psbend()
2152 				decoder->psb_ip = decoder->last_ip;  in intel_pt_walk_psbend()
2157 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psbend()
2161 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psbend()
2162 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_psbend()
2163 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_psbend()
2167 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_psbend()
2178 	decoder->in_psb = false;  in intel_pt_walk_psbend()
2183 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup_tip()  argument
2187 	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {  in intel_pt_walk_fup_tip()
2188 		decoder->tx_flags = 0;  in intel_pt_walk_fup_tip()
2189 		decoder->state.flags &= ~INTEL_PT_IN_TX;  in intel_pt_walk_fup_tip()
2190 		decoder->state.flags |= INTEL_PT_ABORT_TX;  in intel_pt_walk_fup_tip()
2192 		decoder->state.flags |= INTEL_PT_ASYNC;  in intel_pt_walk_fup_tip()
2196 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_fup_tip()
2200 		switch (decoder->packet.type) {  in intel_pt_walk_fup_tip()
2225 			decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_fup_tip()
2226 			decoder->pkt_step = 0;  in intel_pt_walk_fup_tip()
2230 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_fup_tip()
2234 			return intel_pt_overflow(decoder);  in intel_pt_walk_fup_tip()
2237 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2238 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
2239 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
2241 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
2242 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2244 			decoder->pge = false;  in intel_pt_walk_fup_tip()
2245 			decoder->continuous_period = false;  in intel_pt_walk_fup_tip()
2246 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_fup_tip()
2247 			intel_pt_update_nr(decoder);  in intel_pt_walk_fup_tip()
2251 			decoder->pge = true;  in intel_pt_walk_fup_tip()
2253 				     decoder->ip);  in intel_pt_walk_fup_tip()
2254 			decoder->state.from_ip = 0;  in intel_pt_walk_fup_tip()
2255 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
2256 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
2258 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
2259 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2261 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_fup_tip()
2262 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_fup_tip()
2263 			intel_pt_set_nr(decoder);  in intel_pt_walk_fup_tip()
2267 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2268 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
2269 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
2271 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
2272 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2274 			intel_pt_update_nr(decoder);  in intel_pt_walk_fup_tip()
2275 			intel_pt_sample_iflag_chg(decoder);  in intel_pt_walk_fup_tip()
2279 			intel_pt_update_pip(decoder);  in intel_pt_walk_fup_tip()
2283 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_fup_tip()
2284 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_fup_tip()
2285 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_fup_tip()
2289 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_fup_tip()
2293 			intel_pt_mode_exec(decoder);  in intel_pt_walk_fup_tip()
2302 			return intel_pt_bug(decoder);  in intel_pt_walk_fup_tip()
2307 static int intel_pt_resample(struct intel_pt_decoder *decoder)  in intel_pt_resample()  argument
2309 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_resample()
2310 	decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_resample()
2311 	decoder->state.from_ip = decoder->ip;  in intel_pt_resample()
2312 	decoder->state.to_ip = 0;  in intel_pt_resample()
2412 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)  in intel_pt_ovf_fup_lookahead()  argument
2419 	intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);  in intel_pt_ovf_fup_lookahead()
2439 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)  in intel_pt_ctc_to_tsc()  argument
2441 	if (decoder->tsc_ctc_mult)  in intel_pt_ctc_to_tsc()
2442 		return ctc * decoder->tsc_ctc_mult;  in intel_pt_ctc_to_tsc()
2444 		return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);  in intel_pt_ctc_to_tsc()
2447 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,  in intel_pt_calc_expected_tsc()  argument
2470 	expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;  in intel_pt_calc_expected_tsc()
2481 		intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));  in intel_pt_calc_expected_tsc()
2488 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,  in intel_pt_expected_tsc()  argument
2494 	return intel_pt_calc_expected_tsc(decoder, ctc, fc,  in intel_pt_expected_tsc()
2495 					  decoder->ctc_timestamp,  in intel_pt_expected_tsc()
2499 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,  in intel_pt_translate_vm_tsc()  argument
2502 	uint64_t payload = decoder->packet.payload;  in intel_pt_translate_vm_tsc()
2505 	decoder->packet.payload -= vmcs_info->tsc_offset;  in intel_pt_translate_vm_tsc()
2507 	decoder->packet.payload &= SEVEN_BYTES;  in intel_pt_translate_vm_tsc()
2513 	if (!decoder->vm_tm_corr_dry_run)  in intel_pt_translate_vm_tsc()
2514 		memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);  in intel_pt_translate_vm_tsc()
2518 		     payload, decoder->packet.payload, vmcs_info->vmcs,  in intel_pt_translate_vm_tsc()
2522 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,  in intel_pt_translate_vm_tsc_offset()  argument
2530 	intel_pt_translate_vm_tsc(decoder, &vmcs_info);  in intel_pt_translate_vm_tsc_offset()
2549 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_psb()  argument
2553 	data->ctc_delta = decoder->ctc_delta;  in intel_pt_vm_tm_corr_psb()
2554 	data->last_ctc = decoder->last_ctc;  in intel_pt_vm_tm_corr_psb()
2555 	intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);  in intel_pt_vm_tm_corr_psb()
2558 	decoder->in_psb = data->psbend;  in intel_pt_vm_tm_corr_psb()
2561 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_first_tsc()  argument
2566 	if (!decoder->in_psb) {  in intel_pt_vm_tm_corr_first_tsc()
2574 				intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_first_tsc()
2575 				decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_first_tsc()
2580 			decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_first_tsc()
2583 		decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_first_tsc()
2584 		if (intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_first_tsc()
2589 				intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_first_tsc()
2596 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_tsc()  argument
2611 	if (!decoder->in_psb) {  in intel_pt_vm_tm_corr_tsc()
2613 		data->ctc_delta = decoder->ctc_delta;  in intel_pt_vm_tm_corr_tsc()
2614 		data->last_ctc = decoder->last_ctc;  in intel_pt_vm_tm_corr_tsc()
2616 		intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);  in intel_pt_vm_tm_corr_tsc()
2617 		if (decoder->pge) {  in intel_pt_vm_tm_corr_tsc()
2619 			data->pip_packet.payload = decoder->pip_payload;  in intel_pt_vm_tm_corr_tsc()
2629 	vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;  in intel_pt_vm_tm_corr_tsc()
2633 	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);  in intel_pt_vm_tm_corr_tsc()
2635 	ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;  in intel_pt_vm_tm_corr_tsc()
2636 	host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);  in intel_pt_vm_tm_corr_tsc()
2638 	if (!decoder->ctc_timestamp) {  in intel_pt_vm_tm_corr_tsc()
2639 		intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);  in intel_pt_vm_tm_corr_tsc()
2643 	expected_tsc = intel_pt_expected_tsc(decoder, data);  in intel_pt_vm_tm_corr_tsc()
2653 				intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);  in intel_pt_vm_tm_corr_tsc()
2654 				decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_tsc()
2658 			decoder->last_reliable_timestamp = host_tsc;  in intel_pt_vm_tm_corr_tsc()
2659 			decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_tsc()
2664 		if (decoder->in_psb) {  in intel_pt_vm_tm_corr_tsc()
2687 		if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_tsc()
2689 			decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_tsc()
2700 		} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&  in intel_pt_vm_tm_corr_tsc()
2701 			   decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {  in intel_pt_vm_tm_corr_tsc()
2705 		} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {  in intel_pt_vm_tm_corr_tsc()
2730 		intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_tsc()
2740 			if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))  in intel_pt_vm_tm_corr_tsc()
2743 		intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);  in intel_pt_vm_tm_corr_tsc()
2746 	decoder->vm_tm_corr_reliable = reliable;  in intel_pt_vm_tm_corr_tsc()
2749 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)  in intel_pt_vm_tm_corr_pebs_tsc()  argument
2751 	uint64_t host_tsc = decoder->packet.payload;  in intel_pt_vm_tm_corr_pebs_tsc()
2752 	uint64_t guest_tsc = decoder->packet.payload;  in intel_pt_vm_tm_corr_pebs_tsc()
2756 	vmcs = decoder->vmcs;  in intel_pt_vm_tm_corr_pebs_tsc()
2760 	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);  in intel_pt_vm_tm_corr_pebs_tsc()
2762 	if (decoder->pge) {  in intel_pt_vm_tm_corr_pebs_tsc()
2763 		if (in_vm(decoder->pip_payload)) { /* Guest */  in intel_pt_vm_tm_corr_pebs_tsc()
2772 		if (intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_pebs_tsc()
2781 		host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);  in intel_pt_vm_tm_corr_pebs_tsc()
2786 		if (!intel_pt_time_in_range(decoder, host_tsc) &&  in intel_pt_vm_tm_corr_pebs_tsc()
2787 		    intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))  in intel_pt_vm_tm_corr_pebs_tsc()
2790 		if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))  in intel_pt_vm_tm_corr_pebs_tsc()
2792 		host_tsc = decoder->timestamp;  in intel_pt_vm_tm_corr_pebs_tsc()
2795 	decoder->packet.payload = host_tsc;  in intel_pt_vm_tm_corr_pebs_tsc()
2797 	if (!decoder->vm_tm_corr_dry_run)  in intel_pt_vm_tm_corr_pebs_tsc()
2798 		memcpy((void *)decoder->buf + 1, &host_tsc, 8);  in intel_pt_vm_tm_corr_pebs_tsc()
2801 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)  in intel_pt_vm_time_correlation()  argument
2807 	if (decoder->in_psb)  in intel_pt_vm_time_correlation()
2808 		intel_pt_vm_tm_corr_psb(decoder, &data);  in intel_pt_vm_time_correlation()
2811 		err = intel_pt_get_next_packet(decoder);  in intel_pt_vm_time_correlation()
2817 		switch (decoder->packet.type) {  in intel_pt_vm_time_correlation()
2819 			decoder->pge = false;  in intel_pt_vm_time_correlation()
2820 			decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2826 			decoder->pge = true;  in intel_pt_vm_time_correlation()
2830 			decoder->in_psb = false;  in intel_pt_vm_time_correlation()
2831 			pge = decoder->pge;  in intel_pt_vm_time_correlation()
2832 			decoder->pge = intel_pt_ovf_fup_lookahead(decoder);  in intel_pt_vm_time_correlation()
2833 			if (pge != decoder->pge)  in intel_pt_vm_time_correlation()
2835 			if (!decoder->pge)  in intel_pt_vm_time_correlation()
2836 				decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2840 			if (decoder->in_psb)  in intel_pt_vm_time_correlation()
2841 				decoder->pge = true;  in intel_pt_vm_time_correlation()
2845 			decoder->pge = false;  in intel_pt_vm_time_correlation()
2846 			decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2847 			decoder->have_tma = false;  in intel_pt_vm_time_correlation()
2851 			intel_pt_vm_tm_corr_psb(decoder, &data);  in intel_pt_vm_time_correlation()
2855 			decoder->pip_payload = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2859 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2863 			intel_pt_vm_tm_corr_tsc(decoder, &data);  in intel_pt_vm_time_correlation()
2864 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2865 			decoder->vm_tm_corr_same_buf = true;  in intel_pt_vm_time_correlation()
2866 			decoder->vm_tm_corr_continuous = decoder->pge;  in intel_pt_vm_time_correlation()
2870 			intel_pt_calc_tma(decoder);  in intel_pt_vm_time_correlation()
2874 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2878 			intel_pt_calc_cbr(decoder);  in intel_pt_vm_time_correlation()
2882 			decoder->in_psb = false;  in intel_pt_vm_time_correlation()
2887 			if (decoder->packet.payload != NO_VMCS)  in intel_pt_vm_time_correlation()
2888 				decoder->vmcs = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2892 			decoder->blk_type = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2896 			if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&  in intel_pt_vm_time_correlation()
2897 			    decoder->packet.count == 2)  in intel_pt_vm_time_correlation()
2898 				intel_pt_vm_tm_corr_pebs_tsc(decoder);  in intel_pt_vm_time_correlation()
2903 			decoder->blk_type = 0;  in intel_pt_vm_time_correlation()
2934 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2937 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)  in intel_pt_hop_trace()  argument
2942 	if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {  in intel_pt_hop_trace()
2943 		*err = intel_pt_scan_for_psb(decoder);  in intel_pt_hop_trace()
2948 	switch (decoder->packet.type) {  in intel_pt_hop_trace()
2953 		decoder->pge = false;  in intel_pt_hop_trace()
2954 		if (!decoder->packet.count) {  in intel_pt_hop_trace()
2955 			intel_pt_set_nr(decoder);  in intel_pt_hop_trace()
2958 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2959 		decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_hop_trace()
2960 		decoder->state.from_ip = 0;  in intel_pt_hop_trace()
2961 		decoder->state.to_ip = decoder->ip;  in intel_pt_hop_trace()
2962 		intel_pt_update_nr(decoder);  in intel_pt_hop_trace()
2966 		if (!decoder->packet.count) {  in intel_pt_hop_trace()
2967 			intel_pt_set_nr(decoder);  in intel_pt_hop_trace()
2970 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2971 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
2972 		decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
2973 		decoder->state.to_ip = 0;  in intel_pt_hop_trace()
2974 		intel_pt_update_nr(decoder);  in intel_pt_hop_trace()
2975 		intel_pt_sample_iflag_chg(decoder);  in intel_pt_hop_trace()
2979 		if (!decoder->packet.count)  in intel_pt_hop_trace()
2981 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2982 		if (decoder->set_fup_mwait || decoder->set_fup_pwre)  in intel_pt_hop_trace()
2984 		if (!decoder->branch_enable || !decoder->pge)  in intel_pt_hop_trace()
2987 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
2988 			decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
2989 			decoder->state.to_ip = 0;  in intel_pt_hop_trace()
2990 			intel_pt_fup_event(decoder, *no_tip);  in intel_pt_hop_trace()
2993 		intel_pt_fup_event(decoder, *no_tip);  in intel_pt_hop_trace()
2994 		decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH;  in intel_pt_hop_trace()
2995 		*err = intel_pt_walk_fup_tip(decoder);  in intel_pt_hop_trace()
2996 		if (!*err && decoder->state.to_ip)  in intel_pt_hop_trace()
2997 			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_hop_trace()
3001 		decoder->state.psb_offset = decoder->pos;  in intel_pt_hop_trace()
3002 		decoder->psb_ip = 0;  in intel_pt_hop_trace()
3003 		decoder->last_ip = 0;  in intel_pt_hop_trace()
3004 		decoder->have_last_ip = true;  in intel_pt_hop_trace()
3005 		*err = intel_pt_walk_psbend(decoder);  in intel_pt_hop_trace()
3010 		decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_hop_trace()
3011 		if (decoder->psb_ip) {  in intel_pt_hop_trace()
3012 			decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
3013 			decoder->ip = decoder->psb_ip;  in intel_pt_hop_trace()
3015 		decoder->state.from_ip = decoder->psb_ip;  in intel_pt_hop_trace()
3016 		decoder->state.to_ip = 0;  in intel_pt_hop_trace()
3142 static int intel_pt_psb(struct intel_pt_decoder *decoder)  in intel_pt_psb()  argument
3146 	decoder->last_ip = 0;  in intel_pt_psb()
3147 	decoder->psb_ip = 0;  in intel_pt_psb()
3148 	decoder->have_last_ip = true;  in intel_pt_psb()
3149 	intel_pt_clear_stack(&decoder->stack);  in intel_pt_psb()
3150 	err = intel_pt_walk_psbend(decoder);  in intel_pt_psb()
3153 	decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_psb()
3154 	decoder->state.from_ip = decoder->psb_ip;  in intel_pt_psb()
3155 	decoder->state.to_ip = 0;  in intel_pt_psb()
3159 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)  in intel_pt_fup_in_psb()  argument
3163 	if (decoder->ip != decoder->last_ip) {  in intel_pt_fup_in_psb()
3164 		err = intel_pt_walk_fup(decoder);  in intel_pt_fup_in_psb()
3169 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_fup_in_psb()
3170 	err = intel_pt_psb(decoder);  in intel_pt_fup_in_psb()
3172 		decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_fup_in_psb()
3179 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)  in intel_pt_psb_with_fup()  argument
3183 	if (!decoder->branch_enable)  in intel_pt_psb_with_fup()
3186 	intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);  in intel_pt_psb_with_fup()
3190 	decoder->packet = data.fup_packet;  in intel_pt_psb_with_fup()
3191 	intel_pt_set_last_ip(decoder);  in intel_pt_psb_with_fup()
3192 	decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;  in intel_pt_psb_with_fup()
3194 	*err = intel_pt_fup_in_psb(decoder);  in intel_pt_psb_with_fup()
3199 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)  in intel_pt_walk_trace()  argument
3206 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3211 		if (decoder->cyc_threshold) {  in intel_pt_walk_trace()
3212 			if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)  in intel_pt_walk_trace()
3213 				decoder->sample_cyc = false;  in intel_pt_walk_trace()
3214 			last_packet_type = decoder->packet.type;  in intel_pt_walk_trace()
3217 		if (decoder->hop) {  in intel_pt_walk_trace()
3218 			switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {  in intel_pt_walk_trace()
3230 		switch (decoder->packet.type) {  in intel_pt_walk_trace()
3232 			if (!decoder->packet.count)  in intel_pt_walk_trace()
3234 			decoder->tnt = decoder->packet;  in intel_pt_walk_trace()
3235 			decoder->pkt_state = INTEL_PT_STATE_TNT;  in intel_pt_walk_trace()
3236 			err = intel_pt_walk_tnt(decoder);  in intel_pt_walk_trace()
3242 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
3243 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
3244 			decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;  in intel_pt_walk_trace()
3245 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
3248 			decoder->pge = true;  in intel_pt_walk_trace()
3249 			decoder->overflow = false;  in intel_pt_walk_trace()
3250 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_trace()
3251 			intel_pt_set_nr(decoder);  in intel_pt_walk_trace()
3252 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
3254 						decoder->pos);  in intel_pt_walk_trace()
3257 			intel_pt_sample_iflag_chg(decoder);  in intel_pt_walk_trace()
3258 			intel_pt_set_ip(decoder);  in intel_pt_walk_trace()
3259 			decoder->state.from_ip = 0;  in intel_pt_walk_trace()
3260 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_trace()
3261 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_trace()
3266 			if (decoder->hop)  in intel_pt_walk_trace()
3267 				decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_walk_trace()
3272 			return intel_pt_overflow(decoder);  in intel_pt_walk_trace()
3275 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
3276 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
3277 			decoder->pkt_state = INTEL_PT_STATE_TIP;  in intel_pt_walk_trace()
3278 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
3281 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
3283 						decoder->pos);  in intel_pt_walk_trace()
3287 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
3288 			if (!decoder->branch_enable || !decoder->pge) {  in intel_pt_walk_trace()
3289 				decoder->ip = decoder->last_ip;  in intel_pt_walk_trace()
3290 				if (intel_pt_fup_event(decoder, no_tip))  in intel_pt_walk_trace()
3295 			if (decoder->set_fup_mwait)  in intel_pt_walk_trace()
3298 				decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;  in intel_pt_walk_trace()
3300 				decoder->pkt_state = INTEL_PT_STATE_FUP;  in intel_pt_walk_trace()
3301 			err = intel_pt_walk_fup(decoder);  in intel_pt_walk_trace()
3308 			return intel_pt_walk_fup_tip(decoder);  in intel_pt_walk_trace()
3311 			decoder->pge = false;  in intel_pt_walk_trace()
3312 			decoder->continuous_period = false;  in intel_pt_walk_trace()
3313 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_trace()
3314 			decoder->have_tma = false;  in intel_pt_walk_trace()
3318 			decoder->state.psb_offset = decoder->pos;  in intel_pt_walk_trace()
3319 			decoder->psb_ip = 0;  in intel_pt_walk_trace()
3320 			if (intel_pt_psb_with_fup(decoder, &err))  in intel_pt_walk_trace()
3322 			err = intel_pt_psb(decoder);  in intel_pt_walk_trace()
3328 			intel_pt_update_pip(decoder);  in intel_pt_walk_trace()
3332 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_trace()
3333 			if (decoder->period_type != INTEL_PT_PERIOD_MTC)  in intel_pt_walk_trace()
3339 			if (!decoder->mtc_insn)  in intel_pt_walk_trace()
3341 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
3343 			if (!decoder->timestamp)  in intel_pt_walk_trace()
3345 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_trace()
3346 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3347 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3348 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
3352 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_trace()
3356 			intel_pt_calc_tma(decoder);  in intel_pt_walk_trace()
3360 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_trace()
3364 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_trace()
3365 			if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_walk_trace()
3366 				decoder->state.type = 0;  in intel_pt_walk_trace()
3372 			intel_pt_mode_exec(decoder);  in intel_pt_walk_trace()
3373 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3376 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3377 				decoder->set_fup_mode_exec = true;  in intel_pt_walk_trace()
3384 			if (!decoder->pge || decoder->in_psb) {  in intel_pt_walk_trace()
3385 				intel_pt_update_in_tx(decoder);  in intel_pt_walk_trace()
3388 			err = intel_pt_mode_tsx(decoder, &no_tip);  in intel_pt_walk_trace()
3394 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
3403 			decoder->fup_ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3404 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3407 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3408 				decoder->set_fup_ptw = true;  in intel_pt_walk_trace()
3412 						decoder->pos);  in intel_pt_walk_trace()
3417 			decoder->state.type = INTEL_PT_PTW;  in intel_pt_walk_trace()
3418 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3419 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3420 			decoder->state.ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3424 			decoder->fup_mwait_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3425 			decoder->set_fup_mwait = true;  in intel_pt_walk_trace()
3429 			if (decoder->set_fup_mwait) {  in intel_pt_walk_trace()
3430 				decoder->fup_pwre_payload =  in intel_pt_walk_trace()
3431 							decoder->packet.payload;  in intel_pt_walk_trace()
3432 				decoder->set_fup_pwre = true;  in intel_pt_walk_trace()
3435 			decoder->state.type = INTEL_PT_PWR_ENTRY;  in intel_pt_walk_trace()
3436 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3437 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3438 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3442 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3445 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3446 				decoder->set_fup_exstop = true;  in intel_pt_walk_trace()
3450 						decoder->pos);  in intel_pt_walk_trace()
3455 			decoder->state.type = INTEL_PT_EX_STOP;  in intel_pt_walk_trace()
3456 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3457 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3461 			decoder->state.type = INTEL_PT_PWR_EXIT;  in intel_pt_walk_trace()
3462 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3463 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3464 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3468 			intel_pt_bbp(decoder);  in intel_pt_walk_trace()
3472 			intel_pt_bip(decoder);  in intel_pt_walk_trace()
3476 			decoder->state.type = INTEL_PT_BLK_ITEMS;  in intel_pt_walk_trace()
3477 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3478 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3482 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3485 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3486 				decoder->set_fup_bep = true;  in intel_pt_walk_trace()
3490 						decoder->pos);  in intel_pt_walk_trace()
3495 			decoder->fup_cfe_pkt = decoder->packet;  in intel_pt_walk_trace()
3496 			decoder->set_fup_cfe = true;  in intel_pt_walk_trace()
3497 			if (!decoder->pge) {  in intel_pt_walk_trace()
3498 				intel_pt_fup_event(decoder, true);  in intel_pt_walk_trace()
3504 			decoder->fup_cfe_pkt = decoder->packet;  in intel_pt_walk_trace()
3505 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3508 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3509 				decoder->set_fup_cfe_ip = true;  in intel_pt_walk_trace()
3513 						decoder->pos);  in intel_pt_walk_trace()
3518 			err = intel_pt_evd(decoder);  in intel_pt_walk_trace()
3524 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
3529 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)  in intel_pt_have_ip()  argument
3531 	return decoder->packet.count &&  in intel_pt_have_ip()
3532 	       (decoder->have_last_ip || decoder->packet.count == 3 ||  in intel_pt_have_ip()
3533 		decoder->packet.count == 6);  in intel_pt_have_ip()
3537 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)  in intel_pt_walk_psb()  argument
3541 	decoder->in_psb = true;  in intel_pt_walk_psb()
3544 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psb()
3548 		switch (decoder->packet.type) {  in intel_pt_walk_psb()
3550 			decoder->continuous_period = false;  in intel_pt_walk_psb()
3573 			decoder->pge = true;  in intel_pt_walk_psb()
3574 			if (intel_pt_have_ip(decoder)) {  in intel_pt_walk_psb()
3575 				uint64_t current_ip = decoder->ip;  in intel_pt_walk_psb()
3577 				intel_pt_set_ip(decoder);  in intel_pt_walk_psb()
3578 				decoder->psb_ip = decoder->ip;  in intel_pt_walk_psb()
3581 							decoder->ip);  in intel_pt_walk_psb()
3586 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psb()
3590 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psb()
3594 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psb()
3598 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_psb()
3602 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psb()
3606 			intel_pt_set_pip(decoder);  in intel_pt_walk_psb()
3610 			intel_pt_mode_exec_status(decoder);  in intel_pt_walk_psb()
3614 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psb()
3618 			decoder->pge = false;  in intel_pt_walk_psb()
3619 			decoder->continuous_period = false;  in intel_pt_walk_psb()
3620 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_psb()
3624 			decoder->have_tma = false;  in intel_pt_walk_psb()
3626 			if (decoder->ip)  in intel_pt_walk_psb()
3627 				decoder->pkt_state = INTEL_PT_STATE_ERR4;  in intel_pt_walk_psb()
3629 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_psb()
3634 			err = intel_pt_bug(decoder);  in intel_pt_walk_psb()
3638 			err = intel_pt_overflow(decoder);  in intel_pt_walk_psb()
3654 	decoder->in_psb = false;  in intel_pt_walk_psb()
3659 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)  in intel_pt_walk_to_ip()  argument
3664 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_to_ip()
3668 		switch (decoder->packet.type) {  in intel_pt_walk_to_ip()
3670 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
3671 			decoder->pge = false;  in intel_pt_walk_to_ip()
3672 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3673 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3674 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3676 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_to_ip()
3680 			decoder->pge = true;  in intel_pt_walk_to_ip()
3681 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_to_ip()
3682 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3683 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3684 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3686 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_to_ip()
3690 			decoder->pge = true;  in intel_pt_walk_to_ip()
3691 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3692 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3693 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3698 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3699 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3700 			if (decoder->ip)  in intel_pt_walk_to_ip()
3705 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_to_ip()
3709 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_to_ip()
3713 			intel_pt_calc_tma(decoder);  in intel_pt_walk_to_ip()
3717 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_to_ip()
3721 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_to_ip()
3725 			intel_pt_set_pip(decoder);  in intel_pt_walk_to_ip()
3729 			intel_pt_mode_exec_status(decoder);  in intel_pt_walk_to_ip()
3733 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_to_ip()
3737 			return intel_pt_overflow(decoder);  in intel_pt_walk_to_ip()
3740 			return intel_pt_bug(decoder);  in intel_pt_walk_to_ip()
3743 			decoder->pge = false;  in intel_pt_walk_to_ip()
3744 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
3745 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_to_ip()
3746 			decoder->have_tma = false;  in intel_pt_walk_to_ip()
3750 			decoder->state.psb_offset = decoder->pos;  in intel_pt_walk_to_ip()
3751 			decoder->psb_ip = 0;  in intel_pt_walk_to_ip()
3752 			decoder->last_ip = 0;  in intel_pt_walk_to_ip()
3753 			decoder->have_last_ip = true;  in intel_pt_walk_to_ip()
3754 			intel_pt_clear_stack(&decoder->stack);  in intel_pt_walk_to_ip()
3755 			err = intel_pt_walk_psb(decoder);  in intel_pt_walk_to_ip()
3758 			decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_walk_to_ip()
3759 			decoder->state.from_ip = decoder->psb_ip;  in intel_pt_walk_to_ip()
3760 			decoder->state.to_ip = 0;  in intel_pt_walk_to_ip()
3788 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)  in intel_pt_sync_ip()  argument
3792 	intel_pt_clear_fup_event(decoder);  in intel_pt_sync_ip()
3793 	decoder->overflow = false;  in intel_pt_sync_ip()
3795 	if (!decoder->branch_enable) {  in intel_pt_sync_ip()
3796 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
3797 		decoder->state.type = 0; /* Do not have a sample */  in intel_pt_sync_ip()
3802 	err = intel_pt_walk_to_ip(decoder);  in intel_pt_sync_ip()
3803 	if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))  in intel_pt_sync_ip()
3807 	if (decoder->hop)  in intel_pt_sync_ip()
3808 		decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_sync_ip()
3810 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
3812 	decoder->state.from_ip = 0;  in intel_pt_sync_ip()
3813 	decoder->state.to_ip = decoder->ip;  in intel_pt_sync_ip()
3814 	intel_pt_log_to("Setting IP", decoder->ip);  in intel_pt_sync_ip()
3819 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)  in intel_pt_part_psb()  argument
3821 	const unsigned char *end = decoder->buf + decoder->len;  in intel_pt_part_psb()
3825 		if (i > decoder->len)  in intel_pt_part_psb()
3833 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)  in intel_pt_rest_psb()  argument
3838 	if (rest_psb > decoder->len ||  in intel_pt_rest_psb()
3839 	    memcmp(decoder->buf, psb + part_psb, rest_psb))  in intel_pt_rest_psb()
3845 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,  in intel_pt_get_split_psb()  argument
3850 	decoder->pos += decoder->len;  in intel_pt_get_split_psb()
3851 	decoder->len = 0;  in intel_pt_get_split_psb()
3853 	ret = intel_pt_get_next_data(decoder, false);  in intel_pt_get_split_psb()
3857 	rest_psb = intel_pt_rest_psb(decoder, part_psb);  in intel_pt_get_split_psb()
3861 	decoder->pos -= part_psb;  in intel_pt_get_split_psb()
3862 	decoder->next_buf = decoder->buf + rest_psb;  in intel_pt_get_split_psb()
3863 	decoder->next_len = decoder->len - rest_psb;  in intel_pt_get_split_psb()
3864 	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);  in intel_pt_get_split_psb()
3865 	decoder->buf = decoder->temp_buf;  in intel_pt_get_split_psb()
3866 	decoder->len = INTEL_PT_PSB_LEN;  in intel_pt_get_split_psb()
3871 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)  in intel_pt_scan_for_psb()  argument
3878 		if (!decoder->len) {  in intel_pt_scan_for_psb()
3879 			ret = intel_pt_get_next_data(decoder, false);  in intel_pt_scan_for_psb()
3884 		next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,  in intel_pt_scan_for_psb()
3889 			part_psb = intel_pt_part_psb(decoder);  in intel_pt_scan_for_psb()
3891 				ret = intel_pt_get_split_psb(decoder, part_psb);  in intel_pt_scan_for_psb()
3895 				decoder->pos += decoder->len;  in intel_pt_scan_for_psb()
3896 				decoder->len = 0;  in intel_pt_scan_for_psb()
3901 		decoder->pkt_step = next - decoder->buf;  in intel_pt_scan_for_psb()
3902 		return intel_pt_get_next_packet(decoder);  in intel_pt_scan_for_psb()
3906 static int intel_pt_sync(struct intel_pt_decoder *decoder)  in intel_pt_sync()  argument
3910 	decoder->pge = false;  in intel_pt_sync()
3911 	decoder->continuous_period = false;  in intel_pt_sync()
3912 	decoder->have_last_ip = false;  in intel_pt_sync()
3913 	decoder->last_ip = 0;  in intel_pt_sync()
3914 	decoder->psb_ip = 0;  in intel_pt_sync()
3915 	decoder->ip = 0;  in intel_pt_sync()
3916 	intel_pt_clear_stack(&decoder->stack);  in intel_pt_sync()
3918 	err = intel_pt_scan_for_psb(decoder);  in intel_pt_sync()
3922 	if (decoder->vm_time_correlation) {  in intel_pt_sync()
3923 		decoder->in_psb = true;  in intel_pt_sync()
3924 		if (!decoder->timestamp)  in intel_pt_sync()
3925 			decoder->timestamp = 1;  in intel_pt_sync()
3926 		decoder->state.type = 0;  in intel_pt_sync()
3927 		decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;  in intel_pt_sync()
3931 	decoder->have_last_ip = true;  in intel_pt_sync()
3932 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync()
3934 	err = intel_pt_walk_psb(decoder);  in intel_pt_sync()
3938 	decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */  in intel_pt_sync()
3939 	decoder->state.from_ip = decoder->psb_ip;  in intel_pt_sync()
3940 	decoder->state.to_ip = 0;  in intel_pt_sync()
3942 	if (decoder->ip) {  in intel_pt_sync()
3947 		if (decoder->hop)  in intel_pt_sync()
3948 			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_sync()
3950 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync()
3956 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_est_timestamp()  argument
3958 	uint64_t est = decoder->sample_insn_cnt << 1;  in intel_pt_est_timestamp()
3960 	if (!decoder->cbr || !decoder->max_non_turbo_ratio)  in intel_pt_est_timestamp()
3963 	est *= decoder->max_non_turbo_ratio;  in intel_pt_est_timestamp()
3964 	est /= decoder->cbr;  in intel_pt_est_timestamp()
3966 	return decoder->sample_timestamp + est;  in intel_pt_est_timestamp()
3969 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)  in intel_pt_decode()  argument
3974 		decoder->state.type = INTEL_PT_BRANCH;  in intel_pt_decode()
3975 		decoder->state.flags = 0;  in intel_pt_decode()
3977 		switch (decoder->pkt_state) {  in intel_pt_decode()
3979 			err = intel_pt_sync(decoder);  in intel_pt_decode()
3982 			decoder->have_last_ip = false;  in intel_pt_decode()
3983 			decoder->last_ip = 0;  in intel_pt_decode()
3984 			decoder->ip = 0;  in intel_pt_decode()
3987 			err = intel_pt_sync_ip(decoder);  in intel_pt_decode()
3990 			err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
3994 			err = intel_pt_walk_tnt(decoder);  in intel_pt_decode()
3996 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
4000 			err = intel_pt_walk_tip(decoder);  in intel_pt_decode()
4003 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
4005 				err = intel_pt_walk_fup_tip(decoder);  in intel_pt_decode()
4008 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
4010 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
4013 			err = intel_pt_fup_in_psb(decoder);  in intel_pt_decode()
4016 			err = intel_pt_resample(decoder);  in intel_pt_decode()
4019 			err = intel_pt_vm_time_correlation(decoder);  in intel_pt_decode()
4022 			err = intel_pt_bug(decoder);  in intel_pt_decode()
4028 		decoder->state.err = intel_pt_ext_err(err);  in intel_pt_decode()
4030 			decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
4031 		intel_pt_update_sample_time(decoder);  in intel_pt_decode()
4032 		decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_decode()
4033 		intel_pt_set_nr(decoder);  in intel_pt_decode()
4035 		decoder->state.err = 0;  in intel_pt_decode()
4036 		if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_decode()
4037 			decoder->cbr_seen = decoder->cbr;  in intel_pt_decode()
4038 			if (!decoder->state.type) {  in intel_pt_decode()
4039 				decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
4040 				decoder->state.to_ip = 0;  in intel_pt_decode()
4042 			decoder->state.type |= INTEL_PT_CBR_CHG;  in intel_pt_decode()
4043 			decoder->state.cbr_payload = decoder->cbr_payload;  in intel_pt_decode()
4044 			decoder->state.cbr = decoder->cbr;  in intel_pt_decode()
4046 		if (intel_pt_sample_time(decoder->pkt_state)) {  in intel_pt_decode()
4047 			intel_pt_update_sample_time(decoder);  in intel_pt_decode()
4048 			if (decoder->sample_cyc) {  in intel_pt_decode()
4049 				decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_decode()
4050 				decoder->state.flags |= INTEL_PT_SAMPLE_IPC;  in intel_pt_decode()
4051 				decoder->sample_cyc = false;  in intel_pt_decode()
4058 		if (!decoder->have_cyc)  in intel_pt_decode()
4059 			decoder->state.flags |= INTEL_PT_SAMPLE_IPC;  in intel_pt_decode()
4063 	if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)  in intel_pt_decode()
4064 		decoder->sample_timestamp = decoder->tsc_timestamp;  in intel_pt_decode()
4066 	decoder->state.from_nr = decoder->nr;  in intel_pt_decode()
4067 	decoder->state.to_nr = decoder->next_nr;  in intel_pt_decode()
4068 	decoder->nr = decoder->next_nr;  in intel_pt_decode()
4070 	decoder->state.timestamp = decoder->sample_timestamp;  in intel_pt_decode()
4071 	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);  in intel_pt_decode()
4072 	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;  in intel_pt_decode()
4073 	decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;  in intel_pt_decode()
4075 	return &decoder->state;  in intel_pt_decode()
4443  * intel_pt_fast_forward - reposition decoder forwards.
4444  * @decoder: Intel PT decoder
4447  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4451 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)  in intel_pt_fast_forward()  argument
4461 	err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);  in intel_pt_fast_forward()
4468 			decoder->pos += decoder->len;  in intel_pt_fast_forward()
4469 			decoder->len = 0;  in intel_pt_fast_forward()
4470 			err = intel_pt_get_next_data(decoder, true);  in intel_pt_fast_forward()
4474 		} while (decoder->buf_timestamp != d.buf_timestamp);  in intel_pt_fast_forward()
4477 	if (!decoder->buf)  in intel_pt_fast_forward()
4480 	buf = (unsigned char *)decoder->buf;  in intel_pt_fast_forward()
4481 	len = decoder->len;  in intel_pt_fast_forward()
4496 		tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);  in intel_pt_fast_forward()
4505 			decoder->pos += decoder->len - len;  in intel_pt_fast_forward()
4506 			decoder->buf = buf;  in intel_pt_fast_forward()
4507 			decoder->len = len;  in intel_pt_fast_forward()
4508 			intel_pt_reposition(decoder);  in intel_pt_fast_forward()