1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define SEVEN_BYTES 0xffffffffffffffULL 39 40 #define NO_VMCS 0xffffffffffULL 41 42 #define INTEL_PT_RETURN 1 43 44 /* 45 * Default maximum number of loops with no packets consumed i.e. stuck in a 46 * loop. 47 */ 48 #define INTEL_PT_MAX_LOOPS 100000 49 50 struct intel_pt_blk { 51 struct intel_pt_blk *prev; 52 uint64_t ip[INTEL_PT_BLK_SIZE]; 53 }; 54 55 struct intel_pt_stack { 56 struct intel_pt_blk *blk; 57 struct intel_pt_blk *spare; 58 int pos; 59 }; 60 61 enum intel_pt_p_once { 62 INTEL_PT_PRT_ONCE_UNK_VMCS, 63 INTEL_PT_PRT_ONCE_ERANGE, 64 }; 65 66 enum intel_pt_pkt_state { 67 INTEL_PT_STATE_NO_PSB, 68 INTEL_PT_STATE_NO_IP, 69 INTEL_PT_STATE_ERR_RESYNC, 70 INTEL_PT_STATE_IN_SYNC, 71 INTEL_PT_STATE_TNT_CONT, 72 INTEL_PT_STATE_TNT, 73 INTEL_PT_STATE_TIP, 74 INTEL_PT_STATE_TIP_PGD, 75 INTEL_PT_STATE_FUP, 76 INTEL_PT_STATE_FUP_NO_TIP, 77 INTEL_PT_STATE_FUP_IN_PSB, 78 INTEL_PT_STATE_RESAMPLE, 79 INTEL_PT_STATE_VM_TIME_CORRELATION, 80 }; 81 82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 83 { 84 switch (pkt_state) { 85 case INTEL_PT_STATE_NO_PSB: 86 case INTEL_PT_STATE_NO_IP: 87 case INTEL_PT_STATE_ERR_RESYNC: 88 case INTEL_PT_STATE_IN_SYNC: 89 case INTEL_PT_STATE_TNT_CONT: 90 case INTEL_PT_STATE_RESAMPLE: 91 case INTEL_PT_STATE_VM_TIME_CORRELATION: 92 return true; 93 case INTEL_PT_STATE_TNT: 94 case INTEL_PT_STATE_TIP: 95 case INTEL_PT_STATE_TIP_PGD: 96 case INTEL_PT_STATE_FUP: 97 case INTEL_PT_STATE_FUP_NO_TIP: 98 case INTEL_PT_STATE_FUP_IN_PSB: 99 return false; 100 default: 101 return true; 102 }; 103 } 104 105 #ifdef INTEL_PT_STRICT 106 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 107 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 108 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 109 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 110 #else 111 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 112 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 113 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 114 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 115 #endif 116 117 struct intel_pt_decoder { 118 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 119 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 120 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 121 uint64_t max_insn_cnt, void *data); 122 bool (*pgd_ip)(uint64_t ip, void *data); 123 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 124 struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); 125 void *data; 126 struct intel_pt_state state; 127 const unsigned char *buf; 128 size_t len; 129 bool return_compression; 130 bool branch_enable; 131 bool mtc_insn; 132 bool pge; 133 bool have_tma; 134 bool have_cyc; 135 bool fixup_last_mtc; 136 bool have_last_ip; 137 bool in_psb; 138 bool hop; 139 bool leap; 140 bool vm_time_correlation; 141 bool vm_tm_corr_dry_run; 142 bool vm_tm_corr_reliable; 143 bool vm_tm_corr_same_buf; 144 bool vm_tm_corr_continuous; 145 bool nr; 146 bool next_nr; 147 enum intel_pt_param_flags flags; 148 uint64_t pos; 149 uint64_t last_ip; 150 uint64_t ip; 151 uint64_t pip_payload; 152 uint64_t timestamp; 153 uint64_t tsc_timestamp; 154 uint64_t ref_timestamp; 155 uint64_t buf_timestamp; 156 uint64_t sample_timestamp; 157 uint64_t ret_addr; 158 uint64_t ctc_timestamp; 159 uint64_t ctc_delta; 160 uint64_t cycle_cnt; 161 uint64_t cyc_ref_timestamp; 162 uint64_t first_timestamp; 163 uint64_t last_reliable_timestamp; 164 uint64_t vmcs; 165 uint64_t print_once; 166 uint64_t last_ctc; 167 uint32_t last_mtc; 168 uint32_t tsc_ctc_ratio_n; 169 uint32_t tsc_ctc_ratio_d; 170 uint32_t tsc_ctc_mult; 171 uint32_t tsc_slip; 172 uint32_t ctc_rem_mask; 173 int mtc_shift; 174 struct intel_pt_stack stack; 175 enum intel_pt_pkt_state pkt_state; 176 enum intel_pt_pkt_ctx pkt_ctx; 177 enum intel_pt_pkt_ctx prev_pkt_ctx; 178 enum intel_pt_blk_type blk_type; 179 int blk_type_pos; 180 struct intel_pt_pkt packet; 181 struct intel_pt_pkt tnt; 182 int pkt_step; 183 int pkt_len; 184 int last_packet_type; 185 unsigned int cbr; 186 unsigned int cbr_seen; 187 unsigned int max_non_turbo_ratio; 188 double max_non_turbo_ratio_fp; 189 double cbr_cyc_to_tsc; 190 double calc_cyc_to_tsc; 191 bool have_calc_cyc_to_tsc; 192 int exec_mode; 193 unsigned int insn_bytes; 194 uint64_t period; 195 enum intel_pt_period_type period_type; 196 uint64_t tot_insn_cnt; 197 uint64_t period_insn_cnt; 198 uint64_t period_mask; 199 uint64_t period_ticks; 200 uint64_t last_masked_timestamp; 201 uint64_t tot_cyc_cnt; 202 uint64_t sample_tot_cyc_cnt; 203 uint64_t base_cyc_cnt; 204 uint64_t cyc_cnt_timestamp; 205 uint64_t ctl; 206 uint64_t cyc_threshold; 207 double tsc_to_cyc; 208 bool continuous_period; 209 bool overflow; 210 bool set_fup_tx_flags; 211 bool set_fup_ptw; 212 bool set_fup_mwait; 213 bool set_fup_pwre; 214 bool set_fup_exstop; 215 bool set_fup_bep; 216 bool sample_cyc; 217 unsigned int fup_tx_flags; 218 unsigned int tx_flags; 219 uint64_t fup_ptw_payload; 220 uint64_t fup_mwait_payload; 221 uint64_t fup_pwre_payload; 222 uint64_t cbr_payload; 223 uint64_t timestamp_insn_cnt; 224 uint64_t sample_insn_cnt; 225 uint64_t stuck_ip; 226 int max_loops; 227 int no_progress; 228 int stuck_ip_prd; 229 int stuck_ip_cnt; 230 uint64_t psb_ip; 231 const unsigned char *next_buf; 232 size_t next_len; 233 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 234 }; 235 236 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 237 { 238 int i; 239 240 for (i = 0; x != 1; i++) 241 x >>= 1; 242 243 return x << i; 244 } 245 246 __printf(1, 2) 247 static void p_log(const char *fmt, ...) 248 { 249 char buf[512]; 250 va_list args; 251 252 va_start(args, fmt); 253 vsnprintf(buf, sizeof(buf), fmt, args); 254 va_end(args); 255 256 fprintf(stderr, "%s\n", buf); 257 intel_pt_log("%s\n", buf); 258 } 259 260 static bool intel_pt_print_once(struct intel_pt_decoder *decoder, 261 enum intel_pt_p_once id) 262 { 263 uint64_t bit = 1ULL << id; 264 265 if (decoder->print_once & bit) 266 return false; 267 decoder->print_once |= bit; 268 return true; 269 } 270 271 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 272 { 273 if (!(ctl & INTEL_PT_CYC_ENABLE)) 274 return 0; 275 276 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 277 } 278 279 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 280 { 281 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 282 uint64_t period; 283 284 period = intel_pt_lower_power_of_2(decoder->period); 285 decoder->period_mask = ~(period - 1); 286 decoder->period_ticks = period; 287 } 288 } 289 290 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 291 { 292 if (!d) 293 return 0; 294 return (t / d) * n + ((t % d) * n) / d; 295 } 296 297 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 298 { 299 struct intel_pt_decoder *decoder; 300 301 if (!params->get_trace || !params->walk_insn) 302 return NULL; 303 304 decoder = zalloc(sizeof(struct intel_pt_decoder)); 305 if (!decoder) 306 return NULL; 307 308 decoder->get_trace = params->get_trace; 309 decoder->walk_insn = params->walk_insn; 310 decoder->pgd_ip = params->pgd_ip; 311 decoder->lookahead = params->lookahead; 312 decoder->findnew_vmcs_info = params->findnew_vmcs_info; 313 decoder->data = params->data; 314 decoder->return_compression = params->return_compression; 315 decoder->branch_enable = params->branch_enable; 316 decoder->hop = params->quick >= 1; 317 decoder->leap = params->quick >= 2; 318 decoder->vm_time_correlation = params->vm_time_correlation; 319 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; 320 decoder->first_timestamp = params->first_timestamp; 321 decoder->last_reliable_timestamp = params->first_timestamp; 322 decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS; 323 324 decoder->flags = params->flags; 325 326 decoder->ctl = params->ctl; 327 decoder->period = params->period; 328 decoder->period_type = params->period_type; 329 330 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 331 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 332 333 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 334 335 intel_pt_setup_period(decoder); 336 337 decoder->mtc_shift = params->mtc_period; 338 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 339 340 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 341 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 342 343 if (!decoder->tsc_ctc_ratio_n) 344 decoder->tsc_ctc_ratio_d = 0; 345 346 if (decoder->tsc_ctc_ratio_d) { 347 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 348 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 349 decoder->tsc_ctc_ratio_d; 350 } 351 352 /* 353 * A TSC packet can slip past MTC packets so that the timestamp appears 354 * to go backwards. One estimate is that can be up to about 40 CPU 355 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 356 * slippage an order of magnitude more to be on the safe side. 357 */ 358 decoder->tsc_slip = 0x10000; 359 360 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 361 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 362 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 363 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 364 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 365 366 if (decoder->hop) 367 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 368 369 return decoder; 370 } 371 372 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, 373 uint64_t first_timestamp) 374 { 375 decoder->first_timestamp = first_timestamp; 376 } 377 378 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 379 { 380 struct intel_pt_blk *blk = stack->blk; 381 382 stack->blk = blk->prev; 383 if (!stack->spare) 384 stack->spare = blk; 385 else 386 free(blk); 387 } 388 389 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 390 { 391 if (!stack->pos) { 392 if (!stack->blk) 393 return 0; 394 intel_pt_pop_blk(stack); 395 if (!stack->blk) 396 return 0; 397 stack->pos = INTEL_PT_BLK_SIZE; 398 } 399 return stack->blk->ip[--stack->pos]; 400 } 401 402 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 403 { 404 struct intel_pt_blk *blk; 405 406 if (stack->spare) { 407 blk = stack->spare; 408 stack->spare = NULL; 409 } else { 410 blk = malloc(sizeof(struct intel_pt_blk)); 411 if (!blk) 412 return -ENOMEM; 413 } 414 415 blk->prev = stack->blk; 416 stack->blk = blk; 417 stack->pos = 0; 418 return 0; 419 } 420 421 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 422 { 423 int err; 424 425 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 426 err = intel_pt_alloc_blk(stack); 427 if (err) 428 return err; 429 } 430 431 stack->blk->ip[stack->pos++] = ip; 432 return 0; 433 } 434 435 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 436 { 437 while (stack->blk) 438 intel_pt_pop_blk(stack); 439 stack->pos = 0; 440 } 441 442 static void intel_pt_free_stack(struct intel_pt_stack *stack) 443 { 444 intel_pt_clear_stack(stack); 445 zfree(&stack->blk); 446 zfree(&stack->spare); 447 } 448 449 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 450 { 451 intel_pt_free_stack(&decoder->stack); 452 free(decoder); 453 } 454 455 static int intel_pt_ext_err(int code) 456 { 457 switch (code) { 458 case -ENOMEM: 459 return INTEL_PT_ERR_NOMEM; 460 case -ENOSYS: 461 return INTEL_PT_ERR_INTERN; 462 case -EBADMSG: 463 return INTEL_PT_ERR_BADPKT; 464 case -ENODATA: 465 return INTEL_PT_ERR_NODATA; 466 case -EILSEQ: 467 return INTEL_PT_ERR_NOINSN; 468 case -ENOENT: 469 return INTEL_PT_ERR_MISMAT; 470 case -EOVERFLOW: 471 return INTEL_PT_ERR_OVR; 472 case -ENOSPC: 473 return INTEL_PT_ERR_LOST; 474 case -ELOOP: 475 return INTEL_PT_ERR_NELOOP; 476 default: 477 return INTEL_PT_ERR_UNK; 478 } 479 } 480 481 static const char *intel_pt_err_msgs[] = { 482 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 483 [INTEL_PT_ERR_INTERN] = "Internal error", 484 [INTEL_PT_ERR_BADPKT] = "Bad packet", 485 [INTEL_PT_ERR_NODATA] = "No more data", 486 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 487 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 488 [INTEL_PT_ERR_OVR] = "Overflow packet", 489 [INTEL_PT_ERR_LOST] = "Lost trace data", 490 [INTEL_PT_ERR_UNK] = "Unknown error!", 491 [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)", 492 }; 493 494 int intel_pt__strerror(int code, char *buf, size_t buflen) 495 { 496 if (code < 1 || code >= INTEL_PT_ERR_MAX) 497 code = INTEL_PT_ERR_UNK; 498 strlcpy(buf, intel_pt_err_msgs[code], buflen); 499 return 0; 500 } 501 502 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 503 uint64_t last_ip) 504 { 505 uint64_t ip; 506 507 switch (packet->count) { 508 case 1: 509 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 510 packet->payload; 511 break; 512 case 2: 513 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 514 packet->payload; 515 break; 516 case 3: 517 ip = packet->payload; 518 /* Sign-extend 6-byte ip */ 519 if (ip & (uint64_t)0x800000000000ULL) 520 ip |= (uint64_t)0xffff000000000000ULL; 521 break; 522 case 4: 523 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 524 packet->payload; 525 break; 526 case 6: 527 ip = packet->payload; 528 break; 529 default: 530 return 0; 531 } 532 533 return ip; 534 } 535 536 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 537 { 538 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 539 decoder->have_last_ip = true; 540 } 541 542 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 543 { 544 intel_pt_set_last_ip(decoder); 545 decoder->ip = decoder->last_ip; 546 } 547 548 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 549 { 550 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 551 decoder->buf); 552 } 553 554 static int intel_pt_bug(struct intel_pt_decoder *decoder) 555 { 556 intel_pt_log("ERROR: Internal error\n"); 557 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 558 return -ENOSYS; 559 } 560 561 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 562 { 563 decoder->tx_flags = 0; 564 } 565 566 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 567 { 568 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 569 } 570 571 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 572 { 573 decoder->pip_payload = decoder->packet.payload; 574 } 575 576 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 577 { 578 decoder->next_nr = decoder->pip_payload & 1; 579 } 580 581 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 582 { 583 decoder->nr = decoder->pip_payload & 1; 584 decoder->next_nr = decoder->nr; 585 } 586 587 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 588 { 589 intel_pt_update_pip(decoder); 590 intel_pt_set_nr(decoder); 591 } 592 593 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 594 { 595 intel_pt_clear_tx_flags(decoder); 596 decoder->have_tma = false; 597 decoder->pkt_len = 1; 598 decoder->pkt_step = 1; 599 intel_pt_decoder_log_packet(decoder); 600 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 601 intel_pt_log("ERROR: Bad packet\n"); 602 decoder->pkt_state = INTEL_PT_STATE_ERR1; 603 } 604 return -EBADMSG; 605 } 606 607 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 608 { 609 decoder->sample_timestamp = decoder->timestamp; 610 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 611 decoder->state.cycles = decoder->tot_cyc_cnt; 612 } 613 614 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 615 { 616 decoder->ip = 0; 617 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 618 decoder->timestamp = 0; 619 decoder->have_tma = false; 620 } 621 622 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 623 { 624 struct intel_pt_buffer buffer = { .buf = 0, }; 625 int ret; 626 627 decoder->pkt_step = 0; 628 629 intel_pt_log("Getting more data\n"); 630 ret = decoder->get_trace(&buffer, decoder->data); 631 if (ret) 632 return ret; 633 decoder->buf = buffer.buf; 634 decoder->len = buffer.len; 635 if (!decoder->len) { 636 intel_pt_log("No more data\n"); 637 return -ENODATA; 638 } 639 decoder->buf_timestamp = buffer.ref_timestamp; 640 if (!buffer.consecutive || reposition) { 641 intel_pt_reposition(decoder); 642 decoder->ref_timestamp = buffer.ref_timestamp; 643 decoder->state.trace_nr = buffer.trace_nr; 644 decoder->vm_tm_corr_same_buf = false; 645 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 646 decoder->ref_timestamp); 647 return -ENOLINK; 648 } 649 650 return 0; 651 } 652 653 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 654 bool reposition) 655 { 656 if (!decoder->next_buf) 657 return intel_pt_get_data(decoder, reposition); 658 659 decoder->buf = decoder->next_buf; 660 decoder->len = decoder->next_len; 661 decoder->next_buf = 0; 662 decoder->next_len = 0; 663 return 0; 664 } 665 666 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 667 { 668 unsigned char *buf = decoder->temp_buf; 669 size_t old_len, len, n; 670 int ret; 671 672 old_len = decoder->len; 673 len = decoder->len; 674 memcpy(buf, decoder->buf, len); 675 676 ret = intel_pt_get_data(decoder, false); 677 if (ret) { 678 decoder->pos += old_len; 679 return ret < 0 ? ret : -EINVAL; 680 } 681 682 n = INTEL_PT_PKT_MAX_SZ - len; 683 if (n > decoder->len) 684 n = decoder->len; 685 memcpy(buf + len, decoder->buf, n); 686 len += n; 687 688 decoder->prev_pkt_ctx = decoder->pkt_ctx; 689 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 690 if (ret < (int)old_len) { 691 decoder->next_buf = decoder->buf; 692 decoder->next_len = decoder->len; 693 decoder->buf = buf; 694 decoder->len = old_len; 695 return intel_pt_bad_packet(decoder); 696 } 697 698 decoder->next_buf = decoder->buf + (ret - old_len); 699 decoder->next_len = decoder->len - (ret - old_len); 700 701 decoder->buf = buf; 702 decoder->len = ret; 703 704 return ret; 705 } 706 707 struct intel_pt_pkt_info { 708 struct intel_pt_decoder *decoder; 709 struct intel_pt_pkt packet; 710 uint64_t pos; 711 int pkt_len; 712 int last_packet_type; 713 void *data; 714 }; 715 716 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 717 718 /* Lookahead packets in current buffer */ 719 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 720 intel_pt_pkt_cb_t cb, void *data) 721 { 722 struct intel_pt_pkt_info pkt_info; 723 const unsigned char *buf = decoder->buf; 724 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 725 size_t len = decoder->len; 726 int ret; 727 728 pkt_info.decoder = decoder; 729 pkt_info.pos = decoder->pos; 730 pkt_info.pkt_len = decoder->pkt_step; 731 pkt_info.last_packet_type = decoder->last_packet_type; 732 pkt_info.data = data; 733 734 while (1) { 735 do { 736 pkt_info.pos += pkt_info.pkt_len; 737 buf += pkt_info.pkt_len; 738 len -= pkt_info.pkt_len; 739 740 if (!len) 741 return INTEL_PT_NEED_MORE_BYTES; 742 743 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 744 &pkt_ctx); 745 if (!ret) 746 return INTEL_PT_NEED_MORE_BYTES; 747 if (ret < 0) 748 return ret; 749 750 pkt_info.pkt_len = ret; 751 } while (pkt_info.packet.type == INTEL_PT_PAD); 752 753 ret = cb(&pkt_info); 754 if (ret) 755 return 0; 756 757 pkt_info.last_packet_type = pkt_info.packet.type; 758 } 759 } 760 761 struct intel_pt_calc_cyc_to_tsc_info { 762 uint64_t cycle_cnt; 763 unsigned int cbr; 764 uint32_t last_mtc; 765 uint64_t ctc_timestamp; 766 uint64_t ctc_delta; 767 uint64_t tsc_timestamp; 768 uint64_t timestamp; 769 bool have_tma; 770 bool fixup_last_mtc; 771 bool from_mtc; 772 double cbr_cyc_to_tsc; 773 }; 774 775 /* 776 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 777 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 778 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 779 * packet by copying the missing bits from the current MTC assuming the least 780 * difference between the two, and that the current MTC comes after last_mtc. 781 */ 782 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 783 uint32_t *last_mtc) 784 { 785 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 786 uint32_t mask = ~(first_missing_bit - 1); 787 788 *last_mtc |= mtc & mask; 789 if (*last_mtc >= mtc) { 790 *last_mtc -= first_missing_bit; 791 *last_mtc &= 0xff; 792 } 793 } 794 795 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 796 { 797 struct intel_pt_decoder *decoder = pkt_info->decoder; 798 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 799 uint64_t timestamp; 800 double cyc_to_tsc; 801 unsigned int cbr; 802 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 803 804 switch (pkt_info->packet.type) { 805 case INTEL_PT_TNT: 806 case INTEL_PT_TIP_PGE: 807 case INTEL_PT_TIP: 808 case INTEL_PT_FUP: 809 case INTEL_PT_PSB: 810 case INTEL_PT_PIP: 811 case INTEL_PT_MODE_EXEC: 812 case INTEL_PT_MODE_TSX: 813 case INTEL_PT_PSBEND: 814 case INTEL_PT_PAD: 815 case INTEL_PT_VMCS: 816 case INTEL_PT_MNT: 817 case INTEL_PT_PTWRITE: 818 case INTEL_PT_PTWRITE_IP: 819 case INTEL_PT_BBP: 820 case INTEL_PT_BIP: 821 case INTEL_PT_BEP: 822 case INTEL_PT_BEP_IP: 823 return 0; 824 825 case INTEL_PT_MTC: 826 if (!data->have_tma) 827 return 0; 828 829 mtc = pkt_info->packet.payload; 830 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 831 data->fixup_last_mtc = false; 832 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 833 &data->last_mtc); 834 } 835 if (mtc > data->last_mtc) 836 mtc_delta = mtc - data->last_mtc; 837 else 838 mtc_delta = mtc + 256 - data->last_mtc; 839 data->ctc_delta += mtc_delta << decoder->mtc_shift; 840 data->last_mtc = mtc; 841 842 if (decoder->tsc_ctc_mult) { 843 timestamp = data->ctc_timestamp + 844 data->ctc_delta * decoder->tsc_ctc_mult; 845 } else { 846 timestamp = data->ctc_timestamp + 847 multdiv(data->ctc_delta, 848 decoder->tsc_ctc_ratio_n, 849 decoder->tsc_ctc_ratio_d); 850 } 851 852 if (timestamp < data->timestamp) 853 return 1; 854 855 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 856 data->timestamp = timestamp; 857 return 0; 858 } 859 860 break; 861 862 case INTEL_PT_TSC: 863 /* 864 * For now, do not support using TSC packets - refer 865 * intel_pt_calc_cyc_to_tsc(). 866 */ 867 if (data->from_mtc) 868 return 1; 869 timestamp = pkt_info->packet.payload | 870 (data->timestamp & (0xffULL << 56)); 871 if (data->from_mtc && timestamp < data->timestamp && 872 data->timestamp - timestamp < decoder->tsc_slip) 873 return 1; 874 if (timestamp < data->timestamp) 875 timestamp += (1ULL << 56); 876 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 877 if (data->from_mtc) 878 return 1; 879 data->tsc_timestamp = timestamp; 880 data->timestamp = timestamp; 881 return 0; 882 } 883 break; 884 885 case INTEL_PT_TMA: 886 if (data->from_mtc) 887 return 1; 888 889 if (!decoder->tsc_ctc_ratio_d) 890 return 0; 891 892 ctc = pkt_info->packet.payload; 893 fc = pkt_info->packet.count; 894 ctc_rem = ctc & decoder->ctc_rem_mask; 895 896 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 897 898 data->ctc_timestamp = data->tsc_timestamp - fc; 899 if (decoder->tsc_ctc_mult) { 900 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 901 } else { 902 data->ctc_timestamp -= 903 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 904 decoder->tsc_ctc_ratio_d); 905 } 906 907 data->ctc_delta = 0; 908 data->have_tma = true; 909 data->fixup_last_mtc = true; 910 911 return 0; 912 913 case INTEL_PT_CYC: 914 data->cycle_cnt += pkt_info->packet.payload; 915 return 0; 916 917 case INTEL_PT_CBR: 918 cbr = pkt_info->packet.payload; 919 if (data->cbr && data->cbr != cbr) 920 return 1; 921 data->cbr = cbr; 922 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 923 return 0; 924 925 case INTEL_PT_TIP_PGD: 926 case INTEL_PT_TRACESTOP: 927 case INTEL_PT_EXSTOP: 928 case INTEL_PT_EXSTOP_IP: 929 case INTEL_PT_MWAIT: 930 case INTEL_PT_PWRE: 931 case INTEL_PT_PWRX: 932 case INTEL_PT_OVF: 933 case INTEL_PT_BAD: /* Does not happen */ 934 default: 935 return 1; 936 } 937 938 if (!data->cbr && decoder->cbr) { 939 data->cbr = decoder->cbr; 940 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 941 } 942 943 if (!data->cycle_cnt) 944 return 1; 945 946 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 947 948 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 949 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 950 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 951 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 952 return 1; 953 } 954 955 decoder->calc_cyc_to_tsc = cyc_to_tsc; 956 decoder->have_calc_cyc_to_tsc = true; 957 958 if (data->cbr) { 959 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 960 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 961 } else { 962 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 963 cyc_to_tsc, pkt_info->pos); 964 } 965 966 return 1; 967 } 968 969 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 970 bool from_mtc) 971 { 972 struct intel_pt_calc_cyc_to_tsc_info data = { 973 .cycle_cnt = 0, 974 .cbr = 0, 975 .last_mtc = decoder->last_mtc, 976 .ctc_timestamp = decoder->ctc_timestamp, 977 .ctc_delta = decoder->ctc_delta, 978 .tsc_timestamp = decoder->tsc_timestamp, 979 .timestamp = decoder->timestamp, 980 .have_tma = decoder->have_tma, 981 .fixup_last_mtc = decoder->fixup_last_mtc, 982 .from_mtc = from_mtc, 983 .cbr_cyc_to_tsc = 0, 984 }; 985 986 /* 987 * For now, do not support using TSC packets for at least the reasons: 988 * 1) timing might have stopped 989 * 2) TSC packets within PSB+ can slip against CYC packets 990 */ 991 if (!from_mtc) 992 return; 993 994 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 995 } 996 997 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 998 { 999 int ret; 1000 1001 decoder->last_packet_type = decoder->packet.type; 1002 1003 do { 1004 decoder->pos += decoder->pkt_step; 1005 decoder->buf += decoder->pkt_step; 1006 decoder->len -= decoder->pkt_step; 1007 1008 if (!decoder->len) { 1009 ret = intel_pt_get_next_data(decoder, false); 1010 if (ret) 1011 return ret; 1012 } 1013 1014 decoder->prev_pkt_ctx = decoder->pkt_ctx; 1015 ret = intel_pt_get_packet(decoder->buf, decoder->len, 1016 &decoder->packet, &decoder->pkt_ctx); 1017 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 1018 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 1019 ret = intel_pt_get_split_packet(decoder); 1020 if (ret < 0) 1021 return ret; 1022 } 1023 if (ret <= 0) 1024 return intel_pt_bad_packet(decoder); 1025 1026 decoder->pkt_len = ret; 1027 decoder->pkt_step = ret; 1028 intel_pt_decoder_log_packet(decoder); 1029 } while (decoder->packet.type == INTEL_PT_PAD); 1030 1031 return 0; 1032 } 1033 1034 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 1035 { 1036 uint64_t timestamp, masked_timestamp; 1037 1038 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1039 masked_timestamp = timestamp & decoder->period_mask; 1040 if (decoder->continuous_period) { 1041 if (masked_timestamp > decoder->last_masked_timestamp) 1042 return 1; 1043 } else { 1044 timestamp += 1; 1045 masked_timestamp = timestamp & decoder->period_mask; 1046 if (masked_timestamp > decoder->last_masked_timestamp) { 1047 decoder->last_masked_timestamp = masked_timestamp; 1048 decoder->continuous_period = true; 1049 } 1050 } 1051 1052 if (masked_timestamp < decoder->last_masked_timestamp) 1053 return decoder->period_ticks; 1054 1055 return decoder->period_ticks - (timestamp - masked_timestamp); 1056 } 1057 1058 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 1059 { 1060 switch (decoder->period_type) { 1061 case INTEL_PT_PERIOD_INSTRUCTIONS: 1062 return decoder->period - decoder->period_insn_cnt; 1063 case INTEL_PT_PERIOD_TICKS: 1064 return intel_pt_next_period(decoder); 1065 case INTEL_PT_PERIOD_NONE: 1066 case INTEL_PT_PERIOD_MTC: 1067 default: 1068 return 0; 1069 } 1070 } 1071 1072 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1073 { 1074 uint64_t timestamp, masked_timestamp; 1075 1076 switch (decoder->period_type) { 1077 case INTEL_PT_PERIOD_INSTRUCTIONS: 1078 decoder->period_insn_cnt = 0; 1079 break; 1080 case INTEL_PT_PERIOD_TICKS: 1081 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1082 masked_timestamp = timestamp & decoder->period_mask; 1083 if (masked_timestamp > decoder->last_masked_timestamp) 1084 decoder->last_masked_timestamp = masked_timestamp; 1085 else 1086 decoder->last_masked_timestamp += decoder->period_ticks; 1087 break; 1088 case INTEL_PT_PERIOD_NONE: 1089 case INTEL_PT_PERIOD_MTC: 1090 default: 1091 break; 1092 } 1093 1094 decoder->state.type |= INTEL_PT_INSTRUCTION; 1095 } 1096 1097 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1098 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1099 { 1100 uint64_t max_insn_cnt, insn_cnt = 0; 1101 int err; 1102 1103 if (!decoder->mtc_insn) 1104 decoder->mtc_insn = true; 1105 1106 max_insn_cnt = intel_pt_next_sample(decoder); 1107 1108 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1109 max_insn_cnt, decoder->data); 1110 1111 decoder->tot_insn_cnt += insn_cnt; 1112 decoder->timestamp_insn_cnt += insn_cnt; 1113 decoder->sample_insn_cnt += insn_cnt; 1114 decoder->period_insn_cnt += insn_cnt; 1115 1116 if (err) { 1117 decoder->no_progress = 0; 1118 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1119 intel_pt_log_at("ERROR: Failed to get instruction", 1120 decoder->ip); 1121 if (err == -ENOENT) 1122 return -ENOLINK; 1123 return -EILSEQ; 1124 } 1125 1126 if (ip && decoder->ip == ip) { 1127 err = -EAGAIN; 1128 goto out; 1129 } 1130 1131 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1132 intel_pt_sample_insn(decoder); 1133 1134 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1135 decoder->state.type = INTEL_PT_INSTRUCTION; 1136 decoder->state.from_ip = decoder->ip; 1137 decoder->state.to_ip = 0; 1138 decoder->ip += intel_pt_insn->length; 1139 err = INTEL_PT_RETURN; 1140 goto out; 1141 } 1142 1143 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1144 /* Zero-length calls are excluded */ 1145 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1146 intel_pt_insn->rel) { 1147 err = intel_pt_push(&decoder->stack, decoder->ip + 1148 intel_pt_insn->length); 1149 if (err) 1150 goto out; 1151 } 1152 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1153 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1154 } 1155 1156 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1157 int cnt = decoder->no_progress++; 1158 1159 decoder->state.from_ip = decoder->ip; 1160 decoder->ip += intel_pt_insn->length + 1161 intel_pt_insn->rel; 1162 decoder->state.to_ip = decoder->ip; 1163 err = INTEL_PT_RETURN; 1164 1165 /* 1166 * Check for being stuck in a loop. This can happen if a 1167 * decoder error results in the decoder erroneously setting the 1168 * ip to an address that is itself in an infinite loop that 1169 * consumes no packets. When that happens, there must be an 1170 * unconditional branch. 1171 */ 1172 if (cnt) { 1173 if (cnt == 1) { 1174 decoder->stuck_ip = decoder->state.to_ip; 1175 decoder->stuck_ip_prd = 1; 1176 decoder->stuck_ip_cnt = 1; 1177 } else if (cnt > decoder->max_loops || 1178 decoder->state.to_ip == decoder->stuck_ip) { 1179 intel_pt_log_at("ERROR: Never-ending loop", 1180 decoder->state.to_ip); 1181 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1182 err = -ELOOP; 1183 goto out; 1184 } else if (!--decoder->stuck_ip_cnt) { 1185 decoder->stuck_ip_prd += 1; 1186 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1187 decoder->stuck_ip = decoder->state.to_ip; 1188 } 1189 } 1190 goto out_no_progress; 1191 } 1192 out: 1193 decoder->no_progress = 0; 1194 out_no_progress: 1195 decoder->state.insn_op = intel_pt_insn->op; 1196 decoder->state.insn_len = intel_pt_insn->length; 1197 memcpy(decoder->state.insn, intel_pt_insn->buf, 1198 INTEL_PT_INSN_BUF_SZ); 1199 1200 if (decoder->tx_flags & INTEL_PT_IN_TX) 1201 decoder->state.flags |= INTEL_PT_IN_TX; 1202 1203 return err; 1204 } 1205 1206 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1207 { 1208 bool ret = false; 1209 1210 if (decoder->set_fup_tx_flags) { 1211 decoder->set_fup_tx_flags = false; 1212 decoder->tx_flags = decoder->fup_tx_flags; 1213 decoder->state.type = INTEL_PT_TRANSACTION; 1214 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) 1215 decoder->state.type |= INTEL_PT_BRANCH; 1216 decoder->state.from_ip = decoder->ip; 1217 decoder->state.to_ip = 0; 1218 decoder->state.flags = decoder->fup_tx_flags; 1219 return true; 1220 } 1221 if (decoder->set_fup_ptw) { 1222 decoder->set_fup_ptw = false; 1223 decoder->state.type = INTEL_PT_PTW; 1224 decoder->state.flags |= INTEL_PT_FUP_IP; 1225 decoder->state.from_ip = decoder->ip; 1226 decoder->state.to_ip = 0; 1227 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1228 return true; 1229 } 1230 if (decoder->set_fup_mwait) { 1231 decoder->set_fup_mwait = false; 1232 decoder->state.type = INTEL_PT_MWAIT_OP; 1233 decoder->state.from_ip = decoder->ip; 1234 decoder->state.to_ip = 0; 1235 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1236 ret = true; 1237 } 1238 if (decoder->set_fup_pwre) { 1239 decoder->set_fup_pwre = false; 1240 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1241 decoder->state.type &= ~INTEL_PT_BRANCH; 1242 decoder->state.from_ip = decoder->ip; 1243 decoder->state.to_ip = 0; 1244 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1245 ret = true; 1246 } 1247 if (decoder->set_fup_exstop) { 1248 decoder->set_fup_exstop = false; 1249 decoder->state.type |= INTEL_PT_EX_STOP; 1250 decoder->state.type &= ~INTEL_PT_BRANCH; 1251 decoder->state.flags |= INTEL_PT_FUP_IP; 1252 decoder->state.from_ip = decoder->ip; 1253 decoder->state.to_ip = 0; 1254 ret = true; 1255 } 1256 if (decoder->set_fup_bep) { 1257 decoder->set_fup_bep = false; 1258 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1259 decoder->state.type &= ~INTEL_PT_BRANCH; 1260 decoder->state.from_ip = decoder->ip; 1261 decoder->state.to_ip = 0; 1262 ret = true; 1263 } 1264 return ret; 1265 } 1266 1267 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1268 struct intel_pt_insn *intel_pt_insn, 1269 uint64_t ip, int err) 1270 { 1271 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1272 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1273 ip == decoder->ip + intel_pt_insn->length; 1274 } 1275 1276 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1277 { 1278 struct intel_pt_insn intel_pt_insn; 1279 uint64_t ip; 1280 int err; 1281 1282 ip = decoder->last_ip; 1283 1284 while (1) { 1285 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1286 if (err == INTEL_PT_RETURN) 1287 return 0; 1288 if (err == -EAGAIN || 1289 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1290 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; 1291 1292 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1293 if (intel_pt_fup_event(decoder) && no_tip) 1294 return 0; 1295 return -EAGAIN; 1296 } 1297 decoder->set_fup_tx_flags = false; 1298 if (err) 1299 return err; 1300 1301 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1302 intel_pt_log_at("ERROR: Unexpected indirect branch", 1303 decoder->ip); 1304 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1305 return -ENOENT; 1306 } 1307 1308 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1309 intel_pt_log_at("ERROR: Unexpected conditional branch", 1310 decoder->ip); 1311 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1312 return -ENOENT; 1313 } 1314 1315 intel_pt_bug(decoder); 1316 } 1317 } 1318 1319 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1320 { 1321 struct intel_pt_insn intel_pt_insn; 1322 int err; 1323 1324 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1325 if (err == INTEL_PT_RETURN && 1326 decoder->pgd_ip && 1327 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1328 (decoder->state.type & INTEL_PT_BRANCH) && 1329 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1330 /* Unconditional branch leaving filter region */ 1331 decoder->no_progress = 0; 1332 decoder->pge = false; 1333 decoder->continuous_period = false; 1334 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1335 decoder->state.type |= INTEL_PT_TRACE_END; 1336 intel_pt_update_nr(decoder); 1337 return 0; 1338 } 1339 if (err == INTEL_PT_RETURN) 1340 return 0; 1341 if (err) 1342 return err; 1343 1344 intel_pt_update_nr(decoder); 1345 1346 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1347 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1348 decoder->pge = false; 1349 decoder->continuous_period = false; 1350 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1351 decoder->state.from_ip = decoder->ip; 1352 if (decoder->packet.count == 0) { 1353 decoder->state.to_ip = 0; 1354 } else { 1355 decoder->state.to_ip = decoder->last_ip; 1356 decoder->ip = decoder->last_ip; 1357 } 1358 decoder->state.type |= INTEL_PT_TRACE_END; 1359 } else { 1360 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1361 decoder->state.from_ip = decoder->ip; 1362 if (decoder->packet.count == 0) { 1363 decoder->state.to_ip = 0; 1364 } else { 1365 decoder->state.to_ip = decoder->last_ip; 1366 decoder->ip = decoder->last_ip; 1367 } 1368 } 1369 return 0; 1370 } 1371 1372 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1373 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1374 intel_pt_insn.rel; 1375 1376 if (decoder->pgd_ip && 1377 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1378 decoder->pgd_ip(to_ip, decoder->data)) { 1379 /* Conditional branch leaving filter region */ 1380 decoder->pge = false; 1381 decoder->continuous_period = false; 1382 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1383 decoder->ip = to_ip; 1384 decoder->state.from_ip = decoder->ip; 1385 decoder->state.to_ip = to_ip; 1386 decoder->state.type |= INTEL_PT_TRACE_END; 1387 return 0; 1388 } 1389 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1390 decoder->ip); 1391 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1392 return -ENOENT; 1393 } 1394 1395 return intel_pt_bug(decoder); 1396 } 1397 1398 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1399 { 1400 struct intel_pt_insn intel_pt_insn; 1401 int err; 1402 1403 while (1) { 1404 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1405 if (err == INTEL_PT_RETURN) 1406 return 0; 1407 if (err) 1408 return err; 1409 1410 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1411 if (!decoder->return_compression) { 1412 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1413 decoder->ip); 1414 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1415 return -ENOENT; 1416 } 1417 if (!decoder->ret_addr) { 1418 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1419 decoder->ip); 1420 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1421 return -ENOENT; 1422 } 1423 if (!(decoder->tnt.payload & BIT63)) { 1424 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1425 decoder->ip); 1426 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1427 return -ENOENT; 1428 } 1429 decoder->tnt.count -= 1; 1430 if (decoder->tnt.count) 1431 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1432 else 1433 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1434 decoder->tnt.payload <<= 1; 1435 decoder->state.from_ip = decoder->ip; 1436 decoder->ip = decoder->ret_addr; 1437 decoder->state.to_ip = decoder->ip; 1438 return 0; 1439 } 1440 1441 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1442 /* Handle deferred TIPs */ 1443 err = intel_pt_get_next_packet(decoder); 1444 if (err) 1445 return err; 1446 if (decoder->packet.type != INTEL_PT_TIP || 1447 decoder->packet.count == 0) { 1448 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1449 decoder->ip); 1450 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1451 decoder->pkt_step = 0; 1452 return -ENOENT; 1453 } 1454 intel_pt_set_last_ip(decoder); 1455 decoder->state.from_ip = decoder->ip; 1456 decoder->state.to_ip = decoder->last_ip; 1457 decoder->ip = decoder->last_ip; 1458 intel_pt_update_nr(decoder); 1459 return 0; 1460 } 1461 1462 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1463 decoder->tnt.count -= 1; 1464 if (decoder->tnt.count) 1465 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1466 else 1467 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1468 if (decoder->tnt.payload & BIT63) { 1469 decoder->tnt.payload <<= 1; 1470 decoder->state.from_ip = decoder->ip; 1471 decoder->ip += intel_pt_insn.length + 1472 intel_pt_insn.rel; 1473 decoder->state.to_ip = decoder->ip; 1474 return 0; 1475 } 1476 /* Instruction sample for a non-taken branch */ 1477 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1478 decoder->tnt.payload <<= 1; 1479 decoder->state.type = INTEL_PT_INSTRUCTION; 1480 decoder->state.from_ip = decoder->ip; 1481 decoder->state.to_ip = 0; 1482 decoder->ip += intel_pt_insn.length; 1483 return 0; 1484 } 1485 decoder->sample_cyc = false; 1486 decoder->ip += intel_pt_insn.length; 1487 if (!decoder->tnt.count) { 1488 intel_pt_update_sample_time(decoder); 1489 return -EAGAIN; 1490 } 1491 decoder->tnt.payload <<= 1; 1492 continue; 1493 } 1494 1495 return intel_pt_bug(decoder); 1496 } 1497 } 1498 1499 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1500 { 1501 unsigned int fup_tx_flags; 1502 int err; 1503 1504 fup_tx_flags = decoder->packet.payload & 1505 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1506 err = intel_pt_get_next_packet(decoder); 1507 if (err) 1508 return err; 1509 if (decoder->packet.type == INTEL_PT_FUP) { 1510 decoder->fup_tx_flags = fup_tx_flags; 1511 decoder->set_fup_tx_flags = true; 1512 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1513 *no_tip = true; 1514 } else { 1515 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1516 decoder->pos); 1517 intel_pt_update_in_tx(decoder); 1518 } 1519 return 0; 1520 } 1521 1522 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1523 { 1524 timestamp |= (ref_timestamp & (0xffULL << 56)); 1525 1526 if (timestamp < ref_timestamp) { 1527 if (ref_timestamp - timestamp > (1ULL << 55)) 1528 timestamp += (1ULL << 56); 1529 } else { 1530 if (timestamp - ref_timestamp > (1ULL << 55)) 1531 timestamp -= (1ULL << 56); 1532 } 1533 1534 return timestamp; 1535 } 1536 1537 /* For use only when decoder->vm_time_correlation is true */ 1538 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, 1539 uint64_t timestamp) 1540 { 1541 uint64_t max_timestamp = decoder->buf_timestamp; 1542 1543 if (!max_timestamp) { 1544 max_timestamp = decoder->last_reliable_timestamp + 1545 0x400000000ULL; 1546 } 1547 return timestamp >= decoder->last_reliable_timestamp && 1548 timestamp < decoder->buf_timestamp; 1549 } 1550 1551 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1552 { 1553 uint64_t timestamp; 1554 bool bad = false; 1555 1556 decoder->have_tma = false; 1557 1558 if (decoder->ref_timestamp) { 1559 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1560 decoder->ref_timestamp); 1561 decoder->tsc_timestamp = timestamp; 1562 decoder->timestamp = timestamp; 1563 decoder->ref_timestamp = 0; 1564 decoder->timestamp_insn_cnt = 0; 1565 } else if (decoder->timestamp) { 1566 timestamp = decoder->packet.payload | 1567 (decoder->timestamp & (0xffULL << 56)); 1568 decoder->tsc_timestamp = timestamp; 1569 if (timestamp < decoder->timestamp && 1570 decoder->timestamp - timestamp < decoder->tsc_slip) { 1571 intel_pt_log_to("Suppressing backwards timestamp", 1572 timestamp); 1573 timestamp = decoder->timestamp; 1574 } 1575 if (timestamp < decoder->timestamp) { 1576 if (!decoder->buf_timestamp || 1577 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { 1578 intel_pt_log_to("Wraparound timestamp", timestamp); 1579 timestamp += (1ULL << 56); 1580 decoder->tsc_timestamp = timestamp; 1581 } else { 1582 intel_pt_log_to("Suppressing bad timestamp", timestamp); 1583 timestamp = decoder->timestamp; 1584 bad = true; 1585 } 1586 } 1587 if (decoder->vm_time_correlation && 1588 (bad || !intel_pt_time_in_range(decoder, timestamp)) && 1589 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 1590 p_log("Timestamp out of range"); 1591 decoder->timestamp = timestamp; 1592 decoder->timestamp_insn_cnt = 0; 1593 } 1594 1595 if (decoder->last_packet_type == INTEL_PT_CYC) { 1596 decoder->cyc_ref_timestamp = decoder->timestamp; 1597 decoder->cycle_cnt = 0; 1598 decoder->have_calc_cyc_to_tsc = false; 1599 intel_pt_calc_cyc_to_tsc(decoder, false); 1600 } 1601 1602 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1603 } 1604 1605 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1606 { 1607 intel_pt_log("ERROR: Buffer overflow\n"); 1608 intel_pt_clear_tx_flags(decoder); 1609 intel_pt_set_nr(decoder); 1610 decoder->timestamp_insn_cnt = 0; 1611 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1612 decoder->overflow = true; 1613 return -EOVERFLOW; 1614 } 1615 1616 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1617 { 1618 if (decoder->have_cyc) 1619 return; 1620 1621 decoder->cyc_cnt_timestamp = decoder->timestamp; 1622 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1623 } 1624 1625 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1626 { 1627 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1628 1629 if (decoder->pge) 1630 intel_pt_mtc_cyc_cnt_pge(decoder); 1631 } 1632 1633 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1634 { 1635 uint64_t tot_cyc_cnt, tsc_delta; 1636 1637 if (decoder->have_cyc) 1638 return; 1639 1640 decoder->sample_cyc = true; 1641 1642 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1643 return; 1644 1645 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1646 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1647 1648 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1649 decoder->tot_cyc_cnt = tot_cyc_cnt; 1650 } 1651 1652 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1653 { 1654 uint32_t ctc = decoder->packet.payload; 1655 uint32_t fc = decoder->packet.count; 1656 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1657 1658 if (!decoder->tsc_ctc_ratio_d) 1659 return; 1660 1661 if (decoder->pge && !decoder->in_psb) 1662 intel_pt_mtc_cyc_cnt_pge(decoder); 1663 else 1664 intel_pt_mtc_cyc_cnt_upd(decoder); 1665 1666 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1667 decoder->last_ctc = ctc - ctc_rem; 1668 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1669 if (decoder->tsc_ctc_mult) { 1670 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1671 } else { 1672 decoder->ctc_timestamp -= multdiv(ctc_rem, 1673 decoder->tsc_ctc_ratio_n, 1674 decoder->tsc_ctc_ratio_d); 1675 } 1676 decoder->ctc_delta = 0; 1677 decoder->have_tma = true; 1678 decoder->fixup_last_mtc = true; 1679 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1680 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1681 } 1682 1683 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1684 { 1685 uint64_t timestamp; 1686 uint32_t mtc, mtc_delta; 1687 1688 if (!decoder->have_tma) 1689 return; 1690 1691 mtc = decoder->packet.payload; 1692 1693 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1694 decoder->fixup_last_mtc = false; 1695 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1696 &decoder->last_mtc); 1697 } 1698 1699 if (mtc > decoder->last_mtc) 1700 mtc_delta = mtc - decoder->last_mtc; 1701 else 1702 mtc_delta = mtc + 256 - decoder->last_mtc; 1703 1704 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1705 1706 if (decoder->tsc_ctc_mult) { 1707 timestamp = decoder->ctc_timestamp + 1708 decoder->ctc_delta * decoder->tsc_ctc_mult; 1709 } else { 1710 timestamp = decoder->ctc_timestamp + 1711 multdiv(decoder->ctc_delta, 1712 decoder->tsc_ctc_ratio_n, 1713 decoder->tsc_ctc_ratio_d); 1714 } 1715 1716 if (timestamp < decoder->timestamp) 1717 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1718 timestamp, decoder->timestamp); 1719 else 1720 decoder->timestamp = timestamp; 1721 1722 intel_pt_mtc_cyc_cnt_upd(decoder); 1723 1724 decoder->timestamp_insn_cnt = 0; 1725 decoder->last_mtc = mtc; 1726 1727 if (decoder->last_packet_type == INTEL_PT_CYC) { 1728 decoder->cyc_ref_timestamp = decoder->timestamp; 1729 decoder->cycle_cnt = 0; 1730 decoder->have_calc_cyc_to_tsc = false; 1731 intel_pt_calc_cyc_to_tsc(decoder, true); 1732 } 1733 1734 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1735 } 1736 1737 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1738 { 1739 unsigned int cbr = decoder->packet.payload & 0xff; 1740 1741 decoder->cbr_payload = decoder->packet.payload; 1742 1743 if (decoder->cbr == cbr) 1744 return; 1745 1746 decoder->cbr = cbr; 1747 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1748 1749 intel_pt_mtc_cyc_cnt_cbr(decoder); 1750 } 1751 1752 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1753 { 1754 uint64_t timestamp = decoder->cyc_ref_timestamp; 1755 1756 decoder->have_cyc = true; 1757 1758 decoder->cycle_cnt += decoder->packet.payload; 1759 if (decoder->pge) 1760 decoder->tot_cyc_cnt += decoder->packet.payload; 1761 decoder->sample_cyc = true; 1762 1763 if (!decoder->cyc_ref_timestamp) 1764 return; 1765 1766 if (decoder->have_calc_cyc_to_tsc) 1767 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1768 else if (decoder->cbr) 1769 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1770 else 1771 return; 1772 1773 if (timestamp < decoder->timestamp) 1774 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1775 timestamp, decoder->timestamp); 1776 else 1777 decoder->timestamp = timestamp; 1778 1779 decoder->timestamp_insn_cnt = 0; 1780 1781 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1782 } 1783 1784 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1785 { 1786 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1787 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1788 decoder->state.items.is_32_bit = false; 1789 } 1790 decoder->blk_type = decoder->packet.payload; 1791 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1792 if (decoder->blk_type == INTEL_PT_GP_REGS) 1793 decoder->state.items.is_32_bit = decoder->packet.count; 1794 if (decoder->blk_type_pos < 0) { 1795 intel_pt_log("WARNING: Unknown block type %u\n", 1796 decoder->blk_type); 1797 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1798 intel_pt_log("WARNING: Duplicate block type %u\n", 1799 decoder->blk_type); 1800 } 1801 } 1802 1803 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1804 { 1805 uint32_t id = decoder->packet.count; 1806 uint32_t bit = 1 << id; 1807 int pos = decoder->blk_type_pos; 1808 1809 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1810 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1811 id, decoder->blk_type); 1812 return; 1813 } 1814 1815 if (decoder->state.items.mask[pos] & bit) { 1816 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1817 id, decoder->blk_type); 1818 } 1819 1820 decoder->state.items.mask[pos] |= bit; 1821 decoder->state.items.val[pos][id] = decoder->packet.payload; 1822 } 1823 1824 /* Walk PSB+ packets when already in sync. */ 1825 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1826 { 1827 int err; 1828 1829 decoder->in_psb = true; 1830 1831 while (1) { 1832 err = intel_pt_get_next_packet(decoder); 1833 if (err) 1834 goto out; 1835 1836 switch (decoder->packet.type) { 1837 case INTEL_PT_PSBEND: 1838 err = 0; 1839 goto out; 1840 1841 case INTEL_PT_TIP_PGD: 1842 case INTEL_PT_TIP_PGE: 1843 case INTEL_PT_TIP: 1844 case INTEL_PT_TNT: 1845 case INTEL_PT_TRACESTOP: 1846 case INTEL_PT_BAD: 1847 case INTEL_PT_PSB: 1848 case INTEL_PT_PTWRITE: 1849 case INTEL_PT_PTWRITE_IP: 1850 case INTEL_PT_EXSTOP: 1851 case INTEL_PT_EXSTOP_IP: 1852 case INTEL_PT_MWAIT: 1853 case INTEL_PT_PWRE: 1854 case INTEL_PT_PWRX: 1855 case INTEL_PT_BBP: 1856 case INTEL_PT_BIP: 1857 case INTEL_PT_BEP: 1858 case INTEL_PT_BEP_IP: 1859 decoder->have_tma = false; 1860 intel_pt_log("ERROR: Unexpected packet\n"); 1861 err = -EAGAIN; 1862 goto out; 1863 1864 case INTEL_PT_OVF: 1865 err = intel_pt_overflow(decoder); 1866 goto out; 1867 1868 case INTEL_PT_TSC: 1869 intel_pt_calc_tsc_timestamp(decoder); 1870 break; 1871 1872 case INTEL_PT_TMA: 1873 intel_pt_calc_tma(decoder); 1874 break; 1875 1876 case INTEL_PT_CBR: 1877 intel_pt_calc_cbr(decoder); 1878 break; 1879 1880 case INTEL_PT_MODE_EXEC: 1881 decoder->exec_mode = decoder->packet.payload; 1882 break; 1883 1884 case INTEL_PT_PIP: 1885 intel_pt_set_pip(decoder); 1886 break; 1887 1888 case INTEL_PT_FUP: 1889 decoder->pge = true; 1890 if (decoder->packet.count) { 1891 intel_pt_set_last_ip(decoder); 1892 decoder->psb_ip = decoder->last_ip; 1893 } 1894 break; 1895 1896 case INTEL_PT_MODE_TSX: 1897 intel_pt_update_in_tx(decoder); 1898 break; 1899 1900 case INTEL_PT_MTC: 1901 intel_pt_calc_mtc_timestamp(decoder); 1902 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1903 decoder->state.type |= INTEL_PT_INSTRUCTION; 1904 break; 1905 1906 case INTEL_PT_CYC: 1907 intel_pt_calc_cyc_timestamp(decoder); 1908 break; 1909 1910 case INTEL_PT_VMCS: 1911 case INTEL_PT_MNT: 1912 case INTEL_PT_PAD: 1913 default: 1914 break; 1915 } 1916 } 1917 out: 1918 decoder->in_psb = false; 1919 1920 return err; 1921 } 1922 1923 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1924 { 1925 int err; 1926 1927 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1928 decoder->tx_flags = 0; 1929 decoder->state.flags &= ~INTEL_PT_IN_TX; 1930 decoder->state.flags |= INTEL_PT_ABORT_TX; 1931 } else { 1932 decoder->state.flags |= INTEL_PT_ASYNC; 1933 } 1934 1935 while (1) { 1936 err = intel_pt_get_next_packet(decoder); 1937 if (err) 1938 return err; 1939 1940 switch (decoder->packet.type) { 1941 case INTEL_PT_TNT: 1942 case INTEL_PT_FUP: 1943 case INTEL_PT_TRACESTOP: 1944 case INTEL_PT_PSB: 1945 case INTEL_PT_TSC: 1946 case INTEL_PT_TMA: 1947 case INTEL_PT_MODE_TSX: 1948 case INTEL_PT_BAD: 1949 case INTEL_PT_PSBEND: 1950 case INTEL_PT_PTWRITE: 1951 case INTEL_PT_PTWRITE_IP: 1952 case INTEL_PT_EXSTOP: 1953 case INTEL_PT_EXSTOP_IP: 1954 case INTEL_PT_MWAIT: 1955 case INTEL_PT_PWRE: 1956 case INTEL_PT_PWRX: 1957 case INTEL_PT_BBP: 1958 case INTEL_PT_BIP: 1959 case INTEL_PT_BEP: 1960 case INTEL_PT_BEP_IP: 1961 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1962 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1963 decoder->pkt_step = 0; 1964 return -ENOENT; 1965 1966 case INTEL_PT_CBR: 1967 intel_pt_calc_cbr(decoder); 1968 break; 1969 1970 case INTEL_PT_OVF: 1971 return intel_pt_overflow(decoder); 1972 1973 case INTEL_PT_TIP_PGD: 1974 decoder->state.from_ip = decoder->ip; 1975 if (decoder->packet.count == 0) { 1976 decoder->state.to_ip = 0; 1977 } else { 1978 intel_pt_set_ip(decoder); 1979 decoder->state.to_ip = decoder->ip; 1980 } 1981 decoder->pge = false; 1982 decoder->continuous_period = false; 1983 decoder->state.type |= INTEL_PT_TRACE_END; 1984 intel_pt_update_nr(decoder); 1985 return 0; 1986 1987 case INTEL_PT_TIP_PGE: 1988 decoder->pge = true; 1989 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1990 decoder->ip); 1991 decoder->state.from_ip = 0; 1992 if (decoder->packet.count == 0) { 1993 decoder->state.to_ip = 0; 1994 } else { 1995 intel_pt_set_ip(decoder); 1996 decoder->state.to_ip = decoder->ip; 1997 } 1998 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1999 intel_pt_mtc_cyc_cnt_pge(decoder); 2000 intel_pt_set_nr(decoder); 2001 return 0; 2002 2003 case INTEL_PT_TIP: 2004 decoder->state.from_ip = decoder->ip; 2005 if (decoder->packet.count == 0) { 2006 decoder->state.to_ip = 0; 2007 } else { 2008 intel_pt_set_ip(decoder); 2009 decoder->state.to_ip = decoder->ip; 2010 } 2011 intel_pt_update_nr(decoder); 2012 return 0; 2013 2014 case INTEL_PT_PIP: 2015 intel_pt_update_pip(decoder); 2016 break; 2017 2018 case INTEL_PT_MTC: 2019 intel_pt_calc_mtc_timestamp(decoder); 2020 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2021 decoder->state.type |= INTEL_PT_INSTRUCTION; 2022 break; 2023 2024 case INTEL_PT_CYC: 2025 intel_pt_calc_cyc_timestamp(decoder); 2026 break; 2027 2028 case INTEL_PT_MODE_EXEC: 2029 decoder->exec_mode = decoder->packet.payload; 2030 break; 2031 2032 case INTEL_PT_VMCS: 2033 case INTEL_PT_MNT: 2034 case INTEL_PT_PAD: 2035 break; 2036 2037 default: 2038 return intel_pt_bug(decoder); 2039 } 2040 } 2041 } 2042 2043 static int intel_pt_resample(struct intel_pt_decoder *decoder) 2044 { 2045 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2046 decoder->state.type = INTEL_PT_INSTRUCTION; 2047 decoder->state.from_ip = decoder->ip; 2048 decoder->state.to_ip = 0; 2049 return 0; 2050 } 2051 2052 struct intel_pt_vm_tsc_info { 2053 struct intel_pt_pkt pip_packet; 2054 struct intel_pt_pkt vmcs_packet; 2055 struct intel_pt_pkt tma_packet; 2056 bool tsc, pip, vmcs, tma, psbend; 2057 uint64_t ctc_delta; 2058 uint64_t last_ctc; 2059 int max_lookahead; 2060 }; 2061 2062 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */ 2063 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2064 { 2065 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2066 2067 switch (pkt_info->packet.type) { 2068 case INTEL_PT_PAD: 2069 case INTEL_PT_MNT: 2070 case INTEL_PT_MODE_EXEC: 2071 case INTEL_PT_MODE_TSX: 2072 case INTEL_PT_MTC: 2073 case INTEL_PT_FUP: 2074 case INTEL_PT_CYC: 2075 case INTEL_PT_CBR: 2076 break; 2077 2078 case INTEL_PT_TSC: 2079 data->tsc = true; 2080 break; 2081 2082 case INTEL_PT_TMA: 2083 data->tma_packet = pkt_info->packet; 2084 data->tma = true; 2085 break; 2086 2087 case INTEL_PT_PIP: 2088 data->pip_packet = pkt_info->packet; 2089 data->pip = true; 2090 break; 2091 2092 case INTEL_PT_VMCS: 2093 data->vmcs_packet = pkt_info->packet; 2094 data->vmcs = true; 2095 break; 2096 2097 case INTEL_PT_PSBEND: 2098 data->psbend = true; 2099 return 1; 2100 2101 case INTEL_PT_TIP_PGE: 2102 case INTEL_PT_PTWRITE: 2103 case INTEL_PT_PTWRITE_IP: 2104 case INTEL_PT_EXSTOP: 2105 case INTEL_PT_EXSTOP_IP: 2106 case INTEL_PT_MWAIT: 2107 case INTEL_PT_PWRE: 2108 case INTEL_PT_PWRX: 2109 case INTEL_PT_BBP: 2110 case INTEL_PT_BIP: 2111 case INTEL_PT_BEP: 2112 case INTEL_PT_BEP_IP: 2113 case INTEL_PT_OVF: 2114 case INTEL_PT_BAD: 2115 case INTEL_PT_TNT: 2116 case INTEL_PT_TIP_PGD: 2117 case INTEL_PT_TIP: 2118 case INTEL_PT_PSB: 2119 case INTEL_PT_TRACESTOP: 2120 default: 2121 return 1; 2122 } 2123 2124 return 0; 2125 } 2126 2127 struct intel_pt_ovf_fup_info { 2128 int max_lookahead; 2129 bool found; 2130 }; 2131 2132 /* Lookahead to detect a FUP packet after OVF */ 2133 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2134 { 2135 struct intel_pt_ovf_fup_info *data = pkt_info->data; 2136 2137 if (pkt_info->packet.type == INTEL_PT_CYC || 2138 pkt_info->packet.type == INTEL_PT_MTC || 2139 pkt_info->packet.type == INTEL_PT_TSC) 2140 return !--(data->max_lookahead); 2141 data->found = pkt_info->packet.type == INTEL_PT_FUP; 2142 return 1; 2143 } 2144 2145 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder) 2146 { 2147 struct intel_pt_ovf_fup_info data = { 2148 .max_lookahead = 16, 2149 .found = false, 2150 }; 2151 2152 intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); 2153 return data.found; 2154 } 2155 2156 /* Lookahead and get the TMA packet after TSC */ 2157 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2158 { 2159 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2160 2161 if (pkt_info->packet.type == INTEL_PT_CYC || 2162 pkt_info->packet.type == INTEL_PT_MTC) 2163 return !--(data->max_lookahead); 2164 2165 if (pkt_info->packet.type == INTEL_PT_TMA) { 2166 data->tma_packet = pkt_info->packet; 2167 data->tma = true; 2168 } 2169 return 1; 2170 } 2171 2172 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc) 2173 { 2174 if (decoder->tsc_ctc_mult) 2175 return ctc * decoder->tsc_ctc_mult; 2176 else 2177 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); 2178 } 2179 2180 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, 2181 uint32_t ctc, 2182 uint32_t fc, 2183 uint64_t last_ctc_timestamp, 2184 uint64_t ctc_delta, 2185 uint32_t last_ctc) 2186 { 2187 /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ 2188 uint64_t last_mtc_ctc = last_ctc + ctc_delta; 2189 /* 2190 * Number of CTC ticks from there until current TMA packet. We would 2191 * expect last_mtc_ctc to be before ctc, but the TSC packet can slip 2192 * past an MTC, so a sign-extended value is used. 2193 */ 2194 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); 2195 /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ 2196 uint64_t new_ctc_delta = ctc_delta + delta; 2197 uint64_t expected_tsc; 2198 2199 /* 2200 * Convert CTC ticks to TSC ticks, add the starting point 2201 * (last_ctc_timestamp) and the fast counter from the TMA packet. 2202 */ 2203 expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc; 2204 2205 if (intel_pt_enable_logging) { 2206 intel_pt_log_x64(last_mtc_ctc); 2207 intel_pt_log_x32(last_ctc); 2208 intel_pt_log_x64(ctc_delta); 2209 intel_pt_log_x64(delta); 2210 intel_pt_log_x32(ctc); 2211 intel_pt_log_x64(new_ctc_delta); 2212 intel_pt_log_x64(last_ctc_timestamp); 2213 intel_pt_log_x32(fc); 2214 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); 2215 intel_pt_log_x64(expected_tsc); 2216 } 2217 2218 return expected_tsc; 2219 } 2220 2221 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, 2222 struct intel_pt_vm_tsc_info *data) 2223 { 2224 uint32_t ctc = data->tma_packet.payload; 2225 uint32_t fc = data->tma_packet.count; 2226 2227 return intel_pt_calc_expected_tsc(decoder, ctc, fc, 2228 decoder->ctc_timestamp, 2229 data->ctc_delta, data->last_ctc); 2230 } 2231 2232 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, 2233 struct intel_pt_vmcs_info *vmcs_info) 2234 { 2235 uint64_t payload = decoder->packet.payload; 2236 2237 /* VMX adds the TSC Offset, so subtract to get host TSC */ 2238 decoder->packet.payload -= vmcs_info->tsc_offset; 2239 /* TSC packet has only 7 bytes */ 2240 decoder->packet.payload &= SEVEN_BYTES; 2241 2242 /* 2243 * The buffer is mmapped from the data file, so this also updates the 2244 * data file. 2245 */ 2246 if (!decoder->vm_tm_corr_dry_run) 2247 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); 2248 2249 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2250 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2251 payload, decoder->packet.payload, vmcs_info->vmcs, 2252 vmcs_info->tsc_offset); 2253 } 2254 2255 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, 2256 uint64_t tsc_offset) 2257 { 2258 struct intel_pt_vmcs_info vmcs_info = { 2259 .vmcs = NO_VMCS, 2260 .tsc_offset = tsc_offset 2261 }; 2262 2263 intel_pt_translate_vm_tsc(decoder, &vmcs_info); 2264 } 2265 2266 static inline bool in_vm(uint64_t pip_payload) 2267 { 2268 return pip_payload & 1; 2269 } 2270 2271 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet) 2272 { 2273 return pip_packet->payload & 1; 2274 } 2275 2276 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info) 2277 { 2278 p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, 2279 vmcs_info->vmcs, vmcs_info->tsc_offset); 2280 } 2281 2282 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, 2283 struct intel_pt_vm_tsc_info *data) 2284 { 2285 memset(data, 0, sizeof(*data)); 2286 data->ctc_delta = decoder->ctc_delta; 2287 data->last_ctc = decoder->last_ctc; 2288 intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); 2289 if (data->tsc && !data->psbend) 2290 p_log("ERROR: PSB without PSBEND"); 2291 decoder->in_psb = data->psbend; 2292 } 2293 2294 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, 2295 struct intel_pt_vm_tsc_info *data, 2296 struct intel_pt_vmcs_info *vmcs_info, 2297 uint64_t host_tsc) 2298 { 2299 if (!decoder->in_psb) { 2300 /* Can't happen */ 2301 p_log("ERROR: First TSC is not in PSB+"); 2302 } 2303 2304 if (data->pip) { 2305 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2306 if (vmcs_info && vmcs_info->tsc_offset) { 2307 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2308 decoder->vm_tm_corr_reliable = true; 2309 } else { 2310 p_log("ERROR: First TSC, unknown TSC Offset"); 2311 } 2312 } else { /* Host */ 2313 decoder->vm_tm_corr_reliable = true; 2314 } 2315 } else { /* Host or Guest */ 2316 decoder->vm_tm_corr_reliable = false; 2317 if (intel_pt_time_in_range(decoder, host_tsc)) { 2318 /* Assume Host */ 2319 } else { 2320 /* Assume Guest */ 2321 if (vmcs_info && vmcs_info->tsc_offset) 2322 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2323 else 2324 p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); 2325 } 2326 } 2327 } 2328 2329 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, 2330 struct intel_pt_vm_tsc_info *data) 2331 { 2332 struct intel_pt_vmcs_info *vmcs_info; 2333 uint64_t tsc_offset = 0; 2334 uint64_t vmcs; 2335 bool reliable = true; 2336 uint64_t expected_tsc; 2337 uint64_t host_tsc; 2338 uint64_t ref_timestamp; 2339 2340 bool assign = false; 2341 bool assign_reliable = false; 2342 2343 /* Already have 'data' for the in_psb case */ 2344 if (!decoder->in_psb) { 2345 memset(data, 0, sizeof(*data)); 2346 data->ctc_delta = decoder->ctc_delta; 2347 data->last_ctc = decoder->last_ctc; 2348 data->max_lookahead = 16; 2349 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); 2350 if (decoder->pge) { 2351 data->pip = true; 2352 data->pip_packet.payload = decoder->pip_payload; 2353 } 2354 } 2355 2356 /* Calculations depend on having TMA packets */ 2357 if (!data->tma) { 2358 p_log("ERROR: TSC without TMA"); 2359 return; 2360 } 2361 2362 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; 2363 if (vmcs == NO_VMCS) 2364 vmcs = 0; 2365 2366 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2367 2368 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; 2369 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); 2370 2371 if (!decoder->ctc_timestamp) { 2372 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); 2373 return; 2374 } 2375 2376 expected_tsc = intel_pt_expected_tsc(decoder, data); 2377 2378 tsc_offset = host_tsc - expected_tsc; 2379 2380 /* Determine if TSC is from Host or Guest */ 2381 if (data->pip) { 2382 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2383 if (!vmcs_info) { 2384 /* PIP NR=1 without VMCS cannot happen */ 2385 p_log("ERROR: Missing VMCS"); 2386 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2387 decoder->vm_tm_corr_reliable = false; 2388 return; 2389 } 2390 } else { /* Host */ 2391 decoder->last_reliable_timestamp = host_tsc; 2392 decoder->vm_tm_corr_reliable = true; 2393 return; 2394 } 2395 } else { /* Host or Guest */ 2396 reliable = false; /* Host/Guest is a guess, so not reliable */ 2397 if (decoder->in_psb) { 2398 if (!tsc_offset) 2399 return; /* Zero TSC Offset, assume Host */ 2400 /* 2401 * TSC packet has only 7 bytes of TSC. We have no 2402 * information about the Guest's 8th byte, but it 2403 * doesn't matter because we only need 7 bytes. 2404 * Here, since the 8th byte is unreliable and 2405 * irrelevant, compare only 7 byes. 2406 */ 2407 if (vmcs_info && 2408 (tsc_offset & SEVEN_BYTES) == 2409 (vmcs_info->tsc_offset & SEVEN_BYTES)) { 2410 /* Same TSC Offset as last VMCS, assume Guest */ 2411 goto guest; 2412 } 2413 } 2414 /* 2415 * Check if the host_tsc is within the expected range. 2416 * Note, we could narrow the range more by looking ahead for 2417 * the next host TSC in the same buffer, but we don't bother to 2418 * do that because this is probably good enough. 2419 */ 2420 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { 2421 /* Within expected range for Host TSC, assume Host */ 2422 decoder->vm_tm_corr_reliable = false; 2423 return; 2424 } 2425 } 2426 2427 guest: /* Assuming Guest */ 2428 2429 /* Determine whether to assign TSC Offset */ 2430 if (vmcs_info && vmcs_info->vmcs) { 2431 if (vmcs_info->tsc_offset && vmcs_info->reliable) { 2432 assign = false; 2433 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && 2434 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { 2435 /* Continuous tracing, TSC in a PSB is not a time loss */ 2436 assign = true; 2437 assign_reliable = true; 2438 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { 2439 /* 2440 * Unlikely to be a time loss TSC in a PSB which is not 2441 * at the start of a buffer. 2442 */ 2443 assign = true; 2444 assign_reliable = false; 2445 } 2446 } 2447 2448 /* Record VMCS TSC Offset */ 2449 if (assign && (vmcs_info->tsc_offset != tsc_offset || 2450 vmcs_info->reliable != assign_reliable)) { 2451 bool print = vmcs_info->tsc_offset != tsc_offset; 2452 2453 vmcs_info->tsc_offset = tsc_offset; 2454 vmcs_info->reliable = assign_reliable; 2455 if (print) 2456 intel_pt_print_vmcs_info(vmcs_info); 2457 } 2458 2459 /* Determine what TSC Offset to use */ 2460 if (vmcs_info && vmcs_info->tsc_offset) { 2461 if (!vmcs_info->reliable) 2462 reliable = false; 2463 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2464 } else { 2465 reliable = false; 2466 if (vmcs_info) { 2467 if (!vmcs_info->error_printed) { 2468 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, 2469 vmcs_info->vmcs); 2470 vmcs_info->error_printed = true; 2471 } 2472 } else { 2473 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2474 p_log("ERROR: Unknown VMCS"); 2475 } 2476 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2477 } 2478 2479 decoder->vm_tm_corr_reliable = reliable; 2480 } 2481 2482 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder) 2483 { 2484 uint64_t host_tsc = decoder->packet.payload; 2485 uint64_t guest_tsc = decoder->packet.payload; 2486 struct intel_pt_vmcs_info *vmcs_info; 2487 uint64_t vmcs; 2488 2489 vmcs = decoder->vmcs; 2490 if (vmcs == NO_VMCS) 2491 vmcs = 0; 2492 2493 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2494 2495 if (decoder->pge) { 2496 if (in_vm(decoder->pip_payload)) { /* Guest */ 2497 if (!vmcs_info) { 2498 /* PIP NR=1 without VMCS cannot happen */ 2499 p_log("ERROR: Missing VMCS"); 2500 } 2501 } else { /* Host */ 2502 return; 2503 } 2504 } else { /* Host or Guest */ 2505 if (intel_pt_time_in_range(decoder, host_tsc)) { 2506 /* Within expected range for Host TSC, assume Host */ 2507 return; 2508 } 2509 } 2510 2511 if (vmcs_info) { 2512 /* Translate Guest TSC to Host TSC */ 2513 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; 2514 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); 2515 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2516 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2517 guest_tsc, host_tsc, vmcs_info->vmcs, 2518 vmcs_info->tsc_offset); 2519 if (!intel_pt_time_in_range(decoder, host_tsc) && 2520 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 2521 p_log("Timestamp out of range"); 2522 } else { 2523 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2524 p_log("ERROR: Unknown VMCS"); 2525 host_tsc = decoder->timestamp; 2526 } 2527 2528 decoder->packet.payload = host_tsc; 2529 2530 if (!decoder->vm_tm_corr_dry_run) 2531 memcpy((void *)decoder->buf + 1, &host_tsc, 8); 2532 } 2533 2534 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder) 2535 { 2536 struct intel_pt_vm_tsc_info data = { .psbend = false }; 2537 bool pge; 2538 int err; 2539 2540 if (decoder->in_psb) 2541 intel_pt_vm_tm_corr_psb(decoder, &data); 2542 2543 while (1) { 2544 err = intel_pt_get_next_packet(decoder); 2545 if (err == -ENOLINK) 2546 continue; 2547 if (err) 2548 break; 2549 2550 switch (decoder->packet.type) { 2551 case INTEL_PT_TIP_PGD: 2552 decoder->pge = false; 2553 decoder->vm_tm_corr_continuous = false; 2554 break; 2555 2556 case INTEL_PT_TNT: 2557 case INTEL_PT_TIP: 2558 case INTEL_PT_TIP_PGE: 2559 decoder->pge = true; 2560 break; 2561 2562 case INTEL_PT_OVF: 2563 decoder->in_psb = false; 2564 pge = decoder->pge; 2565 decoder->pge = intel_pt_ovf_fup_lookahead(decoder); 2566 if (pge != decoder->pge) 2567 intel_pt_log("Surprising PGE change in OVF!"); 2568 if (!decoder->pge) 2569 decoder->vm_tm_corr_continuous = false; 2570 break; 2571 2572 case INTEL_PT_FUP: 2573 if (decoder->in_psb) 2574 decoder->pge = true; 2575 break; 2576 2577 case INTEL_PT_TRACESTOP: 2578 decoder->pge = false; 2579 decoder->vm_tm_corr_continuous = false; 2580 decoder->have_tma = false; 2581 break; 2582 2583 case INTEL_PT_PSB: 2584 intel_pt_vm_tm_corr_psb(decoder, &data); 2585 break; 2586 2587 case INTEL_PT_PIP: 2588 decoder->pip_payload = decoder->packet.payload; 2589 break; 2590 2591 case INTEL_PT_MTC: 2592 intel_pt_calc_mtc_timestamp(decoder); 2593 break; 2594 2595 case INTEL_PT_TSC: 2596 intel_pt_vm_tm_corr_tsc(decoder, &data); 2597 intel_pt_calc_tsc_timestamp(decoder); 2598 decoder->vm_tm_corr_same_buf = true; 2599 decoder->vm_tm_corr_continuous = decoder->pge; 2600 break; 2601 2602 case INTEL_PT_TMA: 2603 intel_pt_calc_tma(decoder); 2604 break; 2605 2606 case INTEL_PT_CYC: 2607 intel_pt_calc_cyc_timestamp(decoder); 2608 break; 2609 2610 case INTEL_PT_CBR: 2611 intel_pt_calc_cbr(decoder); 2612 break; 2613 2614 case INTEL_PT_PSBEND: 2615 decoder->in_psb = false; 2616 data.psbend = false; 2617 break; 2618 2619 case INTEL_PT_VMCS: 2620 if (decoder->packet.payload != NO_VMCS) 2621 decoder->vmcs = decoder->packet.payload; 2622 break; 2623 2624 case INTEL_PT_BBP: 2625 decoder->blk_type = decoder->packet.payload; 2626 break; 2627 2628 case INTEL_PT_BIP: 2629 if (decoder->blk_type == INTEL_PT_PEBS_BASIC && 2630 decoder->packet.count == 2) 2631 intel_pt_vm_tm_corr_pebs_tsc(decoder); 2632 break; 2633 2634 case INTEL_PT_BEP: 2635 case INTEL_PT_BEP_IP: 2636 decoder->blk_type = 0; 2637 break; 2638 2639 case INTEL_PT_MODE_EXEC: 2640 case INTEL_PT_MODE_TSX: 2641 case INTEL_PT_MNT: 2642 case INTEL_PT_PAD: 2643 case INTEL_PT_PTWRITE_IP: 2644 case INTEL_PT_PTWRITE: 2645 case INTEL_PT_MWAIT: 2646 case INTEL_PT_PWRE: 2647 case INTEL_PT_EXSTOP_IP: 2648 case INTEL_PT_EXSTOP: 2649 case INTEL_PT_PWRX: 2650 case INTEL_PT_BAD: /* Does not happen */ 2651 default: 2652 break; 2653 } 2654 } 2655 2656 return err; 2657 } 2658 2659 #define HOP_PROCESS 0 2660 #define HOP_IGNORE 1 2661 #define HOP_RETURN 2 2662 #define HOP_AGAIN 3 2663 2664 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 2665 2666 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 2667 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 2668 { 2669 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 2670 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 2671 *err = intel_pt_scan_for_psb(decoder); 2672 if (*err) 2673 return HOP_RETURN; 2674 } 2675 2676 switch (decoder->packet.type) { 2677 case INTEL_PT_TNT: 2678 return HOP_IGNORE; 2679 2680 case INTEL_PT_TIP_PGD: 2681 if (!decoder->packet.count) { 2682 intel_pt_set_nr(decoder); 2683 return HOP_IGNORE; 2684 } 2685 intel_pt_set_ip(decoder); 2686 decoder->state.type |= INTEL_PT_TRACE_END; 2687 decoder->state.from_ip = 0; 2688 decoder->state.to_ip = decoder->ip; 2689 intel_pt_update_nr(decoder); 2690 return HOP_RETURN; 2691 2692 case INTEL_PT_TIP: 2693 if (!decoder->packet.count) { 2694 intel_pt_set_nr(decoder); 2695 return HOP_IGNORE; 2696 } 2697 intel_pt_set_ip(decoder); 2698 decoder->state.type = INTEL_PT_INSTRUCTION; 2699 decoder->state.from_ip = decoder->ip; 2700 decoder->state.to_ip = 0; 2701 intel_pt_update_nr(decoder); 2702 return HOP_RETURN; 2703 2704 case INTEL_PT_FUP: 2705 if (!decoder->packet.count) 2706 return HOP_IGNORE; 2707 intel_pt_set_ip(decoder); 2708 if (intel_pt_fup_event(decoder)) 2709 return HOP_RETURN; 2710 if (!decoder->branch_enable) 2711 *no_tip = true; 2712 if (*no_tip) { 2713 decoder->state.type = INTEL_PT_INSTRUCTION; 2714 decoder->state.from_ip = decoder->ip; 2715 decoder->state.to_ip = 0; 2716 return HOP_RETURN; 2717 } 2718 *err = intel_pt_walk_fup_tip(decoder); 2719 if (!*err) 2720 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2721 return HOP_RETURN; 2722 2723 case INTEL_PT_PSB: 2724 decoder->state.psb_offset = decoder->pos; 2725 decoder->psb_ip = 0; 2726 decoder->last_ip = 0; 2727 decoder->have_last_ip = true; 2728 *err = intel_pt_walk_psbend(decoder); 2729 if (*err == -EAGAIN) 2730 return HOP_AGAIN; 2731 if (*err) 2732 return HOP_RETURN; 2733 decoder->state.type = INTEL_PT_PSB_EVT; 2734 if (decoder->psb_ip) { 2735 decoder->state.type |= INTEL_PT_INSTRUCTION; 2736 decoder->ip = decoder->psb_ip; 2737 } 2738 decoder->state.from_ip = decoder->psb_ip; 2739 decoder->state.to_ip = 0; 2740 return HOP_RETURN; 2741 2742 case INTEL_PT_BAD: 2743 case INTEL_PT_PAD: 2744 case INTEL_PT_TIP_PGE: 2745 case INTEL_PT_TSC: 2746 case INTEL_PT_TMA: 2747 case INTEL_PT_MODE_EXEC: 2748 case INTEL_PT_MODE_TSX: 2749 case INTEL_PT_MTC: 2750 case INTEL_PT_CYC: 2751 case INTEL_PT_VMCS: 2752 case INTEL_PT_PSBEND: 2753 case INTEL_PT_CBR: 2754 case INTEL_PT_TRACESTOP: 2755 case INTEL_PT_PIP: 2756 case INTEL_PT_OVF: 2757 case INTEL_PT_MNT: 2758 case INTEL_PT_PTWRITE: 2759 case INTEL_PT_PTWRITE_IP: 2760 case INTEL_PT_EXSTOP: 2761 case INTEL_PT_EXSTOP_IP: 2762 case INTEL_PT_MWAIT: 2763 case INTEL_PT_PWRE: 2764 case INTEL_PT_PWRX: 2765 case INTEL_PT_BBP: 2766 case INTEL_PT_BIP: 2767 case INTEL_PT_BEP: 2768 case INTEL_PT_BEP_IP: 2769 default: 2770 return HOP_PROCESS; 2771 } 2772 } 2773 2774 struct intel_pt_psb_info { 2775 struct intel_pt_pkt fup_packet; 2776 bool fup; 2777 int after_psbend; 2778 }; 2779 2780 /* Lookahead and get the FUP packet from PSB+ */ 2781 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2782 { 2783 struct intel_pt_psb_info *data = pkt_info->data; 2784 2785 switch (pkt_info->packet.type) { 2786 case INTEL_PT_PAD: 2787 case INTEL_PT_MNT: 2788 case INTEL_PT_TSC: 2789 case INTEL_PT_TMA: 2790 case INTEL_PT_MODE_EXEC: 2791 case INTEL_PT_MODE_TSX: 2792 case INTEL_PT_MTC: 2793 case INTEL_PT_CYC: 2794 case INTEL_PT_VMCS: 2795 case INTEL_PT_CBR: 2796 case INTEL_PT_PIP: 2797 if (data->after_psbend) { 2798 data->after_psbend -= 1; 2799 if (!data->after_psbend) 2800 return 1; 2801 } 2802 break; 2803 2804 case INTEL_PT_FUP: 2805 if (data->after_psbend) 2806 return 1; 2807 if (data->fup || pkt_info->packet.count == 0) 2808 return 1; 2809 data->fup_packet = pkt_info->packet; 2810 data->fup = true; 2811 break; 2812 2813 case INTEL_PT_PSBEND: 2814 if (!data->fup) 2815 return 1; 2816 /* Keep going to check for a TIP.PGE */ 2817 data->after_psbend = 6; 2818 break; 2819 2820 case INTEL_PT_TIP_PGE: 2821 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 2822 if (data->after_psbend) 2823 data->fup = false; 2824 return 1; 2825 2826 case INTEL_PT_PTWRITE: 2827 case INTEL_PT_PTWRITE_IP: 2828 case INTEL_PT_EXSTOP: 2829 case INTEL_PT_EXSTOP_IP: 2830 case INTEL_PT_MWAIT: 2831 case INTEL_PT_PWRE: 2832 case INTEL_PT_PWRX: 2833 case INTEL_PT_BBP: 2834 case INTEL_PT_BIP: 2835 case INTEL_PT_BEP: 2836 case INTEL_PT_BEP_IP: 2837 if (data->after_psbend) { 2838 data->after_psbend -= 1; 2839 if (!data->after_psbend) 2840 return 1; 2841 break; 2842 } 2843 return 1; 2844 2845 case INTEL_PT_OVF: 2846 case INTEL_PT_BAD: 2847 case INTEL_PT_TNT: 2848 case INTEL_PT_TIP_PGD: 2849 case INTEL_PT_TIP: 2850 case INTEL_PT_PSB: 2851 case INTEL_PT_TRACESTOP: 2852 default: 2853 return 1; 2854 } 2855 2856 return 0; 2857 } 2858 2859 static int intel_pt_psb(struct intel_pt_decoder *decoder) 2860 { 2861 int err; 2862 2863 decoder->last_ip = 0; 2864 decoder->psb_ip = 0; 2865 decoder->have_last_ip = true; 2866 intel_pt_clear_stack(&decoder->stack); 2867 err = intel_pt_walk_psbend(decoder); 2868 if (err) 2869 return err; 2870 decoder->state.type = INTEL_PT_PSB_EVT; 2871 decoder->state.from_ip = decoder->psb_ip; 2872 decoder->state.to_ip = 0; 2873 return 0; 2874 } 2875 2876 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 2877 { 2878 int err; 2879 2880 if (decoder->ip != decoder->last_ip) { 2881 err = intel_pt_walk_fup(decoder); 2882 if (!err || err != -EAGAIN) 2883 return err; 2884 } 2885 2886 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2887 err = intel_pt_psb(decoder); 2888 if (err) { 2889 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2890 return -ENOENT; 2891 } 2892 2893 return 0; 2894 } 2895 2896 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 2897 { 2898 struct intel_pt_psb_info data = { .fup = false }; 2899 2900 if (!decoder->branch_enable || !decoder->pge) 2901 return false; 2902 2903 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 2904 if (!data.fup) 2905 return false; 2906 2907 decoder->packet = data.fup_packet; 2908 intel_pt_set_last_ip(decoder); 2909 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 2910 2911 *err = intel_pt_fup_in_psb(decoder); 2912 2913 return true; 2914 } 2915 2916 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2917 { 2918 int last_packet_type = INTEL_PT_PAD; 2919 bool no_tip = false; 2920 int err; 2921 2922 while (1) { 2923 err = intel_pt_get_next_packet(decoder); 2924 if (err) 2925 return err; 2926 next: 2927 if (decoder->cyc_threshold) { 2928 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 2929 decoder->sample_cyc = false; 2930 last_packet_type = decoder->packet.type; 2931 } 2932 2933 if (decoder->hop) { 2934 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2935 case HOP_IGNORE: 2936 continue; 2937 case HOP_RETURN: 2938 return err; 2939 case HOP_AGAIN: 2940 goto next; 2941 default: 2942 break; 2943 } 2944 } 2945 2946 switch (decoder->packet.type) { 2947 case INTEL_PT_TNT: 2948 if (!decoder->packet.count) 2949 break; 2950 decoder->tnt = decoder->packet; 2951 decoder->pkt_state = INTEL_PT_STATE_TNT; 2952 err = intel_pt_walk_tnt(decoder); 2953 if (err == -EAGAIN) 2954 break; 2955 return err; 2956 2957 case INTEL_PT_TIP_PGD: 2958 if (decoder->packet.count != 0) 2959 intel_pt_set_last_ip(decoder); 2960 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2961 return intel_pt_walk_tip(decoder); 2962 2963 case INTEL_PT_TIP_PGE: { 2964 decoder->pge = true; 2965 intel_pt_mtc_cyc_cnt_pge(decoder); 2966 intel_pt_set_nr(decoder); 2967 if (decoder->packet.count == 0) { 2968 intel_pt_log_at("Skipping zero TIP.PGE", 2969 decoder->pos); 2970 break; 2971 } 2972 intel_pt_set_ip(decoder); 2973 decoder->state.from_ip = 0; 2974 decoder->state.to_ip = decoder->ip; 2975 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2976 /* 2977 * In hop mode, resample to get the to_ip as an 2978 * "instruction" sample. 2979 */ 2980 if (decoder->hop) 2981 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2982 return 0; 2983 } 2984 2985 case INTEL_PT_OVF: 2986 return intel_pt_overflow(decoder); 2987 2988 case INTEL_PT_TIP: 2989 if (decoder->packet.count != 0) 2990 intel_pt_set_last_ip(decoder); 2991 decoder->pkt_state = INTEL_PT_STATE_TIP; 2992 return intel_pt_walk_tip(decoder); 2993 2994 case INTEL_PT_FUP: 2995 if (decoder->packet.count == 0) { 2996 intel_pt_log_at("Skipping zero FUP", 2997 decoder->pos); 2998 no_tip = false; 2999 break; 3000 } 3001 intel_pt_set_last_ip(decoder); 3002 if (!decoder->branch_enable) { 3003 decoder->ip = decoder->last_ip; 3004 if (intel_pt_fup_event(decoder)) 3005 return 0; 3006 no_tip = false; 3007 break; 3008 } 3009 if (decoder->set_fup_mwait) 3010 no_tip = true; 3011 if (no_tip) 3012 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 3013 else 3014 decoder->pkt_state = INTEL_PT_STATE_FUP; 3015 err = intel_pt_walk_fup(decoder); 3016 if (err != -EAGAIN) 3017 return err; 3018 if (no_tip) { 3019 no_tip = false; 3020 break; 3021 } 3022 return intel_pt_walk_fup_tip(decoder); 3023 3024 case INTEL_PT_TRACESTOP: 3025 decoder->pge = false; 3026 decoder->continuous_period = false; 3027 intel_pt_clear_tx_flags(decoder); 3028 decoder->have_tma = false; 3029 break; 3030 3031 case INTEL_PT_PSB: 3032 decoder->state.psb_offset = decoder->pos; 3033 decoder->psb_ip = 0; 3034 if (intel_pt_psb_with_fup(decoder, &err)) 3035 return err; 3036 err = intel_pt_psb(decoder); 3037 if (err == -EAGAIN) 3038 goto next; 3039 return err; 3040 3041 case INTEL_PT_PIP: 3042 intel_pt_update_pip(decoder); 3043 break; 3044 3045 case INTEL_PT_MTC: 3046 intel_pt_calc_mtc_timestamp(decoder); 3047 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 3048 break; 3049 /* 3050 * Ensure that there has been an instruction since the 3051 * last MTC. 3052 */ 3053 if (!decoder->mtc_insn) 3054 break; 3055 decoder->mtc_insn = false; 3056 /* Ensure that there is a timestamp */ 3057 if (!decoder->timestamp) 3058 break; 3059 decoder->state.type = INTEL_PT_INSTRUCTION; 3060 decoder->state.from_ip = decoder->ip; 3061 decoder->state.to_ip = 0; 3062 decoder->mtc_insn = false; 3063 return 0; 3064 3065 case INTEL_PT_TSC: 3066 intel_pt_calc_tsc_timestamp(decoder); 3067 break; 3068 3069 case INTEL_PT_TMA: 3070 intel_pt_calc_tma(decoder); 3071 break; 3072 3073 case INTEL_PT_CYC: 3074 intel_pt_calc_cyc_timestamp(decoder); 3075 break; 3076 3077 case INTEL_PT_CBR: 3078 intel_pt_calc_cbr(decoder); 3079 if (decoder->cbr != decoder->cbr_seen) { 3080 decoder->state.type = 0; 3081 return 0; 3082 } 3083 break; 3084 3085 case INTEL_PT_MODE_EXEC: 3086 decoder->exec_mode = decoder->packet.payload; 3087 break; 3088 3089 case INTEL_PT_MODE_TSX: 3090 /* MODE_TSX need not be followed by FUP */ 3091 if (!decoder->pge || decoder->in_psb) { 3092 intel_pt_update_in_tx(decoder); 3093 break; 3094 } 3095 err = intel_pt_mode_tsx(decoder, &no_tip); 3096 if (err) 3097 return err; 3098 goto next; 3099 3100 case INTEL_PT_BAD: /* Does not happen */ 3101 return intel_pt_bug(decoder); 3102 3103 case INTEL_PT_PSBEND: 3104 case INTEL_PT_VMCS: 3105 case INTEL_PT_MNT: 3106 case INTEL_PT_PAD: 3107 break; 3108 3109 case INTEL_PT_PTWRITE_IP: 3110 decoder->fup_ptw_payload = decoder->packet.payload; 3111 err = intel_pt_get_next_packet(decoder); 3112 if (err) 3113 return err; 3114 if (decoder->packet.type == INTEL_PT_FUP) { 3115 decoder->set_fup_ptw = true; 3116 no_tip = true; 3117 } else { 3118 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 3119 decoder->pos); 3120 } 3121 goto next; 3122 3123 case INTEL_PT_PTWRITE: 3124 decoder->state.type = INTEL_PT_PTW; 3125 decoder->state.from_ip = decoder->ip; 3126 decoder->state.to_ip = 0; 3127 decoder->state.ptw_payload = decoder->packet.payload; 3128 return 0; 3129 3130 case INTEL_PT_MWAIT: 3131 decoder->fup_mwait_payload = decoder->packet.payload; 3132 decoder->set_fup_mwait = true; 3133 break; 3134 3135 case INTEL_PT_PWRE: 3136 if (decoder->set_fup_mwait) { 3137 decoder->fup_pwre_payload = 3138 decoder->packet.payload; 3139 decoder->set_fup_pwre = true; 3140 break; 3141 } 3142 decoder->state.type = INTEL_PT_PWR_ENTRY; 3143 decoder->state.from_ip = decoder->ip; 3144 decoder->state.to_ip = 0; 3145 decoder->state.pwrx_payload = decoder->packet.payload; 3146 return 0; 3147 3148 case INTEL_PT_EXSTOP_IP: 3149 err = intel_pt_get_next_packet(decoder); 3150 if (err) 3151 return err; 3152 if (decoder->packet.type == INTEL_PT_FUP) { 3153 decoder->set_fup_exstop = true; 3154 no_tip = true; 3155 } else { 3156 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 3157 decoder->pos); 3158 } 3159 goto next; 3160 3161 case INTEL_PT_EXSTOP: 3162 decoder->state.type = INTEL_PT_EX_STOP; 3163 decoder->state.from_ip = decoder->ip; 3164 decoder->state.to_ip = 0; 3165 return 0; 3166 3167 case INTEL_PT_PWRX: 3168 decoder->state.type = INTEL_PT_PWR_EXIT; 3169 decoder->state.from_ip = decoder->ip; 3170 decoder->state.to_ip = 0; 3171 decoder->state.pwrx_payload = decoder->packet.payload; 3172 return 0; 3173 3174 case INTEL_PT_BBP: 3175 intel_pt_bbp(decoder); 3176 break; 3177 3178 case INTEL_PT_BIP: 3179 intel_pt_bip(decoder); 3180 break; 3181 3182 case INTEL_PT_BEP: 3183 decoder->state.type = INTEL_PT_BLK_ITEMS; 3184 decoder->state.from_ip = decoder->ip; 3185 decoder->state.to_ip = 0; 3186 return 0; 3187 3188 case INTEL_PT_BEP_IP: 3189 err = intel_pt_get_next_packet(decoder); 3190 if (err) 3191 return err; 3192 if (decoder->packet.type == INTEL_PT_FUP) { 3193 decoder->set_fup_bep = true; 3194 no_tip = true; 3195 } else { 3196 intel_pt_log_at("ERROR: Missing FUP after BEP", 3197 decoder->pos); 3198 } 3199 goto next; 3200 3201 default: 3202 return intel_pt_bug(decoder); 3203 } 3204 } 3205 } 3206 3207 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 3208 { 3209 return decoder->packet.count && 3210 (decoder->have_last_ip || decoder->packet.count == 3 || 3211 decoder->packet.count == 6); 3212 } 3213 3214 /* Walk PSB+ packets to get in sync. */ 3215 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 3216 { 3217 int err; 3218 3219 decoder->in_psb = true; 3220 3221 while (1) { 3222 err = intel_pt_get_next_packet(decoder); 3223 if (err) 3224 goto out; 3225 3226 switch (decoder->packet.type) { 3227 case INTEL_PT_TIP_PGD: 3228 decoder->continuous_period = false; 3229 __fallthrough; 3230 case INTEL_PT_TIP_PGE: 3231 case INTEL_PT_TIP: 3232 case INTEL_PT_PTWRITE: 3233 case INTEL_PT_PTWRITE_IP: 3234 case INTEL_PT_EXSTOP: 3235 case INTEL_PT_EXSTOP_IP: 3236 case INTEL_PT_MWAIT: 3237 case INTEL_PT_PWRE: 3238 case INTEL_PT_PWRX: 3239 case INTEL_PT_BBP: 3240 case INTEL_PT_BIP: 3241 case INTEL_PT_BEP: 3242 case INTEL_PT_BEP_IP: 3243 intel_pt_log("ERROR: Unexpected packet\n"); 3244 err = -ENOENT; 3245 goto out; 3246 3247 case INTEL_PT_FUP: 3248 decoder->pge = true; 3249 if (intel_pt_have_ip(decoder)) { 3250 uint64_t current_ip = decoder->ip; 3251 3252 intel_pt_set_ip(decoder); 3253 decoder->psb_ip = decoder->ip; 3254 if (current_ip) 3255 intel_pt_log_to("Setting IP", 3256 decoder->ip); 3257 } 3258 break; 3259 3260 case INTEL_PT_MTC: 3261 intel_pt_calc_mtc_timestamp(decoder); 3262 break; 3263 3264 case INTEL_PT_TSC: 3265 intel_pt_calc_tsc_timestamp(decoder); 3266 break; 3267 3268 case INTEL_PT_TMA: 3269 intel_pt_calc_tma(decoder); 3270 break; 3271 3272 case INTEL_PT_CYC: 3273 intel_pt_calc_cyc_timestamp(decoder); 3274 break; 3275 3276 case INTEL_PT_CBR: 3277 intel_pt_calc_cbr(decoder); 3278 break; 3279 3280 case INTEL_PT_PIP: 3281 intel_pt_set_pip(decoder); 3282 break; 3283 3284 case INTEL_PT_MODE_EXEC: 3285 decoder->exec_mode = decoder->packet.payload; 3286 break; 3287 3288 case INTEL_PT_MODE_TSX: 3289 intel_pt_update_in_tx(decoder); 3290 break; 3291 3292 case INTEL_PT_TRACESTOP: 3293 decoder->pge = false; 3294 decoder->continuous_period = false; 3295 intel_pt_clear_tx_flags(decoder); 3296 __fallthrough; 3297 3298 case INTEL_PT_TNT: 3299 decoder->have_tma = false; 3300 intel_pt_log("ERROR: Unexpected packet\n"); 3301 if (decoder->ip) 3302 decoder->pkt_state = INTEL_PT_STATE_ERR4; 3303 else 3304 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3305 err = -ENOENT; 3306 goto out; 3307 3308 case INTEL_PT_BAD: /* Does not happen */ 3309 err = intel_pt_bug(decoder); 3310 goto out; 3311 3312 case INTEL_PT_OVF: 3313 err = intel_pt_overflow(decoder); 3314 goto out; 3315 3316 case INTEL_PT_PSBEND: 3317 err = 0; 3318 goto out; 3319 3320 case INTEL_PT_PSB: 3321 case INTEL_PT_VMCS: 3322 case INTEL_PT_MNT: 3323 case INTEL_PT_PAD: 3324 default: 3325 break; 3326 } 3327 } 3328 out: 3329 decoder->in_psb = false; 3330 3331 return err; 3332 } 3333 3334 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 3335 { 3336 int err; 3337 3338 while (1) { 3339 err = intel_pt_get_next_packet(decoder); 3340 if (err) 3341 return err; 3342 3343 switch (decoder->packet.type) { 3344 case INTEL_PT_TIP_PGD: 3345 decoder->continuous_period = false; 3346 decoder->pge = false; 3347 if (intel_pt_have_ip(decoder)) 3348 intel_pt_set_ip(decoder); 3349 if (!decoder->ip) 3350 break; 3351 decoder->state.type |= INTEL_PT_TRACE_END; 3352 return 0; 3353 3354 case INTEL_PT_TIP_PGE: 3355 decoder->pge = true; 3356 intel_pt_mtc_cyc_cnt_pge(decoder); 3357 if (intel_pt_have_ip(decoder)) 3358 intel_pt_set_ip(decoder); 3359 if (!decoder->ip) 3360 break; 3361 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3362 return 0; 3363 3364 case INTEL_PT_TIP: 3365 decoder->pge = true; 3366 if (intel_pt_have_ip(decoder)) 3367 intel_pt_set_ip(decoder); 3368 if (!decoder->ip) 3369 break; 3370 return 0; 3371 3372 case INTEL_PT_FUP: 3373 if (intel_pt_have_ip(decoder)) 3374 intel_pt_set_ip(decoder); 3375 if (decoder->ip) 3376 return 0; 3377 break; 3378 3379 case INTEL_PT_MTC: 3380 intel_pt_calc_mtc_timestamp(decoder); 3381 break; 3382 3383 case INTEL_PT_TSC: 3384 intel_pt_calc_tsc_timestamp(decoder); 3385 break; 3386 3387 case INTEL_PT_TMA: 3388 intel_pt_calc_tma(decoder); 3389 break; 3390 3391 case INTEL_PT_CYC: 3392 intel_pt_calc_cyc_timestamp(decoder); 3393 break; 3394 3395 case INTEL_PT_CBR: 3396 intel_pt_calc_cbr(decoder); 3397 break; 3398 3399 case INTEL_PT_PIP: 3400 intel_pt_set_pip(decoder); 3401 break; 3402 3403 case INTEL_PT_MODE_EXEC: 3404 decoder->exec_mode = decoder->packet.payload; 3405 break; 3406 3407 case INTEL_PT_MODE_TSX: 3408 intel_pt_update_in_tx(decoder); 3409 break; 3410 3411 case INTEL_PT_OVF: 3412 return intel_pt_overflow(decoder); 3413 3414 case INTEL_PT_BAD: /* Does not happen */ 3415 return intel_pt_bug(decoder); 3416 3417 case INTEL_PT_TRACESTOP: 3418 decoder->pge = false; 3419 decoder->continuous_period = false; 3420 intel_pt_clear_tx_flags(decoder); 3421 decoder->have_tma = false; 3422 break; 3423 3424 case INTEL_PT_PSB: 3425 decoder->state.psb_offset = decoder->pos; 3426 decoder->psb_ip = 0; 3427 decoder->last_ip = 0; 3428 decoder->have_last_ip = true; 3429 intel_pt_clear_stack(&decoder->stack); 3430 err = intel_pt_walk_psb(decoder); 3431 if (err) 3432 return err; 3433 decoder->state.type = INTEL_PT_PSB_EVT; 3434 decoder->state.from_ip = decoder->psb_ip; 3435 decoder->state.to_ip = 0; 3436 return 0; 3437 3438 case INTEL_PT_TNT: 3439 case INTEL_PT_PSBEND: 3440 case INTEL_PT_VMCS: 3441 case INTEL_PT_MNT: 3442 case INTEL_PT_PAD: 3443 case INTEL_PT_PTWRITE: 3444 case INTEL_PT_PTWRITE_IP: 3445 case INTEL_PT_EXSTOP: 3446 case INTEL_PT_EXSTOP_IP: 3447 case INTEL_PT_MWAIT: 3448 case INTEL_PT_PWRE: 3449 case INTEL_PT_PWRX: 3450 case INTEL_PT_BBP: 3451 case INTEL_PT_BIP: 3452 case INTEL_PT_BEP: 3453 case INTEL_PT_BEP_IP: 3454 default: 3455 break; 3456 } 3457 } 3458 } 3459 3460 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 3461 { 3462 int err; 3463 3464 decoder->set_fup_tx_flags = false; 3465 decoder->set_fup_ptw = false; 3466 decoder->set_fup_mwait = false; 3467 decoder->set_fup_pwre = false; 3468 decoder->set_fup_exstop = false; 3469 decoder->set_fup_bep = false; 3470 3471 if (!decoder->branch_enable) { 3472 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3473 decoder->overflow = false; 3474 decoder->state.type = 0; /* Do not have a sample */ 3475 return 0; 3476 } 3477 3478 intel_pt_log("Scanning for full IP\n"); 3479 err = intel_pt_walk_to_ip(decoder); 3480 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 3481 return err; 3482 3483 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 3484 if (decoder->hop) 3485 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3486 else 3487 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3488 decoder->overflow = false; 3489 3490 decoder->state.from_ip = 0; 3491 decoder->state.to_ip = decoder->ip; 3492 intel_pt_log_to("Setting IP", decoder->ip); 3493 3494 return 0; 3495 } 3496 3497 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 3498 { 3499 const unsigned char *end = decoder->buf + decoder->len; 3500 size_t i; 3501 3502 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 3503 if (i > decoder->len) 3504 continue; 3505 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 3506 return i; 3507 } 3508 return 0; 3509 } 3510 3511 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 3512 { 3513 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 3514 const char *psb = INTEL_PT_PSB_STR; 3515 3516 if (rest_psb > decoder->len || 3517 memcmp(decoder->buf, psb + part_psb, rest_psb)) 3518 return 0; 3519 3520 return rest_psb; 3521 } 3522 3523 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 3524 int part_psb) 3525 { 3526 int rest_psb, ret; 3527 3528 decoder->pos += decoder->len; 3529 decoder->len = 0; 3530 3531 ret = intel_pt_get_next_data(decoder, false); 3532 if (ret) 3533 return ret; 3534 3535 rest_psb = intel_pt_rest_psb(decoder, part_psb); 3536 if (!rest_psb) 3537 return 0; 3538 3539 decoder->pos -= part_psb; 3540 decoder->next_buf = decoder->buf + rest_psb; 3541 decoder->next_len = decoder->len - rest_psb; 3542 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3543 decoder->buf = decoder->temp_buf; 3544 decoder->len = INTEL_PT_PSB_LEN; 3545 3546 return 0; 3547 } 3548 3549 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 3550 { 3551 unsigned char *next; 3552 int ret; 3553 3554 intel_pt_log("Scanning for PSB\n"); 3555 while (1) { 3556 if (!decoder->len) { 3557 ret = intel_pt_get_next_data(decoder, false); 3558 if (ret) 3559 return ret; 3560 } 3561 3562 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 3563 INTEL_PT_PSB_LEN); 3564 if (!next) { 3565 int part_psb; 3566 3567 part_psb = intel_pt_part_psb(decoder); 3568 if (part_psb) { 3569 ret = intel_pt_get_split_psb(decoder, part_psb); 3570 if (ret) 3571 return ret; 3572 } else { 3573 decoder->pos += decoder->len; 3574 decoder->len = 0; 3575 } 3576 continue; 3577 } 3578 3579 decoder->pkt_step = next - decoder->buf; 3580 return intel_pt_get_next_packet(decoder); 3581 } 3582 } 3583 3584 static int intel_pt_sync(struct intel_pt_decoder *decoder) 3585 { 3586 int err; 3587 3588 decoder->pge = false; 3589 decoder->continuous_period = false; 3590 decoder->have_last_ip = false; 3591 decoder->last_ip = 0; 3592 decoder->psb_ip = 0; 3593 decoder->ip = 0; 3594 intel_pt_clear_stack(&decoder->stack); 3595 3596 err = intel_pt_scan_for_psb(decoder); 3597 if (err) 3598 return err; 3599 3600 if (decoder->vm_time_correlation) { 3601 decoder->in_psb = true; 3602 if (!decoder->timestamp) 3603 decoder->timestamp = 1; 3604 decoder->state.type = 0; 3605 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; 3606 return 0; 3607 } 3608 3609 decoder->have_last_ip = true; 3610 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 3611 3612 err = intel_pt_walk_psb(decoder); 3613 if (err) 3614 return err; 3615 3616 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 3617 decoder->state.from_ip = decoder->psb_ip; 3618 decoder->state.to_ip = 0; 3619 3620 if (decoder->ip) { 3621 /* 3622 * In hop mode, resample to get the PSB FUP ip as an 3623 * "instruction" sample. 3624 */ 3625 if (decoder->hop) 3626 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3627 else 3628 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3629 } 3630 3631 return 0; 3632 } 3633 3634 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 3635 { 3636 uint64_t est = decoder->sample_insn_cnt << 1; 3637 3638 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 3639 goto out; 3640 3641 est *= decoder->max_non_turbo_ratio; 3642 est /= decoder->cbr; 3643 out: 3644 return decoder->sample_timestamp + est; 3645 } 3646 3647 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 3648 { 3649 int err; 3650 3651 do { 3652 decoder->state.type = INTEL_PT_BRANCH; 3653 decoder->state.flags = 0; 3654 3655 switch (decoder->pkt_state) { 3656 case INTEL_PT_STATE_NO_PSB: 3657 err = intel_pt_sync(decoder); 3658 break; 3659 case INTEL_PT_STATE_NO_IP: 3660 decoder->have_last_ip = false; 3661 decoder->last_ip = 0; 3662 decoder->ip = 0; 3663 __fallthrough; 3664 case INTEL_PT_STATE_ERR_RESYNC: 3665 err = intel_pt_sync_ip(decoder); 3666 break; 3667 case INTEL_PT_STATE_IN_SYNC: 3668 err = intel_pt_walk_trace(decoder); 3669 break; 3670 case INTEL_PT_STATE_TNT: 3671 case INTEL_PT_STATE_TNT_CONT: 3672 err = intel_pt_walk_tnt(decoder); 3673 if (err == -EAGAIN) 3674 err = intel_pt_walk_trace(decoder); 3675 break; 3676 case INTEL_PT_STATE_TIP: 3677 case INTEL_PT_STATE_TIP_PGD: 3678 err = intel_pt_walk_tip(decoder); 3679 break; 3680 case INTEL_PT_STATE_FUP: 3681 err = intel_pt_walk_fup(decoder); 3682 if (err == -EAGAIN) 3683 err = intel_pt_walk_fup_tip(decoder); 3684 break; 3685 case INTEL_PT_STATE_FUP_NO_TIP: 3686 err = intel_pt_walk_fup(decoder); 3687 if (err == -EAGAIN) 3688 err = intel_pt_walk_trace(decoder); 3689 break; 3690 case INTEL_PT_STATE_FUP_IN_PSB: 3691 err = intel_pt_fup_in_psb(decoder); 3692 break; 3693 case INTEL_PT_STATE_RESAMPLE: 3694 err = intel_pt_resample(decoder); 3695 break; 3696 case INTEL_PT_STATE_VM_TIME_CORRELATION: 3697 err = intel_pt_vm_time_correlation(decoder); 3698 break; 3699 default: 3700 err = intel_pt_bug(decoder); 3701 break; 3702 } 3703 } while (err == -ENOLINK); 3704 3705 if (err) { 3706 decoder->state.err = intel_pt_ext_err(err); 3707 decoder->state.from_ip = decoder->ip; 3708 intel_pt_update_sample_time(decoder); 3709 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3710 intel_pt_set_nr(decoder); 3711 } else { 3712 decoder->state.err = 0; 3713 if (decoder->cbr != decoder->cbr_seen) { 3714 decoder->cbr_seen = decoder->cbr; 3715 if (!decoder->state.type) { 3716 decoder->state.from_ip = decoder->ip; 3717 decoder->state.to_ip = 0; 3718 } 3719 decoder->state.type |= INTEL_PT_CBR_CHG; 3720 decoder->state.cbr_payload = decoder->cbr_payload; 3721 decoder->state.cbr = decoder->cbr; 3722 } 3723 if (intel_pt_sample_time(decoder->pkt_state)) { 3724 intel_pt_update_sample_time(decoder); 3725 if (decoder->sample_cyc) { 3726 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3727 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3728 decoder->sample_cyc = false; 3729 } 3730 } 3731 /* 3732 * When using only TSC/MTC to compute cycles, IPC can be 3733 * sampled as soon as the cycle count changes. 3734 */ 3735 if (!decoder->have_cyc) 3736 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3737 } 3738 3739 /* Let PSB event always have TSC timestamp */ 3740 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 3741 decoder->sample_timestamp = decoder->tsc_timestamp; 3742 3743 decoder->state.from_nr = decoder->nr; 3744 decoder->state.to_nr = decoder->next_nr; 3745 decoder->nr = decoder->next_nr; 3746 3747 decoder->state.timestamp = decoder->sample_timestamp; 3748 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 3749 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 3750 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 3751 3752 return &decoder->state; 3753 } 3754 3755 /** 3756 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 3757 * @buf: pointer to buffer pointer 3758 * @len: size of buffer 3759 * 3760 * Updates the buffer pointer to point to the start of the next PSB packet if 3761 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 3762 * @len is adjusted accordingly. 3763 * 3764 * Return: %true if a PSB packet is found, %false otherwise. 3765 */ 3766 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 3767 { 3768 unsigned char *next; 3769 3770 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3771 if (next) { 3772 *len -= next - *buf; 3773 *buf = next; 3774 return true; 3775 } 3776 return false; 3777 } 3778 3779 /** 3780 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 3781 * packet. 3782 * @buf: pointer to buffer pointer 3783 * @len: size of buffer 3784 * 3785 * Updates the buffer pointer to point to the start of the following PSB packet 3786 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 3787 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 3788 * 3789 * Return: %true if a PSB packet is found, %false otherwise. 3790 */ 3791 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 3792 { 3793 unsigned char *next; 3794 3795 if (!*len) 3796 return false; 3797 3798 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3799 if (next) { 3800 *len -= next - *buf; 3801 *buf = next; 3802 return true; 3803 } 3804 return false; 3805 } 3806 3807 /** 3808 * intel_pt_last_psb - find the last PSB packet in a buffer. 3809 * @buf: buffer 3810 * @len: size of buffer 3811 * 3812 * This function finds the last PSB in a buffer. 3813 * 3814 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 3815 */ 3816 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 3817 { 3818 const char *n = INTEL_PT_PSB_STR; 3819 unsigned char *p; 3820 size_t k; 3821 3822 if (len < INTEL_PT_PSB_LEN) 3823 return NULL; 3824 3825 k = len - INTEL_PT_PSB_LEN + 1; 3826 while (1) { 3827 p = memrchr(buf, n[0], k); 3828 if (!p) 3829 return NULL; 3830 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 3831 return p; 3832 k = p - buf; 3833 if (!k) 3834 return NULL; 3835 } 3836 } 3837 3838 /** 3839 * intel_pt_next_tsc - find and return next TSC. 3840 * @buf: buffer 3841 * @len: size of buffer 3842 * @tsc: TSC value returned 3843 * @rem: returns remaining size when TSC is found 3844 * 3845 * Find a TSC packet in @buf and return the TSC value. This function assumes 3846 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 3847 * PSBEND packet is found. 3848 * 3849 * Return: %true if TSC is found, false otherwise. 3850 */ 3851 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 3852 size_t *rem) 3853 { 3854 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 3855 struct intel_pt_pkt packet; 3856 int ret; 3857 3858 while (len) { 3859 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 3860 if (ret <= 0) 3861 return false; 3862 if (packet.type == INTEL_PT_TSC) { 3863 *tsc = packet.payload; 3864 *rem = len; 3865 return true; 3866 } 3867 if (packet.type == INTEL_PT_PSBEND) 3868 return false; 3869 buf += ret; 3870 len -= ret; 3871 } 3872 return false; 3873 } 3874 3875 /** 3876 * intel_pt_tsc_cmp - compare 7-byte TSCs. 3877 * @tsc1: first TSC to compare 3878 * @tsc2: second TSC to compare 3879 * 3880 * This function compares 7-byte TSC values allowing for the possibility that 3881 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 3882 * around so for that purpose this function assumes the absolute difference is 3883 * less than half the maximum difference. 3884 * 3885 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 3886 * after @tsc2. 3887 */ 3888 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 3889 { 3890 const uint64_t halfway = (1ULL << 55); 3891 3892 if (tsc1 == tsc2) 3893 return 0; 3894 3895 if (tsc1 < tsc2) { 3896 if (tsc2 - tsc1 < halfway) 3897 return -1; 3898 else 3899 return 1; 3900 } else { 3901 if (tsc1 - tsc2 < halfway) 3902 return 1; 3903 else 3904 return -1; 3905 } 3906 } 3907 3908 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 3909 3910 /** 3911 * adj_for_padding - adjust overlap to account for padding. 3912 * @buf_b: second buffer 3913 * @buf_a: first buffer 3914 * @len_a: size of first buffer 3915 * 3916 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 3917 * accordingly. 3918 * 3919 * Return: A pointer into @buf_b from where non-overlapped data starts 3920 */ 3921 static unsigned char *adj_for_padding(unsigned char *buf_b, 3922 unsigned char *buf_a, size_t len_a) 3923 { 3924 unsigned char *p = buf_b - MAX_PADDING; 3925 unsigned char *q = buf_a + len_a - MAX_PADDING; 3926 int i; 3927 3928 for (i = MAX_PADDING; i; i--, p++, q++) { 3929 if (*p != *q) 3930 break; 3931 } 3932 3933 return p; 3934 } 3935 3936 /** 3937 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3938 * using TSC. 3939 * @buf_a: first buffer 3940 * @len_a: size of first buffer 3941 * @buf_b: second buffer 3942 * @len_b: size of second buffer 3943 * @consecutive: returns true if there is data in buf_b that is consecutive 3944 * to buf_a 3945 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 3946 * 3947 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3948 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3949 * walk forward in @buf_b until a later TSC is found. A precondition is that 3950 * @buf_a and @buf_b are positioned at a PSB. 3951 * 3952 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3953 * @buf_b + @len_b if there is no non-overlapped data. 3954 */ 3955 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3956 size_t len_a, 3957 unsigned char *buf_b, 3958 size_t len_b, bool *consecutive, 3959 bool ooo_tsc) 3960 { 3961 uint64_t tsc_a, tsc_b; 3962 unsigned char *p; 3963 size_t len, rem_a, rem_b; 3964 3965 p = intel_pt_last_psb(buf_a, len_a); 3966 if (!p) 3967 return buf_b; /* No PSB in buf_a => no overlap */ 3968 3969 len = len_a - (p - buf_a); 3970 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3971 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3972 len_a -= len; 3973 p = intel_pt_last_psb(buf_a, len_a); 3974 if (!p) 3975 return buf_b; /* No full PSB+ => assume no overlap */ 3976 len = len_a - (p - buf_a); 3977 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 3978 return buf_b; /* No TSC in buf_a => assume no overlap */ 3979 } 3980 3981 while (1) { 3982 /* Ignore PSB+ with no TSC */ 3983 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 3984 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 3985 3986 /* Same TSC, so buffers are consecutive */ 3987 if (!cmp && rem_b >= rem_a) { 3988 unsigned char *start; 3989 3990 *consecutive = true; 3991 start = buf_b + len_b - (rem_b - rem_a); 3992 return adj_for_padding(start, buf_a, len_a); 3993 } 3994 if (cmp < 0 && !ooo_tsc) 3995 return buf_b; /* tsc_a < tsc_b => no overlap */ 3996 } 3997 3998 if (!intel_pt_step_psb(&buf_b, &len_b)) 3999 return buf_b + len_b; /* No PSB in buf_b => no data */ 4000 } 4001 } 4002 4003 /** 4004 * intel_pt_find_overlap - determine start of non-overlapped trace data. 4005 * @buf_a: first buffer 4006 * @len_a: size of first buffer 4007 * @buf_b: second buffer 4008 * @len_b: size of second buffer 4009 * @have_tsc: can use TSC packets to detect overlap 4010 * @consecutive: returns true if there is data in buf_b that is consecutive 4011 * to buf_a 4012 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4013 * 4014 * When trace samples or snapshots are recorded there is the possibility that 4015 * the data overlaps. Note that, for the purposes of decoding, data is only 4016 * useful if it begins with a PSB packet. 4017 * 4018 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4019 * @buf_b + @len_b if there is no non-overlapped data. 4020 */ 4021 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 4022 unsigned char *buf_b, size_t len_b, 4023 bool have_tsc, bool *consecutive, 4024 bool ooo_tsc) 4025 { 4026 unsigned char *found; 4027 4028 /* Buffer 'b' must start at PSB so throw away everything before that */ 4029 if (!intel_pt_next_psb(&buf_b, &len_b)) 4030 return buf_b + len_b; /* No PSB */ 4031 4032 if (!intel_pt_next_psb(&buf_a, &len_a)) 4033 return buf_b; /* No overlap */ 4034 4035 if (have_tsc) { 4036 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 4037 consecutive, ooo_tsc); 4038 if (found) 4039 return found; 4040 } 4041 4042 /* 4043 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 4044 * we can ignore the first part of buffer 'a'. 4045 */ 4046 while (len_b < len_a) { 4047 if (!intel_pt_step_psb(&buf_a, &len_a)) 4048 return buf_b; /* No overlap */ 4049 } 4050 4051 /* Now len_b >= len_a */ 4052 while (1) { 4053 /* Potential overlap so check the bytes */ 4054 found = memmem(buf_a, len_a, buf_b, len_a); 4055 if (found) { 4056 *consecutive = true; 4057 return adj_for_padding(buf_b + len_a, buf_a, len_a); 4058 } 4059 4060 /* Try again at next PSB in buffer 'a' */ 4061 if (!intel_pt_step_psb(&buf_a, &len_a)) 4062 return buf_b; /* No overlap */ 4063 } 4064 } 4065 4066 /** 4067 * struct fast_forward_data - data used by intel_pt_ff_cb(). 4068 * @timestamp: timestamp to fast forward towards 4069 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 4070 * the fast forward timestamp. 4071 */ 4072 struct fast_forward_data { 4073 uint64_t timestamp; 4074 uint64_t buf_timestamp; 4075 }; 4076 4077 /** 4078 * intel_pt_ff_cb - fast forward lookahead callback. 4079 * @buffer: Intel PT trace buffer 4080 * @data: opaque pointer to fast forward data (struct fast_forward_data) 4081 * 4082 * Determine if @buffer trace is past the fast forward timestamp. 4083 * 4084 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 4085 * timestamp, and 0 otherwise. 4086 */ 4087 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 4088 { 4089 struct fast_forward_data *d = data; 4090 unsigned char *buf; 4091 uint64_t tsc; 4092 size_t rem; 4093 size_t len; 4094 4095 buf = (unsigned char *)buffer->buf; 4096 len = buffer->len; 4097 4098 if (!intel_pt_next_psb(&buf, &len) || 4099 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 4100 return 0; 4101 4102 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 4103 4104 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 4105 tsc, buffer->ref_timestamp); 4106 4107 /* 4108 * If the buffer contains a timestamp earlier that the fast forward 4109 * timestamp, then record it, else stop. 4110 */ 4111 if (tsc < d->timestamp) 4112 d->buf_timestamp = buffer->ref_timestamp; 4113 else 4114 return 1; 4115 4116 return 0; 4117 } 4118 4119 /** 4120 * intel_pt_fast_forward - reposition decoder forwards. 4121 * @decoder: Intel PT decoder 4122 * @timestamp: timestamp to fast forward towards 4123 * 4124 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 4125 * 4126 * Return: 0 on success or negative error code on failure. 4127 */ 4128 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 4129 { 4130 struct fast_forward_data d = { .timestamp = timestamp }; 4131 unsigned char *buf; 4132 size_t len; 4133 int err; 4134 4135 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 4136 4137 /* Find buffer timestamp of buffer to fast forward to */ 4138 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 4139 if (err < 0) 4140 return err; 4141 4142 /* Walk to buffer with same buffer timestamp */ 4143 if (d.buf_timestamp) { 4144 do { 4145 decoder->pos += decoder->len; 4146 decoder->len = 0; 4147 err = intel_pt_get_next_data(decoder, true); 4148 /* -ENOLINK means non-consecutive trace */ 4149 if (err && err != -ENOLINK) 4150 return err; 4151 } while (decoder->buf_timestamp != d.buf_timestamp); 4152 } 4153 4154 if (!decoder->buf) 4155 return 0; 4156 4157 buf = (unsigned char *)decoder->buf; 4158 len = decoder->len; 4159 4160 if (!intel_pt_next_psb(&buf, &len)) 4161 return 0; 4162 4163 /* 4164 * Walk PSBs while the PSB timestamp is less than the fast forward 4165 * timestamp. 4166 */ 4167 do { 4168 uint64_t tsc; 4169 size_t rem; 4170 4171 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 4172 break; 4173 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 4174 /* 4175 * A TSC packet can slip past MTC packets but, after fast 4176 * forward, decoding starts at the TSC timestamp. That means 4177 * the timestamps may not be exactly the same as the timestamps 4178 * that would have been decoded without fast forward. 4179 */ 4180 if (tsc < timestamp) { 4181 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 4182 decoder->pos += decoder->len - len; 4183 decoder->buf = buf; 4184 decoder->len = len; 4185 intel_pt_reposition(decoder); 4186 } else { 4187 break; 4188 } 4189 } while (intel_pt_step_psb(&buf, &len)); 4190 4191 return 0; 4192 } 4193