1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define SEVEN_BYTES 0xffffffffffffffULL 39 40 #define NO_VMCS 0xffffffffffULL 41 42 #define INTEL_PT_RETURN 1 43 44 /* 45 * Default maximum number of loops with no packets consumed i.e. stuck in a 46 * loop. 47 */ 48 #define INTEL_PT_MAX_LOOPS 100000 49 50 struct intel_pt_blk { 51 struct intel_pt_blk *prev; 52 uint64_t ip[INTEL_PT_BLK_SIZE]; 53 }; 54 55 struct intel_pt_stack { 56 struct intel_pt_blk *blk; 57 struct intel_pt_blk *spare; 58 int pos; 59 }; 60 61 enum intel_pt_p_once { 62 INTEL_PT_PRT_ONCE_UNK_VMCS, 63 INTEL_PT_PRT_ONCE_ERANGE, 64 }; 65 66 enum intel_pt_pkt_state { 67 INTEL_PT_STATE_NO_PSB, 68 INTEL_PT_STATE_NO_IP, 69 INTEL_PT_STATE_ERR_RESYNC, 70 INTEL_PT_STATE_IN_SYNC, 71 INTEL_PT_STATE_TNT_CONT, 72 INTEL_PT_STATE_TNT, 73 INTEL_PT_STATE_TIP, 74 INTEL_PT_STATE_TIP_PGD, 75 INTEL_PT_STATE_FUP, 76 INTEL_PT_STATE_FUP_NO_TIP, 77 INTEL_PT_STATE_FUP_IN_PSB, 78 INTEL_PT_STATE_RESAMPLE, 79 INTEL_PT_STATE_VM_TIME_CORRELATION, 80 }; 81 82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 83 { 84 switch (pkt_state) { 85 case INTEL_PT_STATE_NO_PSB: 86 case INTEL_PT_STATE_NO_IP: 87 case INTEL_PT_STATE_ERR_RESYNC: 88 case INTEL_PT_STATE_IN_SYNC: 89 case INTEL_PT_STATE_TNT_CONT: 90 case INTEL_PT_STATE_RESAMPLE: 91 case INTEL_PT_STATE_VM_TIME_CORRELATION: 92 return true; 93 case INTEL_PT_STATE_TNT: 94 case INTEL_PT_STATE_TIP: 95 case INTEL_PT_STATE_TIP_PGD: 96 case INTEL_PT_STATE_FUP: 97 case INTEL_PT_STATE_FUP_NO_TIP: 98 case INTEL_PT_STATE_FUP_IN_PSB: 99 return false; 100 default: 101 return true; 102 }; 103 } 104 105 #ifdef INTEL_PT_STRICT 106 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 107 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 108 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 109 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 110 #else 111 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 112 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 113 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 114 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 115 #endif 116 117 struct intel_pt_decoder { 118 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 119 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 120 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 121 uint64_t max_insn_cnt, void *data); 122 bool (*pgd_ip)(uint64_t ip, void *data); 123 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 124 struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); 125 void *data; 126 struct intel_pt_state state; 127 const unsigned char *buf; 128 size_t len; 129 bool return_compression; 130 bool branch_enable; 131 bool mtc_insn; 132 bool pge; 133 bool have_tma; 134 bool have_cyc; 135 bool fixup_last_mtc; 136 bool have_last_ip; 137 bool in_psb; 138 bool hop; 139 bool leap; 140 bool vm_time_correlation; 141 bool vm_tm_corr_dry_run; 142 bool vm_tm_corr_reliable; 143 bool vm_tm_corr_same_buf; 144 bool vm_tm_corr_continuous; 145 bool nr; 146 bool next_nr; 147 enum intel_pt_param_flags flags; 148 uint64_t pos; 149 uint64_t last_ip; 150 uint64_t ip; 151 uint64_t pip_payload; 152 uint64_t timestamp; 153 uint64_t tsc_timestamp; 154 uint64_t ref_timestamp; 155 uint64_t buf_timestamp; 156 uint64_t sample_timestamp; 157 uint64_t ret_addr; 158 uint64_t ctc_timestamp; 159 uint64_t ctc_delta; 160 uint64_t cycle_cnt; 161 uint64_t cyc_ref_timestamp; 162 uint64_t first_timestamp; 163 uint64_t last_reliable_timestamp; 164 uint64_t vmcs; 165 uint64_t print_once; 166 uint64_t last_ctc; 167 uint32_t last_mtc; 168 uint32_t tsc_ctc_ratio_n; 169 uint32_t tsc_ctc_ratio_d; 170 uint32_t tsc_ctc_mult; 171 uint32_t tsc_slip; 172 uint32_t ctc_rem_mask; 173 int mtc_shift; 174 struct intel_pt_stack stack; 175 enum intel_pt_pkt_state pkt_state; 176 enum intel_pt_pkt_ctx pkt_ctx; 177 enum intel_pt_pkt_ctx prev_pkt_ctx; 178 enum intel_pt_blk_type blk_type; 179 int blk_type_pos; 180 struct intel_pt_pkt packet; 181 struct intel_pt_pkt tnt; 182 int pkt_step; 183 int pkt_len; 184 int last_packet_type; 185 unsigned int cbr; 186 unsigned int cbr_seen; 187 unsigned int max_non_turbo_ratio; 188 double max_non_turbo_ratio_fp; 189 double cbr_cyc_to_tsc; 190 double calc_cyc_to_tsc; 191 bool have_calc_cyc_to_tsc; 192 int exec_mode; 193 unsigned int insn_bytes; 194 uint64_t period; 195 enum intel_pt_period_type period_type; 196 uint64_t tot_insn_cnt; 197 uint64_t period_insn_cnt; 198 uint64_t period_mask; 199 uint64_t period_ticks; 200 uint64_t last_masked_timestamp; 201 uint64_t tot_cyc_cnt; 202 uint64_t sample_tot_cyc_cnt; 203 uint64_t base_cyc_cnt; 204 uint64_t cyc_cnt_timestamp; 205 uint64_t ctl; 206 uint64_t cyc_threshold; 207 double tsc_to_cyc; 208 bool continuous_period; 209 bool overflow; 210 bool set_fup_tx_flags; 211 bool set_fup_ptw; 212 bool set_fup_mwait; 213 bool set_fup_pwre; 214 bool set_fup_exstop; 215 bool set_fup_bep; 216 bool sample_cyc; 217 unsigned int fup_tx_flags; 218 unsigned int tx_flags; 219 uint64_t fup_ptw_payload; 220 uint64_t fup_mwait_payload; 221 uint64_t fup_pwre_payload; 222 uint64_t cbr_payload; 223 uint64_t timestamp_insn_cnt; 224 uint64_t sample_insn_cnt; 225 uint64_t stuck_ip; 226 int max_loops; 227 int no_progress; 228 int stuck_ip_prd; 229 int stuck_ip_cnt; 230 uint64_t psb_ip; 231 const unsigned char *next_buf; 232 size_t next_len; 233 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 234 }; 235 236 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 237 { 238 int i; 239 240 for (i = 0; x != 1; i++) 241 x >>= 1; 242 243 return x << i; 244 } 245 246 __printf(1, 2) 247 static void p_log(const char *fmt, ...) 248 { 249 char buf[512]; 250 va_list args; 251 252 va_start(args, fmt); 253 vsnprintf(buf, sizeof(buf), fmt, args); 254 va_end(args); 255 256 fprintf(stderr, "%s\n", buf); 257 intel_pt_log("%s\n", buf); 258 } 259 260 static bool intel_pt_print_once(struct intel_pt_decoder *decoder, 261 enum intel_pt_p_once id) 262 { 263 uint64_t bit = 1ULL << id; 264 265 if (decoder->print_once & bit) 266 return false; 267 decoder->print_once |= bit; 268 return true; 269 } 270 271 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 272 { 273 if (!(ctl & INTEL_PT_CYC_ENABLE)) 274 return 0; 275 276 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 277 } 278 279 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 280 { 281 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 282 uint64_t period; 283 284 period = intel_pt_lower_power_of_2(decoder->period); 285 decoder->period_mask = ~(period - 1); 286 decoder->period_ticks = period; 287 } 288 } 289 290 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 291 { 292 if (!d) 293 return 0; 294 return (t / d) * n + ((t % d) * n) / d; 295 } 296 297 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 298 { 299 struct intel_pt_decoder *decoder; 300 301 if (!params->get_trace || !params->walk_insn) 302 return NULL; 303 304 decoder = zalloc(sizeof(struct intel_pt_decoder)); 305 if (!decoder) 306 return NULL; 307 308 decoder->get_trace = params->get_trace; 309 decoder->walk_insn = params->walk_insn; 310 decoder->pgd_ip = params->pgd_ip; 311 decoder->lookahead = params->lookahead; 312 decoder->findnew_vmcs_info = params->findnew_vmcs_info; 313 decoder->data = params->data; 314 decoder->return_compression = params->return_compression; 315 decoder->branch_enable = params->branch_enable; 316 decoder->hop = params->quick >= 1; 317 decoder->leap = params->quick >= 2; 318 decoder->vm_time_correlation = params->vm_time_correlation; 319 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; 320 decoder->first_timestamp = params->first_timestamp; 321 decoder->last_reliable_timestamp = params->first_timestamp; 322 decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS; 323 324 decoder->flags = params->flags; 325 326 decoder->ctl = params->ctl; 327 decoder->period = params->period; 328 decoder->period_type = params->period_type; 329 330 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 331 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 332 333 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 334 335 intel_pt_setup_period(decoder); 336 337 decoder->mtc_shift = params->mtc_period; 338 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 339 340 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 341 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 342 343 if (!decoder->tsc_ctc_ratio_n) 344 decoder->tsc_ctc_ratio_d = 0; 345 346 if (decoder->tsc_ctc_ratio_d) { 347 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 348 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 349 decoder->tsc_ctc_ratio_d; 350 } 351 352 /* 353 * A TSC packet can slip past MTC packets so that the timestamp appears 354 * to go backwards. One estimate is that can be up to about 40 CPU 355 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 356 * slippage an order of magnitude more to be on the safe side. 357 */ 358 decoder->tsc_slip = 0x10000; 359 360 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 361 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 362 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 363 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 364 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 365 366 if (decoder->hop) 367 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 368 369 return decoder; 370 } 371 372 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, 373 uint64_t first_timestamp) 374 { 375 decoder->first_timestamp = first_timestamp; 376 } 377 378 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 379 { 380 struct intel_pt_blk *blk = stack->blk; 381 382 stack->blk = blk->prev; 383 if (!stack->spare) 384 stack->spare = blk; 385 else 386 free(blk); 387 } 388 389 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 390 { 391 if (!stack->pos) { 392 if (!stack->blk) 393 return 0; 394 intel_pt_pop_blk(stack); 395 if (!stack->blk) 396 return 0; 397 stack->pos = INTEL_PT_BLK_SIZE; 398 } 399 return stack->blk->ip[--stack->pos]; 400 } 401 402 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 403 { 404 struct intel_pt_blk *blk; 405 406 if (stack->spare) { 407 blk = stack->spare; 408 stack->spare = NULL; 409 } else { 410 blk = malloc(sizeof(struct intel_pt_blk)); 411 if (!blk) 412 return -ENOMEM; 413 } 414 415 blk->prev = stack->blk; 416 stack->blk = blk; 417 stack->pos = 0; 418 return 0; 419 } 420 421 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 422 { 423 int err; 424 425 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 426 err = intel_pt_alloc_blk(stack); 427 if (err) 428 return err; 429 } 430 431 stack->blk->ip[stack->pos++] = ip; 432 return 0; 433 } 434 435 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 436 { 437 while (stack->blk) 438 intel_pt_pop_blk(stack); 439 stack->pos = 0; 440 } 441 442 static void intel_pt_free_stack(struct intel_pt_stack *stack) 443 { 444 intel_pt_clear_stack(stack); 445 zfree(&stack->blk); 446 zfree(&stack->spare); 447 } 448 449 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 450 { 451 intel_pt_free_stack(&decoder->stack); 452 free(decoder); 453 } 454 455 static int intel_pt_ext_err(int code) 456 { 457 switch (code) { 458 case -ENOMEM: 459 return INTEL_PT_ERR_NOMEM; 460 case -ENOSYS: 461 return INTEL_PT_ERR_INTERN; 462 case -EBADMSG: 463 return INTEL_PT_ERR_BADPKT; 464 case -ENODATA: 465 return INTEL_PT_ERR_NODATA; 466 case -EILSEQ: 467 return INTEL_PT_ERR_NOINSN; 468 case -ENOENT: 469 return INTEL_PT_ERR_MISMAT; 470 case -EOVERFLOW: 471 return INTEL_PT_ERR_OVR; 472 case -ENOSPC: 473 return INTEL_PT_ERR_LOST; 474 case -ELOOP: 475 return INTEL_PT_ERR_NELOOP; 476 default: 477 return INTEL_PT_ERR_UNK; 478 } 479 } 480 481 static const char *intel_pt_err_msgs[] = { 482 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 483 [INTEL_PT_ERR_INTERN] = "Internal error", 484 [INTEL_PT_ERR_BADPKT] = "Bad packet", 485 [INTEL_PT_ERR_NODATA] = "No more data", 486 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 487 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 488 [INTEL_PT_ERR_OVR] = "Overflow packet", 489 [INTEL_PT_ERR_LOST] = "Lost trace data", 490 [INTEL_PT_ERR_UNK] = "Unknown error!", 491 [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)", 492 }; 493 494 int intel_pt__strerror(int code, char *buf, size_t buflen) 495 { 496 if (code < 1 || code >= INTEL_PT_ERR_MAX) 497 code = INTEL_PT_ERR_UNK; 498 strlcpy(buf, intel_pt_err_msgs[code], buflen); 499 return 0; 500 } 501 502 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 503 uint64_t last_ip) 504 { 505 uint64_t ip; 506 507 switch (packet->count) { 508 case 1: 509 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 510 packet->payload; 511 break; 512 case 2: 513 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 514 packet->payload; 515 break; 516 case 3: 517 ip = packet->payload; 518 /* Sign-extend 6-byte ip */ 519 if (ip & (uint64_t)0x800000000000ULL) 520 ip |= (uint64_t)0xffff000000000000ULL; 521 break; 522 case 4: 523 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 524 packet->payload; 525 break; 526 case 6: 527 ip = packet->payload; 528 break; 529 default: 530 return 0; 531 } 532 533 return ip; 534 } 535 536 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 537 { 538 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 539 decoder->have_last_ip = true; 540 } 541 542 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 543 { 544 intel_pt_set_last_ip(decoder); 545 decoder->ip = decoder->last_ip; 546 } 547 548 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 549 { 550 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 551 decoder->buf); 552 } 553 554 static int intel_pt_bug(struct intel_pt_decoder *decoder) 555 { 556 intel_pt_log("ERROR: Internal error\n"); 557 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 558 return -ENOSYS; 559 } 560 561 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 562 { 563 decoder->tx_flags = 0; 564 } 565 566 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 567 { 568 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 569 } 570 571 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 572 { 573 decoder->pip_payload = decoder->packet.payload; 574 } 575 576 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 577 { 578 decoder->next_nr = decoder->pip_payload & 1; 579 } 580 581 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 582 { 583 decoder->nr = decoder->pip_payload & 1; 584 decoder->next_nr = decoder->nr; 585 } 586 587 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 588 { 589 intel_pt_update_pip(decoder); 590 intel_pt_set_nr(decoder); 591 } 592 593 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 594 { 595 intel_pt_clear_tx_flags(decoder); 596 decoder->have_tma = false; 597 decoder->pkt_len = 1; 598 decoder->pkt_step = 1; 599 intel_pt_decoder_log_packet(decoder); 600 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 601 intel_pt_log("ERROR: Bad packet\n"); 602 decoder->pkt_state = INTEL_PT_STATE_ERR1; 603 } 604 return -EBADMSG; 605 } 606 607 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 608 { 609 decoder->sample_timestamp = decoder->timestamp; 610 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 611 decoder->state.cycles = decoder->tot_cyc_cnt; 612 } 613 614 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 615 { 616 decoder->ip = 0; 617 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 618 decoder->timestamp = 0; 619 decoder->have_tma = false; 620 } 621 622 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 623 { 624 struct intel_pt_buffer buffer = { .buf = 0, }; 625 int ret; 626 627 decoder->pkt_step = 0; 628 629 intel_pt_log("Getting more data\n"); 630 ret = decoder->get_trace(&buffer, decoder->data); 631 if (ret) 632 return ret; 633 decoder->buf = buffer.buf; 634 decoder->len = buffer.len; 635 if (!decoder->len) { 636 intel_pt_log("No more data\n"); 637 return -ENODATA; 638 } 639 decoder->buf_timestamp = buffer.ref_timestamp; 640 if (!buffer.consecutive || reposition) { 641 intel_pt_reposition(decoder); 642 decoder->ref_timestamp = buffer.ref_timestamp; 643 decoder->state.trace_nr = buffer.trace_nr; 644 decoder->vm_tm_corr_same_buf = false; 645 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 646 decoder->ref_timestamp); 647 return -ENOLINK; 648 } 649 650 return 0; 651 } 652 653 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 654 bool reposition) 655 { 656 if (!decoder->next_buf) 657 return intel_pt_get_data(decoder, reposition); 658 659 decoder->buf = decoder->next_buf; 660 decoder->len = decoder->next_len; 661 decoder->next_buf = 0; 662 decoder->next_len = 0; 663 return 0; 664 } 665 666 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 667 { 668 unsigned char *buf = decoder->temp_buf; 669 size_t old_len, len, n; 670 int ret; 671 672 old_len = decoder->len; 673 len = decoder->len; 674 memcpy(buf, decoder->buf, len); 675 676 ret = intel_pt_get_data(decoder, false); 677 if (ret) { 678 decoder->pos += old_len; 679 return ret < 0 ? ret : -EINVAL; 680 } 681 682 n = INTEL_PT_PKT_MAX_SZ - len; 683 if (n > decoder->len) 684 n = decoder->len; 685 memcpy(buf + len, decoder->buf, n); 686 len += n; 687 688 decoder->prev_pkt_ctx = decoder->pkt_ctx; 689 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 690 if (ret < (int)old_len) { 691 decoder->next_buf = decoder->buf; 692 decoder->next_len = decoder->len; 693 decoder->buf = buf; 694 decoder->len = old_len; 695 return intel_pt_bad_packet(decoder); 696 } 697 698 decoder->next_buf = decoder->buf + (ret - old_len); 699 decoder->next_len = decoder->len - (ret - old_len); 700 701 decoder->buf = buf; 702 decoder->len = ret; 703 704 return ret; 705 } 706 707 struct intel_pt_pkt_info { 708 struct intel_pt_decoder *decoder; 709 struct intel_pt_pkt packet; 710 uint64_t pos; 711 int pkt_len; 712 int last_packet_type; 713 void *data; 714 }; 715 716 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 717 718 /* Lookahead packets in current buffer */ 719 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 720 intel_pt_pkt_cb_t cb, void *data) 721 { 722 struct intel_pt_pkt_info pkt_info; 723 const unsigned char *buf = decoder->buf; 724 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 725 size_t len = decoder->len; 726 int ret; 727 728 pkt_info.decoder = decoder; 729 pkt_info.pos = decoder->pos; 730 pkt_info.pkt_len = decoder->pkt_step; 731 pkt_info.last_packet_type = decoder->last_packet_type; 732 pkt_info.data = data; 733 734 while (1) { 735 do { 736 pkt_info.pos += pkt_info.pkt_len; 737 buf += pkt_info.pkt_len; 738 len -= pkt_info.pkt_len; 739 740 if (!len) 741 return INTEL_PT_NEED_MORE_BYTES; 742 743 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 744 &pkt_ctx); 745 if (!ret) 746 return INTEL_PT_NEED_MORE_BYTES; 747 if (ret < 0) 748 return ret; 749 750 pkt_info.pkt_len = ret; 751 } while (pkt_info.packet.type == INTEL_PT_PAD); 752 753 ret = cb(&pkt_info); 754 if (ret) 755 return 0; 756 757 pkt_info.last_packet_type = pkt_info.packet.type; 758 } 759 } 760 761 struct intel_pt_calc_cyc_to_tsc_info { 762 uint64_t cycle_cnt; 763 unsigned int cbr; 764 uint32_t last_mtc; 765 uint64_t ctc_timestamp; 766 uint64_t ctc_delta; 767 uint64_t tsc_timestamp; 768 uint64_t timestamp; 769 bool have_tma; 770 bool fixup_last_mtc; 771 bool from_mtc; 772 double cbr_cyc_to_tsc; 773 }; 774 775 /* 776 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 777 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 778 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 779 * packet by copying the missing bits from the current MTC assuming the least 780 * difference between the two, and that the current MTC comes after last_mtc. 781 */ 782 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 783 uint32_t *last_mtc) 784 { 785 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 786 uint32_t mask = ~(first_missing_bit - 1); 787 788 *last_mtc |= mtc & mask; 789 if (*last_mtc >= mtc) { 790 *last_mtc -= first_missing_bit; 791 *last_mtc &= 0xff; 792 } 793 } 794 795 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 796 { 797 struct intel_pt_decoder *decoder = pkt_info->decoder; 798 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 799 uint64_t timestamp; 800 double cyc_to_tsc; 801 unsigned int cbr; 802 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 803 804 switch (pkt_info->packet.type) { 805 case INTEL_PT_TNT: 806 case INTEL_PT_TIP_PGE: 807 case INTEL_PT_TIP: 808 case INTEL_PT_FUP: 809 case INTEL_PT_PSB: 810 case INTEL_PT_PIP: 811 case INTEL_PT_MODE_EXEC: 812 case INTEL_PT_MODE_TSX: 813 case INTEL_PT_PSBEND: 814 case INTEL_PT_PAD: 815 case INTEL_PT_VMCS: 816 case INTEL_PT_MNT: 817 case INTEL_PT_PTWRITE: 818 case INTEL_PT_PTWRITE_IP: 819 case INTEL_PT_BBP: 820 case INTEL_PT_BIP: 821 case INTEL_PT_BEP: 822 case INTEL_PT_BEP_IP: 823 return 0; 824 825 case INTEL_PT_MTC: 826 if (!data->have_tma) 827 return 0; 828 829 mtc = pkt_info->packet.payload; 830 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 831 data->fixup_last_mtc = false; 832 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 833 &data->last_mtc); 834 } 835 if (mtc > data->last_mtc) 836 mtc_delta = mtc - data->last_mtc; 837 else 838 mtc_delta = mtc + 256 - data->last_mtc; 839 data->ctc_delta += mtc_delta << decoder->mtc_shift; 840 data->last_mtc = mtc; 841 842 if (decoder->tsc_ctc_mult) { 843 timestamp = data->ctc_timestamp + 844 data->ctc_delta * decoder->tsc_ctc_mult; 845 } else { 846 timestamp = data->ctc_timestamp + 847 multdiv(data->ctc_delta, 848 decoder->tsc_ctc_ratio_n, 849 decoder->tsc_ctc_ratio_d); 850 } 851 852 if (timestamp < data->timestamp) 853 return 1; 854 855 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 856 data->timestamp = timestamp; 857 return 0; 858 } 859 860 break; 861 862 case INTEL_PT_TSC: 863 /* 864 * For now, do not support using TSC packets - refer 865 * intel_pt_calc_cyc_to_tsc(). 866 */ 867 if (data->from_mtc) 868 return 1; 869 timestamp = pkt_info->packet.payload | 870 (data->timestamp & (0xffULL << 56)); 871 if (data->from_mtc && timestamp < data->timestamp && 872 data->timestamp - timestamp < decoder->tsc_slip) 873 return 1; 874 if (timestamp < data->timestamp) 875 timestamp += (1ULL << 56); 876 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 877 if (data->from_mtc) 878 return 1; 879 data->tsc_timestamp = timestamp; 880 data->timestamp = timestamp; 881 return 0; 882 } 883 break; 884 885 case INTEL_PT_TMA: 886 if (data->from_mtc) 887 return 1; 888 889 if (!decoder->tsc_ctc_ratio_d) 890 return 0; 891 892 ctc = pkt_info->packet.payload; 893 fc = pkt_info->packet.count; 894 ctc_rem = ctc & decoder->ctc_rem_mask; 895 896 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 897 898 data->ctc_timestamp = data->tsc_timestamp - fc; 899 if (decoder->tsc_ctc_mult) { 900 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 901 } else { 902 data->ctc_timestamp -= 903 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 904 decoder->tsc_ctc_ratio_d); 905 } 906 907 data->ctc_delta = 0; 908 data->have_tma = true; 909 data->fixup_last_mtc = true; 910 911 return 0; 912 913 case INTEL_PT_CYC: 914 data->cycle_cnt += pkt_info->packet.payload; 915 return 0; 916 917 case INTEL_PT_CBR: 918 cbr = pkt_info->packet.payload; 919 if (data->cbr && data->cbr != cbr) 920 return 1; 921 data->cbr = cbr; 922 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 923 return 0; 924 925 case INTEL_PT_TIP_PGD: 926 case INTEL_PT_TRACESTOP: 927 case INTEL_PT_EXSTOP: 928 case INTEL_PT_EXSTOP_IP: 929 case INTEL_PT_MWAIT: 930 case INTEL_PT_PWRE: 931 case INTEL_PT_PWRX: 932 case INTEL_PT_OVF: 933 case INTEL_PT_BAD: /* Does not happen */ 934 default: 935 return 1; 936 } 937 938 if (!data->cbr && decoder->cbr) { 939 data->cbr = decoder->cbr; 940 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 941 } 942 943 if (!data->cycle_cnt) 944 return 1; 945 946 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 947 948 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 949 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 950 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 951 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 952 return 1; 953 } 954 955 decoder->calc_cyc_to_tsc = cyc_to_tsc; 956 decoder->have_calc_cyc_to_tsc = true; 957 958 if (data->cbr) { 959 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 960 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 961 } else { 962 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 963 cyc_to_tsc, pkt_info->pos); 964 } 965 966 return 1; 967 } 968 969 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 970 bool from_mtc) 971 { 972 struct intel_pt_calc_cyc_to_tsc_info data = { 973 .cycle_cnt = 0, 974 .cbr = 0, 975 .last_mtc = decoder->last_mtc, 976 .ctc_timestamp = decoder->ctc_timestamp, 977 .ctc_delta = decoder->ctc_delta, 978 .tsc_timestamp = decoder->tsc_timestamp, 979 .timestamp = decoder->timestamp, 980 .have_tma = decoder->have_tma, 981 .fixup_last_mtc = decoder->fixup_last_mtc, 982 .from_mtc = from_mtc, 983 .cbr_cyc_to_tsc = 0, 984 }; 985 986 /* 987 * For now, do not support using TSC packets for at least the reasons: 988 * 1) timing might have stopped 989 * 2) TSC packets within PSB+ can slip against CYC packets 990 */ 991 if (!from_mtc) 992 return; 993 994 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 995 } 996 997 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 998 { 999 int ret; 1000 1001 decoder->last_packet_type = decoder->packet.type; 1002 1003 do { 1004 decoder->pos += decoder->pkt_step; 1005 decoder->buf += decoder->pkt_step; 1006 decoder->len -= decoder->pkt_step; 1007 1008 if (!decoder->len) { 1009 ret = intel_pt_get_next_data(decoder, false); 1010 if (ret) 1011 return ret; 1012 } 1013 1014 decoder->prev_pkt_ctx = decoder->pkt_ctx; 1015 ret = intel_pt_get_packet(decoder->buf, decoder->len, 1016 &decoder->packet, &decoder->pkt_ctx); 1017 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 1018 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 1019 ret = intel_pt_get_split_packet(decoder); 1020 if (ret < 0) 1021 return ret; 1022 } 1023 if (ret <= 0) 1024 return intel_pt_bad_packet(decoder); 1025 1026 decoder->pkt_len = ret; 1027 decoder->pkt_step = ret; 1028 intel_pt_decoder_log_packet(decoder); 1029 } while (decoder->packet.type == INTEL_PT_PAD); 1030 1031 return 0; 1032 } 1033 1034 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 1035 { 1036 uint64_t timestamp, masked_timestamp; 1037 1038 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1039 masked_timestamp = timestamp & decoder->period_mask; 1040 if (decoder->continuous_period) { 1041 if (masked_timestamp > decoder->last_masked_timestamp) 1042 return 1; 1043 } else { 1044 timestamp += 1; 1045 masked_timestamp = timestamp & decoder->period_mask; 1046 if (masked_timestamp > decoder->last_masked_timestamp) { 1047 decoder->last_masked_timestamp = masked_timestamp; 1048 decoder->continuous_period = true; 1049 } 1050 } 1051 1052 if (masked_timestamp < decoder->last_masked_timestamp) 1053 return decoder->period_ticks; 1054 1055 return decoder->period_ticks - (timestamp - masked_timestamp); 1056 } 1057 1058 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 1059 { 1060 switch (decoder->period_type) { 1061 case INTEL_PT_PERIOD_INSTRUCTIONS: 1062 return decoder->period - decoder->period_insn_cnt; 1063 case INTEL_PT_PERIOD_TICKS: 1064 return intel_pt_next_period(decoder); 1065 case INTEL_PT_PERIOD_NONE: 1066 case INTEL_PT_PERIOD_MTC: 1067 default: 1068 return 0; 1069 } 1070 } 1071 1072 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1073 { 1074 uint64_t timestamp, masked_timestamp; 1075 1076 switch (decoder->period_type) { 1077 case INTEL_PT_PERIOD_INSTRUCTIONS: 1078 decoder->period_insn_cnt = 0; 1079 break; 1080 case INTEL_PT_PERIOD_TICKS: 1081 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1082 masked_timestamp = timestamp & decoder->period_mask; 1083 if (masked_timestamp > decoder->last_masked_timestamp) 1084 decoder->last_masked_timestamp = masked_timestamp; 1085 else 1086 decoder->last_masked_timestamp += decoder->period_ticks; 1087 break; 1088 case INTEL_PT_PERIOD_NONE: 1089 case INTEL_PT_PERIOD_MTC: 1090 default: 1091 break; 1092 } 1093 1094 decoder->state.type |= INTEL_PT_INSTRUCTION; 1095 } 1096 1097 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1098 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1099 { 1100 uint64_t max_insn_cnt, insn_cnt = 0; 1101 int err; 1102 1103 if (!decoder->mtc_insn) 1104 decoder->mtc_insn = true; 1105 1106 max_insn_cnt = intel_pt_next_sample(decoder); 1107 1108 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1109 max_insn_cnt, decoder->data); 1110 1111 decoder->tot_insn_cnt += insn_cnt; 1112 decoder->timestamp_insn_cnt += insn_cnt; 1113 decoder->sample_insn_cnt += insn_cnt; 1114 decoder->period_insn_cnt += insn_cnt; 1115 1116 if (err) { 1117 decoder->no_progress = 0; 1118 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1119 intel_pt_log_at("ERROR: Failed to get instruction", 1120 decoder->ip); 1121 if (err == -ENOENT) 1122 return -ENOLINK; 1123 return -EILSEQ; 1124 } 1125 1126 if (ip && decoder->ip == ip) { 1127 err = -EAGAIN; 1128 goto out; 1129 } 1130 1131 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1132 intel_pt_sample_insn(decoder); 1133 1134 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1135 decoder->state.type = INTEL_PT_INSTRUCTION; 1136 decoder->state.from_ip = decoder->ip; 1137 decoder->state.to_ip = 0; 1138 decoder->ip += intel_pt_insn->length; 1139 err = INTEL_PT_RETURN; 1140 goto out; 1141 } 1142 1143 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1144 /* Zero-length calls are excluded */ 1145 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1146 intel_pt_insn->rel) { 1147 err = intel_pt_push(&decoder->stack, decoder->ip + 1148 intel_pt_insn->length); 1149 if (err) 1150 goto out; 1151 } 1152 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1153 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1154 } 1155 1156 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1157 int cnt = decoder->no_progress++; 1158 1159 decoder->state.from_ip = decoder->ip; 1160 decoder->ip += intel_pt_insn->length + 1161 intel_pt_insn->rel; 1162 decoder->state.to_ip = decoder->ip; 1163 err = INTEL_PT_RETURN; 1164 1165 /* 1166 * Check for being stuck in a loop. This can happen if a 1167 * decoder error results in the decoder erroneously setting the 1168 * ip to an address that is itself in an infinite loop that 1169 * consumes no packets. When that happens, there must be an 1170 * unconditional branch. 1171 */ 1172 if (cnt) { 1173 if (cnt == 1) { 1174 decoder->stuck_ip = decoder->state.to_ip; 1175 decoder->stuck_ip_prd = 1; 1176 decoder->stuck_ip_cnt = 1; 1177 } else if (cnt > decoder->max_loops || 1178 decoder->state.to_ip == decoder->stuck_ip) { 1179 intel_pt_log_at("ERROR: Never-ending loop", 1180 decoder->state.to_ip); 1181 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1182 err = -ELOOP; 1183 goto out; 1184 } else if (!--decoder->stuck_ip_cnt) { 1185 decoder->stuck_ip_prd += 1; 1186 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1187 decoder->stuck_ip = decoder->state.to_ip; 1188 } 1189 } 1190 goto out_no_progress; 1191 } 1192 out: 1193 decoder->no_progress = 0; 1194 out_no_progress: 1195 decoder->state.insn_op = intel_pt_insn->op; 1196 decoder->state.insn_len = intel_pt_insn->length; 1197 memcpy(decoder->state.insn, intel_pt_insn->buf, 1198 INTEL_PT_INSN_BUF_SZ); 1199 1200 if (decoder->tx_flags & INTEL_PT_IN_TX) 1201 decoder->state.flags |= INTEL_PT_IN_TX; 1202 1203 return err; 1204 } 1205 1206 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1207 { 1208 enum intel_pt_sample_type type = decoder->state.type; 1209 bool ret = false; 1210 1211 decoder->state.type &= ~INTEL_PT_BRANCH; 1212 1213 if (decoder->set_fup_tx_flags) { 1214 decoder->set_fup_tx_flags = false; 1215 decoder->tx_flags = decoder->fup_tx_flags; 1216 decoder->state.type |= INTEL_PT_TRANSACTION; 1217 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) 1218 decoder->state.type |= INTEL_PT_BRANCH; 1219 decoder->state.flags = decoder->fup_tx_flags; 1220 ret = true; 1221 } 1222 if (decoder->set_fup_ptw) { 1223 decoder->set_fup_ptw = false; 1224 decoder->state.type |= INTEL_PT_PTW; 1225 decoder->state.flags |= INTEL_PT_FUP_IP; 1226 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1227 ret = true; 1228 } 1229 if (decoder->set_fup_mwait) { 1230 decoder->set_fup_mwait = false; 1231 decoder->state.type |= INTEL_PT_MWAIT_OP; 1232 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1233 ret = true; 1234 } 1235 if (decoder->set_fup_pwre) { 1236 decoder->set_fup_pwre = false; 1237 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1238 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1239 ret = true; 1240 } 1241 if (decoder->set_fup_exstop) { 1242 decoder->set_fup_exstop = false; 1243 decoder->state.type |= INTEL_PT_EX_STOP; 1244 decoder->state.flags |= INTEL_PT_FUP_IP; 1245 ret = true; 1246 } 1247 if (decoder->set_fup_bep) { 1248 decoder->set_fup_bep = false; 1249 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1250 ret = true; 1251 } 1252 if (decoder->overflow) { 1253 decoder->overflow = false; 1254 if (!ret && !decoder->pge) { 1255 if (decoder->hop) { 1256 decoder->state.type = 0; 1257 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 1258 } 1259 decoder->pge = true; 1260 decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; 1261 decoder->state.from_ip = 0; 1262 decoder->state.to_ip = decoder->ip; 1263 return true; 1264 } 1265 } 1266 if (ret) { 1267 decoder->state.from_ip = decoder->ip; 1268 decoder->state.to_ip = 0; 1269 } else { 1270 decoder->state.type = type; 1271 } 1272 return ret; 1273 } 1274 1275 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1276 struct intel_pt_insn *intel_pt_insn, 1277 uint64_t ip, int err) 1278 { 1279 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1280 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1281 ip == decoder->ip + intel_pt_insn->length; 1282 } 1283 1284 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1285 { 1286 struct intel_pt_insn intel_pt_insn; 1287 uint64_t ip; 1288 int err; 1289 1290 ip = decoder->last_ip; 1291 1292 while (1) { 1293 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1294 if (err == INTEL_PT_RETURN) 1295 return 0; 1296 if (err == -EAGAIN || 1297 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1298 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; 1299 1300 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1301 if (intel_pt_fup_event(decoder) && no_tip) 1302 return 0; 1303 return -EAGAIN; 1304 } 1305 decoder->set_fup_tx_flags = false; 1306 if (err) 1307 return err; 1308 1309 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1310 intel_pt_log_at("ERROR: Unexpected indirect branch", 1311 decoder->ip); 1312 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1313 return -ENOENT; 1314 } 1315 1316 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1317 intel_pt_log_at("ERROR: Unexpected conditional branch", 1318 decoder->ip); 1319 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1320 return -ENOENT; 1321 } 1322 1323 intel_pt_bug(decoder); 1324 } 1325 } 1326 1327 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1328 { 1329 struct intel_pt_insn intel_pt_insn; 1330 int err; 1331 1332 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1333 if (err == INTEL_PT_RETURN && 1334 decoder->pgd_ip && 1335 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1336 (decoder->state.type & INTEL_PT_BRANCH) && 1337 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1338 /* Unconditional branch leaving filter region */ 1339 decoder->no_progress = 0; 1340 decoder->pge = false; 1341 decoder->continuous_period = false; 1342 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1343 decoder->state.type |= INTEL_PT_TRACE_END; 1344 intel_pt_update_nr(decoder); 1345 return 0; 1346 } 1347 if (err == INTEL_PT_RETURN) 1348 return 0; 1349 if (err) 1350 return err; 1351 1352 intel_pt_update_nr(decoder); 1353 1354 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1355 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1356 decoder->pge = false; 1357 decoder->continuous_period = false; 1358 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1359 decoder->state.from_ip = decoder->ip; 1360 if (decoder->packet.count == 0) { 1361 decoder->state.to_ip = 0; 1362 } else { 1363 decoder->state.to_ip = decoder->last_ip; 1364 decoder->ip = decoder->last_ip; 1365 } 1366 decoder->state.type |= INTEL_PT_TRACE_END; 1367 } else { 1368 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1369 decoder->state.from_ip = decoder->ip; 1370 if (decoder->packet.count == 0) { 1371 decoder->state.to_ip = 0; 1372 } else { 1373 decoder->state.to_ip = decoder->last_ip; 1374 decoder->ip = decoder->last_ip; 1375 } 1376 } 1377 return 0; 1378 } 1379 1380 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1381 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1382 intel_pt_insn.rel; 1383 1384 if (decoder->pgd_ip && 1385 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1386 decoder->pgd_ip(to_ip, decoder->data)) { 1387 /* Conditional branch leaving filter region */ 1388 decoder->pge = false; 1389 decoder->continuous_period = false; 1390 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1391 decoder->ip = to_ip; 1392 decoder->state.from_ip = decoder->ip; 1393 decoder->state.to_ip = to_ip; 1394 decoder->state.type |= INTEL_PT_TRACE_END; 1395 return 0; 1396 } 1397 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1398 decoder->ip); 1399 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1400 return -ENOENT; 1401 } 1402 1403 return intel_pt_bug(decoder); 1404 } 1405 1406 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1407 { 1408 struct intel_pt_insn intel_pt_insn; 1409 int err; 1410 1411 while (1) { 1412 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1413 if (err == INTEL_PT_RETURN) 1414 return 0; 1415 if (err) 1416 return err; 1417 1418 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1419 if (!decoder->return_compression) { 1420 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1421 decoder->ip); 1422 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1423 return -ENOENT; 1424 } 1425 if (!decoder->ret_addr) { 1426 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1427 decoder->ip); 1428 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1429 return -ENOENT; 1430 } 1431 if (!(decoder->tnt.payload & BIT63)) { 1432 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1433 decoder->ip); 1434 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1435 return -ENOENT; 1436 } 1437 decoder->tnt.count -= 1; 1438 if (decoder->tnt.count) 1439 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1440 else 1441 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1442 decoder->tnt.payload <<= 1; 1443 decoder->state.from_ip = decoder->ip; 1444 decoder->ip = decoder->ret_addr; 1445 decoder->state.to_ip = decoder->ip; 1446 return 0; 1447 } 1448 1449 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1450 /* Handle deferred TIPs */ 1451 err = intel_pt_get_next_packet(decoder); 1452 if (err) 1453 return err; 1454 if (decoder->packet.type != INTEL_PT_TIP || 1455 decoder->packet.count == 0) { 1456 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1457 decoder->ip); 1458 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1459 decoder->pkt_step = 0; 1460 return -ENOENT; 1461 } 1462 intel_pt_set_last_ip(decoder); 1463 decoder->state.from_ip = decoder->ip; 1464 decoder->state.to_ip = decoder->last_ip; 1465 decoder->ip = decoder->last_ip; 1466 intel_pt_update_nr(decoder); 1467 return 0; 1468 } 1469 1470 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1471 decoder->tnt.count -= 1; 1472 if (decoder->tnt.count) 1473 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1474 else 1475 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1476 if (decoder->tnt.payload & BIT63) { 1477 decoder->tnt.payload <<= 1; 1478 decoder->state.from_ip = decoder->ip; 1479 decoder->ip += intel_pt_insn.length + 1480 intel_pt_insn.rel; 1481 decoder->state.to_ip = decoder->ip; 1482 return 0; 1483 } 1484 /* Instruction sample for a non-taken branch */ 1485 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1486 decoder->tnt.payload <<= 1; 1487 decoder->state.type = INTEL_PT_INSTRUCTION; 1488 decoder->state.from_ip = decoder->ip; 1489 decoder->state.to_ip = 0; 1490 decoder->ip += intel_pt_insn.length; 1491 return 0; 1492 } 1493 decoder->sample_cyc = false; 1494 decoder->ip += intel_pt_insn.length; 1495 if (!decoder->tnt.count) { 1496 intel_pt_update_sample_time(decoder); 1497 return -EAGAIN; 1498 } 1499 decoder->tnt.payload <<= 1; 1500 continue; 1501 } 1502 1503 return intel_pt_bug(decoder); 1504 } 1505 } 1506 1507 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1508 { 1509 unsigned int fup_tx_flags; 1510 int err; 1511 1512 fup_tx_flags = decoder->packet.payload & 1513 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1514 err = intel_pt_get_next_packet(decoder); 1515 if (err) 1516 return err; 1517 if (decoder->packet.type == INTEL_PT_FUP) { 1518 decoder->fup_tx_flags = fup_tx_flags; 1519 decoder->set_fup_tx_flags = true; 1520 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1521 *no_tip = true; 1522 } else { 1523 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1524 decoder->pos); 1525 intel_pt_update_in_tx(decoder); 1526 } 1527 return 0; 1528 } 1529 1530 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1531 { 1532 timestamp |= (ref_timestamp & (0xffULL << 56)); 1533 1534 if (timestamp < ref_timestamp) { 1535 if (ref_timestamp - timestamp > (1ULL << 55)) 1536 timestamp += (1ULL << 56); 1537 } else { 1538 if (timestamp - ref_timestamp > (1ULL << 55)) 1539 timestamp -= (1ULL << 56); 1540 } 1541 1542 return timestamp; 1543 } 1544 1545 /* For use only when decoder->vm_time_correlation is true */ 1546 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, 1547 uint64_t timestamp) 1548 { 1549 uint64_t max_timestamp = decoder->buf_timestamp; 1550 1551 if (!max_timestamp) { 1552 max_timestamp = decoder->last_reliable_timestamp + 1553 0x400000000ULL; 1554 } 1555 return timestamp >= decoder->last_reliable_timestamp && 1556 timestamp < decoder->buf_timestamp; 1557 } 1558 1559 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1560 { 1561 uint64_t timestamp; 1562 bool bad = false; 1563 1564 decoder->have_tma = false; 1565 1566 if (decoder->ref_timestamp) { 1567 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1568 decoder->ref_timestamp); 1569 decoder->tsc_timestamp = timestamp; 1570 decoder->timestamp = timestamp; 1571 decoder->ref_timestamp = 0; 1572 decoder->timestamp_insn_cnt = 0; 1573 } else if (decoder->timestamp) { 1574 timestamp = decoder->packet.payload | 1575 (decoder->timestamp & (0xffULL << 56)); 1576 decoder->tsc_timestamp = timestamp; 1577 if (timestamp < decoder->timestamp && 1578 decoder->timestamp - timestamp < decoder->tsc_slip) { 1579 intel_pt_log_to("Suppressing backwards timestamp", 1580 timestamp); 1581 timestamp = decoder->timestamp; 1582 } 1583 if (timestamp < decoder->timestamp) { 1584 if (!decoder->buf_timestamp || 1585 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { 1586 intel_pt_log_to("Wraparound timestamp", timestamp); 1587 timestamp += (1ULL << 56); 1588 decoder->tsc_timestamp = timestamp; 1589 } else { 1590 intel_pt_log_to("Suppressing bad timestamp", timestamp); 1591 timestamp = decoder->timestamp; 1592 bad = true; 1593 } 1594 } 1595 if (decoder->vm_time_correlation && 1596 (bad || !intel_pt_time_in_range(decoder, timestamp)) && 1597 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 1598 p_log("Timestamp out of range"); 1599 decoder->timestamp = timestamp; 1600 decoder->timestamp_insn_cnt = 0; 1601 } 1602 1603 if (decoder->last_packet_type == INTEL_PT_CYC) { 1604 decoder->cyc_ref_timestamp = decoder->timestamp; 1605 decoder->cycle_cnt = 0; 1606 decoder->have_calc_cyc_to_tsc = false; 1607 intel_pt_calc_cyc_to_tsc(decoder, false); 1608 } 1609 1610 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1611 } 1612 1613 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1614 { 1615 intel_pt_log("ERROR: Buffer overflow\n"); 1616 intel_pt_clear_tx_flags(decoder); 1617 intel_pt_set_nr(decoder); 1618 decoder->timestamp_insn_cnt = 0; 1619 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1620 decoder->state.from_ip = decoder->ip; 1621 decoder->ip = 0; 1622 decoder->pge = false; 1623 decoder->set_fup_tx_flags = false; 1624 decoder->set_fup_ptw = false; 1625 decoder->set_fup_mwait = false; 1626 decoder->set_fup_pwre = false; 1627 decoder->set_fup_exstop = false; 1628 decoder->set_fup_bep = false; 1629 decoder->overflow = true; 1630 return -EOVERFLOW; 1631 } 1632 1633 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1634 { 1635 if (decoder->have_cyc) 1636 return; 1637 1638 decoder->cyc_cnt_timestamp = decoder->timestamp; 1639 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1640 } 1641 1642 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1643 { 1644 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1645 1646 if (decoder->pge) 1647 intel_pt_mtc_cyc_cnt_pge(decoder); 1648 } 1649 1650 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1651 { 1652 uint64_t tot_cyc_cnt, tsc_delta; 1653 1654 if (decoder->have_cyc) 1655 return; 1656 1657 decoder->sample_cyc = true; 1658 1659 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1660 return; 1661 1662 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1663 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1664 1665 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1666 decoder->tot_cyc_cnt = tot_cyc_cnt; 1667 } 1668 1669 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1670 { 1671 uint32_t ctc = decoder->packet.payload; 1672 uint32_t fc = decoder->packet.count; 1673 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1674 1675 if (!decoder->tsc_ctc_ratio_d) 1676 return; 1677 1678 if (decoder->pge && !decoder->in_psb) 1679 intel_pt_mtc_cyc_cnt_pge(decoder); 1680 else 1681 intel_pt_mtc_cyc_cnt_upd(decoder); 1682 1683 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1684 decoder->last_ctc = ctc - ctc_rem; 1685 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1686 if (decoder->tsc_ctc_mult) { 1687 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1688 } else { 1689 decoder->ctc_timestamp -= multdiv(ctc_rem, 1690 decoder->tsc_ctc_ratio_n, 1691 decoder->tsc_ctc_ratio_d); 1692 } 1693 decoder->ctc_delta = 0; 1694 decoder->have_tma = true; 1695 decoder->fixup_last_mtc = true; 1696 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1697 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1698 } 1699 1700 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1701 { 1702 uint64_t timestamp; 1703 uint32_t mtc, mtc_delta; 1704 1705 if (!decoder->have_tma) 1706 return; 1707 1708 mtc = decoder->packet.payload; 1709 1710 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1711 decoder->fixup_last_mtc = false; 1712 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1713 &decoder->last_mtc); 1714 } 1715 1716 if (mtc > decoder->last_mtc) 1717 mtc_delta = mtc - decoder->last_mtc; 1718 else 1719 mtc_delta = mtc + 256 - decoder->last_mtc; 1720 1721 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1722 1723 if (decoder->tsc_ctc_mult) { 1724 timestamp = decoder->ctc_timestamp + 1725 decoder->ctc_delta * decoder->tsc_ctc_mult; 1726 } else { 1727 timestamp = decoder->ctc_timestamp + 1728 multdiv(decoder->ctc_delta, 1729 decoder->tsc_ctc_ratio_n, 1730 decoder->tsc_ctc_ratio_d); 1731 } 1732 1733 if (timestamp < decoder->timestamp) 1734 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1735 timestamp, decoder->timestamp); 1736 else 1737 decoder->timestamp = timestamp; 1738 1739 intel_pt_mtc_cyc_cnt_upd(decoder); 1740 1741 decoder->timestamp_insn_cnt = 0; 1742 decoder->last_mtc = mtc; 1743 1744 if (decoder->last_packet_type == INTEL_PT_CYC) { 1745 decoder->cyc_ref_timestamp = decoder->timestamp; 1746 decoder->cycle_cnt = 0; 1747 decoder->have_calc_cyc_to_tsc = false; 1748 intel_pt_calc_cyc_to_tsc(decoder, true); 1749 } 1750 1751 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1752 } 1753 1754 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1755 { 1756 unsigned int cbr = decoder->packet.payload & 0xff; 1757 1758 decoder->cbr_payload = decoder->packet.payload; 1759 1760 if (decoder->cbr == cbr) 1761 return; 1762 1763 decoder->cbr = cbr; 1764 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1765 1766 intel_pt_mtc_cyc_cnt_cbr(decoder); 1767 } 1768 1769 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1770 { 1771 uint64_t timestamp = decoder->cyc_ref_timestamp; 1772 1773 decoder->have_cyc = true; 1774 1775 decoder->cycle_cnt += decoder->packet.payload; 1776 if (decoder->pge) 1777 decoder->tot_cyc_cnt += decoder->packet.payload; 1778 decoder->sample_cyc = true; 1779 1780 if (!decoder->cyc_ref_timestamp) 1781 return; 1782 1783 if (decoder->have_calc_cyc_to_tsc) 1784 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1785 else if (decoder->cbr) 1786 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1787 else 1788 return; 1789 1790 if (timestamp < decoder->timestamp) 1791 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1792 timestamp, decoder->timestamp); 1793 else 1794 decoder->timestamp = timestamp; 1795 1796 decoder->timestamp_insn_cnt = 0; 1797 1798 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1799 } 1800 1801 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1802 { 1803 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1804 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1805 decoder->state.items.is_32_bit = false; 1806 } 1807 decoder->blk_type = decoder->packet.payload; 1808 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1809 if (decoder->blk_type == INTEL_PT_GP_REGS) 1810 decoder->state.items.is_32_bit = decoder->packet.count; 1811 if (decoder->blk_type_pos < 0) { 1812 intel_pt_log("WARNING: Unknown block type %u\n", 1813 decoder->blk_type); 1814 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1815 intel_pt_log("WARNING: Duplicate block type %u\n", 1816 decoder->blk_type); 1817 } 1818 } 1819 1820 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1821 { 1822 uint32_t id = decoder->packet.count; 1823 uint32_t bit = 1 << id; 1824 int pos = decoder->blk_type_pos; 1825 1826 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1827 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1828 id, decoder->blk_type); 1829 return; 1830 } 1831 1832 if (decoder->state.items.mask[pos] & bit) { 1833 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1834 id, decoder->blk_type); 1835 } 1836 1837 decoder->state.items.mask[pos] |= bit; 1838 decoder->state.items.val[pos][id] = decoder->packet.payload; 1839 } 1840 1841 /* Walk PSB+ packets when already in sync. */ 1842 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1843 { 1844 int err; 1845 1846 decoder->in_psb = true; 1847 1848 while (1) { 1849 err = intel_pt_get_next_packet(decoder); 1850 if (err) 1851 goto out; 1852 1853 switch (decoder->packet.type) { 1854 case INTEL_PT_PSBEND: 1855 err = 0; 1856 goto out; 1857 1858 case INTEL_PT_TIP_PGD: 1859 case INTEL_PT_TIP_PGE: 1860 case INTEL_PT_TIP: 1861 case INTEL_PT_TNT: 1862 case INTEL_PT_TRACESTOP: 1863 case INTEL_PT_BAD: 1864 case INTEL_PT_PSB: 1865 case INTEL_PT_PTWRITE: 1866 case INTEL_PT_PTWRITE_IP: 1867 case INTEL_PT_EXSTOP: 1868 case INTEL_PT_EXSTOP_IP: 1869 case INTEL_PT_MWAIT: 1870 case INTEL_PT_PWRE: 1871 case INTEL_PT_PWRX: 1872 case INTEL_PT_BBP: 1873 case INTEL_PT_BIP: 1874 case INTEL_PT_BEP: 1875 case INTEL_PT_BEP_IP: 1876 decoder->have_tma = false; 1877 intel_pt_log("ERROR: Unexpected packet\n"); 1878 err = -EAGAIN; 1879 goto out; 1880 1881 case INTEL_PT_OVF: 1882 err = intel_pt_overflow(decoder); 1883 goto out; 1884 1885 case INTEL_PT_TSC: 1886 intel_pt_calc_tsc_timestamp(decoder); 1887 break; 1888 1889 case INTEL_PT_TMA: 1890 intel_pt_calc_tma(decoder); 1891 break; 1892 1893 case INTEL_PT_CBR: 1894 intel_pt_calc_cbr(decoder); 1895 break; 1896 1897 case INTEL_PT_MODE_EXEC: 1898 decoder->exec_mode = decoder->packet.payload; 1899 break; 1900 1901 case INTEL_PT_PIP: 1902 intel_pt_set_pip(decoder); 1903 break; 1904 1905 case INTEL_PT_FUP: 1906 decoder->pge = true; 1907 if (decoder->packet.count) { 1908 intel_pt_set_last_ip(decoder); 1909 decoder->psb_ip = decoder->last_ip; 1910 } 1911 break; 1912 1913 case INTEL_PT_MODE_TSX: 1914 intel_pt_update_in_tx(decoder); 1915 break; 1916 1917 case INTEL_PT_MTC: 1918 intel_pt_calc_mtc_timestamp(decoder); 1919 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1920 decoder->state.type |= INTEL_PT_INSTRUCTION; 1921 break; 1922 1923 case INTEL_PT_CYC: 1924 intel_pt_calc_cyc_timestamp(decoder); 1925 break; 1926 1927 case INTEL_PT_VMCS: 1928 case INTEL_PT_MNT: 1929 case INTEL_PT_PAD: 1930 default: 1931 break; 1932 } 1933 } 1934 out: 1935 decoder->in_psb = false; 1936 1937 return err; 1938 } 1939 1940 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1941 { 1942 int err; 1943 1944 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1945 decoder->tx_flags = 0; 1946 decoder->state.flags &= ~INTEL_PT_IN_TX; 1947 decoder->state.flags |= INTEL_PT_ABORT_TX; 1948 } else { 1949 decoder->state.flags |= INTEL_PT_ASYNC; 1950 } 1951 1952 while (1) { 1953 err = intel_pt_get_next_packet(decoder); 1954 if (err) 1955 return err; 1956 1957 switch (decoder->packet.type) { 1958 case INTEL_PT_TNT: 1959 case INTEL_PT_FUP: 1960 case INTEL_PT_TRACESTOP: 1961 case INTEL_PT_PSB: 1962 case INTEL_PT_TSC: 1963 case INTEL_PT_TMA: 1964 case INTEL_PT_MODE_TSX: 1965 case INTEL_PT_BAD: 1966 case INTEL_PT_PSBEND: 1967 case INTEL_PT_PTWRITE: 1968 case INTEL_PT_PTWRITE_IP: 1969 case INTEL_PT_EXSTOP: 1970 case INTEL_PT_EXSTOP_IP: 1971 case INTEL_PT_MWAIT: 1972 case INTEL_PT_PWRE: 1973 case INTEL_PT_PWRX: 1974 case INTEL_PT_BBP: 1975 case INTEL_PT_BIP: 1976 case INTEL_PT_BEP: 1977 case INTEL_PT_BEP_IP: 1978 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1979 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1980 decoder->pkt_step = 0; 1981 return -ENOENT; 1982 1983 case INTEL_PT_CBR: 1984 intel_pt_calc_cbr(decoder); 1985 break; 1986 1987 case INTEL_PT_OVF: 1988 return intel_pt_overflow(decoder); 1989 1990 case INTEL_PT_TIP_PGD: 1991 decoder->state.from_ip = decoder->ip; 1992 if (decoder->packet.count == 0) { 1993 decoder->state.to_ip = 0; 1994 } else { 1995 intel_pt_set_ip(decoder); 1996 decoder->state.to_ip = decoder->ip; 1997 } 1998 decoder->pge = false; 1999 decoder->continuous_period = false; 2000 decoder->state.type |= INTEL_PT_TRACE_END; 2001 intel_pt_update_nr(decoder); 2002 return 0; 2003 2004 case INTEL_PT_TIP_PGE: 2005 decoder->pge = true; 2006 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 2007 decoder->ip); 2008 decoder->state.from_ip = 0; 2009 if (decoder->packet.count == 0) { 2010 decoder->state.to_ip = 0; 2011 } else { 2012 intel_pt_set_ip(decoder); 2013 decoder->state.to_ip = decoder->ip; 2014 } 2015 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2016 intel_pt_mtc_cyc_cnt_pge(decoder); 2017 intel_pt_set_nr(decoder); 2018 return 0; 2019 2020 case INTEL_PT_TIP: 2021 decoder->state.from_ip = decoder->ip; 2022 if (decoder->packet.count == 0) { 2023 decoder->state.to_ip = 0; 2024 } else { 2025 intel_pt_set_ip(decoder); 2026 decoder->state.to_ip = decoder->ip; 2027 } 2028 intel_pt_update_nr(decoder); 2029 return 0; 2030 2031 case INTEL_PT_PIP: 2032 intel_pt_update_pip(decoder); 2033 break; 2034 2035 case INTEL_PT_MTC: 2036 intel_pt_calc_mtc_timestamp(decoder); 2037 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2038 decoder->state.type |= INTEL_PT_INSTRUCTION; 2039 break; 2040 2041 case INTEL_PT_CYC: 2042 intel_pt_calc_cyc_timestamp(decoder); 2043 break; 2044 2045 case INTEL_PT_MODE_EXEC: 2046 decoder->exec_mode = decoder->packet.payload; 2047 break; 2048 2049 case INTEL_PT_VMCS: 2050 case INTEL_PT_MNT: 2051 case INTEL_PT_PAD: 2052 break; 2053 2054 default: 2055 return intel_pt_bug(decoder); 2056 } 2057 } 2058 } 2059 2060 static int intel_pt_resample(struct intel_pt_decoder *decoder) 2061 { 2062 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2063 decoder->state.type = INTEL_PT_INSTRUCTION; 2064 decoder->state.from_ip = decoder->ip; 2065 decoder->state.to_ip = 0; 2066 return 0; 2067 } 2068 2069 struct intel_pt_vm_tsc_info { 2070 struct intel_pt_pkt pip_packet; 2071 struct intel_pt_pkt vmcs_packet; 2072 struct intel_pt_pkt tma_packet; 2073 bool tsc, pip, vmcs, tma, psbend; 2074 uint64_t ctc_delta; 2075 uint64_t last_ctc; 2076 int max_lookahead; 2077 }; 2078 2079 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */ 2080 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2081 { 2082 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2083 2084 switch (pkt_info->packet.type) { 2085 case INTEL_PT_PAD: 2086 case INTEL_PT_MNT: 2087 case INTEL_PT_MODE_EXEC: 2088 case INTEL_PT_MODE_TSX: 2089 case INTEL_PT_MTC: 2090 case INTEL_PT_FUP: 2091 case INTEL_PT_CYC: 2092 case INTEL_PT_CBR: 2093 break; 2094 2095 case INTEL_PT_TSC: 2096 data->tsc = true; 2097 break; 2098 2099 case INTEL_PT_TMA: 2100 data->tma_packet = pkt_info->packet; 2101 data->tma = true; 2102 break; 2103 2104 case INTEL_PT_PIP: 2105 data->pip_packet = pkt_info->packet; 2106 data->pip = true; 2107 break; 2108 2109 case INTEL_PT_VMCS: 2110 data->vmcs_packet = pkt_info->packet; 2111 data->vmcs = true; 2112 break; 2113 2114 case INTEL_PT_PSBEND: 2115 data->psbend = true; 2116 return 1; 2117 2118 case INTEL_PT_TIP_PGE: 2119 case INTEL_PT_PTWRITE: 2120 case INTEL_PT_PTWRITE_IP: 2121 case INTEL_PT_EXSTOP: 2122 case INTEL_PT_EXSTOP_IP: 2123 case INTEL_PT_MWAIT: 2124 case INTEL_PT_PWRE: 2125 case INTEL_PT_PWRX: 2126 case INTEL_PT_BBP: 2127 case INTEL_PT_BIP: 2128 case INTEL_PT_BEP: 2129 case INTEL_PT_BEP_IP: 2130 case INTEL_PT_OVF: 2131 case INTEL_PT_BAD: 2132 case INTEL_PT_TNT: 2133 case INTEL_PT_TIP_PGD: 2134 case INTEL_PT_TIP: 2135 case INTEL_PT_PSB: 2136 case INTEL_PT_TRACESTOP: 2137 default: 2138 return 1; 2139 } 2140 2141 return 0; 2142 } 2143 2144 struct intel_pt_ovf_fup_info { 2145 int max_lookahead; 2146 bool found; 2147 }; 2148 2149 /* Lookahead to detect a FUP packet after OVF */ 2150 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2151 { 2152 struct intel_pt_ovf_fup_info *data = pkt_info->data; 2153 2154 if (pkt_info->packet.type == INTEL_PT_CYC || 2155 pkt_info->packet.type == INTEL_PT_MTC || 2156 pkt_info->packet.type == INTEL_PT_TSC) 2157 return !--(data->max_lookahead); 2158 data->found = pkt_info->packet.type == INTEL_PT_FUP; 2159 return 1; 2160 } 2161 2162 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder) 2163 { 2164 struct intel_pt_ovf_fup_info data = { 2165 .max_lookahead = 16, 2166 .found = false, 2167 }; 2168 2169 intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); 2170 return data.found; 2171 } 2172 2173 /* Lookahead and get the TMA packet after TSC */ 2174 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2175 { 2176 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2177 2178 if (pkt_info->packet.type == INTEL_PT_CYC || 2179 pkt_info->packet.type == INTEL_PT_MTC) 2180 return !--(data->max_lookahead); 2181 2182 if (pkt_info->packet.type == INTEL_PT_TMA) { 2183 data->tma_packet = pkt_info->packet; 2184 data->tma = true; 2185 } 2186 return 1; 2187 } 2188 2189 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc) 2190 { 2191 if (decoder->tsc_ctc_mult) 2192 return ctc * decoder->tsc_ctc_mult; 2193 else 2194 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); 2195 } 2196 2197 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, 2198 uint32_t ctc, 2199 uint32_t fc, 2200 uint64_t last_ctc_timestamp, 2201 uint64_t ctc_delta, 2202 uint32_t last_ctc) 2203 { 2204 /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ 2205 uint64_t last_mtc_ctc = last_ctc + ctc_delta; 2206 /* 2207 * Number of CTC ticks from there until current TMA packet. We would 2208 * expect last_mtc_ctc to be before ctc, but the TSC packet can slip 2209 * past an MTC, so a sign-extended value is used. 2210 */ 2211 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); 2212 /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ 2213 uint64_t new_ctc_delta = ctc_delta + delta; 2214 uint64_t expected_tsc; 2215 2216 /* 2217 * Convert CTC ticks to TSC ticks, add the starting point 2218 * (last_ctc_timestamp) and the fast counter from the TMA packet. 2219 */ 2220 expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc; 2221 2222 if (intel_pt_enable_logging) { 2223 intel_pt_log_x64(last_mtc_ctc); 2224 intel_pt_log_x32(last_ctc); 2225 intel_pt_log_x64(ctc_delta); 2226 intel_pt_log_x64(delta); 2227 intel_pt_log_x32(ctc); 2228 intel_pt_log_x64(new_ctc_delta); 2229 intel_pt_log_x64(last_ctc_timestamp); 2230 intel_pt_log_x32(fc); 2231 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); 2232 intel_pt_log_x64(expected_tsc); 2233 } 2234 2235 return expected_tsc; 2236 } 2237 2238 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, 2239 struct intel_pt_vm_tsc_info *data) 2240 { 2241 uint32_t ctc = data->tma_packet.payload; 2242 uint32_t fc = data->tma_packet.count; 2243 2244 return intel_pt_calc_expected_tsc(decoder, ctc, fc, 2245 decoder->ctc_timestamp, 2246 data->ctc_delta, data->last_ctc); 2247 } 2248 2249 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, 2250 struct intel_pt_vmcs_info *vmcs_info) 2251 { 2252 uint64_t payload = decoder->packet.payload; 2253 2254 /* VMX adds the TSC Offset, so subtract to get host TSC */ 2255 decoder->packet.payload -= vmcs_info->tsc_offset; 2256 /* TSC packet has only 7 bytes */ 2257 decoder->packet.payload &= SEVEN_BYTES; 2258 2259 /* 2260 * The buffer is mmapped from the data file, so this also updates the 2261 * data file. 2262 */ 2263 if (!decoder->vm_tm_corr_dry_run) 2264 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); 2265 2266 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2267 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2268 payload, decoder->packet.payload, vmcs_info->vmcs, 2269 vmcs_info->tsc_offset); 2270 } 2271 2272 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, 2273 uint64_t tsc_offset) 2274 { 2275 struct intel_pt_vmcs_info vmcs_info = { 2276 .vmcs = NO_VMCS, 2277 .tsc_offset = tsc_offset 2278 }; 2279 2280 intel_pt_translate_vm_tsc(decoder, &vmcs_info); 2281 } 2282 2283 static inline bool in_vm(uint64_t pip_payload) 2284 { 2285 return pip_payload & 1; 2286 } 2287 2288 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet) 2289 { 2290 return pip_packet->payload & 1; 2291 } 2292 2293 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info) 2294 { 2295 p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, 2296 vmcs_info->vmcs, vmcs_info->tsc_offset); 2297 } 2298 2299 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, 2300 struct intel_pt_vm_tsc_info *data) 2301 { 2302 memset(data, 0, sizeof(*data)); 2303 data->ctc_delta = decoder->ctc_delta; 2304 data->last_ctc = decoder->last_ctc; 2305 intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); 2306 if (data->tsc && !data->psbend) 2307 p_log("ERROR: PSB without PSBEND"); 2308 decoder->in_psb = data->psbend; 2309 } 2310 2311 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, 2312 struct intel_pt_vm_tsc_info *data, 2313 struct intel_pt_vmcs_info *vmcs_info, 2314 uint64_t host_tsc) 2315 { 2316 if (!decoder->in_psb) { 2317 /* Can't happen */ 2318 p_log("ERROR: First TSC is not in PSB+"); 2319 } 2320 2321 if (data->pip) { 2322 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2323 if (vmcs_info && vmcs_info->tsc_offset) { 2324 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2325 decoder->vm_tm_corr_reliable = true; 2326 } else { 2327 p_log("ERROR: First TSC, unknown TSC Offset"); 2328 } 2329 } else { /* Host */ 2330 decoder->vm_tm_corr_reliable = true; 2331 } 2332 } else { /* Host or Guest */ 2333 decoder->vm_tm_corr_reliable = false; 2334 if (intel_pt_time_in_range(decoder, host_tsc)) { 2335 /* Assume Host */ 2336 } else { 2337 /* Assume Guest */ 2338 if (vmcs_info && vmcs_info->tsc_offset) 2339 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2340 else 2341 p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); 2342 } 2343 } 2344 } 2345 2346 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, 2347 struct intel_pt_vm_tsc_info *data) 2348 { 2349 struct intel_pt_vmcs_info *vmcs_info; 2350 uint64_t tsc_offset = 0; 2351 uint64_t vmcs; 2352 bool reliable = true; 2353 uint64_t expected_tsc; 2354 uint64_t host_tsc; 2355 uint64_t ref_timestamp; 2356 2357 bool assign = false; 2358 bool assign_reliable = false; 2359 2360 /* Already have 'data' for the in_psb case */ 2361 if (!decoder->in_psb) { 2362 memset(data, 0, sizeof(*data)); 2363 data->ctc_delta = decoder->ctc_delta; 2364 data->last_ctc = decoder->last_ctc; 2365 data->max_lookahead = 16; 2366 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); 2367 if (decoder->pge) { 2368 data->pip = true; 2369 data->pip_packet.payload = decoder->pip_payload; 2370 } 2371 } 2372 2373 /* Calculations depend on having TMA packets */ 2374 if (!data->tma) { 2375 p_log("ERROR: TSC without TMA"); 2376 return; 2377 } 2378 2379 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; 2380 if (vmcs == NO_VMCS) 2381 vmcs = 0; 2382 2383 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2384 2385 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; 2386 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); 2387 2388 if (!decoder->ctc_timestamp) { 2389 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); 2390 return; 2391 } 2392 2393 expected_tsc = intel_pt_expected_tsc(decoder, data); 2394 2395 tsc_offset = host_tsc - expected_tsc; 2396 2397 /* Determine if TSC is from Host or Guest */ 2398 if (data->pip) { 2399 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2400 if (!vmcs_info) { 2401 /* PIP NR=1 without VMCS cannot happen */ 2402 p_log("ERROR: Missing VMCS"); 2403 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2404 decoder->vm_tm_corr_reliable = false; 2405 return; 2406 } 2407 } else { /* Host */ 2408 decoder->last_reliable_timestamp = host_tsc; 2409 decoder->vm_tm_corr_reliable = true; 2410 return; 2411 } 2412 } else { /* Host or Guest */ 2413 reliable = false; /* Host/Guest is a guess, so not reliable */ 2414 if (decoder->in_psb) { 2415 if (!tsc_offset) 2416 return; /* Zero TSC Offset, assume Host */ 2417 /* 2418 * TSC packet has only 7 bytes of TSC. We have no 2419 * information about the Guest's 8th byte, but it 2420 * doesn't matter because we only need 7 bytes. 2421 * Here, since the 8th byte is unreliable and 2422 * irrelevant, compare only 7 byes. 2423 */ 2424 if (vmcs_info && 2425 (tsc_offset & SEVEN_BYTES) == 2426 (vmcs_info->tsc_offset & SEVEN_BYTES)) { 2427 /* Same TSC Offset as last VMCS, assume Guest */ 2428 goto guest; 2429 } 2430 } 2431 /* 2432 * Check if the host_tsc is within the expected range. 2433 * Note, we could narrow the range more by looking ahead for 2434 * the next host TSC in the same buffer, but we don't bother to 2435 * do that because this is probably good enough. 2436 */ 2437 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { 2438 /* Within expected range for Host TSC, assume Host */ 2439 decoder->vm_tm_corr_reliable = false; 2440 return; 2441 } 2442 } 2443 2444 guest: /* Assuming Guest */ 2445 2446 /* Determine whether to assign TSC Offset */ 2447 if (vmcs_info && vmcs_info->vmcs) { 2448 if (vmcs_info->tsc_offset && vmcs_info->reliable) { 2449 assign = false; 2450 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && 2451 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { 2452 /* Continuous tracing, TSC in a PSB is not a time loss */ 2453 assign = true; 2454 assign_reliable = true; 2455 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { 2456 /* 2457 * Unlikely to be a time loss TSC in a PSB which is not 2458 * at the start of a buffer. 2459 */ 2460 assign = true; 2461 assign_reliable = false; 2462 } 2463 } 2464 2465 /* Record VMCS TSC Offset */ 2466 if (assign && (vmcs_info->tsc_offset != tsc_offset || 2467 vmcs_info->reliable != assign_reliable)) { 2468 bool print = vmcs_info->tsc_offset != tsc_offset; 2469 2470 vmcs_info->tsc_offset = tsc_offset; 2471 vmcs_info->reliable = assign_reliable; 2472 if (print) 2473 intel_pt_print_vmcs_info(vmcs_info); 2474 } 2475 2476 /* Determine what TSC Offset to use */ 2477 if (vmcs_info && vmcs_info->tsc_offset) { 2478 if (!vmcs_info->reliable) 2479 reliable = false; 2480 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2481 } else { 2482 reliable = false; 2483 if (vmcs_info) { 2484 if (!vmcs_info->error_printed) { 2485 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, 2486 vmcs_info->vmcs); 2487 vmcs_info->error_printed = true; 2488 } 2489 } else { 2490 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2491 p_log("ERROR: Unknown VMCS"); 2492 } 2493 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2494 } 2495 2496 decoder->vm_tm_corr_reliable = reliable; 2497 } 2498 2499 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder) 2500 { 2501 uint64_t host_tsc = decoder->packet.payload; 2502 uint64_t guest_tsc = decoder->packet.payload; 2503 struct intel_pt_vmcs_info *vmcs_info; 2504 uint64_t vmcs; 2505 2506 vmcs = decoder->vmcs; 2507 if (vmcs == NO_VMCS) 2508 vmcs = 0; 2509 2510 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2511 2512 if (decoder->pge) { 2513 if (in_vm(decoder->pip_payload)) { /* Guest */ 2514 if (!vmcs_info) { 2515 /* PIP NR=1 without VMCS cannot happen */ 2516 p_log("ERROR: Missing VMCS"); 2517 } 2518 } else { /* Host */ 2519 return; 2520 } 2521 } else { /* Host or Guest */ 2522 if (intel_pt_time_in_range(decoder, host_tsc)) { 2523 /* Within expected range for Host TSC, assume Host */ 2524 return; 2525 } 2526 } 2527 2528 if (vmcs_info) { 2529 /* Translate Guest TSC to Host TSC */ 2530 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; 2531 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); 2532 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2533 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2534 guest_tsc, host_tsc, vmcs_info->vmcs, 2535 vmcs_info->tsc_offset); 2536 if (!intel_pt_time_in_range(decoder, host_tsc) && 2537 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 2538 p_log("Timestamp out of range"); 2539 } else { 2540 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2541 p_log("ERROR: Unknown VMCS"); 2542 host_tsc = decoder->timestamp; 2543 } 2544 2545 decoder->packet.payload = host_tsc; 2546 2547 if (!decoder->vm_tm_corr_dry_run) 2548 memcpy((void *)decoder->buf + 1, &host_tsc, 8); 2549 } 2550 2551 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder) 2552 { 2553 struct intel_pt_vm_tsc_info data = { .psbend = false }; 2554 bool pge; 2555 int err; 2556 2557 if (decoder->in_psb) 2558 intel_pt_vm_tm_corr_psb(decoder, &data); 2559 2560 while (1) { 2561 err = intel_pt_get_next_packet(decoder); 2562 if (err == -ENOLINK) 2563 continue; 2564 if (err) 2565 break; 2566 2567 switch (decoder->packet.type) { 2568 case INTEL_PT_TIP_PGD: 2569 decoder->pge = false; 2570 decoder->vm_tm_corr_continuous = false; 2571 break; 2572 2573 case INTEL_PT_TNT: 2574 case INTEL_PT_TIP: 2575 case INTEL_PT_TIP_PGE: 2576 decoder->pge = true; 2577 break; 2578 2579 case INTEL_PT_OVF: 2580 decoder->in_psb = false; 2581 pge = decoder->pge; 2582 decoder->pge = intel_pt_ovf_fup_lookahead(decoder); 2583 if (pge != decoder->pge) 2584 intel_pt_log("Surprising PGE change in OVF!"); 2585 if (!decoder->pge) 2586 decoder->vm_tm_corr_continuous = false; 2587 break; 2588 2589 case INTEL_PT_FUP: 2590 if (decoder->in_psb) 2591 decoder->pge = true; 2592 break; 2593 2594 case INTEL_PT_TRACESTOP: 2595 decoder->pge = false; 2596 decoder->vm_tm_corr_continuous = false; 2597 decoder->have_tma = false; 2598 break; 2599 2600 case INTEL_PT_PSB: 2601 intel_pt_vm_tm_corr_psb(decoder, &data); 2602 break; 2603 2604 case INTEL_PT_PIP: 2605 decoder->pip_payload = decoder->packet.payload; 2606 break; 2607 2608 case INTEL_PT_MTC: 2609 intel_pt_calc_mtc_timestamp(decoder); 2610 break; 2611 2612 case INTEL_PT_TSC: 2613 intel_pt_vm_tm_corr_tsc(decoder, &data); 2614 intel_pt_calc_tsc_timestamp(decoder); 2615 decoder->vm_tm_corr_same_buf = true; 2616 decoder->vm_tm_corr_continuous = decoder->pge; 2617 break; 2618 2619 case INTEL_PT_TMA: 2620 intel_pt_calc_tma(decoder); 2621 break; 2622 2623 case INTEL_PT_CYC: 2624 intel_pt_calc_cyc_timestamp(decoder); 2625 break; 2626 2627 case INTEL_PT_CBR: 2628 intel_pt_calc_cbr(decoder); 2629 break; 2630 2631 case INTEL_PT_PSBEND: 2632 decoder->in_psb = false; 2633 data.psbend = false; 2634 break; 2635 2636 case INTEL_PT_VMCS: 2637 if (decoder->packet.payload != NO_VMCS) 2638 decoder->vmcs = decoder->packet.payload; 2639 break; 2640 2641 case INTEL_PT_BBP: 2642 decoder->blk_type = decoder->packet.payload; 2643 break; 2644 2645 case INTEL_PT_BIP: 2646 if (decoder->blk_type == INTEL_PT_PEBS_BASIC && 2647 decoder->packet.count == 2) 2648 intel_pt_vm_tm_corr_pebs_tsc(decoder); 2649 break; 2650 2651 case INTEL_PT_BEP: 2652 case INTEL_PT_BEP_IP: 2653 decoder->blk_type = 0; 2654 break; 2655 2656 case INTEL_PT_MODE_EXEC: 2657 case INTEL_PT_MODE_TSX: 2658 case INTEL_PT_MNT: 2659 case INTEL_PT_PAD: 2660 case INTEL_PT_PTWRITE_IP: 2661 case INTEL_PT_PTWRITE: 2662 case INTEL_PT_MWAIT: 2663 case INTEL_PT_PWRE: 2664 case INTEL_PT_EXSTOP_IP: 2665 case INTEL_PT_EXSTOP: 2666 case INTEL_PT_PWRX: 2667 case INTEL_PT_BAD: /* Does not happen */ 2668 default: 2669 break; 2670 } 2671 } 2672 2673 return err; 2674 } 2675 2676 #define HOP_PROCESS 0 2677 #define HOP_IGNORE 1 2678 #define HOP_RETURN 2 2679 #define HOP_AGAIN 3 2680 2681 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 2682 2683 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 2684 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 2685 { 2686 *err = 0; 2687 2688 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 2689 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 2690 *err = intel_pt_scan_for_psb(decoder); 2691 if (*err) 2692 return HOP_RETURN; 2693 } 2694 2695 switch (decoder->packet.type) { 2696 case INTEL_PT_TNT: 2697 return HOP_IGNORE; 2698 2699 case INTEL_PT_TIP_PGD: 2700 decoder->pge = false; 2701 if (!decoder->packet.count) { 2702 intel_pt_set_nr(decoder); 2703 return HOP_IGNORE; 2704 } 2705 intel_pt_set_ip(decoder); 2706 decoder->state.type |= INTEL_PT_TRACE_END; 2707 decoder->state.from_ip = 0; 2708 decoder->state.to_ip = decoder->ip; 2709 intel_pt_update_nr(decoder); 2710 return HOP_RETURN; 2711 2712 case INTEL_PT_TIP: 2713 if (!decoder->packet.count) { 2714 intel_pt_set_nr(decoder); 2715 return HOP_IGNORE; 2716 } 2717 intel_pt_set_ip(decoder); 2718 decoder->state.type = INTEL_PT_INSTRUCTION; 2719 decoder->state.from_ip = decoder->ip; 2720 decoder->state.to_ip = 0; 2721 intel_pt_update_nr(decoder); 2722 return HOP_RETURN; 2723 2724 case INTEL_PT_FUP: 2725 if (!decoder->packet.count) 2726 return HOP_IGNORE; 2727 intel_pt_set_ip(decoder); 2728 if (decoder->set_fup_mwait || decoder->set_fup_pwre) 2729 *no_tip = true; 2730 if (!decoder->branch_enable || !decoder->pge) 2731 *no_tip = true; 2732 if (*no_tip) { 2733 decoder->state.type = INTEL_PT_INSTRUCTION; 2734 decoder->state.from_ip = decoder->ip; 2735 decoder->state.to_ip = 0; 2736 intel_pt_fup_event(decoder); 2737 return HOP_RETURN; 2738 } 2739 intel_pt_fup_event(decoder); 2740 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH; 2741 *err = intel_pt_walk_fup_tip(decoder); 2742 if (!*err && decoder->state.to_ip) 2743 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2744 return HOP_RETURN; 2745 2746 case INTEL_PT_PSB: 2747 decoder->state.psb_offset = decoder->pos; 2748 decoder->psb_ip = 0; 2749 decoder->last_ip = 0; 2750 decoder->have_last_ip = true; 2751 *err = intel_pt_walk_psbend(decoder); 2752 if (*err == -EAGAIN) 2753 return HOP_AGAIN; 2754 if (*err) 2755 return HOP_RETURN; 2756 decoder->state.type = INTEL_PT_PSB_EVT; 2757 if (decoder->psb_ip) { 2758 decoder->state.type |= INTEL_PT_INSTRUCTION; 2759 decoder->ip = decoder->psb_ip; 2760 } 2761 decoder->state.from_ip = decoder->psb_ip; 2762 decoder->state.to_ip = 0; 2763 return HOP_RETURN; 2764 2765 case INTEL_PT_BAD: 2766 case INTEL_PT_PAD: 2767 case INTEL_PT_TIP_PGE: 2768 case INTEL_PT_TSC: 2769 case INTEL_PT_TMA: 2770 case INTEL_PT_MODE_EXEC: 2771 case INTEL_PT_MODE_TSX: 2772 case INTEL_PT_MTC: 2773 case INTEL_PT_CYC: 2774 case INTEL_PT_VMCS: 2775 case INTEL_PT_PSBEND: 2776 case INTEL_PT_CBR: 2777 case INTEL_PT_TRACESTOP: 2778 case INTEL_PT_PIP: 2779 case INTEL_PT_OVF: 2780 case INTEL_PT_MNT: 2781 case INTEL_PT_PTWRITE: 2782 case INTEL_PT_PTWRITE_IP: 2783 case INTEL_PT_EXSTOP: 2784 case INTEL_PT_EXSTOP_IP: 2785 case INTEL_PT_MWAIT: 2786 case INTEL_PT_PWRE: 2787 case INTEL_PT_PWRX: 2788 case INTEL_PT_BBP: 2789 case INTEL_PT_BIP: 2790 case INTEL_PT_BEP: 2791 case INTEL_PT_BEP_IP: 2792 default: 2793 return HOP_PROCESS; 2794 } 2795 } 2796 2797 struct intel_pt_psb_info { 2798 struct intel_pt_pkt fup_packet; 2799 bool fup; 2800 int after_psbend; 2801 }; 2802 2803 /* Lookahead and get the FUP packet from PSB+ */ 2804 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2805 { 2806 struct intel_pt_psb_info *data = pkt_info->data; 2807 2808 switch (pkt_info->packet.type) { 2809 case INTEL_PT_PAD: 2810 case INTEL_PT_MNT: 2811 case INTEL_PT_TSC: 2812 case INTEL_PT_TMA: 2813 case INTEL_PT_MODE_EXEC: 2814 case INTEL_PT_MODE_TSX: 2815 case INTEL_PT_MTC: 2816 case INTEL_PT_CYC: 2817 case INTEL_PT_VMCS: 2818 case INTEL_PT_CBR: 2819 case INTEL_PT_PIP: 2820 if (data->after_psbend) { 2821 data->after_psbend -= 1; 2822 if (!data->after_psbend) 2823 return 1; 2824 } 2825 break; 2826 2827 case INTEL_PT_FUP: 2828 if (data->after_psbend) 2829 return 1; 2830 if (data->fup || pkt_info->packet.count == 0) 2831 return 1; 2832 data->fup_packet = pkt_info->packet; 2833 data->fup = true; 2834 break; 2835 2836 case INTEL_PT_PSBEND: 2837 if (!data->fup) 2838 return 1; 2839 /* Keep going to check for a TIP.PGE */ 2840 data->after_psbend = 6; 2841 break; 2842 2843 case INTEL_PT_TIP_PGE: 2844 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 2845 if (data->after_psbend) 2846 data->fup = false; 2847 return 1; 2848 2849 case INTEL_PT_PTWRITE: 2850 case INTEL_PT_PTWRITE_IP: 2851 case INTEL_PT_EXSTOP: 2852 case INTEL_PT_EXSTOP_IP: 2853 case INTEL_PT_MWAIT: 2854 case INTEL_PT_PWRE: 2855 case INTEL_PT_PWRX: 2856 case INTEL_PT_BBP: 2857 case INTEL_PT_BIP: 2858 case INTEL_PT_BEP: 2859 case INTEL_PT_BEP_IP: 2860 if (data->after_psbend) { 2861 data->after_psbend -= 1; 2862 if (!data->after_psbend) 2863 return 1; 2864 break; 2865 } 2866 return 1; 2867 2868 case INTEL_PT_OVF: 2869 case INTEL_PT_BAD: 2870 case INTEL_PT_TNT: 2871 case INTEL_PT_TIP_PGD: 2872 case INTEL_PT_TIP: 2873 case INTEL_PT_PSB: 2874 case INTEL_PT_TRACESTOP: 2875 default: 2876 return 1; 2877 } 2878 2879 return 0; 2880 } 2881 2882 static int intel_pt_psb(struct intel_pt_decoder *decoder) 2883 { 2884 int err; 2885 2886 decoder->last_ip = 0; 2887 decoder->psb_ip = 0; 2888 decoder->have_last_ip = true; 2889 intel_pt_clear_stack(&decoder->stack); 2890 err = intel_pt_walk_psbend(decoder); 2891 if (err) 2892 return err; 2893 decoder->state.type = INTEL_PT_PSB_EVT; 2894 decoder->state.from_ip = decoder->psb_ip; 2895 decoder->state.to_ip = 0; 2896 return 0; 2897 } 2898 2899 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 2900 { 2901 int err; 2902 2903 if (decoder->ip != decoder->last_ip) { 2904 err = intel_pt_walk_fup(decoder); 2905 if (!err || err != -EAGAIN) 2906 return err; 2907 } 2908 2909 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2910 err = intel_pt_psb(decoder); 2911 if (err) { 2912 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2913 return -ENOENT; 2914 } 2915 2916 return 0; 2917 } 2918 2919 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 2920 { 2921 struct intel_pt_psb_info data = { .fup = false }; 2922 2923 if (!decoder->branch_enable) 2924 return false; 2925 2926 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 2927 if (!data.fup) 2928 return false; 2929 2930 decoder->packet = data.fup_packet; 2931 intel_pt_set_last_ip(decoder); 2932 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 2933 2934 *err = intel_pt_fup_in_psb(decoder); 2935 2936 return true; 2937 } 2938 2939 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2940 { 2941 int last_packet_type = INTEL_PT_PAD; 2942 bool no_tip = false; 2943 int err; 2944 2945 while (1) { 2946 err = intel_pt_get_next_packet(decoder); 2947 if (err) 2948 return err; 2949 next: 2950 err = 0; 2951 if (decoder->cyc_threshold) { 2952 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 2953 decoder->sample_cyc = false; 2954 last_packet_type = decoder->packet.type; 2955 } 2956 2957 if (decoder->hop) { 2958 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2959 case HOP_IGNORE: 2960 continue; 2961 case HOP_RETURN: 2962 return err; 2963 case HOP_AGAIN: 2964 goto next; 2965 default: 2966 break; 2967 } 2968 } 2969 2970 switch (decoder->packet.type) { 2971 case INTEL_PT_TNT: 2972 if (!decoder->packet.count) 2973 break; 2974 decoder->tnt = decoder->packet; 2975 decoder->pkt_state = INTEL_PT_STATE_TNT; 2976 err = intel_pt_walk_tnt(decoder); 2977 if (err == -EAGAIN) 2978 break; 2979 return err; 2980 2981 case INTEL_PT_TIP_PGD: 2982 if (decoder->packet.count != 0) 2983 intel_pt_set_last_ip(decoder); 2984 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2985 return intel_pt_walk_tip(decoder); 2986 2987 case INTEL_PT_TIP_PGE: { 2988 decoder->pge = true; 2989 decoder->overflow = false; 2990 intel_pt_mtc_cyc_cnt_pge(decoder); 2991 intel_pt_set_nr(decoder); 2992 if (decoder->packet.count == 0) { 2993 intel_pt_log_at("Skipping zero TIP.PGE", 2994 decoder->pos); 2995 break; 2996 } 2997 intel_pt_set_ip(decoder); 2998 decoder->state.from_ip = 0; 2999 decoder->state.to_ip = decoder->ip; 3000 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3001 /* 3002 * In hop mode, resample to get the to_ip as an 3003 * "instruction" sample. 3004 */ 3005 if (decoder->hop) 3006 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3007 return 0; 3008 } 3009 3010 case INTEL_PT_OVF: 3011 return intel_pt_overflow(decoder); 3012 3013 case INTEL_PT_TIP: 3014 if (decoder->packet.count != 0) 3015 intel_pt_set_last_ip(decoder); 3016 decoder->pkt_state = INTEL_PT_STATE_TIP; 3017 return intel_pt_walk_tip(decoder); 3018 3019 case INTEL_PT_FUP: 3020 if (decoder->packet.count == 0) { 3021 intel_pt_log_at("Skipping zero FUP", 3022 decoder->pos); 3023 no_tip = false; 3024 break; 3025 } 3026 intel_pt_set_last_ip(decoder); 3027 if (!decoder->branch_enable || !decoder->pge) { 3028 decoder->ip = decoder->last_ip; 3029 if (intel_pt_fup_event(decoder)) 3030 return 0; 3031 no_tip = false; 3032 break; 3033 } 3034 if (decoder->set_fup_mwait) 3035 no_tip = true; 3036 if (no_tip) 3037 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 3038 else 3039 decoder->pkt_state = INTEL_PT_STATE_FUP; 3040 err = intel_pt_walk_fup(decoder); 3041 if (err != -EAGAIN) 3042 return err; 3043 if (no_tip) { 3044 no_tip = false; 3045 break; 3046 } 3047 return intel_pt_walk_fup_tip(decoder); 3048 3049 case INTEL_PT_TRACESTOP: 3050 decoder->pge = false; 3051 decoder->continuous_period = false; 3052 intel_pt_clear_tx_flags(decoder); 3053 decoder->have_tma = false; 3054 break; 3055 3056 case INTEL_PT_PSB: 3057 decoder->state.psb_offset = decoder->pos; 3058 decoder->psb_ip = 0; 3059 if (intel_pt_psb_with_fup(decoder, &err)) 3060 return err; 3061 err = intel_pt_psb(decoder); 3062 if (err == -EAGAIN) 3063 goto next; 3064 return err; 3065 3066 case INTEL_PT_PIP: 3067 intel_pt_update_pip(decoder); 3068 break; 3069 3070 case INTEL_PT_MTC: 3071 intel_pt_calc_mtc_timestamp(decoder); 3072 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 3073 break; 3074 /* 3075 * Ensure that there has been an instruction since the 3076 * last MTC. 3077 */ 3078 if (!decoder->mtc_insn) 3079 break; 3080 decoder->mtc_insn = false; 3081 /* Ensure that there is a timestamp */ 3082 if (!decoder->timestamp) 3083 break; 3084 decoder->state.type = INTEL_PT_INSTRUCTION; 3085 decoder->state.from_ip = decoder->ip; 3086 decoder->state.to_ip = 0; 3087 decoder->mtc_insn = false; 3088 return 0; 3089 3090 case INTEL_PT_TSC: 3091 intel_pt_calc_tsc_timestamp(decoder); 3092 break; 3093 3094 case INTEL_PT_TMA: 3095 intel_pt_calc_tma(decoder); 3096 break; 3097 3098 case INTEL_PT_CYC: 3099 intel_pt_calc_cyc_timestamp(decoder); 3100 break; 3101 3102 case INTEL_PT_CBR: 3103 intel_pt_calc_cbr(decoder); 3104 if (decoder->cbr != decoder->cbr_seen) { 3105 decoder->state.type = 0; 3106 return 0; 3107 } 3108 break; 3109 3110 case INTEL_PT_MODE_EXEC: 3111 decoder->exec_mode = decoder->packet.payload; 3112 break; 3113 3114 case INTEL_PT_MODE_TSX: 3115 /* MODE_TSX need not be followed by FUP */ 3116 if (!decoder->pge || decoder->in_psb) { 3117 intel_pt_update_in_tx(decoder); 3118 break; 3119 } 3120 err = intel_pt_mode_tsx(decoder, &no_tip); 3121 if (err) 3122 return err; 3123 goto next; 3124 3125 case INTEL_PT_BAD: /* Does not happen */ 3126 return intel_pt_bug(decoder); 3127 3128 case INTEL_PT_PSBEND: 3129 case INTEL_PT_VMCS: 3130 case INTEL_PT_MNT: 3131 case INTEL_PT_PAD: 3132 break; 3133 3134 case INTEL_PT_PTWRITE_IP: 3135 decoder->fup_ptw_payload = decoder->packet.payload; 3136 err = intel_pt_get_next_packet(decoder); 3137 if (err) 3138 return err; 3139 if (decoder->packet.type == INTEL_PT_FUP) { 3140 decoder->set_fup_ptw = true; 3141 no_tip = true; 3142 } else { 3143 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 3144 decoder->pos); 3145 } 3146 goto next; 3147 3148 case INTEL_PT_PTWRITE: 3149 decoder->state.type = INTEL_PT_PTW; 3150 decoder->state.from_ip = decoder->ip; 3151 decoder->state.to_ip = 0; 3152 decoder->state.ptw_payload = decoder->packet.payload; 3153 return 0; 3154 3155 case INTEL_PT_MWAIT: 3156 decoder->fup_mwait_payload = decoder->packet.payload; 3157 decoder->set_fup_mwait = true; 3158 break; 3159 3160 case INTEL_PT_PWRE: 3161 if (decoder->set_fup_mwait) { 3162 decoder->fup_pwre_payload = 3163 decoder->packet.payload; 3164 decoder->set_fup_pwre = true; 3165 break; 3166 } 3167 decoder->state.type = INTEL_PT_PWR_ENTRY; 3168 decoder->state.from_ip = decoder->ip; 3169 decoder->state.to_ip = 0; 3170 decoder->state.pwrx_payload = decoder->packet.payload; 3171 return 0; 3172 3173 case INTEL_PT_EXSTOP_IP: 3174 err = intel_pt_get_next_packet(decoder); 3175 if (err) 3176 return err; 3177 if (decoder->packet.type == INTEL_PT_FUP) { 3178 decoder->set_fup_exstop = true; 3179 no_tip = true; 3180 } else { 3181 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 3182 decoder->pos); 3183 } 3184 goto next; 3185 3186 case INTEL_PT_EXSTOP: 3187 decoder->state.type = INTEL_PT_EX_STOP; 3188 decoder->state.from_ip = decoder->ip; 3189 decoder->state.to_ip = 0; 3190 return 0; 3191 3192 case INTEL_PT_PWRX: 3193 decoder->state.type = INTEL_PT_PWR_EXIT; 3194 decoder->state.from_ip = decoder->ip; 3195 decoder->state.to_ip = 0; 3196 decoder->state.pwrx_payload = decoder->packet.payload; 3197 return 0; 3198 3199 case INTEL_PT_BBP: 3200 intel_pt_bbp(decoder); 3201 break; 3202 3203 case INTEL_PT_BIP: 3204 intel_pt_bip(decoder); 3205 break; 3206 3207 case INTEL_PT_BEP: 3208 decoder->state.type = INTEL_PT_BLK_ITEMS; 3209 decoder->state.from_ip = decoder->ip; 3210 decoder->state.to_ip = 0; 3211 return 0; 3212 3213 case INTEL_PT_BEP_IP: 3214 err = intel_pt_get_next_packet(decoder); 3215 if (err) 3216 return err; 3217 if (decoder->packet.type == INTEL_PT_FUP) { 3218 decoder->set_fup_bep = true; 3219 no_tip = true; 3220 } else { 3221 intel_pt_log_at("ERROR: Missing FUP after BEP", 3222 decoder->pos); 3223 } 3224 goto next; 3225 3226 default: 3227 return intel_pt_bug(decoder); 3228 } 3229 } 3230 } 3231 3232 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 3233 { 3234 return decoder->packet.count && 3235 (decoder->have_last_ip || decoder->packet.count == 3 || 3236 decoder->packet.count == 6); 3237 } 3238 3239 /* Walk PSB+ packets to get in sync. */ 3240 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 3241 { 3242 int err; 3243 3244 decoder->in_psb = true; 3245 3246 while (1) { 3247 err = intel_pt_get_next_packet(decoder); 3248 if (err) 3249 goto out; 3250 3251 switch (decoder->packet.type) { 3252 case INTEL_PT_TIP_PGD: 3253 decoder->continuous_period = false; 3254 __fallthrough; 3255 case INTEL_PT_TIP_PGE: 3256 case INTEL_PT_TIP: 3257 case INTEL_PT_PTWRITE: 3258 case INTEL_PT_PTWRITE_IP: 3259 case INTEL_PT_EXSTOP: 3260 case INTEL_PT_EXSTOP_IP: 3261 case INTEL_PT_MWAIT: 3262 case INTEL_PT_PWRE: 3263 case INTEL_PT_PWRX: 3264 case INTEL_PT_BBP: 3265 case INTEL_PT_BIP: 3266 case INTEL_PT_BEP: 3267 case INTEL_PT_BEP_IP: 3268 intel_pt_log("ERROR: Unexpected packet\n"); 3269 err = -ENOENT; 3270 goto out; 3271 3272 case INTEL_PT_FUP: 3273 decoder->pge = true; 3274 if (intel_pt_have_ip(decoder)) { 3275 uint64_t current_ip = decoder->ip; 3276 3277 intel_pt_set_ip(decoder); 3278 decoder->psb_ip = decoder->ip; 3279 if (current_ip) 3280 intel_pt_log_to("Setting IP", 3281 decoder->ip); 3282 } 3283 break; 3284 3285 case INTEL_PT_MTC: 3286 intel_pt_calc_mtc_timestamp(decoder); 3287 break; 3288 3289 case INTEL_PT_TSC: 3290 intel_pt_calc_tsc_timestamp(decoder); 3291 break; 3292 3293 case INTEL_PT_TMA: 3294 intel_pt_calc_tma(decoder); 3295 break; 3296 3297 case INTEL_PT_CYC: 3298 intel_pt_calc_cyc_timestamp(decoder); 3299 break; 3300 3301 case INTEL_PT_CBR: 3302 intel_pt_calc_cbr(decoder); 3303 break; 3304 3305 case INTEL_PT_PIP: 3306 intel_pt_set_pip(decoder); 3307 break; 3308 3309 case INTEL_PT_MODE_EXEC: 3310 decoder->exec_mode = decoder->packet.payload; 3311 break; 3312 3313 case INTEL_PT_MODE_TSX: 3314 intel_pt_update_in_tx(decoder); 3315 break; 3316 3317 case INTEL_PT_TRACESTOP: 3318 decoder->pge = false; 3319 decoder->continuous_period = false; 3320 intel_pt_clear_tx_flags(decoder); 3321 __fallthrough; 3322 3323 case INTEL_PT_TNT: 3324 decoder->have_tma = false; 3325 intel_pt_log("ERROR: Unexpected packet\n"); 3326 if (decoder->ip) 3327 decoder->pkt_state = INTEL_PT_STATE_ERR4; 3328 else 3329 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3330 err = -ENOENT; 3331 goto out; 3332 3333 case INTEL_PT_BAD: /* Does not happen */ 3334 err = intel_pt_bug(decoder); 3335 goto out; 3336 3337 case INTEL_PT_OVF: 3338 err = intel_pt_overflow(decoder); 3339 goto out; 3340 3341 case INTEL_PT_PSBEND: 3342 err = 0; 3343 goto out; 3344 3345 case INTEL_PT_PSB: 3346 case INTEL_PT_VMCS: 3347 case INTEL_PT_MNT: 3348 case INTEL_PT_PAD: 3349 default: 3350 break; 3351 } 3352 } 3353 out: 3354 decoder->in_psb = false; 3355 3356 return err; 3357 } 3358 3359 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 3360 { 3361 int err; 3362 3363 while (1) { 3364 err = intel_pt_get_next_packet(decoder); 3365 if (err) 3366 return err; 3367 3368 switch (decoder->packet.type) { 3369 case INTEL_PT_TIP_PGD: 3370 decoder->continuous_period = false; 3371 decoder->pge = false; 3372 if (intel_pt_have_ip(decoder)) 3373 intel_pt_set_ip(decoder); 3374 if (!decoder->ip) 3375 break; 3376 decoder->state.type |= INTEL_PT_TRACE_END; 3377 return 0; 3378 3379 case INTEL_PT_TIP_PGE: 3380 decoder->pge = true; 3381 intel_pt_mtc_cyc_cnt_pge(decoder); 3382 if (intel_pt_have_ip(decoder)) 3383 intel_pt_set_ip(decoder); 3384 if (!decoder->ip) 3385 break; 3386 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3387 return 0; 3388 3389 case INTEL_PT_TIP: 3390 decoder->pge = true; 3391 if (intel_pt_have_ip(decoder)) 3392 intel_pt_set_ip(decoder); 3393 if (!decoder->ip) 3394 break; 3395 return 0; 3396 3397 case INTEL_PT_FUP: 3398 if (intel_pt_have_ip(decoder)) 3399 intel_pt_set_ip(decoder); 3400 if (decoder->ip) 3401 return 0; 3402 break; 3403 3404 case INTEL_PT_MTC: 3405 intel_pt_calc_mtc_timestamp(decoder); 3406 break; 3407 3408 case INTEL_PT_TSC: 3409 intel_pt_calc_tsc_timestamp(decoder); 3410 break; 3411 3412 case INTEL_PT_TMA: 3413 intel_pt_calc_tma(decoder); 3414 break; 3415 3416 case INTEL_PT_CYC: 3417 intel_pt_calc_cyc_timestamp(decoder); 3418 break; 3419 3420 case INTEL_PT_CBR: 3421 intel_pt_calc_cbr(decoder); 3422 break; 3423 3424 case INTEL_PT_PIP: 3425 intel_pt_set_pip(decoder); 3426 break; 3427 3428 case INTEL_PT_MODE_EXEC: 3429 decoder->exec_mode = decoder->packet.payload; 3430 break; 3431 3432 case INTEL_PT_MODE_TSX: 3433 intel_pt_update_in_tx(decoder); 3434 break; 3435 3436 case INTEL_PT_OVF: 3437 return intel_pt_overflow(decoder); 3438 3439 case INTEL_PT_BAD: /* Does not happen */ 3440 return intel_pt_bug(decoder); 3441 3442 case INTEL_PT_TRACESTOP: 3443 decoder->pge = false; 3444 decoder->continuous_period = false; 3445 intel_pt_clear_tx_flags(decoder); 3446 decoder->have_tma = false; 3447 break; 3448 3449 case INTEL_PT_PSB: 3450 decoder->state.psb_offset = decoder->pos; 3451 decoder->psb_ip = 0; 3452 decoder->last_ip = 0; 3453 decoder->have_last_ip = true; 3454 intel_pt_clear_stack(&decoder->stack); 3455 err = intel_pt_walk_psb(decoder); 3456 if (err) 3457 return err; 3458 decoder->state.type = INTEL_PT_PSB_EVT; 3459 decoder->state.from_ip = decoder->psb_ip; 3460 decoder->state.to_ip = 0; 3461 return 0; 3462 3463 case INTEL_PT_TNT: 3464 case INTEL_PT_PSBEND: 3465 case INTEL_PT_VMCS: 3466 case INTEL_PT_MNT: 3467 case INTEL_PT_PAD: 3468 case INTEL_PT_PTWRITE: 3469 case INTEL_PT_PTWRITE_IP: 3470 case INTEL_PT_EXSTOP: 3471 case INTEL_PT_EXSTOP_IP: 3472 case INTEL_PT_MWAIT: 3473 case INTEL_PT_PWRE: 3474 case INTEL_PT_PWRX: 3475 case INTEL_PT_BBP: 3476 case INTEL_PT_BIP: 3477 case INTEL_PT_BEP: 3478 case INTEL_PT_BEP_IP: 3479 default: 3480 break; 3481 } 3482 } 3483 } 3484 3485 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 3486 { 3487 int err; 3488 3489 decoder->set_fup_tx_flags = false; 3490 decoder->set_fup_ptw = false; 3491 decoder->set_fup_mwait = false; 3492 decoder->set_fup_pwre = false; 3493 decoder->set_fup_exstop = false; 3494 decoder->set_fup_bep = false; 3495 decoder->overflow = false; 3496 3497 if (!decoder->branch_enable) { 3498 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3499 decoder->state.type = 0; /* Do not have a sample */ 3500 return 0; 3501 } 3502 3503 intel_pt_log("Scanning for full IP\n"); 3504 err = intel_pt_walk_to_ip(decoder); 3505 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 3506 return err; 3507 3508 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 3509 if (decoder->hop) 3510 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3511 else 3512 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3513 3514 decoder->state.from_ip = 0; 3515 decoder->state.to_ip = decoder->ip; 3516 intel_pt_log_to("Setting IP", decoder->ip); 3517 3518 return 0; 3519 } 3520 3521 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 3522 { 3523 const unsigned char *end = decoder->buf + decoder->len; 3524 size_t i; 3525 3526 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 3527 if (i > decoder->len) 3528 continue; 3529 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 3530 return i; 3531 } 3532 return 0; 3533 } 3534 3535 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 3536 { 3537 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 3538 const char *psb = INTEL_PT_PSB_STR; 3539 3540 if (rest_psb > decoder->len || 3541 memcmp(decoder->buf, psb + part_psb, rest_psb)) 3542 return 0; 3543 3544 return rest_psb; 3545 } 3546 3547 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 3548 int part_psb) 3549 { 3550 int rest_psb, ret; 3551 3552 decoder->pos += decoder->len; 3553 decoder->len = 0; 3554 3555 ret = intel_pt_get_next_data(decoder, false); 3556 if (ret) 3557 return ret; 3558 3559 rest_psb = intel_pt_rest_psb(decoder, part_psb); 3560 if (!rest_psb) 3561 return 0; 3562 3563 decoder->pos -= part_psb; 3564 decoder->next_buf = decoder->buf + rest_psb; 3565 decoder->next_len = decoder->len - rest_psb; 3566 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3567 decoder->buf = decoder->temp_buf; 3568 decoder->len = INTEL_PT_PSB_LEN; 3569 3570 return 0; 3571 } 3572 3573 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 3574 { 3575 unsigned char *next; 3576 int ret; 3577 3578 intel_pt_log("Scanning for PSB\n"); 3579 while (1) { 3580 if (!decoder->len) { 3581 ret = intel_pt_get_next_data(decoder, false); 3582 if (ret) 3583 return ret; 3584 } 3585 3586 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 3587 INTEL_PT_PSB_LEN); 3588 if (!next) { 3589 int part_psb; 3590 3591 part_psb = intel_pt_part_psb(decoder); 3592 if (part_psb) { 3593 ret = intel_pt_get_split_psb(decoder, part_psb); 3594 if (ret) 3595 return ret; 3596 } else { 3597 decoder->pos += decoder->len; 3598 decoder->len = 0; 3599 } 3600 continue; 3601 } 3602 3603 decoder->pkt_step = next - decoder->buf; 3604 return intel_pt_get_next_packet(decoder); 3605 } 3606 } 3607 3608 static int intel_pt_sync(struct intel_pt_decoder *decoder) 3609 { 3610 int err; 3611 3612 decoder->pge = false; 3613 decoder->continuous_period = false; 3614 decoder->have_last_ip = false; 3615 decoder->last_ip = 0; 3616 decoder->psb_ip = 0; 3617 decoder->ip = 0; 3618 intel_pt_clear_stack(&decoder->stack); 3619 3620 err = intel_pt_scan_for_psb(decoder); 3621 if (err) 3622 return err; 3623 3624 if (decoder->vm_time_correlation) { 3625 decoder->in_psb = true; 3626 if (!decoder->timestamp) 3627 decoder->timestamp = 1; 3628 decoder->state.type = 0; 3629 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; 3630 return 0; 3631 } 3632 3633 decoder->have_last_ip = true; 3634 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3635 3636 err = intel_pt_walk_psb(decoder); 3637 if (err) 3638 return err; 3639 3640 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 3641 decoder->state.from_ip = decoder->psb_ip; 3642 decoder->state.to_ip = 0; 3643 3644 if (decoder->ip) { 3645 /* 3646 * In hop mode, resample to get the PSB FUP ip as an 3647 * "instruction" sample. 3648 */ 3649 if (decoder->hop) 3650 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3651 else 3652 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3653 } 3654 3655 return 0; 3656 } 3657 3658 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 3659 { 3660 uint64_t est = decoder->sample_insn_cnt << 1; 3661 3662 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 3663 goto out; 3664 3665 est *= decoder->max_non_turbo_ratio; 3666 est /= decoder->cbr; 3667 out: 3668 return decoder->sample_timestamp + est; 3669 } 3670 3671 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 3672 { 3673 int err; 3674 3675 do { 3676 decoder->state.type = INTEL_PT_BRANCH; 3677 decoder->state.flags = 0; 3678 3679 switch (decoder->pkt_state) { 3680 case INTEL_PT_STATE_NO_PSB: 3681 err = intel_pt_sync(decoder); 3682 break; 3683 case INTEL_PT_STATE_NO_IP: 3684 decoder->have_last_ip = false; 3685 decoder->last_ip = 0; 3686 decoder->ip = 0; 3687 __fallthrough; 3688 case INTEL_PT_STATE_ERR_RESYNC: 3689 err = intel_pt_sync_ip(decoder); 3690 break; 3691 case INTEL_PT_STATE_IN_SYNC: 3692 err = intel_pt_walk_trace(decoder); 3693 break; 3694 case INTEL_PT_STATE_TNT: 3695 case INTEL_PT_STATE_TNT_CONT: 3696 err = intel_pt_walk_tnt(decoder); 3697 if (err == -EAGAIN) 3698 err = intel_pt_walk_trace(decoder); 3699 break; 3700 case INTEL_PT_STATE_TIP: 3701 case INTEL_PT_STATE_TIP_PGD: 3702 err = intel_pt_walk_tip(decoder); 3703 break; 3704 case INTEL_PT_STATE_FUP: 3705 err = intel_pt_walk_fup(decoder); 3706 if (err == -EAGAIN) 3707 err = intel_pt_walk_fup_tip(decoder); 3708 break; 3709 case INTEL_PT_STATE_FUP_NO_TIP: 3710 err = intel_pt_walk_fup(decoder); 3711 if (err == -EAGAIN) 3712 err = intel_pt_walk_trace(decoder); 3713 break; 3714 case INTEL_PT_STATE_FUP_IN_PSB: 3715 err = intel_pt_fup_in_psb(decoder); 3716 break; 3717 case INTEL_PT_STATE_RESAMPLE: 3718 err = intel_pt_resample(decoder); 3719 break; 3720 case INTEL_PT_STATE_VM_TIME_CORRELATION: 3721 err = intel_pt_vm_time_correlation(decoder); 3722 break; 3723 default: 3724 err = intel_pt_bug(decoder); 3725 break; 3726 } 3727 } while (err == -ENOLINK); 3728 3729 if (err) { 3730 decoder->state.err = intel_pt_ext_err(err); 3731 if (err != -EOVERFLOW) 3732 decoder->state.from_ip = decoder->ip; 3733 intel_pt_update_sample_time(decoder); 3734 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3735 intel_pt_set_nr(decoder); 3736 } else { 3737 decoder->state.err = 0; 3738 if (decoder->cbr != decoder->cbr_seen) { 3739 decoder->cbr_seen = decoder->cbr; 3740 if (!decoder->state.type) { 3741 decoder->state.from_ip = decoder->ip; 3742 decoder->state.to_ip = 0; 3743 } 3744 decoder->state.type |= INTEL_PT_CBR_CHG; 3745 decoder->state.cbr_payload = decoder->cbr_payload; 3746 decoder->state.cbr = decoder->cbr; 3747 } 3748 if (intel_pt_sample_time(decoder->pkt_state)) { 3749 intel_pt_update_sample_time(decoder); 3750 if (decoder->sample_cyc) { 3751 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3752 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3753 decoder->sample_cyc = false; 3754 } 3755 } 3756 /* 3757 * When using only TSC/MTC to compute cycles, IPC can be 3758 * sampled as soon as the cycle count changes. 3759 */ 3760 if (!decoder->have_cyc) 3761 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3762 } 3763 3764 /* Let PSB event always have TSC timestamp */ 3765 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 3766 decoder->sample_timestamp = decoder->tsc_timestamp; 3767 3768 decoder->state.from_nr = decoder->nr; 3769 decoder->state.to_nr = decoder->next_nr; 3770 decoder->nr = decoder->next_nr; 3771 3772 decoder->state.timestamp = decoder->sample_timestamp; 3773 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 3774 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 3775 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 3776 3777 return &decoder->state; 3778 } 3779 3780 /** 3781 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 3782 * @buf: pointer to buffer pointer 3783 * @len: size of buffer 3784 * 3785 * Updates the buffer pointer to point to the start of the next PSB packet if 3786 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 3787 * @len is adjusted accordingly. 3788 * 3789 * Return: %true if a PSB packet is found, %false otherwise. 3790 */ 3791 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 3792 { 3793 unsigned char *next; 3794 3795 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3796 if (next) { 3797 *len -= next - *buf; 3798 *buf = next; 3799 return true; 3800 } 3801 return false; 3802 } 3803 3804 /** 3805 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 3806 * packet. 3807 * @buf: pointer to buffer pointer 3808 * @len: size of buffer 3809 * 3810 * Updates the buffer pointer to point to the start of the following PSB packet 3811 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 3812 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 3813 * 3814 * Return: %true if a PSB packet is found, %false otherwise. 3815 */ 3816 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 3817 { 3818 unsigned char *next; 3819 3820 if (!*len) 3821 return false; 3822 3823 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3824 if (next) { 3825 *len -= next - *buf; 3826 *buf = next; 3827 return true; 3828 } 3829 return false; 3830 } 3831 3832 /** 3833 * intel_pt_last_psb - find the last PSB packet in a buffer. 3834 * @buf: buffer 3835 * @len: size of buffer 3836 * 3837 * This function finds the last PSB in a buffer. 3838 * 3839 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 3840 */ 3841 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 3842 { 3843 const char *n = INTEL_PT_PSB_STR; 3844 unsigned char *p; 3845 size_t k; 3846 3847 if (len < INTEL_PT_PSB_LEN) 3848 return NULL; 3849 3850 k = len - INTEL_PT_PSB_LEN + 1; 3851 while (1) { 3852 p = memrchr(buf, n[0], k); 3853 if (!p) 3854 return NULL; 3855 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 3856 return p; 3857 k = p - buf; 3858 if (!k) 3859 return NULL; 3860 } 3861 } 3862 3863 /** 3864 * intel_pt_next_tsc - find and return next TSC. 3865 * @buf: buffer 3866 * @len: size of buffer 3867 * @tsc: TSC value returned 3868 * @rem: returns remaining size when TSC is found 3869 * 3870 * Find a TSC packet in @buf and return the TSC value. This function assumes 3871 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 3872 * PSBEND packet is found. 3873 * 3874 * Return: %true if TSC is found, false otherwise. 3875 */ 3876 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 3877 size_t *rem) 3878 { 3879 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 3880 struct intel_pt_pkt packet; 3881 int ret; 3882 3883 while (len) { 3884 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 3885 if (ret <= 0) 3886 return false; 3887 if (packet.type == INTEL_PT_TSC) { 3888 *tsc = packet.payload; 3889 *rem = len; 3890 return true; 3891 } 3892 if (packet.type == INTEL_PT_PSBEND) 3893 return false; 3894 buf += ret; 3895 len -= ret; 3896 } 3897 return false; 3898 } 3899 3900 /** 3901 * intel_pt_tsc_cmp - compare 7-byte TSCs. 3902 * @tsc1: first TSC to compare 3903 * @tsc2: second TSC to compare 3904 * 3905 * This function compares 7-byte TSC values allowing for the possibility that 3906 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 3907 * around so for that purpose this function assumes the absolute difference is 3908 * less than half the maximum difference. 3909 * 3910 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 3911 * after @tsc2. 3912 */ 3913 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 3914 { 3915 const uint64_t halfway = (1ULL << 55); 3916 3917 if (tsc1 == tsc2) 3918 return 0; 3919 3920 if (tsc1 < tsc2) { 3921 if (tsc2 - tsc1 < halfway) 3922 return -1; 3923 else 3924 return 1; 3925 } else { 3926 if (tsc1 - tsc2 < halfway) 3927 return 1; 3928 else 3929 return -1; 3930 } 3931 } 3932 3933 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 3934 3935 /** 3936 * adj_for_padding - adjust overlap to account for padding. 3937 * @buf_b: second buffer 3938 * @buf_a: first buffer 3939 * @len_a: size of first buffer 3940 * 3941 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 3942 * accordingly. 3943 * 3944 * Return: A pointer into @buf_b from where non-overlapped data starts 3945 */ 3946 static unsigned char *adj_for_padding(unsigned char *buf_b, 3947 unsigned char *buf_a, size_t len_a) 3948 { 3949 unsigned char *p = buf_b - MAX_PADDING; 3950 unsigned char *q = buf_a + len_a - MAX_PADDING; 3951 int i; 3952 3953 for (i = MAX_PADDING; i; i--, p++, q++) { 3954 if (*p != *q) 3955 break; 3956 } 3957 3958 return p; 3959 } 3960 3961 /** 3962 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3963 * using TSC. 3964 * @buf_a: first buffer 3965 * @len_a: size of first buffer 3966 * @buf_b: second buffer 3967 * @len_b: size of second buffer 3968 * @consecutive: returns true if there is data in buf_b that is consecutive 3969 * to buf_a 3970 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 3971 * 3972 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3973 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3974 * walk forward in @buf_b until a later TSC is found. A precondition is that 3975 * @buf_a and @buf_b are positioned at a PSB. 3976 * 3977 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3978 * @buf_b + @len_b if there is no non-overlapped data. 3979 */ 3980 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3981 size_t len_a, 3982 unsigned char *buf_b, 3983 size_t len_b, bool *consecutive, 3984 bool ooo_tsc) 3985 { 3986 uint64_t tsc_a, tsc_b; 3987 unsigned char *p; 3988 size_t len, rem_a, rem_b; 3989 3990 p = intel_pt_last_psb(buf_a, len_a); 3991 if (!p) 3992 return buf_b; /* No PSB in buf_a => no overlap */ 3993 3994 len = len_a - (p - buf_a); 3995 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3996 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3997 len_a -= len; 3998 p = intel_pt_last_psb(buf_a, len_a); 3999 if (!p) 4000 return buf_b; /* No full PSB+ => assume no overlap */ 4001 len = len_a - (p - buf_a); 4002 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 4003 return buf_b; /* No TSC in buf_a => assume no overlap */ 4004 } 4005 4006 while (1) { 4007 /* Ignore PSB+ with no TSC */ 4008 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 4009 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 4010 4011 /* Same TSC, so buffers are consecutive */ 4012 if (!cmp && rem_b >= rem_a) { 4013 unsigned char *start; 4014 4015 *consecutive = true; 4016 start = buf_b + len_b - (rem_b - rem_a); 4017 return adj_for_padding(start, buf_a, len_a); 4018 } 4019 if (cmp < 0 && !ooo_tsc) 4020 return buf_b; /* tsc_a < tsc_b => no overlap */ 4021 } 4022 4023 if (!intel_pt_step_psb(&buf_b, &len_b)) 4024 return buf_b + len_b; /* No PSB in buf_b => no data */ 4025 } 4026 } 4027 4028 /** 4029 * intel_pt_find_overlap - determine start of non-overlapped trace data. 4030 * @buf_a: first buffer 4031 * @len_a: size of first buffer 4032 * @buf_b: second buffer 4033 * @len_b: size of second buffer 4034 * @have_tsc: can use TSC packets to detect overlap 4035 * @consecutive: returns true if there is data in buf_b that is consecutive 4036 * to buf_a 4037 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4038 * 4039 * When trace samples or snapshots are recorded there is the possibility that 4040 * the data overlaps. Note that, for the purposes of decoding, data is only 4041 * useful if it begins with a PSB packet. 4042 * 4043 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4044 * @buf_b + @len_b if there is no non-overlapped data. 4045 */ 4046 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 4047 unsigned char *buf_b, size_t len_b, 4048 bool have_tsc, bool *consecutive, 4049 bool ooo_tsc) 4050 { 4051 unsigned char *found; 4052 4053 /* Buffer 'b' must start at PSB so throw away everything before that */ 4054 if (!intel_pt_next_psb(&buf_b, &len_b)) 4055 return buf_b + len_b; /* No PSB */ 4056 4057 if (!intel_pt_next_psb(&buf_a, &len_a)) 4058 return buf_b; /* No overlap */ 4059 4060 if (have_tsc) { 4061 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 4062 consecutive, ooo_tsc); 4063 if (found) 4064 return found; 4065 } 4066 4067 /* 4068 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 4069 * we can ignore the first part of buffer 'a'. 4070 */ 4071 while (len_b < len_a) { 4072 if (!intel_pt_step_psb(&buf_a, &len_a)) 4073 return buf_b; /* No overlap */ 4074 } 4075 4076 /* Now len_b >= len_a */ 4077 while (1) { 4078 /* Potential overlap so check the bytes */ 4079 found = memmem(buf_a, len_a, buf_b, len_a); 4080 if (found) { 4081 *consecutive = true; 4082 return adj_for_padding(buf_b + len_a, buf_a, len_a); 4083 } 4084 4085 /* Try again at next PSB in buffer 'a' */ 4086 if (!intel_pt_step_psb(&buf_a, &len_a)) 4087 return buf_b; /* No overlap */ 4088 } 4089 } 4090 4091 /** 4092 * struct fast_forward_data - data used by intel_pt_ff_cb(). 4093 * @timestamp: timestamp to fast forward towards 4094 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 4095 * the fast forward timestamp. 4096 */ 4097 struct fast_forward_data { 4098 uint64_t timestamp; 4099 uint64_t buf_timestamp; 4100 }; 4101 4102 /** 4103 * intel_pt_ff_cb - fast forward lookahead callback. 4104 * @buffer: Intel PT trace buffer 4105 * @data: opaque pointer to fast forward data (struct fast_forward_data) 4106 * 4107 * Determine if @buffer trace is past the fast forward timestamp. 4108 * 4109 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 4110 * timestamp, and 0 otherwise. 4111 */ 4112 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 4113 { 4114 struct fast_forward_data *d = data; 4115 unsigned char *buf; 4116 uint64_t tsc; 4117 size_t rem; 4118 size_t len; 4119 4120 buf = (unsigned char *)buffer->buf; 4121 len = buffer->len; 4122 4123 if (!intel_pt_next_psb(&buf, &len) || 4124 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 4125 return 0; 4126 4127 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 4128 4129 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 4130 tsc, buffer->ref_timestamp); 4131 4132 /* 4133 * If the buffer contains a timestamp earlier that the fast forward 4134 * timestamp, then record it, else stop. 4135 */ 4136 if (tsc < d->timestamp) 4137 d->buf_timestamp = buffer->ref_timestamp; 4138 else 4139 return 1; 4140 4141 return 0; 4142 } 4143 4144 /** 4145 * intel_pt_fast_forward - reposition decoder forwards. 4146 * @decoder: Intel PT decoder 4147 * @timestamp: timestamp to fast forward towards 4148 * 4149 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 4150 * 4151 * Return: 0 on success or negative error code on failure. 4152 */ 4153 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 4154 { 4155 struct fast_forward_data d = { .timestamp = timestamp }; 4156 unsigned char *buf; 4157 size_t len; 4158 int err; 4159 4160 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 4161 4162 /* Find buffer timestamp of buffer to fast forward to */ 4163 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 4164 if (err < 0) 4165 return err; 4166 4167 /* Walk to buffer with same buffer timestamp */ 4168 if (d.buf_timestamp) { 4169 do { 4170 decoder->pos += decoder->len; 4171 decoder->len = 0; 4172 err = intel_pt_get_next_data(decoder, true); 4173 /* -ENOLINK means non-consecutive trace */ 4174 if (err && err != -ENOLINK) 4175 return err; 4176 } while (decoder->buf_timestamp != d.buf_timestamp); 4177 } 4178 4179 if (!decoder->buf) 4180 return 0; 4181 4182 buf = (unsigned char *)decoder->buf; 4183 len = decoder->len; 4184 4185 if (!intel_pt_next_psb(&buf, &len)) 4186 return 0; 4187 4188 /* 4189 * Walk PSBs while the PSB timestamp is less than the fast forward 4190 * timestamp. 4191 */ 4192 do { 4193 uint64_t tsc; 4194 size_t rem; 4195 4196 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 4197 break; 4198 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 4199 /* 4200 * A TSC packet can slip past MTC packets but, after fast 4201 * forward, decoding starts at the TSC timestamp. That means 4202 * the timestamps may not be exactly the same as the timestamps 4203 * that would have been decoded without fast forward. 4204 */ 4205 if (tsc < timestamp) { 4206 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 4207 decoder->pos += decoder->len - len; 4208 decoder->buf = buf; 4209 decoder->len = len; 4210 intel_pt_reposition(decoder); 4211 } else { 4212 break; 4213 } 4214 } while (intel_pt_step_psb(&buf, &len)); 4215 4216 return 0; 4217 } 4218