1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define SEVEN_BYTES 0xffffffffffffffULL 39 40 #define NO_VMCS 0xffffffffffULL 41 42 #define INTEL_PT_RETURN 1 43 44 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 45 #define INTEL_PT_MAX_LOOPS 10000 46 47 struct intel_pt_blk { 48 struct intel_pt_blk *prev; 49 uint64_t ip[INTEL_PT_BLK_SIZE]; 50 }; 51 52 struct intel_pt_stack { 53 struct intel_pt_blk *blk; 54 struct intel_pt_blk *spare; 55 int pos; 56 }; 57 58 enum intel_pt_p_once { 59 INTEL_PT_PRT_ONCE_UNK_VMCS, 60 INTEL_PT_PRT_ONCE_ERANGE, 61 }; 62 63 enum intel_pt_pkt_state { 64 INTEL_PT_STATE_NO_PSB, 65 INTEL_PT_STATE_NO_IP, 66 INTEL_PT_STATE_ERR_RESYNC, 67 INTEL_PT_STATE_IN_SYNC, 68 INTEL_PT_STATE_TNT_CONT, 69 INTEL_PT_STATE_TNT, 70 INTEL_PT_STATE_TIP, 71 INTEL_PT_STATE_TIP_PGD, 72 INTEL_PT_STATE_FUP, 73 INTEL_PT_STATE_FUP_NO_TIP, 74 INTEL_PT_STATE_FUP_IN_PSB, 75 INTEL_PT_STATE_RESAMPLE, 76 INTEL_PT_STATE_VM_TIME_CORRELATION, 77 }; 78 79 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 80 { 81 switch (pkt_state) { 82 case INTEL_PT_STATE_NO_PSB: 83 case INTEL_PT_STATE_NO_IP: 84 case INTEL_PT_STATE_ERR_RESYNC: 85 case INTEL_PT_STATE_IN_SYNC: 86 case INTEL_PT_STATE_TNT_CONT: 87 case INTEL_PT_STATE_RESAMPLE: 88 case INTEL_PT_STATE_VM_TIME_CORRELATION: 89 return true; 90 case INTEL_PT_STATE_TNT: 91 case INTEL_PT_STATE_TIP: 92 case INTEL_PT_STATE_TIP_PGD: 93 case INTEL_PT_STATE_FUP: 94 case INTEL_PT_STATE_FUP_NO_TIP: 95 case INTEL_PT_STATE_FUP_IN_PSB: 96 return false; 97 default: 98 return true; 99 }; 100 } 101 102 #ifdef INTEL_PT_STRICT 103 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 104 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 105 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 106 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 107 #else 108 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 109 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 110 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 111 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 112 #endif 113 114 struct intel_pt_decoder { 115 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 116 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 117 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 118 uint64_t max_insn_cnt, void *data); 119 bool (*pgd_ip)(uint64_t ip, void *data); 120 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 121 struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); 122 void *data; 123 struct intel_pt_state state; 124 const unsigned char *buf; 125 size_t len; 126 bool return_compression; 127 bool branch_enable; 128 bool mtc_insn; 129 bool pge; 130 bool have_tma; 131 bool have_cyc; 132 bool fixup_last_mtc; 133 bool have_last_ip; 134 bool in_psb; 135 bool hop; 136 bool leap; 137 bool vm_time_correlation; 138 bool vm_tm_corr_dry_run; 139 bool vm_tm_corr_reliable; 140 bool vm_tm_corr_same_buf; 141 bool vm_tm_corr_continuous; 142 bool nr; 143 bool next_nr; 144 enum intel_pt_param_flags flags; 145 uint64_t pos; 146 uint64_t last_ip; 147 uint64_t ip; 148 uint64_t pip_payload; 149 uint64_t timestamp; 150 uint64_t tsc_timestamp; 151 uint64_t ref_timestamp; 152 uint64_t buf_timestamp; 153 uint64_t sample_timestamp; 154 uint64_t ret_addr; 155 uint64_t ctc_timestamp; 156 uint64_t ctc_delta; 157 uint64_t cycle_cnt; 158 uint64_t cyc_ref_timestamp; 159 uint64_t first_timestamp; 160 uint64_t last_reliable_timestamp; 161 uint64_t vmcs; 162 uint64_t print_once; 163 uint64_t last_ctc; 164 uint32_t last_mtc; 165 uint32_t tsc_ctc_ratio_n; 166 uint32_t tsc_ctc_ratio_d; 167 uint32_t tsc_ctc_mult; 168 uint32_t tsc_slip; 169 uint32_t ctc_rem_mask; 170 int mtc_shift; 171 struct intel_pt_stack stack; 172 enum intel_pt_pkt_state pkt_state; 173 enum intel_pt_pkt_ctx pkt_ctx; 174 enum intel_pt_pkt_ctx prev_pkt_ctx; 175 enum intel_pt_blk_type blk_type; 176 int blk_type_pos; 177 struct intel_pt_pkt packet; 178 struct intel_pt_pkt tnt; 179 int pkt_step; 180 int pkt_len; 181 int last_packet_type; 182 unsigned int cbr; 183 unsigned int cbr_seen; 184 unsigned int max_non_turbo_ratio; 185 double max_non_turbo_ratio_fp; 186 double cbr_cyc_to_tsc; 187 double calc_cyc_to_tsc; 188 bool have_calc_cyc_to_tsc; 189 int exec_mode; 190 unsigned int insn_bytes; 191 uint64_t period; 192 enum intel_pt_period_type period_type; 193 uint64_t tot_insn_cnt; 194 uint64_t period_insn_cnt; 195 uint64_t period_mask; 196 uint64_t period_ticks; 197 uint64_t last_masked_timestamp; 198 uint64_t tot_cyc_cnt; 199 uint64_t sample_tot_cyc_cnt; 200 uint64_t base_cyc_cnt; 201 uint64_t cyc_cnt_timestamp; 202 uint64_t ctl; 203 uint64_t cyc_threshold; 204 double tsc_to_cyc; 205 bool continuous_period; 206 bool overflow; 207 bool set_fup_tx_flags; 208 bool set_fup_ptw; 209 bool set_fup_mwait; 210 bool set_fup_pwre; 211 bool set_fup_exstop; 212 bool set_fup_bep; 213 bool sample_cyc; 214 unsigned int fup_tx_flags; 215 unsigned int tx_flags; 216 uint64_t fup_ptw_payload; 217 uint64_t fup_mwait_payload; 218 uint64_t fup_pwre_payload; 219 uint64_t cbr_payload; 220 uint64_t timestamp_insn_cnt; 221 uint64_t sample_insn_cnt; 222 uint64_t stuck_ip; 223 int no_progress; 224 int stuck_ip_prd; 225 int stuck_ip_cnt; 226 uint64_t psb_ip; 227 const unsigned char *next_buf; 228 size_t next_len; 229 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 230 }; 231 232 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 233 { 234 int i; 235 236 for (i = 0; x != 1; i++) 237 x >>= 1; 238 239 return x << i; 240 } 241 242 __printf(1, 2) 243 static void p_log(const char *fmt, ...) 244 { 245 char buf[512]; 246 va_list args; 247 248 va_start(args, fmt); 249 vsnprintf(buf, sizeof(buf), fmt, args); 250 va_end(args); 251 252 fprintf(stderr, "%s\n", buf); 253 intel_pt_log("%s\n", buf); 254 } 255 256 static bool intel_pt_print_once(struct intel_pt_decoder *decoder, 257 enum intel_pt_p_once id) 258 { 259 uint64_t bit = 1ULL << id; 260 261 if (decoder->print_once & bit) 262 return false; 263 decoder->print_once |= bit; 264 return true; 265 } 266 267 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 268 { 269 if (!(ctl & INTEL_PT_CYC_ENABLE)) 270 return 0; 271 272 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 273 } 274 275 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 276 { 277 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 278 uint64_t period; 279 280 period = intel_pt_lower_power_of_2(decoder->period); 281 decoder->period_mask = ~(period - 1); 282 decoder->period_ticks = period; 283 } 284 } 285 286 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 287 { 288 if (!d) 289 return 0; 290 return (t / d) * n + ((t % d) * n) / d; 291 } 292 293 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 294 { 295 struct intel_pt_decoder *decoder; 296 297 if (!params->get_trace || !params->walk_insn) 298 return NULL; 299 300 decoder = zalloc(sizeof(struct intel_pt_decoder)); 301 if (!decoder) 302 return NULL; 303 304 decoder->get_trace = params->get_trace; 305 decoder->walk_insn = params->walk_insn; 306 decoder->pgd_ip = params->pgd_ip; 307 decoder->lookahead = params->lookahead; 308 decoder->findnew_vmcs_info = params->findnew_vmcs_info; 309 decoder->data = params->data; 310 decoder->return_compression = params->return_compression; 311 decoder->branch_enable = params->branch_enable; 312 decoder->hop = params->quick >= 1; 313 decoder->leap = params->quick >= 2; 314 decoder->vm_time_correlation = params->vm_time_correlation; 315 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; 316 decoder->first_timestamp = params->first_timestamp; 317 decoder->last_reliable_timestamp = params->first_timestamp; 318 319 decoder->flags = params->flags; 320 321 decoder->ctl = params->ctl; 322 decoder->period = params->period; 323 decoder->period_type = params->period_type; 324 325 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 326 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 327 328 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 329 330 intel_pt_setup_period(decoder); 331 332 decoder->mtc_shift = params->mtc_period; 333 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 334 335 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 336 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 337 338 if (!decoder->tsc_ctc_ratio_n) 339 decoder->tsc_ctc_ratio_d = 0; 340 341 if (decoder->tsc_ctc_ratio_d) { 342 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 343 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 344 decoder->tsc_ctc_ratio_d; 345 } 346 347 /* 348 * A TSC packet can slip past MTC packets so that the timestamp appears 349 * to go backwards. One estimate is that can be up to about 40 CPU 350 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 351 * slippage an order of magnitude more to be on the safe side. 352 */ 353 decoder->tsc_slip = 0x10000; 354 355 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 356 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 357 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 358 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 359 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 360 361 if (decoder->hop) 362 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 363 364 return decoder; 365 } 366 367 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, 368 uint64_t first_timestamp) 369 { 370 decoder->first_timestamp = first_timestamp; 371 } 372 373 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 374 { 375 struct intel_pt_blk *blk = stack->blk; 376 377 stack->blk = blk->prev; 378 if (!stack->spare) 379 stack->spare = blk; 380 else 381 free(blk); 382 } 383 384 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 385 { 386 if (!stack->pos) { 387 if (!stack->blk) 388 return 0; 389 intel_pt_pop_blk(stack); 390 if (!stack->blk) 391 return 0; 392 stack->pos = INTEL_PT_BLK_SIZE; 393 } 394 return stack->blk->ip[--stack->pos]; 395 } 396 397 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 398 { 399 struct intel_pt_blk *blk; 400 401 if (stack->spare) { 402 blk = stack->spare; 403 stack->spare = NULL; 404 } else { 405 blk = malloc(sizeof(struct intel_pt_blk)); 406 if (!blk) 407 return -ENOMEM; 408 } 409 410 blk->prev = stack->blk; 411 stack->blk = blk; 412 stack->pos = 0; 413 return 0; 414 } 415 416 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 417 { 418 int err; 419 420 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 421 err = intel_pt_alloc_blk(stack); 422 if (err) 423 return err; 424 } 425 426 stack->blk->ip[stack->pos++] = ip; 427 return 0; 428 } 429 430 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 431 { 432 while (stack->blk) 433 intel_pt_pop_blk(stack); 434 stack->pos = 0; 435 } 436 437 static void intel_pt_free_stack(struct intel_pt_stack *stack) 438 { 439 intel_pt_clear_stack(stack); 440 zfree(&stack->blk); 441 zfree(&stack->spare); 442 } 443 444 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 445 { 446 intel_pt_free_stack(&decoder->stack); 447 free(decoder); 448 } 449 450 static int intel_pt_ext_err(int code) 451 { 452 switch (code) { 453 case -ENOMEM: 454 return INTEL_PT_ERR_NOMEM; 455 case -ENOSYS: 456 return INTEL_PT_ERR_INTERN; 457 case -EBADMSG: 458 return INTEL_PT_ERR_BADPKT; 459 case -ENODATA: 460 return INTEL_PT_ERR_NODATA; 461 case -EILSEQ: 462 return INTEL_PT_ERR_NOINSN; 463 case -ENOENT: 464 return INTEL_PT_ERR_MISMAT; 465 case -EOVERFLOW: 466 return INTEL_PT_ERR_OVR; 467 case -ENOSPC: 468 return INTEL_PT_ERR_LOST; 469 case -ELOOP: 470 return INTEL_PT_ERR_NELOOP; 471 default: 472 return INTEL_PT_ERR_UNK; 473 } 474 } 475 476 static const char *intel_pt_err_msgs[] = { 477 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 478 [INTEL_PT_ERR_INTERN] = "Internal error", 479 [INTEL_PT_ERR_BADPKT] = "Bad packet", 480 [INTEL_PT_ERR_NODATA] = "No more data", 481 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 482 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 483 [INTEL_PT_ERR_OVR] = "Overflow packet", 484 [INTEL_PT_ERR_LOST] = "Lost trace data", 485 [INTEL_PT_ERR_UNK] = "Unknown error!", 486 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 487 }; 488 489 int intel_pt__strerror(int code, char *buf, size_t buflen) 490 { 491 if (code < 1 || code >= INTEL_PT_ERR_MAX) 492 code = INTEL_PT_ERR_UNK; 493 strlcpy(buf, intel_pt_err_msgs[code], buflen); 494 return 0; 495 } 496 497 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 498 uint64_t last_ip) 499 { 500 uint64_t ip; 501 502 switch (packet->count) { 503 case 1: 504 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 505 packet->payload; 506 break; 507 case 2: 508 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 509 packet->payload; 510 break; 511 case 3: 512 ip = packet->payload; 513 /* Sign-extend 6-byte ip */ 514 if (ip & (uint64_t)0x800000000000ULL) 515 ip |= (uint64_t)0xffff000000000000ULL; 516 break; 517 case 4: 518 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 519 packet->payload; 520 break; 521 case 6: 522 ip = packet->payload; 523 break; 524 default: 525 return 0; 526 } 527 528 return ip; 529 } 530 531 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 532 { 533 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 534 decoder->have_last_ip = true; 535 } 536 537 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 538 { 539 intel_pt_set_last_ip(decoder); 540 decoder->ip = decoder->last_ip; 541 } 542 543 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 544 { 545 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 546 decoder->buf); 547 } 548 549 static int intel_pt_bug(struct intel_pt_decoder *decoder) 550 { 551 intel_pt_log("ERROR: Internal error\n"); 552 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 553 return -ENOSYS; 554 } 555 556 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 557 { 558 decoder->tx_flags = 0; 559 } 560 561 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 562 { 563 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 564 } 565 566 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 567 { 568 decoder->pip_payload = decoder->packet.payload; 569 } 570 571 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 572 { 573 decoder->next_nr = decoder->pip_payload & 1; 574 } 575 576 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 577 { 578 decoder->nr = decoder->pip_payload & 1; 579 decoder->next_nr = decoder->nr; 580 } 581 582 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 583 { 584 intel_pt_update_pip(decoder); 585 intel_pt_set_nr(decoder); 586 } 587 588 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 589 { 590 intel_pt_clear_tx_flags(decoder); 591 decoder->have_tma = false; 592 decoder->pkt_len = 1; 593 decoder->pkt_step = 1; 594 intel_pt_decoder_log_packet(decoder); 595 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 596 intel_pt_log("ERROR: Bad packet\n"); 597 decoder->pkt_state = INTEL_PT_STATE_ERR1; 598 } 599 return -EBADMSG; 600 } 601 602 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 603 { 604 decoder->sample_timestamp = decoder->timestamp; 605 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 606 } 607 608 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 609 { 610 decoder->ip = 0; 611 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 612 decoder->timestamp = 0; 613 decoder->have_tma = false; 614 } 615 616 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 617 { 618 struct intel_pt_buffer buffer = { .buf = 0, }; 619 int ret; 620 621 decoder->pkt_step = 0; 622 623 intel_pt_log("Getting more data\n"); 624 ret = decoder->get_trace(&buffer, decoder->data); 625 if (ret) 626 return ret; 627 decoder->buf = buffer.buf; 628 decoder->len = buffer.len; 629 if (!decoder->len) { 630 intel_pt_log("No more data\n"); 631 return -ENODATA; 632 } 633 decoder->buf_timestamp = buffer.ref_timestamp; 634 if (!buffer.consecutive || reposition) { 635 intel_pt_reposition(decoder); 636 decoder->ref_timestamp = buffer.ref_timestamp; 637 decoder->state.trace_nr = buffer.trace_nr; 638 decoder->vm_tm_corr_same_buf = false; 639 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 640 decoder->ref_timestamp); 641 return -ENOLINK; 642 } 643 644 return 0; 645 } 646 647 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 648 bool reposition) 649 { 650 if (!decoder->next_buf) 651 return intel_pt_get_data(decoder, reposition); 652 653 decoder->buf = decoder->next_buf; 654 decoder->len = decoder->next_len; 655 decoder->next_buf = 0; 656 decoder->next_len = 0; 657 return 0; 658 } 659 660 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 661 { 662 unsigned char *buf = decoder->temp_buf; 663 size_t old_len, len, n; 664 int ret; 665 666 old_len = decoder->len; 667 len = decoder->len; 668 memcpy(buf, decoder->buf, len); 669 670 ret = intel_pt_get_data(decoder, false); 671 if (ret) { 672 decoder->pos += old_len; 673 return ret < 0 ? ret : -EINVAL; 674 } 675 676 n = INTEL_PT_PKT_MAX_SZ - len; 677 if (n > decoder->len) 678 n = decoder->len; 679 memcpy(buf + len, decoder->buf, n); 680 len += n; 681 682 decoder->prev_pkt_ctx = decoder->pkt_ctx; 683 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 684 if (ret < (int)old_len) { 685 decoder->next_buf = decoder->buf; 686 decoder->next_len = decoder->len; 687 decoder->buf = buf; 688 decoder->len = old_len; 689 return intel_pt_bad_packet(decoder); 690 } 691 692 decoder->next_buf = decoder->buf + (ret - old_len); 693 decoder->next_len = decoder->len - (ret - old_len); 694 695 decoder->buf = buf; 696 decoder->len = ret; 697 698 return ret; 699 } 700 701 struct intel_pt_pkt_info { 702 struct intel_pt_decoder *decoder; 703 struct intel_pt_pkt packet; 704 uint64_t pos; 705 int pkt_len; 706 int last_packet_type; 707 void *data; 708 }; 709 710 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 711 712 /* Lookahead packets in current buffer */ 713 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 714 intel_pt_pkt_cb_t cb, void *data) 715 { 716 struct intel_pt_pkt_info pkt_info; 717 const unsigned char *buf = decoder->buf; 718 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 719 size_t len = decoder->len; 720 int ret; 721 722 pkt_info.decoder = decoder; 723 pkt_info.pos = decoder->pos; 724 pkt_info.pkt_len = decoder->pkt_step; 725 pkt_info.last_packet_type = decoder->last_packet_type; 726 pkt_info.data = data; 727 728 while (1) { 729 do { 730 pkt_info.pos += pkt_info.pkt_len; 731 buf += pkt_info.pkt_len; 732 len -= pkt_info.pkt_len; 733 734 if (!len) 735 return INTEL_PT_NEED_MORE_BYTES; 736 737 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 738 &pkt_ctx); 739 if (!ret) 740 return INTEL_PT_NEED_MORE_BYTES; 741 if (ret < 0) 742 return ret; 743 744 pkt_info.pkt_len = ret; 745 } while (pkt_info.packet.type == INTEL_PT_PAD); 746 747 ret = cb(&pkt_info); 748 if (ret) 749 return 0; 750 751 pkt_info.last_packet_type = pkt_info.packet.type; 752 } 753 } 754 755 struct intel_pt_calc_cyc_to_tsc_info { 756 uint64_t cycle_cnt; 757 unsigned int cbr; 758 uint32_t last_mtc; 759 uint64_t ctc_timestamp; 760 uint64_t ctc_delta; 761 uint64_t tsc_timestamp; 762 uint64_t timestamp; 763 bool have_tma; 764 bool fixup_last_mtc; 765 bool from_mtc; 766 double cbr_cyc_to_tsc; 767 }; 768 769 /* 770 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 771 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 772 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 773 * packet by copying the missing bits from the current MTC assuming the least 774 * difference between the two, and that the current MTC comes after last_mtc. 775 */ 776 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 777 uint32_t *last_mtc) 778 { 779 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 780 uint32_t mask = ~(first_missing_bit - 1); 781 782 *last_mtc |= mtc & mask; 783 if (*last_mtc >= mtc) { 784 *last_mtc -= first_missing_bit; 785 *last_mtc &= 0xff; 786 } 787 } 788 789 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 790 { 791 struct intel_pt_decoder *decoder = pkt_info->decoder; 792 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 793 uint64_t timestamp; 794 double cyc_to_tsc; 795 unsigned int cbr; 796 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 797 798 switch (pkt_info->packet.type) { 799 case INTEL_PT_TNT: 800 case INTEL_PT_TIP_PGE: 801 case INTEL_PT_TIP: 802 case INTEL_PT_FUP: 803 case INTEL_PT_PSB: 804 case INTEL_PT_PIP: 805 case INTEL_PT_MODE_EXEC: 806 case INTEL_PT_MODE_TSX: 807 case INTEL_PT_PSBEND: 808 case INTEL_PT_PAD: 809 case INTEL_PT_VMCS: 810 case INTEL_PT_MNT: 811 case INTEL_PT_PTWRITE: 812 case INTEL_PT_PTWRITE_IP: 813 case INTEL_PT_BBP: 814 case INTEL_PT_BIP: 815 case INTEL_PT_BEP: 816 case INTEL_PT_BEP_IP: 817 return 0; 818 819 case INTEL_PT_MTC: 820 if (!data->have_tma) 821 return 0; 822 823 mtc = pkt_info->packet.payload; 824 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 825 data->fixup_last_mtc = false; 826 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 827 &data->last_mtc); 828 } 829 if (mtc > data->last_mtc) 830 mtc_delta = mtc - data->last_mtc; 831 else 832 mtc_delta = mtc + 256 - data->last_mtc; 833 data->ctc_delta += mtc_delta << decoder->mtc_shift; 834 data->last_mtc = mtc; 835 836 if (decoder->tsc_ctc_mult) { 837 timestamp = data->ctc_timestamp + 838 data->ctc_delta * decoder->tsc_ctc_mult; 839 } else { 840 timestamp = data->ctc_timestamp + 841 multdiv(data->ctc_delta, 842 decoder->tsc_ctc_ratio_n, 843 decoder->tsc_ctc_ratio_d); 844 } 845 846 if (timestamp < data->timestamp) 847 return 1; 848 849 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 850 data->timestamp = timestamp; 851 return 0; 852 } 853 854 break; 855 856 case INTEL_PT_TSC: 857 /* 858 * For now, do not support using TSC packets - refer 859 * intel_pt_calc_cyc_to_tsc(). 860 */ 861 if (data->from_mtc) 862 return 1; 863 timestamp = pkt_info->packet.payload | 864 (data->timestamp & (0xffULL << 56)); 865 if (data->from_mtc && timestamp < data->timestamp && 866 data->timestamp - timestamp < decoder->tsc_slip) 867 return 1; 868 if (timestamp < data->timestamp) 869 timestamp += (1ULL << 56); 870 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 871 if (data->from_mtc) 872 return 1; 873 data->tsc_timestamp = timestamp; 874 data->timestamp = timestamp; 875 return 0; 876 } 877 break; 878 879 case INTEL_PT_TMA: 880 if (data->from_mtc) 881 return 1; 882 883 if (!decoder->tsc_ctc_ratio_d) 884 return 0; 885 886 ctc = pkt_info->packet.payload; 887 fc = pkt_info->packet.count; 888 ctc_rem = ctc & decoder->ctc_rem_mask; 889 890 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 891 892 data->ctc_timestamp = data->tsc_timestamp - fc; 893 if (decoder->tsc_ctc_mult) { 894 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 895 } else { 896 data->ctc_timestamp -= 897 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 898 decoder->tsc_ctc_ratio_d); 899 } 900 901 data->ctc_delta = 0; 902 data->have_tma = true; 903 data->fixup_last_mtc = true; 904 905 return 0; 906 907 case INTEL_PT_CYC: 908 data->cycle_cnt += pkt_info->packet.payload; 909 return 0; 910 911 case INTEL_PT_CBR: 912 cbr = pkt_info->packet.payload; 913 if (data->cbr && data->cbr != cbr) 914 return 1; 915 data->cbr = cbr; 916 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 917 return 0; 918 919 case INTEL_PT_TIP_PGD: 920 case INTEL_PT_TRACESTOP: 921 case INTEL_PT_EXSTOP: 922 case INTEL_PT_EXSTOP_IP: 923 case INTEL_PT_MWAIT: 924 case INTEL_PT_PWRE: 925 case INTEL_PT_PWRX: 926 case INTEL_PT_OVF: 927 case INTEL_PT_BAD: /* Does not happen */ 928 default: 929 return 1; 930 } 931 932 if (!data->cbr && decoder->cbr) { 933 data->cbr = decoder->cbr; 934 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 935 } 936 937 if (!data->cycle_cnt) 938 return 1; 939 940 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 941 942 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 943 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 944 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 945 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 946 return 1; 947 } 948 949 decoder->calc_cyc_to_tsc = cyc_to_tsc; 950 decoder->have_calc_cyc_to_tsc = true; 951 952 if (data->cbr) { 953 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 954 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 955 } else { 956 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 957 cyc_to_tsc, pkt_info->pos); 958 } 959 960 return 1; 961 } 962 963 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 964 bool from_mtc) 965 { 966 struct intel_pt_calc_cyc_to_tsc_info data = { 967 .cycle_cnt = 0, 968 .cbr = 0, 969 .last_mtc = decoder->last_mtc, 970 .ctc_timestamp = decoder->ctc_timestamp, 971 .ctc_delta = decoder->ctc_delta, 972 .tsc_timestamp = decoder->tsc_timestamp, 973 .timestamp = decoder->timestamp, 974 .have_tma = decoder->have_tma, 975 .fixup_last_mtc = decoder->fixup_last_mtc, 976 .from_mtc = from_mtc, 977 .cbr_cyc_to_tsc = 0, 978 }; 979 980 /* 981 * For now, do not support using TSC packets for at least the reasons: 982 * 1) timing might have stopped 983 * 2) TSC packets within PSB+ can slip against CYC packets 984 */ 985 if (!from_mtc) 986 return; 987 988 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 989 } 990 991 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 992 { 993 int ret; 994 995 decoder->last_packet_type = decoder->packet.type; 996 997 do { 998 decoder->pos += decoder->pkt_step; 999 decoder->buf += decoder->pkt_step; 1000 decoder->len -= decoder->pkt_step; 1001 1002 if (!decoder->len) { 1003 ret = intel_pt_get_next_data(decoder, false); 1004 if (ret) 1005 return ret; 1006 } 1007 1008 decoder->prev_pkt_ctx = decoder->pkt_ctx; 1009 ret = intel_pt_get_packet(decoder->buf, decoder->len, 1010 &decoder->packet, &decoder->pkt_ctx); 1011 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 1012 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 1013 ret = intel_pt_get_split_packet(decoder); 1014 if (ret < 0) 1015 return ret; 1016 } 1017 if (ret <= 0) 1018 return intel_pt_bad_packet(decoder); 1019 1020 decoder->pkt_len = ret; 1021 decoder->pkt_step = ret; 1022 intel_pt_decoder_log_packet(decoder); 1023 } while (decoder->packet.type == INTEL_PT_PAD); 1024 1025 return 0; 1026 } 1027 1028 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 1029 { 1030 uint64_t timestamp, masked_timestamp; 1031 1032 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1033 masked_timestamp = timestamp & decoder->period_mask; 1034 if (decoder->continuous_period) { 1035 if (masked_timestamp > decoder->last_masked_timestamp) 1036 return 1; 1037 } else { 1038 timestamp += 1; 1039 masked_timestamp = timestamp & decoder->period_mask; 1040 if (masked_timestamp > decoder->last_masked_timestamp) { 1041 decoder->last_masked_timestamp = masked_timestamp; 1042 decoder->continuous_period = true; 1043 } 1044 } 1045 1046 if (masked_timestamp < decoder->last_masked_timestamp) 1047 return decoder->period_ticks; 1048 1049 return decoder->period_ticks - (timestamp - masked_timestamp); 1050 } 1051 1052 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 1053 { 1054 switch (decoder->period_type) { 1055 case INTEL_PT_PERIOD_INSTRUCTIONS: 1056 return decoder->period - decoder->period_insn_cnt; 1057 case INTEL_PT_PERIOD_TICKS: 1058 return intel_pt_next_period(decoder); 1059 case INTEL_PT_PERIOD_NONE: 1060 case INTEL_PT_PERIOD_MTC: 1061 default: 1062 return 0; 1063 } 1064 } 1065 1066 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1067 { 1068 uint64_t timestamp, masked_timestamp; 1069 1070 switch (decoder->period_type) { 1071 case INTEL_PT_PERIOD_INSTRUCTIONS: 1072 decoder->period_insn_cnt = 0; 1073 break; 1074 case INTEL_PT_PERIOD_TICKS: 1075 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1076 masked_timestamp = timestamp & decoder->period_mask; 1077 if (masked_timestamp > decoder->last_masked_timestamp) 1078 decoder->last_masked_timestamp = masked_timestamp; 1079 else 1080 decoder->last_masked_timestamp += decoder->period_ticks; 1081 break; 1082 case INTEL_PT_PERIOD_NONE: 1083 case INTEL_PT_PERIOD_MTC: 1084 default: 1085 break; 1086 } 1087 1088 decoder->state.type |= INTEL_PT_INSTRUCTION; 1089 } 1090 1091 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1092 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1093 { 1094 uint64_t max_insn_cnt, insn_cnt = 0; 1095 int err; 1096 1097 if (!decoder->mtc_insn) 1098 decoder->mtc_insn = true; 1099 1100 max_insn_cnt = intel_pt_next_sample(decoder); 1101 1102 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1103 max_insn_cnt, decoder->data); 1104 1105 decoder->tot_insn_cnt += insn_cnt; 1106 decoder->timestamp_insn_cnt += insn_cnt; 1107 decoder->sample_insn_cnt += insn_cnt; 1108 decoder->period_insn_cnt += insn_cnt; 1109 1110 if (err) { 1111 decoder->no_progress = 0; 1112 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1113 intel_pt_log_at("ERROR: Failed to get instruction", 1114 decoder->ip); 1115 if (err == -ENOENT) 1116 return -ENOLINK; 1117 return -EILSEQ; 1118 } 1119 1120 if (ip && decoder->ip == ip) { 1121 err = -EAGAIN; 1122 goto out; 1123 } 1124 1125 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1126 intel_pt_sample_insn(decoder); 1127 1128 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1129 decoder->state.type = INTEL_PT_INSTRUCTION; 1130 decoder->state.from_ip = decoder->ip; 1131 decoder->state.to_ip = 0; 1132 decoder->ip += intel_pt_insn->length; 1133 err = INTEL_PT_RETURN; 1134 goto out; 1135 } 1136 1137 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1138 /* Zero-length calls are excluded */ 1139 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1140 intel_pt_insn->rel) { 1141 err = intel_pt_push(&decoder->stack, decoder->ip + 1142 intel_pt_insn->length); 1143 if (err) 1144 goto out; 1145 } 1146 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1147 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1148 } 1149 1150 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1151 int cnt = decoder->no_progress++; 1152 1153 decoder->state.from_ip = decoder->ip; 1154 decoder->ip += intel_pt_insn->length + 1155 intel_pt_insn->rel; 1156 decoder->state.to_ip = decoder->ip; 1157 err = INTEL_PT_RETURN; 1158 1159 /* 1160 * Check for being stuck in a loop. This can happen if a 1161 * decoder error results in the decoder erroneously setting the 1162 * ip to an address that is itself in an infinite loop that 1163 * consumes no packets. When that happens, there must be an 1164 * unconditional branch. 1165 */ 1166 if (cnt) { 1167 if (cnt == 1) { 1168 decoder->stuck_ip = decoder->state.to_ip; 1169 decoder->stuck_ip_prd = 1; 1170 decoder->stuck_ip_cnt = 1; 1171 } else if (cnt > INTEL_PT_MAX_LOOPS || 1172 decoder->state.to_ip == decoder->stuck_ip) { 1173 intel_pt_log_at("ERROR: Never-ending loop", 1174 decoder->state.to_ip); 1175 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1176 err = -ELOOP; 1177 goto out; 1178 } else if (!--decoder->stuck_ip_cnt) { 1179 decoder->stuck_ip_prd += 1; 1180 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1181 decoder->stuck_ip = decoder->state.to_ip; 1182 } 1183 } 1184 goto out_no_progress; 1185 } 1186 out: 1187 decoder->no_progress = 0; 1188 out_no_progress: 1189 decoder->state.insn_op = intel_pt_insn->op; 1190 decoder->state.insn_len = intel_pt_insn->length; 1191 memcpy(decoder->state.insn, intel_pt_insn->buf, 1192 INTEL_PT_INSN_BUF_SZ); 1193 1194 if (decoder->tx_flags & INTEL_PT_IN_TX) 1195 decoder->state.flags |= INTEL_PT_IN_TX; 1196 1197 return err; 1198 } 1199 1200 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1201 { 1202 bool ret = false; 1203 1204 if (decoder->set_fup_tx_flags) { 1205 decoder->set_fup_tx_flags = false; 1206 decoder->tx_flags = decoder->fup_tx_flags; 1207 decoder->state.type = INTEL_PT_TRANSACTION; 1208 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) 1209 decoder->state.type |= INTEL_PT_BRANCH; 1210 decoder->state.from_ip = decoder->ip; 1211 decoder->state.to_ip = 0; 1212 decoder->state.flags = decoder->fup_tx_flags; 1213 return true; 1214 } 1215 if (decoder->set_fup_ptw) { 1216 decoder->set_fup_ptw = false; 1217 decoder->state.type = INTEL_PT_PTW; 1218 decoder->state.flags |= INTEL_PT_FUP_IP; 1219 decoder->state.from_ip = decoder->ip; 1220 decoder->state.to_ip = 0; 1221 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1222 return true; 1223 } 1224 if (decoder->set_fup_mwait) { 1225 decoder->set_fup_mwait = false; 1226 decoder->state.type = INTEL_PT_MWAIT_OP; 1227 decoder->state.from_ip = decoder->ip; 1228 decoder->state.to_ip = 0; 1229 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1230 ret = true; 1231 } 1232 if (decoder->set_fup_pwre) { 1233 decoder->set_fup_pwre = false; 1234 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1235 decoder->state.type &= ~INTEL_PT_BRANCH; 1236 decoder->state.from_ip = decoder->ip; 1237 decoder->state.to_ip = 0; 1238 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1239 ret = true; 1240 } 1241 if (decoder->set_fup_exstop) { 1242 decoder->set_fup_exstop = false; 1243 decoder->state.type |= INTEL_PT_EX_STOP; 1244 decoder->state.type &= ~INTEL_PT_BRANCH; 1245 decoder->state.flags |= INTEL_PT_FUP_IP; 1246 decoder->state.from_ip = decoder->ip; 1247 decoder->state.to_ip = 0; 1248 ret = true; 1249 } 1250 if (decoder->set_fup_bep) { 1251 decoder->set_fup_bep = false; 1252 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1253 decoder->state.type &= ~INTEL_PT_BRANCH; 1254 decoder->state.from_ip = decoder->ip; 1255 decoder->state.to_ip = 0; 1256 ret = true; 1257 } 1258 return ret; 1259 } 1260 1261 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1262 struct intel_pt_insn *intel_pt_insn, 1263 uint64_t ip, int err) 1264 { 1265 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1266 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1267 ip == decoder->ip + intel_pt_insn->length; 1268 } 1269 1270 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1271 { 1272 struct intel_pt_insn intel_pt_insn; 1273 uint64_t ip; 1274 int err; 1275 1276 ip = decoder->last_ip; 1277 1278 while (1) { 1279 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1280 if (err == INTEL_PT_RETURN) 1281 return 0; 1282 if (err == -EAGAIN || 1283 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1284 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; 1285 1286 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1287 if (intel_pt_fup_event(decoder) && no_tip) 1288 return 0; 1289 return -EAGAIN; 1290 } 1291 decoder->set_fup_tx_flags = false; 1292 if (err) 1293 return err; 1294 1295 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1296 intel_pt_log_at("ERROR: Unexpected indirect branch", 1297 decoder->ip); 1298 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1299 return -ENOENT; 1300 } 1301 1302 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1303 intel_pt_log_at("ERROR: Unexpected conditional branch", 1304 decoder->ip); 1305 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1306 return -ENOENT; 1307 } 1308 1309 intel_pt_bug(decoder); 1310 } 1311 } 1312 1313 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1314 { 1315 struct intel_pt_insn intel_pt_insn; 1316 int err; 1317 1318 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1319 if (err == INTEL_PT_RETURN && 1320 decoder->pgd_ip && 1321 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1322 (decoder->state.type & INTEL_PT_BRANCH) && 1323 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1324 /* Unconditional branch leaving filter region */ 1325 decoder->no_progress = 0; 1326 decoder->pge = false; 1327 decoder->continuous_period = false; 1328 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1329 decoder->state.type |= INTEL_PT_TRACE_END; 1330 intel_pt_update_nr(decoder); 1331 return 0; 1332 } 1333 if (err == INTEL_PT_RETURN) 1334 return 0; 1335 if (err) 1336 return err; 1337 1338 intel_pt_update_nr(decoder); 1339 1340 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1341 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1342 decoder->pge = false; 1343 decoder->continuous_period = false; 1344 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1345 decoder->state.from_ip = decoder->ip; 1346 if (decoder->packet.count == 0) { 1347 decoder->state.to_ip = 0; 1348 } else { 1349 decoder->state.to_ip = decoder->last_ip; 1350 decoder->ip = decoder->last_ip; 1351 } 1352 decoder->state.type |= INTEL_PT_TRACE_END; 1353 } else { 1354 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1355 decoder->state.from_ip = decoder->ip; 1356 if (decoder->packet.count == 0) { 1357 decoder->state.to_ip = 0; 1358 } else { 1359 decoder->state.to_ip = decoder->last_ip; 1360 decoder->ip = decoder->last_ip; 1361 } 1362 } 1363 return 0; 1364 } 1365 1366 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1367 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1368 intel_pt_insn.rel; 1369 1370 if (decoder->pgd_ip && 1371 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1372 decoder->pgd_ip(to_ip, decoder->data)) { 1373 /* Conditional branch leaving filter region */ 1374 decoder->pge = false; 1375 decoder->continuous_period = false; 1376 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1377 decoder->ip = to_ip; 1378 decoder->state.from_ip = decoder->ip; 1379 decoder->state.to_ip = to_ip; 1380 decoder->state.type |= INTEL_PT_TRACE_END; 1381 return 0; 1382 } 1383 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1384 decoder->ip); 1385 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1386 return -ENOENT; 1387 } 1388 1389 return intel_pt_bug(decoder); 1390 } 1391 1392 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1393 { 1394 struct intel_pt_insn intel_pt_insn; 1395 int err; 1396 1397 while (1) { 1398 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1399 if (err == INTEL_PT_RETURN) 1400 return 0; 1401 if (err) 1402 return err; 1403 1404 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1405 if (!decoder->return_compression) { 1406 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1407 decoder->ip); 1408 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1409 return -ENOENT; 1410 } 1411 if (!decoder->ret_addr) { 1412 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1413 decoder->ip); 1414 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1415 return -ENOENT; 1416 } 1417 if (!(decoder->tnt.payload & BIT63)) { 1418 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1419 decoder->ip); 1420 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1421 return -ENOENT; 1422 } 1423 decoder->tnt.count -= 1; 1424 if (decoder->tnt.count) 1425 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1426 else 1427 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1428 decoder->tnt.payload <<= 1; 1429 decoder->state.from_ip = decoder->ip; 1430 decoder->ip = decoder->ret_addr; 1431 decoder->state.to_ip = decoder->ip; 1432 return 0; 1433 } 1434 1435 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1436 /* Handle deferred TIPs */ 1437 err = intel_pt_get_next_packet(decoder); 1438 if (err) 1439 return err; 1440 if (decoder->packet.type != INTEL_PT_TIP || 1441 decoder->packet.count == 0) { 1442 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1443 decoder->ip); 1444 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1445 decoder->pkt_step = 0; 1446 return -ENOENT; 1447 } 1448 intel_pt_set_last_ip(decoder); 1449 decoder->state.from_ip = decoder->ip; 1450 decoder->state.to_ip = decoder->last_ip; 1451 decoder->ip = decoder->last_ip; 1452 intel_pt_update_nr(decoder); 1453 return 0; 1454 } 1455 1456 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1457 decoder->tnt.count -= 1; 1458 if (decoder->tnt.count) 1459 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1460 else 1461 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1462 if (decoder->tnt.payload & BIT63) { 1463 decoder->tnt.payload <<= 1; 1464 decoder->state.from_ip = decoder->ip; 1465 decoder->ip += intel_pt_insn.length + 1466 intel_pt_insn.rel; 1467 decoder->state.to_ip = decoder->ip; 1468 return 0; 1469 } 1470 /* Instruction sample for a non-taken branch */ 1471 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1472 decoder->tnt.payload <<= 1; 1473 decoder->state.type = INTEL_PT_INSTRUCTION; 1474 decoder->state.from_ip = decoder->ip; 1475 decoder->state.to_ip = 0; 1476 decoder->ip += intel_pt_insn.length; 1477 return 0; 1478 } 1479 decoder->sample_cyc = false; 1480 decoder->ip += intel_pt_insn.length; 1481 if (!decoder->tnt.count) { 1482 intel_pt_update_sample_time(decoder); 1483 return -EAGAIN; 1484 } 1485 decoder->tnt.payload <<= 1; 1486 continue; 1487 } 1488 1489 return intel_pt_bug(decoder); 1490 } 1491 } 1492 1493 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1494 { 1495 unsigned int fup_tx_flags; 1496 int err; 1497 1498 fup_tx_flags = decoder->packet.payload & 1499 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1500 err = intel_pt_get_next_packet(decoder); 1501 if (err) 1502 return err; 1503 if (decoder->packet.type == INTEL_PT_FUP) { 1504 decoder->fup_tx_flags = fup_tx_flags; 1505 decoder->set_fup_tx_flags = true; 1506 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1507 *no_tip = true; 1508 } else { 1509 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1510 decoder->pos); 1511 intel_pt_update_in_tx(decoder); 1512 } 1513 return 0; 1514 } 1515 1516 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1517 { 1518 timestamp |= (ref_timestamp & (0xffULL << 56)); 1519 1520 if (timestamp < ref_timestamp) { 1521 if (ref_timestamp - timestamp > (1ULL << 55)) 1522 timestamp += (1ULL << 56); 1523 } else { 1524 if (timestamp - ref_timestamp > (1ULL << 55)) 1525 timestamp -= (1ULL << 56); 1526 } 1527 1528 return timestamp; 1529 } 1530 1531 /* For use only when decoder->vm_time_correlation is true */ 1532 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, 1533 uint64_t timestamp) 1534 { 1535 uint64_t max_timestamp = decoder->buf_timestamp; 1536 1537 if (!max_timestamp) { 1538 max_timestamp = decoder->last_reliable_timestamp + 1539 0x400000000ULL; 1540 } 1541 return timestamp >= decoder->last_reliable_timestamp && 1542 timestamp < decoder->buf_timestamp; 1543 } 1544 1545 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1546 { 1547 uint64_t timestamp; 1548 bool bad = false; 1549 1550 decoder->have_tma = false; 1551 1552 if (decoder->ref_timestamp) { 1553 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1554 decoder->ref_timestamp); 1555 decoder->tsc_timestamp = timestamp; 1556 decoder->timestamp = timestamp; 1557 decoder->ref_timestamp = 0; 1558 decoder->timestamp_insn_cnt = 0; 1559 } else if (decoder->timestamp) { 1560 timestamp = decoder->packet.payload | 1561 (decoder->timestamp & (0xffULL << 56)); 1562 decoder->tsc_timestamp = timestamp; 1563 if (timestamp < decoder->timestamp && 1564 decoder->timestamp - timestamp < decoder->tsc_slip) { 1565 intel_pt_log_to("Suppressing backwards timestamp", 1566 timestamp); 1567 timestamp = decoder->timestamp; 1568 } 1569 if (timestamp < decoder->timestamp) { 1570 if (!decoder->buf_timestamp || 1571 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { 1572 intel_pt_log_to("Wraparound timestamp", timestamp); 1573 timestamp += (1ULL << 56); 1574 decoder->tsc_timestamp = timestamp; 1575 } else { 1576 intel_pt_log_to("Suppressing bad timestamp", timestamp); 1577 timestamp = decoder->timestamp; 1578 bad = true; 1579 } 1580 } 1581 if (decoder->vm_time_correlation && 1582 (bad || !intel_pt_time_in_range(decoder, timestamp)) && 1583 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 1584 p_log("Timestamp out of range"); 1585 decoder->timestamp = timestamp; 1586 decoder->timestamp_insn_cnt = 0; 1587 } 1588 1589 if (decoder->last_packet_type == INTEL_PT_CYC) { 1590 decoder->cyc_ref_timestamp = decoder->timestamp; 1591 decoder->cycle_cnt = 0; 1592 decoder->have_calc_cyc_to_tsc = false; 1593 intel_pt_calc_cyc_to_tsc(decoder, false); 1594 } 1595 1596 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1597 } 1598 1599 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1600 { 1601 intel_pt_log("ERROR: Buffer overflow\n"); 1602 intel_pt_clear_tx_flags(decoder); 1603 intel_pt_set_nr(decoder); 1604 decoder->timestamp_insn_cnt = 0; 1605 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1606 decoder->overflow = true; 1607 return -EOVERFLOW; 1608 } 1609 1610 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1611 { 1612 if (decoder->have_cyc) 1613 return; 1614 1615 decoder->cyc_cnt_timestamp = decoder->timestamp; 1616 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1617 } 1618 1619 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1620 { 1621 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1622 1623 if (decoder->pge) 1624 intel_pt_mtc_cyc_cnt_pge(decoder); 1625 } 1626 1627 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1628 { 1629 uint64_t tot_cyc_cnt, tsc_delta; 1630 1631 if (decoder->have_cyc) 1632 return; 1633 1634 decoder->sample_cyc = true; 1635 1636 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1637 return; 1638 1639 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1640 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1641 1642 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1643 decoder->tot_cyc_cnt = tot_cyc_cnt; 1644 } 1645 1646 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1647 { 1648 uint32_t ctc = decoder->packet.payload; 1649 uint32_t fc = decoder->packet.count; 1650 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1651 1652 if (!decoder->tsc_ctc_ratio_d) 1653 return; 1654 1655 if (decoder->pge && !decoder->in_psb) 1656 intel_pt_mtc_cyc_cnt_pge(decoder); 1657 else 1658 intel_pt_mtc_cyc_cnt_upd(decoder); 1659 1660 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1661 decoder->last_ctc = ctc - ctc_rem; 1662 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1663 if (decoder->tsc_ctc_mult) { 1664 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1665 } else { 1666 decoder->ctc_timestamp -= multdiv(ctc_rem, 1667 decoder->tsc_ctc_ratio_n, 1668 decoder->tsc_ctc_ratio_d); 1669 } 1670 decoder->ctc_delta = 0; 1671 decoder->have_tma = true; 1672 decoder->fixup_last_mtc = true; 1673 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1674 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1675 } 1676 1677 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1678 { 1679 uint64_t timestamp; 1680 uint32_t mtc, mtc_delta; 1681 1682 if (!decoder->have_tma) 1683 return; 1684 1685 mtc = decoder->packet.payload; 1686 1687 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1688 decoder->fixup_last_mtc = false; 1689 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1690 &decoder->last_mtc); 1691 } 1692 1693 if (mtc > decoder->last_mtc) 1694 mtc_delta = mtc - decoder->last_mtc; 1695 else 1696 mtc_delta = mtc + 256 - decoder->last_mtc; 1697 1698 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1699 1700 if (decoder->tsc_ctc_mult) { 1701 timestamp = decoder->ctc_timestamp + 1702 decoder->ctc_delta * decoder->tsc_ctc_mult; 1703 } else { 1704 timestamp = decoder->ctc_timestamp + 1705 multdiv(decoder->ctc_delta, 1706 decoder->tsc_ctc_ratio_n, 1707 decoder->tsc_ctc_ratio_d); 1708 } 1709 1710 if (timestamp < decoder->timestamp) 1711 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1712 timestamp, decoder->timestamp); 1713 else 1714 decoder->timestamp = timestamp; 1715 1716 intel_pt_mtc_cyc_cnt_upd(decoder); 1717 1718 decoder->timestamp_insn_cnt = 0; 1719 decoder->last_mtc = mtc; 1720 1721 if (decoder->last_packet_type == INTEL_PT_CYC) { 1722 decoder->cyc_ref_timestamp = decoder->timestamp; 1723 decoder->cycle_cnt = 0; 1724 decoder->have_calc_cyc_to_tsc = false; 1725 intel_pt_calc_cyc_to_tsc(decoder, true); 1726 } 1727 1728 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1729 } 1730 1731 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1732 { 1733 unsigned int cbr = decoder->packet.payload & 0xff; 1734 1735 decoder->cbr_payload = decoder->packet.payload; 1736 1737 if (decoder->cbr == cbr) 1738 return; 1739 1740 decoder->cbr = cbr; 1741 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1742 1743 intel_pt_mtc_cyc_cnt_cbr(decoder); 1744 } 1745 1746 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1747 { 1748 uint64_t timestamp = decoder->cyc_ref_timestamp; 1749 1750 decoder->have_cyc = true; 1751 1752 decoder->cycle_cnt += decoder->packet.payload; 1753 if (decoder->pge) 1754 decoder->tot_cyc_cnt += decoder->packet.payload; 1755 decoder->sample_cyc = true; 1756 1757 if (!decoder->cyc_ref_timestamp) 1758 return; 1759 1760 if (decoder->have_calc_cyc_to_tsc) 1761 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1762 else if (decoder->cbr) 1763 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1764 else 1765 return; 1766 1767 if (timestamp < decoder->timestamp) 1768 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1769 timestamp, decoder->timestamp); 1770 else 1771 decoder->timestamp = timestamp; 1772 1773 decoder->timestamp_insn_cnt = 0; 1774 1775 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1776 } 1777 1778 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1779 { 1780 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1781 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1782 decoder->state.items.is_32_bit = false; 1783 } 1784 decoder->blk_type = decoder->packet.payload; 1785 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1786 if (decoder->blk_type == INTEL_PT_GP_REGS) 1787 decoder->state.items.is_32_bit = decoder->packet.count; 1788 if (decoder->blk_type_pos < 0) { 1789 intel_pt_log("WARNING: Unknown block type %u\n", 1790 decoder->blk_type); 1791 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1792 intel_pt_log("WARNING: Duplicate block type %u\n", 1793 decoder->blk_type); 1794 } 1795 } 1796 1797 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1798 { 1799 uint32_t id = decoder->packet.count; 1800 uint32_t bit = 1 << id; 1801 int pos = decoder->blk_type_pos; 1802 1803 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1804 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1805 id, decoder->blk_type); 1806 return; 1807 } 1808 1809 if (decoder->state.items.mask[pos] & bit) { 1810 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1811 id, decoder->blk_type); 1812 } 1813 1814 decoder->state.items.mask[pos] |= bit; 1815 decoder->state.items.val[pos][id] = decoder->packet.payload; 1816 } 1817 1818 /* Walk PSB+ packets when already in sync. */ 1819 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1820 { 1821 int err; 1822 1823 decoder->in_psb = true; 1824 1825 while (1) { 1826 err = intel_pt_get_next_packet(decoder); 1827 if (err) 1828 goto out; 1829 1830 switch (decoder->packet.type) { 1831 case INTEL_PT_PSBEND: 1832 err = 0; 1833 goto out; 1834 1835 case INTEL_PT_TIP_PGD: 1836 case INTEL_PT_TIP_PGE: 1837 case INTEL_PT_TIP: 1838 case INTEL_PT_TNT: 1839 case INTEL_PT_TRACESTOP: 1840 case INTEL_PT_BAD: 1841 case INTEL_PT_PSB: 1842 case INTEL_PT_PTWRITE: 1843 case INTEL_PT_PTWRITE_IP: 1844 case INTEL_PT_EXSTOP: 1845 case INTEL_PT_EXSTOP_IP: 1846 case INTEL_PT_MWAIT: 1847 case INTEL_PT_PWRE: 1848 case INTEL_PT_PWRX: 1849 case INTEL_PT_BBP: 1850 case INTEL_PT_BIP: 1851 case INTEL_PT_BEP: 1852 case INTEL_PT_BEP_IP: 1853 decoder->have_tma = false; 1854 intel_pt_log("ERROR: Unexpected packet\n"); 1855 err = -EAGAIN; 1856 goto out; 1857 1858 case INTEL_PT_OVF: 1859 err = intel_pt_overflow(decoder); 1860 goto out; 1861 1862 case INTEL_PT_TSC: 1863 intel_pt_calc_tsc_timestamp(decoder); 1864 break; 1865 1866 case INTEL_PT_TMA: 1867 intel_pt_calc_tma(decoder); 1868 break; 1869 1870 case INTEL_PT_CBR: 1871 intel_pt_calc_cbr(decoder); 1872 break; 1873 1874 case INTEL_PT_MODE_EXEC: 1875 decoder->exec_mode = decoder->packet.payload; 1876 break; 1877 1878 case INTEL_PT_PIP: 1879 intel_pt_set_pip(decoder); 1880 break; 1881 1882 case INTEL_PT_FUP: 1883 decoder->pge = true; 1884 if (decoder->packet.count) { 1885 intel_pt_set_last_ip(decoder); 1886 decoder->psb_ip = decoder->last_ip; 1887 } 1888 break; 1889 1890 case INTEL_PT_MODE_TSX: 1891 intel_pt_update_in_tx(decoder); 1892 break; 1893 1894 case INTEL_PT_MTC: 1895 intel_pt_calc_mtc_timestamp(decoder); 1896 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1897 decoder->state.type |= INTEL_PT_INSTRUCTION; 1898 break; 1899 1900 case INTEL_PT_CYC: 1901 intel_pt_calc_cyc_timestamp(decoder); 1902 break; 1903 1904 case INTEL_PT_VMCS: 1905 case INTEL_PT_MNT: 1906 case INTEL_PT_PAD: 1907 default: 1908 break; 1909 } 1910 } 1911 out: 1912 decoder->in_psb = false; 1913 1914 return err; 1915 } 1916 1917 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1918 { 1919 int err; 1920 1921 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1922 decoder->tx_flags = 0; 1923 decoder->state.flags &= ~INTEL_PT_IN_TX; 1924 decoder->state.flags |= INTEL_PT_ABORT_TX; 1925 } else { 1926 decoder->state.flags |= INTEL_PT_ASYNC; 1927 } 1928 1929 while (1) { 1930 err = intel_pt_get_next_packet(decoder); 1931 if (err) 1932 return err; 1933 1934 switch (decoder->packet.type) { 1935 case INTEL_PT_TNT: 1936 case INTEL_PT_FUP: 1937 case INTEL_PT_TRACESTOP: 1938 case INTEL_PT_PSB: 1939 case INTEL_PT_TSC: 1940 case INTEL_PT_TMA: 1941 case INTEL_PT_MODE_TSX: 1942 case INTEL_PT_BAD: 1943 case INTEL_PT_PSBEND: 1944 case INTEL_PT_PTWRITE: 1945 case INTEL_PT_PTWRITE_IP: 1946 case INTEL_PT_EXSTOP: 1947 case INTEL_PT_EXSTOP_IP: 1948 case INTEL_PT_MWAIT: 1949 case INTEL_PT_PWRE: 1950 case INTEL_PT_PWRX: 1951 case INTEL_PT_BBP: 1952 case INTEL_PT_BIP: 1953 case INTEL_PT_BEP: 1954 case INTEL_PT_BEP_IP: 1955 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1956 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1957 decoder->pkt_step = 0; 1958 return -ENOENT; 1959 1960 case INTEL_PT_CBR: 1961 intel_pt_calc_cbr(decoder); 1962 break; 1963 1964 case INTEL_PT_OVF: 1965 return intel_pt_overflow(decoder); 1966 1967 case INTEL_PT_TIP_PGD: 1968 decoder->state.from_ip = decoder->ip; 1969 if (decoder->packet.count == 0) { 1970 decoder->state.to_ip = 0; 1971 } else { 1972 intel_pt_set_ip(decoder); 1973 decoder->state.to_ip = decoder->ip; 1974 } 1975 decoder->pge = false; 1976 decoder->continuous_period = false; 1977 decoder->state.type |= INTEL_PT_TRACE_END; 1978 intel_pt_update_nr(decoder); 1979 return 0; 1980 1981 case INTEL_PT_TIP_PGE: 1982 decoder->pge = true; 1983 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1984 decoder->ip); 1985 decoder->state.from_ip = 0; 1986 if (decoder->packet.count == 0) { 1987 decoder->state.to_ip = 0; 1988 } else { 1989 intel_pt_set_ip(decoder); 1990 decoder->state.to_ip = decoder->ip; 1991 } 1992 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1993 intel_pt_mtc_cyc_cnt_pge(decoder); 1994 intel_pt_set_nr(decoder); 1995 return 0; 1996 1997 case INTEL_PT_TIP: 1998 decoder->state.from_ip = decoder->ip; 1999 if (decoder->packet.count == 0) { 2000 decoder->state.to_ip = 0; 2001 } else { 2002 intel_pt_set_ip(decoder); 2003 decoder->state.to_ip = decoder->ip; 2004 } 2005 intel_pt_update_nr(decoder); 2006 return 0; 2007 2008 case INTEL_PT_PIP: 2009 intel_pt_update_pip(decoder); 2010 break; 2011 2012 case INTEL_PT_MTC: 2013 intel_pt_calc_mtc_timestamp(decoder); 2014 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2015 decoder->state.type |= INTEL_PT_INSTRUCTION; 2016 break; 2017 2018 case INTEL_PT_CYC: 2019 intel_pt_calc_cyc_timestamp(decoder); 2020 break; 2021 2022 case INTEL_PT_MODE_EXEC: 2023 decoder->exec_mode = decoder->packet.payload; 2024 break; 2025 2026 case INTEL_PT_VMCS: 2027 case INTEL_PT_MNT: 2028 case INTEL_PT_PAD: 2029 break; 2030 2031 default: 2032 return intel_pt_bug(decoder); 2033 } 2034 } 2035 } 2036 2037 static int intel_pt_resample(struct intel_pt_decoder *decoder) 2038 { 2039 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2040 decoder->state.type = INTEL_PT_INSTRUCTION; 2041 decoder->state.from_ip = decoder->ip; 2042 decoder->state.to_ip = 0; 2043 return 0; 2044 } 2045 2046 struct intel_pt_vm_tsc_info { 2047 struct intel_pt_pkt pip_packet; 2048 struct intel_pt_pkt vmcs_packet; 2049 struct intel_pt_pkt tma_packet; 2050 bool tsc, pip, vmcs, tma, psbend; 2051 uint64_t ctc_delta; 2052 uint64_t last_ctc; 2053 int max_lookahead; 2054 }; 2055 2056 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */ 2057 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2058 { 2059 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2060 2061 switch (pkt_info->packet.type) { 2062 case INTEL_PT_PAD: 2063 case INTEL_PT_MNT: 2064 case INTEL_PT_MODE_EXEC: 2065 case INTEL_PT_MODE_TSX: 2066 case INTEL_PT_MTC: 2067 case INTEL_PT_FUP: 2068 case INTEL_PT_CYC: 2069 case INTEL_PT_CBR: 2070 break; 2071 2072 case INTEL_PT_TSC: 2073 data->tsc = true; 2074 break; 2075 2076 case INTEL_PT_TMA: 2077 data->tma_packet = pkt_info->packet; 2078 data->tma = true; 2079 break; 2080 2081 case INTEL_PT_PIP: 2082 data->pip_packet = pkt_info->packet; 2083 data->pip = true; 2084 break; 2085 2086 case INTEL_PT_VMCS: 2087 data->vmcs_packet = pkt_info->packet; 2088 data->vmcs = true; 2089 break; 2090 2091 case INTEL_PT_PSBEND: 2092 data->psbend = true; 2093 return 1; 2094 2095 case INTEL_PT_TIP_PGE: 2096 case INTEL_PT_PTWRITE: 2097 case INTEL_PT_PTWRITE_IP: 2098 case INTEL_PT_EXSTOP: 2099 case INTEL_PT_EXSTOP_IP: 2100 case INTEL_PT_MWAIT: 2101 case INTEL_PT_PWRE: 2102 case INTEL_PT_PWRX: 2103 case INTEL_PT_BBP: 2104 case INTEL_PT_BIP: 2105 case INTEL_PT_BEP: 2106 case INTEL_PT_BEP_IP: 2107 case INTEL_PT_OVF: 2108 case INTEL_PT_BAD: 2109 case INTEL_PT_TNT: 2110 case INTEL_PT_TIP_PGD: 2111 case INTEL_PT_TIP: 2112 case INTEL_PT_PSB: 2113 case INTEL_PT_TRACESTOP: 2114 default: 2115 return 1; 2116 } 2117 2118 return 0; 2119 } 2120 2121 struct intel_pt_ovf_fup_info { 2122 int max_lookahead; 2123 bool found; 2124 }; 2125 2126 /* Lookahead to detect a FUP packet after OVF */ 2127 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2128 { 2129 struct intel_pt_ovf_fup_info *data = pkt_info->data; 2130 2131 if (pkt_info->packet.type == INTEL_PT_CYC || 2132 pkt_info->packet.type == INTEL_PT_MTC || 2133 pkt_info->packet.type == INTEL_PT_TSC) 2134 return !--(data->max_lookahead); 2135 data->found = pkt_info->packet.type == INTEL_PT_FUP; 2136 return 1; 2137 } 2138 2139 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder) 2140 { 2141 struct intel_pt_ovf_fup_info data = { 2142 .max_lookahead = 16, 2143 .found = false, 2144 }; 2145 2146 intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); 2147 return data.found; 2148 } 2149 2150 /* Lookahead and get the TMA packet after TSC */ 2151 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2152 { 2153 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2154 2155 if (pkt_info->packet.type == INTEL_PT_CYC || 2156 pkt_info->packet.type == INTEL_PT_MTC) 2157 return !--(data->max_lookahead); 2158 2159 if (pkt_info->packet.type == INTEL_PT_TMA) { 2160 data->tma_packet = pkt_info->packet; 2161 data->tma = true; 2162 } 2163 return 1; 2164 } 2165 2166 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc) 2167 { 2168 if (decoder->tsc_ctc_mult) 2169 return ctc * decoder->tsc_ctc_mult; 2170 else 2171 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); 2172 } 2173 2174 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, 2175 uint32_t ctc, 2176 uint32_t fc, 2177 uint64_t last_ctc_timestamp, 2178 uint64_t ctc_delta, 2179 uint32_t last_ctc) 2180 { 2181 /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ 2182 uint64_t last_mtc_ctc = last_ctc + ctc_delta; 2183 /* 2184 * Number of CTC ticks from there until current TMA packet. We would 2185 * expect last_mtc_ctc to be before ctc, but the TSC packet can slip 2186 * past an MTC, so a sign-extended value is used. 2187 */ 2188 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); 2189 /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ 2190 uint64_t new_ctc_delta = ctc_delta + delta; 2191 uint64_t expected_tsc; 2192 2193 /* 2194 * Convert CTC ticks to TSC ticks, add the starting point 2195 * (last_ctc_timestamp) and the fast counter from the TMA packet. 2196 */ 2197 expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc; 2198 2199 if (intel_pt_enable_logging) { 2200 intel_pt_log_x64(last_mtc_ctc); 2201 intel_pt_log_x32(last_ctc); 2202 intel_pt_log_x64(ctc_delta); 2203 intel_pt_log_x64(delta); 2204 intel_pt_log_x32(ctc); 2205 intel_pt_log_x64(new_ctc_delta); 2206 intel_pt_log_x64(last_ctc_timestamp); 2207 intel_pt_log_x32(fc); 2208 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); 2209 intel_pt_log_x64(expected_tsc); 2210 } 2211 2212 return expected_tsc; 2213 } 2214 2215 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, 2216 struct intel_pt_vm_tsc_info *data) 2217 { 2218 uint32_t ctc = data->tma_packet.payload; 2219 uint32_t fc = data->tma_packet.count; 2220 2221 return intel_pt_calc_expected_tsc(decoder, ctc, fc, 2222 decoder->ctc_timestamp, 2223 data->ctc_delta, data->last_ctc); 2224 } 2225 2226 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, 2227 struct intel_pt_vmcs_info *vmcs_info) 2228 { 2229 uint64_t payload = decoder->packet.payload; 2230 2231 /* VMX adds the TSC Offset, so subtract to get host TSC */ 2232 decoder->packet.payload -= vmcs_info->tsc_offset; 2233 /* TSC packet has only 7 bytes */ 2234 decoder->packet.payload &= SEVEN_BYTES; 2235 2236 /* 2237 * The buffer is mmapped from the data file, so this also updates the 2238 * data file. 2239 */ 2240 if (!decoder->vm_tm_corr_dry_run) 2241 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); 2242 2243 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2244 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2245 payload, decoder->packet.payload, vmcs_info->vmcs, 2246 vmcs_info->tsc_offset); 2247 } 2248 2249 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, 2250 uint64_t tsc_offset) 2251 { 2252 struct intel_pt_vmcs_info vmcs_info = { 2253 .vmcs = NO_VMCS, 2254 .tsc_offset = tsc_offset 2255 }; 2256 2257 intel_pt_translate_vm_tsc(decoder, &vmcs_info); 2258 } 2259 2260 static inline bool in_vm(uint64_t pip_payload) 2261 { 2262 return pip_payload & 1; 2263 } 2264 2265 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet) 2266 { 2267 return pip_packet->payload & 1; 2268 } 2269 2270 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info) 2271 { 2272 p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, 2273 vmcs_info->vmcs, vmcs_info->tsc_offset); 2274 } 2275 2276 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, 2277 struct intel_pt_vm_tsc_info *data) 2278 { 2279 memset(data, 0, sizeof(*data)); 2280 data->ctc_delta = decoder->ctc_delta; 2281 data->last_ctc = decoder->last_ctc; 2282 intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); 2283 if (data->tsc && !data->psbend) 2284 p_log("ERROR: PSB without PSBEND"); 2285 decoder->in_psb = data->psbend; 2286 } 2287 2288 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, 2289 struct intel_pt_vm_tsc_info *data, 2290 struct intel_pt_vmcs_info *vmcs_info, 2291 uint64_t host_tsc) 2292 { 2293 if (!decoder->in_psb) { 2294 /* Can't happen */ 2295 p_log("ERROR: First TSC is not in PSB+"); 2296 } 2297 2298 if (data->pip) { 2299 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2300 if (vmcs_info && vmcs_info->tsc_offset) { 2301 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2302 decoder->vm_tm_corr_reliable = true; 2303 } else { 2304 p_log("ERROR: First TSC, unknown TSC Offset"); 2305 } 2306 } else { /* Host */ 2307 decoder->vm_tm_corr_reliable = true; 2308 } 2309 } else { /* Host or Guest */ 2310 decoder->vm_tm_corr_reliable = false; 2311 if (intel_pt_time_in_range(decoder, host_tsc)) { 2312 /* Assume Host */ 2313 } else { 2314 /* Assume Guest */ 2315 if (vmcs_info && vmcs_info->tsc_offset) 2316 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2317 else 2318 p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); 2319 } 2320 } 2321 } 2322 2323 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, 2324 struct intel_pt_vm_tsc_info *data) 2325 { 2326 struct intel_pt_vmcs_info *vmcs_info; 2327 uint64_t tsc_offset = 0; 2328 uint64_t vmcs; 2329 bool reliable = true; 2330 uint64_t expected_tsc; 2331 uint64_t host_tsc; 2332 uint64_t ref_timestamp; 2333 2334 bool assign = false; 2335 bool assign_reliable = false; 2336 2337 /* Already have 'data' for the in_psb case */ 2338 if (!decoder->in_psb) { 2339 memset(data, 0, sizeof(*data)); 2340 data->ctc_delta = decoder->ctc_delta; 2341 data->last_ctc = decoder->last_ctc; 2342 data->max_lookahead = 16; 2343 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); 2344 if (decoder->pge) { 2345 data->pip = true; 2346 data->pip_packet.payload = decoder->pip_payload; 2347 } 2348 } 2349 2350 /* Calculations depend on having TMA packets */ 2351 if (!data->tma) { 2352 p_log("ERROR: TSC without TMA"); 2353 return; 2354 } 2355 2356 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; 2357 if (vmcs == NO_VMCS) 2358 vmcs = 0; 2359 2360 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2361 2362 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; 2363 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); 2364 2365 if (!decoder->ctc_timestamp) { 2366 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); 2367 return; 2368 } 2369 2370 expected_tsc = intel_pt_expected_tsc(decoder, data); 2371 2372 tsc_offset = host_tsc - expected_tsc; 2373 2374 /* Determine if TSC is from Host or Guest */ 2375 if (data->pip) { 2376 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2377 if (!vmcs_info) { 2378 /* PIP NR=1 without VMCS cannot happen */ 2379 p_log("ERROR: Missing VMCS"); 2380 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2381 decoder->vm_tm_corr_reliable = false; 2382 return; 2383 } 2384 } else { /* Host */ 2385 decoder->last_reliable_timestamp = host_tsc; 2386 decoder->vm_tm_corr_reliable = true; 2387 return; 2388 } 2389 } else { /* Host or Guest */ 2390 reliable = false; /* Host/Guest is a guess, so not reliable */ 2391 if (decoder->in_psb) { 2392 if (!tsc_offset) 2393 return; /* Zero TSC Offset, assume Host */ 2394 /* 2395 * TSC packet has only 7 bytes of TSC. We have no 2396 * information about the Guest's 8th byte, but it 2397 * doesn't matter because we only need 7 bytes. 2398 * Here, since the 8th byte is unreliable and 2399 * irrelevant, compare only 7 byes. 2400 */ 2401 if (vmcs_info && 2402 (tsc_offset & SEVEN_BYTES) == 2403 (vmcs_info->tsc_offset & SEVEN_BYTES)) { 2404 /* Same TSC Offset as last VMCS, assume Guest */ 2405 goto guest; 2406 } 2407 } 2408 /* 2409 * Check if the host_tsc is within the expected range. 2410 * Note, we could narrow the range more by looking ahead for 2411 * the next host TSC in the same buffer, but we don't bother to 2412 * do that because this is probably good enough. 2413 */ 2414 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { 2415 /* Within expected range for Host TSC, assume Host */ 2416 decoder->vm_tm_corr_reliable = false; 2417 return; 2418 } 2419 } 2420 2421 guest: /* Assuming Guest */ 2422 2423 /* Determine whether to assign TSC Offset */ 2424 if (vmcs_info && vmcs_info->vmcs) { 2425 if (vmcs_info->tsc_offset && vmcs_info->reliable) { 2426 assign = false; 2427 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && 2428 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { 2429 /* Continuous tracing, TSC in a PSB is not a time loss */ 2430 assign = true; 2431 assign_reliable = true; 2432 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { 2433 /* 2434 * Unlikely to be a time loss TSC in a PSB which is not 2435 * at the start of a buffer. 2436 */ 2437 assign = true; 2438 assign_reliable = false; 2439 } 2440 } 2441 2442 /* Record VMCS TSC Offset */ 2443 if (assign && (vmcs_info->tsc_offset != tsc_offset || 2444 vmcs_info->reliable != assign_reliable)) { 2445 bool print = vmcs_info->tsc_offset != tsc_offset; 2446 2447 vmcs_info->tsc_offset = tsc_offset; 2448 vmcs_info->reliable = assign_reliable; 2449 if (print) 2450 intel_pt_print_vmcs_info(vmcs_info); 2451 } 2452 2453 /* Determine what TSC Offset to use */ 2454 if (vmcs_info && vmcs_info->tsc_offset) { 2455 if (!vmcs_info->reliable) 2456 reliable = false; 2457 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2458 } else { 2459 reliable = false; 2460 if (vmcs_info) { 2461 if (!vmcs_info->error_printed) { 2462 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, 2463 vmcs_info->vmcs); 2464 vmcs_info->error_printed = true; 2465 } 2466 } else { 2467 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2468 p_log("ERROR: Unknown VMCS"); 2469 } 2470 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2471 } 2472 2473 decoder->vm_tm_corr_reliable = reliable; 2474 } 2475 2476 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder) 2477 { 2478 uint64_t host_tsc = decoder->packet.payload; 2479 uint64_t guest_tsc = decoder->packet.payload; 2480 struct intel_pt_vmcs_info *vmcs_info; 2481 uint64_t vmcs; 2482 2483 vmcs = decoder->vmcs; 2484 if (vmcs == NO_VMCS) 2485 vmcs = 0; 2486 2487 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2488 2489 if (decoder->pge) { 2490 if (in_vm(decoder->pip_payload)) { /* Guest */ 2491 if (!vmcs_info) { 2492 /* PIP NR=1 without VMCS cannot happen */ 2493 p_log("ERROR: Missing VMCS"); 2494 } 2495 } else { /* Host */ 2496 return; 2497 } 2498 } else { /* Host or Guest */ 2499 if (intel_pt_time_in_range(decoder, host_tsc)) { 2500 /* Within expected range for Host TSC, assume Host */ 2501 return; 2502 } 2503 } 2504 2505 if (vmcs_info) { 2506 /* Translate Guest TSC to Host TSC */ 2507 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; 2508 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); 2509 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2510 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2511 guest_tsc, host_tsc, vmcs_info->vmcs, 2512 vmcs_info->tsc_offset); 2513 if (!intel_pt_time_in_range(decoder, host_tsc) && 2514 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 2515 p_log("Timestamp out of range"); 2516 } else { 2517 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2518 p_log("ERROR: Unknown VMCS"); 2519 host_tsc = decoder->timestamp; 2520 } 2521 2522 decoder->packet.payload = host_tsc; 2523 2524 if (!decoder->vm_tm_corr_dry_run) 2525 memcpy((void *)decoder->buf + 1, &host_tsc, 8); 2526 } 2527 2528 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder) 2529 { 2530 struct intel_pt_vm_tsc_info data = { .psbend = false }; 2531 bool pge; 2532 int err; 2533 2534 if (decoder->in_psb) 2535 intel_pt_vm_tm_corr_psb(decoder, &data); 2536 2537 while (1) { 2538 err = intel_pt_get_next_packet(decoder); 2539 if (err == -ENOLINK) 2540 continue; 2541 if (err) 2542 break; 2543 2544 switch (decoder->packet.type) { 2545 case INTEL_PT_TIP_PGD: 2546 decoder->pge = false; 2547 decoder->vm_tm_corr_continuous = false; 2548 break; 2549 2550 case INTEL_PT_TNT: 2551 case INTEL_PT_TIP: 2552 case INTEL_PT_TIP_PGE: 2553 decoder->pge = true; 2554 break; 2555 2556 case INTEL_PT_OVF: 2557 decoder->in_psb = false; 2558 pge = decoder->pge; 2559 decoder->pge = intel_pt_ovf_fup_lookahead(decoder); 2560 if (pge != decoder->pge) 2561 intel_pt_log("Surprising PGE change in OVF!"); 2562 if (!decoder->pge) 2563 decoder->vm_tm_corr_continuous = false; 2564 break; 2565 2566 case INTEL_PT_FUP: 2567 if (decoder->in_psb) 2568 decoder->pge = true; 2569 break; 2570 2571 case INTEL_PT_TRACESTOP: 2572 decoder->pge = false; 2573 decoder->vm_tm_corr_continuous = false; 2574 decoder->have_tma = false; 2575 break; 2576 2577 case INTEL_PT_PSB: 2578 intel_pt_vm_tm_corr_psb(decoder, &data); 2579 break; 2580 2581 case INTEL_PT_PIP: 2582 decoder->pip_payload = decoder->packet.payload; 2583 break; 2584 2585 case INTEL_PT_MTC: 2586 intel_pt_calc_mtc_timestamp(decoder); 2587 break; 2588 2589 case INTEL_PT_TSC: 2590 intel_pt_vm_tm_corr_tsc(decoder, &data); 2591 intel_pt_calc_tsc_timestamp(decoder); 2592 decoder->vm_tm_corr_same_buf = true; 2593 decoder->vm_tm_corr_continuous = decoder->pge; 2594 break; 2595 2596 case INTEL_PT_TMA: 2597 intel_pt_calc_tma(decoder); 2598 break; 2599 2600 case INTEL_PT_CYC: 2601 intel_pt_calc_cyc_timestamp(decoder); 2602 break; 2603 2604 case INTEL_PT_CBR: 2605 intel_pt_calc_cbr(decoder); 2606 break; 2607 2608 case INTEL_PT_PSBEND: 2609 decoder->in_psb = false; 2610 data.psbend = false; 2611 break; 2612 2613 case INTEL_PT_VMCS: 2614 if (decoder->packet.payload != NO_VMCS) 2615 decoder->vmcs = decoder->packet.payload; 2616 break; 2617 2618 case INTEL_PT_BBP: 2619 decoder->blk_type = decoder->packet.payload; 2620 break; 2621 2622 case INTEL_PT_BIP: 2623 if (decoder->blk_type == INTEL_PT_PEBS_BASIC && 2624 decoder->packet.count == 2) 2625 intel_pt_vm_tm_corr_pebs_tsc(decoder); 2626 break; 2627 2628 case INTEL_PT_BEP: 2629 case INTEL_PT_BEP_IP: 2630 decoder->blk_type = 0; 2631 break; 2632 2633 case INTEL_PT_MODE_EXEC: 2634 case INTEL_PT_MODE_TSX: 2635 case INTEL_PT_MNT: 2636 case INTEL_PT_PAD: 2637 case INTEL_PT_PTWRITE_IP: 2638 case INTEL_PT_PTWRITE: 2639 case INTEL_PT_MWAIT: 2640 case INTEL_PT_PWRE: 2641 case INTEL_PT_EXSTOP_IP: 2642 case INTEL_PT_EXSTOP: 2643 case INTEL_PT_PWRX: 2644 case INTEL_PT_BAD: /* Does not happen */ 2645 default: 2646 break; 2647 } 2648 } 2649 2650 return err; 2651 } 2652 2653 #define HOP_PROCESS 0 2654 #define HOP_IGNORE 1 2655 #define HOP_RETURN 2 2656 #define HOP_AGAIN 3 2657 2658 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 2659 2660 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 2661 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 2662 { 2663 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 2664 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 2665 *err = intel_pt_scan_for_psb(decoder); 2666 if (*err) 2667 return HOP_RETURN; 2668 } 2669 2670 switch (decoder->packet.type) { 2671 case INTEL_PT_TNT: 2672 return HOP_IGNORE; 2673 2674 case INTEL_PT_TIP_PGD: 2675 if (!decoder->packet.count) { 2676 intel_pt_set_nr(decoder); 2677 return HOP_IGNORE; 2678 } 2679 intel_pt_set_ip(decoder); 2680 decoder->state.type |= INTEL_PT_TRACE_END; 2681 decoder->state.from_ip = 0; 2682 decoder->state.to_ip = decoder->ip; 2683 intel_pt_update_nr(decoder); 2684 return HOP_RETURN; 2685 2686 case INTEL_PT_TIP: 2687 if (!decoder->packet.count) { 2688 intel_pt_set_nr(decoder); 2689 return HOP_IGNORE; 2690 } 2691 intel_pt_set_ip(decoder); 2692 decoder->state.type = INTEL_PT_INSTRUCTION; 2693 decoder->state.from_ip = decoder->ip; 2694 decoder->state.to_ip = 0; 2695 intel_pt_update_nr(decoder); 2696 return HOP_RETURN; 2697 2698 case INTEL_PT_FUP: 2699 if (!decoder->packet.count) 2700 return HOP_IGNORE; 2701 intel_pt_set_ip(decoder); 2702 if (intel_pt_fup_event(decoder)) 2703 return HOP_RETURN; 2704 if (!decoder->branch_enable) 2705 *no_tip = true; 2706 if (*no_tip) { 2707 decoder->state.type = INTEL_PT_INSTRUCTION; 2708 decoder->state.from_ip = decoder->ip; 2709 decoder->state.to_ip = 0; 2710 return HOP_RETURN; 2711 } 2712 *err = intel_pt_walk_fup_tip(decoder); 2713 if (!*err) 2714 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2715 return HOP_RETURN; 2716 2717 case INTEL_PT_PSB: 2718 decoder->state.psb_offset = decoder->pos; 2719 decoder->psb_ip = 0; 2720 decoder->last_ip = 0; 2721 decoder->have_last_ip = true; 2722 *err = intel_pt_walk_psbend(decoder); 2723 if (*err == -EAGAIN) 2724 return HOP_AGAIN; 2725 if (*err) 2726 return HOP_RETURN; 2727 decoder->state.type = INTEL_PT_PSB_EVT; 2728 if (decoder->psb_ip) { 2729 decoder->state.type |= INTEL_PT_INSTRUCTION; 2730 decoder->ip = decoder->psb_ip; 2731 } 2732 decoder->state.from_ip = decoder->psb_ip; 2733 decoder->state.to_ip = 0; 2734 return HOP_RETURN; 2735 2736 case INTEL_PT_BAD: 2737 case INTEL_PT_PAD: 2738 case INTEL_PT_TIP_PGE: 2739 case INTEL_PT_TSC: 2740 case INTEL_PT_TMA: 2741 case INTEL_PT_MODE_EXEC: 2742 case INTEL_PT_MODE_TSX: 2743 case INTEL_PT_MTC: 2744 case INTEL_PT_CYC: 2745 case INTEL_PT_VMCS: 2746 case INTEL_PT_PSBEND: 2747 case INTEL_PT_CBR: 2748 case INTEL_PT_TRACESTOP: 2749 case INTEL_PT_PIP: 2750 case INTEL_PT_OVF: 2751 case INTEL_PT_MNT: 2752 case INTEL_PT_PTWRITE: 2753 case INTEL_PT_PTWRITE_IP: 2754 case INTEL_PT_EXSTOP: 2755 case INTEL_PT_EXSTOP_IP: 2756 case INTEL_PT_MWAIT: 2757 case INTEL_PT_PWRE: 2758 case INTEL_PT_PWRX: 2759 case INTEL_PT_BBP: 2760 case INTEL_PT_BIP: 2761 case INTEL_PT_BEP: 2762 case INTEL_PT_BEP_IP: 2763 default: 2764 return HOP_PROCESS; 2765 } 2766 } 2767 2768 struct intel_pt_psb_info { 2769 struct intel_pt_pkt fup_packet; 2770 bool fup; 2771 int after_psbend; 2772 }; 2773 2774 /* Lookahead and get the FUP packet from PSB+ */ 2775 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2776 { 2777 struct intel_pt_psb_info *data = pkt_info->data; 2778 2779 switch (pkt_info->packet.type) { 2780 case INTEL_PT_PAD: 2781 case INTEL_PT_MNT: 2782 case INTEL_PT_TSC: 2783 case INTEL_PT_TMA: 2784 case INTEL_PT_MODE_EXEC: 2785 case INTEL_PT_MODE_TSX: 2786 case INTEL_PT_MTC: 2787 case INTEL_PT_CYC: 2788 case INTEL_PT_VMCS: 2789 case INTEL_PT_CBR: 2790 case INTEL_PT_PIP: 2791 if (data->after_psbend) { 2792 data->after_psbend -= 1; 2793 if (!data->after_psbend) 2794 return 1; 2795 } 2796 break; 2797 2798 case INTEL_PT_FUP: 2799 if (data->after_psbend) 2800 return 1; 2801 if (data->fup || pkt_info->packet.count == 0) 2802 return 1; 2803 data->fup_packet = pkt_info->packet; 2804 data->fup = true; 2805 break; 2806 2807 case INTEL_PT_PSBEND: 2808 if (!data->fup) 2809 return 1; 2810 /* Keep going to check for a TIP.PGE */ 2811 data->after_psbend = 6; 2812 break; 2813 2814 case INTEL_PT_TIP_PGE: 2815 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 2816 if (data->after_psbend) 2817 data->fup = false; 2818 return 1; 2819 2820 case INTEL_PT_PTWRITE: 2821 case INTEL_PT_PTWRITE_IP: 2822 case INTEL_PT_EXSTOP: 2823 case INTEL_PT_EXSTOP_IP: 2824 case INTEL_PT_MWAIT: 2825 case INTEL_PT_PWRE: 2826 case INTEL_PT_PWRX: 2827 case INTEL_PT_BBP: 2828 case INTEL_PT_BIP: 2829 case INTEL_PT_BEP: 2830 case INTEL_PT_BEP_IP: 2831 if (data->after_psbend) { 2832 data->after_psbend -= 1; 2833 if (!data->after_psbend) 2834 return 1; 2835 break; 2836 } 2837 return 1; 2838 2839 case INTEL_PT_OVF: 2840 case INTEL_PT_BAD: 2841 case INTEL_PT_TNT: 2842 case INTEL_PT_TIP_PGD: 2843 case INTEL_PT_TIP: 2844 case INTEL_PT_PSB: 2845 case INTEL_PT_TRACESTOP: 2846 default: 2847 return 1; 2848 } 2849 2850 return 0; 2851 } 2852 2853 static int intel_pt_psb(struct intel_pt_decoder *decoder) 2854 { 2855 int err; 2856 2857 decoder->last_ip = 0; 2858 decoder->psb_ip = 0; 2859 decoder->have_last_ip = true; 2860 intel_pt_clear_stack(&decoder->stack); 2861 err = intel_pt_walk_psbend(decoder); 2862 if (err) 2863 return err; 2864 decoder->state.type = INTEL_PT_PSB_EVT; 2865 decoder->state.from_ip = decoder->psb_ip; 2866 decoder->state.to_ip = 0; 2867 return 0; 2868 } 2869 2870 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 2871 { 2872 int err; 2873 2874 if (decoder->ip != decoder->last_ip) { 2875 err = intel_pt_walk_fup(decoder); 2876 if (!err || err != -EAGAIN) 2877 return err; 2878 } 2879 2880 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2881 err = intel_pt_psb(decoder); 2882 if (err) { 2883 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2884 return -ENOENT; 2885 } 2886 2887 return 0; 2888 } 2889 2890 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 2891 { 2892 struct intel_pt_psb_info data = { .fup = false }; 2893 2894 if (!decoder->branch_enable || !decoder->pge) 2895 return false; 2896 2897 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 2898 if (!data.fup) 2899 return false; 2900 2901 decoder->packet = data.fup_packet; 2902 intel_pt_set_last_ip(decoder); 2903 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 2904 2905 *err = intel_pt_fup_in_psb(decoder); 2906 2907 return true; 2908 } 2909 2910 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2911 { 2912 int last_packet_type = INTEL_PT_PAD; 2913 bool no_tip = false; 2914 int err; 2915 2916 while (1) { 2917 err = intel_pt_get_next_packet(decoder); 2918 if (err) 2919 return err; 2920 next: 2921 if (decoder->cyc_threshold) { 2922 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 2923 decoder->sample_cyc = false; 2924 last_packet_type = decoder->packet.type; 2925 } 2926 2927 if (decoder->hop) { 2928 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2929 case HOP_IGNORE: 2930 continue; 2931 case HOP_RETURN: 2932 return err; 2933 case HOP_AGAIN: 2934 goto next; 2935 default: 2936 break; 2937 } 2938 } 2939 2940 switch (decoder->packet.type) { 2941 case INTEL_PT_TNT: 2942 if (!decoder->packet.count) 2943 break; 2944 decoder->tnt = decoder->packet; 2945 decoder->pkt_state = INTEL_PT_STATE_TNT; 2946 err = intel_pt_walk_tnt(decoder); 2947 if (err == -EAGAIN) 2948 break; 2949 return err; 2950 2951 case INTEL_PT_TIP_PGD: 2952 if (decoder->packet.count != 0) 2953 intel_pt_set_last_ip(decoder); 2954 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2955 return intel_pt_walk_tip(decoder); 2956 2957 case INTEL_PT_TIP_PGE: { 2958 decoder->pge = true; 2959 intel_pt_mtc_cyc_cnt_pge(decoder); 2960 intel_pt_set_nr(decoder); 2961 if (decoder->packet.count == 0) { 2962 intel_pt_log_at("Skipping zero TIP.PGE", 2963 decoder->pos); 2964 break; 2965 } 2966 intel_pt_set_ip(decoder); 2967 decoder->state.from_ip = 0; 2968 decoder->state.to_ip = decoder->ip; 2969 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2970 /* 2971 * In hop mode, resample to get the to_ip as an 2972 * "instruction" sample. 2973 */ 2974 if (decoder->hop) 2975 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2976 return 0; 2977 } 2978 2979 case INTEL_PT_OVF: 2980 return intel_pt_overflow(decoder); 2981 2982 case INTEL_PT_TIP: 2983 if (decoder->packet.count != 0) 2984 intel_pt_set_last_ip(decoder); 2985 decoder->pkt_state = INTEL_PT_STATE_TIP; 2986 return intel_pt_walk_tip(decoder); 2987 2988 case INTEL_PT_FUP: 2989 if (decoder->packet.count == 0) { 2990 intel_pt_log_at("Skipping zero FUP", 2991 decoder->pos); 2992 no_tip = false; 2993 break; 2994 } 2995 intel_pt_set_last_ip(decoder); 2996 if (!decoder->branch_enable) { 2997 decoder->ip = decoder->last_ip; 2998 if (intel_pt_fup_event(decoder)) 2999 return 0; 3000 no_tip = false; 3001 break; 3002 } 3003 if (decoder->set_fup_mwait) 3004 no_tip = true; 3005 if (no_tip) 3006 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 3007 else 3008 decoder->pkt_state = INTEL_PT_STATE_FUP; 3009 err = intel_pt_walk_fup(decoder); 3010 if (err != -EAGAIN) 3011 return err; 3012 if (no_tip) { 3013 no_tip = false; 3014 break; 3015 } 3016 return intel_pt_walk_fup_tip(decoder); 3017 3018 case INTEL_PT_TRACESTOP: 3019 decoder->pge = false; 3020 decoder->continuous_period = false; 3021 intel_pt_clear_tx_flags(decoder); 3022 decoder->have_tma = false; 3023 break; 3024 3025 case INTEL_PT_PSB: 3026 decoder->state.psb_offset = decoder->pos; 3027 decoder->psb_ip = 0; 3028 if (intel_pt_psb_with_fup(decoder, &err)) 3029 return err; 3030 err = intel_pt_psb(decoder); 3031 if (err == -EAGAIN) 3032 goto next; 3033 return err; 3034 3035 case INTEL_PT_PIP: 3036 intel_pt_update_pip(decoder); 3037 break; 3038 3039 case INTEL_PT_MTC: 3040 intel_pt_calc_mtc_timestamp(decoder); 3041 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 3042 break; 3043 /* 3044 * Ensure that there has been an instruction since the 3045 * last MTC. 3046 */ 3047 if (!decoder->mtc_insn) 3048 break; 3049 decoder->mtc_insn = false; 3050 /* Ensure that there is a timestamp */ 3051 if (!decoder->timestamp) 3052 break; 3053 decoder->state.type = INTEL_PT_INSTRUCTION; 3054 decoder->state.from_ip = decoder->ip; 3055 decoder->state.to_ip = 0; 3056 decoder->mtc_insn = false; 3057 return 0; 3058 3059 case INTEL_PT_TSC: 3060 intel_pt_calc_tsc_timestamp(decoder); 3061 break; 3062 3063 case INTEL_PT_TMA: 3064 intel_pt_calc_tma(decoder); 3065 break; 3066 3067 case INTEL_PT_CYC: 3068 intel_pt_calc_cyc_timestamp(decoder); 3069 break; 3070 3071 case INTEL_PT_CBR: 3072 intel_pt_calc_cbr(decoder); 3073 if (decoder->cbr != decoder->cbr_seen) { 3074 decoder->state.type = 0; 3075 return 0; 3076 } 3077 break; 3078 3079 case INTEL_PT_MODE_EXEC: 3080 decoder->exec_mode = decoder->packet.payload; 3081 break; 3082 3083 case INTEL_PT_MODE_TSX: 3084 /* MODE_TSX need not be followed by FUP */ 3085 if (!decoder->pge || decoder->in_psb) { 3086 intel_pt_update_in_tx(decoder); 3087 break; 3088 } 3089 err = intel_pt_mode_tsx(decoder, &no_tip); 3090 if (err) 3091 return err; 3092 goto next; 3093 3094 case INTEL_PT_BAD: /* Does not happen */ 3095 return intel_pt_bug(decoder); 3096 3097 case INTEL_PT_PSBEND: 3098 case INTEL_PT_VMCS: 3099 case INTEL_PT_MNT: 3100 case INTEL_PT_PAD: 3101 break; 3102 3103 case INTEL_PT_PTWRITE_IP: 3104 decoder->fup_ptw_payload = decoder->packet.payload; 3105 err = intel_pt_get_next_packet(decoder); 3106 if (err) 3107 return err; 3108 if (decoder->packet.type == INTEL_PT_FUP) { 3109 decoder->set_fup_ptw = true; 3110 no_tip = true; 3111 } else { 3112 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 3113 decoder->pos); 3114 } 3115 goto next; 3116 3117 case INTEL_PT_PTWRITE: 3118 decoder->state.type = INTEL_PT_PTW; 3119 decoder->state.from_ip = decoder->ip; 3120 decoder->state.to_ip = 0; 3121 decoder->state.ptw_payload = decoder->packet.payload; 3122 return 0; 3123 3124 case INTEL_PT_MWAIT: 3125 decoder->fup_mwait_payload = decoder->packet.payload; 3126 decoder->set_fup_mwait = true; 3127 break; 3128 3129 case INTEL_PT_PWRE: 3130 if (decoder->set_fup_mwait) { 3131 decoder->fup_pwre_payload = 3132 decoder->packet.payload; 3133 decoder->set_fup_pwre = true; 3134 break; 3135 } 3136 decoder->state.type = INTEL_PT_PWR_ENTRY; 3137 decoder->state.from_ip = decoder->ip; 3138 decoder->state.to_ip = 0; 3139 decoder->state.pwrx_payload = decoder->packet.payload; 3140 return 0; 3141 3142 case INTEL_PT_EXSTOP_IP: 3143 err = intel_pt_get_next_packet(decoder); 3144 if (err) 3145 return err; 3146 if (decoder->packet.type == INTEL_PT_FUP) { 3147 decoder->set_fup_exstop = true; 3148 no_tip = true; 3149 } else { 3150 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 3151 decoder->pos); 3152 } 3153 goto next; 3154 3155 case INTEL_PT_EXSTOP: 3156 decoder->state.type = INTEL_PT_EX_STOP; 3157 decoder->state.from_ip = decoder->ip; 3158 decoder->state.to_ip = 0; 3159 return 0; 3160 3161 case INTEL_PT_PWRX: 3162 decoder->state.type = INTEL_PT_PWR_EXIT; 3163 decoder->state.from_ip = decoder->ip; 3164 decoder->state.to_ip = 0; 3165 decoder->state.pwrx_payload = decoder->packet.payload; 3166 return 0; 3167 3168 case INTEL_PT_BBP: 3169 intel_pt_bbp(decoder); 3170 break; 3171 3172 case INTEL_PT_BIP: 3173 intel_pt_bip(decoder); 3174 break; 3175 3176 case INTEL_PT_BEP: 3177 decoder->state.type = INTEL_PT_BLK_ITEMS; 3178 decoder->state.from_ip = decoder->ip; 3179 decoder->state.to_ip = 0; 3180 return 0; 3181 3182 case INTEL_PT_BEP_IP: 3183 err = intel_pt_get_next_packet(decoder); 3184 if (err) 3185 return err; 3186 if (decoder->packet.type == INTEL_PT_FUP) { 3187 decoder->set_fup_bep = true; 3188 no_tip = true; 3189 } else { 3190 intel_pt_log_at("ERROR: Missing FUP after BEP", 3191 decoder->pos); 3192 } 3193 goto next; 3194 3195 default: 3196 return intel_pt_bug(decoder); 3197 } 3198 } 3199 } 3200 3201 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 3202 { 3203 return decoder->packet.count && 3204 (decoder->have_last_ip || decoder->packet.count == 3 || 3205 decoder->packet.count == 6); 3206 } 3207 3208 /* Walk PSB+ packets to get in sync. */ 3209 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 3210 { 3211 int err; 3212 3213 decoder->in_psb = true; 3214 3215 while (1) { 3216 err = intel_pt_get_next_packet(decoder); 3217 if (err) 3218 goto out; 3219 3220 switch (decoder->packet.type) { 3221 case INTEL_PT_TIP_PGD: 3222 decoder->continuous_period = false; 3223 __fallthrough; 3224 case INTEL_PT_TIP_PGE: 3225 case INTEL_PT_TIP: 3226 case INTEL_PT_PTWRITE: 3227 case INTEL_PT_PTWRITE_IP: 3228 case INTEL_PT_EXSTOP: 3229 case INTEL_PT_EXSTOP_IP: 3230 case INTEL_PT_MWAIT: 3231 case INTEL_PT_PWRE: 3232 case INTEL_PT_PWRX: 3233 case INTEL_PT_BBP: 3234 case INTEL_PT_BIP: 3235 case INTEL_PT_BEP: 3236 case INTEL_PT_BEP_IP: 3237 intel_pt_log("ERROR: Unexpected packet\n"); 3238 err = -ENOENT; 3239 goto out; 3240 3241 case INTEL_PT_FUP: 3242 decoder->pge = true; 3243 if (intel_pt_have_ip(decoder)) { 3244 uint64_t current_ip = decoder->ip; 3245 3246 intel_pt_set_ip(decoder); 3247 decoder->psb_ip = decoder->ip; 3248 if (current_ip) 3249 intel_pt_log_to("Setting IP", 3250 decoder->ip); 3251 } 3252 break; 3253 3254 case INTEL_PT_MTC: 3255 intel_pt_calc_mtc_timestamp(decoder); 3256 break; 3257 3258 case INTEL_PT_TSC: 3259 intel_pt_calc_tsc_timestamp(decoder); 3260 break; 3261 3262 case INTEL_PT_TMA: 3263 intel_pt_calc_tma(decoder); 3264 break; 3265 3266 case INTEL_PT_CYC: 3267 intel_pt_calc_cyc_timestamp(decoder); 3268 break; 3269 3270 case INTEL_PT_CBR: 3271 intel_pt_calc_cbr(decoder); 3272 break; 3273 3274 case INTEL_PT_PIP: 3275 intel_pt_set_pip(decoder); 3276 break; 3277 3278 case INTEL_PT_MODE_EXEC: 3279 decoder->exec_mode = decoder->packet.payload; 3280 break; 3281 3282 case INTEL_PT_MODE_TSX: 3283 intel_pt_update_in_tx(decoder); 3284 break; 3285 3286 case INTEL_PT_TRACESTOP: 3287 decoder->pge = false; 3288 decoder->continuous_period = false; 3289 intel_pt_clear_tx_flags(decoder); 3290 __fallthrough; 3291 3292 case INTEL_PT_TNT: 3293 decoder->have_tma = false; 3294 intel_pt_log("ERROR: Unexpected packet\n"); 3295 if (decoder->ip) 3296 decoder->pkt_state = INTEL_PT_STATE_ERR4; 3297 else 3298 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3299 err = -ENOENT; 3300 goto out; 3301 3302 case INTEL_PT_BAD: /* Does not happen */ 3303 err = intel_pt_bug(decoder); 3304 goto out; 3305 3306 case INTEL_PT_OVF: 3307 err = intel_pt_overflow(decoder); 3308 goto out; 3309 3310 case INTEL_PT_PSBEND: 3311 err = 0; 3312 goto out; 3313 3314 case INTEL_PT_PSB: 3315 case INTEL_PT_VMCS: 3316 case INTEL_PT_MNT: 3317 case INTEL_PT_PAD: 3318 default: 3319 break; 3320 } 3321 } 3322 out: 3323 decoder->in_psb = false; 3324 3325 return err; 3326 } 3327 3328 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 3329 { 3330 int err; 3331 3332 while (1) { 3333 err = intel_pt_get_next_packet(decoder); 3334 if (err) 3335 return err; 3336 3337 switch (decoder->packet.type) { 3338 case INTEL_PT_TIP_PGD: 3339 decoder->continuous_period = false; 3340 decoder->pge = false; 3341 if (intel_pt_have_ip(decoder)) 3342 intel_pt_set_ip(decoder); 3343 if (!decoder->ip) 3344 break; 3345 decoder->state.type |= INTEL_PT_TRACE_END; 3346 return 0; 3347 3348 case INTEL_PT_TIP_PGE: 3349 decoder->pge = true; 3350 intel_pt_mtc_cyc_cnt_pge(decoder); 3351 if (intel_pt_have_ip(decoder)) 3352 intel_pt_set_ip(decoder); 3353 if (!decoder->ip) 3354 break; 3355 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3356 return 0; 3357 3358 case INTEL_PT_TIP: 3359 decoder->pge = true; 3360 if (intel_pt_have_ip(decoder)) 3361 intel_pt_set_ip(decoder); 3362 if (!decoder->ip) 3363 break; 3364 return 0; 3365 3366 case INTEL_PT_FUP: 3367 if (intel_pt_have_ip(decoder)) 3368 intel_pt_set_ip(decoder); 3369 if (decoder->ip) 3370 return 0; 3371 break; 3372 3373 case INTEL_PT_MTC: 3374 intel_pt_calc_mtc_timestamp(decoder); 3375 break; 3376 3377 case INTEL_PT_TSC: 3378 intel_pt_calc_tsc_timestamp(decoder); 3379 break; 3380 3381 case INTEL_PT_TMA: 3382 intel_pt_calc_tma(decoder); 3383 break; 3384 3385 case INTEL_PT_CYC: 3386 intel_pt_calc_cyc_timestamp(decoder); 3387 break; 3388 3389 case INTEL_PT_CBR: 3390 intel_pt_calc_cbr(decoder); 3391 break; 3392 3393 case INTEL_PT_PIP: 3394 intel_pt_set_pip(decoder); 3395 break; 3396 3397 case INTEL_PT_MODE_EXEC: 3398 decoder->exec_mode = decoder->packet.payload; 3399 break; 3400 3401 case INTEL_PT_MODE_TSX: 3402 intel_pt_update_in_tx(decoder); 3403 break; 3404 3405 case INTEL_PT_OVF: 3406 return intel_pt_overflow(decoder); 3407 3408 case INTEL_PT_BAD: /* Does not happen */ 3409 return intel_pt_bug(decoder); 3410 3411 case INTEL_PT_TRACESTOP: 3412 decoder->pge = false; 3413 decoder->continuous_period = false; 3414 intel_pt_clear_tx_flags(decoder); 3415 decoder->have_tma = false; 3416 break; 3417 3418 case INTEL_PT_PSB: 3419 decoder->state.psb_offset = decoder->pos; 3420 decoder->psb_ip = 0; 3421 decoder->last_ip = 0; 3422 decoder->have_last_ip = true; 3423 intel_pt_clear_stack(&decoder->stack); 3424 err = intel_pt_walk_psb(decoder); 3425 if (err) 3426 return err; 3427 decoder->state.type = INTEL_PT_PSB_EVT; 3428 decoder->state.from_ip = decoder->psb_ip; 3429 decoder->state.to_ip = 0; 3430 return 0; 3431 3432 case INTEL_PT_TNT: 3433 case INTEL_PT_PSBEND: 3434 case INTEL_PT_VMCS: 3435 case INTEL_PT_MNT: 3436 case INTEL_PT_PAD: 3437 case INTEL_PT_PTWRITE: 3438 case INTEL_PT_PTWRITE_IP: 3439 case INTEL_PT_EXSTOP: 3440 case INTEL_PT_EXSTOP_IP: 3441 case INTEL_PT_MWAIT: 3442 case INTEL_PT_PWRE: 3443 case INTEL_PT_PWRX: 3444 case INTEL_PT_BBP: 3445 case INTEL_PT_BIP: 3446 case INTEL_PT_BEP: 3447 case INTEL_PT_BEP_IP: 3448 default: 3449 break; 3450 } 3451 } 3452 } 3453 3454 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 3455 { 3456 int err; 3457 3458 decoder->set_fup_tx_flags = false; 3459 decoder->set_fup_ptw = false; 3460 decoder->set_fup_mwait = false; 3461 decoder->set_fup_pwre = false; 3462 decoder->set_fup_exstop = false; 3463 decoder->set_fup_bep = false; 3464 3465 if (!decoder->branch_enable) { 3466 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3467 decoder->overflow = false; 3468 decoder->state.type = 0; /* Do not have a sample */ 3469 return 0; 3470 } 3471 3472 intel_pt_log("Scanning for full IP\n"); 3473 err = intel_pt_walk_to_ip(decoder); 3474 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 3475 return err; 3476 3477 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 3478 if (decoder->hop) 3479 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3480 else 3481 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3482 decoder->overflow = false; 3483 3484 decoder->state.from_ip = 0; 3485 decoder->state.to_ip = decoder->ip; 3486 intel_pt_log_to("Setting IP", decoder->ip); 3487 3488 return 0; 3489 } 3490 3491 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 3492 { 3493 const unsigned char *end = decoder->buf + decoder->len; 3494 size_t i; 3495 3496 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 3497 if (i > decoder->len) 3498 continue; 3499 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 3500 return i; 3501 } 3502 return 0; 3503 } 3504 3505 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 3506 { 3507 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 3508 const char *psb = INTEL_PT_PSB_STR; 3509 3510 if (rest_psb > decoder->len || 3511 memcmp(decoder->buf, psb + part_psb, rest_psb)) 3512 return 0; 3513 3514 return rest_psb; 3515 } 3516 3517 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 3518 int part_psb) 3519 { 3520 int rest_psb, ret; 3521 3522 decoder->pos += decoder->len; 3523 decoder->len = 0; 3524 3525 ret = intel_pt_get_next_data(decoder, false); 3526 if (ret) 3527 return ret; 3528 3529 rest_psb = intel_pt_rest_psb(decoder, part_psb); 3530 if (!rest_psb) 3531 return 0; 3532 3533 decoder->pos -= part_psb; 3534 decoder->next_buf = decoder->buf + rest_psb; 3535 decoder->next_len = decoder->len - rest_psb; 3536 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3537 decoder->buf = decoder->temp_buf; 3538 decoder->len = INTEL_PT_PSB_LEN; 3539 3540 return 0; 3541 } 3542 3543 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 3544 { 3545 unsigned char *next; 3546 int ret; 3547 3548 intel_pt_log("Scanning for PSB\n"); 3549 while (1) { 3550 if (!decoder->len) { 3551 ret = intel_pt_get_next_data(decoder, false); 3552 if (ret) 3553 return ret; 3554 } 3555 3556 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 3557 INTEL_PT_PSB_LEN); 3558 if (!next) { 3559 int part_psb; 3560 3561 part_psb = intel_pt_part_psb(decoder); 3562 if (part_psb) { 3563 ret = intel_pt_get_split_psb(decoder, part_psb); 3564 if (ret) 3565 return ret; 3566 } else { 3567 decoder->pos += decoder->len; 3568 decoder->len = 0; 3569 } 3570 continue; 3571 } 3572 3573 decoder->pkt_step = next - decoder->buf; 3574 return intel_pt_get_next_packet(decoder); 3575 } 3576 } 3577 3578 static int intel_pt_sync(struct intel_pt_decoder *decoder) 3579 { 3580 int err; 3581 3582 decoder->pge = false; 3583 decoder->continuous_period = false; 3584 decoder->have_last_ip = false; 3585 decoder->last_ip = 0; 3586 decoder->psb_ip = 0; 3587 decoder->ip = 0; 3588 intel_pt_clear_stack(&decoder->stack); 3589 3590 err = intel_pt_scan_for_psb(decoder); 3591 if (err) 3592 return err; 3593 3594 if (decoder->vm_time_correlation) { 3595 decoder->in_psb = true; 3596 if (!decoder->timestamp) 3597 decoder->timestamp = 1; 3598 decoder->state.type = 0; 3599 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; 3600 return 0; 3601 } 3602 3603 decoder->have_last_ip = true; 3604 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 3605 3606 err = intel_pt_walk_psb(decoder); 3607 if (err) 3608 return err; 3609 3610 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 3611 decoder->state.from_ip = decoder->psb_ip; 3612 decoder->state.to_ip = 0; 3613 3614 if (decoder->ip) { 3615 /* 3616 * In hop mode, resample to get the PSB FUP ip as an 3617 * "instruction" sample. 3618 */ 3619 if (decoder->hop) 3620 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3621 else 3622 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3623 } 3624 3625 return 0; 3626 } 3627 3628 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 3629 { 3630 uint64_t est = decoder->sample_insn_cnt << 1; 3631 3632 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 3633 goto out; 3634 3635 est *= decoder->max_non_turbo_ratio; 3636 est /= decoder->cbr; 3637 out: 3638 return decoder->sample_timestamp + est; 3639 } 3640 3641 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 3642 { 3643 int err; 3644 3645 do { 3646 decoder->state.type = INTEL_PT_BRANCH; 3647 decoder->state.flags = 0; 3648 3649 switch (decoder->pkt_state) { 3650 case INTEL_PT_STATE_NO_PSB: 3651 err = intel_pt_sync(decoder); 3652 break; 3653 case INTEL_PT_STATE_NO_IP: 3654 decoder->have_last_ip = false; 3655 decoder->last_ip = 0; 3656 decoder->ip = 0; 3657 __fallthrough; 3658 case INTEL_PT_STATE_ERR_RESYNC: 3659 err = intel_pt_sync_ip(decoder); 3660 break; 3661 case INTEL_PT_STATE_IN_SYNC: 3662 err = intel_pt_walk_trace(decoder); 3663 break; 3664 case INTEL_PT_STATE_TNT: 3665 case INTEL_PT_STATE_TNT_CONT: 3666 err = intel_pt_walk_tnt(decoder); 3667 if (err == -EAGAIN) 3668 err = intel_pt_walk_trace(decoder); 3669 break; 3670 case INTEL_PT_STATE_TIP: 3671 case INTEL_PT_STATE_TIP_PGD: 3672 err = intel_pt_walk_tip(decoder); 3673 break; 3674 case INTEL_PT_STATE_FUP: 3675 err = intel_pt_walk_fup(decoder); 3676 if (err == -EAGAIN) 3677 err = intel_pt_walk_fup_tip(decoder); 3678 break; 3679 case INTEL_PT_STATE_FUP_NO_TIP: 3680 err = intel_pt_walk_fup(decoder); 3681 if (err == -EAGAIN) 3682 err = intel_pt_walk_trace(decoder); 3683 break; 3684 case INTEL_PT_STATE_FUP_IN_PSB: 3685 err = intel_pt_fup_in_psb(decoder); 3686 break; 3687 case INTEL_PT_STATE_RESAMPLE: 3688 err = intel_pt_resample(decoder); 3689 break; 3690 case INTEL_PT_STATE_VM_TIME_CORRELATION: 3691 err = intel_pt_vm_time_correlation(decoder); 3692 break; 3693 default: 3694 err = intel_pt_bug(decoder); 3695 break; 3696 } 3697 } while (err == -ENOLINK); 3698 3699 if (err) { 3700 decoder->state.err = intel_pt_ext_err(err); 3701 decoder->state.from_ip = decoder->ip; 3702 intel_pt_update_sample_time(decoder); 3703 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3704 intel_pt_set_nr(decoder); 3705 } else { 3706 decoder->state.err = 0; 3707 if (decoder->cbr != decoder->cbr_seen) { 3708 decoder->cbr_seen = decoder->cbr; 3709 if (!decoder->state.type) { 3710 decoder->state.from_ip = decoder->ip; 3711 decoder->state.to_ip = 0; 3712 } 3713 decoder->state.type |= INTEL_PT_CBR_CHG; 3714 decoder->state.cbr_payload = decoder->cbr_payload; 3715 decoder->state.cbr = decoder->cbr; 3716 } 3717 if (intel_pt_sample_time(decoder->pkt_state)) { 3718 intel_pt_update_sample_time(decoder); 3719 if (decoder->sample_cyc) { 3720 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3721 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3722 decoder->sample_cyc = false; 3723 } 3724 } 3725 /* 3726 * When using only TSC/MTC to compute cycles, IPC can be 3727 * sampled as soon as the cycle count changes. 3728 */ 3729 if (!decoder->have_cyc) 3730 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3731 } 3732 3733 /* Let PSB event always have TSC timestamp */ 3734 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 3735 decoder->sample_timestamp = decoder->tsc_timestamp; 3736 3737 decoder->state.from_nr = decoder->nr; 3738 decoder->state.to_nr = decoder->next_nr; 3739 decoder->nr = decoder->next_nr; 3740 3741 decoder->state.timestamp = decoder->sample_timestamp; 3742 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 3743 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 3744 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 3745 3746 return &decoder->state; 3747 } 3748 3749 /** 3750 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 3751 * @buf: pointer to buffer pointer 3752 * @len: size of buffer 3753 * 3754 * Updates the buffer pointer to point to the start of the next PSB packet if 3755 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 3756 * @len is adjusted accordingly. 3757 * 3758 * Return: %true if a PSB packet is found, %false otherwise. 3759 */ 3760 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 3761 { 3762 unsigned char *next; 3763 3764 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3765 if (next) { 3766 *len -= next - *buf; 3767 *buf = next; 3768 return true; 3769 } 3770 return false; 3771 } 3772 3773 /** 3774 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 3775 * packet. 3776 * @buf: pointer to buffer pointer 3777 * @len: size of buffer 3778 * 3779 * Updates the buffer pointer to point to the start of the following PSB packet 3780 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 3781 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 3782 * 3783 * Return: %true if a PSB packet is found, %false otherwise. 3784 */ 3785 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 3786 { 3787 unsigned char *next; 3788 3789 if (!*len) 3790 return false; 3791 3792 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3793 if (next) { 3794 *len -= next - *buf; 3795 *buf = next; 3796 return true; 3797 } 3798 return false; 3799 } 3800 3801 /** 3802 * intel_pt_last_psb - find the last PSB packet in a buffer. 3803 * @buf: buffer 3804 * @len: size of buffer 3805 * 3806 * This function finds the last PSB in a buffer. 3807 * 3808 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 3809 */ 3810 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 3811 { 3812 const char *n = INTEL_PT_PSB_STR; 3813 unsigned char *p; 3814 size_t k; 3815 3816 if (len < INTEL_PT_PSB_LEN) 3817 return NULL; 3818 3819 k = len - INTEL_PT_PSB_LEN + 1; 3820 while (1) { 3821 p = memrchr(buf, n[0], k); 3822 if (!p) 3823 return NULL; 3824 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 3825 return p; 3826 k = p - buf; 3827 if (!k) 3828 return NULL; 3829 } 3830 } 3831 3832 /** 3833 * intel_pt_next_tsc - find and return next TSC. 3834 * @buf: buffer 3835 * @len: size of buffer 3836 * @tsc: TSC value returned 3837 * @rem: returns remaining size when TSC is found 3838 * 3839 * Find a TSC packet in @buf and return the TSC value. This function assumes 3840 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 3841 * PSBEND packet is found. 3842 * 3843 * Return: %true if TSC is found, false otherwise. 3844 */ 3845 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 3846 size_t *rem) 3847 { 3848 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 3849 struct intel_pt_pkt packet; 3850 int ret; 3851 3852 while (len) { 3853 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 3854 if (ret <= 0) 3855 return false; 3856 if (packet.type == INTEL_PT_TSC) { 3857 *tsc = packet.payload; 3858 *rem = len; 3859 return true; 3860 } 3861 if (packet.type == INTEL_PT_PSBEND) 3862 return false; 3863 buf += ret; 3864 len -= ret; 3865 } 3866 return false; 3867 } 3868 3869 /** 3870 * intel_pt_tsc_cmp - compare 7-byte TSCs. 3871 * @tsc1: first TSC to compare 3872 * @tsc2: second TSC to compare 3873 * 3874 * This function compares 7-byte TSC values allowing for the possibility that 3875 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 3876 * around so for that purpose this function assumes the absolute difference is 3877 * less than half the maximum difference. 3878 * 3879 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 3880 * after @tsc2. 3881 */ 3882 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 3883 { 3884 const uint64_t halfway = (1ULL << 55); 3885 3886 if (tsc1 == tsc2) 3887 return 0; 3888 3889 if (tsc1 < tsc2) { 3890 if (tsc2 - tsc1 < halfway) 3891 return -1; 3892 else 3893 return 1; 3894 } else { 3895 if (tsc1 - tsc2 < halfway) 3896 return 1; 3897 else 3898 return -1; 3899 } 3900 } 3901 3902 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 3903 3904 /** 3905 * adj_for_padding - adjust overlap to account for padding. 3906 * @buf_b: second buffer 3907 * @buf_a: first buffer 3908 * @len_a: size of first buffer 3909 * 3910 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 3911 * accordingly. 3912 * 3913 * Return: A pointer into @buf_b from where non-overlapped data starts 3914 */ 3915 static unsigned char *adj_for_padding(unsigned char *buf_b, 3916 unsigned char *buf_a, size_t len_a) 3917 { 3918 unsigned char *p = buf_b - MAX_PADDING; 3919 unsigned char *q = buf_a + len_a - MAX_PADDING; 3920 int i; 3921 3922 for (i = MAX_PADDING; i; i--, p++, q++) { 3923 if (*p != *q) 3924 break; 3925 } 3926 3927 return p; 3928 } 3929 3930 /** 3931 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3932 * using TSC. 3933 * @buf_a: first buffer 3934 * @len_a: size of first buffer 3935 * @buf_b: second buffer 3936 * @len_b: size of second buffer 3937 * @consecutive: returns true if there is data in buf_b that is consecutive 3938 * to buf_a 3939 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 3940 * 3941 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3942 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3943 * walk forward in @buf_b until a later TSC is found. A precondition is that 3944 * @buf_a and @buf_b are positioned at a PSB. 3945 * 3946 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3947 * @buf_b + @len_b if there is no non-overlapped data. 3948 */ 3949 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3950 size_t len_a, 3951 unsigned char *buf_b, 3952 size_t len_b, bool *consecutive, 3953 bool ooo_tsc) 3954 { 3955 uint64_t tsc_a, tsc_b; 3956 unsigned char *p; 3957 size_t len, rem_a, rem_b; 3958 3959 p = intel_pt_last_psb(buf_a, len_a); 3960 if (!p) 3961 return buf_b; /* No PSB in buf_a => no overlap */ 3962 3963 len = len_a - (p - buf_a); 3964 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3965 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3966 len_a -= len; 3967 p = intel_pt_last_psb(buf_a, len_a); 3968 if (!p) 3969 return buf_b; /* No full PSB+ => assume no overlap */ 3970 len = len_a - (p - buf_a); 3971 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 3972 return buf_b; /* No TSC in buf_a => assume no overlap */ 3973 } 3974 3975 while (1) { 3976 /* Ignore PSB+ with no TSC */ 3977 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 3978 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 3979 3980 /* Same TSC, so buffers are consecutive */ 3981 if (!cmp && rem_b >= rem_a) { 3982 unsigned char *start; 3983 3984 *consecutive = true; 3985 start = buf_b + len_b - (rem_b - rem_a); 3986 return adj_for_padding(start, buf_a, len_a); 3987 } 3988 if (cmp < 0 && !ooo_tsc) 3989 return buf_b; /* tsc_a < tsc_b => no overlap */ 3990 } 3991 3992 if (!intel_pt_step_psb(&buf_b, &len_b)) 3993 return buf_b + len_b; /* No PSB in buf_b => no data */ 3994 } 3995 } 3996 3997 /** 3998 * intel_pt_find_overlap - determine start of non-overlapped trace data. 3999 * @buf_a: first buffer 4000 * @len_a: size of first buffer 4001 * @buf_b: second buffer 4002 * @len_b: size of second buffer 4003 * @have_tsc: can use TSC packets to detect overlap 4004 * @consecutive: returns true if there is data in buf_b that is consecutive 4005 * to buf_a 4006 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4007 * 4008 * When trace samples or snapshots are recorded there is the possibility that 4009 * the data overlaps. Note that, for the purposes of decoding, data is only 4010 * useful if it begins with a PSB packet. 4011 * 4012 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4013 * @buf_b + @len_b if there is no non-overlapped data. 4014 */ 4015 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 4016 unsigned char *buf_b, size_t len_b, 4017 bool have_tsc, bool *consecutive, 4018 bool ooo_tsc) 4019 { 4020 unsigned char *found; 4021 4022 /* Buffer 'b' must start at PSB so throw away everything before that */ 4023 if (!intel_pt_next_psb(&buf_b, &len_b)) 4024 return buf_b + len_b; /* No PSB */ 4025 4026 if (!intel_pt_next_psb(&buf_a, &len_a)) 4027 return buf_b; /* No overlap */ 4028 4029 if (have_tsc) { 4030 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 4031 consecutive, ooo_tsc); 4032 if (found) 4033 return found; 4034 } 4035 4036 /* 4037 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 4038 * we can ignore the first part of buffer 'a'. 4039 */ 4040 while (len_b < len_a) { 4041 if (!intel_pt_step_psb(&buf_a, &len_a)) 4042 return buf_b; /* No overlap */ 4043 } 4044 4045 /* Now len_b >= len_a */ 4046 while (1) { 4047 /* Potential overlap so check the bytes */ 4048 found = memmem(buf_a, len_a, buf_b, len_a); 4049 if (found) { 4050 *consecutive = true; 4051 return adj_for_padding(buf_b + len_a, buf_a, len_a); 4052 } 4053 4054 /* Try again at next PSB in buffer 'a' */ 4055 if (!intel_pt_step_psb(&buf_a, &len_a)) 4056 return buf_b; /* No overlap */ 4057 } 4058 } 4059 4060 /** 4061 * struct fast_forward_data - data used by intel_pt_ff_cb(). 4062 * @timestamp: timestamp to fast forward towards 4063 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 4064 * the fast forward timestamp. 4065 */ 4066 struct fast_forward_data { 4067 uint64_t timestamp; 4068 uint64_t buf_timestamp; 4069 }; 4070 4071 /** 4072 * intel_pt_ff_cb - fast forward lookahead callback. 4073 * @buffer: Intel PT trace buffer 4074 * @data: opaque pointer to fast forward data (struct fast_forward_data) 4075 * 4076 * Determine if @buffer trace is past the fast forward timestamp. 4077 * 4078 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 4079 * timestamp, and 0 otherwise. 4080 */ 4081 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 4082 { 4083 struct fast_forward_data *d = data; 4084 unsigned char *buf; 4085 uint64_t tsc; 4086 size_t rem; 4087 size_t len; 4088 4089 buf = (unsigned char *)buffer->buf; 4090 len = buffer->len; 4091 4092 if (!intel_pt_next_psb(&buf, &len) || 4093 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 4094 return 0; 4095 4096 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 4097 4098 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 4099 tsc, buffer->ref_timestamp); 4100 4101 /* 4102 * If the buffer contains a timestamp earlier that the fast forward 4103 * timestamp, then record it, else stop. 4104 */ 4105 if (tsc < d->timestamp) 4106 d->buf_timestamp = buffer->ref_timestamp; 4107 else 4108 return 1; 4109 4110 return 0; 4111 } 4112 4113 /** 4114 * intel_pt_fast_forward - reposition decoder forwards. 4115 * @decoder: Intel PT decoder 4116 * @timestamp: timestamp to fast forward towards 4117 * 4118 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 4119 * 4120 * Return: 0 on success or negative error code on failure. 4121 */ 4122 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 4123 { 4124 struct fast_forward_data d = { .timestamp = timestamp }; 4125 unsigned char *buf; 4126 size_t len; 4127 int err; 4128 4129 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 4130 4131 /* Find buffer timestamp of buffer to fast forward to */ 4132 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 4133 if (err < 0) 4134 return err; 4135 4136 /* Walk to buffer with same buffer timestamp */ 4137 if (d.buf_timestamp) { 4138 do { 4139 decoder->pos += decoder->len; 4140 decoder->len = 0; 4141 err = intel_pt_get_next_data(decoder, true); 4142 /* -ENOLINK means non-consecutive trace */ 4143 if (err && err != -ENOLINK) 4144 return err; 4145 } while (decoder->buf_timestamp != d.buf_timestamp); 4146 } 4147 4148 if (!decoder->buf) 4149 return 0; 4150 4151 buf = (unsigned char *)decoder->buf; 4152 len = decoder->len; 4153 4154 if (!intel_pt_next_psb(&buf, &len)) 4155 return 0; 4156 4157 /* 4158 * Walk PSBs while the PSB timestamp is less than the fast forward 4159 * timestamp. 4160 */ 4161 do { 4162 uint64_t tsc; 4163 size_t rem; 4164 4165 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 4166 break; 4167 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 4168 /* 4169 * A TSC packet can slip past MTC packets but, after fast 4170 * forward, decoding starts at the TSC timestamp. That means 4171 * the timestamps may not be exactly the same as the timestamps 4172 * that would have been decoded without fast forward. 4173 */ 4174 if (tsc < timestamp) { 4175 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 4176 decoder->pos += decoder->len - len; 4177 decoder->buf = buf; 4178 decoder->len = len; 4179 intel_pt_reposition(decoder); 4180 } else { 4181 break; 4182 } 4183 } while (intel_pt_step_psb(&buf, &len)); 4184 4185 return 0; 4186 } 4187