1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define INTEL_PT_BLK_SIZE 1024 28 29 #define BIT63 (((uint64_t)1 << 63)) 30 31 #define INTEL_PT_RETURN 1 32 33 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 34 #define INTEL_PT_MAX_LOOPS 10000 35 36 struct intel_pt_blk { 37 struct intel_pt_blk *prev; 38 uint64_t ip[INTEL_PT_BLK_SIZE]; 39 }; 40 41 struct intel_pt_stack { 42 struct intel_pt_blk *blk; 43 struct intel_pt_blk *spare; 44 int pos; 45 }; 46 47 enum intel_pt_pkt_state { 48 INTEL_PT_STATE_NO_PSB, 49 INTEL_PT_STATE_NO_IP, 50 INTEL_PT_STATE_ERR_RESYNC, 51 INTEL_PT_STATE_IN_SYNC, 52 INTEL_PT_STATE_TNT_CONT, 53 INTEL_PT_STATE_TNT, 54 INTEL_PT_STATE_TIP, 55 INTEL_PT_STATE_TIP_PGD, 56 INTEL_PT_STATE_FUP, 57 INTEL_PT_STATE_FUP_NO_TIP, 58 INTEL_PT_STATE_RESAMPLE, 59 }; 60 61 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 62 { 63 switch (pkt_state) { 64 case INTEL_PT_STATE_NO_PSB: 65 case INTEL_PT_STATE_NO_IP: 66 case INTEL_PT_STATE_ERR_RESYNC: 67 case INTEL_PT_STATE_IN_SYNC: 68 case INTEL_PT_STATE_TNT_CONT: 69 case INTEL_PT_STATE_RESAMPLE: 70 return true; 71 case INTEL_PT_STATE_TNT: 72 case INTEL_PT_STATE_TIP: 73 case INTEL_PT_STATE_TIP_PGD: 74 case INTEL_PT_STATE_FUP: 75 case INTEL_PT_STATE_FUP_NO_TIP: 76 return false; 77 default: 78 return true; 79 }; 80 } 81 82 #ifdef INTEL_PT_STRICT 83 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 84 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 85 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 86 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 87 #else 88 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 89 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 90 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 91 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 92 #endif 93 94 struct intel_pt_decoder { 95 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 96 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 97 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 98 uint64_t max_insn_cnt, void *data); 99 bool (*pgd_ip)(uint64_t ip, void *data); 100 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 101 void *data; 102 struct intel_pt_state state; 103 const unsigned char *buf; 104 size_t len; 105 bool return_compression; 106 bool branch_enable; 107 bool mtc_insn; 108 bool pge; 109 bool have_tma; 110 bool have_cyc; 111 bool fixup_last_mtc; 112 bool have_last_ip; 113 bool in_psb; 114 bool hop; 115 bool hop_psb_fup; 116 bool leap; 117 enum intel_pt_param_flags flags; 118 uint64_t pos; 119 uint64_t last_ip; 120 uint64_t ip; 121 uint64_t cr3; 122 uint64_t timestamp; 123 uint64_t tsc_timestamp; 124 uint64_t ref_timestamp; 125 uint64_t buf_timestamp; 126 uint64_t sample_timestamp; 127 uint64_t ret_addr; 128 uint64_t ctc_timestamp; 129 uint64_t ctc_delta; 130 uint64_t cycle_cnt; 131 uint64_t cyc_ref_timestamp; 132 uint32_t last_mtc; 133 uint32_t tsc_ctc_ratio_n; 134 uint32_t tsc_ctc_ratio_d; 135 uint32_t tsc_ctc_mult; 136 uint32_t tsc_slip; 137 uint32_t ctc_rem_mask; 138 int mtc_shift; 139 struct intel_pt_stack stack; 140 enum intel_pt_pkt_state pkt_state; 141 enum intel_pt_pkt_ctx pkt_ctx; 142 enum intel_pt_pkt_ctx prev_pkt_ctx; 143 enum intel_pt_blk_type blk_type; 144 int blk_type_pos; 145 struct intel_pt_pkt packet; 146 struct intel_pt_pkt tnt; 147 int pkt_step; 148 int pkt_len; 149 int last_packet_type; 150 unsigned int cbr; 151 unsigned int cbr_seen; 152 unsigned int max_non_turbo_ratio; 153 double max_non_turbo_ratio_fp; 154 double cbr_cyc_to_tsc; 155 double calc_cyc_to_tsc; 156 bool have_calc_cyc_to_tsc; 157 int exec_mode; 158 unsigned int insn_bytes; 159 uint64_t period; 160 enum intel_pt_period_type period_type; 161 uint64_t tot_insn_cnt; 162 uint64_t period_insn_cnt; 163 uint64_t period_mask; 164 uint64_t period_ticks; 165 uint64_t last_masked_timestamp; 166 uint64_t tot_cyc_cnt; 167 uint64_t sample_tot_cyc_cnt; 168 uint64_t base_cyc_cnt; 169 uint64_t cyc_cnt_timestamp; 170 double tsc_to_cyc; 171 bool continuous_period; 172 bool overflow; 173 bool set_fup_tx_flags; 174 bool set_fup_ptw; 175 bool set_fup_mwait; 176 bool set_fup_pwre; 177 bool set_fup_exstop; 178 bool set_fup_bep; 179 bool sample_cyc; 180 unsigned int fup_tx_flags; 181 unsigned int tx_flags; 182 uint64_t fup_ptw_payload; 183 uint64_t fup_mwait_payload; 184 uint64_t fup_pwre_payload; 185 uint64_t cbr_payload; 186 uint64_t timestamp_insn_cnt; 187 uint64_t sample_insn_cnt; 188 uint64_t stuck_ip; 189 int no_progress; 190 int stuck_ip_prd; 191 int stuck_ip_cnt; 192 const unsigned char *next_buf; 193 size_t next_len; 194 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 195 }; 196 197 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 198 { 199 int i; 200 201 for (i = 0; x != 1; i++) 202 x >>= 1; 203 204 return x << i; 205 } 206 207 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 208 { 209 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 210 uint64_t period; 211 212 period = intel_pt_lower_power_of_2(decoder->period); 213 decoder->period_mask = ~(period - 1); 214 decoder->period_ticks = period; 215 } 216 } 217 218 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 219 { 220 if (!d) 221 return 0; 222 return (t / d) * n + ((t % d) * n) / d; 223 } 224 225 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 226 { 227 struct intel_pt_decoder *decoder; 228 229 if (!params->get_trace || !params->walk_insn) 230 return NULL; 231 232 decoder = zalloc(sizeof(struct intel_pt_decoder)); 233 if (!decoder) 234 return NULL; 235 236 decoder->get_trace = params->get_trace; 237 decoder->walk_insn = params->walk_insn; 238 decoder->pgd_ip = params->pgd_ip; 239 decoder->lookahead = params->lookahead; 240 decoder->data = params->data; 241 decoder->return_compression = params->return_compression; 242 decoder->branch_enable = params->branch_enable; 243 decoder->hop = params->quick >= 1; 244 decoder->leap = params->quick >= 2; 245 246 decoder->flags = params->flags; 247 248 decoder->period = params->period; 249 decoder->period_type = params->period_type; 250 251 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 252 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 253 254 intel_pt_setup_period(decoder); 255 256 decoder->mtc_shift = params->mtc_period; 257 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 258 259 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 260 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 261 262 if (!decoder->tsc_ctc_ratio_n) 263 decoder->tsc_ctc_ratio_d = 0; 264 265 if (decoder->tsc_ctc_ratio_d) { 266 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 267 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 268 decoder->tsc_ctc_ratio_d; 269 } 270 271 /* 272 * A TSC packet can slip past MTC packets so that the timestamp appears 273 * to go backwards. One estimate is that can be up to about 40 CPU 274 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 275 * slippage an order of magnitude more to be on the safe side. 276 */ 277 decoder->tsc_slip = 0x10000; 278 279 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 280 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 281 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 282 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 283 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 284 285 if (decoder->hop) 286 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 287 288 return decoder; 289 } 290 291 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 292 { 293 struct intel_pt_blk *blk = stack->blk; 294 295 stack->blk = blk->prev; 296 if (!stack->spare) 297 stack->spare = blk; 298 else 299 free(blk); 300 } 301 302 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 303 { 304 if (!stack->pos) { 305 if (!stack->blk) 306 return 0; 307 intel_pt_pop_blk(stack); 308 if (!stack->blk) 309 return 0; 310 stack->pos = INTEL_PT_BLK_SIZE; 311 } 312 return stack->blk->ip[--stack->pos]; 313 } 314 315 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 316 { 317 struct intel_pt_blk *blk; 318 319 if (stack->spare) { 320 blk = stack->spare; 321 stack->spare = NULL; 322 } else { 323 blk = malloc(sizeof(struct intel_pt_blk)); 324 if (!blk) 325 return -ENOMEM; 326 } 327 328 blk->prev = stack->blk; 329 stack->blk = blk; 330 stack->pos = 0; 331 return 0; 332 } 333 334 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 335 { 336 int err; 337 338 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 339 err = intel_pt_alloc_blk(stack); 340 if (err) 341 return err; 342 } 343 344 stack->blk->ip[stack->pos++] = ip; 345 return 0; 346 } 347 348 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 349 { 350 while (stack->blk) 351 intel_pt_pop_blk(stack); 352 stack->pos = 0; 353 } 354 355 static void intel_pt_free_stack(struct intel_pt_stack *stack) 356 { 357 intel_pt_clear_stack(stack); 358 zfree(&stack->blk); 359 zfree(&stack->spare); 360 } 361 362 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 363 { 364 intel_pt_free_stack(&decoder->stack); 365 free(decoder); 366 } 367 368 static int intel_pt_ext_err(int code) 369 { 370 switch (code) { 371 case -ENOMEM: 372 return INTEL_PT_ERR_NOMEM; 373 case -ENOSYS: 374 return INTEL_PT_ERR_INTERN; 375 case -EBADMSG: 376 return INTEL_PT_ERR_BADPKT; 377 case -ENODATA: 378 return INTEL_PT_ERR_NODATA; 379 case -EILSEQ: 380 return INTEL_PT_ERR_NOINSN; 381 case -ENOENT: 382 return INTEL_PT_ERR_MISMAT; 383 case -EOVERFLOW: 384 return INTEL_PT_ERR_OVR; 385 case -ENOSPC: 386 return INTEL_PT_ERR_LOST; 387 case -ELOOP: 388 return INTEL_PT_ERR_NELOOP; 389 default: 390 return INTEL_PT_ERR_UNK; 391 } 392 } 393 394 static const char *intel_pt_err_msgs[] = { 395 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 396 [INTEL_PT_ERR_INTERN] = "Internal error", 397 [INTEL_PT_ERR_BADPKT] = "Bad packet", 398 [INTEL_PT_ERR_NODATA] = "No more data", 399 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 400 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 401 [INTEL_PT_ERR_OVR] = "Overflow packet", 402 [INTEL_PT_ERR_LOST] = "Lost trace data", 403 [INTEL_PT_ERR_UNK] = "Unknown error!", 404 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 405 }; 406 407 int intel_pt__strerror(int code, char *buf, size_t buflen) 408 { 409 if (code < 1 || code >= INTEL_PT_ERR_MAX) 410 code = INTEL_PT_ERR_UNK; 411 strlcpy(buf, intel_pt_err_msgs[code], buflen); 412 return 0; 413 } 414 415 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 416 uint64_t last_ip) 417 { 418 uint64_t ip; 419 420 switch (packet->count) { 421 case 1: 422 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 423 packet->payload; 424 break; 425 case 2: 426 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 427 packet->payload; 428 break; 429 case 3: 430 ip = packet->payload; 431 /* Sign-extend 6-byte ip */ 432 if (ip & (uint64_t)0x800000000000ULL) 433 ip |= (uint64_t)0xffff000000000000ULL; 434 break; 435 case 4: 436 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 437 packet->payload; 438 break; 439 case 6: 440 ip = packet->payload; 441 break; 442 default: 443 return 0; 444 } 445 446 return ip; 447 } 448 449 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 450 { 451 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 452 decoder->have_last_ip = true; 453 } 454 455 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 456 { 457 intel_pt_set_last_ip(decoder); 458 decoder->ip = decoder->last_ip; 459 } 460 461 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 462 { 463 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 464 decoder->buf); 465 } 466 467 static int intel_pt_bug(struct intel_pt_decoder *decoder) 468 { 469 intel_pt_log("ERROR: Internal error\n"); 470 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 471 return -ENOSYS; 472 } 473 474 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 475 { 476 decoder->tx_flags = 0; 477 } 478 479 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 480 { 481 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 482 } 483 484 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 485 { 486 intel_pt_clear_tx_flags(decoder); 487 decoder->have_tma = false; 488 decoder->pkt_len = 1; 489 decoder->pkt_step = 1; 490 intel_pt_decoder_log_packet(decoder); 491 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 492 intel_pt_log("ERROR: Bad packet\n"); 493 decoder->pkt_state = INTEL_PT_STATE_ERR1; 494 } 495 return -EBADMSG; 496 } 497 498 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 499 { 500 decoder->sample_timestamp = decoder->timestamp; 501 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 502 } 503 504 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 505 { 506 decoder->ip = 0; 507 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 508 decoder->timestamp = 0; 509 decoder->have_tma = false; 510 } 511 512 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 513 { 514 struct intel_pt_buffer buffer = { .buf = 0, }; 515 int ret; 516 517 decoder->pkt_step = 0; 518 519 intel_pt_log("Getting more data\n"); 520 ret = decoder->get_trace(&buffer, decoder->data); 521 if (ret) 522 return ret; 523 decoder->buf = buffer.buf; 524 decoder->len = buffer.len; 525 if (!decoder->len) { 526 intel_pt_log("No more data\n"); 527 return -ENODATA; 528 } 529 decoder->buf_timestamp = buffer.ref_timestamp; 530 if (!buffer.consecutive || reposition) { 531 intel_pt_reposition(decoder); 532 decoder->ref_timestamp = buffer.ref_timestamp; 533 decoder->state.trace_nr = buffer.trace_nr; 534 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 535 decoder->ref_timestamp); 536 return -ENOLINK; 537 } 538 539 return 0; 540 } 541 542 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 543 bool reposition) 544 { 545 if (!decoder->next_buf) 546 return intel_pt_get_data(decoder, reposition); 547 548 decoder->buf = decoder->next_buf; 549 decoder->len = decoder->next_len; 550 decoder->next_buf = 0; 551 decoder->next_len = 0; 552 return 0; 553 } 554 555 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 556 { 557 unsigned char *buf = decoder->temp_buf; 558 size_t old_len, len, n; 559 int ret; 560 561 old_len = decoder->len; 562 len = decoder->len; 563 memcpy(buf, decoder->buf, len); 564 565 ret = intel_pt_get_data(decoder, false); 566 if (ret) { 567 decoder->pos += old_len; 568 return ret < 0 ? ret : -EINVAL; 569 } 570 571 n = INTEL_PT_PKT_MAX_SZ - len; 572 if (n > decoder->len) 573 n = decoder->len; 574 memcpy(buf + len, decoder->buf, n); 575 len += n; 576 577 decoder->prev_pkt_ctx = decoder->pkt_ctx; 578 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 579 if (ret < (int)old_len) { 580 decoder->next_buf = decoder->buf; 581 decoder->next_len = decoder->len; 582 decoder->buf = buf; 583 decoder->len = old_len; 584 return intel_pt_bad_packet(decoder); 585 } 586 587 decoder->next_buf = decoder->buf + (ret - old_len); 588 decoder->next_len = decoder->len - (ret - old_len); 589 590 decoder->buf = buf; 591 decoder->len = ret; 592 593 return ret; 594 } 595 596 struct intel_pt_pkt_info { 597 struct intel_pt_decoder *decoder; 598 struct intel_pt_pkt packet; 599 uint64_t pos; 600 int pkt_len; 601 int last_packet_type; 602 void *data; 603 }; 604 605 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 606 607 /* Lookahead packets in current buffer */ 608 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 609 intel_pt_pkt_cb_t cb, void *data) 610 { 611 struct intel_pt_pkt_info pkt_info; 612 const unsigned char *buf = decoder->buf; 613 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 614 size_t len = decoder->len; 615 int ret; 616 617 pkt_info.decoder = decoder; 618 pkt_info.pos = decoder->pos; 619 pkt_info.pkt_len = decoder->pkt_step; 620 pkt_info.last_packet_type = decoder->last_packet_type; 621 pkt_info.data = data; 622 623 while (1) { 624 do { 625 pkt_info.pos += pkt_info.pkt_len; 626 buf += pkt_info.pkt_len; 627 len -= pkt_info.pkt_len; 628 629 if (!len) 630 return INTEL_PT_NEED_MORE_BYTES; 631 632 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 633 &pkt_ctx); 634 if (!ret) 635 return INTEL_PT_NEED_MORE_BYTES; 636 if (ret < 0) 637 return ret; 638 639 pkt_info.pkt_len = ret; 640 } while (pkt_info.packet.type == INTEL_PT_PAD); 641 642 ret = cb(&pkt_info); 643 if (ret) 644 return 0; 645 646 pkt_info.last_packet_type = pkt_info.packet.type; 647 } 648 } 649 650 struct intel_pt_calc_cyc_to_tsc_info { 651 uint64_t cycle_cnt; 652 unsigned int cbr; 653 uint32_t last_mtc; 654 uint64_t ctc_timestamp; 655 uint64_t ctc_delta; 656 uint64_t tsc_timestamp; 657 uint64_t timestamp; 658 bool have_tma; 659 bool fixup_last_mtc; 660 bool from_mtc; 661 double cbr_cyc_to_tsc; 662 }; 663 664 /* 665 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 666 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 667 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 668 * packet by copying the missing bits from the current MTC assuming the least 669 * difference between the two, and that the current MTC comes after last_mtc. 670 */ 671 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 672 uint32_t *last_mtc) 673 { 674 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 675 uint32_t mask = ~(first_missing_bit - 1); 676 677 *last_mtc |= mtc & mask; 678 if (*last_mtc >= mtc) { 679 *last_mtc -= first_missing_bit; 680 *last_mtc &= 0xff; 681 } 682 } 683 684 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 685 { 686 struct intel_pt_decoder *decoder = pkt_info->decoder; 687 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 688 uint64_t timestamp; 689 double cyc_to_tsc; 690 unsigned int cbr; 691 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 692 693 switch (pkt_info->packet.type) { 694 case INTEL_PT_TNT: 695 case INTEL_PT_TIP_PGE: 696 case INTEL_PT_TIP: 697 case INTEL_PT_FUP: 698 case INTEL_PT_PSB: 699 case INTEL_PT_PIP: 700 case INTEL_PT_MODE_EXEC: 701 case INTEL_PT_MODE_TSX: 702 case INTEL_PT_PSBEND: 703 case INTEL_PT_PAD: 704 case INTEL_PT_VMCS: 705 case INTEL_PT_MNT: 706 case INTEL_PT_PTWRITE: 707 case INTEL_PT_PTWRITE_IP: 708 case INTEL_PT_BBP: 709 case INTEL_PT_BIP: 710 case INTEL_PT_BEP: 711 case INTEL_PT_BEP_IP: 712 return 0; 713 714 case INTEL_PT_MTC: 715 if (!data->have_tma) 716 return 0; 717 718 mtc = pkt_info->packet.payload; 719 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 720 data->fixup_last_mtc = false; 721 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 722 &data->last_mtc); 723 } 724 if (mtc > data->last_mtc) 725 mtc_delta = mtc - data->last_mtc; 726 else 727 mtc_delta = mtc + 256 - data->last_mtc; 728 data->ctc_delta += mtc_delta << decoder->mtc_shift; 729 data->last_mtc = mtc; 730 731 if (decoder->tsc_ctc_mult) { 732 timestamp = data->ctc_timestamp + 733 data->ctc_delta * decoder->tsc_ctc_mult; 734 } else { 735 timestamp = data->ctc_timestamp + 736 multdiv(data->ctc_delta, 737 decoder->tsc_ctc_ratio_n, 738 decoder->tsc_ctc_ratio_d); 739 } 740 741 if (timestamp < data->timestamp) 742 return 1; 743 744 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 745 data->timestamp = timestamp; 746 return 0; 747 } 748 749 break; 750 751 case INTEL_PT_TSC: 752 /* 753 * For now, do not support using TSC packets - refer 754 * intel_pt_calc_cyc_to_tsc(). 755 */ 756 if (data->from_mtc) 757 return 1; 758 timestamp = pkt_info->packet.payload | 759 (data->timestamp & (0xffULL << 56)); 760 if (data->from_mtc && timestamp < data->timestamp && 761 data->timestamp - timestamp < decoder->tsc_slip) 762 return 1; 763 if (timestamp < data->timestamp) 764 timestamp += (1ULL << 56); 765 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 766 if (data->from_mtc) 767 return 1; 768 data->tsc_timestamp = timestamp; 769 data->timestamp = timestamp; 770 return 0; 771 } 772 break; 773 774 case INTEL_PT_TMA: 775 if (data->from_mtc) 776 return 1; 777 778 if (!decoder->tsc_ctc_ratio_d) 779 return 0; 780 781 ctc = pkt_info->packet.payload; 782 fc = pkt_info->packet.count; 783 ctc_rem = ctc & decoder->ctc_rem_mask; 784 785 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 786 787 data->ctc_timestamp = data->tsc_timestamp - fc; 788 if (decoder->tsc_ctc_mult) { 789 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 790 } else { 791 data->ctc_timestamp -= 792 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 793 decoder->tsc_ctc_ratio_d); 794 } 795 796 data->ctc_delta = 0; 797 data->have_tma = true; 798 data->fixup_last_mtc = true; 799 800 return 0; 801 802 case INTEL_PT_CYC: 803 data->cycle_cnt += pkt_info->packet.payload; 804 return 0; 805 806 case INTEL_PT_CBR: 807 cbr = pkt_info->packet.payload; 808 if (data->cbr && data->cbr != cbr) 809 return 1; 810 data->cbr = cbr; 811 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 812 return 0; 813 814 case INTEL_PT_TIP_PGD: 815 case INTEL_PT_TRACESTOP: 816 case INTEL_PT_EXSTOP: 817 case INTEL_PT_EXSTOP_IP: 818 case INTEL_PT_MWAIT: 819 case INTEL_PT_PWRE: 820 case INTEL_PT_PWRX: 821 case INTEL_PT_OVF: 822 case INTEL_PT_BAD: /* Does not happen */ 823 default: 824 return 1; 825 } 826 827 if (!data->cbr && decoder->cbr) { 828 data->cbr = decoder->cbr; 829 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 830 } 831 832 if (!data->cycle_cnt) 833 return 1; 834 835 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 836 837 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 838 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 839 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 840 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 841 return 1; 842 } 843 844 decoder->calc_cyc_to_tsc = cyc_to_tsc; 845 decoder->have_calc_cyc_to_tsc = true; 846 847 if (data->cbr) { 848 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 849 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 850 } else { 851 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 852 cyc_to_tsc, pkt_info->pos); 853 } 854 855 return 1; 856 } 857 858 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 859 bool from_mtc) 860 { 861 struct intel_pt_calc_cyc_to_tsc_info data = { 862 .cycle_cnt = 0, 863 .cbr = 0, 864 .last_mtc = decoder->last_mtc, 865 .ctc_timestamp = decoder->ctc_timestamp, 866 .ctc_delta = decoder->ctc_delta, 867 .tsc_timestamp = decoder->tsc_timestamp, 868 .timestamp = decoder->timestamp, 869 .have_tma = decoder->have_tma, 870 .fixup_last_mtc = decoder->fixup_last_mtc, 871 .from_mtc = from_mtc, 872 .cbr_cyc_to_tsc = 0, 873 }; 874 875 /* 876 * For now, do not support using TSC packets for at least the reasons: 877 * 1) timing might have stopped 878 * 2) TSC packets within PSB+ can slip against CYC packets 879 */ 880 if (!from_mtc) 881 return; 882 883 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 884 } 885 886 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 887 { 888 int ret; 889 890 decoder->last_packet_type = decoder->packet.type; 891 892 do { 893 decoder->pos += decoder->pkt_step; 894 decoder->buf += decoder->pkt_step; 895 decoder->len -= decoder->pkt_step; 896 897 if (!decoder->len) { 898 ret = intel_pt_get_next_data(decoder, false); 899 if (ret) 900 return ret; 901 } 902 903 decoder->prev_pkt_ctx = decoder->pkt_ctx; 904 ret = intel_pt_get_packet(decoder->buf, decoder->len, 905 &decoder->packet, &decoder->pkt_ctx); 906 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 907 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 908 ret = intel_pt_get_split_packet(decoder); 909 if (ret < 0) 910 return ret; 911 } 912 if (ret <= 0) 913 return intel_pt_bad_packet(decoder); 914 915 decoder->pkt_len = ret; 916 decoder->pkt_step = ret; 917 intel_pt_decoder_log_packet(decoder); 918 } while (decoder->packet.type == INTEL_PT_PAD); 919 920 return 0; 921 } 922 923 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 924 { 925 uint64_t timestamp, masked_timestamp; 926 927 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 928 masked_timestamp = timestamp & decoder->period_mask; 929 if (decoder->continuous_period) { 930 if (masked_timestamp > decoder->last_masked_timestamp) 931 return 1; 932 } else { 933 timestamp += 1; 934 masked_timestamp = timestamp & decoder->period_mask; 935 if (masked_timestamp > decoder->last_masked_timestamp) { 936 decoder->last_masked_timestamp = masked_timestamp; 937 decoder->continuous_period = true; 938 } 939 } 940 941 if (masked_timestamp < decoder->last_masked_timestamp) 942 return decoder->period_ticks; 943 944 return decoder->period_ticks - (timestamp - masked_timestamp); 945 } 946 947 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 948 { 949 switch (decoder->period_type) { 950 case INTEL_PT_PERIOD_INSTRUCTIONS: 951 return decoder->period - decoder->period_insn_cnt; 952 case INTEL_PT_PERIOD_TICKS: 953 return intel_pt_next_period(decoder); 954 case INTEL_PT_PERIOD_NONE: 955 case INTEL_PT_PERIOD_MTC: 956 default: 957 return 0; 958 } 959 } 960 961 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 962 { 963 uint64_t timestamp, masked_timestamp; 964 965 switch (decoder->period_type) { 966 case INTEL_PT_PERIOD_INSTRUCTIONS: 967 decoder->period_insn_cnt = 0; 968 break; 969 case INTEL_PT_PERIOD_TICKS: 970 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 971 masked_timestamp = timestamp & decoder->period_mask; 972 if (masked_timestamp > decoder->last_masked_timestamp) 973 decoder->last_masked_timestamp = masked_timestamp; 974 else 975 decoder->last_masked_timestamp += decoder->period_ticks; 976 break; 977 case INTEL_PT_PERIOD_NONE: 978 case INTEL_PT_PERIOD_MTC: 979 default: 980 break; 981 } 982 983 decoder->state.type |= INTEL_PT_INSTRUCTION; 984 } 985 986 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 987 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 988 { 989 uint64_t max_insn_cnt, insn_cnt = 0; 990 int err; 991 992 if (!decoder->mtc_insn) 993 decoder->mtc_insn = true; 994 995 max_insn_cnt = intel_pt_next_sample(decoder); 996 997 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 998 max_insn_cnt, decoder->data); 999 1000 decoder->tot_insn_cnt += insn_cnt; 1001 decoder->timestamp_insn_cnt += insn_cnt; 1002 decoder->sample_insn_cnt += insn_cnt; 1003 decoder->period_insn_cnt += insn_cnt; 1004 1005 if (err) { 1006 decoder->no_progress = 0; 1007 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1008 intel_pt_log_at("ERROR: Failed to get instruction", 1009 decoder->ip); 1010 if (err == -ENOENT) 1011 return -ENOLINK; 1012 return -EILSEQ; 1013 } 1014 1015 if (ip && decoder->ip == ip) { 1016 err = -EAGAIN; 1017 goto out; 1018 } 1019 1020 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1021 intel_pt_sample_insn(decoder); 1022 1023 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1024 decoder->state.type = INTEL_PT_INSTRUCTION; 1025 decoder->state.from_ip = decoder->ip; 1026 decoder->state.to_ip = 0; 1027 decoder->ip += intel_pt_insn->length; 1028 err = INTEL_PT_RETURN; 1029 goto out; 1030 } 1031 1032 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1033 /* Zero-length calls are excluded */ 1034 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1035 intel_pt_insn->rel) { 1036 err = intel_pt_push(&decoder->stack, decoder->ip + 1037 intel_pt_insn->length); 1038 if (err) 1039 goto out; 1040 } 1041 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1042 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1043 } 1044 1045 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1046 int cnt = decoder->no_progress++; 1047 1048 decoder->state.from_ip = decoder->ip; 1049 decoder->ip += intel_pt_insn->length + 1050 intel_pt_insn->rel; 1051 decoder->state.to_ip = decoder->ip; 1052 err = INTEL_PT_RETURN; 1053 1054 /* 1055 * Check for being stuck in a loop. This can happen if a 1056 * decoder error results in the decoder erroneously setting the 1057 * ip to an address that is itself in an infinite loop that 1058 * consumes no packets. When that happens, there must be an 1059 * unconditional branch. 1060 */ 1061 if (cnt) { 1062 if (cnt == 1) { 1063 decoder->stuck_ip = decoder->state.to_ip; 1064 decoder->stuck_ip_prd = 1; 1065 decoder->stuck_ip_cnt = 1; 1066 } else if (cnt > INTEL_PT_MAX_LOOPS || 1067 decoder->state.to_ip == decoder->stuck_ip) { 1068 intel_pt_log_at("ERROR: Never-ending loop", 1069 decoder->state.to_ip); 1070 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1071 err = -ELOOP; 1072 goto out; 1073 } else if (!--decoder->stuck_ip_cnt) { 1074 decoder->stuck_ip_prd += 1; 1075 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1076 decoder->stuck_ip = decoder->state.to_ip; 1077 } 1078 } 1079 goto out_no_progress; 1080 } 1081 out: 1082 decoder->no_progress = 0; 1083 out_no_progress: 1084 decoder->state.insn_op = intel_pt_insn->op; 1085 decoder->state.insn_len = intel_pt_insn->length; 1086 memcpy(decoder->state.insn, intel_pt_insn->buf, 1087 INTEL_PT_INSN_BUF_SZ); 1088 1089 if (decoder->tx_flags & INTEL_PT_IN_TX) 1090 decoder->state.flags |= INTEL_PT_IN_TX; 1091 1092 return err; 1093 } 1094 1095 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1096 { 1097 bool ret = false; 1098 1099 if (decoder->set_fup_tx_flags) { 1100 decoder->set_fup_tx_flags = false; 1101 decoder->tx_flags = decoder->fup_tx_flags; 1102 decoder->state.type = INTEL_PT_TRANSACTION; 1103 decoder->state.from_ip = decoder->ip; 1104 decoder->state.to_ip = 0; 1105 decoder->state.flags = decoder->fup_tx_flags; 1106 return true; 1107 } 1108 if (decoder->set_fup_ptw) { 1109 decoder->set_fup_ptw = false; 1110 decoder->state.type = INTEL_PT_PTW; 1111 decoder->state.flags |= INTEL_PT_FUP_IP; 1112 decoder->state.from_ip = decoder->ip; 1113 decoder->state.to_ip = 0; 1114 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1115 return true; 1116 } 1117 if (decoder->set_fup_mwait) { 1118 decoder->set_fup_mwait = false; 1119 decoder->state.type = INTEL_PT_MWAIT_OP; 1120 decoder->state.from_ip = decoder->ip; 1121 decoder->state.to_ip = 0; 1122 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1123 ret = true; 1124 } 1125 if (decoder->set_fup_pwre) { 1126 decoder->set_fup_pwre = false; 1127 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1128 decoder->state.type &= ~INTEL_PT_BRANCH; 1129 decoder->state.from_ip = decoder->ip; 1130 decoder->state.to_ip = 0; 1131 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1132 ret = true; 1133 } 1134 if (decoder->set_fup_exstop) { 1135 decoder->set_fup_exstop = false; 1136 decoder->state.type |= INTEL_PT_EX_STOP; 1137 decoder->state.type &= ~INTEL_PT_BRANCH; 1138 decoder->state.flags |= INTEL_PT_FUP_IP; 1139 decoder->state.from_ip = decoder->ip; 1140 decoder->state.to_ip = 0; 1141 ret = true; 1142 } 1143 if (decoder->set_fup_bep) { 1144 decoder->set_fup_bep = false; 1145 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1146 decoder->state.type &= ~INTEL_PT_BRANCH; 1147 decoder->state.from_ip = decoder->ip; 1148 decoder->state.to_ip = 0; 1149 ret = true; 1150 } 1151 return ret; 1152 } 1153 1154 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1155 struct intel_pt_insn *intel_pt_insn, 1156 uint64_t ip, int err) 1157 { 1158 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1159 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1160 ip == decoder->ip + intel_pt_insn->length; 1161 } 1162 1163 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1164 { 1165 struct intel_pt_insn intel_pt_insn; 1166 uint64_t ip; 1167 int err; 1168 1169 ip = decoder->last_ip; 1170 1171 while (1) { 1172 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1173 if (err == INTEL_PT_RETURN) 1174 return 0; 1175 if (err == -EAGAIN || 1176 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1177 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1178 if (intel_pt_fup_event(decoder)) 1179 return 0; 1180 return -EAGAIN; 1181 } 1182 decoder->set_fup_tx_flags = false; 1183 if (err) 1184 return err; 1185 1186 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1187 intel_pt_log_at("ERROR: Unexpected indirect branch", 1188 decoder->ip); 1189 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1190 return -ENOENT; 1191 } 1192 1193 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1194 intel_pt_log_at("ERROR: Unexpected conditional branch", 1195 decoder->ip); 1196 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1197 return -ENOENT; 1198 } 1199 1200 intel_pt_bug(decoder); 1201 } 1202 } 1203 1204 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1205 { 1206 struct intel_pt_insn intel_pt_insn; 1207 int err; 1208 1209 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1210 if (err == INTEL_PT_RETURN && 1211 decoder->pgd_ip && 1212 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1213 (decoder->state.type & INTEL_PT_BRANCH) && 1214 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1215 /* Unconditional branch leaving filter region */ 1216 decoder->no_progress = 0; 1217 decoder->pge = false; 1218 decoder->continuous_period = false; 1219 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1220 decoder->state.type |= INTEL_PT_TRACE_END; 1221 return 0; 1222 } 1223 if (err == INTEL_PT_RETURN) 1224 return 0; 1225 if (err) 1226 return err; 1227 1228 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1229 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1230 decoder->pge = false; 1231 decoder->continuous_period = false; 1232 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1233 decoder->state.from_ip = decoder->ip; 1234 if (decoder->packet.count == 0) { 1235 decoder->state.to_ip = 0; 1236 } else { 1237 decoder->state.to_ip = decoder->last_ip; 1238 decoder->ip = decoder->last_ip; 1239 } 1240 decoder->state.type |= INTEL_PT_TRACE_END; 1241 } else { 1242 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1243 decoder->state.from_ip = decoder->ip; 1244 if (decoder->packet.count == 0) { 1245 decoder->state.to_ip = 0; 1246 } else { 1247 decoder->state.to_ip = decoder->last_ip; 1248 decoder->ip = decoder->last_ip; 1249 } 1250 } 1251 return 0; 1252 } 1253 1254 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1255 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1256 intel_pt_insn.rel; 1257 1258 if (decoder->pgd_ip && 1259 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1260 decoder->pgd_ip(to_ip, decoder->data)) { 1261 /* Conditional branch leaving filter region */ 1262 decoder->pge = false; 1263 decoder->continuous_period = false; 1264 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1265 decoder->ip = to_ip; 1266 decoder->state.from_ip = decoder->ip; 1267 decoder->state.to_ip = to_ip; 1268 decoder->state.type |= INTEL_PT_TRACE_END; 1269 return 0; 1270 } 1271 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1272 decoder->ip); 1273 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1274 return -ENOENT; 1275 } 1276 1277 return intel_pt_bug(decoder); 1278 } 1279 1280 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1281 { 1282 struct intel_pt_insn intel_pt_insn; 1283 int err; 1284 1285 while (1) { 1286 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1287 if (err == INTEL_PT_RETURN) 1288 return 0; 1289 if (err) 1290 return err; 1291 1292 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1293 if (!decoder->return_compression) { 1294 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1295 decoder->ip); 1296 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1297 return -ENOENT; 1298 } 1299 if (!decoder->ret_addr) { 1300 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1301 decoder->ip); 1302 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1303 return -ENOENT; 1304 } 1305 if (!(decoder->tnt.payload & BIT63)) { 1306 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1307 decoder->ip); 1308 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1309 return -ENOENT; 1310 } 1311 decoder->tnt.count -= 1; 1312 if (decoder->tnt.count) 1313 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1314 else 1315 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1316 decoder->tnt.payload <<= 1; 1317 decoder->state.from_ip = decoder->ip; 1318 decoder->ip = decoder->ret_addr; 1319 decoder->state.to_ip = decoder->ip; 1320 return 0; 1321 } 1322 1323 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1324 /* Handle deferred TIPs */ 1325 err = intel_pt_get_next_packet(decoder); 1326 if (err) 1327 return err; 1328 if (decoder->packet.type != INTEL_PT_TIP || 1329 decoder->packet.count == 0) { 1330 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1331 decoder->ip); 1332 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1333 decoder->pkt_step = 0; 1334 return -ENOENT; 1335 } 1336 intel_pt_set_last_ip(decoder); 1337 decoder->state.from_ip = decoder->ip; 1338 decoder->state.to_ip = decoder->last_ip; 1339 decoder->ip = decoder->last_ip; 1340 return 0; 1341 } 1342 1343 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1344 decoder->tnt.count -= 1; 1345 if (decoder->tnt.count) 1346 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1347 else 1348 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1349 if (decoder->tnt.payload & BIT63) { 1350 decoder->tnt.payload <<= 1; 1351 decoder->state.from_ip = decoder->ip; 1352 decoder->ip += intel_pt_insn.length + 1353 intel_pt_insn.rel; 1354 decoder->state.to_ip = decoder->ip; 1355 return 0; 1356 } 1357 /* Instruction sample for a non-taken branch */ 1358 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1359 decoder->tnt.payload <<= 1; 1360 decoder->state.type = INTEL_PT_INSTRUCTION; 1361 decoder->state.from_ip = decoder->ip; 1362 decoder->state.to_ip = 0; 1363 decoder->ip += intel_pt_insn.length; 1364 return 0; 1365 } 1366 decoder->sample_cyc = false; 1367 decoder->ip += intel_pt_insn.length; 1368 if (!decoder->tnt.count) { 1369 intel_pt_update_sample_time(decoder); 1370 return -EAGAIN; 1371 } 1372 decoder->tnt.payload <<= 1; 1373 continue; 1374 } 1375 1376 return intel_pt_bug(decoder); 1377 } 1378 } 1379 1380 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1381 { 1382 unsigned int fup_tx_flags; 1383 int err; 1384 1385 fup_tx_flags = decoder->packet.payload & 1386 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1387 err = intel_pt_get_next_packet(decoder); 1388 if (err) 1389 return err; 1390 if (decoder->packet.type == INTEL_PT_FUP) { 1391 decoder->fup_tx_flags = fup_tx_flags; 1392 decoder->set_fup_tx_flags = true; 1393 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1394 *no_tip = true; 1395 } else { 1396 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1397 decoder->pos); 1398 intel_pt_update_in_tx(decoder); 1399 } 1400 return 0; 1401 } 1402 1403 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1404 { 1405 timestamp |= (ref_timestamp & (0xffULL << 56)); 1406 1407 if (timestamp < ref_timestamp) { 1408 if (ref_timestamp - timestamp > (1ULL << 55)) 1409 timestamp += (1ULL << 56); 1410 } else { 1411 if (timestamp - ref_timestamp > (1ULL << 55)) 1412 timestamp -= (1ULL << 56); 1413 } 1414 1415 return timestamp; 1416 } 1417 1418 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1419 { 1420 uint64_t timestamp; 1421 1422 decoder->have_tma = false; 1423 1424 if (decoder->ref_timestamp) { 1425 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1426 decoder->ref_timestamp); 1427 decoder->tsc_timestamp = timestamp; 1428 decoder->timestamp = timestamp; 1429 decoder->ref_timestamp = 0; 1430 decoder->timestamp_insn_cnt = 0; 1431 } else if (decoder->timestamp) { 1432 timestamp = decoder->packet.payload | 1433 (decoder->timestamp & (0xffULL << 56)); 1434 decoder->tsc_timestamp = timestamp; 1435 if (timestamp < decoder->timestamp && 1436 decoder->timestamp - timestamp < decoder->tsc_slip) { 1437 intel_pt_log_to("Suppressing backwards timestamp", 1438 timestamp); 1439 timestamp = decoder->timestamp; 1440 } 1441 if (timestamp < decoder->timestamp) { 1442 intel_pt_log_to("Wraparound timestamp", timestamp); 1443 timestamp += (1ULL << 56); 1444 decoder->tsc_timestamp = timestamp; 1445 } 1446 decoder->timestamp = timestamp; 1447 decoder->timestamp_insn_cnt = 0; 1448 } 1449 1450 if (decoder->last_packet_type == INTEL_PT_CYC) { 1451 decoder->cyc_ref_timestamp = decoder->timestamp; 1452 decoder->cycle_cnt = 0; 1453 decoder->have_calc_cyc_to_tsc = false; 1454 intel_pt_calc_cyc_to_tsc(decoder, false); 1455 } 1456 1457 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1458 } 1459 1460 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1461 { 1462 intel_pt_log("ERROR: Buffer overflow\n"); 1463 intel_pt_clear_tx_flags(decoder); 1464 decoder->timestamp_insn_cnt = 0; 1465 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1466 decoder->overflow = true; 1467 return -EOVERFLOW; 1468 } 1469 1470 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1471 { 1472 if (decoder->have_cyc) 1473 return; 1474 1475 decoder->cyc_cnt_timestamp = decoder->timestamp; 1476 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1477 } 1478 1479 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1480 { 1481 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1482 1483 if (decoder->pge) 1484 intel_pt_mtc_cyc_cnt_pge(decoder); 1485 } 1486 1487 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1488 { 1489 uint64_t tot_cyc_cnt, tsc_delta; 1490 1491 if (decoder->have_cyc) 1492 return; 1493 1494 decoder->sample_cyc = true; 1495 1496 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1497 return; 1498 1499 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1500 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1501 1502 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1503 decoder->tot_cyc_cnt = tot_cyc_cnt; 1504 } 1505 1506 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1507 { 1508 uint32_t ctc = decoder->packet.payload; 1509 uint32_t fc = decoder->packet.count; 1510 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1511 1512 if (!decoder->tsc_ctc_ratio_d) 1513 return; 1514 1515 if (decoder->pge && !decoder->in_psb) 1516 intel_pt_mtc_cyc_cnt_pge(decoder); 1517 else 1518 intel_pt_mtc_cyc_cnt_upd(decoder); 1519 1520 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1521 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1522 if (decoder->tsc_ctc_mult) { 1523 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1524 } else { 1525 decoder->ctc_timestamp -= multdiv(ctc_rem, 1526 decoder->tsc_ctc_ratio_n, 1527 decoder->tsc_ctc_ratio_d); 1528 } 1529 decoder->ctc_delta = 0; 1530 decoder->have_tma = true; 1531 decoder->fixup_last_mtc = true; 1532 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1533 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1534 } 1535 1536 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1537 { 1538 uint64_t timestamp; 1539 uint32_t mtc, mtc_delta; 1540 1541 if (!decoder->have_tma) 1542 return; 1543 1544 mtc = decoder->packet.payload; 1545 1546 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1547 decoder->fixup_last_mtc = false; 1548 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1549 &decoder->last_mtc); 1550 } 1551 1552 if (mtc > decoder->last_mtc) 1553 mtc_delta = mtc - decoder->last_mtc; 1554 else 1555 mtc_delta = mtc + 256 - decoder->last_mtc; 1556 1557 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1558 1559 if (decoder->tsc_ctc_mult) { 1560 timestamp = decoder->ctc_timestamp + 1561 decoder->ctc_delta * decoder->tsc_ctc_mult; 1562 } else { 1563 timestamp = decoder->ctc_timestamp + 1564 multdiv(decoder->ctc_delta, 1565 decoder->tsc_ctc_ratio_n, 1566 decoder->tsc_ctc_ratio_d); 1567 } 1568 1569 if (timestamp < decoder->timestamp) 1570 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1571 timestamp, decoder->timestamp); 1572 else 1573 decoder->timestamp = timestamp; 1574 1575 intel_pt_mtc_cyc_cnt_upd(decoder); 1576 1577 decoder->timestamp_insn_cnt = 0; 1578 decoder->last_mtc = mtc; 1579 1580 if (decoder->last_packet_type == INTEL_PT_CYC) { 1581 decoder->cyc_ref_timestamp = decoder->timestamp; 1582 decoder->cycle_cnt = 0; 1583 decoder->have_calc_cyc_to_tsc = false; 1584 intel_pt_calc_cyc_to_tsc(decoder, true); 1585 } 1586 1587 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1588 } 1589 1590 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1591 { 1592 unsigned int cbr = decoder->packet.payload & 0xff; 1593 1594 decoder->cbr_payload = decoder->packet.payload; 1595 1596 if (decoder->cbr == cbr) 1597 return; 1598 1599 decoder->cbr = cbr; 1600 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1601 1602 intel_pt_mtc_cyc_cnt_cbr(decoder); 1603 } 1604 1605 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1606 { 1607 uint64_t timestamp = decoder->cyc_ref_timestamp; 1608 1609 decoder->have_cyc = true; 1610 1611 decoder->cycle_cnt += decoder->packet.payload; 1612 if (decoder->pge) 1613 decoder->tot_cyc_cnt += decoder->packet.payload; 1614 decoder->sample_cyc = true; 1615 1616 if (!decoder->cyc_ref_timestamp) 1617 return; 1618 1619 if (decoder->have_calc_cyc_to_tsc) 1620 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1621 else if (decoder->cbr) 1622 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1623 else 1624 return; 1625 1626 if (timestamp < decoder->timestamp) 1627 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1628 timestamp, decoder->timestamp); 1629 else 1630 decoder->timestamp = timestamp; 1631 1632 decoder->timestamp_insn_cnt = 0; 1633 1634 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1635 } 1636 1637 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1638 { 1639 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1640 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1641 decoder->state.items.is_32_bit = false; 1642 } 1643 decoder->blk_type = decoder->packet.payload; 1644 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1645 if (decoder->blk_type == INTEL_PT_GP_REGS) 1646 decoder->state.items.is_32_bit = decoder->packet.count; 1647 if (decoder->blk_type_pos < 0) { 1648 intel_pt_log("WARNING: Unknown block type %u\n", 1649 decoder->blk_type); 1650 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1651 intel_pt_log("WARNING: Duplicate block type %u\n", 1652 decoder->blk_type); 1653 } 1654 } 1655 1656 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1657 { 1658 uint32_t id = decoder->packet.count; 1659 uint32_t bit = 1 << id; 1660 int pos = decoder->blk_type_pos; 1661 1662 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1663 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1664 id, decoder->blk_type); 1665 return; 1666 } 1667 1668 if (decoder->state.items.mask[pos] & bit) { 1669 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1670 id, decoder->blk_type); 1671 } 1672 1673 decoder->state.items.mask[pos] |= bit; 1674 decoder->state.items.val[pos][id] = decoder->packet.payload; 1675 } 1676 1677 /* Walk PSB+ packets when already in sync. */ 1678 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1679 { 1680 int err; 1681 1682 decoder->in_psb = true; 1683 1684 while (1) { 1685 err = intel_pt_get_next_packet(decoder); 1686 if (err) 1687 goto out; 1688 1689 switch (decoder->packet.type) { 1690 case INTEL_PT_PSBEND: 1691 err = 0; 1692 goto out; 1693 1694 case INTEL_PT_TIP_PGD: 1695 case INTEL_PT_TIP_PGE: 1696 case INTEL_PT_TIP: 1697 case INTEL_PT_TNT: 1698 case INTEL_PT_TRACESTOP: 1699 case INTEL_PT_BAD: 1700 case INTEL_PT_PSB: 1701 case INTEL_PT_PTWRITE: 1702 case INTEL_PT_PTWRITE_IP: 1703 case INTEL_PT_EXSTOP: 1704 case INTEL_PT_EXSTOP_IP: 1705 case INTEL_PT_MWAIT: 1706 case INTEL_PT_PWRE: 1707 case INTEL_PT_PWRX: 1708 case INTEL_PT_BBP: 1709 case INTEL_PT_BIP: 1710 case INTEL_PT_BEP: 1711 case INTEL_PT_BEP_IP: 1712 decoder->have_tma = false; 1713 intel_pt_log("ERROR: Unexpected packet\n"); 1714 err = -EAGAIN; 1715 goto out; 1716 1717 case INTEL_PT_OVF: 1718 err = intel_pt_overflow(decoder); 1719 goto out; 1720 1721 case INTEL_PT_TSC: 1722 intel_pt_calc_tsc_timestamp(decoder); 1723 break; 1724 1725 case INTEL_PT_TMA: 1726 intel_pt_calc_tma(decoder); 1727 break; 1728 1729 case INTEL_PT_CBR: 1730 intel_pt_calc_cbr(decoder); 1731 break; 1732 1733 case INTEL_PT_MODE_EXEC: 1734 decoder->exec_mode = decoder->packet.payload; 1735 break; 1736 1737 case INTEL_PT_PIP: 1738 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1739 break; 1740 1741 case INTEL_PT_FUP: 1742 decoder->pge = true; 1743 if (decoder->packet.count) { 1744 intel_pt_set_last_ip(decoder); 1745 if (decoder->hop) { 1746 /* Act on FUP at PSBEND */ 1747 decoder->ip = decoder->last_ip; 1748 decoder->hop_psb_fup = true; 1749 } 1750 } 1751 break; 1752 1753 case INTEL_PT_MODE_TSX: 1754 intel_pt_update_in_tx(decoder); 1755 break; 1756 1757 case INTEL_PT_MTC: 1758 intel_pt_calc_mtc_timestamp(decoder); 1759 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1760 decoder->state.type |= INTEL_PT_INSTRUCTION; 1761 break; 1762 1763 case INTEL_PT_CYC: 1764 case INTEL_PT_VMCS: 1765 case INTEL_PT_MNT: 1766 case INTEL_PT_PAD: 1767 default: 1768 break; 1769 } 1770 } 1771 out: 1772 decoder->in_psb = false; 1773 1774 return err; 1775 } 1776 1777 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1778 { 1779 int err; 1780 1781 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1782 decoder->tx_flags = 0; 1783 decoder->state.flags &= ~INTEL_PT_IN_TX; 1784 decoder->state.flags |= INTEL_PT_ABORT_TX; 1785 } else { 1786 decoder->state.flags |= INTEL_PT_ASYNC; 1787 } 1788 1789 while (1) { 1790 err = intel_pt_get_next_packet(decoder); 1791 if (err) 1792 return err; 1793 1794 switch (decoder->packet.type) { 1795 case INTEL_PT_TNT: 1796 case INTEL_PT_FUP: 1797 case INTEL_PT_TRACESTOP: 1798 case INTEL_PT_PSB: 1799 case INTEL_PT_TSC: 1800 case INTEL_PT_TMA: 1801 case INTEL_PT_MODE_TSX: 1802 case INTEL_PT_BAD: 1803 case INTEL_PT_PSBEND: 1804 case INTEL_PT_PTWRITE: 1805 case INTEL_PT_PTWRITE_IP: 1806 case INTEL_PT_EXSTOP: 1807 case INTEL_PT_EXSTOP_IP: 1808 case INTEL_PT_MWAIT: 1809 case INTEL_PT_PWRE: 1810 case INTEL_PT_PWRX: 1811 case INTEL_PT_BBP: 1812 case INTEL_PT_BIP: 1813 case INTEL_PT_BEP: 1814 case INTEL_PT_BEP_IP: 1815 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1816 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1817 decoder->pkt_step = 0; 1818 return -ENOENT; 1819 1820 case INTEL_PT_CBR: 1821 intel_pt_calc_cbr(decoder); 1822 break; 1823 1824 case INTEL_PT_OVF: 1825 return intel_pt_overflow(decoder); 1826 1827 case INTEL_PT_TIP_PGD: 1828 decoder->state.from_ip = decoder->ip; 1829 if (decoder->packet.count == 0) { 1830 decoder->state.to_ip = 0; 1831 } else { 1832 intel_pt_set_ip(decoder); 1833 decoder->state.to_ip = decoder->ip; 1834 } 1835 decoder->pge = false; 1836 decoder->continuous_period = false; 1837 decoder->state.type |= INTEL_PT_TRACE_END; 1838 return 0; 1839 1840 case INTEL_PT_TIP_PGE: 1841 decoder->pge = true; 1842 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1843 decoder->ip); 1844 decoder->state.from_ip = 0; 1845 if (decoder->packet.count == 0) { 1846 decoder->state.to_ip = 0; 1847 } else { 1848 intel_pt_set_ip(decoder); 1849 decoder->state.to_ip = decoder->ip; 1850 } 1851 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1852 intel_pt_mtc_cyc_cnt_pge(decoder); 1853 return 0; 1854 1855 case INTEL_PT_TIP: 1856 decoder->state.from_ip = decoder->ip; 1857 if (decoder->packet.count == 0) { 1858 decoder->state.to_ip = 0; 1859 } else { 1860 intel_pt_set_ip(decoder); 1861 decoder->state.to_ip = decoder->ip; 1862 } 1863 return 0; 1864 1865 case INTEL_PT_PIP: 1866 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1867 break; 1868 1869 case INTEL_PT_MTC: 1870 intel_pt_calc_mtc_timestamp(decoder); 1871 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1872 decoder->state.type |= INTEL_PT_INSTRUCTION; 1873 break; 1874 1875 case INTEL_PT_CYC: 1876 intel_pt_calc_cyc_timestamp(decoder); 1877 break; 1878 1879 case INTEL_PT_MODE_EXEC: 1880 decoder->exec_mode = decoder->packet.payload; 1881 break; 1882 1883 case INTEL_PT_VMCS: 1884 case INTEL_PT_MNT: 1885 case INTEL_PT_PAD: 1886 break; 1887 1888 default: 1889 return intel_pt_bug(decoder); 1890 } 1891 } 1892 } 1893 1894 static int intel_pt_resample(struct intel_pt_decoder *decoder) 1895 { 1896 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1897 decoder->state.type = INTEL_PT_INSTRUCTION; 1898 decoder->state.from_ip = decoder->ip; 1899 decoder->state.to_ip = 0; 1900 return 0; 1901 } 1902 1903 #define HOP_PROCESS 0 1904 #define HOP_IGNORE 1 1905 #define HOP_RETURN 2 1906 #define HOP_AGAIN 3 1907 1908 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 1909 1910 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 1911 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 1912 { 1913 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 1914 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 1915 *err = intel_pt_scan_for_psb(decoder); 1916 if (*err) 1917 return HOP_RETURN; 1918 } 1919 1920 switch (decoder->packet.type) { 1921 case INTEL_PT_TNT: 1922 return HOP_IGNORE; 1923 1924 case INTEL_PT_TIP_PGD: 1925 if (!decoder->packet.count) 1926 return HOP_IGNORE; 1927 intel_pt_set_ip(decoder); 1928 decoder->state.type |= INTEL_PT_TRACE_END; 1929 decoder->state.from_ip = 0; 1930 decoder->state.to_ip = decoder->ip; 1931 return HOP_RETURN; 1932 1933 case INTEL_PT_TIP: 1934 if (!decoder->packet.count) 1935 return HOP_IGNORE; 1936 intel_pt_set_ip(decoder); 1937 decoder->state.type = INTEL_PT_INSTRUCTION; 1938 decoder->state.from_ip = decoder->ip; 1939 decoder->state.to_ip = 0; 1940 return HOP_RETURN; 1941 1942 case INTEL_PT_FUP: 1943 if (!decoder->packet.count) 1944 return HOP_IGNORE; 1945 intel_pt_set_ip(decoder); 1946 if (intel_pt_fup_event(decoder)) 1947 return HOP_RETURN; 1948 if (!decoder->branch_enable) 1949 *no_tip = true; 1950 if (*no_tip) { 1951 decoder->state.type = INTEL_PT_INSTRUCTION; 1952 decoder->state.from_ip = decoder->ip; 1953 decoder->state.to_ip = 0; 1954 return HOP_RETURN; 1955 } 1956 *err = intel_pt_walk_fup_tip(decoder); 1957 if (!*err) 1958 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 1959 return HOP_RETURN; 1960 1961 case INTEL_PT_PSB: 1962 decoder->last_ip = 0; 1963 decoder->have_last_ip = true; 1964 decoder->hop_psb_fup = false; 1965 *err = intel_pt_walk_psbend(decoder); 1966 if (*err == -EAGAIN) 1967 return HOP_AGAIN; 1968 if (*err) 1969 return HOP_RETURN; 1970 if (decoder->hop_psb_fup) { 1971 decoder->hop_psb_fup = false; 1972 decoder->state.type = INTEL_PT_INSTRUCTION; 1973 decoder->state.from_ip = decoder->ip; 1974 decoder->state.to_ip = 0; 1975 return HOP_RETURN; 1976 } 1977 if (decoder->cbr != decoder->cbr_seen) { 1978 decoder->state.type = 0; 1979 return HOP_RETURN; 1980 } 1981 return HOP_IGNORE; 1982 1983 case INTEL_PT_BAD: 1984 case INTEL_PT_PAD: 1985 case INTEL_PT_TIP_PGE: 1986 case INTEL_PT_TSC: 1987 case INTEL_PT_TMA: 1988 case INTEL_PT_MODE_EXEC: 1989 case INTEL_PT_MODE_TSX: 1990 case INTEL_PT_MTC: 1991 case INTEL_PT_CYC: 1992 case INTEL_PT_VMCS: 1993 case INTEL_PT_PSBEND: 1994 case INTEL_PT_CBR: 1995 case INTEL_PT_TRACESTOP: 1996 case INTEL_PT_PIP: 1997 case INTEL_PT_OVF: 1998 case INTEL_PT_MNT: 1999 case INTEL_PT_PTWRITE: 2000 case INTEL_PT_PTWRITE_IP: 2001 case INTEL_PT_EXSTOP: 2002 case INTEL_PT_EXSTOP_IP: 2003 case INTEL_PT_MWAIT: 2004 case INTEL_PT_PWRE: 2005 case INTEL_PT_PWRX: 2006 case INTEL_PT_BBP: 2007 case INTEL_PT_BIP: 2008 case INTEL_PT_BEP: 2009 case INTEL_PT_BEP_IP: 2010 default: 2011 return HOP_PROCESS; 2012 } 2013 } 2014 2015 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2016 { 2017 bool no_tip = false; 2018 int err; 2019 2020 while (1) { 2021 err = intel_pt_get_next_packet(decoder); 2022 if (err) 2023 return err; 2024 next: 2025 if (decoder->hop) { 2026 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2027 case HOP_IGNORE: 2028 continue; 2029 case HOP_RETURN: 2030 return err; 2031 case HOP_AGAIN: 2032 goto next; 2033 default: 2034 break; 2035 } 2036 } 2037 2038 switch (decoder->packet.type) { 2039 case INTEL_PT_TNT: 2040 if (!decoder->packet.count) 2041 break; 2042 decoder->tnt = decoder->packet; 2043 decoder->pkt_state = INTEL_PT_STATE_TNT; 2044 err = intel_pt_walk_tnt(decoder); 2045 if (err == -EAGAIN) 2046 break; 2047 return err; 2048 2049 case INTEL_PT_TIP_PGD: 2050 if (decoder->packet.count != 0) 2051 intel_pt_set_last_ip(decoder); 2052 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2053 return intel_pt_walk_tip(decoder); 2054 2055 case INTEL_PT_TIP_PGE: { 2056 decoder->pge = true; 2057 intel_pt_mtc_cyc_cnt_pge(decoder); 2058 if (decoder->packet.count == 0) { 2059 intel_pt_log_at("Skipping zero TIP.PGE", 2060 decoder->pos); 2061 break; 2062 } 2063 intel_pt_set_ip(decoder); 2064 decoder->state.from_ip = 0; 2065 decoder->state.to_ip = decoder->ip; 2066 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2067 /* 2068 * In hop mode, resample to get the to_ip as an 2069 * "instruction" sample. 2070 */ 2071 if (decoder->hop) 2072 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2073 return 0; 2074 } 2075 2076 case INTEL_PT_OVF: 2077 return intel_pt_overflow(decoder); 2078 2079 case INTEL_PT_TIP: 2080 if (decoder->packet.count != 0) 2081 intel_pt_set_last_ip(decoder); 2082 decoder->pkt_state = INTEL_PT_STATE_TIP; 2083 return intel_pt_walk_tip(decoder); 2084 2085 case INTEL_PT_FUP: 2086 if (decoder->packet.count == 0) { 2087 intel_pt_log_at("Skipping zero FUP", 2088 decoder->pos); 2089 no_tip = false; 2090 break; 2091 } 2092 intel_pt_set_last_ip(decoder); 2093 if (!decoder->branch_enable) { 2094 decoder->ip = decoder->last_ip; 2095 if (intel_pt_fup_event(decoder)) 2096 return 0; 2097 no_tip = false; 2098 break; 2099 } 2100 if (decoder->set_fup_mwait) 2101 no_tip = true; 2102 if (no_tip) 2103 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 2104 else 2105 decoder->pkt_state = INTEL_PT_STATE_FUP; 2106 err = intel_pt_walk_fup(decoder); 2107 if (err != -EAGAIN) 2108 return err; 2109 if (no_tip) { 2110 no_tip = false; 2111 break; 2112 } 2113 return intel_pt_walk_fup_tip(decoder); 2114 2115 case INTEL_PT_TRACESTOP: 2116 decoder->pge = false; 2117 decoder->continuous_period = false; 2118 intel_pt_clear_tx_flags(decoder); 2119 decoder->have_tma = false; 2120 break; 2121 2122 case INTEL_PT_PSB: 2123 decoder->last_ip = 0; 2124 decoder->have_last_ip = true; 2125 intel_pt_clear_stack(&decoder->stack); 2126 err = intel_pt_walk_psbend(decoder); 2127 if (err == -EAGAIN) 2128 goto next; 2129 if (err) 2130 return err; 2131 /* 2132 * PSB+ CBR will not have changed but cater for the 2133 * possibility of another CBR change that gets caught up 2134 * in the PSB+. 2135 */ 2136 if (decoder->cbr != decoder->cbr_seen) { 2137 decoder->state.type = 0; 2138 return 0; 2139 } 2140 break; 2141 2142 case INTEL_PT_PIP: 2143 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2144 break; 2145 2146 case INTEL_PT_MTC: 2147 intel_pt_calc_mtc_timestamp(decoder); 2148 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 2149 break; 2150 /* 2151 * Ensure that there has been an instruction since the 2152 * last MTC. 2153 */ 2154 if (!decoder->mtc_insn) 2155 break; 2156 decoder->mtc_insn = false; 2157 /* Ensure that there is a timestamp */ 2158 if (!decoder->timestamp) 2159 break; 2160 decoder->state.type = INTEL_PT_INSTRUCTION; 2161 decoder->state.from_ip = decoder->ip; 2162 decoder->state.to_ip = 0; 2163 decoder->mtc_insn = false; 2164 return 0; 2165 2166 case INTEL_PT_TSC: 2167 intel_pt_calc_tsc_timestamp(decoder); 2168 break; 2169 2170 case INTEL_PT_TMA: 2171 intel_pt_calc_tma(decoder); 2172 break; 2173 2174 case INTEL_PT_CYC: 2175 intel_pt_calc_cyc_timestamp(decoder); 2176 break; 2177 2178 case INTEL_PT_CBR: 2179 intel_pt_calc_cbr(decoder); 2180 if (decoder->cbr != decoder->cbr_seen) { 2181 decoder->state.type = 0; 2182 return 0; 2183 } 2184 break; 2185 2186 case INTEL_PT_MODE_EXEC: 2187 decoder->exec_mode = decoder->packet.payload; 2188 break; 2189 2190 case INTEL_PT_MODE_TSX: 2191 /* MODE_TSX need not be followed by FUP */ 2192 if (!decoder->pge || decoder->in_psb) { 2193 intel_pt_update_in_tx(decoder); 2194 break; 2195 } 2196 err = intel_pt_mode_tsx(decoder, &no_tip); 2197 if (err) 2198 return err; 2199 goto next; 2200 2201 case INTEL_PT_BAD: /* Does not happen */ 2202 return intel_pt_bug(decoder); 2203 2204 case INTEL_PT_PSBEND: 2205 case INTEL_PT_VMCS: 2206 case INTEL_PT_MNT: 2207 case INTEL_PT_PAD: 2208 break; 2209 2210 case INTEL_PT_PTWRITE_IP: 2211 decoder->fup_ptw_payload = decoder->packet.payload; 2212 err = intel_pt_get_next_packet(decoder); 2213 if (err) 2214 return err; 2215 if (decoder->packet.type == INTEL_PT_FUP) { 2216 decoder->set_fup_ptw = true; 2217 no_tip = true; 2218 } else { 2219 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 2220 decoder->pos); 2221 } 2222 goto next; 2223 2224 case INTEL_PT_PTWRITE: 2225 decoder->state.type = INTEL_PT_PTW; 2226 decoder->state.from_ip = decoder->ip; 2227 decoder->state.to_ip = 0; 2228 decoder->state.ptw_payload = decoder->packet.payload; 2229 return 0; 2230 2231 case INTEL_PT_MWAIT: 2232 decoder->fup_mwait_payload = decoder->packet.payload; 2233 decoder->set_fup_mwait = true; 2234 break; 2235 2236 case INTEL_PT_PWRE: 2237 if (decoder->set_fup_mwait) { 2238 decoder->fup_pwre_payload = 2239 decoder->packet.payload; 2240 decoder->set_fup_pwre = true; 2241 break; 2242 } 2243 decoder->state.type = INTEL_PT_PWR_ENTRY; 2244 decoder->state.from_ip = decoder->ip; 2245 decoder->state.to_ip = 0; 2246 decoder->state.pwrx_payload = decoder->packet.payload; 2247 return 0; 2248 2249 case INTEL_PT_EXSTOP_IP: 2250 err = intel_pt_get_next_packet(decoder); 2251 if (err) 2252 return err; 2253 if (decoder->packet.type == INTEL_PT_FUP) { 2254 decoder->set_fup_exstop = true; 2255 no_tip = true; 2256 } else { 2257 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 2258 decoder->pos); 2259 } 2260 goto next; 2261 2262 case INTEL_PT_EXSTOP: 2263 decoder->state.type = INTEL_PT_EX_STOP; 2264 decoder->state.from_ip = decoder->ip; 2265 decoder->state.to_ip = 0; 2266 return 0; 2267 2268 case INTEL_PT_PWRX: 2269 decoder->state.type = INTEL_PT_PWR_EXIT; 2270 decoder->state.from_ip = decoder->ip; 2271 decoder->state.to_ip = 0; 2272 decoder->state.pwrx_payload = decoder->packet.payload; 2273 return 0; 2274 2275 case INTEL_PT_BBP: 2276 intel_pt_bbp(decoder); 2277 break; 2278 2279 case INTEL_PT_BIP: 2280 intel_pt_bip(decoder); 2281 break; 2282 2283 case INTEL_PT_BEP: 2284 decoder->state.type = INTEL_PT_BLK_ITEMS; 2285 decoder->state.from_ip = decoder->ip; 2286 decoder->state.to_ip = 0; 2287 return 0; 2288 2289 case INTEL_PT_BEP_IP: 2290 err = intel_pt_get_next_packet(decoder); 2291 if (err) 2292 return err; 2293 if (decoder->packet.type == INTEL_PT_FUP) { 2294 decoder->set_fup_bep = true; 2295 no_tip = true; 2296 } else { 2297 intel_pt_log_at("ERROR: Missing FUP after BEP", 2298 decoder->pos); 2299 } 2300 goto next; 2301 2302 default: 2303 return intel_pt_bug(decoder); 2304 } 2305 } 2306 } 2307 2308 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 2309 { 2310 return decoder->packet.count && 2311 (decoder->have_last_ip || decoder->packet.count == 3 || 2312 decoder->packet.count == 6); 2313 } 2314 2315 /* Walk PSB+ packets to get in sync. */ 2316 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 2317 { 2318 int err; 2319 2320 decoder->in_psb = true; 2321 2322 while (1) { 2323 err = intel_pt_get_next_packet(decoder); 2324 if (err) 2325 goto out; 2326 2327 switch (decoder->packet.type) { 2328 case INTEL_PT_TIP_PGD: 2329 decoder->continuous_period = false; 2330 __fallthrough; 2331 case INTEL_PT_TIP_PGE: 2332 case INTEL_PT_TIP: 2333 case INTEL_PT_PTWRITE: 2334 case INTEL_PT_PTWRITE_IP: 2335 case INTEL_PT_EXSTOP: 2336 case INTEL_PT_EXSTOP_IP: 2337 case INTEL_PT_MWAIT: 2338 case INTEL_PT_PWRE: 2339 case INTEL_PT_PWRX: 2340 case INTEL_PT_BBP: 2341 case INTEL_PT_BIP: 2342 case INTEL_PT_BEP: 2343 case INTEL_PT_BEP_IP: 2344 intel_pt_log("ERROR: Unexpected packet\n"); 2345 err = -ENOENT; 2346 goto out; 2347 2348 case INTEL_PT_FUP: 2349 decoder->pge = true; 2350 if (intel_pt_have_ip(decoder)) { 2351 uint64_t current_ip = decoder->ip; 2352 2353 intel_pt_set_ip(decoder); 2354 if (current_ip) 2355 intel_pt_log_to("Setting IP", 2356 decoder->ip); 2357 } 2358 break; 2359 2360 case INTEL_PT_MTC: 2361 intel_pt_calc_mtc_timestamp(decoder); 2362 break; 2363 2364 case INTEL_PT_TSC: 2365 intel_pt_calc_tsc_timestamp(decoder); 2366 break; 2367 2368 case INTEL_PT_TMA: 2369 intel_pt_calc_tma(decoder); 2370 break; 2371 2372 case INTEL_PT_CYC: 2373 intel_pt_calc_cyc_timestamp(decoder); 2374 break; 2375 2376 case INTEL_PT_CBR: 2377 intel_pt_calc_cbr(decoder); 2378 break; 2379 2380 case INTEL_PT_PIP: 2381 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2382 break; 2383 2384 case INTEL_PT_MODE_EXEC: 2385 decoder->exec_mode = decoder->packet.payload; 2386 break; 2387 2388 case INTEL_PT_MODE_TSX: 2389 intel_pt_update_in_tx(decoder); 2390 break; 2391 2392 case INTEL_PT_TRACESTOP: 2393 decoder->pge = false; 2394 decoder->continuous_period = false; 2395 intel_pt_clear_tx_flags(decoder); 2396 __fallthrough; 2397 2398 case INTEL_PT_TNT: 2399 decoder->have_tma = false; 2400 intel_pt_log("ERROR: Unexpected packet\n"); 2401 if (decoder->ip) 2402 decoder->pkt_state = INTEL_PT_STATE_ERR4; 2403 else 2404 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2405 err = -ENOENT; 2406 goto out; 2407 2408 case INTEL_PT_BAD: /* Does not happen */ 2409 err = intel_pt_bug(decoder); 2410 goto out; 2411 2412 case INTEL_PT_OVF: 2413 err = intel_pt_overflow(decoder); 2414 goto out; 2415 2416 case INTEL_PT_PSBEND: 2417 err = 0; 2418 goto out; 2419 2420 case INTEL_PT_PSB: 2421 case INTEL_PT_VMCS: 2422 case INTEL_PT_MNT: 2423 case INTEL_PT_PAD: 2424 default: 2425 break; 2426 } 2427 } 2428 out: 2429 decoder->in_psb = false; 2430 2431 return err; 2432 } 2433 2434 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 2435 { 2436 int err; 2437 2438 while (1) { 2439 err = intel_pt_get_next_packet(decoder); 2440 if (err) 2441 return err; 2442 2443 switch (decoder->packet.type) { 2444 case INTEL_PT_TIP_PGD: 2445 decoder->continuous_period = false; 2446 decoder->pge = false; 2447 if (intel_pt_have_ip(decoder)) 2448 intel_pt_set_ip(decoder); 2449 if (!decoder->ip) 2450 break; 2451 decoder->state.type |= INTEL_PT_TRACE_END; 2452 return 0; 2453 2454 case INTEL_PT_TIP_PGE: 2455 decoder->pge = true; 2456 intel_pt_mtc_cyc_cnt_pge(decoder); 2457 if (intel_pt_have_ip(decoder)) 2458 intel_pt_set_ip(decoder); 2459 if (!decoder->ip) 2460 break; 2461 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2462 return 0; 2463 2464 case INTEL_PT_TIP: 2465 decoder->pge = true; 2466 if (intel_pt_have_ip(decoder)) 2467 intel_pt_set_ip(decoder); 2468 if (!decoder->ip) 2469 break; 2470 return 0; 2471 2472 case INTEL_PT_FUP: 2473 if (intel_pt_have_ip(decoder)) 2474 intel_pt_set_ip(decoder); 2475 if (decoder->ip) 2476 return 0; 2477 break; 2478 2479 case INTEL_PT_MTC: 2480 intel_pt_calc_mtc_timestamp(decoder); 2481 break; 2482 2483 case INTEL_PT_TSC: 2484 intel_pt_calc_tsc_timestamp(decoder); 2485 break; 2486 2487 case INTEL_PT_TMA: 2488 intel_pt_calc_tma(decoder); 2489 break; 2490 2491 case INTEL_PT_CYC: 2492 intel_pt_calc_cyc_timestamp(decoder); 2493 break; 2494 2495 case INTEL_PT_CBR: 2496 intel_pt_calc_cbr(decoder); 2497 break; 2498 2499 case INTEL_PT_PIP: 2500 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2501 break; 2502 2503 case INTEL_PT_MODE_EXEC: 2504 decoder->exec_mode = decoder->packet.payload; 2505 break; 2506 2507 case INTEL_PT_MODE_TSX: 2508 intel_pt_update_in_tx(decoder); 2509 break; 2510 2511 case INTEL_PT_OVF: 2512 return intel_pt_overflow(decoder); 2513 2514 case INTEL_PT_BAD: /* Does not happen */ 2515 return intel_pt_bug(decoder); 2516 2517 case INTEL_PT_TRACESTOP: 2518 decoder->pge = false; 2519 decoder->continuous_period = false; 2520 intel_pt_clear_tx_flags(decoder); 2521 decoder->have_tma = false; 2522 break; 2523 2524 case INTEL_PT_PSB: 2525 decoder->last_ip = 0; 2526 decoder->have_last_ip = true; 2527 intel_pt_clear_stack(&decoder->stack); 2528 err = intel_pt_walk_psb(decoder); 2529 if (err) 2530 return err; 2531 if (decoder->ip) { 2532 /* Do not have a sample */ 2533 decoder->state.type = 0; 2534 return 0; 2535 } 2536 break; 2537 2538 case INTEL_PT_TNT: 2539 case INTEL_PT_PSBEND: 2540 case INTEL_PT_VMCS: 2541 case INTEL_PT_MNT: 2542 case INTEL_PT_PAD: 2543 case INTEL_PT_PTWRITE: 2544 case INTEL_PT_PTWRITE_IP: 2545 case INTEL_PT_EXSTOP: 2546 case INTEL_PT_EXSTOP_IP: 2547 case INTEL_PT_MWAIT: 2548 case INTEL_PT_PWRE: 2549 case INTEL_PT_PWRX: 2550 case INTEL_PT_BBP: 2551 case INTEL_PT_BIP: 2552 case INTEL_PT_BEP: 2553 case INTEL_PT_BEP_IP: 2554 default: 2555 break; 2556 } 2557 } 2558 } 2559 2560 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 2561 { 2562 int err; 2563 2564 decoder->set_fup_tx_flags = false; 2565 decoder->set_fup_ptw = false; 2566 decoder->set_fup_mwait = false; 2567 decoder->set_fup_pwre = false; 2568 decoder->set_fup_exstop = false; 2569 decoder->set_fup_bep = false; 2570 2571 if (!decoder->branch_enable) { 2572 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2573 decoder->overflow = false; 2574 decoder->state.type = 0; /* Do not have a sample */ 2575 return 0; 2576 } 2577 2578 intel_pt_log("Scanning for full IP\n"); 2579 err = intel_pt_walk_to_ip(decoder); 2580 if (err) 2581 return err; 2582 2583 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 2584 if (decoder->hop) 2585 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2586 else 2587 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2588 decoder->overflow = false; 2589 2590 decoder->state.from_ip = 0; 2591 decoder->state.to_ip = decoder->ip; 2592 intel_pt_log_to("Setting IP", decoder->ip); 2593 2594 return 0; 2595 } 2596 2597 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 2598 { 2599 const unsigned char *end = decoder->buf + decoder->len; 2600 size_t i; 2601 2602 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 2603 if (i > decoder->len) 2604 continue; 2605 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 2606 return i; 2607 } 2608 return 0; 2609 } 2610 2611 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 2612 { 2613 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 2614 const char *psb = INTEL_PT_PSB_STR; 2615 2616 if (rest_psb > decoder->len || 2617 memcmp(decoder->buf, psb + part_psb, rest_psb)) 2618 return 0; 2619 2620 return rest_psb; 2621 } 2622 2623 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 2624 int part_psb) 2625 { 2626 int rest_psb, ret; 2627 2628 decoder->pos += decoder->len; 2629 decoder->len = 0; 2630 2631 ret = intel_pt_get_next_data(decoder, false); 2632 if (ret) 2633 return ret; 2634 2635 rest_psb = intel_pt_rest_psb(decoder, part_psb); 2636 if (!rest_psb) 2637 return 0; 2638 2639 decoder->pos -= part_psb; 2640 decoder->next_buf = decoder->buf + rest_psb; 2641 decoder->next_len = decoder->len - rest_psb; 2642 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2643 decoder->buf = decoder->temp_buf; 2644 decoder->len = INTEL_PT_PSB_LEN; 2645 2646 return 0; 2647 } 2648 2649 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 2650 { 2651 unsigned char *next; 2652 int ret; 2653 2654 intel_pt_log("Scanning for PSB\n"); 2655 while (1) { 2656 if (!decoder->len) { 2657 ret = intel_pt_get_next_data(decoder, false); 2658 if (ret) 2659 return ret; 2660 } 2661 2662 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2663 INTEL_PT_PSB_LEN); 2664 if (!next) { 2665 int part_psb; 2666 2667 part_psb = intel_pt_part_psb(decoder); 2668 if (part_psb) { 2669 ret = intel_pt_get_split_psb(decoder, part_psb); 2670 if (ret) 2671 return ret; 2672 } else { 2673 decoder->pos += decoder->len; 2674 decoder->len = 0; 2675 } 2676 continue; 2677 } 2678 2679 decoder->pkt_step = next - decoder->buf; 2680 return intel_pt_get_next_packet(decoder); 2681 } 2682 } 2683 2684 static int intel_pt_sync(struct intel_pt_decoder *decoder) 2685 { 2686 int err; 2687 2688 decoder->pge = false; 2689 decoder->continuous_period = false; 2690 decoder->have_last_ip = false; 2691 decoder->last_ip = 0; 2692 decoder->ip = 0; 2693 intel_pt_clear_stack(&decoder->stack); 2694 2695 leap: 2696 err = intel_pt_scan_for_psb(decoder); 2697 if (err) 2698 return err; 2699 2700 decoder->have_last_ip = true; 2701 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 2702 2703 err = intel_pt_walk_psb(decoder); 2704 if (err) 2705 return err; 2706 2707 if (decoder->ip) { 2708 decoder->state.type = 0; /* Do not have a sample */ 2709 /* 2710 * In hop mode, resample to get the PSB FUP ip as an 2711 * "instruction" sample. 2712 */ 2713 if (decoder->hop) 2714 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2715 else 2716 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2717 } else if (decoder->leap) { 2718 /* 2719 * In leap mode, only PSB+ is decoded, so keeping leaping to the 2720 * next PSB until there is an ip. 2721 */ 2722 goto leap; 2723 } else { 2724 return intel_pt_sync_ip(decoder); 2725 } 2726 2727 return 0; 2728 } 2729 2730 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2731 { 2732 uint64_t est = decoder->sample_insn_cnt << 1; 2733 2734 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2735 goto out; 2736 2737 est *= decoder->max_non_turbo_ratio; 2738 est /= decoder->cbr; 2739 out: 2740 return decoder->sample_timestamp + est; 2741 } 2742 2743 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2744 { 2745 int err; 2746 2747 do { 2748 decoder->state.type = INTEL_PT_BRANCH; 2749 decoder->state.flags = 0; 2750 2751 switch (decoder->pkt_state) { 2752 case INTEL_PT_STATE_NO_PSB: 2753 err = intel_pt_sync(decoder); 2754 break; 2755 case INTEL_PT_STATE_NO_IP: 2756 decoder->have_last_ip = false; 2757 decoder->last_ip = 0; 2758 decoder->ip = 0; 2759 __fallthrough; 2760 case INTEL_PT_STATE_ERR_RESYNC: 2761 err = intel_pt_sync_ip(decoder); 2762 break; 2763 case INTEL_PT_STATE_IN_SYNC: 2764 err = intel_pt_walk_trace(decoder); 2765 break; 2766 case INTEL_PT_STATE_TNT: 2767 case INTEL_PT_STATE_TNT_CONT: 2768 err = intel_pt_walk_tnt(decoder); 2769 if (err == -EAGAIN) 2770 err = intel_pt_walk_trace(decoder); 2771 break; 2772 case INTEL_PT_STATE_TIP: 2773 case INTEL_PT_STATE_TIP_PGD: 2774 err = intel_pt_walk_tip(decoder); 2775 break; 2776 case INTEL_PT_STATE_FUP: 2777 err = intel_pt_walk_fup(decoder); 2778 if (err == -EAGAIN) 2779 err = intel_pt_walk_fup_tip(decoder); 2780 break; 2781 case INTEL_PT_STATE_FUP_NO_TIP: 2782 err = intel_pt_walk_fup(decoder); 2783 if (err == -EAGAIN) 2784 err = intel_pt_walk_trace(decoder); 2785 break; 2786 case INTEL_PT_STATE_RESAMPLE: 2787 err = intel_pt_resample(decoder); 2788 break; 2789 default: 2790 err = intel_pt_bug(decoder); 2791 break; 2792 } 2793 } while (err == -ENOLINK); 2794 2795 if (err) { 2796 decoder->state.err = intel_pt_ext_err(err); 2797 decoder->state.from_ip = decoder->ip; 2798 intel_pt_update_sample_time(decoder); 2799 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 2800 } else { 2801 decoder->state.err = 0; 2802 if (decoder->cbr != decoder->cbr_seen) { 2803 decoder->cbr_seen = decoder->cbr; 2804 if (!decoder->state.type) { 2805 decoder->state.from_ip = decoder->ip; 2806 decoder->state.to_ip = 0; 2807 } 2808 decoder->state.type |= INTEL_PT_CBR_CHG; 2809 decoder->state.cbr_payload = decoder->cbr_payload; 2810 decoder->state.cbr = decoder->cbr; 2811 } 2812 if (intel_pt_sample_time(decoder->pkt_state)) { 2813 intel_pt_update_sample_time(decoder); 2814 if (decoder->sample_cyc) 2815 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 2816 } 2817 } 2818 2819 decoder->state.timestamp = decoder->sample_timestamp; 2820 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2821 decoder->state.cr3 = decoder->cr3; 2822 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2823 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 2824 2825 return &decoder->state; 2826 } 2827 2828 /** 2829 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2830 * @buf: pointer to buffer pointer 2831 * @len: size of buffer 2832 * 2833 * Updates the buffer pointer to point to the start of the next PSB packet if 2834 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2835 * @len is adjusted accordingly. 2836 * 2837 * Return: %true if a PSB packet is found, %false otherwise. 2838 */ 2839 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2840 { 2841 unsigned char *next; 2842 2843 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2844 if (next) { 2845 *len -= next - *buf; 2846 *buf = next; 2847 return true; 2848 } 2849 return false; 2850 } 2851 2852 /** 2853 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2854 * packet. 2855 * @buf: pointer to buffer pointer 2856 * @len: size of buffer 2857 * 2858 * Updates the buffer pointer to point to the start of the following PSB packet 2859 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2860 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2861 * 2862 * Return: %true if a PSB packet is found, %false otherwise. 2863 */ 2864 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2865 { 2866 unsigned char *next; 2867 2868 if (!*len) 2869 return false; 2870 2871 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2872 if (next) { 2873 *len -= next - *buf; 2874 *buf = next; 2875 return true; 2876 } 2877 return false; 2878 } 2879 2880 /** 2881 * intel_pt_last_psb - find the last PSB packet in a buffer. 2882 * @buf: buffer 2883 * @len: size of buffer 2884 * 2885 * This function finds the last PSB in a buffer. 2886 * 2887 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2888 */ 2889 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2890 { 2891 const char *n = INTEL_PT_PSB_STR; 2892 unsigned char *p; 2893 size_t k; 2894 2895 if (len < INTEL_PT_PSB_LEN) 2896 return NULL; 2897 2898 k = len - INTEL_PT_PSB_LEN + 1; 2899 while (1) { 2900 p = memrchr(buf, n[0], k); 2901 if (!p) 2902 return NULL; 2903 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2904 return p; 2905 k = p - buf; 2906 if (!k) 2907 return NULL; 2908 } 2909 } 2910 2911 /** 2912 * intel_pt_next_tsc - find and return next TSC. 2913 * @buf: buffer 2914 * @len: size of buffer 2915 * @tsc: TSC value returned 2916 * @rem: returns remaining size when TSC is found 2917 * 2918 * Find a TSC packet in @buf and return the TSC value. This function assumes 2919 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2920 * PSBEND packet is found. 2921 * 2922 * Return: %true if TSC is found, false otherwise. 2923 */ 2924 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 2925 size_t *rem) 2926 { 2927 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 2928 struct intel_pt_pkt packet; 2929 int ret; 2930 2931 while (len) { 2932 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 2933 if (ret <= 0) 2934 return false; 2935 if (packet.type == INTEL_PT_TSC) { 2936 *tsc = packet.payload; 2937 *rem = len; 2938 return true; 2939 } 2940 if (packet.type == INTEL_PT_PSBEND) 2941 return false; 2942 buf += ret; 2943 len -= ret; 2944 } 2945 return false; 2946 } 2947 2948 /** 2949 * intel_pt_tsc_cmp - compare 7-byte TSCs. 2950 * @tsc1: first TSC to compare 2951 * @tsc2: second TSC to compare 2952 * 2953 * This function compares 7-byte TSC values allowing for the possibility that 2954 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 2955 * around so for that purpose this function assumes the absolute difference is 2956 * less than half the maximum difference. 2957 * 2958 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 2959 * after @tsc2. 2960 */ 2961 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 2962 { 2963 const uint64_t halfway = (1ULL << 55); 2964 2965 if (tsc1 == tsc2) 2966 return 0; 2967 2968 if (tsc1 < tsc2) { 2969 if (tsc2 - tsc1 < halfway) 2970 return -1; 2971 else 2972 return 1; 2973 } else { 2974 if (tsc1 - tsc2 < halfway) 2975 return 1; 2976 else 2977 return -1; 2978 } 2979 } 2980 2981 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 2982 2983 /** 2984 * adj_for_padding - adjust overlap to account for padding. 2985 * @buf_b: second buffer 2986 * @buf_a: first buffer 2987 * @len_a: size of first buffer 2988 * 2989 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 2990 * accordingly. 2991 * 2992 * Return: A pointer into @buf_b from where non-overlapped data starts 2993 */ 2994 static unsigned char *adj_for_padding(unsigned char *buf_b, 2995 unsigned char *buf_a, size_t len_a) 2996 { 2997 unsigned char *p = buf_b - MAX_PADDING; 2998 unsigned char *q = buf_a + len_a - MAX_PADDING; 2999 int i; 3000 3001 for (i = MAX_PADDING; i; i--, p++, q++) { 3002 if (*p != *q) 3003 break; 3004 } 3005 3006 return p; 3007 } 3008 3009 /** 3010 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3011 * using TSC. 3012 * @buf_a: first buffer 3013 * @len_a: size of first buffer 3014 * @buf_b: second buffer 3015 * @len_b: size of second buffer 3016 * @consecutive: returns true if there is data in buf_b that is consecutive 3017 * to buf_a 3018 * 3019 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3020 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3021 * walk forward in @buf_b until a later TSC is found. A precondition is that 3022 * @buf_a and @buf_b are positioned at a PSB. 3023 * 3024 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3025 * @buf_b + @len_b if there is no non-overlapped data. 3026 */ 3027 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3028 size_t len_a, 3029 unsigned char *buf_b, 3030 size_t len_b, bool *consecutive) 3031 { 3032 uint64_t tsc_a, tsc_b; 3033 unsigned char *p; 3034 size_t len, rem_a, rem_b; 3035 3036 p = intel_pt_last_psb(buf_a, len_a); 3037 if (!p) 3038 return buf_b; /* No PSB in buf_a => no overlap */ 3039 3040 len = len_a - (p - buf_a); 3041 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3042 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3043 len_a -= len; 3044 p = intel_pt_last_psb(buf_a, len_a); 3045 if (!p) 3046 return buf_b; /* No full PSB+ => assume no overlap */ 3047 len = len_a - (p - buf_a); 3048 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 3049 return buf_b; /* No TSC in buf_a => assume no overlap */ 3050 } 3051 3052 while (1) { 3053 /* Ignore PSB+ with no TSC */ 3054 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 3055 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 3056 3057 /* Same TSC, so buffers are consecutive */ 3058 if (!cmp && rem_b >= rem_a) { 3059 unsigned char *start; 3060 3061 *consecutive = true; 3062 start = buf_b + len_b - (rem_b - rem_a); 3063 return adj_for_padding(start, buf_a, len_a); 3064 } 3065 if (cmp < 0) 3066 return buf_b; /* tsc_a < tsc_b => no overlap */ 3067 } 3068 3069 if (!intel_pt_step_psb(&buf_b, &len_b)) 3070 return buf_b + len_b; /* No PSB in buf_b => no data */ 3071 } 3072 } 3073 3074 /** 3075 * intel_pt_find_overlap - determine start of non-overlapped trace data. 3076 * @buf_a: first buffer 3077 * @len_a: size of first buffer 3078 * @buf_b: second buffer 3079 * @len_b: size of second buffer 3080 * @have_tsc: can use TSC packets to detect overlap 3081 * @consecutive: returns true if there is data in buf_b that is consecutive 3082 * to buf_a 3083 * 3084 * When trace samples or snapshots are recorded there is the possibility that 3085 * the data overlaps. Note that, for the purposes of decoding, data is only 3086 * useful if it begins with a PSB packet. 3087 * 3088 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3089 * @buf_b + @len_b if there is no non-overlapped data. 3090 */ 3091 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 3092 unsigned char *buf_b, size_t len_b, 3093 bool have_tsc, bool *consecutive) 3094 { 3095 unsigned char *found; 3096 3097 /* Buffer 'b' must start at PSB so throw away everything before that */ 3098 if (!intel_pt_next_psb(&buf_b, &len_b)) 3099 return buf_b + len_b; /* No PSB */ 3100 3101 if (!intel_pt_next_psb(&buf_a, &len_a)) 3102 return buf_b; /* No overlap */ 3103 3104 if (have_tsc) { 3105 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 3106 consecutive); 3107 if (found) 3108 return found; 3109 } 3110 3111 /* 3112 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 3113 * we can ignore the first part of buffer 'a'. 3114 */ 3115 while (len_b < len_a) { 3116 if (!intel_pt_step_psb(&buf_a, &len_a)) 3117 return buf_b; /* No overlap */ 3118 } 3119 3120 /* Now len_b >= len_a */ 3121 while (1) { 3122 /* Potential overlap so check the bytes */ 3123 found = memmem(buf_a, len_a, buf_b, len_a); 3124 if (found) { 3125 *consecutive = true; 3126 return adj_for_padding(buf_b + len_a, buf_a, len_a); 3127 } 3128 3129 /* Try again at next PSB in buffer 'a' */ 3130 if (!intel_pt_step_psb(&buf_a, &len_a)) 3131 return buf_b; /* No overlap */ 3132 } 3133 } 3134 3135 /** 3136 * struct fast_forward_data - data used by intel_pt_ff_cb(). 3137 * @timestamp: timestamp to fast forward towards 3138 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 3139 * the fast forward timestamp. 3140 */ 3141 struct fast_forward_data { 3142 uint64_t timestamp; 3143 uint64_t buf_timestamp; 3144 }; 3145 3146 /** 3147 * intel_pt_ff_cb - fast forward lookahead callback. 3148 * @buffer: Intel PT trace buffer 3149 * @data: opaque pointer to fast forward data (struct fast_forward_data) 3150 * 3151 * Determine if @buffer trace is past the fast forward timestamp. 3152 * 3153 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 3154 * timestamp, and 0 otherwise. 3155 */ 3156 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 3157 { 3158 struct fast_forward_data *d = data; 3159 unsigned char *buf; 3160 uint64_t tsc; 3161 size_t rem; 3162 size_t len; 3163 3164 buf = (unsigned char *)buffer->buf; 3165 len = buffer->len; 3166 3167 if (!intel_pt_next_psb(&buf, &len) || 3168 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 3169 return 0; 3170 3171 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 3172 3173 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 3174 tsc, buffer->ref_timestamp); 3175 3176 /* 3177 * If the buffer contains a timestamp earlier that the fast forward 3178 * timestamp, then record it, else stop. 3179 */ 3180 if (tsc < d->timestamp) 3181 d->buf_timestamp = buffer->ref_timestamp; 3182 else 3183 return 1; 3184 3185 return 0; 3186 } 3187 3188 /** 3189 * intel_pt_fast_forward - reposition decoder forwards. 3190 * @decoder: Intel PT decoder 3191 * @timestamp: timestamp to fast forward towards 3192 * 3193 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 3194 * 3195 * Return: 0 on success or negative error code on failure. 3196 */ 3197 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 3198 { 3199 struct fast_forward_data d = { .timestamp = timestamp }; 3200 unsigned char *buf; 3201 size_t len; 3202 int err; 3203 3204 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 3205 3206 /* Find buffer timestamp of buffer to fast forward to */ 3207 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 3208 if (err < 0) 3209 return err; 3210 3211 /* Walk to buffer with same buffer timestamp */ 3212 if (d.buf_timestamp) { 3213 do { 3214 decoder->pos += decoder->len; 3215 decoder->len = 0; 3216 err = intel_pt_get_next_data(decoder, true); 3217 /* -ENOLINK means non-consecutive trace */ 3218 if (err && err != -ENOLINK) 3219 return err; 3220 } while (decoder->buf_timestamp != d.buf_timestamp); 3221 } 3222 3223 if (!decoder->buf) 3224 return 0; 3225 3226 buf = (unsigned char *)decoder->buf; 3227 len = decoder->len; 3228 3229 if (!intel_pt_next_psb(&buf, &len)) 3230 return 0; 3231 3232 /* 3233 * Walk PSBs while the PSB timestamp is less than the fast forward 3234 * timestamp. 3235 */ 3236 do { 3237 uint64_t tsc; 3238 size_t rem; 3239 3240 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 3241 break; 3242 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 3243 /* 3244 * A TSC packet can slip past MTC packets but, after fast 3245 * forward, decoding starts at the TSC timestamp. That means 3246 * the timestamps may not be exactly the same as the timestamps 3247 * that would have been decoded without fast forward. 3248 */ 3249 if (tsc < timestamp) { 3250 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 3251 decoder->pos += decoder->len - len; 3252 decoder->buf = buf; 3253 decoder->len = len; 3254 intel_pt_reposition(decoder); 3255 } else { 3256 break; 3257 } 3258 } while (intel_pt_step_psb(&buf, &len)); 3259 3260 return 0; 3261 } 3262