1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define INTEL_PT_BLK_SIZE 1024 28 29 #define BIT63 (((uint64_t)1 << 63)) 30 31 #define INTEL_PT_RETURN 1 32 33 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 34 #define INTEL_PT_MAX_LOOPS 10000 35 36 struct intel_pt_blk { 37 struct intel_pt_blk *prev; 38 uint64_t ip[INTEL_PT_BLK_SIZE]; 39 }; 40 41 struct intel_pt_stack { 42 struct intel_pt_blk *blk; 43 struct intel_pt_blk *spare; 44 int pos; 45 }; 46 47 enum intel_pt_pkt_state { 48 INTEL_PT_STATE_NO_PSB, 49 INTEL_PT_STATE_NO_IP, 50 INTEL_PT_STATE_ERR_RESYNC, 51 INTEL_PT_STATE_IN_SYNC, 52 INTEL_PT_STATE_TNT_CONT, 53 INTEL_PT_STATE_TNT, 54 INTEL_PT_STATE_TIP, 55 INTEL_PT_STATE_TIP_PGD, 56 INTEL_PT_STATE_FUP, 57 INTEL_PT_STATE_FUP_NO_TIP, 58 }; 59 60 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 61 { 62 switch (pkt_state) { 63 case INTEL_PT_STATE_NO_PSB: 64 case INTEL_PT_STATE_NO_IP: 65 case INTEL_PT_STATE_ERR_RESYNC: 66 case INTEL_PT_STATE_IN_SYNC: 67 case INTEL_PT_STATE_TNT_CONT: 68 return true; 69 case INTEL_PT_STATE_TNT: 70 case INTEL_PT_STATE_TIP: 71 case INTEL_PT_STATE_TIP_PGD: 72 case INTEL_PT_STATE_FUP: 73 case INTEL_PT_STATE_FUP_NO_TIP: 74 return false; 75 default: 76 return true; 77 }; 78 } 79 80 #ifdef INTEL_PT_STRICT 81 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 82 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 83 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 84 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 85 #else 86 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 87 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 88 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 89 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 90 #endif 91 92 struct intel_pt_decoder { 93 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 94 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 95 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 96 uint64_t max_insn_cnt, void *data); 97 bool (*pgd_ip)(uint64_t ip, void *data); 98 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 99 void *data; 100 struct intel_pt_state state; 101 const unsigned char *buf; 102 size_t len; 103 bool return_compression; 104 bool branch_enable; 105 bool mtc_insn; 106 bool pge; 107 bool have_tma; 108 bool have_cyc; 109 bool fixup_last_mtc; 110 bool have_last_ip; 111 bool in_psb; 112 enum intel_pt_param_flags flags; 113 uint64_t pos; 114 uint64_t last_ip; 115 uint64_t ip; 116 uint64_t cr3; 117 uint64_t timestamp; 118 uint64_t tsc_timestamp; 119 uint64_t ref_timestamp; 120 uint64_t buf_timestamp; 121 uint64_t sample_timestamp; 122 uint64_t ret_addr; 123 uint64_t ctc_timestamp; 124 uint64_t ctc_delta; 125 uint64_t cycle_cnt; 126 uint64_t cyc_ref_timestamp; 127 uint32_t last_mtc; 128 uint32_t tsc_ctc_ratio_n; 129 uint32_t tsc_ctc_ratio_d; 130 uint32_t tsc_ctc_mult; 131 uint32_t tsc_slip; 132 uint32_t ctc_rem_mask; 133 int mtc_shift; 134 struct intel_pt_stack stack; 135 enum intel_pt_pkt_state pkt_state; 136 enum intel_pt_pkt_ctx pkt_ctx; 137 enum intel_pt_pkt_ctx prev_pkt_ctx; 138 enum intel_pt_blk_type blk_type; 139 int blk_type_pos; 140 struct intel_pt_pkt packet; 141 struct intel_pt_pkt tnt; 142 int pkt_step; 143 int pkt_len; 144 int last_packet_type; 145 unsigned int cbr; 146 unsigned int cbr_seen; 147 unsigned int max_non_turbo_ratio; 148 double max_non_turbo_ratio_fp; 149 double cbr_cyc_to_tsc; 150 double calc_cyc_to_tsc; 151 bool have_calc_cyc_to_tsc; 152 int exec_mode; 153 unsigned int insn_bytes; 154 uint64_t period; 155 enum intel_pt_period_type period_type; 156 uint64_t tot_insn_cnt; 157 uint64_t period_insn_cnt; 158 uint64_t period_mask; 159 uint64_t period_ticks; 160 uint64_t last_masked_timestamp; 161 uint64_t tot_cyc_cnt; 162 uint64_t sample_tot_cyc_cnt; 163 uint64_t base_cyc_cnt; 164 uint64_t cyc_cnt_timestamp; 165 double tsc_to_cyc; 166 bool continuous_period; 167 bool overflow; 168 bool set_fup_tx_flags; 169 bool set_fup_ptw; 170 bool set_fup_mwait; 171 bool set_fup_pwre; 172 bool set_fup_exstop; 173 bool set_fup_bep; 174 bool sample_cyc; 175 unsigned int fup_tx_flags; 176 unsigned int tx_flags; 177 uint64_t fup_ptw_payload; 178 uint64_t fup_mwait_payload; 179 uint64_t fup_pwre_payload; 180 uint64_t cbr_payload; 181 uint64_t timestamp_insn_cnt; 182 uint64_t sample_insn_cnt; 183 uint64_t stuck_ip; 184 int no_progress; 185 int stuck_ip_prd; 186 int stuck_ip_cnt; 187 const unsigned char *next_buf; 188 size_t next_len; 189 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 190 }; 191 192 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 193 { 194 int i; 195 196 for (i = 0; x != 1; i++) 197 x >>= 1; 198 199 return x << i; 200 } 201 202 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 203 { 204 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 205 uint64_t period; 206 207 period = intel_pt_lower_power_of_2(decoder->period); 208 decoder->period_mask = ~(period - 1); 209 decoder->period_ticks = period; 210 } 211 } 212 213 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 214 { 215 if (!d) 216 return 0; 217 return (t / d) * n + ((t % d) * n) / d; 218 } 219 220 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 221 { 222 struct intel_pt_decoder *decoder; 223 224 if (!params->get_trace || !params->walk_insn) 225 return NULL; 226 227 decoder = zalloc(sizeof(struct intel_pt_decoder)); 228 if (!decoder) 229 return NULL; 230 231 decoder->get_trace = params->get_trace; 232 decoder->walk_insn = params->walk_insn; 233 decoder->pgd_ip = params->pgd_ip; 234 decoder->lookahead = params->lookahead; 235 decoder->data = params->data; 236 decoder->return_compression = params->return_compression; 237 decoder->branch_enable = params->branch_enable; 238 239 decoder->flags = params->flags; 240 241 decoder->period = params->period; 242 decoder->period_type = params->period_type; 243 244 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 245 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 246 247 intel_pt_setup_period(decoder); 248 249 decoder->mtc_shift = params->mtc_period; 250 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 251 252 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 253 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 254 255 if (!decoder->tsc_ctc_ratio_n) 256 decoder->tsc_ctc_ratio_d = 0; 257 258 if (decoder->tsc_ctc_ratio_d) { 259 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 260 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 261 decoder->tsc_ctc_ratio_d; 262 } 263 264 /* 265 * A TSC packet can slip past MTC packets so that the timestamp appears 266 * to go backwards. One estimate is that can be up to about 40 CPU 267 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 268 * slippage an order of magnitude more to be on the safe side. 269 */ 270 decoder->tsc_slip = 0x10000; 271 272 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 273 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 274 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 275 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 276 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 277 278 return decoder; 279 } 280 281 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 282 { 283 struct intel_pt_blk *blk = stack->blk; 284 285 stack->blk = blk->prev; 286 if (!stack->spare) 287 stack->spare = blk; 288 else 289 free(blk); 290 } 291 292 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 293 { 294 if (!stack->pos) { 295 if (!stack->blk) 296 return 0; 297 intel_pt_pop_blk(stack); 298 if (!stack->blk) 299 return 0; 300 stack->pos = INTEL_PT_BLK_SIZE; 301 } 302 return stack->blk->ip[--stack->pos]; 303 } 304 305 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 306 { 307 struct intel_pt_blk *blk; 308 309 if (stack->spare) { 310 blk = stack->spare; 311 stack->spare = NULL; 312 } else { 313 blk = malloc(sizeof(struct intel_pt_blk)); 314 if (!blk) 315 return -ENOMEM; 316 } 317 318 blk->prev = stack->blk; 319 stack->blk = blk; 320 stack->pos = 0; 321 return 0; 322 } 323 324 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 325 { 326 int err; 327 328 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 329 err = intel_pt_alloc_blk(stack); 330 if (err) 331 return err; 332 } 333 334 stack->blk->ip[stack->pos++] = ip; 335 return 0; 336 } 337 338 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 339 { 340 while (stack->blk) 341 intel_pt_pop_blk(stack); 342 stack->pos = 0; 343 } 344 345 static void intel_pt_free_stack(struct intel_pt_stack *stack) 346 { 347 intel_pt_clear_stack(stack); 348 zfree(&stack->blk); 349 zfree(&stack->spare); 350 } 351 352 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 353 { 354 intel_pt_free_stack(&decoder->stack); 355 free(decoder); 356 } 357 358 static int intel_pt_ext_err(int code) 359 { 360 switch (code) { 361 case -ENOMEM: 362 return INTEL_PT_ERR_NOMEM; 363 case -ENOSYS: 364 return INTEL_PT_ERR_INTERN; 365 case -EBADMSG: 366 return INTEL_PT_ERR_BADPKT; 367 case -ENODATA: 368 return INTEL_PT_ERR_NODATA; 369 case -EILSEQ: 370 return INTEL_PT_ERR_NOINSN; 371 case -ENOENT: 372 return INTEL_PT_ERR_MISMAT; 373 case -EOVERFLOW: 374 return INTEL_PT_ERR_OVR; 375 case -ENOSPC: 376 return INTEL_PT_ERR_LOST; 377 case -ELOOP: 378 return INTEL_PT_ERR_NELOOP; 379 default: 380 return INTEL_PT_ERR_UNK; 381 } 382 } 383 384 static const char *intel_pt_err_msgs[] = { 385 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 386 [INTEL_PT_ERR_INTERN] = "Internal error", 387 [INTEL_PT_ERR_BADPKT] = "Bad packet", 388 [INTEL_PT_ERR_NODATA] = "No more data", 389 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 390 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 391 [INTEL_PT_ERR_OVR] = "Overflow packet", 392 [INTEL_PT_ERR_LOST] = "Lost trace data", 393 [INTEL_PT_ERR_UNK] = "Unknown error!", 394 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 395 }; 396 397 int intel_pt__strerror(int code, char *buf, size_t buflen) 398 { 399 if (code < 1 || code >= INTEL_PT_ERR_MAX) 400 code = INTEL_PT_ERR_UNK; 401 strlcpy(buf, intel_pt_err_msgs[code], buflen); 402 return 0; 403 } 404 405 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 406 uint64_t last_ip) 407 { 408 uint64_t ip; 409 410 switch (packet->count) { 411 case 1: 412 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 413 packet->payload; 414 break; 415 case 2: 416 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 417 packet->payload; 418 break; 419 case 3: 420 ip = packet->payload; 421 /* Sign-extend 6-byte ip */ 422 if (ip & (uint64_t)0x800000000000ULL) 423 ip |= (uint64_t)0xffff000000000000ULL; 424 break; 425 case 4: 426 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 427 packet->payload; 428 break; 429 case 6: 430 ip = packet->payload; 431 break; 432 default: 433 return 0; 434 } 435 436 return ip; 437 } 438 439 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 440 { 441 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 442 decoder->have_last_ip = true; 443 } 444 445 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 446 { 447 intel_pt_set_last_ip(decoder); 448 decoder->ip = decoder->last_ip; 449 } 450 451 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 452 { 453 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 454 decoder->buf); 455 } 456 457 static int intel_pt_bug(struct intel_pt_decoder *decoder) 458 { 459 intel_pt_log("ERROR: Internal error\n"); 460 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 461 return -ENOSYS; 462 } 463 464 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 465 { 466 decoder->tx_flags = 0; 467 } 468 469 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 470 { 471 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 472 } 473 474 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 475 { 476 intel_pt_clear_tx_flags(decoder); 477 decoder->have_tma = false; 478 decoder->pkt_len = 1; 479 decoder->pkt_step = 1; 480 intel_pt_decoder_log_packet(decoder); 481 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 482 intel_pt_log("ERROR: Bad packet\n"); 483 decoder->pkt_state = INTEL_PT_STATE_ERR1; 484 } 485 return -EBADMSG; 486 } 487 488 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 489 { 490 decoder->sample_timestamp = decoder->timestamp; 491 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 492 } 493 494 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 495 { 496 decoder->ip = 0; 497 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 498 decoder->timestamp = 0; 499 decoder->have_tma = false; 500 } 501 502 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 503 { 504 struct intel_pt_buffer buffer = { .buf = 0, }; 505 int ret; 506 507 decoder->pkt_step = 0; 508 509 intel_pt_log("Getting more data\n"); 510 ret = decoder->get_trace(&buffer, decoder->data); 511 if (ret) 512 return ret; 513 decoder->buf = buffer.buf; 514 decoder->len = buffer.len; 515 if (!decoder->len) { 516 intel_pt_log("No more data\n"); 517 return -ENODATA; 518 } 519 decoder->buf_timestamp = buffer.ref_timestamp; 520 if (!buffer.consecutive || reposition) { 521 intel_pt_reposition(decoder); 522 decoder->ref_timestamp = buffer.ref_timestamp; 523 decoder->state.trace_nr = buffer.trace_nr; 524 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 525 decoder->ref_timestamp); 526 return -ENOLINK; 527 } 528 529 return 0; 530 } 531 532 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 533 bool reposition) 534 { 535 if (!decoder->next_buf) 536 return intel_pt_get_data(decoder, reposition); 537 538 decoder->buf = decoder->next_buf; 539 decoder->len = decoder->next_len; 540 decoder->next_buf = 0; 541 decoder->next_len = 0; 542 return 0; 543 } 544 545 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 546 { 547 unsigned char *buf = decoder->temp_buf; 548 size_t old_len, len, n; 549 int ret; 550 551 old_len = decoder->len; 552 len = decoder->len; 553 memcpy(buf, decoder->buf, len); 554 555 ret = intel_pt_get_data(decoder, false); 556 if (ret) { 557 decoder->pos += old_len; 558 return ret < 0 ? ret : -EINVAL; 559 } 560 561 n = INTEL_PT_PKT_MAX_SZ - len; 562 if (n > decoder->len) 563 n = decoder->len; 564 memcpy(buf + len, decoder->buf, n); 565 len += n; 566 567 decoder->prev_pkt_ctx = decoder->pkt_ctx; 568 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 569 if (ret < (int)old_len) { 570 decoder->next_buf = decoder->buf; 571 decoder->next_len = decoder->len; 572 decoder->buf = buf; 573 decoder->len = old_len; 574 return intel_pt_bad_packet(decoder); 575 } 576 577 decoder->next_buf = decoder->buf + (ret - old_len); 578 decoder->next_len = decoder->len - (ret - old_len); 579 580 decoder->buf = buf; 581 decoder->len = ret; 582 583 return ret; 584 } 585 586 struct intel_pt_pkt_info { 587 struct intel_pt_decoder *decoder; 588 struct intel_pt_pkt packet; 589 uint64_t pos; 590 int pkt_len; 591 int last_packet_type; 592 void *data; 593 }; 594 595 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 596 597 /* Lookahead packets in current buffer */ 598 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 599 intel_pt_pkt_cb_t cb, void *data) 600 { 601 struct intel_pt_pkt_info pkt_info; 602 const unsigned char *buf = decoder->buf; 603 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 604 size_t len = decoder->len; 605 int ret; 606 607 pkt_info.decoder = decoder; 608 pkt_info.pos = decoder->pos; 609 pkt_info.pkt_len = decoder->pkt_step; 610 pkt_info.last_packet_type = decoder->last_packet_type; 611 pkt_info.data = data; 612 613 while (1) { 614 do { 615 pkt_info.pos += pkt_info.pkt_len; 616 buf += pkt_info.pkt_len; 617 len -= pkt_info.pkt_len; 618 619 if (!len) 620 return INTEL_PT_NEED_MORE_BYTES; 621 622 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 623 &pkt_ctx); 624 if (!ret) 625 return INTEL_PT_NEED_MORE_BYTES; 626 if (ret < 0) 627 return ret; 628 629 pkt_info.pkt_len = ret; 630 } while (pkt_info.packet.type == INTEL_PT_PAD); 631 632 ret = cb(&pkt_info); 633 if (ret) 634 return 0; 635 636 pkt_info.last_packet_type = pkt_info.packet.type; 637 } 638 } 639 640 struct intel_pt_calc_cyc_to_tsc_info { 641 uint64_t cycle_cnt; 642 unsigned int cbr; 643 uint32_t last_mtc; 644 uint64_t ctc_timestamp; 645 uint64_t ctc_delta; 646 uint64_t tsc_timestamp; 647 uint64_t timestamp; 648 bool have_tma; 649 bool fixup_last_mtc; 650 bool from_mtc; 651 double cbr_cyc_to_tsc; 652 }; 653 654 /* 655 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 656 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 657 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 658 * packet by copying the missing bits from the current MTC assuming the least 659 * difference between the two, and that the current MTC comes after last_mtc. 660 */ 661 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 662 uint32_t *last_mtc) 663 { 664 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 665 uint32_t mask = ~(first_missing_bit - 1); 666 667 *last_mtc |= mtc & mask; 668 if (*last_mtc >= mtc) { 669 *last_mtc -= first_missing_bit; 670 *last_mtc &= 0xff; 671 } 672 } 673 674 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 675 { 676 struct intel_pt_decoder *decoder = pkt_info->decoder; 677 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 678 uint64_t timestamp; 679 double cyc_to_tsc; 680 unsigned int cbr; 681 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 682 683 switch (pkt_info->packet.type) { 684 case INTEL_PT_TNT: 685 case INTEL_PT_TIP_PGE: 686 case INTEL_PT_TIP: 687 case INTEL_PT_FUP: 688 case INTEL_PT_PSB: 689 case INTEL_PT_PIP: 690 case INTEL_PT_MODE_EXEC: 691 case INTEL_PT_MODE_TSX: 692 case INTEL_PT_PSBEND: 693 case INTEL_PT_PAD: 694 case INTEL_PT_VMCS: 695 case INTEL_PT_MNT: 696 case INTEL_PT_PTWRITE: 697 case INTEL_PT_PTWRITE_IP: 698 case INTEL_PT_BBP: 699 case INTEL_PT_BIP: 700 case INTEL_PT_BEP: 701 case INTEL_PT_BEP_IP: 702 return 0; 703 704 case INTEL_PT_MTC: 705 if (!data->have_tma) 706 return 0; 707 708 mtc = pkt_info->packet.payload; 709 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 710 data->fixup_last_mtc = false; 711 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 712 &data->last_mtc); 713 } 714 if (mtc > data->last_mtc) 715 mtc_delta = mtc - data->last_mtc; 716 else 717 mtc_delta = mtc + 256 - data->last_mtc; 718 data->ctc_delta += mtc_delta << decoder->mtc_shift; 719 data->last_mtc = mtc; 720 721 if (decoder->tsc_ctc_mult) { 722 timestamp = data->ctc_timestamp + 723 data->ctc_delta * decoder->tsc_ctc_mult; 724 } else { 725 timestamp = data->ctc_timestamp + 726 multdiv(data->ctc_delta, 727 decoder->tsc_ctc_ratio_n, 728 decoder->tsc_ctc_ratio_d); 729 } 730 731 if (timestamp < data->timestamp) 732 return 1; 733 734 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 735 data->timestamp = timestamp; 736 return 0; 737 } 738 739 break; 740 741 case INTEL_PT_TSC: 742 /* 743 * For now, do not support using TSC packets - refer 744 * intel_pt_calc_cyc_to_tsc(). 745 */ 746 if (data->from_mtc) 747 return 1; 748 timestamp = pkt_info->packet.payload | 749 (data->timestamp & (0xffULL << 56)); 750 if (data->from_mtc && timestamp < data->timestamp && 751 data->timestamp - timestamp < decoder->tsc_slip) 752 return 1; 753 if (timestamp < data->timestamp) 754 timestamp += (1ULL << 56); 755 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 756 if (data->from_mtc) 757 return 1; 758 data->tsc_timestamp = timestamp; 759 data->timestamp = timestamp; 760 return 0; 761 } 762 break; 763 764 case INTEL_PT_TMA: 765 if (data->from_mtc) 766 return 1; 767 768 if (!decoder->tsc_ctc_ratio_d) 769 return 0; 770 771 ctc = pkt_info->packet.payload; 772 fc = pkt_info->packet.count; 773 ctc_rem = ctc & decoder->ctc_rem_mask; 774 775 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 776 777 data->ctc_timestamp = data->tsc_timestamp - fc; 778 if (decoder->tsc_ctc_mult) { 779 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 780 } else { 781 data->ctc_timestamp -= 782 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 783 decoder->tsc_ctc_ratio_d); 784 } 785 786 data->ctc_delta = 0; 787 data->have_tma = true; 788 data->fixup_last_mtc = true; 789 790 return 0; 791 792 case INTEL_PT_CYC: 793 data->cycle_cnt += pkt_info->packet.payload; 794 return 0; 795 796 case INTEL_PT_CBR: 797 cbr = pkt_info->packet.payload; 798 if (data->cbr && data->cbr != cbr) 799 return 1; 800 data->cbr = cbr; 801 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 802 return 0; 803 804 case INTEL_PT_TIP_PGD: 805 case INTEL_PT_TRACESTOP: 806 case INTEL_PT_EXSTOP: 807 case INTEL_PT_EXSTOP_IP: 808 case INTEL_PT_MWAIT: 809 case INTEL_PT_PWRE: 810 case INTEL_PT_PWRX: 811 case INTEL_PT_OVF: 812 case INTEL_PT_BAD: /* Does not happen */ 813 default: 814 return 1; 815 } 816 817 if (!data->cbr && decoder->cbr) { 818 data->cbr = decoder->cbr; 819 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 820 } 821 822 if (!data->cycle_cnt) 823 return 1; 824 825 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 826 827 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 828 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 829 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 830 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 831 return 1; 832 } 833 834 decoder->calc_cyc_to_tsc = cyc_to_tsc; 835 decoder->have_calc_cyc_to_tsc = true; 836 837 if (data->cbr) { 838 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 839 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 840 } else { 841 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 842 cyc_to_tsc, pkt_info->pos); 843 } 844 845 return 1; 846 } 847 848 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 849 bool from_mtc) 850 { 851 struct intel_pt_calc_cyc_to_tsc_info data = { 852 .cycle_cnt = 0, 853 .cbr = 0, 854 .last_mtc = decoder->last_mtc, 855 .ctc_timestamp = decoder->ctc_timestamp, 856 .ctc_delta = decoder->ctc_delta, 857 .tsc_timestamp = decoder->tsc_timestamp, 858 .timestamp = decoder->timestamp, 859 .have_tma = decoder->have_tma, 860 .fixup_last_mtc = decoder->fixup_last_mtc, 861 .from_mtc = from_mtc, 862 .cbr_cyc_to_tsc = 0, 863 }; 864 865 /* 866 * For now, do not support using TSC packets for at least the reasons: 867 * 1) timing might have stopped 868 * 2) TSC packets within PSB+ can slip against CYC packets 869 */ 870 if (!from_mtc) 871 return; 872 873 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 874 } 875 876 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 877 { 878 int ret; 879 880 decoder->last_packet_type = decoder->packet.type; 881 882 do { 883 decoder->pos += decoder->pkt_step; 884 decoder->buf += decoder->pkt_step; 885 decoder->len -= decoder->pkt_step; 886 887 if (!decoder->len) { 888 ret = intel_pt_get_next_data(decoder, false); 889 if (ret) 890 return ret; 891 } 892 893 decoder->prev_pkt_ctx = decoder->pkt_ctx; 894 ret = intel_pt_get_packet(decoder->buf, decoder->len, 895 &decoder->packet, &decoder->pkt_ctx); 896 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 897 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 898 ret = intel_pt_get_split_packet(decoder); 899 if (ret < 0) 900 return ret; 901 } 902 if (ret <= 0) 903 return intel_pt_bad_packet(decoder); 904 905 decoder->pkt_len = ret; 906 decoder->pkt_step = ret; 907 intel_pt_decoder_log_packet(decoder); 908 } while (decoder->packet.type == INTEL_PT_PAD); 909 910 return 0; 911 } 912 913 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 914 { 915 uint64_t timestamp, masked_timestamp; 916 917 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 918 masked_timestamp = timestamp & decoder->period_mask; 919 if (decoder->continuous_period) { 920 if (masked_timestamp > decoder->last_masked_timestamp) 921 return 1; 922 } else { 923 timestamp += 1; 924 masked_timestamp = timestamp & decoder->period_mask; 925 if (masked_timestamp > decoder->last_masked_timestamp) { 926 decoder->last_masked_timestamp = masked_timestamp; 927 decoder->continuous_period = true; 928 } 929 } 930 931 if (masked_timestamp < decoder->last_masked_timestamp) 932 return decoder->period_ticks; 933 934 return decoder->period_ticks - (timestamp - masked_timestamp); 935 } 936 937 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 938 { 939 switch (decoder->period_type) { 940 case INTEL_PT_PERIOD_INSTRUCTIONS: 941 return decoder->period - decoder->period_insn_cnt; 942 case INTEL_PT_PERIOD_TICKS: 943 return intel_pt_next_period(decoder); 944 case INTEL_PT_PERIOD_NONE: 945 case INTEL_PT_PERIOD_MTC: 946 default: 947 return 0; 948 } 949 } 950 951 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 952 { 953 uint64_t timestamp, masked_timestamp; 954 955 switch (decoder->period_type) { 956 case INTEL_PT_PERIOD_INSTRUCTIONS: 957 decoder->period_insn_cnt = 0; 958 break; 959 case INTEL_PT_PERIOD_TICKS: 960 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 961 masked_timestamp = timestamp & decoder->period_mask; 962 if (masked_timestamp > decoder->last_masked_timestamp) 963 decoder->last_masked_timestamp = masked_timestamp; 964 else 965 decoder->last_masked_timestamp += decoder->period_ticks; 966 break; 967 case INTEL_PT_PERIOD_NONE: 968 case INTEL_PT_PERIOD_MTC: 969 default: 970 break; 971 } 972 973 decoder->state.type |= INTEL_PT_INSTRUCTION; 974 } 975 976 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 977 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 978 { 979 uint64_t max_insn_cnt, insn_cnt = 0; 980 int err; 981 982 if (!decoder->mtc_insn) 983 decoder->mtc_insn = true; 984 985 max_insn_cnt = intel_pt_next_sample(decoder); 986 987 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 988 max_insn_cnt, decoder->data); 989 990 decoder->tot_insn_cnt += insn_cnt; 991 decoder->timestamp_insn_cnt += insn_cnt; 992 decoder->sample_insn_cnt += insn_cnt; 993 decoder->period_insn_cnt += insn_cnt; 994 995 if (err) { 996 decoder->no_progress = 0; 997 decoder->pkt_state = INTEL_PT_STATE_ERR2; 998 intel_pt_log_at("ERROR: Failed to get instruction", 999 decoder->ip); 1000 if (err == -ENOENT) 1001 return -ENOLINK; 1002 return -EILSEQ; 1003 } 1004 1005 if (ip && decoder->ip == ip) { 1006 err = -EAGAIN; 1007 goto out; 1008 } 1009 1010 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1011 intel_pt_sample_insn(decoder); 1012 1013 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1014 decoder->state.type = INTEL_PT_INSTRUCTION; 1015 decoder->state.from_ip = decoder->ip; 1016 decoder->state.to_ip = 0; 1017 decoder->ip += intel_pt_insn->length; 1018 err = INTEL_PT_RETURN; 1019 goto out; 1020 } 1021 1022 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1023 /* Zero-length calls are excluded */ 1024 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1025 intel_pt_insn->rel) { 1026 err = intel_pt_push(&decoder->stack, decoder->ip + 1027 intel_pt_insn->length); 1028 if (err) 1029 goto out; 1030 } 1031 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1032 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1033 } 1034 1035 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1036 int cnt = decoder->no_progress++; 1037 1038 decoder->state.from_ip = decoder->ip; 1039 decoder->ip += intel_pt_insn->length + 1040 intel_pt_insn->rel; 1041 decoder->state.to_ip = decoder->ip; 1042 err = INTEL_PT_RETURN; 1043 1044 /* 1045 * Check for being stuck in a loop. This can happen if a 1046 * decoder error results in the decoder erroneously setting the 1047 * ip to an address that is itself in an infinite loop that 1048 * consumes no packets. When that happens, there must be an 1049 * unconditional branch. 1050 */ 1051 if (cnt) { 1052 if (cnt == 1) { 1053 decoder->stuck_ip = decoder->state.to_ip; 1054 decoder->stuck_ip_prd = 1; 1055 decoder->stuck_ip_cnt = 1; 1056 } else if (cnt > INTEL_PT_MAX_LOOPS || 1057 decoder->state.to_ip == decoder->stuck_ip) { 1058 intel_pt_log_at("ERROR: Never-ending loop", 1059 decoder->state.to_ip); 1060 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1061 err = -ELOOP; 1062 goto out; 1063 } else if (!--decoder->stuck_ip_cnt) { 1064 decoder->stuck_ip_prd += 1; 1065 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1066 decoder->stuck_ip = decoder->state.to_ip; 1067 } 1068 } 1069 goto out_no_progress; 1070 } 1071 out: 1072 decoder->no_progress = 0; 1073 out_no_progress: 1074 decoder->state.insn_op = intel_pt_insn->op; 1075 decoder->state.insn_len = intel_pt_insn->length; 1076 memcpy(decoder->state.insn, intel_pt_insn->buf, 1077 INTEL_PT_INSN_BUF_SZ); 1078 1079 if (decoder->tx_flags & INTEL_PT_IN_TX) 1080 decoder->state.flags |= INTEL_PT_IN_TX; 1081 1082 return err; 1083 } 1084 1085 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1086 { 1087 bool ret = false; 1088 1089 if (decoder->set_fup_tx_flags) { 1090 decoder->set_fup_tx_flags = false; 1091 decoder->tx_flags = decoder->fup_tx_flags; 1092 decoder->state.type = INTEL_PT_TRANSACTION; 1093 decoder->state.from_ip = decoder->ip; 1094 decoder->state.to_ip = 0; 1095 decoder->state.flags = decoder->fup_tx_flags; 1096 return true; 1097 } 1098 if (decoder->set_fup_ptw) { 1099 decoder->set_fup_ptw = false; 1100 decoder->state.type = INTEL_PT_PTW; 1101 decoder->state.flags |= INTEL_PT_FUP_IP; 1102 decoder->state.from_ip = decoder->ip; 1103 decoder->state.to_ip = 0; 1104 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1105 return true; 1106 } 1107 if (decoder->set_fup_mwait) { 1108 decoder->set_fup_mwait = false; 1109 decoder->state.type = INTEL_PT_MWAIT_OP; 1110 decoder->state.from_ip = decoder->ip; 1111 decoder->state.to_ip = 0; 1112 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1113 ret = true; 1114 } 1115 if (decoder->set_fup_pwre) { 1116 decoder->set_fup_pwre = false; 1117 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1118 decoder->state.type &= ~INTEL_PT_BRANCH; 1119 decoder->state.from_ip = decoder->ip; 1120 decoder->state.to_ip = 0; 1121 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1122 ret = true; 1123 } 1124 if (decoder->set_fup_exstop) { 1125 decoder->set_fup_exstop = false; 1126 decoder->state.type |= INTEL_PT_EX_STOP; 1127 decoder->state.type &= ~INTEL_PT_BRANCH; 1128 decoder->state.flags |= INTEL_PT_FUP_IP; 1129 decoder->state.from_ip = decoder->ip; 1130 decoder->state.to_ip = 0; 1131 ret = true; 1132 } 1133 if (decoder->set_fup_bep) { 1134 decoder->set_fup_bep = false; 1135 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1136 decoder->state.type &= ~INTEL_PT_BRANCH; 1137 decoder->state.from_ip = decoder->ip; 1138 decoder->state.to_ip = 0; 1139 ret = true; 1140 } 1141 return ret; 1142 } 1143 1144 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1145 struct intel_pt_insn *intel_pt_insn, 1146 uint64_t ip, int err) 1147 { 1148 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1149 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1150 ip == decoder->ip + intel_pt_insn->length; 1151 } 1152 1153 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1154 { 1155 struct intel_pt_insn intel_pt_insn; 1156 uint64_t ip; 1157 int err; 1158 1159 ip = decoder->last_ip; 1160 1161 while (1) { 1162 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1163 if (err == INTEL_PT_RETURN) 1164 return 0; 1165 if (err == -EAGAIN || 1166 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1167 if (intel_pt_fup_event(decoder)) 1168 return 0; 1169 return -EAGAIN; 1170 } 1171 decoder->set_fup_tx_flags = false; 1172 if (err) 1173 return err; 1174 1175 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1176 intel_pt_log_at("ERROR: Unexpected indirect branch", 1177 decoder->ip); 1178 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1179 return -ENOENT; 1180 } 1181 1182 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1183 intel_pt_log_at("ERROR: Unexpected conditional branch", 1184 decoder->ip); 1185 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1186 return -ENOENT; 1187 } 1188 1189 intel_pt_bug(decoder); 1190 } 1191 } 1192 1193 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1194 { 1195 struct intel_pt_insn intel_pt_insn; 1196 int err; 1197 1198 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1199 if (err == INTEL_PT_RETURN && 1200 decoder->pgd_ip && 1201 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1202 (decoder->state.type & INTEL_PT_BRANCH) && 1203 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1204 /* Unconditional branch leaving filter region */ 1205 decoder->no_progress = 0; 1206 decoder->pge = false; 1207 decoder->continuous_period = false; 1208 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1209 decoder->state.type |= INTEL_PT_TRACE_END; 1210 return 0; 1211 } 1212 if (err == INTEL_PT_RETURN) 1213 return 0; 1214 if (err) 1215 return err; 1216 1217 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1218 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1219 decoder->pge = false; 1220 decoder->continuous_period = false; 1221 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1222 decoder->state.from_ip = decoder->ip; 1223 if (decoder->packet.count == 0) { 1224 decoder->state.to_ip = 0; 1225 } else { 1226 decoder->state.to_ip = decoder->last_ip; 1227 decoder->ip = decoder->last_ip; 1228 } 1229 decoder->state.type |= INTEL_PT_TRACE_END; 1230 } else { 1231 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1232 decoder->state.from_ip = decoder->ip; 1233 if (decoder->packet.count == 0) { 1234 decoder->state.to_ip = 0; 1235 } else { 1236 decoder->state.to_ip = decoder->last_ip; 1237 decoder->ip = decoder->last_ip; 1238 } 1239 } 1240 return 0; 1241 } 1242 1243 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1244 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1245 intel_pt_insn.rel; 1246 1247 if (decoder->pgd_ip && 1248 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1249 decoder->pgd_ip(to_ip, decoder->data)) { 1250 /* Conditional branch leaving filter region */ 1251 decoder->pge = false; 1252 decoder->continuous_period = false; 1253 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1254 decoder->ip = to_ip; 1255 decoder->state.from_ip = decoder->ip; 1256 decoder->state.to_ip = to_ip; 1257 decoder->state.type |= INTEL_PT_TRACE_END; 1258 return 0; 1259 } 1260 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1261 decoder->ip); 1262 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1263 return -ENOENT; 1264 } 1265 1266 return intel_pt_bug(decoder); 1267 } 1268 1269 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1270 { 1271 struct intel_pt_insn intel_pt_insn; 1272 int err; 1273 1274 while (1) { 1275 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1276 if (err == INTEL_PT_RETURN) 1277 return 0; 1278 if (err) 1279 return err; 1280 1281 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1282 if (!decoder->return_compression) { 1283 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1284 decoder->ip); 1285 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1286 return -ENOENT; 1287 } 1288 if (!decoder->ret_addr) { 1289 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1290 decoder->ip); 1291 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1292 return -ENOENT; 1293 } 1294 if (!(decoder->tnt.payload & BIT63)) { 1295 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1296 decoder->ip); 1297 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1298 return -ENOENT; 1299 } 1300 decoder->tnt.count -= 1; 1301 if (decoder->tnt.count) 1302 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1303 else 1304 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1305 decoder->tnt.payload <<= 1; 1306 decoder->state.from_ip = decoder->ip; 1307 decoder->ip = decoder->ret_addr; 1308 decoder->state.to_ip = decoder->ip; 1309 return 0; 1310 } 1311 1312 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1313 /* Handle deferred TIPs */ 1314 err = intel_pt_get_next_packet(decoder); 1315 if (err) 1316 return err; 1317 if (decoder->packet.type != INTEL_PT_TIP || 1318 decoder->packet.count == 0) { 1319 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1320 decoder->ip); 1321 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1322 decoder->pkt_step = 0; 1323 return -ENOENT; 1324 } 1325 intel_pt_set_last_ip(decoder); 1326 decoder->state.from_ip = decoder->ip; 1327 decoder->state.to_ip = decoder->last_ip; 1328 decoder->ip = decoder->last_ip; 1329 return 0; 1330 } 1331 1332 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1333 decoder->tnt.count -= 1; 1334 if (decoder->tnt.count) 1335 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1336 else 1337 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1338 if (decoder->tnt.payload & BIT63) { 1339 decoder->tnt.payload <<= 1; 1340 decoder->state.from_ip = decoder->ip; 1341 decoder->ip += intel_pt_insn.length + 1342 intel_pt_insn.rel; 1343 decoder->state.to_ip = decoder->ip; 1344 return 0; 1345 } 1346 /* Instruction sample for a non-taken branch */ 1347 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1348 decoder->tnt.payload <<= 1; 1349 decoder->state.type = INTEL_PT_INSTRUCTION; 1350 decoder->state.from_ip = decoder->ip; 1351 decoder->state.to_ip = 0; 1352 decoder->ip += intel_pt_insn.length; 1353 return 0; 1354 } 1355 decoder->sample_cyc = false; 1356 decoder->ip += intel_pt_insn.length; 1357 if (!decoder->tnt.count) { 1358 intel_pt_update_sample_time(decoder); 1359 return -EAGAIN; 1360 } 1361 decoder->tnt.payload <<= 1; 1362 continue; 1363 } 1364 1365 return intel_pt_bug(decoder); 1366 } 1367 } 1368 1369 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1370 { 1371 unsigned int fup_tx_flags; 1372 int err; 1373 1374 fup_tx_flags = decoder->packet.payload & 1375 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1376 err = intel_pt_get_next_packet(decoder); 1377 if (err) 1378 return err; 1379 if (decoder->packet.type == INTEL_PT_FUP) { 1380 decoder->fup_tx_flags = fup_tx_flags; 1381 decoder->set_fup_tx_flags = true; 1382 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1383 *no_tip = true; 1384 } else { 1385 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1386 decoder->pos); 1387 intel_pt_update_in_tx(decoder); 1388 } 1389 return 0; 1390 } 1391 1392 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1393 { 1394 timestamp |= (ref_timestamp & (0xffULL << 56)); 1395 1396 if (timestamp < ref_timestamp) { 1397 if (ref_timestamp - timestamp > (1ULL << 55)) 1398 timestamp += (1ULL << 56); 1399 } else { 1400 if (timestamp - ref_timestamp > (1ULL << 55)) 1401 timestamp -= (1ULL << 56); 1402 } 1403 1404 return timestamp; 1405 } 1406 1407 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1408 { 1409 uint64_t timestamp; 1410 1411 decoder->have_tma = false; 1412 1413 if (decoder->ref_timestamp) { 1414 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1415 decoder->ref_timestamp); 1416 decoder->tsc_timestamp = timestamp; 1417 decoder->timestamp = timestamp; 1418 decoder->ref_timestamp = 0; 1419 decoder->timestamp_insn_cnt = 0; 1420 } else if (decoder->timestamp) { 1421 timestamp = decoder->packet.payload | 1422 (decoder->timestamp & (0xffULL << 56)); 1423 decoder->tsc_timestamp = timestamp; 1424 if (timestamp < decoder->timestamp && 1425 decoder->timestamp - timestamp < decoder->tsc_slip) { 1426 intel_pt_log_to("Suppressing backwards timestamp", 1427 timestamp); 1428 timestamp = decoder->timestamp; 1429 } 1430 if (timestamp < decoder->timestamp) { 1431 intel_pt_log_to("Wraparound timestamp", timestamp); 1432 timestamp += (1ULL << 56); 1433 decoder->tsc_timestamp = timestamp; 1434 } 1435 decoder->timestamp = timestamp; 1436 decoder->timestamp_insn_cnt = 0; 1437 } 1438 1439 if (decoder->last_packet_type == INTEL_PT_CYC) { 1440 decoder->cyc_ref_timestamp = decoder->timestamp; 1441 decoder->cycle_cnt = 0; 1442 decoder->have_calc_cyc_to_tsc = false; 1443 intel_pt_calc_cyc_to_tsc(decoder, false); 1444 } 1445 1446 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1447 } 1448 1449 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1450 { 1451 intel_pt_log("ERROR: Buffer overflow\n"); 1452 intel_pt_clear_tx_flags(decoder); 1453 decoder->timestamp_insn_cnt = 0; 1454 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1455 decoder->overflow = true; 1456 return -EOVERFLOW; 1457 } 1458 1459 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1460 { 1461 if (decoder->have_cyc) 1462 return; 1463 1464 decoder->cyc_cnt_timestamp = decoder->timestamp; 1465 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1466 } 1467 1468 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1469 { 1470 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1471 1472 if (decoder->pge) 1473 intel_pt_mtc_cyc_cnt_pge(decoder); 1474 } 1475 1476 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1477 { 1478 uint64_t tot_cyc_cnt, tsc_delta; 1479 1480 if (decoder->have_cyc) 1481 return; 1482 1483 decoder->sample_cyc = true; 1484 1485 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1486 return; 1487 1488 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1489 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1490 1491 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1492 decoder->tot_cyc_cnt = tot_cyc_cnt; 1493 } 1494 1495 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1496 { 1497 uint32_t ctc = decoder->packet.payload; 1498 uint32_t fc = decoder->packet.count; 1499 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1500 1501 if (!decoder->tsc_ctc_ratio_d) 1502 return; 1503 1504 if (decoder->pge && !decoder->in_psb) 1505 intel_pt_mtc_cyc_cnt_pge(decoder); 1506 else 1507 intel_pt_mtc_cyc_cnt_upd(decoder); 1508 1509 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1510 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1511 if (decoder->tsc_ctc_mult) { 1512 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1513 } else { 1514 decoder->ctc_timestamp -= multdiv(ctc_rem, 1515 decoder->tsc_ctc_ratio_n, 1516 decoder->tsc_ctc_ratio_d); 1517 } 1518 decoder->ctc_delta = 0; 1519 decoder->have_tma = true; 1520 decoder->fixup_last_mtc = true; 1521 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1522 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1523 } 1524 1525 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1526 { 1527 uint64_t timestamp; 1528 uint32_t mtc, mtc_delta; 1529 1530 if (!decoder->have_tma) 1531 return; 1532 1533 mtc = decoder->packet.payload; 1534 1535 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1536 decoder->fixup_last_mtc = false; 1537 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1538 &decoder->last_mtc); 1539 } 1540 1541 if (mtc > decoder->last_mtc) 1542 mtc_delta = mtc - decoder->last_mtc; 1543 else 1544 mtc_delta = mtc + 256 - decoder->last_mtc; 1545 1546 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1547 1548 if (decoder->tsc_ctc_mult) { 1549 timestamp = decoder->ctc_timestamp + 1550 decoder->ctc_delta * decoder->tsc_ctc_mult; 1551 } else { 1552 timestamp = decoder->ctc_timestamp + 1553 multdiv(decoder->ctc_delta, 1554 decoder->tsc_ctc_ratio_n, 1555 decoder->tsc_ctc_ratio_d); 1556 } 1557 1558 if (timestamp < decoder->timestamp) 1559 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1560 timestamp, decoder->timestamp); 1561 else 1562 decoder->timestamp = timestamp; 1563 1564 intel_pt_mtc_cyc_cnt_upd(decoder); 1565 1566 decoder->timestamp_insn_cnt = 0; 1567 decoder->last_mtc = mtc; 1568 1569 if (decoder->last_packet_type == INTEL_PT_CYC) { 1570 decoder->cyc_ref_timestamp = decoder->timestamp; 1571 decoder->cycle_cnt = 0; 1572 decoder->have_calc_cyc_to_tsc = false; 1573 intel_pt_calc_cyc_to_tsc(decoder, true); 1574 } 1575 1576 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1577 } 1578 1579 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1580 { 1581 unsigned int cbr = decoder->packet.payload & 0xff; 1582 1583 decoder->cbr_payload = decoder->packet.payload; 1584 1585 if (decoder->cbr == cbr) 1586 return; 1587 1588 decoder->cbr = cbr; 1589 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1590 1591 intel_pt_mtc_cyc_cnt_cbr(decoder); 1592 } 1593 1594 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1595 { 1596 uint64_t timestamp = decoder->cyc_ref_timestamp; 1597 1598 decoder->have_cyc = true; 1599 1600 decoder->cycle_cnt += decoder->packet.payload; 1601 if (decoder->pge) 1602 decoder->tot_cyc_cnt += decoder->packet.payload; 1603 decoder->sample_cyc = true; 1604 1605 if (!decoder->cyc_ref_timestamp) 1606 return; 1607 1608 if (decoder->have_calc_cyc_to_tsc) 1609 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1610 else if (decoder->cbr) 1611 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1612 else 1613 return; 1614 1615 if (timestamp < decoder->timestamp) 1616 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1617 timestamp, decoder->timestamp); 1618 else 1619 decoder->timestamp = timestamp; 1620 1621 decoder->timestamp_insn_cnt = 0; 1622 1623 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1624 } 1625 1626 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1627 { 1628 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1629 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1630 decoder->state.items.is_32_bit = false; 1631 } 1632 decoder->blk_type = decoder->packet.payload; 1633 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1634 if (decoder->blk_type == INTEL_PT_GP_REGS) 1635 decoder->state.items.is_32_bit = decoder->packet.count; 1636 if (decoder->blk_type_pos < 0) { 1637 intel_pt_log("WARNING: Unknown block type %u\n", 1638 decoder->blk_type); 1639 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1640 intel_pt_log("WARNING: Duplicate block type %u\n", 1641 decoder->blk_type); 1642 } 1643 } 1644 1645 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1646 { 1647 uint32_t id = decoder->packet.count; 1648 uint32_t bit = 1 << id; 1649 int pos = decoder->blk_type_pos; 1650 1651 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1652 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1653 id, decoder->blk_type); 1654 return; 1655 } 1656 1657 if (decoder->state.items.mask[pos] & bit) { 1658 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1659 id, decoder->blk_type); 1660 } 1661 1662 decoder->state.items.mask[pos] |= bit; 1663 decoder->state.items.val[pos][id] = decoder->packet.payload; 1664 } 1665 1666 /* Walk PSB+ packets when already in sync. */ 1667 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1668 { 1669 int err; 1670 1671 decoder->in_psb = true; 1672 1673 while (1) { 1674 err = intel_pt_get_next_packet(decoder); 1675 if (err) 1676 goto out; 1677 1678 switch (decoder->packet.type) { 1679 case INTEL_PT_PSBEND: 1680 err = 0; 1681 goto out; 1682 1683 case INTEL_PT_TIP_PGD: 1684 case INTEL_PT_TIP_PGE: 1685 case INTEL_PT_TIP: 1686 case INTEL_PT_TNT: 1687 case INTEL_PT_TRACESTOP: 1688 case INTEL_PT_BAD: 1689 case INTEL_PT_PSB: 1690 case INTEL_PT_PTWRITE: 1691 case INTEL_PT_PTWRITE_IP: 1692 case INTEL_PT_EXSTOP: 1693 case INTEL_PT_EXSTOP_IP: 1694 case INTEL_PT_MWAIT: 1695 case INTEL_PT_PWRE: 1696 case INTEL_PT_PWRX: 1697 case INTEL_PT_BBP: 1698 case INTEL_PT_BIP: 1699 case INTEL_PT_BEP: 1700 case INTEL_PT_BEP_IP: 1701 decoder->have_tma = false; 1702 intel_pt_log("ERROR: Unexpected packet\n"); 1703 err = -EAGAIN; 1704 goto out; 1705 1706 case INTEL_PT_OVF: 1707 err = intel_pt_overflow(decoder); 1708 goto out; 1709 1710 case INTEL_PT_TSC: 1711 intel_pt_calc_tsc_timestamp(decoder); 1712 break; 1713 1714 case INTEL_PT_TMA: 1715 intel_pt_calc_tma(decoder); 1716 break; 1717 1718 case INTEL_PT_CBR: 1719 intel_pt_calc_cbr(decoder); 1720 break; 1721 1722 case INTEL_PT_MODE_EXEC: 1723 decoder->exec_mode = decoder->packet.payload; 1724 break; 1725 1726 case INTEL_PT_PIP: 1727 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1728 break; 1729 1730 case INTEL_PT_FUP: 1731 decoder->pge = true; 1732 if (decoder->packet.count) 1733 intel_pt_set_last_ip(decoder); 1734 break; 1735 1736 case INTEL_PT_MODE_TSX: 1737 intel_pt_update_in_tx(decoder); 1738 break; 1739 1740 case INTEL_PT_MTC: 1741 intel_pt_calc_mtc_timestamp(decoder); 1742 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1743 decoder->state.type |= INTEL_PT_INSTRUCTION; 1744 break; 1745 1746 case INTEL_PT_CYC: 1747 case INTEL_PT_VMCS: 1748 case INTEL_PT_MNT: 1749 case INTEL_PT_PAD: 1750 default: 1751 break; 1752 } 1753 } 1754 out: 1755 decoder->in_psb = false; 1756 1757 return err; 1758 } 1759 1760 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1761 { 1762 int err; 1763 1764 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1765 decoder->tx_flags = 0; 1766 decoder->state.flags &= ~INTEL_PT_IN_TX; 1767 decoder->state.flags |= INTEL_PT_ABORT_TX; 1768 } else { 1769 decoder->state.flags |= INTEL_PT_ASYNC; 1770 } 1771 1772 while (1) { 1773 err = intel_pt_get_next_packet(decoder); 1774 if (err) 1775 return err; 1776 1777 switch (decoder->packet.type) { 1778 case INTEL_PT_TNT: 1779 case INTEL_PT_FUP: 1780 case INTEL_PT_TRACESTOP: 1781 case INTEL_PT_PSB: 1782 case INTEL_PT_TSC: 1783 case INTEL_PT_TMA: 1784 case INTEL_PT_MODE_TSX: 1785 case INTEL_PT_BAD: 1786 case INTEL_PT_PSBEND: 1787 case INTEL_PT_PTWRITE: 1788 case INTEL_PT_PTWRITE_IP: 1789 case INTEL_PT_EXSTOP: 1790 case INTEL_PT_EXSTOP_IP: 1791 case INTEL_PT_MWAIT: 1792 case INTEL_PT_PWRE: 1793 case INTEL_PT_PWRX: 1794 case INTEL_PT_BBP: 1795 case INTEL_PT_BIP: 1796 case INTEL_PT_BEP: 1797 case INTEL_PT_BEP_IP: 1798 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1799 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1800 decoder->pkt_step = 0; 1801 return -ENOENT; 1802 1803 case INTEL_PT_CBR: 1804 intel_pt_calc_cbr(decoder); 1805 break; 1806 1807 case INTEL_PT_OVF: 1808 return intel_pt_overflow(decoder); 1809 1810 case INTEL_PT_TIP_PGD: 1811 decoder->state.from_ip = decoder->ip; 1812 if (decoder->packet.count == 0) { 1813 decoder->state.to_ip = 0; 1814 } else { 1815 intel_pt_set_ip(decoder); 1816 decoder->state.to_ip = decoder->ip; 1817 } 1818 decoder->pge = false; 1819 decoder->continuous_period = false; 1820 decoder->state.type |= INTEL_PT_TRACE_END; 1821 return 0; 1822 1823 case INTEL_PT_TIP_PGE: 1824 decoder->pge = true; 1825 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1826 decoder->ip); 1827 decoder->state.from_ip = 0; 1828 if (decoder->packet.count == 0) { 1829 decoder->state.to_ip = 0; 1830 } else { 1831 intel_pt_set_ip(decoder); 1832 decoder->state.to_ip = decoder->ip; 1833 } 1834 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1835 intel_pt_mtc_cyc_cnt_pge(decoder); 1836 return 0; 1837 1838 case INTEL_PT_TIP: 1839 decoder->state.from_ip = decoder->ip; 1840 if (decoder->packet.count == 0) { 1841 decoder->state.to_ip = 0; 1842 } else { 1843 intel_pt_set_ip(decoder); 1844 decoder->state.to_ip = decoder->ip; 1845 } 1846 return 0; 1847 1848 case INTEL_PT_PIP: 1849 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1850 break; 1851 1852 case INTEL_PT_MTC: 1853 intel_pt_calc_mtc_timestamp(decoder); 1854 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1855 decoder->state.type |= INTEL_PT_INSTRUCTION; 1856 break; 1857 1858 case INTEL_PT_CYC: 1859 intel_pt_calc_cyc_timestamp(decoder); 1860 break; 1861 1862 case INTEL_PT_MODE_EXEC: 1863 decoder->exec_mode = decoder->packet.payload; 1864 break; 1865 1866 case INTEL_PT_VMCS: 1867 case INTEL_PT_MNT: 1868 case INTEL_PT_PAD: 1869 break; 1870 1871 default: 1872 return intel_pt_bug(decoder); 1873 } 1874 } 1875 } 1876 1877 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 1878 { 1879 bool no_tip = false; 1880 int err; 1881 1882 while (1) { 1883 err = intel_pt_get_next_packet(decoder); 1884 if (err) 1885 return err; 1886 next: 1887 switch (decoder->packet.type) { 1888 case INTEL_PT_TNT: 1889 if (!decoder->packet.count) 1890 break; 1891 decoder->tnt = decoder->packet; 1892 decoder->pkt_state = INTEL_PT_STATE_TNT; 1893 err = intel_pt_walk_tnt(decoder); 1894 if (err == -EAGAIN) 1895 break; 1896 return err; 1897 1898 case INTEL_PT_TIP_PGD: 1899 if (decoder->packet.count != 0) 1900 intel_pt_set_last_ip(decoder); 1901 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 1902 return intel_pt_walk_tip(decoder); 1903 1904 case INTEL_PT_TIP_PGE: { 1905 decoder->pge = true; 1906 intel_pt_mtc_cyc_cnt_pge(decoder); 1907 if (decoder->packet.count == 0) { 1908 intel_pt_log_at("Skipping zero TIP.PGE", 1909 decoder->pos); 1910 break; 1911 } 1912 intel_pt_set_ip(decoder); 1913 decoder->state.from_ip = 0; 1914 decoder->state.to_ip = decoder->ip; 1915 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1916 return 0; 1917 } 1918 1919 case INTEL_PT_OVF: 1920 return intel_pt_overflow(decoder); 1921 1922 case INTEL_PT_TIP: 1923 if (decoder->packet.count != 0) 1924 intel_pt_set_last_ip(decoder); 1925 decoder->pkt_state = INTEL_PT_STATE_TIP; 1926 return intel_pt_walk_tip(decoder); 1927 1928 case INTEL_PT_FUP: 1929 if (decoder->packet.count == 0) { 1930 intel_pt_log_at("Skipping zero FUP", 1931 decoder->pos); 1932 no_tip = false; 1933 break; 1934 } 1935 intel_pt_set_last_ip(decoder); 1936 if (!decoder->branch_enable) { 1937 decoder->ip = decoder->last_ip; 1938 if (intel_pt_fup_event(decoder)) 1939 return 0; 1940 no_tip = false; 1941 break; 1942 } 1943 if (decoder->set_fup_mwait) 1944 no_tip = true; 1945 err = intel_pt_walk_fup(decoder); 1946 if (err != -EAGAIN) { 1947 if (err) 1948 return err; 1949 if (no_tip) 1950 decoder->pkt_state = 1951 INTEL_PT_STATE_FUP_NO_TIP; 1952 else 1953 decoder->pkt_state = INTEL_PT_STATE_FUP; 1954 return 0; 1955 } 1956 if (no_tip) { 1957 no_tip = false; 1958 break; 1959 } 1960 return intel_pt_walk_fup_tip(decoder); 1961 1962 case INTEL_PT_TRACESTOP: 1963 decoder->pge = false; 1964 decoder->continuous_period = false; 1965 intel_pt_clear_tx_flags(decoder); 1966 decoder->have_tma = false; 1967 break; 1968 1969 case INTEL_PT_PSB: 1970 decoder->last_ip = 0; 1971 decoder->have_last_ip = true; 1972 intel_pt_clear_stack(&decoder->stack); 1973 err = intel_pt_walk_psbend(decoder); 1974 if (err == -EAGAIN) 1975 goto next; 1976 if (err) 1977 return err; 1978 /* 1979 * PSB+ CBR will not have changed but cater for the 1980 * possibility of another CBR change that gets caught up 1981 * in the PSB+. 1982 */ 1983 if (decoder->cbr != decoder->cbr_seen) 1984 return 0; 1985 break; 1986 1987 case INTEL_PT_PIP: 1988 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1989 break; 1990 1991 case INTEL_PT_MTC: 1992 intel_pt_calc_mtc_timestamp(decoder); 1993 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 1994 break; 1995 /* 1996 * Ensure that there has been an instruction since the 1997 * last MTC. 1998 */ 1999 if (!decoder->mtc_insn) 2000 break; 2001 decoder->mtc_insn = false; 2002 /* Ensure that there is a timestamp */ 2003 if (!decoder->timestamp) 2004 break; 2005 decoder->state.type = INTEL_PT_INSTRUCTION; 2006 decoder->state.from_ip = decoder->ip; 2007 decoder->state.to_ip = 0; 2008 decoder->mtc_insn = false; 2009 return 0; 2010 2011 case INTEL_PT_TSC: 2012 intel_pt_calc_tsc_timestamp(decoder); 2013 break; 2014 2015 case INTEL_PT_TMA: 2016 intel_pt_calc_tma(decoder); 2017 break; 2018 2019 case INTEL_PT_CYC: 2020 intel_pt_calc_cyc_timestamp(decoder); 2021 break; 2022 2023 case INTEL_PT_CBR: 2024 intel_pt_calc_cbr(decoder); 2025 if (decoder->cbr != decoder->cbr_seen) 2026 return 0; 2027 break; 2028 2029 case INTEL_PT_MODE_EXEC: 2030 decoder->exec_mode = decoder->packet.payload; 2031 break; 2032 2033 case INTEL_PT_MODE_TSX: 2034 /* MODE_TSX need not be followed by FUP */ 2035 if (!decoder->pge) { 2036 intel_pt_update_in_tx(decoder); 2037 break; 2038 } 2039 err = intel_pt_mode_tsx(decoder, &no_tip); 2040 if (err) 2041 return err; 2042 goto next; 2043 2044 case INTEL_PT_BAD: /* Does not happen */ 2045 return intel_pt_bug(decoder); 2046 2047 case INTEL_PT_PSBEND: 2048 case INTEL_PT_VMCS: 2049 case INTEL_PT_MNT: 2050 case INTEL_PT_PAD: 2051 break; 2052 2053 case INTEL_PT_PTWRITE_IP: 2054 decoder->fup_ptw_payload = decoder->packet.payload; 2055 err = intel_pt_get_next_packet(decoder); 2056 if (err) 2057 return err; 2058 if (decoder->packet.type == INTEL_PT_FUP) { 2059 decoder->set_fup_ptw = true; 2060 no_tip = true; 2061 } else { 2062 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 2063 decoder->pos); 2064 } 2065 goto next; 2066 2067 case INTEL_PT_PTWRITE: 2068 decoder->state.type = INTEL_PT_PTW; 2069 decoder->state.from_ip = decoder->ip; 2070 decoder->state.to_ip = 0; 2071 decoder->state.ptw_payload = decoder->packet.payload; 2072 return 0; 2073 2074 case INTEL_PT_MWAIT: 2075 decoder->fup_mwait_payload = decoder->packet.payload; 2076 decoder->set_fup_mwait = true; 2077 break; 2078 2079 case INTEL_PT_PWRE: 2080 if (decoder->set_fup_mwait) { 2081 decoder->fup_pwre_payload = 2082 decoder->packet.payload; 2083 decoder->set_fup_pwre = true; 2084 break; 2085 } 2086 decoder->state.type = INTEL_PT_PWR_ENTRY; 2087 decoder->state.from_ip = decoder->ip; 2088 decoder->state.to_ip = 0; 2089 decoder->state.pwrx_payload = decoder->packet.payload; 2090 return 0; 2091 2092 case INTEL_PT_EXSTOP_IP: 2093 err = intel_pt_get_next_packet(decoder); 2094 if (err) 2095 return err; 2096 if (decoder->packet.type == INTEL_PT_FUP) { 2097 decoder->set_fup_exstop = true; 2098 no_tip = true; 2099 } else { 2100 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 2101 decoder->pos); 2102 } 2103 goto next; 2104 2105 case INTEL_PT_EXSTOP: 2106 decoder->state.type = INTEL_PT_EX_STOP; 2107 decoder->state.from_ip = decoder->ip; 2108 decoder->state.to_ip = 0; 2109 return 0; 2110 2111 case INTEL_PT_PWRX: 2112 decoder->state.type = INTEL_PT_PWR_EXIT; 2113 decoder->state.from_ip = decoder->ip; 2114 decoder->state.to_ip = 0; 2115 decoder->state.pwrx_payload = decoder->packet.payload; 2116 return 0; 2117 2118 case INTEL_PT_BBP: 2119 intel_pt_bbp(decoder); 2120 break; 2121 2122 case INTEL_PT_BIP: 2123 intel_pt_bip(decoder); 2124 break; 2125 2126 case INTEL_PT_BEP: 2127 decoder->state.type = INTEL_PT_BLK_ITEMS; 2128 decoder->state.from_ip = decoder->ip; 2129 decoder->state.to_ip = 0; 2130 return 0; 2131 2132 case INTEL_PT_BEP_IP: 2133 err = intel_pt_get_next_packet(decoder); 2134 if (err) 2135 return err; 2136 if (decoder->packet.type == INTEL_PT_FUP) { 2137 decoder->set_fup_bep = true; 2138 no_tip = true; 2139 } else { 2140 intel_pt_log_at("ERROR: Missing FUP after BEP", 2141 decoder->pos); 2142 } 2143 goto next; 2144 2145 default: 2146 return intel_pt_bug(decoder); 2147 } 2148 } 2149 } 2150 2151 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 2152 { 2153 return decoder->packet.count && 2154 (decoder->have_last_ip || decoder->packet.count == 3 || 2155 decoder->packet.count == 6); 2156 } 2157 2158 /* Walk PSB+ packets to get in sync. */ 2159 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 2160 { 2161 int err; 2162 2163 decoder->in_psb = true; 2164 2165 while (1) { 2166 err = intel_pt_get_next_packet(decoder); 2167 if (err) 2168 goto out; 2169 2170 switch (decoder->packet.type) { 2171 case INTEL_PT_TIP_PGD: 2172 decoder->continuous_period = false; 2173 __fallthrough; 2174 case INTEL_PT_TIP_PGE: 2175 case INTEL_PT_TIP: 2176 case INTEL_PT_PTWRITE: 2177 case INTEL_PT_PTWRITE_IP: 2178 case INTEL_PT_EXSTOP: 2179 case INTEL_PT_EXSTOP_IP: 2180 case INTEL_PT_MWAIT: 2181 case INTEL_PT_PWRE: 2182 case INTEL_PT_PWRX: 2183 case INTEL_PT_BBP: 2184 case INTEL_PT_BIP: 2185 case INTEL_PT_BEP: 2186 case INTEL_PT_BEP_IP: 2187 intel_pt_log("ERROR: Unexpected packet\n"); 2188 err = -ENOENT; 2189 goto out; 2190 2191 case INTEL_PT_FUP: 2192 decoder->pge = true; 2193 if (intel_pt_have_ip(decoder)) { 2194 uint64_t current_ip = decoder->ip; 2195 2196 intel_pt_set_ip(decoder); 2197 if (current_ip) 2198 intel_pt_log_to("Setting IP", 2199 decoder->ip); 2200 } 2201 break; 2202 2203 case INTEL_PT_MTC: 2204 intel_pt_calc_mtc_timestamp(decoder); 2205 break; 2206 2207 case INTEL_PT_TSC: 2208 intel_pt_calc_tsc_timestamp(decoder); 2209 break; 2210 2211 case INTEL_PT_TMA: 2212 intel_pt_calc_tma(decoder); 2213 break; 2214 2215 case INTEL_PT_CYC: 2216 intel_pt_calc_cyc_timestamp(decoder); 2217 break; 2218 2219 case INTEL_PT_CBR: 2220 intel_pt_calc_cbr(decoder); 2221 break; 2222 2223 case INTEL_PT_PIP: 2224 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2225 break; 2226 2227 case INTEL_PT_MODE_EXEC: 2228 decoder->exec_mode = decoder->packet.payload; 2229 break; 2230 2231 case INTEL_PT_MODE_TSX: 2232 intel_pt_update_in_tx(decoder); 2233 break; 2234 2235 case INTEL_PT_TRACESTOP: 2236 decoder->pge = false; 2237 decoder->continuous_period = false; 2238 intel_pt_clear_tx_flags(decoder); 2239 __fallthrough; 2240 2241 case INTEL_PT_TNT: 2242 decoder->have_tma = false; 2243 intel_pt_log("ERROR: Unexpected packet\n"); 2244 if (decoder->ip) 2245 decoder->pkt_state = INTEL_PT_STATE_ERR4; 2246 else 2247 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2248 err = -ENOENT; 2249 goto out; 2250 2251 case INTEL_PT_BAD: /* Does not happen */ 2252 err = intel_pt_bug(decoder); 2253 goto out; 2254 2255 case INTEL_PT_OVF: 2256 err = intel_pt_overflow(decoder); 2257 goto out; 2258 2259 case INTEL_PT_PSBEND: 2260 err = 0; 2261 goto out; 2262 2263 case INTEL_PT_PSB: 2264 case INTEL_PT_VMCS: 2265 case INTEL_PT_MNT: 2266 case INTEL_PT_PAD: 2267 default: 2268 break; 2269 } 2270 } 2271 out: 2272 decoder->in_psb = false; 2273 2274 return err; 2275 } 2276 2277 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 2278 { 2279 int err; 2280 2281 while (1) { 2282 err = intel_pt_get_next_packet(decoder); 2283 if (err) 2284 return err; 2285 2286 switch (decoder->packet.type) { 2287 case INTEL_PT_TIP_PGD: 2288 decoder->continuous_period = false; 2289 decoder->pge = false; 2290 if (intel_pt_have_ip(decoder)) 2291 intel_pt_set_ip(decoder); 2292 if (!decoder->ip) 2293 break; 2294 decoder->state.type |= INTEL_PT_TRACE_END; 2295 return 0; 2296 2297 case INTEL_PT_TIP_PGE: 2298 decoder->pge = true; 2299 intel_pt_mtc_cyc_cnt_pge(decoder); 2300 if (intel_pt_have_ip(decoder)) 2301 intel_pt_set_ip(decoder); 2302 if (!decoder->ip) 2303 break; 2304 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2305 return 0; 2306 2307 case INTEL_PT_TIP: 2308 decoder->pge = true; 2309 if (intel_pt_have_ip(decoder)) 2310 intel_pt_set_ip(decoder); 2311 if (!decoder->ip) 2312 break; 2313 return 0; 2314 2315 case INTEL_PT_FUP: 2316 if (intel_pt_have_ip(decoder)) 2317 intel_pt_set_ip(decoder); 2318 if (decoder->ip) 2319 return 0; 2320 break; 2321 2322 case INTEL_PT_MTC: 2323 intel_pt_calc_mtc_timestamp(decoder); 2324 break; 2325 2326 case INTEL_PT_TSC: 2327 intel_pt_calc_tsc_timestamp(decoder); 2328 break; 2329 2330 case INTEL_PT_TMA: 2331 intel_pt_calc_tma(decoder); 2332 break; 2333 2334 case INTEL_PT_CYC: 2335 intel_pt_calc_cyc_timestamp(decoder); 2336 break; 2337 2338 case INTEL_PT_CBR: 2339 intel_pt_calc_cbr(decoder); 2340 break; 2341 2342 case INTEL_PT_PIP: 2343 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2344 break; 2345 2346 case INTEL_PT_MODE_EXEC: 2347 decoder->exec_mode = decoder->packet.payload; 2348 break; 2349 2350 case INTEL_PT_MODE_TSX: 2351 intel_pt_update_in_tx(decoder); 2352 break; 2353 2354 case INTEL_PT_OVF: 2355 return intel_pt_overflow(decoder); 2356 2357 case INTEL_PT_BAD: /* Does not happen */ 2358 return intel_pt_bug(decoder); 2359 2360 case INTEL_PT_TRACESTOP: 2361 decoder->pge = false; 2362 decoder->continuous_period = false; 2363 intel_pt_clear_tx_flags(decoder); 2364 decoder->have_tma = false; 2365 break; 2366 2367 case INTEL_PT_PSB: 2368 decoder->last_ip = 0; 2369 decoder->have_last_ip = true; 2370 intel_pt_clear_stack(&decoder->stack); 2371 err = intel_pt_walk_psb(decoder); 2372 if (err) 2373 return err; 2374 if (decoder->ip) { 2375 /* Do not have a sample */ 2376 decoder->state.type = 0; 2377 return 0; 2378 } 2379 break; 2380 2381 case INTEL_PT_TNT: 2382 case INTEL_PT_PSBEND: 2383 case INTEL_PT_VMCS: 2384 case INTEL_PT_MNT: 2385 case INTEL_PT_PAD: 2386 case INTEL_PT_PTWRITE: 2387 case INTEL_PT_PTWRITE_IP: 2388 case INTEL_PT_EXSTOP: 2389 case INTEL_PT_EXSTOP_IP: 2390 case INTEL_PT_MWAIT: 2391 case INTEL_PT_PWRE: 2392 case INTEL_PT_PWRX: 2393 case INTEL_PT_BBP: 2394 case INTEL_PT_BIP: 2395 case INTEL_PT_BEP: 2396 case INTEL_PT_BEP_IP: 2397 default: 2398 break; 2399 } 2400 } 2401 } 2402 2403 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 2404 { 2405 int err; 2406 2407 decoder->set_fup_tx_flags = false; 2408 decoder->set_fup_ptw = false; 2409 decoder->set_fup_mwait = false; 2410 decoder->set_fup_pwre = false; 2411 decoder->set_fup_exstop = false; 2412 decoder->set_fup_bep = false; 2413 2414 if (!decoder->branch_enable) { 2415 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2416 decoder->overflow = false; 2417 decoder->state.type = 0; /* Do not have a sample */ 2418 return 0; 2419 } 2420 2421 intel_pt_log("Scanning for full IP\n"); 2422 err = intel_pt_walk_to_ip(decoder); 2423 if (err) 2424 return err; 2425 2426 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2427 decoder->overflow = false; 2428 2429 decoder->state.from_ip = 0; 2430 decoder->state.to_ip = decoder->ip; 2431 intel_pt_log_to("Setting IP", decoder->ip); 2432 2433 return 0; 2434 } 2435 2436 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 2437 { 2438 const unsigned char *end = decoder->buf + decoder->len; 2439 size_t i; 2440 2441 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 2442 if (i > decoder->len) 2443 continue; 2444 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 2445 return i; 2446 } 2447 return 0; 2448 } 2449 2450 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 2451 { 2452 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 2453 const char *psb = INTEL_PT_PSB_STR; 2454 2455 if (rest_psb > decoder->len || 2456 memcmp(decoder->buf, psb + part_psb, rest_psb)) 2457 return 0; 2458 2459 return rest_psb; 2460 } 2461 2462 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 2463 int part_psb) 2464 { 2465 int rest_psb, ret; 2466 2467 decoder->pos += decoder->len; 2468 decoder->len = 0; 2469 2470 ret = intel_pt_get_next_data(decoder, false); 2471 if (ret) 2472 return ret; 2473 2474 rest_psb = intel_pt_rest_psb(decoder, part_psb); 2475 if (!rest_psb) 2476 return 0; 2477 2478 decoder->pos -= part_psb; 2479 decoder->next_buf = decoder->buf + rest_psb; 2480 decoder->next_len = decoder->len - rest_psb; 2481 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2482 decoder->buf = decoder->temp_buf; 2483 decoder->len = INTEL_PT_PSB_LEN; 2484 2485 return 0; 2486 } 2487 2488 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 2489 { 2490 unsigned char *next; 2491 int ret; 2492 2493 intel_pt_log("Scanning for PSB\n"); 2494 while (1) { 2495 if (!decoder->len) { 2496 ret = intel_pt_get_next_data(decoder, false); 2497 if (ret) 2498 return ret; 2499 } 2500 2501 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2502 INTEL_PT_PSB_LEN); 2503 if (!next) { 2504 int part_psb; 2505 2506 part_psb = intel_pt_part_psb(decoder); 2507 if (part_psb) { 2508 ret = intel_pt_get_split_psb(decoder, part_psb); 2509 if (ret) 2510 return ret; 2511 } else { 2512 decoder->pos += decoder->len; 2513 decoder->len = 0; 2514 } 2515 continue; 2516 } 2517 2518 decoder->pkt_step = next - decoder->buf; 2519 return intel_pt_get_next_packet(decoder); 2520 } 2521 } 2522 2523 static int intel_pt_sync(struct intel_pt_decoder *decoder) 2524 { 2525 int err; 2526 2527 decoder->pge = false; 2528 decoder->continuous_period = false; 2529 decoder->have_last_ip = false; 2530 decoder->last_ip = 0; 2531 decoder->ip = 0; 2532 intel_pt_clear_stack(&decoder->stack); 2533 2534 err = intel_pt_scan_for_psb(decoder); 2535 if (err) 2536 return err; 2537 2538 decoder->have_last_ip = true; 2539 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 2540 2541 err = intel_pt_walk_psb(decoder); 2542 if (err) 2543 return err; 2544 2545 if (decoder->ip) { 2546 decoder->state.type = 0; /* Do not have a sample */ 2547 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2548 } else { 2549 return intel_pt_sync_ip(decoder); 2550 } 2551 2552 return 0; 2553 } 2554 2555 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2556 { 2557 uint64_t est = decoder->sample_insn_cnt << 1; 2558 2559 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2560 goto out; 2561 2562 est *= decoder->max_non_turbo_ratio; 2563 est /= decoder->cbr; 2564 out: 2565 return decoder->sample_timestamp + est; 2566 } 2567 2568 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2569 { 2570 int err; 2571 2572 do { 2573 decoder->state.type = INTEL_PT_BRANCH; 2574 decoder->state.flags = 0; 2575 2576 switch (decoder->pkt_state) { 2577 case INTEL_PT_STATE_NO_PSB: 2578 err = intel_pt_sync(decoder); 2579 break; 2580 case INTEL_PT_STATE_NO_IP: 2581 decoder->have_last_ip = false; 2582 decoder->last_ip = 0; 2583 decoder->ip = 0; 2584 __fallthrough; 2585 case INTEL_PT_STATE_ERR_RESYNC: 2586 err = intel_pt_sync_ip(decoder); 2587 break; 2588 case INTEL_PT_STATE_IN_SYNC: 2589 err = intel_pt_walk_trace(decoder); 2590 break; 2591 case INTEL_PT_STATE_TNT: 2592 case INTEL_PT_STATE_TNT_CONT: 2593 err = intel_pt_walk_tnt(decoder); 2594 if (err == -EAGAIN) 2595 err = intel_pt_walk_trace(decoder); 2596 break; 2597 case INTEL_PT_STATE_TIP: 2598 case INTEL_PT_STATE_TIP_PGD: 2599 err = intel_pt_walk_tip(decoder); 2600 break; 2601 case INTEL_PT_STATE_FUP: 2602 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2603 err = intel_pt_walk_fup(decoder); 2604 if (err == -EAGAIN) 2605 err = intel_pt_walk_fup_tip(decoder); 2606 else if (!err) 2607 decoder->pkt_state = INTEL_PT_STATE_FUP; 2608 break; 2609 case INTEL_PT_STATE_FUP_NO_TIP: 2610 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2611 err = intel_pt_walk_fup(decoder); 2612 if (err == -EAGAIN) 2613 err = intel_pt_walk_trace(decoder); 2614 break; 2615 default: 2616 err = intel_pt_bug(decoder); 2617 break; 2618 } 2619 } while (err == -ENOLINK); 2620 2621 if (err) { 2622 decoder->state.err = intel_pt_ext_err(err); 2623 decoder->state.from_ip = decoder->ip; 2624 intel_pt_update_sample_time(decoder); 2625 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 2626 } else { 2627 decoder->state.err = 0; 2628 if (decoder->cbr != decoder->cbr_seen) { 2629 decoder->cbr_seen = decoder->cbr; 2630 if (!decoder->state.type) { 2631 decoder->state.from_ip = decoder->ip; 2632 decoder->state.to_ip = 0; 2633 } 2634 decoder->state.type |= INTEL_PT_CBR_CHG; 2635 decoder->state.cbr_payload = decoder->cbr_payload; 2636 decoder->state.cbr = decoder->cbr; 2637 } 2638 if (intel_pt_sample_time(decoder->pkt_state)) { 2639 intel_pt_update_sample_time(decoder); 2640 if (decoder->sample_cyc) 2641 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 2642 } 2643 } 2644 2645 decoder->state.timestamp = decoder->sample_timestamp; 2646 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2647 decoder->state.cr3 = decoder->cr3; 2648 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2649 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 2650 2651 return &decoder->state; 2652 } 2653 2654 /** 2655 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2656 * @buf: pointer to buffer pointer 2657 * @len: size of buffer 2658 * 2659 * Updates the buffer pointer to point to the start of the next PSB packet if 2660 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2661 * @len is adjusted accordingly. 2662 * 2663 * Return: %true if a PSB packet is found, %false otherwise. 2664 */ 2665 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2666 { 2667 unsigned char *next; 2668 2669 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2670 if (next) { 2671 *len -= next - *buf; 2672 *buf = next; 2673 return true; 2674 } 2675 return false; 2676 } 2677 2678 /** 2679 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2680 * packet. 2681 * @buf: pointer to buffer pointer 2682 * @len: size of buffer 2683 * 2684 * Updates the buffer pointer to point to the start of the following PSB packet 2685 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2686 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2687 * 2688 * Return: %true if a PSB packet is found, %false otherwise. 2689 */ 2690 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2691 { 2692 unsigned char *next; 2693 2694 if (!*len) 2695 return false; 2696 2697 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2698 if (next) { 2699 *len -= next - *buf; 2700 *buf = next; 2701 return true; 2702 } 2703 return false; 2704 } 2705 2706 /** 2707 * intel_pt_last_psb - find the last PSB packet in a buffer. 2708 * @buf: buffer 2709 * @len: size of buffer 2710 * 2711 * This function finds the last PSB in a buffer. 2712 * 2713 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2714 */ 2715 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2716 { 2717 const char *n = INTEL_PT_PSB_STR; 2718 unsigned char *p; 2719 size_t k; 2720 2721 if (len < INTEL_PT_PSB_LEN) 2722 return NULL; 2723 2724 k = len - INTEL_PT_PSB_LEN + 1; 2725 while (1) { 2726 p = memrchr(buf, n[0], k); 2727 if (!p) 2728 return NULL; 2729 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2730 return p; 2731 k = p - buf; 2732 if (!k) 2733 return NULL; 2734 } 2735 } 2736 2737 /** 2738 * intel_pt_next_tsc - find and return next TSC. 2739 * @buf: buffer 2740 * @len: size of buffer 2741 * @tsc: TSC value returned 2742 * @rem: returns remaining size when TSC is found 2743 * 2744 * Find a TSC packet in @buf and return the TSC value. This function assumes 2745 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2746 * PSBEND packet is found. 2747 * 2748 * Return: %true if TSC is found, false otherwise. 2749 */ 2750 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 2751 size_t *rem) 2752 { 2753 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 2754 struct intel_pt_pkt packet; 2755 int ret; 2756 2757 while (len) { 2758 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 2759 if (ret <= 0) 2760 return false; 2761 if (packet.type == INTEL_PT_TSC) { 2762 *tsc = packet.payload; 2763 *rem = len; 2764 return true; 2765 } 2766 if (packet.type == INTEL_PT_PSBEND) 2767 return false; 2768 buf += ret; 2769 len -= ret; 2770 } 2771 return false; 2772 } 2773 2774 /** 2775 * intel_pt_tsc_cmp - compare 7-byte TSCs. 2776 * @tsc1: first TSC to compare 2777 * @tsc2: second TSC to compare 2778 * 2779 * This function compares 7-byte TSC values allowing for the possibility that 2780 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 2781 * around so for that purpose this function assumes the absolute difference is 2782 * less than half the maximum difference. 2783 * 2784 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 2785 * after @tsc2. 2786 */ 2787 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 2788 { 2789 const uint64_t halfway = (1ULL << 55); 2790 2791 if (tsc1 == tsc2) 2792 return 0; 2793 2794 if (tsc1 < tsc2) { 2795 if (tsc2 - tsc1 < halfway) 2796 return -1; 2797 else 2798 return 1; 2799 } else { 2800 if (tsc1 - tsc2 < halfway) 2801 return 1; 2802 else 2803 return -1; 2804 } 2805 } 2806 2807 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 2808 2809 /** 2810 * adj_for_padding - adjust overlap to account for padding. 2811 * @buf_b: second buffer 2812 * @buf_a: first buffer 2813 * @len_a: size of first buffer 2814 * 2815 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 2816 * accordingly. 2817 * 2818 * Return: A pointer into @buf_b from where non-overlapped data starts 2819 */ 2820 static unsigned char *adj_for_padding(unsigned char *buf_b, 2821 unsigned char *buf_a, size_t len_a) 2822 { 2823 unsigned char *p = buf_b - MAX_PADDING; 2824 unsigned char *q = buf_a + len_a - MAX_PADDING; 2825 int i; 2826 2827 for (i = MAX_PADDING; i; i--, p++, q++) { 2828 if (*p != *q) 2829 break; 2830 } 2831 2832 return p; 2833 } 2834 2835 /** 2836 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 2837 * using TSC. 2838 * @buf_a: first buffer 2839 * @len_a: size of first buffer 2840 * @buf_b: second buffer 2841 * @len_b: size of second buffer 2842 * @consecutive: returns true if there is data in buf_b that is consecutive 2843 * to buf_a 2844 * 2845 * If the trace contains TSC we can look at the last TSC of @buf_a and the 2846 * first TSC of @buf_b in order to determine if the buffers overlap, and then 2847 * walk forward in @buf_b until a later TSC is found. A precondition is that 2848 * @buf_a and @buf_b are positioned at a PSB. 2849 * 2850 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2851 * @buf_b + @len_b if there is no non-overlapped data. 2852 */ 2853 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 2854 size_t len_a, 2855 unsigned char *buf_b, 2856 size_t len_b, bool *consecutive) 2857 { 2858 uint64_t tsc_a, tsc_b; 2859 unsigned char *p; 2860 size_t len, rem_a, rem_b; 2861 2862 p = intel_pt_last_psb(buf_a, len_a); 2863 if (!p) 2864 return buf_b; /* No PSB in buf_a => no overlap */ 2865 2866 len = len_a - (p - buf_a); 2867 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 2868 /* The last PSB+ in buf_a is incomplete, so go back one more */ 2869 len_a -= len; 2870 p = intel_pt_last_psb(buf_a, len_a); 2871 if (!p) 2872 return buf_b; /* No full PSB+ => assume no overlap */ 2873 len = len_a - (p - buf_a); 2874 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 2875 return buf_b; /* No TSC in buf_a => assume no overlap */ 2876 } 2877 2878 while (1) { 2879 /* Ignore PSB+ with no TSC */ 2880 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 2881 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 2882 2883 /* Same TSC, so buffers are consecutive */ 2884 if (!cmp && rem_b >= rem_a) { 2885 unsigned char *start; 2886 2887 *consecutive = true; 2888 start = buf_b + len_b - (rem_b - rem_a); 2889 return adj_for_padding(start, buf_a, len_a); 2890 } 2891 if (cmp < 0) 2892 return buf_b; /* tsc_a < tsc_b => no overlap */ 2893 } 2894 2895 if (!intel_pt_step_psb(&buf_b, &len_b)) 2896 return buf_b + len_b; /* No PSB in buf_b => no data */ 2897 } 2898 } 2899 2900 /** 2901 * intel_pt_find_overlap - determine start of non-overlapped trace data. 2902 * @buf_a: first buffer 2903 * @len_a: size of first buffer 2904 * @buf_b: second buffer 2905 * @len_b: size of second buffer 2906 * @have_tsc: can use TSC packets to detect overlap 2907 * @consecutive: returns true if there is data in buf_b that is consecutive 2908 * to buf_a 2909 * 2910 * When trace samples or snapshots are recorded there is the possibility that 2911 * the data overlaps. Note that, for the purposes of decoding, data is only 2912 * useful if it begins with a PSB packet. 2913 * 2914 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2915 * @buf_b + @len_b if there is no non-overlapped data. 2916 */ 2917 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 2918 unsigned char *buf_b, size_t len_b, 2919 bool have_tsc, bool *consecutive) 2920 { 2921 unsigned char *found; 2922 2923 /* Buffer 'b' must start at PSB so throw away everything before that */ 2924 if (!intel_pt_next_psb(&buf_b, &len_b)) 2925 return buf_b + len_b; /* No PSB */ 2926 2927 if (!intel_pt_next_psb(&buf_a, &len_a)) 2928 return buf_b; /* No overlap */ 2929 2930 if (have_tsc) { 2931 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 2932 consecutive); 2933 if (found) 2934 return found; 2935 } 2936 2937 /* 2938 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 2939 * we can ignore the first part of buffer 'a'. 2940 */ 2941 while (len_b < len_a) { 2942 if (!intel_pt_step_psb(&buf_a, &len_a)) 2943 return buf_b; /* No overlap */ 2944 } 2945 2946 /* Now len_b >= len_a */ 2947 while (1) { 2948 /* Potential overlap so check the bytes */ 2949 found = memmem(buf_a, len_a, buf_b, len_a); 2950 if (found) { 2951 *consecutive = true; 2952 return adj_for_padding(buf_b + len_a, buf_a, len_a); 2953 } 2954 2955 /* Try again at next PSB in buffer 'a' */ 2956 if (!intel_pt_step_psb(&buf_a, &len_a)) 2957 return buf_b; /* No overlap */ 2958 } 2959 } 2960 2961 /** 2962 * struct fast_forward_data - data used by intel_pt_ff_cb(). 2963 * @timestamp: timestamp to fast forward towards 2964 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 2965 * the fast forward timestamp. 2966 */ 2967 struct fast_forward_data { 2968 uint64_t timestamp; 2969 uint64_t buf_timestamp; 2970 }; 2971 2972 /** 2973 * intel_pt_ff_cb - fast forward lookahead callback. 2974 * @buffer: Intel PT trace buffer 2975 * @data: opaque pointer to fast forward data (struct fast_forward_data) 2976 * 2977 * Determine if @buffer trace is past the fast forward timestamp. 2978 * 2979 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 2980 * timestamp, and 0 otherwise. 2981 */ 2982 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 2983 { 2984 struct fast_forward_data *d = data; 2985 unsigned char *buf; 2986 uint64_t tsc; 2987 size_t rem; 2988 size_t len; 2989 2990 buf = (unsigned char *)buffer->buf; 2991 len = buffer->len; 2992 2993 if (!intel_pt_next_psb(&buf, &len) || 2994 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 2995 return 0; 2996 2997 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 2998 2999 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 3000 tsc, buffer->ref_timestamp); 3001 3002 /* 3003 * If the buffer contains a timestamp earlier that the fast forward 3004 * timestamp, then record it, else stop. 3005 */ 3006 if (tsc < d->timestamp) 3007 d->buf_timestamp = buffer->ref_timestamp; 3008 else 3009 return 1; 3010 3011 return 0; 3012 } 3013 3014 /** 3015 * intel_pt_fast_forward - reposition decoder forwards. 3016 * @decoder: Intel PT decoder 3017 * @timestamp: timestamp to fast forward towards 3018 * 3019 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 3020 * 3021 * Return: 0 on success or negative error code on failure. 3022 */ 3023 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 3024 { 3025 struct fast_forward_data d = { .timestamp = timestamp }; 3026 unsigned char *buf; 3027 size_t len; 3028 int err; 3029 3030 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 3031 3032 /* Find buffer timestamp of buffer to fast forward to */ 3033 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 3034 if (err < 0) 3035 return err; 3036 3037 /* Walk to buffer with same buffer timestamp */ 3038 if (d.buf_timestamp) { 3039 do { 3040 decoder->pos += decoder->len; 3041 decoder->len = 0; 3042 err = intel_pt_get_next_data(decoder, true); 3043 /* -ENOLINK means non-consecutive trace */ 3044 if (err && err != -ENOLINK) 3045 return err; 3046 } while (decoder->buf_timestamp != d.buf_timestamp); 3047 } 3048 3049 if (!decoder->buf) 3050 return 0; 3051 3052 buf = (unsigned char *)decoder->buf; 3053 len = decoder->len; 3054 3055 if (!intel_pt_next_psb(&buf, &len)) 3056 return 0; 3057 3058 /* 3059 * Walk PSBs while the PSB timestamp is less than the fast forward 3060 * timestamp. 3061 */ 3062 do { 3063 uint64_t tsc; 3064 size_t rem; 3065 3066 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 3067 break; 3068 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 3069 /* 3070 * A TSC packet can slip past MTC packets but, after fast 3071 * forward, decoding starts at the TSC timestamp. That means 3072 * the timestamps may not be exactly the same as the timestamps 3073 * that would have been decoded without fast forward. 3074 */ 3075 if (tsc < timestamp) { 3076 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 3077 decoder->pos += decoder->len - len; 3078 decoder->buf = buf; 3079 decoder->len = len; 3080 intel_pt_reposition(decoder); 3081 } else { 3082 break; 3083 } 3084 } while (intel_pt_step_psb(&buf, &len)); 3085 3086 return 0; 3087 } 3088