1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define INTEL_PT_RETURN 1 39 40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 41 #define INTEL_PT_MAX_LOOPS 10000 42 43 struct intel_pt_blk { 44 struct intel_pt_blk *prev; 45 uint64_t ip[INTEL_PT_BLK_SIZE]; 46 }; 47 48 struct intel_pt_stack { 49 struct intel_pt_blk *blk; 50 struct intel_pt_blk *spare; 51 int pos; 52 }; 53 54 enum intel_pt_pkt_state { 55 INTEL_PT_STATE_NO_PSB, 56 INTEL_PT_STATE_NO_IP, 57 INTEL_PT_STATE_ERR_RESYNC, 58 INTEL_PT_STATE_IN_SYNC, 59 INTEL_PT_STATE_TNT_CONT, 60 INTEL_PT_STATE_TNT, 61 INTEL_PT_STATE_TIP, 62 INTEL_PT_STATE_TIP_PGD, 63 INTEL_PT_STATE_FUP, 64 INTEL_PT_STATE_FUP_NO_TIP, 65 INTEL_PT_STATE_FUP_IN_PSB, 66 INTEL_PT_STATE_RESAMPLE, 67 }; 68 69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 70 { 71 switch (pkt_state) { 72 case INTEL_PT_STATE_NO_PSB: 73 case INTEL_PT_STATE_NO_IP: 74 case INTEL_PT_STATE_ERR_RESYNC: 75 case INTEL_PT_STATE_IN_SYNC: 76 case INTEL_PT_STATE_TNT_CONT: 77 case INTEL_PT_STATE_RESAMPLE: 78 return true; 79 case INTEL_PT_STATE_TNT: 80 case INTEL_PT_STATE_TIP: 81 case INTEL_PT_STATE_TIP_PGD: 82 case INTEL_PT_STATE_FUP: 83 case INTEL_PT_STATE_FUP_NO_TIP: 84 case INTEL_PT_STATE_FUP_IN_PSB: 85 return false; 86 default: 87 return true; 88 }; 89 } 90 91 #ifdef INTEL_PT_STRICT 92 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 93 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 94 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 95 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 96 #else 97 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 98 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 99 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 100 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 101 #endif 102 103 struct intel_pt_decoder { 104 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 105 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 106 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 107 uint64_t max_insn_cnt, void *data); 108 bool (*pgd_ip)(uint64_t ip, void *data); 109 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 110 void *data; 111 struct intel_pt_state state; 112 const unsigned char *buf; 113 size_t len; 114 bool return_compression; 115 bool branch_enable; 116 bool mtc_insn; 117 bool pge; 118 bool have_tma; 119 bool have_cyc; 120 bool fixup_last_mtc; 121 bool have_last_ip; 122 bool in_psb; 123 bool hop; 124 bool leap; 125 bool nr; 126 bool next_nr; 127 enum intel_pt_param_flags flags; 128 uint64_t pos; 129 uint64_t last_ip; 130 uint64_t ip; 131 uint64_t pip_payload; 132 uint64_t timestamp; 133 uint64_t tsc_timestamp; 134 uint64_t ref_timestamp; 135 uint64_t buf_timestamp; 136 uint64_t sample_timestamp; 137 uint64_t ret_addr; 138 uint64_t ctc_timestamp; 139 uint64_t ctc_delta; 140 uint64_t cycle_cnt; 141 uint64_t cyc_ref_timestamp; 142 uint32_t last_mtc; 143 uint32_t tsc_ctc_ratio_n; 144 uint32_t tsc_ctc_ratio_d; 145 uint32_t tsc_ctc_mult; 146 uint32_t tsc_slip; 147 uint32_t ctc_rem_mask; 148 int mtc_shift; 149 struct intel_pt_stack stack; 150 enum intel_pt_pkt_state pkt_state; 151 enum intel_pt_pkt_ctx pkt_ctx; 152 enum intel_pt_pkt_ctx prev_pkt_ctx; 153 enum intel_pt_blk_type blk_type; 154 int blk_type_pos; 155 struct intel_pt_pkt packet; 156 struct intel_pt_pkt tnt; 157 int pkt_step; 158 int pkt_len; 159 int last_packet_type; 160 unsigned int cbr; 161 unsigned int cbr_seen; 162 unsigned int max_non_turbo_ratio; 163 double max_non_turbo_ratio_fp; 164 double cbr_cyc_to_tsc; 165 double calc_cyc_to_tsc; 166 bool have_calc_cyc_to_tsc; 167 int exec_mode; 168 unsigned int insn_bytes; 169 uint64_t period; 170 enum intel_pt_period_type period_type; 171 uint64_t tot_insn_cnt; 172 uint64_t period_insn_cnt; 173 uint64_t period_mask; 174 uint64_t period_ticks; 175 uint64_t last_masked_timestamp; 176 uint64_t tot_cyc_cnt; 177 uint64_t sample_tot_cyc_cnt; 178 uint64_t base_cyc_cnt; 179 uint64_t cyc_cnt_timestamp; 180 uint64_t ctl; 181 uint64_t cyc_threshold; 182 double tsc_to_cyc; 183 bool continuous_period; 184 bool overflow; 185 bool set_fup_tx_flags; 186 bool set_fup_ptw; 187 bool set_fup_mwait; 188 bool set_fup_pwre; 189 bool set_fup_exstop; 190 bool set_fup_bep; 191 bool sample_cyc; 192 unsigned int fup_tx_flags; 193 unsigned int tx_flags; 194 uint64_t fup_ptw_payload; 195 uint64_t fup_mwait_payload; 196 uint64_t fup_pwre_payload; 197 uint64_t cbr_payload; 198 uint64_t timestamp_insn_cnt; 199 uint64_t sample_insn_cnt; 200 uint64_t stuck_ip; 201 int no_progress; 202 int stuck_ip_prd; 203 int stuck_ip_cnt; 204 uint64_t psb_ip; 205 const unsigned char *next_buf; 206 size_t next_len; 207 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 208 }; 209 210 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 211 { 212 int i; 213 214 for (i = 0; x != 1; i++) 215 x >>= 1; 216 217 return x << i; 218 } 219 220 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 221 { 222 if (!(ctl & INTEL_PT_CYC_ENABLE)) 223 return 0; 224 225 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 226 } 227 228 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 229 { 230 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 231 uint64_t period; 232 233 period = intel_pt_lower_power_of_2(decoder->period); 234 decoder->period_mask = ~(period - 1); 235 decoder->period_ticks = period; 236 } 237 } 238 239 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 240 { 241 if (!d) 242 return 0; 243 return (t / d) * n + ((t % d) * n) / d; 244 } 245 246 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 247 { 248 struct intel_pt_decoder *decoder; 249 250 if (!params->get_trace || !params->walk_insn) 251 return NULL; 252 253 decoder = zalloc(sizeof(struct intel_pt_decoder)); 254 if (!decoder) 255 return NULL; 256 257 decoder->get_trace = params->get_trace; 258 decoder->walk_insn = params->walk_insn; 259 decoder->pgd_ip = params->pgd_ip; 260 decoder->lookahead = params->lookahead; 261 decoder->data = params->data; 262 decoder->return_compression = params->return_compression; 263 decoder->branch_enable = params->branch_enable; 264 decoder->hop = params->quick >= 1; 265 decoder->leap = params->quick >= 2; 266 267 decoder->flags = params->flags; 268 269 decoder->ctl = params->ctl; 270 decoder->period = params->period; 271 decoder->period_type = params->period_type; 272 273 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 274 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 275 276 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 277 278 intel_pt_setup_period(decoder); 279 280 decoder->mtc_shift = params->mtc_period; 281 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 282 283 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 284 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 285 286 if (!decoder->tsc_ctc_ratio_n) 287 decoder->tsc_ctc_ratio_d = 0; 288 289 if (decoder->tsc_ctc_ratio_d) { 290 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 291 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 292 decoder->tsc_ctc_ratio_d; 293 } 294 295 /* 296 * A TSC packet can slip past MTC packets so that the timestamp appears 297 * to go backwards. One estimate is that can be up to about 40 CPU 298 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 299 * slippage an order of magnitude more to be on the safe side. 300 */ 301 decoder->tsc_slip = 0x10000; 302 303 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 304 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 305 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 306 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 307 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 308 309 if (decoder->hop) 310 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 311 312 return decoder; 313 } 314 315 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 316 { 317 struct intel_pt_blk *blk = stack->blk; 318 319 stack->blk = blk->prev; 320 if (!stack->spare) 321 stack->spare = blk; 322 else 323 free(blk); 324 } 325 326 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 327 { 328 if (!stack->pos) { 329 if (!stack->blk) 330 return 0; 331 intel_pt_pop_blk(stack); 332 if (!stack->blk) 333 return 0; 334 stack->pos = INTEL_PT_BLK_SIZE; 335 } 336 return stack->blk->ip[--stack->pos]; 337 } 338 339 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 340 { 341 struct intel_pt_blk *blk; 342 343 if (stack->spare) { 344 blk = stack->spare; 345 stack->spare = NULL; 346 } else { 347 blk = malloc(sizeof(struct intel_pt_blk)); 348 if (!blk) 349 return -ENOMEM; 350 } 351 352 blk->prev = stack->blk; 353 stack->blk = blk; 354 stack->pos = 0; 355 return 0; 356 } 357 358 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 359 { 360 int err; 361 362 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 363 err = intel_pt_alloc_blk(stack); 364 if (err) 365 return err; 366 } 367 368 stack->blk->ip[stack->pos++] = ip; 369 return 0; 370 } 371 372 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 373 { 374 while (stack->blk) 375 intel_pt_pop_blk(stack); 376 stack->pos = 0; 377 } 378 379 static void intel_pt_free_stack(struct intel_pt_stack *stack) 380 { 381 intel_pt_clear_stack(stack); 382 zfree(&stack->blk); 383 zfree(&stack->spare); 384 } 385 386 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 387 { 388 intel_pt_free_stack(&decoder->stack); 389 free(decoder); 390 } 391 392 static int intel_pt_ext_err(int code) 393 { 394 switch (code) { 395 case -ENOMEM: 396 return INTEL_PT_ERR_NOMEM; 397 case -ENOSYS: 398 return INTEL_PT_ERR_INTERN; 399 case -EBADMSG: 400 return INTEL_PT_ERR_BADPKT; 401 case -ENODATA: 402 return INTEL_PT_ERR_NODATA; 403 case -EILSEQ: 404 return INTEL_PT_ERR_NOINSN; 405 case -ENOENT: 406 return INTEL_PT_ERR_MISMAT; 407 case -EOVERFLOW: 408 return INTEL_PT_ERR_OVR; 409 case -ENOSPC: 410 return INTEL_PT_ERR_LOST; 411 case -ELOOP: 412 return INTEL_PT_ERR_NELOOP; 413 default: 414 return INTEL_PT_ERR_UNK; 415 } 416 } 417 418 static const char *intel_pt_err_msgs[] = { 419 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 420 [INTEL_PT_ERR_INTERN] = "Internal error", 421 [INTEL_PT_ERR_BADPKT] = "Bad packet", 422 [INTEL_PT_ERR_NODATA] = "No more data", 423 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 424 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 425 [INTEL_PT_ERR_OVR] = "Overflow packet", 426 [INTEL_PT_ERR_LOST] = "Lost trace data", 427 [INTEL_PT_ERR_UNK] = "Unknown error!", 428 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 429 }; 430 431 int intel_pt__strerror(int code, char *buf, size_t buflen) 432 { 433 if (code < 1 || code >= INTEL_PT_ERR_MAX) 434 code = INTEL_PT_ERR_UNK; 435 strlcpy(buf, intel_pt_err_msgs[code], buflen); 436 return 0; 437 } 438 439 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 440 uint64_t last_ip) 441 { 442 uint64_t ip; 443 444 switch (packet->count) { 445 case 1: 446 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 447 packet->payload; 448 break; 449 case 2: 450 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 451 packet->payload; 452 break; 453 case 3: 454 ip = packet->payload; 455 /* Sign-extend 6-byte ip */ 456 if (ip & (uint64_t)0x800000000000ULL) 457 ip |= (uint64_t)0xffff000000000000ULL; 458 break; 459 case 4: 460 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 461 packet->payload; 462 break; 463 case 6: 464 ip = packet->payload; 465 break; 466 default: 467 return 0; 468 } 469 470 return ip; 471 } 472 473 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 474 { 475 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 476 decoder->have_last_ip = true; 477 } 478 479 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 480 { 481 intel_pt_set_last_ip(decoder); 482 decoder->ip = decoder->last_ip; 483 } 484 485 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 486 { 487 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 488 decoder->buf); 489 } 490 491 static int intel_pt_bug(struct intel_pt_decoder *decoder) 492 { 493 intel_pt_log("ERROR: Internal error\n"); 494 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 495 return -ENOSYS; 496 } 497 498 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 499 { 500 decoder->tx_flags = 0; 501 } 502 503 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 504 { 505 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 506 } 507 508 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 509 { 510 decoder->pip_payload = decoder->packet.payload; 511 } 512 513 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 514 { 515 decoder->next_nr = decoder->pip_payload & 1; 516 } 517 518 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 519 { 520 decoder->nr = decoder->pip_payload & 1; 521 decoder->next_nr = decoder->nr; 522 } 523 524 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 525 { 526 intel_pt_update_pip(decoder); 527 intel_pt_set_nr(decoder); 528 } 529 530 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 531 { 532 intel_pt_clear_tx_flags(decoder); 533 decoder->have_tma = false; 534 decoder->pkt_len = 1; 535 decoder->pkt_step = 1; 536 intel_pt_decoder_log_packet(decoder); 537 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 538 intel_pt_log("ERROR: Bad packet\n"); 539 decoder->pkt_state = INTEL_PT_STATE_ERR1; 540 } 541 return -EBADMSG; 542 } 543 544 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 545 { 546 decoder->sample_timestamp = decoder->timestamp; 547 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 548 } 549 550 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 551 { 552 decoder->ip = 0; 553 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 554 decoder->timestamp = 0; 555 decoder->have_tma = false; 556 } 557 558 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 559 { 560 struct intel_pt_buffer buffer = { .buf = 0, }; 561 int ret; 562 563 decoder->pkt_step = 0; 564 565 intel_pt_log("Getting more data\n"); 566 ret = decoder->get_trace(&buffer, decoder->data); 567 if (ret) 568 return ret; 569 decoder->buf = buffer.buf; 570 decoder->len = buffer.len; 571 if (!decoder->len) { 572 intel_pt_log("No more data\n"); 573 return -ENODATA; 574 } 575 decoder->buf_timestamp = buffer.ref_timestamp; 576 if (!buffer.consecutive || reposition) { 577 intel_pt_reposition(decoder); 578 decoder->ref_timestamp = buffer.ref_timestamp; 579 decoder->state.trace_nr = buffer.trace_nr; 580 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 581 decoder->ref_timestamp); 582 return -ENOLINK; 583 } 584 585 return 0; 586 } 587 588 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 589 bool reposition) 590 { 591 if (!decoder->next_buf) 592 return intel_pt_get_data(decoder, reposition); 593 594 decoder->buf = decoder->next_buf; 595 decoder->len = decoder->next_len; 596 decoder->next_buf = 0; 597 decoder->next_len = 0; 598 return 0; 599 } 600 601 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 602 { 603 unsigned char *buf = decoder->temp_buf; 604 size_t old_len, len, n; 605 int ret; 606 607 old_len = decoder->len; 608 len = decoder->len; 609 memcpy(buf, decoder->buf, len); 610 611 ret = intel_pt_get_data(decoder, false); 612 if (ret) { 613 decoder->pos += old_len; 614 return ret < 0 ? ret : -EINVAL; 615 } 616 617 n = INTEL_PT_PKT_MAX_SZ - len; 618 if (n > decoder->len) 619 n = decoder->len; 620 memcpy(buf + len, decoder->buf, n); 621 len += n; 622 623 decoder->prev_pkt_ctx = decoder->pkt_ctx; 624 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 625 if (ret < (int)old_len) { 626 decoder->next_buf = decoder->buf; 627 decoder->next_len = decoder->len; 628 decoder->buf = buf; 629 decoder->len = old_len; 630 return intel_pt_bad_packet(decoder); 631 } 632 633 decoder->next_buf = decoder->buf + (ret - old_len); 634 decoder->next_len = decoder->len - (ret - old_len); 635 636 decoder->buf = buf; 637 decoder->len = ret; 638 639 return ret; 640 } 641 642 struct intel_pt_pkt_info { 643 struct intel_pt_decoder *decoder; 644 struct intel_pt_pkt packet; 645 uint64_t pos; 646 int pkt_len; 647 int last_packet_type; 648 void *data; 649 }; 650 651 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 652 653 /* Lookahead packets in current buffer */ 654 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 655 intel_pt_pkt_cb_t cb, void *data) 656 { 657 struct intel_pt_pkt_info pkt_info; 658 const unsigned char *buf = decoder->buf; 659 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 660 size_t len = decoder->len; 661 int ret; 662 663 pkt_info.decoder = decoder; 664 pkt_info.pos = decoder->pos; 665 pkt_info.pkt_len = decoder->pkt_step; 666 pkt_info.last_packet_type = decoder->last_packet_type; 667 pkt_info.data = data; 668 669 while (1) { 670 do { 671 pkt_info.pos += pkt_info.pkt_len; 672 buf += pkt_info.pkt_len; 673 len -= pkt_info.pkt_len; 674 675 if (!len) 676 return INTEL_PT_NEED_MORE_BYTES; 677 678 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 679 &pkt_ctx); 680 if (!ret) 681 return INTEL_PT_NEED_MORE_BYTES; 682 if (ret < 0) 683 return ret; 684 685 pkt_info.pkt_len = ret; 686 } while (pkt_info.packet.type == INTEL_PT_PAD); 687 688 ret = cb(&pkt_info); 689 if (ret) 690 return 0; 691 692 pkt_info.last_packet_type = pkt_info.packet.type; 693 } 694 } 695 696 struct intel_pt_calc_cyc_to_tsc_info { 697 uint64_t cycle_cnt; 698 unsigned int cbr; 699 uint32_t last_mtc; 700 uint64_t ctc_timestamp; 701 uint64_t ctc_delta; 702 uint64_t tsc_timestamp; 703 uint64_t timestamp; 704 bool have_tma; 705 bool fixup_last_mtc; 706 bool from_mtc; 707 double cbr_cyc_to_tsc; 708 }; 709 710 /* 711 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 712 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 713 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 714 * packet by copying the missing bits from the current MTC assuming the least 715 * difference between the two, and that the current MTC comes after last_mtc. 716 */ 717 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 718 uint32_t *last_mtc) 719 { 720 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 721 uint32_t mask = ~(first_missing_bit - 1); 722 723 *last_mtc |= mtc & mask; 724 if (*last_mtc >= mtc) { 725 *last_mtc -= first_missing_bit; 726 *last_mtc &= 0xff; 727 } 728 } 729 730 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 731 { 732 struct intel_pt_decoder *decoder = pkt_info->decoder; 733 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 734 uint64_t timestamp; 735 double cyc_to_tsc; 736 unsigned int cbr; 737 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 738 739 switch (pkt_info->packet.type) { 740 case INTEL_PT_TNT: 741 case INTEL_PT_TIP_PGE: 742 case INTEL_PT_TIP: 743 case INTEL_PT_FUP: 744 case INTEL_PT_PSB: 745 case INTEL_PT_PIP: 746 case INTEL_PT_MODE_EXEC: 747 case INTEL_PT_MODE_TSX: 748 case INTEL_PT_PSBEND: 749 case INTEL_PT_PAD: 750 case INTEL_PT_VMCS: 751 case INTEL_PT_MNT: 752 case INTEL_PT_PTWRITE: 753 case INTEL_PT_PTWRITE_IP: 754 case INTEL_PT_BBP: 755 case INTEL_PT_BIP: 756 case INTEL_PT_BEP: 757 case INTEL_PT_BEP_IP: 758 return 0; 759 760 case INTEL_PT_MTC: 761 if (!data->have_tma) 762 return 0; 763 764 mtc = pkt_info->packet.payload; 765 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 766 data->fixup_last_mtc = false; 767 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 768 &data->last_mtc); 769 } 770 if (mtc > data->last_mtc) 771 mtc_delta = mtc - data->last_mtc; 772 else 773 mtc_delta = mtc + 256 - data->last_mtc; 774 data->ctc_delta += mtc_delta << decoder->mtc_shift; 775 data->last_mtc = mtc; 776 777 if (decoder->tsc_ctc_mult) { 778 timestamp = data->ctc_timestamp + 779 data->ctc_delta * decoder->tsc_ctc_mult; 780 } else { 781 timestamp = data->ctc_timestamp + 782 multdiv(data->ctc_delta, 783 decoder->tsc_ctc_ratio_n, 784 decoder->tsc_ctc_ratio_d); 785 } 786 787 if (timestamp < data->timestamp) 788 return 1; 789 790 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 791 data->timestamp = timestamp; 792 return 0; 793 } 794 795 break; 796 797 case INTEL_PT_TSC: 798 /* 799 * For now, do not support using TSC packets - refer 800 * intel_pt_calc_cyc_to_tsc(). 801 */ 802 if (data->from_mtc) 803 return 1; 804 timestamp = pkt_info->packet.payload | 805 (data->timestamp & (0xffULL << 56)); 806 if (data->from_mtc && timestamp < data->timestamp && 807 data->timestamp - timestamp < decoder->tsc_slip) 808 return 1; 809 if (timestamp < data->timestamp) 810 timestamp += (1ULL << 56); 811 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 812 if (data->from_mtc) 813 return 1; 814 data->tsc_timestamp = timestamp; 815 data->timestamp = timestamp; 816 return 0; 817 } 818 break; 819 820 case INTEL_PT_TMA: 821 if (data->from_mtc) 822 return 1; 823 824 if (!decoder->tsc_ctc_ratio_d) 825 return 0; 826 827 ctc = pkt_info->packet.payload; 828 fc = pkt_info->packet.count; 829 ctc_rem = ctc & decoder->ctc_rem_mask; 830 831 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 832 833 data->ctc_timestamp = data->tsc_timestamp - fc; 834 if (decoder->tsc_ctc_mult) { 835 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 836 } else { 837 data->ctc_timestamp -= 838 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 839 decoder->tsc_ctc_ratio_d); 840 } 841 842 data->ctc_delta = 0; 843 data->have_tma = true; 844 data->fixup_last_mtc = true; 845 846 return 0; 847 848 case INTEL_PT_CYC: 849 data->cycle_cnt += pkt_info->packet.payload; 850 return 0; 851 852 case INTEL_PT_CBR: 853 cbr = pkt_info->packet.payload; 854 if (data->cbr && data->cbr != cbr) 855 return 1; 856 data->cbr = cbr; 857 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 858 return 0; 859 860 case INTEL_PT_TIP_PGD: 861 case INTEL_PT_TRACESTOP: 862 case INTEL_PT_EXSTOP: 863 case INTEL_PT_EXSTOP_IP: 864 case INTEL_PT_MWAIT: 865 case INTEL_PT_PWRE: 866 case INTEL_PT_PWRX: 867 case INTEL_PT_OVF: 868 case INTEL_PT_BAD: /* Does not happen */ 869 default: 870 return 1; 871 } 872 873 if (!data->cbr && decoder->cbr) { 874 data->cbr = decoder->cbr; 875 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 876 } 877 878 if (!data->cycle_cnt) 879 return 1; 880 881 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 882 883 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 884 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 885 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 886 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 887 return 1; 888 } 889 890 decoder->calc_cyc_to_tsc = cyc_to_tsc; 891 decoder->have_calc_cyc_to_tsc = true; 892 893 if (data->cbr) { 894 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 895 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 896 } else { 897 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 898 cyc_to_tsc, pkt_info->pos); 899 } 900 901 return 1; 902 } 903 904 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 905 bool from_mtc) 906 { 907 struct intel_pt_calc_cyc_to_tsc_info data = { 908 .cycle_cnt = 0, 909 .cbr = 0, 910 .last_mtc = decoder->last_mtc, 911 .ctc_timestamp = decoder->ctc_timestamp, 912 .ctc_delta = decoder->ctc_delta, 913 .tsc_timestamp = decoder->tsc_timestamp, 914 .timestamp = decoder->timestamp, 915 .have_tma = decoder->have_tma, 916 .fixup_last_mtc = decoder->fixup_last_mtc, 917 .from_mtc = from_mtc, 918 .cbr_cyc_to_tsc = 0, 919 }; 920 921 /* 922 * For now, do not support using TSC packets for at least the reasons: 923 * 1) timing might have stopped 924 * 2) TSC packets within PSB+ can slip against CYC packets 925 */ 926 if (!from_mtc) 927 return; 928 929 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 930 } 931 932 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 933 { 934 int ret; 935 936 decoder->last_packet_type = decoder->packet.type; 937 938 do { 939 decoder->pos += decoder->pkt_step; 940 decoder->buf += decoder->pkt_step; 941 decoder->len -= decoder->pkt_step; 942 943 if (!decoder->len) { 944 ret = intel_pt_get_next_data(decoder, false); 945 if (ret) 946 return ret; 947 } 948 949 decoder->prev_pkt_ctx = decoder->pkt_ctx; 950 ret = intel_pt_get_packet(decoder->buf, decoder->len, 951 &decoder->packet, &decoder->pkt_ctx); 952 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 953 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 954 ret = intel_pt_get_split_packet(decoder); 955 if (ret < 0) 956 return ret; 957 } 958 if (ret <= 0) 959 return intel_pt_bad_packet(decoder); 960 961 decoder->pkt_len = ret; 962 decoder->pkt_step = ret; 963 intel_pt_decoder_log_packet(decoder); 964 } while (decoder->packet.type == INTEL_PT_PAD); 965 966 return 0; 967 } 968 969 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 970 { 971 uint64_t timestamp, masked_timestamp; 972 973 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 974 masked_timestamp = timestamp & decoder->period_mask; 975 if (decoder->continuous_period) { 976 if (masked_timestamp > decoder->last_masked_timestamp) 977 return 1; 978 } else { 979 timestamp += 1; 980 masked_timestamp = timestamp & decoder->period_mask; 981 if (masked_timestamp > decoder->last_masked_timestamp) { 982 decoder->last_masked_timestamp = masked_timestamp; 983 decoder->continuous_period = true; 984 } 985 } 986 987 if (masked_timestamp < decoder->last_masked_timestamp) 988 return decoder->period_ticks; 989 990 return decoder->period_ticks - (timestamp - masked_timestamp); 991 } 992 993 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 994 { 995 switch (decoder->period_type) { 996 case INTEL_PT_PERIOD_INSTRUCTIONS: 997 return decoder->period - decoder->period_insn_cnt; 998 case INTEL_PT_PERIOD_TICKS: 999 return intel_pt_next_period(decoder); 1000 case INTEL_PT_PERIOD_NONE: 1001 case INTEL_PT_PERIOD_MTC: 1002 default: 1003 return 0; 1004 } 1005 } 1006 1007 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1008 { 1009 uint64_t timestamp, masked_timestamp; 1010 1011 switch (decoder->period_type) { 1012 case INTEL_PT_PERIOD_INSTRUCTIONS: 1013 decoder->period_insn_cnt = 0; 1014 break; 1015 case INTEL_PT_PERIOD_TICKS: 1016 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1017 masked_timestamp = timestamp & decoder->period_mask; 1018 if (masked_timestamp > decoder->last_masked_timestamp) 1019 decoder->last_masked_timestamp = masked_timestamp; 1020 else 1021 decoder->last_masked_timestamp += decoder->period_ticks; 1022 break; 1023 case INTEL_PT_PERIOD_NONE: 1024 case INTEL_PT_PERIOD_MTC: 1025 default: 1026 break; 1027 } 1028 1029 decoder->state.type |= INTEL_PT_INSTRUCTION; 1030 } 1031 1032 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1033 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1034 { 1035 uint64_t max_insn_cnt, insn_cnt = 0; 1036 int err; 1037 1038 if (!decoder->mtc_insn) 1039 decoder->mtc_insn = true; 1040 1041 max_insn_cnt = intel_pt_next_sample(decoder); 1042 1043 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1044 max_insn_cnt, decoder->data); 1045 1046 decoder->tot_insn_cnt += insn_cnt; 1047 decoder->timestamp_insn_cnt += insn_cnt; 1048 decoder->sample_insn_cnt += insn_cnt; 1049 decoder->period_insn_cnt += insn_cnt; 1050 1051 if (err) { 1052 decoder->no_progress = 0; 1053 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1054 intel_pt_log_at("ERROR: Failed to get instruction", 1055 decoder->ip); 1056 if (err == -ENOENT) 1057 return -ENOLINK; 1058 return -EILSEQ; 1059 } 1060 1061 if (ip && decoder->ip == ip) { 1062 err = -EAGAIN; 1063 goto out; 1064 } 1065 1066 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1067 intel_pt_sample_insn(decoder); 1068 1069 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1070 decoder->state.type = INTEL_PT_INSTRUCTION; 1071 decoder->state.from_ip = decoder->ip; 1072 decoder->state.to_ip = 0; 1073 decoder->ip += intel_pt_insn->length; 1074 err = INTEL_PT_RETURN; 1075 goto out; 1076 } 1077 1078 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1079 /* Zero-length calls are excluded */ 1080 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1081 intel_pt_insn->rel) { 1082 err = intel_pt_push(&decoder->stack, decoder->ip + 1083 intel_pt_insn->length); 1084 if (err) 1085 goto out; 1086 } 1087 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1088 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1089 } 1090 1091 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1092 int cnt = decoder->no_progress++; 1093 1094 decoder->state.from_ip = decoder->ip; 1095 decoder->ip += intel_pt_insn->length + 1096 intel_pt_insn->rel; 1097 decoder->state.to_ip = decoder->ip; 1098 err = INTEL_PT_RETURN; 1099 1100 /* 1101 * Check for being stuck in a loop. This can happen if a 1102 * decoder error results in the decoder erroneously setting the 1103 * ip to an address that is itself in an infinite loop that 1104 * consumes no packets. When that happens, there must be an 1105 * unconditional branch. 1106 */ 1107 if (cnt) { 1108 if (cnt == 1) { 1109 decoder->stuck_ip = decoder->state.to_ip; 1110 decoder->stuck_ip_prd = 1; 1111 decoder->stuck_ip_cnt = 1; 1112 } else if (cnt > INTEL_PT_MAX_LOOPS || 1113 decoder->state.to_ip == decoder->stuck_ip) { 1114 intel_pt_log_at("ERROR: Never-ending loop", 1115 decoder->state.to_ip); 1116 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1117 err = -ELOOP; 1118 goto out; 1119 } else if (!--decoder->stuck_ip_cnt) { 1120 decoder->stuck_ip_prd += 1; 1121 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1122 decoder->stuck_ip = decoder->state.to_ip; 1123 } 1124 } 1125 goto out_no_progress; 1126 } 1127 out: 1128 decoder->no_progress = 0; 1129 out_no_progress: 1130 decoder->state.insn_op = intel_pt_insn->op; 1131 decoder->state.insn_len = intel_pt_insn->length; 1132 memcpy(decoder->state.insn, intel_pt_insn->buf, 1133 INTEL_PT_INSN_BUF_SZ); 1134 1135 if (decoder->tx_flags & INTEL_PT_IN_TX) 1136 decoder->state.flags |= INTEL_PT_IN_TX; 1137 1138 return err; 1139 } 1140 1141 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1142 { 1143 bool ret = false; 1144 1145 if (decoder->set_fup_tx_flags) { 1146 decoder->set_fup_tx_flags = false; 1147 decoder->tx_flags = decoder->fup_tx_flags; 1148 decoder->state.type = INTEL_PT_TRANSACTION; 1149 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) 1150 decoder->state.type |= INTEL_PT_BRANCH; 1151 decoder->state.from_ip = decoder->ip; 1152 decoder->state.to_ip = 0; 1153 decoder->state.flags = decoder->fup_tx_flags; 1154 return true; 1155 } 1156 if (decoder->set_fup_ptw) { 1157 decoder->set_fup_ptw = false; 1158 decoder->state.type = INTEL_PT_PTW; 1159 decoder->state.flags |= INTEL_PT_FUP_IP; 1160 decoder->state.from_ip = decoder->ip; 1161 decoder->state.to_ip = 0; 1162 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1163 return true; 1164 } 1165 if (decoder->set_fup_mwait) { 1166 decoder->set_fup_mwait = false; 1167 decoder->state.type = INTEL_PT_MWAIT_OP; 1168 decoder->state.from_ip = decoder->ip; 1169 decoder->state.to_ip = 0; 1170 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1171 ret = true; 1172 } 1173 if (decoder->set_fup_pwre) { 1174 decoder->set_fup_pwre = false; 1175 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1176 decoder->state.type &= ~INTEL_PT_BRANCH; 1177 decoder->state.from_ip = decoder->ip; 1178 decoder->state.to_ip = 0; 1179 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1180 ret = true; 1181 } 1182 if (decoder->set_fup_exstop) { 1183 decoder->set_fup_exstop = false; 1184 decoder->state.type |= INTEL_PT_EX_STOP; 1185 decoder->state.type &= ~INTEL_PT_BRANCH; 1186 decoder->state.flags |= INTEL_PT_FUP_IP; 1187 decoder->state.from_ip = decoder->ip; 1188 decoder->state.to_ip = 0; 1189 ret = true; 1190 } 1191 if (decoder->set_fup_bep) { 1192 decoder->set_fup_bep = false; 1193 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1194 decoder->state.type &= ~INTEL_PT_BRANCH; 1195 decoder->state.from_ip = decoder->ip; 1196 decoder->state.to_ip = 0; 1197 ret = true; 1198 } 1199 return ret; 1200 } 1201 1202 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1203 struct intel_pt_insn *intel_pt_insn, 1204 uint64_t ip, int err) 1205 { 1206 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1207 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1208 ip == decoder->ip + intel_pt_insn->length; 1209 } 1210 1211 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1212 { 1213 struct intel_pt_insn intel_pt_insn; 1214 uint64_t ip; 1215 int err; 1216 1217 ip = decoder->last_ip; 1218 1219 while (1) { 1220 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1221 if (err == INTEL_PT_RETURN) 1222 return 0; 1223 if (err == -EAGAIN || 1224 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1225 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; 1226 1227 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1228 if (intel_pt_fup_event(decoder) && no_tip) 1229 return 0; 1230 return -EAGAIN; 1231 } 1232 decoder->set_fup_tx_flags = false; 1233 if (err) 1234 return err; 1235 1236 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1237 intel_pt_log_at("ERROR: Unexpected indirect branch", 1238 decoder->ip); 1239 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1240 return -ENOENT; 1241 } 1242 1243 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1244 intel_pt_log_at("ERROR: Unexpected conditional branch", 1245 decoder->ip); 1246 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1247 return -ENOENT; 1248 } 1249 1250 intel_pt_bug(decoder); 1251 } 1252 } 1253 1254 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1255 { 1256 struct intel_pt_insn intel_pt_insn; 1257 int err; 1258 1259 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1260 if (err == INTEL_PT_RETURN && 1261 decoder->pgd_ip && 1262 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1263 (decoder->state.type & INTEL_PT_BRANCH) && 1264 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1265 /* Unconditional branch leaving filter region */ 1266 decoder->no_progress = 0; 1267 decoder->pge = false; 1268 decoder->continuous_period = false; 1269 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1270 decoder->state.type |= INTEL_PT_TRACE_END; 1271 intel_pt_update_nr(decoder); 1272 return 0; 1273 } 1274 if (err == INTEL_PT_RETURN) 1275 return 0; 1276 if (err) 1277 return err; 1278 1279 intel_pt_update_nr(decoder); 1280 1281 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1282 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1283 decoder->pge = false; 1284 decoder->continuous_period = false; 1285 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1286 decoder->state.from_ip = decoder->ip; 1287 if (decoder->packet.count == 0) { 1288 decoder->state.to_ip = 0; 1289 } else { 1290 decoder->state.to_ip = decoder->last_ip; 1291 decoder->ip = decoder->last_ip; 1292 } 1293 decoder->state.type |= INTEL_PT_TRACE_END; 1294 } else { 1295 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1296 decoder->state.from_ip = decoder->ip; 1297 if (decoder->packet.count == 0) { 1298 decoder->state.to_ip = 0; 1299 } else { 1300 decoder->state.to_ip = decoder->last_ip; 1301 decoder->ip = decoder->last_ip; 1302 } 1303 } 1304 return 0; 1305 } 1306 1307 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1308 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1309 intel_pt_insn.rel; 1310 1311 if (decoder->pgd_ip && 1312 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1313 decoder->pgd_ip(to_ip, decoder->data)) { 1314 /* Conditional branch leaving filter region */ 1315 decoder->pge = false; 1316 decoder->continuous_period = false; 1317 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1318 decoder->ip = to_ip; 1319 decoder->state.from_ip = decoder->ip; 1320 decoder->state.to_ip = to_ip; 1321 decoder->state.type |= INTEL_PT_TRACE_END; 1322 return 0; 1323 } 1324 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1325 decoder->ip); 1326 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1327 return -ENOENT; 1328 } 1329 1330 return intel_pt_bug(decoder); 1331 } 1332 1333 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1334 { 1335 struct intel_pt_insn intel_pt_insn; 1336 int err; 1337 1338 while (1) { 1339 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1340 if (err == INTEL_PT_RETURN) 1341 return 0; 1342 if (err) 1343 return err; 1344 1345 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1346 if (!decoder->return_compression) { 1347 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1348 decoder->ip); 1349 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1350 return -ENOENT; 1351 } 1352 if (!decoder->ret_addr) { 1353 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1354 decoder->ip); 1355 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1356 return -ENOENT; 1357 } 1358 if (!(decoder->tnt.payload & BIT63)) { 1359 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1360 decoder->ip); 1361 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1362 return -ENOENT; 1363 } 1364 decoder->tnt.count -= 1; 1365 if (decoder->tnt.count) 1366 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1367 else 1368 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1369 decoder->tnt.payload <<= 1; 1370 decoder->state.from_ip = decoder->ip; 1371 decoder->ip = decoder->ret_addr; 1372 decoder->state.to_ip = decoder->ip; 1373 return 0; 1374 } 1375 1376 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1377 /* Handle deferred TIPs */ 1378 err = intel_pt_get_next_packet(decoder); 1379 if (err) 1380 return err; 1381 if (decoder->packet.type != INTEL_PT_TIP || 1382 decoder->packet.count == 0) { 1383 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1384 decoder->ip); 1385 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1386 decoder->pkt_step = 0; 1387 return -ENOENT; 1388 } 1389 intel_pt_set_last_ip(decoder); 1390 decoder->state.from_ip = decoder->ip; 1391 decoder->state.to_ip = decoder->last_ip; 1392 decoder->ip = decoder->last_ip; 1393 intel_pt_update_nr(decoder); 1394 return 0; 1395 } 1396 1397 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1398 decoder->tnt.count -= 1; 1399 if (decoder->tnt.count) 1400 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1401 else 1402 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1403 if (decoder->tnt.payload & BIT63) { 1404 decoder->tnt.payload <<= 1; 1405 decoder->state.from_ip = decoder->ip; 1406 decoder->ip += intel_pt_insn.length + 1407 intel_pt_insn.rel; 1408 decoder->state.to_ip = decoder->ip; 1409 return 0; 1410 } 1411 /* Instruction sample for a non-taken branch */ 1412 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1413 decoder->tnt.payload <<= 1; 1414 decoder->state.type = INTEL_PT_INSTRUCTION; 1415 decoder->state.from_ip = decoder->ip; 1416 decoder->state.to_ip = 0; 1417 decoder->ip += intel_pt_insn.length; 1418 return 0; 1419 } 1420 decoder->sample_cyc = false; 1421 decoder->ip += intel_pt_insn.length; 1422 if (!decoder->tnt.count) { 1423 intel_pt_update_sample_time(decoder); 1424 return -EAGAIN; 1425 } 1426 decoder->tnt.payload <<= 1; 1427 continue; 1428 } 1429 1430 return intel_pt_bug(decoder); 1431 } 1432 } 1433 1434 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1435 { 1436 unsigned int fup_tx_flags; 1437 int err; 1438 1439 fup_tx_flags = decoder->packet.payload & 1440 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1441 err = intel_pt_get_next_packet(decoder); 1442 if (err) 1443 return err; 1444 if (decoder->packet.type == INTEL_PT_FUP) { 1445 decoder->fup_tx_flags = fup_tx_flags; 1446 decoder->set_fup_tx_flags = true; 1447 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1448 *no_tip = true; 1449 } else { 1450 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1451 decoder->pos); 1452 intel_pt_update_in_tx(decoder); 1453 } 1454 return 0; 1455 } 1456 1457 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1458 { 1459 timestamp |= (ref_timestamp & (0xffULL << 56)); 1460 1461 if (timestamp < ref_timestamp) { 1462 if (ref_timestamp - timestamp > (1ULL << 55)) 1463 timestamp += (1ULL << 56); 1464 } else { 1465 if (timestamp - ref_timestamp > (1ULL << 55)) 1466 timestamp -= (1ULL << 56); 1467 } 1468 1469 return timestamp; 1470 } 1471 1472 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1473 { 1474 uint64_t timestamp; 1475 1476 decoder->have_tma = false; 1477 1478 if (decoder->ref_timestamp) { 1479 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1480 decoder->ref_timestamp); 1481 decoder->tsc_timestamp = timestamp; 1482 decoder->timestamp = timestamp; 1483 decoder->ref_timestamp = 0; 1484 decoder->timestamp_insn_cnt = 0; 1485 } else if (decoder->timestamp) { 1486 timestamp = decoder->packet.payload | 1487 (decoder->timestamp & (0xffULL << 56)); 1488 decoder->tsc_timestamp = timestamp; 1489 if (timestamp < decoder->timestamp && 1490 decoder->timestamp - timestamp < decoder->tsc_slip) { 1491 intel_pt_log_to("Suppressing backwards timestamp", 1492 timestamp); 1493 timestamp = decoder->timestamp; 1494 } 1495 if (timestamp < decoder->timestamp) { 1496 intel_pt_log_to("Wraparound timestamp", timestamp); 1497 timestamp += (1ULL << 56); 1498 decoder->tsc_timestamp = timestamp; 1499 } 1500 decoder->timestamp = timestamp; 1501 decoder->timestamp_insn_cnt = 0; 1502 } 1503 1504 if (decoder->last_packet_type == INTEL_PT_CYC) { 1505 decoder->cyc_ref_timestamp = decoder->timestamp; 1506 decoder->cycle_cnt = 0; 1507 decoder->have_calc_cyc_to_tsc = false; 1508 intel_pt_calc_cyc_to_tsc(decoder, false); 1509 } 1510 1511 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1512 } 1513 1514 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1515 { 1516 intel_pt_log("ERROR: Buffer overflow\n"); 1517 intel_pt_clear_tx_flags(decoder); 1518 intel_pt_set_nr(decoder); 1519 decoder->timestamp_insn_cnt = 0; 1520 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1521 decoder->overflow = true; 1522 return -EOVERFLOW; 1523 } 1524 1525 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1526 { 1527 if (decoder->have_cyc) 1528 return; 1529 1530 decoder->cyc_cnt_timestamp = decoder->timestamp; 1531 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1532 } 1533 1534 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1535 { 1536 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1537 1538 if (decoder->pge) 1539 intel_pt_mtc_cyc_cnt_pge(decoder); 1540 } 1541 1542 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1543 { 1544 uint64_t tot_cyc_cnt, tsc_delta; 1545 1546 if (decoder->have_cyc) 1547 return; 1548 1549 decoder->sample_cyc = true; 1550 1551 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1552 return; 1553 1554 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1555 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1556 1557 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1558 decoder->tot_cyc_cnt = tot_cyc_cnt; 1559 } 1560 1561 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1562 { 1563 uint32_t ctc = decoder->packet.payload; 1564 uint32_t fc = decoder->packet.count; 1565 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1566 1567 if (!decoder->tsc_ctc_ratio_d) 1568 return; 1569 1570 if (decoder->pge && !decoder->in_psb) 1571 intel_pt_mtc_cyc_cnt_pge(decoder); 1572 else 1573 intel_pt_mtc_cyc_cnt_upd(decoder); 1574 1575 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1576 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1577 if (decoder->tsc_ctc_mult) { 1578 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1579 } else { 1580 decoder->ctc_timestamp -= multdiv(ctc_rem, 1581 decoder->tsc_ctc_ratio_n, 1582 decoder->tsc_ctc_ratio_d); 1583 } 1584 decoder->ctc_delta = 0; 1585 decoder->have_tma = true; 1586 decoder->fixup_last_mtc = true; 1587 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1588 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1589 } 1590 1591 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1592 { 1593 uint64_t timestamp; 1594 uint32_t mtc, mtc_delta; 1595 1596 if (!decoder->have_tma) 1597 return; 1598 1599 mtc = decoder->packet.payload; 1600 1601 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1602 decoder->fixup_last_mtc = false; 1603 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1604 &decoder->last_mtc); 1605 } 1606 1607 if (mtc > decoder->last_mtc) 1608 mtc_delta = mtc - decoder->last_mtc; 1609 else 1610 mtc_delta = mtc + 256 - decoder->last_mtc; 1611 1612 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1613 1614 if (decoder->tsc_ctc_mult) { 1615 timestamp = decoder->ctc_timestamp + 1616 decoder->ctc_delta * decoder->tsc_ctc_mult; 1617 } else { 1618 timestamp = decoder->ctc_timestamp + 1619 multdiv(decoder->ctc_delta, 1620 decoder->tsc_ctc_ratio_n, 1621 decoder->tsc_ctc_ratio_d); 1622 } 1623 1624 if (timestamp < decoder->timestamp) 1625 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1626 timestamp, decoder->timestamp); 1627 else 1628 decoder->timestamp = timestamp; 1629 1630 intel_pt_mtc_cyc_cnt_upd(decoder); 1631 1632 decoder->timestamp_insn_cnt = 0; 1633 decoder->last_mtc = mtc; 1634 1635 if (decoder->last_packet_type == INTEL_PT_CYC) { 1636 decoder->cyc_ref_timestamp = decoder->timestamp; 1637 decoder->cycle_cnt = 0; 1638 decoder->have_calc_cyc_to_tsc = false; 1639 intel_pt_calc_cyc_to_tsc(decoder, true); 1640 } 1641 1642 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1643 } 1644 1645 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1646 { 1647 unsigned int cbr = decoder->packet.payload & 0xff; 1648 1649 decoder->cbr_payload = decoder->packet.payload; 1650 1651 if (decoder->cbr == cbr) 1652 return; 1653 1654 decoder->cbr = cbr; 1655 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1656 1657 intel_pt_mtc_cyc_cnt_cbr(decoder); 1658 } 1659 1660 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1661 { 1662 uint64_t timestamp = decoder->cyc_ref_timestamp; 1663 1664 decoder->have_cyc = true; 1665 1666 decoder->cycle_cnt += decoder->packet.payload; 1667 if (decoder->pge) 1668 decoder->tot_cyc_cnt += decoder->packet.payload; 1669 decoder->sample_cyc = true; 1670 1671 if (!decoder->cyc_ref_timestamp) 1672 return; 1673 1674 if (decoder->have_calc_cyc_to_tsc) 1675 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1676 else if (decoder->cbr) 1677 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1678 else 1679 return; 1680 1681 if (timestamp < decoder->timestamp) 1682 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1683 timestamp, decoder->timestamp); 1684 else 1685 decoder->timestamp = timestamp; 1686 1687 decoder->timestamp_insn_cnt = 0; 1688 1689 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1690 } 1691 1692 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1693 { 1694 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1695 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1696 decoder->state.items.is_32_bit = false; 1697 } 1698 decoder->blk_type = decoder->packet.payload; 1699 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1700 if (decoder->blk_type == INTEL_PT_GP_REGS) 1701 decoder->state.items.is_32_bit = decoder->packet.count; 1702 if (decoder->blk_type_pos < 0) { 1703 intel_pt_log("WARNING: Unknown block type %u\n", 1704 decoder->blk_type); 1705 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1706 intel_pt_log("WARNING: Duplicate block type %u\n", 1707 decoder->blk_type); 1708 } 1709 } 1710 1711 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1712 { 1713 uint32_t id = decoder->packet.count; 1714 uint32_t bit = 1 << id; 1715 int pos = decoder->blk_type_pos; 1716 1717 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1718 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1719 id, decoder->blk_type); 1720 return; 1721 } 1722 1723 if (decoder->state.items.mask[pos] & bit) { 1724 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1725 id, decoder->blk_type); 1726 } 1727 1728 decoder->state.items.mask[pos] |= bit; 1729 decoder->state.items.val[pos][id] = decoder->packet.payload; 1730 } 1731 1732 /* Walk PSB+ packets when already in sync. */ 1733 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1734 { 1735 int err; 1736 1737 decoder->in_psb = true; 1738 1739 while (1) { 1740 err = intel_pt_get_next_packet(decoder); 1741 if (err) 1742 goto out; 1743 1744 switch (decoder->packet.type) { 1745 case INTEL_PT_PSBEND: 1746 err = 0; 1747 goto out; 1748 1749 case INTEL_PT_TIP_PGD: 1750 case INTEL_PT_TIP_PGE: 1751 case INTEL_PT_TIP: 1752 case INTEL_PT_TNT: 1753 case INTEL_PT_TRACESTOP: 1754 case INTEL_PT_BAD: 1755 case INTEL_PT_PSB: 1756 case INTEL_PT_PTWRITE: 1757 case INTEL_PT_PTWRITE_IP: 1758 case INTEL_PT_EXSTOP: 1759 case INTEL_PT_EXSTOP_IP: 1760 case INTEL_PT_MWAIT: 1761 case INTEL_PT_PWRE: 1762 case INTEL_PT_PWRX: 1763 case INTEL_PT_BBP: 1764 case INTEL_PT_BIP: 1765 case INTEL_PT_BEP: 1766 case INTEL_PT_BEP_IP: 1767 decoder->have_tma = false; 1768 intel_pt_log("ERROR: Unexpected packet\n"); 1769 err = -EAGAIN; 1770 goto out; 1771 1772 case INTEL_PT_OVF: 1773 err = intel_pt_overflow(decoder); 1774 goto out; 1775 1776 case INTEL_PT_TSC: 1777 intel_pt_calc_tsc_timestamp(decoder); 1778 break; 1779 1780 case INTEL_PT_TMA: 1781 intel_pt_calc_tma(decoder); 1782 break; 1783 1784 case INTEL_PT_CBR: 1785 intel_pt_calc_cbr(decoder); 1786 break; 1787 1788 case INTEL_PT_MODE_EXEC: 1789 decoder->exec_mode = decoder->packet.payload; 1790 break; 1791 1792 case INTEL_PT_PIP: 1793 intel_pt_set_pip(decoder); 1794 break; 1795 1796 case INTEL_PT_FUP: 1797 decoder->pge = true; 1798 if (decoder->packet.count) { 1799 intel_pt_set_last_ip(decoder); 1800 decoder->psb_ip = decoder->last_ip; 1801 } 1802 break; 1803 1804 case INTEL_PT_MODE_TSX: 1805 intel_pt_update_in_tx(decoder); 1806 break; 1807 1808 case INTEL_PT_MTC: 1809 intel_pt_calc_mtc_timestamp(decoder); 1810 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1811 decoder->state.type |= INTEL_PT_INSTRUCTION; 1812 break; 1813 1814 case INTEL_PT_CYC: 1815 intel_pt_calc_cyc_timestamp(decoder); 1816 break; 1817 1818 case INTEL_PT_VMCS: 1819 case INTEL_PT_MNT: 1820 case INTEL_PT_PAD: 1821 default: 1822 break; 1823 } 1824 } 1825 out: 1826 decoder->in_psb = false; 1827 1828 return err; 1829 } 1830 1831 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1832 { 1833 int err; 1834 1835 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1836 decoder->tx_flags = 0; 1837 decoder->state.flags &= ~INTEL_PT_IN_TX; 1838 decoder->state.flags |= INTEL_PT_ABORT_TX; 1839 } else { 1840 decoder->state.flags |= INTEL_PT_ASYNC; 1841 } 1842 1843 while (1) { 1844 err = intel_pt_get_next_packet(decoder); 1845 if (err) 1846 return err; 1847 1848 switch (decoder->packet.type) { 1849 case INTEL_PT_TNT: 1850 case INTEL_PT_FUP: 1851 case INTEL_PT_TRACESTOP: 1852 case INTEL_PT_PSB: 1853 case INTEL_PT_TSC: 1854 case INTEL_PT_TMA: 1855 case INTEL_PT_MODE_TSX: 1856 case INTEL_PT_BAD: 1857 case INTEL_PT_PSBEND: 1858 case INTEL_PT_PTWRITE: 1859 case INTEL_PT_PTWRITE_IP: 1860 case INTEL_PT_EXSTOP: 1861 case INTEL_PT_EXSTOP_IP: 1862 case INTEL_PT_MWAIT: 1863 case INTEL_PT_PWRE: 1864 case INTEL_PT_PWRX: 1865 case INTEL_PT_BBP: 1866 case INTEL_PT_BIP: 1867 case INTEL_PT_BEP: 1868 case INTEL_PT_BEP_IP: 1869 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1870 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1871 decoder->pkt_step = 0; 1872 return -ENOENT; 1873 1874 case INTEL_PT_CBR: 1875 intel_pt_calc_cbr(decoder); 1876 break; 1877 1878 case INTEL_PT_OVF: 1879 return intel_pt_overflow(decoder); 1880 1881 case INTEL_PT_TIP_PGD: 1882 decoder->state.from_ip = decoder->ip; 1883 if (decoder->packet.count == 0) { 1884 decoder->state.to_ip = 0; 1885 } else { 1886 intel_pt_set_ip(decoder); 1887 decoder->state.to_ip = decoder->ip; 1888 } 1889 decoder->pge = false; 1890 decoder->continuous_period = false; 1891 decoder->state.type |= INTEL_PT_TRACE_END; 1892 intel_pt_update_nr(decoder); 1893 return 0; 1894 1895 case INTEL_PT_TIP_PGE: 1896 decoder->pge = true; 1897 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1898 decoder->ip); 1899 decoder->state.from_ip = 0; 1900 if (decoder->packet.count == 0) { 1901 decoder->state.to_ip = 0; 1902 } else { 1903 intel_pt_set_ip(decoder); 1904 decoder->state.to_ip = decoder->ip; 1905 } 1906 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1907 intel_pt_mtc_cyc_cnt_pge(decoder); 1908 intel_pt_set_nr(decoder); 1909 return 0; 1910 1911 case INTEL_PT_TIP: 1912 decoder->state.from_ip = decoder->ip; 1913 if (decoder->packet.count == 0) { 1914 decoder->state.to_ip = 0; 1915 } else { 1916 intel_pt_set_ip(decoder); 1917 decoder->state.to_ip = decoder->ip; 1918 } 1919 intel_pt_update_nr(decoder); 1920 return 0; 1921 1922 case INTEL_PT_PIP: 1923 intel_pt_update_pip(decoder); 1924 break; 1925 1926 case INTEL_PT_MTC: 1927 intel_pt_calc_mtc_timestamp(decoder); 1928 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1929 decoder->state.type |= INTEL_PT_INSTRUCTION; 1930 break; 1931 1932 case INTEL_PT_CYC: 1933 intel_pt_calc_cyc_timestamp(decoder); 1934 break; 1935 1936 case INTEL_PT_MODE_EXEC: 1937 decoder->exec_mode = decoder->packet.payload; 1938 break; 1939 1940 case INTEL_PT_VMCS: 1941 case INTEL_PT_MNT: 1942 case INTEL_PT_PAD: 1943 break; 1944 1945 default: 1946 return intel_pt_bug(decoder); 1947 } 1948 } 1949 } 1950 1951 static int intel_pt_resample(struct intel_pt_decoder *decoder) 1952 { 1953 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1954 decoder->state.type = INTEL_PT_INSTRUCTION; 1955 decoder->state.from_ip = decoder->ip; 1956 decoder->state.to_ip = 0; 1957 return 0; 1958 } 1959 1960 #define HOP_PROCESS 0 1961 #define HOP_IGNORE 1 1962 #define HOP_RETURN 2 1963 #define HOP_AGAIN 3 1964 1965 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 1966 1967 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 1968 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 1969 { 1970 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 1971 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 1972 *err = intel_pt_scan_for_psb(decoder); 1973 if (*err) 1974 return HOP_RETURN; 1975 } 1976 1977 switch (decoder->packet.type) { 1978 case INTEL_PT_TNT: 1979 return HOP_IGNORE; 1980 1981 case INTEL_PT_TIP_PGD: 1982 if (!decoder->packet.count) { 1983 intel_pt_set_nr(decoder); 1984 return HOP_IGNORE; 1985 } 1986 intel_pt_set_ip(decoder); 1987 decoder->state.type |= INTEL_PT_TRACE_END; 1988 decoder->state.from_ip = 0; 1989 decoder->state.to_ip = decoder->ip; 1990 intel_pt_update_nr(decoder); 1991 return HOP_RETURN; 1992 1993 case INTEL_PT_TIP: 1994 if (!decoder->packet.count) { 1995 intel_pt_set_nr(decoder); 1996 return HOP_IGNORE; 1997 } 1998 intel_pt_set_ip(decoder); 1999 decoder->state.type = INTEL_PT_INSTRUCTION; 2000 decoder->state.from_ip = decoder->ip; 2001 decoder->state.to_ip = 0; 2002 intel_pt_update_nr(decoder); 2003 return HOP_RETURN; 2004 2005 case INTEL_PT_FUP: 2006 if (!decoder->packet.count) 2007 return HOP_IGNORE; 2008 intel_pt_set_ip(decoder); 2009 if (intel_pt_fup_event(decoder)) 2010 return HOP_RETURN; 2011 if (!decoder->branch_enable) 2012 *no_tip = true; 2013 if (*no_tip) { 2014 decoder->state.type = INTEL_PT_INSTRUCTION; 2015 decoder->state.from_ip = decoder->ip; 2016 decoder->state.to_ip = 0; 2017 return HOP_RETURN; 2018 } 2019 *err = intel_pt_walk_fup_tip(decoder); 2020 if (!*err) 2021 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2022 return HOP_RETURN; 2023 2024 case INTEL_PT_PSB: 2025 decoder->state.psb_offset = decoder->pos; 2026 decoder->psb_ip = 0; 2027 decoder->last_ip = 0; 2028 decoder->have_last_ip = true; 2029 *err = intel_pt_walk_psbend(decoder); 2030 if (*err == -EAGAIN) 2031 return HOP_AGAIN; 2032 if (*err) 2033 return HOP_RETURN; 2034 decoder->state.type = INTEL_PT_PSB_EVT; 2035 if (decoder->psb_ip) { 2036 decoder->state.type |= INTEL_PT_INSTRUCTION; 2037 decoder->ip = decoder->psb_ip; 2038 } 2039 decoder->state.from_ip = decoder->psb_ip; 2040 decoder->state.to_ip = 0; 2041 return HOP_RETURN; 2042 2043 case INTEL_PT_BAD: 2044 case INTEL_PT_PAD: 2045 case INTEL_PT_TIP_PGE: 2046 case INTEL_PT_TSC: 2047 case INTEL_PT_TMA: 2048 case INTEL_PT_MODE_EXEC: 2049 case INTEL_PT_MODE_TSX: 2050 case INTEL_PT_MTC: 2051 case INTEL_PT_CYC: 2052 case INTEL_PT_VMCS: 2053 case INTEL_PT_PSBEND: 2054 case INTEL_PT_CBR: 2055 case INTEL_PT_TRACESTOP: 2056 case INTEL_PT_PIP: 2057 case INTEL_PT_OVF: 2058 case INTEL_PT_MNT: 2059 case INTEL_PT_PTWRITE: 2060 case INTEL_PT_PTWRITE_IP: 2061 case INTEL_PT_EXSTOP: 2062 case INTEL_PT_EXSTOP_IP: 2063 case INTEL_PT_MWAIT: 2064 case INTEL_PT_PWRE: 2065 case INTEL_PT_PWRX: 2066 case INTEL_PT_BBP: 2067 case INTEL_PT_BIP: 2068 case INTEL_PT_BEP: 2069 case INTEL_PT_BEP_IP: 2070 default: 2071 return HOP_PROCESS; 2072 } 2073 } 2074 2075 struct intel_pt_psb_info { 2076 struct intel_pt_pkt fup_packet; 2077 bool fup; 2078 int after_psbend; 2079 }; 2080 2081 /* Lookahead and get the FUP packet from PSB+ */ 2082 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2083 { 2084 struct intel_pt_psb_info *data = pkt_info->data; 2085 2086 switch (pkt_info->packet.type) { 2087 case INTEL_PT_PAD: 2088 case INTEL_PT_MNT: 2089 case INTEL_PT_TSC: 2090 case INTEL_PT_TMA: 2091 case INTEL_PT_MODE_EXEC: 2092 case INTEL_PT_MODE_TSX: 2093 case INTEL_PT_MTC: 2094 case INTEL_PT_CYC: 2095 case INTEL_PT_VMCS: 2096 case INTEL_PT_CBR: 2097 case INTEL_PT_PIP: 2098 if (data->after_psbend) { 2099 data->after_psbend -= 1; 2100 if (!data->after_psbend) 2101 return 1; 2102 } 2103 break; 2104 2105 case INTEL_PT_FUP: 2106 if (data->after_psbend) 2107 return 1; 2108 if (data->fup || pkt_info->packet.count == 0) 2109 return 1; 2110 data->fup_packet = pkt_info->packet; 2111 data->fup = true; 2112 break; 2113 2114 case INTEL_PT_PSBEND: 2115 if (!data->fup) 2116 return 1; 2117 /* Keep going to check for a TIP.PGE */ 2118 data->after_psbend = 6; 2119 break; 2120 2121 case INTEL_PT_TIP_PGE: 2122 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 2123 if (data->after_psbend) 2124 data->fup = false; 2125 return 1; 2126 2127 case INTEL_PT_PTWRITE: 2128 case INTEL_PT_PTWRITE_IP: 2129 case INTEL_PT_EXSTOP: 2130 case INTEL_PT_EXSTOP_IP: 2131 case INTEL_PT_MWAIT: 2132 case INTEL_PT_PWRE: 2133 case INTEL_PT_PWRX: 2134 case INTEL_PT_BBP: 2135 case INTEL_PT_BIP: 2136 case INTEL_PT_BEP: 2137 case INTEL_PT_BEP_IP: 2138 if (data->after_psbend) { 2139 data->after_psbend -= 1; 2140 if (!data->after_psbend) 2141 return 1; 2142 break; 2143 } 2144 return 1; 2145 2146 case INTEL_PT_OVF: 2147 case INTEL_PT_BAD: 2148 case INTEL_PT_TNT: 2149 case INTEL_PT_TIP_PGD: 2150 case INTEL_PT_TIP: 2151 case INTEL_PT_PSB: 2152 case INTEL_PT_TRACESTOP: 2153 default: 2154 return 1; 2155 } 2156 2157 return 0; 2158 } 2159 2160 static int intel_pt_psb(struct intel_pt_decoder *decoder) 2161 { 2162 int err; 2163 2164 decoder->last_ip = 0; 2165 decoder->psb_ip = 0; 2166 decoder->have_last_ip = true; 2167 intel_pt_clear_stack(&decoder->stack); 2168 err = intel_pt_walk_psbend(decoder); 2169 if (err) 2170 return err; 2171 decoder->state.type = INTEL_PT_PSB_EVT; 2172 decoder->state.from_ip = decoder->psb_ip; 2173 decoder->state.to_ip = 0; 2174 return 0; 2175 } 2176 2177 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 2178 { 2179 int err; 2180 2181 if (decoder->ip != decoder->last_ip) { 2182 err = intel_pt_walk_fup(decoder); 2183 if (!err || err != -EAGAIN) 2184 return err; 2185 } 2186 2187 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2188 err = intel_pt_psb(decoder); 2189 if (err) { 2190 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2191 return -ENOENT; 2192 } 2193 2194 return 0; 2195 } 2196 2197 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 2198 { 2199 struct intel_pt_psb_info data = { .fup = false }; 2200 2201 if (!decoder->branch_enable || !decoder->pge) 2202 return false; 2203 2204 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 2205 if (!data.fup) 2206 return false; 2207 2208 decoder->packet = data.fup_packet; 2209 intel_pt_set_last_ip(decoder); 2210 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 2211 2212 *err = intel_pt_fup_in_psb(decoder); 2213 2214 return true; 2215 } 2216 2217 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2218 { 2219 int last_packet_type = INTEL_PT_PAD; 2220 bool no_tip = false; 2221 int err; 2222 2223 while (1) { 2224 err = intel_pt_get_next_packet(decoder); 2225 if (err) 2226 return err; 2227 next: 2228 if (decoder->cyc_threshold) { 2229 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 2230 decoder->sample_cyc = false; 2231 last_packet_type = decoder->packet.type; 2232 } 2233 2234 if (decoder->hop) { 2235 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2236 case HOP_IGNORE: 2237 continue; 2238 case HOP_RETURN: 2239 return err; 2240 case HOP_AGAIN: 2241 goto next; 2242 default: 2243 break; 2244 } 2245 } 2246 2247 switch (decoder->packet.type) { 2248 case INTEL_PT_TNT: 2249 if (!decoder->packet.count) 2250 break; 2251 decoder->tnt = decoder->packet; 2252 decoder->pkt_state = INTEL_PT_STATE_TNT; 2253 err = intel_pt_walk_tnt(decoder); 2254 if (err == -EAGAIN) 2255 break; 2256 return err; 2257 2258 case INTEL_PT_TIP_PGD: 2259 if (decoder->packet.count != 0) 2260 intel_pt_set_last_ip(decoder); 2261 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2262 return intel_pt_walk_tip(decoder); 2263 2264 case INTEL_PT_TIP_PGE: { 2265 decoder->pge = true; 2266 intel_pt_mtc_cyc_cnt_pge(decoder); 2267 intel_pt_set_nr(decoder); 2268 if (decoder->packet.count == 0) { 2269 intel_pt_log_at("Skipping zero TIP.PGE", 2270 decoder->pos); 2271 break; 2272 } 2273 intel_pt_set_ip(decoder); 2274 decoder->state.from_ip = 0; 2275 decoder->state.to_ip = decoder->ip; 2276 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2277 /* 2278 * In hop mode, resample to get the to_ip as an 2279 * "instruction" sample. 2280 */ 2281 if (decoder->hop) 2282 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2283 return 0; 2284 } 2285 2286 case INTEL_PT_OVF: 2287 return intel_pt_overflow(decoder); 2288 2289 case INTEL_PT_TIP: 2290 if (decoder->packet.count != 0) 2291 intel_pt_set_last_ip(decoder); 2292 decoder->pkt_state = INTEL_PT_STATE_TIP; 2293 return intel_pt_walk_tip(decoder); 2294 2295 case INTEL_PT_FUP: 2296 if (decoder->packet.count == 0) { 2297 intel_pt_log_at("Skipping zero FUP", 2298 decoder->pos); 2299 no_tip = false; 2300 break; 2301 } 2302 intel_pt_set_last_ip(decoder); 2303 if (!decoder->branch_enable) { 2304 decoder->ip = decoder->last_ip; 2305 if (intel_pt_fup_event(decoder)) 2306 return 0; 2307 no_tip = false; 2308 break; 2309 } 2310 if (decoder->set_fup_mwait) 2311 no_tip = true; 2312 if (no_tip) 2313 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 2314 else 2315 decoder->pkt_state = INTEL_PT_STATE_FUP; 2316 err = intel_pt_walk_fup(decoder); 2317 if (err != -EAGAIN) 2318 return err; 2319 if (no_tip) { 2320 no_tip = false; 2321 break; 2322 } 2323 return intel_pt_walk_fup_tip(decoder); 2324 2325 case INTEL_PT_TRACESTOP: 2326 decoder->pge = false; 2327 decoder->continuous_period = false; 2328 intel_pt_clear_tx_flags(decoder); 2329 decoder->have_tma = false; 2330 break; 2331 2332 case INTEL_PT_PSB: 2333 decoder->state.psb_offset = decoder->pos; 2334 decoder->psb_ip = 0; 2335 if (intel_pt_psb_with_fup(decoder, &err)) 2336 return err; 2337 err = intel_pt_psb(decoder); 2338 if (err == -EAGAIN) 2339 goto next; 2340 return err; 2341 2342 case INTEL_PT_PIP: 2343 intel_pt_update_pip(decoder); 2344 break; 2345 2346 case INTEL_PT_MTC: 2347 intel_pt_calc_mtc_timestamp(decoder); 2348 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 2349 break; 2350 /* 2351 * Ensure that there has been an instruction since the 2352 * last MTC. 2353 */ 2354 if (!decoder->mtc_insn) 2355 break; 2356 decoder->mtc_insn = false; 2357 /* Ensure that there is a timestamp */ 2358 if (!decoder->timestamp) 2359 break; 2360 decoder->state.type = INTEL_PT_INSTRUCTION; 2361 decoder->state.from_ip = decoder->ip; 2362 decoder->state.to_ip = 0; 2363 decoder->mtc_insn = false; 2364 return 0; 2365 2366 case INTEL_PT_TSC: 2367 intel_pt_calc_tsc_timestamp(decoder); 2368 break; 2369 2370 case INTEL_PT_TMA: 2371 intel_pt_calc_tma(decoder); 2372 break; 2373 2374 case INTEL_PT_CYC: 2375 intel_pt_calc_cyc_timestamp(decoder); 2376 break; 2377 2378 case INTEL_PT_CBR: 2379 intel_pt_calc_cbr(decoder); 2380 if (decoder->cbr != decoder->cbr_seen) { 2381 decoder->state.type = 0; 2382 return 0; 2383 } 2384 break; 2385 2386 case INTEL_PT_MODE_EXEC: 2387 decoder->exec_mode = decoder->packet.payload; 2388 break; 2389 2390 case INTEL_PT_MODE_TSX: 2391 /* MODE_TSX need not be followed by FUP */ 2392 if (!decoder->pge || decoder->in_psb) { 2393 intel_pt_update_in_tx(decoder); 2394 break; 2395 } 2396 err = intel_pt_mode_tsx(decoder, &no_tip); 2397 if (err) 2398 return err; 2399 goto next; 2400 2401 case INTEL_PT_BAD: /* Does not happen */ 2402 return intel_pt_bug(decoder); 2403 2404 case INTEL_PT_PSBEND: 2405 case INTEL_PT_VMCS: 2406 case INTEL_PT_MNT: 2407 case INTEL_PT_PAD: 2408 break; 2409 2410 case INTEL_PT_PTWRITE_IP: 2411 decoder->fup_ptw_payload = decoder->packet.payload; 2412 err = intel_pt_get_next_packet(decoder); 2413 if (err) 2414 return err; 2415 if (decoder->packet.type == INTEL_PT_FUP) { 2416 decoder->set_fup_ptw = true; 2417 no_tip = true; 2418 } else { 2419 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 2420 decoder->pos); 2421 } 2422 goto next; 2423 2424 case INTEL_PT_PTWRITE: 2425 decoder->state.type = INTEL_PT_PTW; 2426 decoder->state.from_ip = decoder->ip; 2427 decoder->state.to_ip = 0; 2428 decoder->state.ptw_payload = decoder->packet.payload; 2429 return 0; 2430 2431 case INTEL_PT_MWAIT: 2432 decoder->fup_mwait_payload = decoder->packet.payload; 2433 decoder->set_fup_mwait = true; 2434 break; 2435 2436 case INTEL_PT_PWRE: 2437 if (decoder->set_fup_mwait) { 2438 decoder->fup_pwre_payload = 2439 decoder->packet.payload; 2440 decoder->set_fup_pwre = true; 2441 break; 2442 } 2443 decoder->state.type = INTEL_PT_PWR_ENTRY; 2444 decoder->state.from_ip = decoder->ip; 2445 decoder->state.to_ip = 0; 2446 decoder->state.pwrx_payload = decoder->packet.payload; 2447 return 0; 2448 2449 case INTEL_PT_EXSTOP_IP: 2450 err = intel_pt_get_next_packet(decoder); 2451 if (err) 2452 return err; 2453 if (decoder->packet.type == INTEL_PT_FUP) { 2454 decoder->set_fup_exstop = true; 2455 no_tip = true; 2456 } else { 2457 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 2458 decoder->pos); 2459 } 2460 goto next; 2461 2462 case INTEL_PT_EXSTOP: 2463 decoder->state.type = INTEL_PT_EX_STOP; 2464 decoder->state.from_ip = decoder->ip; 2465 decoder->state.to_ip = 0; 2466 return 0; 2467 2468 case INTEL_PT_PWRX: 2469 decoder->state.type = INTEL_PT_PWR_EXIT; 2470 decoder->state.from_ip = decoder->ip; 2471 decoder->state.to_ip = 0; 2472 decoder->state.pwrx_payload = decoder->packet.payload; 2473 return 0; 2474 2475 case INTEL_PT_BBP: 2476 intel_pt_bbp(decoder); 2477 break; 2478 2479 case INTEL_PT_BIP: 2480 intel_pt_bip(decoder); 2481 break; 2482 2483 case INTEL_PT_BEP: 2484 decoder->state.type = INTEL_PT_BLK_ITEMS; 2485 decoder->state.from_ip = decoder->ip; 2486 decoder->state.to_ip = 0; 2487 return 0; 2488 2489 case INTEL_PT_BEP_IP: 2490 err = intel_pt_get_next_packet(decoder); 2491 if (err) 2492 return err; 2493 if (decoder->packet.type == INTEL_PT_FUP) { 2494 decoder->set_fup_bep = true; 2495 no_tip = true; 2496 } else { 2497 intel_pt_log_at("ERROR: Missing FUP after BEP", 2498 decoder->pos); 2499 } 2500 goto next; 2501 2502 default: 2503 return intel_pt_bug(decoder); 2504 } 2505 } 2506 } 2507 2508 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 2509 { 2510 return decoder->packet.count && 2511 (decoder->have_last_ip || decoder->packet.count == 3 || 2512 decoder->packet.count == 6); 2513 } 2514 2515 /* Walk PSB+ packets to get in sync. */ 2516 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 2517 { 2518 int err; 2519 2520 decoder->in_psb = true; 2521 2522 while (1) { 2523 err = intel_pt_get_next_packet(decoder); 2524 if (err) 2525 goto out; 2526 2527 switch (decoder->packet.type) { 2528 case INTEL_PT_TIP_PGD: 2529 decoder->continuous_period = false; 2530 __fallthrough; 2531 case INTEL_PT_TIP_PGE: 2532 case INTEL_PT_TIP: 2533 case INTEL_PT_PTWRITE: 2534 case INTEL_PT_PTWRITE_IP: 2535 case INTEL_PT_EXSTOP: 2536 case INTEL_PT_EXSTOP_IP: 2537 case INTEL_PT_MWAIT: 2538 case INTEL_PT_PWRE: 2539 case INTEL_PT_PWRX: 2540 case INTEL_PT_BBP: 2541 case INTEL_PT_BIP: 2542 case INTEL_PT_BEP: 2543 case INTEL_PT_BEP_IP: 2544 intel_pt_log("ERROR: Unexpected packet\n"); 2545 err = -ENOENT; 2546 goto out; 2547 2548 case INTEL_PT_FUP: 2549 decoder->pge = true; 2550 if (intel_pt_have_ip(decoder)) { 2551 uint64_t current_ip = decoder->ip; 2552 2553 intel_pt_set_ip(decoder); 2554 decoder->psb_ip = decoder->ip; 2555 if (current_ip) 2556 intel_pt_log_to("Setting IP", 2557 decoder->ip); 2558 } 2559 break; 2560 2561 case INTEL_PT_MTC: 2562 intel_pt_calc_mtc_timestamp(decoder); 2563 break; 2564 2565 case INTEL_PT_TSC: 2566 intel_pt_calc_tsc_timestamp(decoder); 2567 break; 2568 2569 case INTEL_PT_TMA: 2570 intel_pt_calc_tma(decoder); 2571 break; 2572 2573 case INTEL_PT_CYC: 2574 intel_pt_calc_cyc_timestamp(decoder); 2575 break; 2576 2577 case INTEL_PT_CBR: 2578 intel_pt_calc_cbr(decoder); 2579 break; 2580 2581 case INTEL_PT_PIP: 2582 intel_pt_set_pip(decoder); 2583 break; 2584 2585 case INTEL_PT_MODE_EXEC: 2586 decoder->exec_mode = decoder->packet.payload; 2587 break; 2588 2589 case INTEL_PT_MODE_TSX: 2590 intel_pt_update_in_tx(decoder); 2591 break; 2592 2593 case INTEL_PT_TRACESTOP: 2594 decoder->pge = false; 2595 decoder->continuous_period = false; 2596 intel_pt_clear_tx_flags(decoder); 2597 __fallthrough; 2598 2599 case INTEL_PT_TNT: 2600 decoder->have_tma = false; 2601 intel_pt_log("ERROR: Unexpected packet\n"); 2602 if (decoder->ip) 2603 decoder->pkt_state = INTEL_PT_STATE_ERR4; 2604 else 2605 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2606 err = -ENOENT; 2607 goto out; 2608 2609 case INTEL_PT_BAD: /* Does not happen */ 2610 err = intel_pt_bug(decoder); 2611 goto out; 2612 2613 case INTEL_PT_OVF: 2614 err = intel_pt_overflow(decoder); 2615 goto out; 2616 2617 case INTEL_PT_PSBEND: 2618 err = 0; 2619 goto out; 2620 2621 case INTEL_PT_PSB: 2622 case INTEL_PT_VMCS: 2623 case INTEL_PT_MNT: 2624 case INTEL_PT_PAD: 2625 default: 2626 break; 2627 } 2628 } 2629 out: 2630 decoder->in_psb = false; 2631 2632 return err; 2633 } 2634 2635 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 2636 { 2637 int err; 2638 2639 while (1) { 2640 err = intel_pt_get_next_packet(decoder); 2641 if (err) 2642 return err; 2643 2644 switch (decoder->packet.type) { 2645 case INTEL_PT_TIP_PGD: 2646 decoder->continuous_period = false; 2647 decoder->pge = false; 2648 if (intel_pt_have_ip(decoder)) 2649 intel_pt_set_ip(decoder); 2650 if (!decoder->ip) 2651 break; 2652 decoder->state.type |= INTEL_PT_TRACE_END; 2653 return 0; 2654 2655 case INTEL_PT_TIP_PGE: 2656 decoder->pge = true; 2657 intel_pt_mtc_cyc_cnt_pge(decoder); 2658 if (intel_pt_have_ip(decoder)) 2659 intel_pt_set_ip(decoder); 2660 if (!decoder->ip) 2661 break; 2662 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2663 return 0; 2664 2665 case INTEL_PT_TIP: 2666 decoder->pge = true; 2667 if (intel_pt_have_ip(decoder)) 2668 intel_pt_set_ip(decoder); 2669 if (!decoder->ip) 2670 break; 2671 return 0; 2672 2673 case INTEL_PT_FUP: 2674 if (intel_pt_have_ip(decoder)) 2675 intel_pt_set_ip(decoder); 2676 if (decoder->ip) 2677 return 0; 2678 break; 2679 2680 case INTEL_PT_MTC: 2681 intel_pt_calc_mtc_timestamp(decoder); 2682 break; 2683 2684 case INTEL_PT_TSC: 2685 intel_pt_calc_tsc_timestamp(decoder); 2686 break; 2687 2688 case INTEL_PT_TMA: 2689 intel_pt_calc_tma(decoder); 2690 break; 2691 2692 case INTEL_PT_CYC: 2693 intel_pt_calc_cyc_timestamp(decoder); 2694 break; 2695 2696 case INTEL_PT_CBR: 2697 intel_pt_calc_cbr(decoder); 2698 break; 2699 2700 case INTEL_PT_PIP: 2701 intel_pt_set_pip(decoder); 2702 break; 2703 2704 case INTEL_PT_MODE_EXEC: 2705 decoder->exec_mode = decoder->packet.payload; 2706 break; 2707 2708 case INTEL_PT_MODE_TSX: 2709 intel_pt_update_in_tx(decoder); 2710 break; 2711 2712 case INTEL_PT_OVF: 2713 return intel_pt_overflow(decoder); 2714 2715 case INTEL_PT_BAD: /* Does not happen */ 2716 return intel_pt_bug(decoder); 2717 2718 case INTEL_PT_TRACESTOP: 2719 decoder->pge = false; 2720 decoder->continuous_period = false; 2721 intel_pt_clear_tx_flags(decoder); 2722 decoder->have_tma = false; 2723 break; 2724 2725 case INTEL_PT_PSB: 2726 decoder->state.psb_offset = decoder->pos; 2727 decoder->psb_ip = 0; 2728 decoder->last_ip = 0; 2729 decoder->have_last_ip = true; 2730 intel_pt_clear_stack(&decoder->stack); 2731 err = intel_pt_walk_psb(decoder); 2732 if (err) 2733 return err; 2734 decoder->state.type = INTEL_PT_PSB_EVT; 2735 decoder->state.from_ip = decoder->psb_ip; 2736 decoder->state.to_ip = 0; 2737 return 0; 2738 2739 case INTEL_PT_TNT: 2740 case INTEL_PT_PSBEND: 2741 case INTEL_PT_VMCS: 2742 case INTEL_PT_MNT: 2743 case INTEL_PT_PAD: 2744 case INTEL_PT_PTWRITE: 2745 case INTEL_PT_PTWRITE_IP: 2746 case INTEL_PT_EXSTOP: 2747 case INTEL_PT_EXSTOP_IP: 2748 case INTEL_PT_MWAIT: 2749 case INTEL_PT_PWRE: 2750 case INTEL_PT_PWRX: 2751 case INTEL_PT_BBP: 2752 case INTEL_PT_BIP: 2753 case INTEL_PT_BEP: 2754 case INTEL_PT_BEP_IP: 2755 default: 2756 break; 2757 } 2758 } 2759 } 2760 2761 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 2762 { 2763 int err; 2764 2765 decoder->set_fup_tx_flags = false; 2766 decoder->set_fup_ptw = false; 2767 decoder->set_fup_mwait = false; 2768 decoder->set_fup_pwre = false; 2769 decoder->set_fup_exstop = false; 2770 decoder->set_fup_bep = false; 2771 2772 if (!decoder->branch_enable) { 2773 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2774 decoder->overflow = false; 2775 decoder->state.type = 0; /* Do not have a sample */ 2776 return 0; 2777 } 2778 2779 intel_pt_log("Scanning for full IP\n"); 2780 err = intel_pt_walk_to_ip(decoder); 2781 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 2782 return err; 2783 2784 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 2785 if (decoder->hop) 2786 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2787 else 2788 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2789 decoder->overflow = false; 2790 2791 decoder->state.from_ip = 0; 2792 decoder->state.to_ip = decoder->ip; 2793 intel_pt_log_to("Setting IP", decoder->ip); 2794 2795 return 0; 2796 } 2797 2798 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 2799 { 2800 const unsigned char *end = decoder->buf + decoder->len; 2801 size_t i; 2802 2803 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 2804 if (i > decoder->len) 2805 continue; 2806 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 2807 return i; 2808 } 2809 return 0; 2810 } 2811 2812 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 2813 { 2814 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 2815 const char *psb = INTEL_PT_PSB_STR; 2816 2817 if (rest_psb > decoder->len || 2818 memcmp(decoder->buf, psb + part_psb, rest_psb)) 2819 return 0; 2820 2821 return rest_psb; 2822 } 2823 2824 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 2825 int part_psb) 2826 { 2827 int rest_psb, ret; 2828 2829 decoder->pos += decoder->len; 2830 decoder->len = 0; 2831 2832 ret = intel_pt_get_next_data(decoder, false); 2833 if (ret) 2834 return ret; 2835 2836 rest_psb = intel_pt_rest_psb(decoder, part_psb); 2837 if (!rest_psb) 2838 return 0; 2839 2840 decoder->pos -= part_psb; 2841 decoder->next_buf = decoder->buf + rest_psb; 2842 decoder->next_len = decoder->len - rest_psb; 2843 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2844 decoder->buf = decoder->temp_buf; 2845 decoder->len = INTEL_PT_PSB_LEN; 2846 2847 return 0; 2848 } 2849 2850 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 2851 { 2852 unsigned char *next; 2853 int ret; 2854 2855 intel_pt_log("Scanning for PSB\n"); 2856 while (1) { 2857 if (!decoder->len) { 2858 ret = intel_pt_get_next_data(decoder, false); 2859 if (ret) 2860 return ret; 2861 } 2862 2863 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2864 INTEL_PT_PSB_LEN); 2865 if (!next) { 2866 int part_psb; 2867 2868 part_psb = intel_pt_part_psb(decoder); 2869 if (part_psb) { 2870 ret = intel_pt_get_split_psb(decoder, part_psb); 2871 if (ret) 2872 return ret; 2873 } else { 2874 decoder->pos += decoder->len; 2875 decoder->len = 0; 2876 } 2877 continue; 2878 } 2879 2880 decoder->pkt_step = next - decoder->buf; 2881 return intel_pt_get_next_packet(decoder); 2882 } 2883 } 2884 2885 static int intel_pt_sync(struct intel_pt_decoder *decoder) 2886 { 2887 int err; 2888 2889 decoder->pge = false; 2890 decoder->continuous_period = false; 2891 decoder->have_last_ip = false; 2892 decoder->last_ip = 0; 2893 decoder->psb_ip = 0; 2894 decoder->ip = 0; 2895 intel_pt_clear_stack(&decoder->stack); 2896 2897 err = intel_pt_scan_for_psb(decoder); 2898 if (err) 2899 return err; 2900 2901 decoder->have_last_ip = true; 2902 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 2903 2904 err = intel_pt_walk_psb(decoder); 2905 if (err) 2906 return err; 2907 2908 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 2909 decoder->state.from_ip = decoder->psb_ip; 2910 decoder->state.to_ip = 0; 2911 2912 if (decoder->ip) { 2913 /* 2914 * In hop mode, resample to get the PSB FUP ip as an 2915 * "instruction" sample. 2916 */ 2917 if (decoder->hop) 2918 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2919 else 2920 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2921 } 2922 2923 return 0; 2924 } 2925 2926 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2927 { 2928 uint64_t est = decoder->sample_insn_cnt << 1; 2929 2930 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2931 goto out; 2932 2933 est *= decoder->max_non_turbo_ratio; 2934 est /= decoder->cbr; 2935 out: 2936 return decoder->sample_timestamp + est; 2937 } 2938 2939 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2940 { 2941 int err; 2942 2943 do { 2944 decoder->state.type = INTEL_PT_BRANCH; 2945 decoder->state.flags = 0; 2946 2947 switch (decoder->pkt_state) { 2948 case INTEL_PT_STATE_NO_PSB: 2949 err = intel_pt_sync(decoder); 2950 break; 2951 case INTEL_PT_STATE_NO_IP: 2952 decoder->have_last_ip = false; 2953 decoder->last_ip = 0; 2954 decoder->ip = 0; 2955 __fallthrough; 2956 case INTEL_PT_STATE_ERR_RESYNC: 2957 err = intel_pt_sync_ip(decoder); 2958 break; 2959 case INTEL_PT_STATE_IN_SYNC: 2960 err = intel_pt_walk_trace(decoder); 2961 break; 2962 case INTEL_PT_STATE_TNT: 2963 case INTEL_PT_STATE_TNT_CONT: 2964 err = intel_pt_walk_tnt(decoder); 2965 if (err == -EAGAIN) 2966 err = intel_pt_walk_trace(decoder); 2967 break; 2968 case INTEL_PT_STATE_TIP: 2969 case INTEL_PT_STATE_TIP_PGD: 2970 err = intel_pt_walk_tip(decoder); 2971 break; 2972 case INTEL_PT_STATE_FUP: 2973 err = intel_pt_walk_fup(decoder); 2974 if (err == -EAGAIN) 2975 err = intel_pt_walk_fup_tip(decoder); 2976 break; 2977 case INTEL_PT_STATE_FUP_NO_TIP: 2978 err = intel_pt_walk_fup(decoder); 2979 if (err == -EAGAIN) 2980 err = intel_pt_walk_trace(decoder); 2981 break; 2982 case INTEL_PT_STATE_FUP_IN_PSB: 2983 err = intel_pt_fup_in_psb(decoder); 2984 break; 2985 case INTEL_PT_STATE_RESAMPLE: 2986 err = intel_pt_resample(decoder); 2987 break; 2988 default: 2989 err = intel_pt_bug(decoder); 2990 break; 2991 } 2992 } while (err == -ENOLINK); 2993 2994 if (err) { 2995 decoder->state.err = intel_pt_ext_err(err); 2996 decoder->state.from_ip = decoder->ip; 2997 intel_pt_update_sample_time(decoder); 2998 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 2999 intel_pt_set_nr(decoder); 3000 } else { 3001 decoder->state.err = 0; 3002 if (decoder->cbr != decoder->cbr_seen) { 3003 decoder->cbr_seen = decoder->cbr; 3004 if (!decoder->state.type) { 3005 decoder->state.from_ip = decoder->ip; 3006 decoder->state.to_ip = 0; 3007 } 3008 decoder->state.type |= INTEL_PT_CBR_CHG; 3009 decoder->state.cbr_payload = decoder->cbr_payload; 3010 decoder->state.cbr = decoder->cbr; 3011 } 3012 if (intel_pt_sample_time(decoder->pkt_state)) { 3013 intel_pt_update_sample_time(decoder); 3014 if (decoder->sample_cyc) { 3015 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3016 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3017 decoder->sample_cyc = false; 3018 } 3019 } 3020 /* 3021 * When using only TSC/MTC to compute cycles, IPC can be 3022 * sampled as soon as the cycle count changes. 3023 */ 3024 if (!decoder->have_cyc) 3025 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3026 } 3027 3028 /* Let PSB event always have TSC timestamp */ 3029 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 3030 decoder->sample_timestamp = decoder->tsc_timestamp; 3031 3032 decoder->state.from_nr = decoder->nr; 3033 decoder->state.to_nr = decoder->next_nr; 3034 decoder->nr = decoder->next_nr; 3035 3036 decoder->state.timestamp = decoder->sample_timestamp; 3037 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 3038 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 3039 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 3040 3041 return &decoder->state; 3042 } 3043 3044 /** 3045 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 3046 * @buf: pointer to buffer pointer 3047 * @len: size of buffer 3048 * 3049 * Updates the buffer pointer to point to the start of the next PSB packet if 3050 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 3051 * @len is adjusted accordingly. 3052 * 3053 * Return: %true if a PSB packet is found, %false otherwise. 3054 */ 3055 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 3056 { 3057 unsigned char *next; 3058 3059 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3060 if (next) { 3061 *len -= next - *buf; 3062 *buf = next; 3063 return true; 3064 } 3065 return false; 3066 } 3067 3068 /** 3069 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 3070 * packet. 3071 * @buf: pointer to buffer pointer 3072 * @len: size of buffer 3073 * 3074 * Updates the buffer pointer to point to the start of the following PSB packet 3075 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 3076 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 3077 * 3078 * Return: %true if a PSB packet is found, %false otherwise. 3079 */ 3080 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 3081 { 3082 unsigned char *next; 3083 3084 if (!*len) 3085 return false; 3086 3087 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3088 if (next) { 3089 *len -= next - *buf; 3090 *buf = next; 3091 return true; 3092 } 3093 return false; 3094 } 3095 3096 /** 3097 * intel_pt_last_psb - find the last PSB packet in a buffer. 3098 * @buf: buffer 3099 * @len: size of buffer 3100 * 3101 * This function finds the last PSB in a buffer. 3102 * 3103 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 3104 */ 3105 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 3106 { 3107 const char *n = INTEL_PT_PSB_STR; 3108 unsigned char *p; 3109 size_t k; 3110 3111 if (len < INTEL_PT_PSB_LEN) 3112 return NULL; 3113 3114 k = len - INTEL_PT_PSB_LEN + 1; 3115 while (1) { 3116 p = memrchr(buf, n[0], k); 3117 if (!p) 3118 return NULL; 3119 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 3120 return p; 3121 k = p - buf; 3122 if (!k) 3123 return NULL; 3124 } 3125 } 3126 3127 /** 3128 * intel_pt_next_tsc - find and return next TSC. 3129 * @buf: buffer 3130 * @len: size of buffer 3131 * @tsc: TSC value returned 3132 * @rem: returns remaining size when TSC is found 3133 * 3134 * Find a TSC packet in @buf and return the TSC value. This function assumes 3135 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 3136 * PSBEND packet is found. 3137 * 3138 * Return: %true if TSC is found, false otherwise. 3139 */ 3140 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 3141 size_t *rem) 3142 { 3143 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 3144 struct intel_pt_pkt packet; 3145 int ret; 3146 3147 while (len) { 3148 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 3149 if (ret <= 0) 3150 return false; 3151 if (packet.type == INTEL_PT_TSC) { 3152 *tsc = packet.payload; 3153 *rem = len; 3154 return true; 3155 } 3156 if (packet.type == INTEL_PT_PSBEND) 3157 return false; 3158 buf += ret; 3159 len -= ret; 3160 } 3161 return false; 3162 } 3163 3164 /** 3165 * intel_pt_tsc_cmp - compare 7-byte TSCs. 3166 * @tsc1: first TSC to compare 3167 * @tsc2: second TSC to compare 3168 * 3169 * This function compares 7-byte TSC values allowing for the possibility that 3170 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 3171 * around so for that purpose this function assumes the absolute difference is 3172 * less than half the maximum difference. 3173 * 3174 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 3175 * after @tsc2. 3176 */ 3177 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 3178 { 3179 const uint64_t halfway = (1ULL << 55); 3180 3181 if (tsc1 == tsc2) 3182 return 0; 3183 3184 if (tsc1 < tsc2) { 3185 if (tsc2 - tsc1 < halfway) 3186 return -1; 3187 else 3188 return 1; 3189 } else { 3190 if (tsc1 - tsc2 < halfway) 3191 return 1; 3192 else 3193 return -1; 3194 } 3195 } 3196 3197 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 3198 3199 /** 3200 * adj_for_padding - adjust overlap to account for padding. 3201 * @buf_b: second buffer 3202 * @buf_a: first buffer 3203 * @len_a: size of first buffer 3204 * 3205 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 3206 * accordingly. 3207 * 3208 * Return: A pointer into @buf_b from where non-overlapped data starts 3209 */ 3210 static unsigned char *adj_for_padding(unsigned char *buf_b, 3211 unsigned char *buf_a, size_t len_a) 3212 { 3213 unsigned char *p = buf_b - MAX_PADDING; 3214 unsigned char *q = buf_a + len_a - MAX_PADDING; 3215 int i; 3216 3217 for (i = MAX_PADDING; i; i--, p++, q++) { 3218 if (*p != *q) 3219 break; 3220 } 3221 3222 return p; 3223 } 3224 3225 /** 3226 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3227 * using TSC. 3228 * @buf_a: first buffer 3229 * @len_a: size of first buffer 3230 * @buf_b: second buffer 3231 * @len_b: size of second buffer 3232 * @consecutive: returns true if there is data in buf_b that is consecutive 3233 * to buf_a 3234 * 3235 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3236 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3237 * walk forward in @buf_b until a later TSC is found. A precondition is that 3238 * @buf_a and @buf_b are positioned at a PSB. 3239 * 3240 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3241 * @buf_b + @len_b if there is no non-overlapped data. 3242 */ 3243 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3244 size_t len_a, 3245 unsigned char *buf_b, 3246 size_t len_b, bool *consecutive) 3247 { 3248 uint64_t tsc_a, tsc_b; 3249 unsigned char *p; 3250 size_t len, rem_a, rem_b; 3251 3252 p = intel_pt_last_psb(buf_a, len_a); 3253 if (!p) 3254 return buf_b; /* No PSB in buf_a => no overlap */ 3255 3256 len = len_a - (p - buf_a); 3257 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3258 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3259 len_a -= len; 3260 p = intel_pt_last_psb(buf_a, len_a); 3261 if (!p) 3262 return buf_b; /* No full PSB+ => assume no overlap */ 3263 len = len_a - (p - buf_a); 3264 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 3265 return buf_b; /* No TSC in buf_a => assume no overlap */ 3266 } 3267 3268 while (1) { 3269 /* Ignore PSB+ with no TSC */ 3270 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 3271 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 3272 3273 /* Same TSC, so buffers are consecutive */ 3274 if (!cmp && rem_b >= rem_a) { 3275 unsigned char *start; 3276 3277 *consecutive = true; 3278 start = buf_b + len_b - (rem_b - rem_a); 3279 return adj_for_padding(start, buf_a, len_a); 3280 } 3281 if (cmp < 0) 3282 return buf_b; /* tsc_a < tsc_b => no overlap */ 3283 } 3284 3285 if (!intel_pt_step_psb(&buf_b, &len_b)) 3286 return buf_b + len_b; /* No PSB in buf_b => no data */ 3287 } 3288 } 3289 3290 /** 3291 * intel_pt_find_overlap - determine start of non-overlapped trace data. 3292 * @buf_a: first buffer 3293 * @len_a: size of first buffer 3294 * @buf_b: second buffer 3295 * @len_b: size of second buffer 3296 * @have_tsc: can use TSC packets to detect overlap 3297 * @consecutive: returns true if there is data in buf_b that is consecutive 3298 * to buf_a 3299 * 3300 * When trace samples or snapshots are recorded there is the possibility that 3301 * the data overlaps. Note that, for the purposes of decoding, data is only 3302 * useful if it begins with a PSB packet. 3303 * 3304 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3305 * @buf_b + @len_b if there is no non-overlapped data. 3306 */ 3307 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 3308 unsigned char *buf_b, size_t len_b, 3309 bool have_tsc, bool *consecutive) 3310 { 3311 unsigned char *found; 3312 3313 /* Buffer 'b' must start at PSB so throw away everything before that */ 3314 if (!intel_pt_next_psb(&buf_b, &len_b)) 3315 return buf_b + len_b; /* No PSB */ 3316 3317 if (!intel_pt_next_psb(&buf_a, &len_a)) 3318 return buf_b; /* No overlap */ 3319 3320 if (have_tsc) { 3321 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 3322 consecutive); 3323 if (found) 3324 return found; 3325 } 3326 3327 /* 3328 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 3329 * we can ignore the first part of buffer 'a'. 3330 */ 3331 while (len_b < len_a) { 3332 if (!intel_pt_step_psb(&buf_a, &len_a)) 3333 return buf_b; /* No overlap */ 3334 } 3335 3336 /* Now len_b >= len_a */ 3337 while (1) { 3338 /* Potential overlap so check the bytes */ 3339 found = memmem(buf_a, len_a, buf_b, len_a); 3340 if (found) { 3341 *consecutive = true; 3342 return adj_for_padding(buf_b + len_a, buf_a, len_a); 3343 } 3344 3345 /* Try again at next PSB in buffer 'a' */ 3346 if (!intel_pt_step_psb(&buf_a, &len_a)) 3347 return buf_b; /* No overlap */ 3348 } 3349 } 3350 3351 /** 3352 * struct fast_forward_data - data used by intel_pt_ff_cb(). 3353 * @timestamp: timestamp to fast forward towards 3354 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 3355 * the fast forward timestamp. 3356 */ 3357 struct fast_forward_data { 3358 uint64_t timestamp; 3359 uint64_t buf_timestamp; 3360 }; 3361 3362 /** 3363 * intel_pt_ff_cb - fast forward lookahead callback. 3364 * @buffer: Intel PT trace buffer 3365 * @data: opaque pointer to fast forward data (struct fast_forward_data) 3366 * 3367 * Determine if @buffer trace is past the fast forward timestamp. 3368 * 3369 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 3370 * timestamp, and 0 otherwise. 3371 */ 3372 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 3373 { 3374 struct fast_forward_data *d = data; 3375 unsigned char *buf; 3376 uint64_t tsc; 3377 size_t rem; 3378 size_t len; 3379 3380 buf = (unsigned char *)buffer->buf; 3381 len = buffer->len; 3382 3383 if (!intel_pt_next_psb(&buf, &len) || 3384 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 3385 return 0; 3386 3387 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 3388 3389 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 3390 tsc, buffer->ref_timestamp); 3391 3392 /* 3393 * If the buffer contains a timestamp earlier that the fast forward 3394 * timestamp, then record it, else stop. 3395 */ 3396 if (tsc < d->timestamp) 3397 d->buf_timestamp = buffer->ref_timestamp; 3398 else 3399 return 1; 3400 3401 return 0; 3402 } 3403 3404 /** 3405 * intel_pt_fast_forward - reposition decoder forwards. 3406 * @decoder: Intel PT decoder 3407 * @timestamp: timestamp to fast forward towards 3408 * 3409 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 3410 * 3411 * Return: 0 on success or negative error code on failure. 3412 */ 3413 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 3414 { 3415 struct fast_forward_data d = { .timestamp = timestamp }; 3416 unsigned char *buf; 3417 size_t len; 3418 int err; 3419 3420 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 3421 3422 /* Find buffer timestamp of buffer to fast forward to */ 3423 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 3424 if (err < 0) 3425 return err; 3426 3427 /* Walk to buffer with same buffer timestamp */ 3428 if (d.buf_timestamp) { 3429 do { 3430 decoder->pos += decoder->len; 3431 decoder->len = 0; 3432 err = intel_pt_get_next_data(decoder, true); 3433 /* -ENOLINK means non-consecutive trace */ 3434 if (err && err != -ENOLINK) 3435 return err; 3436 } while (decoder->buf_timestamp != d.buf_timestamp); 3437 } 3438 3439 if (!decoder->buf) 3440 return 0; 3441 3442 buf = (unsigned char *)decoder->buf; 3443 len = decoder->len; 3444 3445 if (!intel_pt_next_psb(&buf, &len)) 3446 return 0; 3447 3448 /* 3449 * Walk PSBs while the PSB timestamp is less than the fast forward 3450 * timestamp. 3451 */ 3452 do { 3453 uint64_t tsc; 3454 size_t rem; 3455 3456 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 3457 break; 3458 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 3459 /* 3460 * A TSC packet can slip past MTC packets but, after fast 3461 * forward, decoding starts at the TSC timestamp. That means 3462 * the timestamps may not be exactly the same as the timestamps 3463 * that would have been decoded without fast forward. 3464 */ 3465 if (tsc < timestamp) { 3466 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 3467 decoder->pos += decoder->len - len; 3468 decoder->buf = buf; 3469 decoder->len = len; 3470 intel_pt_reposition(decoder); 3471 } else { 3472 break; 3473 } 3474 } while (intel_pt_step_psb(&buf, &len)); 3475 3476 return 0; 3477 } 3478