1 /* 2 * intel_pt_decoder.c: Intel Processor Trace support 3 * Copyright (c) 2013-2014, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #ifndef _GNU_SOURCE 17 #define _GNU_SOURCE 18 #endif 19 #include <stdlib.h> 20 #include <stdbool.h> 21 #include <string.h> 22 #include <errno.h> 23 #include <stdint.h> 24 #include <inttypes.h> 25 26 #include "../cache.h" 27 #include "../util.h" 28 29 #include "intel-pt-insn-decoder.h" 30 #include "intel-pt-pkt-decoder.h" 31 #include "intel-pt-decoder.h" 32 #include "intel-pt-log.h" 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define INTEL_PT_RETURN 1 39 40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 41 #define INTEL_PT_MAX_LOOPS 10000 42 43 struct intel_pt_blk { 44 struct intel_pt_blk *prev; 45 uint64_t ip[INTEL_PT_BLK_SIZE]; 46 }; 47 48 struct intel_pt_stack { 49 struct intel_pt_blk *blk; 50 struct intel_pt_blk *spare; 51 int pos; 52 }; 53 54 enum intel_pt_pkt_state { 55 INTEL_PT_STATE_NO_PSB, 56 INTEL_PT_STATE_NO_IP, 57 INTEL_PT_STATE_ERR_RESYNC, 58 INTEL_PT_STATE_IN_SYNC, 59 INTEL_PT_STATE_TNT, 60 INTEL_PT_STATE_TIP, 61 INTEL_PT_STATE_TIP_PGD, 62 INTEL_PT_STATE_FUP, 63 INTEL_PT_STATE_FUP_NO_TIP, 64 }; 65 66 #ifdef INTEL_PT_STRICT 67 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 68 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 69 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 70 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 71 #else 72 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 73 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 74 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 75 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 76 #endif 77 78 struct intel_pt_decoder { 79 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 80 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 81 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 82 uint64_t max_insn_cnt, void *data); 83 void *data; 84 struct intel_pt_state state; 85 const unsigned char *buf; 86 size_t len; 87 bool return_compression; 88 bool mtc_insn; 89 bool pge; 90 bool have_tma; 91 bool have_cyc; 92 uint64_t pos; 93 uint64_t last_ip; 94 uint64_t ip; 95 uint64_t cr3; 96 uint64_t timestamp; 97 uint64_t tsc_timestamp; 98 uint64_t ref_timestamp; 99 uint64_t ret_addr; 100 uint64_t ctc_timestamp; 101 uint64_t ctc_delta; 102 uint64_t cycle_cnt; 103 uint64_t cyc_ref_timestamp; 104 uint32_t last_mtc; 105 uint32_t tsc_ctc_ratio_n; 106 uint32_t tsc_ctc_ratio_d; 107 uint32_t tsc_ctc_mult; 108 uint32_t tsc_slip; 109 uint32_t ctc_rem_mask; 110 int mtc_shift; 111 struct intel_pt_stack stack; 112 enum intel_pt_pkt_state pkt_state; 113 struct intel_pt_pkt packet; 114 struct intel_pt_pkt tnt; 115 int pkt_step; 116 int pkt_len; 117 int last_packet_type; 118 unsigned int cbr; 119 unsigned int max_non_turbo_ratio; 120 double max_non_turbo_ratio_fp; 121 double cbr_cyc_to_tsc; 122 double calc_cyc_to_tsc; 123 bool have_calc_cyc_to_tsc; 124 int exec_mode; 125 unsigned int insn_bytes; 126 uint64_t sign_bit; 127 uint64_t sign_bits; 128 uint64_t period; 129 enum intel_pt_period_type period_type; 130 uint64_t tot_insn_cnt; 131 uint64_t period_insn_cnt; 132 uint64_t period_mask; 133 uint64_t period_ticks; 134 uint64_t last_masked_timestamp; 135 bool continuous_period; 136 bool overflow; 137 bool set_fup_tx_flags; 138 unsigned int fup_tx_flags; 139 unsigned int tx_flags; 140 uint64_t timestamp_insn_cnt; 141 uint64_t stuck_ip; 142 int no_progress; 143 int stuck_ip_prd; 144 int stuck_ip_cnt; 145 const unsigned char *next_buf; 146 size_t next_len; 147 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 148 }; 149 150 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 151 { 152 int i; 153 154 for (i = 0; x != 1; i++) 155 x >>= 1; 156 157 return x << i; 158 } 159 160 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 161 { 162 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 163 uint64_t period; 164 165 period = intel_pt_lower_power_of_2(decoder->period); 166 decoder->period_mask = ~(period - 1); 167 decoder->period_ticks = period; 168 } 169 } 170 171 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 172 { 173 if (!d) 174 return 0; 175 return (t / d) * n + ((t % d) * n) / d; 176 } 177 178 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 179 { 180 struct intel_pt_decoder *decoder; 181 182 if (!params->get_trace || !params->walk_insn) 183 return NULL; 184 185 decoder = zalloc(sizeof(struct intel_pt_decoder)); 186 if (!decoder) 187 return NULL; 188 189 decoder->get_trace = params->get_trace; 190 decoder->walk_insn = params->walk_insn; 191 decoder->data = params->data; 192 decoder->return_compression = params->return_compression; 193 194 decoder->sign_bit = (uint64_t)1 << 47; 195 decoder->sign_bits = ~(((uint64_t)1 << 48) - 1); 196 197 decoder->period = params->period; 198 decoder->period_type = params->period_type; 199 200 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 201 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 202 203 intel_pt_setup_period(decoder); 204 205 decoder->mtc_shift = params->mtc_period; 206 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 207 208 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 209 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 210 211 if (!decoder->tsc_ctc_ratio_n) 212 decoder->tsc_ctc_ratio_d = 0; 213 214 if (decoder->tsc_ctc_ratio_d) { 215 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 216 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 217 decoder->tsc_ctc_ratio_d; 218 219 /* 220 * Allow for timestamps appearing to backwards because a TSC 221 * packet has slipped past a MTC packet, so allow 2 MTC ticks 222 * or ... 223 */ 224 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift, 225 decoder->tsc_ctc_ratio_n, 226 decoder->tsc_ctc_ratio_d); 227 } 228 /* ... or 0x100 paranoia */ 229 if (decoder->tsc_slip < 0x100) 230 decoder->tsc_slip = 0x100; 231 232 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 233 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 234 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 235 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 236 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 237 238 return decoder; 239 } 240 241 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 242 { 243 struct intel_pt_blk *blk = stack->blk; 244 245 stack->blk = blk->prev; 246 if (!stack->spare) 247 stack->spare = blk; 248 else 249 free(blk); 250 } 251 252 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 253 { 254 if (!stack->pos) { 255 if (!stack->blk) 256 return 0; 257 intel_pt_pop_blk(stack); 258 if (!stack->blk) 259 return 0; 260 stack->pos = INTEL_PT_BLK_SIZE; 261 } 262 return stack->blk->ip[--stack->pos]; 263 } 264 265 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 266 { 267 struct intel_pt_blk *blk; 268 269 if (stack->spare) { 270 blk = stack->spare; 271 stack->spare = NULL; 272 } else { 273 blk = malloc(sizeof(struct intel_pt_blk)); 274 if (!blk) 275 return -ENOMEM; 276 } 277 278 blk->prev = stack->blk; 279 stack->blk = blk; 280 stack->pos = 0; 281 return 0; 282 } 283 284 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 285 { 286 int err; 287 288 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 289 err = intel_pt_alloc_blk(stack); 290 if (err) 291 return err; 292 } 293 294 stack->blk->ip[stack->pos++] = ip; 295 return 0; 296 } 297 298 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 299 { 300 while (stack->blk) 301 intel_pt_pop_blk(stack); 302 stack->pos = 0; 303 } 304 305 static void intel_pt_free_stack(struct intel_pt_stack *stack) 306 { 307 intel_pt_clear_stack(stack); 308 zfree(&stack->blk); 309 zfree(&stack->spare); 310 } 311 312 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 313 { 314 intel_pt_free_stack(&decoder->stack); 315 free(decoder); 316 } 317 318 static int intel_pt_ext_err(int code) 319 { 320 switch (code) { 321 case -ENOMEM: 322 return INTEL_PT_ERR_NOMEM; 323 case -ENOSYS: 324 return INTEL_PT_ERR_INTERN; 325 case -EBADMSG: 326 return INTEL_PT_ERR_BADPKT; 327 case -ENODATA: 328 return INTEL_PT_ERR_NODATA; 329 case -EILSEQ: 330 return INTEL_PT_ERR_NOINSN; 331 case -ENOENT: 332 return INTEL_PT_ERR_MISMAT; 333 case -EOVERFLOW: 334 return INTEL_PT_ERR_OVR; 335 case -ENOSPC: 336 return INTEL_PT_ERR_LOST; 337 case -ELOOP: 338 return INTEL_PT_ERR_NELOOP; 339 default: 340 return INTEL_PT_ERR_UNK; 341 } 342 } 343 344 static const char *intel_pt_err_msgs[] = { 345 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 346 [INTEL_PT_ERR_INTERN] = "Internal error", 347 [INTEL_PT_ERR_BADPKT] = "Bad packet", 348 [INTEL_PT_ERR_NODATA] = "No more data", 349 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 350 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 351 [INTEL_PT_ERR_OVR] = "Overflow packet", 352 [INTEL_PT_ERR_LOST] = "Lost trace data", 353 [INTEL_PT_ERR_UNK] = "Unknown error!", 354 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 355 }; 356 357 int intel_pt__strerror(int code, char *buf, size_t buflen) 358 { 359 if (code < 1 || code >= INTEL_PT_ERR_MAX) 360 code = INTEL_PT_ERR_UNK; 361 strlcpy(buf, intel_pt_err_msgs[code], buflen); 362 return 0; 363 } 364 365 static uint64_t intel_pt_calc_ip(struct intel_pt_decoder *decoder, 366 const struct intel_pt_pkt *packet, 367 uint64_t last_ip) 368 { 369 uint64_t ip; 370 371 switch (packet->count) { 372 case 2: 373 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 374 packet->payload; 375 break; 376 case 4: 377 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 378 packet->payload; 379 break; 380 case 6: 381 ip = packet->payload; 382 break; 383 default: 384 return 0; 385 } 386 387 if (ip & decoder->sign_bit) 388 return ip | decoder->sign_bits; 389 390 return ip; 391 } 392 393 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 394 { 395 decoder->last_ip = intel_pt_calc_ip(decoder, &decoder->packet, 396 decoder->last_ip); 397 } 398 399 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 400 { 401 intel_pt_set_last_ip(decoder); 402 decoder->ip = decoder->last_ip; 403 } 404 405 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 406 { 407 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 408 decoder->buf); 409 } 410 411 static int intel_pt_bug(struct intel_pt_decoder *decoder) 412 { 413 intel_pt_log("ERROR: Internal error\n"); 414 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 415 return -ENOSYS; 416 } 417 418 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 419 { 420 decoder->tx_flags = 0; 421 } 422 423 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 424 { 425 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 426 } 427 428 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 429 { 430 intel_pt_clear_tx_flags(decoder); 431 decoder->have_tma = false; 432 decoder->pkt_len = 1; 433 decoder->pkt_step = 1; 434 intel_pt_decoder_log_packet(decoder); 435 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 436 intel_pt_log("ERROR: Bad packet\n"); 437 decoder->pkt_state = INTEL_PT_STATE_ERR1; 438 } 439 return -EBADMSG; 440 } 441 442 static int intel_pt_get_data(struct intel_pt_decoder *decoder) 443 { 444 struct intel_pt_buffer buffer = { .buf = 0, }; 445 int ret; 446 447 decoder->pkt_step = 0; 448 449 intel_pt_log("Getting more data\n"); 450 ret = decoder->get_trace(&buffer, decoder->data); 451 if (ret) 452 return ret; 453 decoder->buf = buffer.buf; 454 decoder->len = buffer.len; 455 if (!decoder->len) { 456 intel_pt_log("No more data\n"); 457 return -ENODATA; 458 } 459 if (!buffer.consecutive) { 460 decoder->ip = 0; 461 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 462 decoder->ref_timestamp = buffer.ref_timestamp; 463 decoder->timestamp = 0; 464 decoder->have_tma = false; 465 decoder->state.trace_nr = buffer.trace_nr; 466 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 467 decoder->ref_timestamp); 468 return -ENOLINK; 469 } 470 471 return 0; 472 } 473 474 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder) 475 { 476 if (!decoder->next_buf) 477 return intel_pt_get_data(decoder); 478 479 decoder->buf = decoder->next_buf; 480 decoder->len = decoder->next_len; 481 decoder->next_buf = 0; 482 decoder->next_len = 0; 483 return 0; 484 } 485 486 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 487 { 488 unsigned char *buf = decoder->temp_buf; 489 size_t old_len, len, n; 490 int ret; 491 492 old_len = decoder->len; 493 len = decoder->len; 494 memcpy(buf, decoder->buf, len); 495 496 ret = intel_pt_get_data(decoder); 497 if (ret) { 498 decoder->pos += old_len; 499 return ret < 0 ? ret : -EINVAL; 500 } 501 502 n = INTEL_PT_PKT_MAX_SZ - len; 503 if (n > decoder->len) 504 n = decoder->len; 505 memcpy(buf + len, decoder->buf, n); 506 len += n; 507 508 ret = intel_pt_get_packet(buf, len, &decoder->packet); 509 if (ret < (int)old_len) { 510 decoder->next_buf = decoder->buf; 511 decoder->next_len = decoder->len; 512 decoder->buf = buf; 513 decoder->len = old_len; 514 return intel_pt_bad_packet(decoder); 515 } 516 517 decoder->next_buf = decoder->buf + (ret - old_len); 518 decoder->next_len = decoder->len - (ret - old_len); 519 520 decoder->buf = buf; 521 decoder->len = ret; 522 523 return ret; 524 } 525 526 struct intel_pt_pkt_info { 527 struct intel_pt_decoder *decoder; 528 struct intel_pt_pkt packet; 529 uint64_t pos; 530 int pkt_len; 531 int last_packet_type; 532 void *data; 533 }; 534 535 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 536 537 /* Lookahead packets in current buffer */ 538 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 539 intel_pt_pkt_cb_t cb, void *data) 540 { 541 struct intel_pt_pkt_info pkt_info; 542 const unsigned char *buf = decoder->buf; 543 size_t len = decoder->len; 544 int ret; 545 546 pkt_info.decoder = decoder; 547 pkt_info.pos = decoder->pos; 548 pkt_info.pkt_len = decoder->pkt_step; 549 pkt_info.last_packet_type = decoder->last_packet_type; 550 pkt_info.data = data; 551 552 while (1) { 553 do { 554 pkt_info.pos += pkt_info.pkt_len; 555 buf += pkt_info.pkt_len; 556 len -= pkt_info.pkt_len; 557 558 if (!len) 559 return INTEL_PT_NEED_MORE_BYTES; 560 561 ret = intel_pt_get_packet(buf, len, &pkt_info.packet); 562 if (!ret) 563 return INTEL_PT_NEED_MORE_BYTES; 564 if (ret < 0) 565 return ret; 566 567 pkt_info.pkt_len = ret; 568 } while (pkt_info.packet.type == INTEL_PT_PAD); 569 570 ret = cb(&pkt_info); 571 if (ret) 572 return 0; 573 574 pkt_info.last_packet_type = pkt_info.packet.type; 575 } 576 } 577 578 struct intel_pt_calc_cyc_to_tsc_info { 579 uint64_t cycle_cnt; 580 unsigned int cbr; 581 uint32_t last_mtc; 582 uint64_t ctc_timestamp; 583 uint64_t ctc_delta; 584 uint64_t tsc_timestamp; 585 uint64_t timestamp; 586 bool have_tma; 587 bool from_mtc; 588 double cbr_cyc_to_tsc; 589 }; 590 591 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 592 { 593 struct intel_pt_decoder *decoder = pkt_info->decoder; 594 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 595 uint64_t timestamp; 596 double cyc_to_tsc; 597 unsigned int cbr; 598 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 599 600 switch (pkt_info->packet.type) { 601 case INTEL_PT_TNT: 602 case INTEL_PT_TIP_PGE: 603 case INTEL_PT_TIP: 604 case INTEL_PT_FUP: 605 case INTEL_PT_PSB: 606 case INTEL_PT_PIP: 607 case INTEL_PT_MODE_EXEC: 608 case INTEL_PT_MODE_TSX: 609 case INTEL_PT_PSBEND: 610 case INTEL_PT_PAD: 611 case INTEL_PT_VMCS: 612 case INTEL_PT_MNT: 613 return 0; 614 615 case INTEL_PT_MTC: 616 if (!data->have_tma) 617 return 0; 618 619 mtc = pkt_info->packet.payload; 620 if (mtc > data->last_mtc) 621 mtc_delta = mtc - data->last_mtc; 622 else 623 mtc_delta = mtc + 256 - data->last_mtc; 624 data->ctc_delta += mtc_delta << decoder->mtc_shift; 625 data->last_mtc = mtc; 626 627 if (decoder->tsc_ctc_mult) { 628 timestamp = data->ctc_timestamp + 629 data->ctc_delta * decoder->tsc_ctc_mult; 630 } else { 631 timestamp = data->ctc_timestamp + 632 multdiv(data->ctc_delta, 633 decoder->tsc_ctc_ratio_n, 634 decoder->tsc_ctc_ratio_d); 635 } 636 637 if (timestamp < data->timestamp) 638 return 1; 639 640 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 641 data->timestamp = timestamp; 642 return 0; 643 } 644 645 break; 646 647 case INTEL_PT_TSC: 648 timestamp = pkt_info->packet.payload | 649 (data->timestamp & (0xffULL << 56)); 650 if (data->from_mtc && timestamp < data->timestamp && 651 data->timestamp - timestamp < decoder->tsc_slip) 652 return 1; 653 if (timestamp < data->timestamp) 654 timestamp += (1ULL << 56); 655 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 656 if (data->from_mtc) 657 return 1; 658 data->tsc_timestamp = timestamp; 659 data->timestamp = timestamp; 660 return 0; 661 } 662 break; 663 664 case INTEL_PT_TMA: 665 if (data->from_mtc) 666 return 1; 667 668 if (!decoder->tsc_ctc_ratio_d) 669 return 0; 670 671 ctc = pkt_info->packet.payload; 672 fc = pkt_info->packet.count; 673 ctc_rem = ctc & decoder->ctc_rem_mask; 674 675 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 676 677 data->ctc_timestamp = data->tsc_timestamp - fc; 678 if (decoder->tsc_ctc_mult) { 679 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 680 } else { 681 data->ctc_timestamp -= 682 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 683 decoder->tsc_ctc_ratio_d); 684 } 685 686 data->ctc_delta = 0; 687 data->have_tma = true; 688 689 return 0; 690 691 case INTEL_PT_CYC: 692 data->cycle_cnt += pkt_info->packet.payload; 693 return 0; 694 695 case INTEL_PT_CBR: 696 cbr = pkt_info->packet.payload; 697 if (data->cbr && data->cbr != cbr) 698 return 1; 699 data->cbr = cbr; 700 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 701 return 0; 702 703 case INTEL_PT_TIP_PGD: 704 case INTEL_PT_TRACESTOP: 705 case INTEL_PT_OVF: 706 case INTEL_PT_BAD: /* Does not happen */ 707 default: 708 return 1; 709 } 710 711 if (!data->cbr && decoder->cbr) { 712 data->cbr = decoder->cbr; 713 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 714 } 715 716 if (!data->cycle_cnt) 717 return 1; 718 719 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 720 721 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 722 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 723 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 724 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 725 return 1; 726 } 727 728 decoder->calc_cyc_to_tsc = cyc_to_tsc; 729 decoder->have_calc_cyc_to_tsc = true; 730 731 if (data->cbr) { 732 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 733 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 734 } else { 735 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 736 cyc_to_tsc, pkt_info->pos); 737 } 738 739 return 1; 740 } 741 742 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 743 bool from_mtc) 744 { 745 struct intel_pt_calc_cyc_to_tsc_info data = { 746 .cycle_cnt = 0, 747 .cbr = 0, 748 .last_mtc = decoder->last_mtc, 749 .ctc_timestamp = decoder->ctc_timestamp, 750 .ctc_delta = decoder->ctc_delta, 751 .tsc_timestamp = decoder->tsc_timestamp, 752 .timestamp = decoder->timestamp, 753 .have_tma = decoder->have_tma, 754 .from_mtc = from_mtc, 755 .cbr_cyc_to_tsc = 0, 756 }; 757 758 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 759 } 760 761 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 762 { 763 int ret; 764 765 decoder->last_packet_type = decoder->packet.type; 766 767 do { 768 decoder->pos += decoder->pkt_step; 769 decoder->buf += decoder->pkt_step; 770 decoder->len -= decoder->pkt_step; 771 772 if (!decoder->len) { 773 ret = intel_pt_get_next_data(decoder); 774 if (ret) 775 return ret; 776 } 777 778 ret = intel_pt_get_packet(decoder->buf, decoder->len, 779 &decoder->packet); 780 if (ret == INTEL_PT_NEED_MORE_BYTES && 781 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 782 ret = intel_pt_get_split_packet(decoder); 783 if (ret < 0) 784 return ret; 785 } 786 if (ret <= 0) 787 return intel_pt_bad_packet(decoder); 788 789 decoder->pkt_len = ret; 790 decoder->pkt_step = ret; 791 intel_pt_decoder_log_packet(decoder); 792 } while (decoder->packet.type == INTEL_PT_PAD); 793 794 return 0; 795 } 796 797 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 798 { 799 uint64_t timestamp, masked_timestamp; 800 801 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 802 masked_timestamp = timestamp & decoder->period_mask; 803 if (decoder->continuous_period) { 804 if (masked_timestamp != decoder->last_masked_timestamp) 805 return 1; 806 } else { 807 timestamp += 1; 808 masked_timestamp = timestamp & decoder->period_mask; 809 if (masked_timestamp != decoder->last_masked_timestamp) { 810 decoder->last_masked_timestamp = masked_timestamp; 811 decoder->continuous_period = true; 812 } 813 } 814 return decoder->period_ticks - (timestamp - masked_timestamp); 815 } 816 817 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 818 { 819 switch (decoder->period_type) { 820 case INTEL_PT_PERIOD_INSTRUCTIONS: 821 return decoder->period - decoder->period_insn_cnt; 822 case INTEL_PT_PERIOD_TICKS: 823 return intel_pt_next_period(decoder); 824 case INTEL_PT_PERIOD_NONE: 825 case INTEL_PT_PERIOD_MTC: 826 default: 827 return 0; 828 } 829 } 830 831 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 832 { 833 uint64_t timestamp, masked_timestamp; 834 835 switch (decoder->period_type) { 836 case INTEL_PT_PERIOD_INSTRUCTIONS: 837 decoder->period_insn_cnt = 0; 838 break; 839 case INTEL_PT_PERIOD_TICKS: 840 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 841 masked_timestamp = timestamp & decoder->period_mask; 842 decoder->last_masked_timestamp = masked_timestamp; 843 break; 844 case INTEL_PT_PERIOD_NONE: 845 case INTEL_PT_PERIOD_MTC: 846 default: 847 break; 848 } 849 850 decoder->state.type |= INTEL_PT_INSTRUCTION; 851 } 852 853 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 854 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 855 { 856 uint64_t max_insn_cnt, insn_cnt = 0; 857 int err; 858 859 if (!decoder->mtc_insn) 860 decoder->mtc_insn = true; 861 862 max_insn_cnt = intel_pt_next_sample(decoder); 863 864 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 865 max_insn_cnt, decoder->data); 866 867 decoder->tot_insn_cnt += insn_cnt; 868 decoder->timestamp_insn_cnt += insn_cnt; 869 decoder->period_insn_cnt += insn_cnt; 870 871 if (err) { 872 decoder->no_progress = 0; 873 decoder->pkt_state = INTEL_PT_STATE_ERR2; 874 intel_pt_log_at("ERROR: Failed to get instruction", 875 decoder->ip); 876 if (err == -ENOENT) 877 return -ENOLINK; 878 return -EILSEQ; 879 } 880 881 if (ip && decoder->ip == ip) { 882 err = -EAGAIN; 883 goto out; 884 } 885 886 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 887 intel_pt_sample_insn(decoder); 888 889 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 890 decoder->state.type = INTEL_PT_INSTRUCTION; 891 decoder->state.from_ip = decoder->ip; 892 decoder->state.to_ip = 0; 893 decoder->ip += intel_pt_insn->length; 894 err = INTEL_PT_RETURN; 895 goto out; 896 } 897 898 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 899 /* Zero-length calls are excluded */ 900 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 901 intel_pt_insn->rel) { 902 err = intel_pt_push(&decoder->stack, decoder->ip + 903 intel_pt_insn->length); 904 if (err) 905 goto out; 906 } 907 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 908 decoder->ret_addr = intel_pt_pop(&decoder->stack); 909 } 910 911 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 912 int cnt = decoder->no_progress++; 913 914 decoder->state.from_ip = decoder->ip; 915 decoder->ip += intel_pt_insn->length + 916 intel_pt_insn->rel; 917 decoder->state.to_ip = decoder->ip; 918 err = INTEL_PT_RETURN; 919 920 /* 921 * Check for being stuck in a loop. This can happen if a 922 * decoder error results in the decoder erroneously setting the 923 * ip to an address that is itself in an infinite loop that 924 * consumes no packets. When that happens, there must be an 925 * unconditional branch. 926 */ 927 if (cnt) { 928 if (cnt == 1) { 929 decoder->stuck_ip = decoder->state.to_ip; 930 decoder->stuck_ip_prd = 1; 931 decoder->stuck_ip_cnt = 1; 932 } else if (cnt > INTEL_PT_MAX_LOOPS || 933 decoder->state.to_ip == decoder->stuck_ip) { 934 intel_pt_log_at("ERROR: Never-ending loop", 935 decoder->state.to_ip); 936 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 937 err = -ELOOP; 938 goto out; 939 } else if (!--decoder->stuck_ip_cnt) { 940 decoder->stuck_ip_prd += 1; 941 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 942 decoder->stuck_ip = decoder->state.to_ip; 943 } 944 } 945 goto out_no_progress; 946 } 947 out: 948 decoder->no_progress = 0; 949 out_no_progress: 950 decoder->state.insn_op = intel_pt_insn->op; 951 decoder->state.insn_len = intel_pt_insn->length; 952 953 if (decoder->tx_flags & INTEL_PT_IN_TX) 954 decoder->state.flags |= INTEL_PT_IN_TX; 955 956 return err; 957 } 958 959 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 960 { 961 struct intel_pt_insn intel_pt_insn; 962 uint64_t ip; 963 int err; 964 965 ip = decoder->last_ip; 966 967 while (1) { 968 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 969 if (err == INTEL_PT_RETURN) 970 return 0; 971 if (err == -EAGAIN) { 972 if (decoder->set_fup_tx_flags) { 973 decoder->set_fup_tx_flags = false; 974 decoder->tx_flags = decoder->fup_tx_flags; 975 decoder->state.type = INTEL_PT_TRANSACTION; 976 decoder->state.from_ip = decoder->ip; 977 decoder->state.to_ip = 0; 978 decoder->state.flags = decoder->fup_tx_flags; 979 return 0; 980 } 981 return err; 982 } 983 decoder->set_fup_tx_flags = false; 984 if (err) 985 return err; 986 987 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 988 intel_pt_log_at("ERROR: Unexpected indirect branch", 989 decoder->ip); 990 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 991 return -ENOENT; 992 } 993 994 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 995 intel_pt_log_at("ERROR: Unexpected conditional branch", 996 decoder->ip); 997 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 998 return -ENOENT; 999 } 1000 1001 intel_pt_bug(decoder); 1002 } 1003 } 1004 1005 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1006 { 1007 struct intel_pt_insn intel_pt_insn; 1008 int err; 1009 1010 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1011 if (err == INTEL_PT_RETURN) 1012 return 0; 1013 if (err) 1014 return err; 1015 1016 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1017 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1018 decoder->pge = false; 1019 decoder->continuous_period = false; 1020 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1021 decoder->state.from_ip = decoder->ip; 1022 decoder->state.to_ip = 0; 1023 if (decoder->packet.count != 0) 1024 decoder->ip = decoder->last_ip; 1025 } else { 1026 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1027 decoder->state.from_ip = decoder->ip; 1028 if (decoder->packet.count == 0) { 1029 decoder->state.to_ip = 0; 1030 } else { 1031 decoder->state.to_ip = decoder->last_ip; 1032 decoder->ip = decoder->last_ip; 1033 } 1034 } 1035 return 0; 1036 } 1037 1038 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1039 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1040 decoder->ip); 1041 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1042 return -ENOENT; 1043 } 1044 1045 return intel_pt_bug(decoder); 1046 } 1047 1048 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1049 { 1050 struct intel_pt_insn intel_pt_insn; 1051 int err; 1052 1053 while (1) { 1054 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1055 if (err == INTEL_PT_RETURN) 1056 return 0; 1057 if (err) 1058 return err; 1059 1060 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1061 if (!decoder->return_compression) { 1062 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1063 decoder->ip); 1064 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1065 return -ENOENT; 1066 } 1067 if (!decoder->ret_addr) { 1068 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1069 decoder->ip); 1070 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1071 return -ENOENT; 1072 } 1073 if (!(decoder->tnt.payload & BIT63)) { 1074 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1075 decoder->ip); 1076 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1077 return -ENOENT; 1078 } 1079 decoder->tnt.count -= 1; 1080 if (!decoder->tnt.count) 1081 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1082 decoder->tnt.payload <<= 1; 1083 decoder->state.from_ip = decoder->ip; 1084 decoder->ip = decoder->ret_addr; 1085 decoder->state.to_ip = decoder->ip; 1086 return 0; 1087 } 1088 1089 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1090 /* Handle deferred TIPs */ 1091 err = intel_pt_get_next_packet(decoder); 1092 if (err) 1093 return err; 1094 if (decoder->packet.type != INTEL_PT_TIP || 1095 decoder->packet.count == 0) { 1096 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1097 decoder->ip); 1098 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1099 decoder->pkt_step = 0; 1100 return -ENOENT; 1101 } 1102 intel_pt_set_last_ip(decoder); 1103 decoder->state.from_ip = decoder->ip; 1104 decoder->state.to_ip = decoder->last_ip; 1105 decoder->ip = decoder->last_ip; 1106 return 0; 1107 } 1108 1109 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1110 decoder->tnt.count -= 1; 1111 if (!decoder->tnt.count) 1112 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1113 if (decoder->tnt.payload & BIT63) { 1114 decoder->tnt.payload <<= 1; 1115 decoder->state.from_ip = decoder->ip; 1116 decoder->ip += intel_pt_insn.length + 1117 intel_pt_insn.rel; 1118 decoder->state.to_ip = decoder->ip; 1119 return 0; 1120 } 1121 /* Instruction sample for a non-taken branch */ 1122 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1123 decoder->tnt.payload <<= 1; 1124 decoder->state.type = INTEL_PT_INSTRUCTION; 1125 decoder->state.from_ip = decoder->ip; 1126 decoder->state.to_ip = 0; 1127 decoder->ip += intel_pt_insn.length; 1128 return 0; 1129 } 1130 decoder->ip += intel_pt_insn.length; 1131 if (!decoder->tnt.count) 1132 return -EAGAIN; 1133 decoder->tnt.payload <<= 1; 1134 continue; 1135 } 1136 1137 return intel_pt_bug(decoder); 1138 } 1139 } 1140 1141 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1142 { 1143 unsigned int fup_tx_flags; 1144 int err; 1145 1146 fup_tx_flags = decoder->packet.payload & 1147 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1148 err = intel_pt_get_next_packet(decoder); 1149 if (err) 1150 return err; 1151 if (decoder->packet.type == INTEL_PT_FUP) { 1152 decoder->fup_tx_flags = fup_tx_flags; 1153 decoder->set_fup_tx_flags = true; 1154 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1155 *no_tip = true; 1156 } else { 1157 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1158 decoder->pos); 1159 intel_pt_update_in_tx(decoder); 1160 } 1161 return 0; 1162 } 1163 1164 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1165 { 1166 uint64_t timestamp; 1167 1168 decoder->have_tma = false; 1169 1170 if (decoder->ref_timestamp) { 1171 timestamp = decoder->packet.payload | 1172 (decoder->ref_timestamp & (0xffULL << 56)); 1173 if (timestamp < decoder->ref_timestamp) { 1174 if (decoder->ref_timestamp - timestamp > (1ULL << 55)) 1175 timestamp += (1ULL << 56); 1176 } else { 1177 if (timestamp - decoder->ref_timestamp > (1ULL << 55)) 1178 timestamp -= (1ULL << 56); 1179 } 1180 decoder->tsc_timestamp = timestamp; 1181 decoder->timestamp = timestamp; 1182 decoder->ref_timestamp = 0; 1183 decoder->timestamp_insn_cnt = 0; 1184 } else if (decoder->timestamp) { 1185 timestamp = decoder->packet.payload | 1186 (decoder->timestamp & (0xffULL << 56)); 1187 decoder->tsc_timestamp = timestamp; 1188 if (timestamp < decoder->timestamp && 1189 decoder->timestamp - timestamp < decoder->tsc_slip) { 1190 intel_pt_log_to("Suppressing backwards timestamp", 1191 timestamp); 1192 timestamp = decoder->timestamp; 1193 } 1194 if (timestamp < decoder->timestamp) { 1195 intel_pt_log_to("Wraparound timestamp", timestamp); 1196 timestamp += (1ULL << 56); 1197 decoder->tsc_timestamp = timestamp; 1198 } 1199 decoder->timestamp = timestamp; 1200 decoder->timestamp_insn_cnt = 0; 1201 } 1202 1203 if (decoder->last_packet_type == INTEL_PT_CYC) { 1204 decoder->cyc_ref_timestamp = decoder->timestamp; 1205 decoder->cycle_cnt = 0; 1206 decoder->have_calc_cyc_to_tsc = false; 1207 intel_pt_calc_cyc_to_tsc(decoder, false); 1208 } 1209 1210 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1211 } 1212 1213 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1214 { 1215 intel_pt_log("ERROR: Buffer overflow\n"); 1216 intel_pt_clear_tx_flags(decoder); 1217 decoder->have_tma = false; 1218 decoder->cbr = 0; 1219 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1220 decoder->overflow = true; 1221 return -EOVERFLOW; 1222 } 1223 1224 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1225 { 1226 uint32_t ctc = decoder->packet.payload; 1227 uint32_t fc = decoder->packet.count; 1228 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1229 1230 if (!decoder->tsc_ctc_ratio_d) 1231 return; 1232 1233 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1234 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1235 if (decoder->tsc_ctc_mult) { 1236 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1237 } else { 1238 decoder->ctc_timestamp -= multdiv(ctc_rem, 1239 decoder->tsc_ctc_ratio_n, 1240 decoder->tsc_ctc_ratio_d); 1241 } 1242 decoder->ctc_delta = 0; 1243 decoder->have_tma = true; 1244 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1245 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1246 } 1247 1248 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1249 { 1250 uint64_t timestamp; 1251 uint32_t mtc, mtc_delta; 1252 1253 if (!decoder->have_tma) 1254 return; 1255 1256 mtc = decoder->packet.payload; 1257 1258 if (mtc > decoder->last_mtc) 1259 mtc_delta = mtc - decoder->last_mtc; 1260 else 1261 mtc_delta = mtc + 256 - decoder->last_mtc; 1262 1263 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1264 1265 if (decoder->tsc_ctc_mult) { 1266 timestamp = decoder->ctc_timestamp + 1267 decoder->ctc_delta * decoder->tsc_ctc_mult; 1268 } else { 1269 timestamp = decoder->ctc_timestamp + 1270 multdiv(decoder->ctc_delta, 1271 decoder->tsc_ctc_ratio_n, 1272 decoder->tsc_ctc_ratio_d); 1273 } 1274 1275 if (timestamp < decoder->timestamp) 1276 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1277 timestamp, decoder->timestamp); 1278 else 1279 decoder->timestamp = timestamp; 1280 1281 decoder->timestamp_insn_cnt = 0; 1282 decoder->last_mtc = mtc; 1283 1284 if (decoder->last_packet_type == INTEL_PT_CYC) { 1285 decoder->cyc_ref_timestamp = decoder->timestamp; 1286 decoder->cycle_cnt = 0; 1287 decoder->have_calc_cyc_to_tsc = false; 1288 intel_pt_calc_cyc_to_tsc(decoder, true); 1289 } 1290 } 1291 1292 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1293 { 1294 unsigned int cbr = decoder->packet.payload; 1295 1296 if (decoder->cbr == cbr) 1297 return; 1298 1299 decoder->cbr = cbr; 1300 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1301 } 1302 1303 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1304 { 1305 uint64_t timestamp = decoder->cyc_ref_timestamp; 1306 1307 decoder->have_cyc = true; 1308 1309 decoder->cycle_cnt += decoder->packet.payload; 1310 1311 if (!decoder->cyc_ref_timestamp) 1312 return; 1313 1314 if (decoder->have_calc_cyc_to_tsc) 1315 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1316 else if (decoder->cbr) 1317 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1318 else 1319 return; 1320 1321 if (timestamp < decoder->timestamp) 1322 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1323 timestamp, decoder->timestamp); 1324 else 1325 decoder->timestamp = timestamp; 1326 } 1327 1328 /* Walk PSB+ packets when already in sync. */ 1329 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1330 { 1331 int err; 1332 1333 while (1) { 1334 err = intel_pt_get_next_packet(decoder); 1335 if (err) 1336 return err; 1337 1338 switch (decoder->packet.type) { 1339 case INTEL_PT_PSBEND: 1340 return 0; 1341 1342 case INTEL_PT_TIP_PGD: 1343 case INTEL_PT_TIP_PGE: 1344 case INTEL_PT_TIP: 1345 case INTEL_PT_TNT: 1346 case INTEL_PT_TRACESTOP: 1347 case INTEL_PT_BAD: 1348 case INTEL_PT_PSB: 1349 decoder->have_tma = false; 1350 intel_pt_log("ERROR: Unexpected packet\n"); 1351 return -EAGAIN; 1352 1353 case INTEL_PT_OVF: 1354 return intel_pt_overflow(decoder); 1355 1356 case INTEL_PT_TSC: 1357 intel_pt_calc_tsc_timestamp(decoder); 1358 break; 1359 1360 case INTEL_PT_TMA: 1361 intel_pt_calc_tma(decoder); 1362 break; 1363 1364 case INTEL_PT_CBR: 1365 intel_pt_calc_cbr(decoder); 1366 break; 1367 1368 case INTEL_PT_MODE_EXEC: 1369 decoder->exec_mode = decoder->packet.payload; 1370 break; 1371 1372 case INTEL_PT_PIP: 1373 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1374 break; 1375 1376 case INTEL_PT_FUP: 1377 decoder->pge = true; 1378 intel_pt_set_last_ip(decoder); 1379 break; 1380 1381 case INTEL_PT_MODE_TSX: 1382 intel_pt_update_in_tx(decoder); 1383 break; 1384 1385 case INTEL_PT_MTC: 1386 intel_pt_calc_mtc_timestamp(decoder); 1387 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1388 decoder->state.type |= INTEL_PT_INSTRUCTION; 1389 break; 1390 1391 case INTEL_PT_CYC: 1392 case INTEL_PT_VMCS: 1393 case INTEL_PT_MNT: 1394 case INTEL_PT_PAD: 1395 default: 1396 break; 1397 } 1398 } 1399 } 1400 1401 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1402 { 1403 int err; 1404 1405 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1406 decoder->tx_flags = 0; 1407 decoder->state.flags &= ~INTEL_PT_IN_TX; 1408 decoder->state.flags |= INTEL_PT_ABORT_TX; 1409 } else { 1410 decoder->state.flags |= INTEL_PT_ASYNC; 1411 } 1412 1413 while (1) { 1414 err = intel_pt_get_next_packet(decoder); 1415 if (err) 1416 return err; 1417 1418 switch (decoder->packet.type) { 1419 case INTEL_PT_TNT: 1420 case INTEL_PT_FUP: 1421 case INTEL_PT_TRACESTOP: 1422 case INTEL_PT_PSB: 1423 case INTEL_PT_TSC: 1424 case INTEL_PT_TMA: 1425 case INTEL_PT_CBR: 1426 case INTEL_PT_MODE_TSX: 1427 case INTEL_PT_BAD: 1428 case INTEL_PT_PSBEND: 1429 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1430 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1431 return -ENOENT; 1432 1433 case INTEL_PT_OVF: 1434 return intel_pt_overflow(decoder); 1435 1436 case INTEL_PT_TIP_PGD: 1437 decoder->state.from_ip = decoder->ip; 1438 decoder->state.to_ip = 0; 1439 if (decoder->packet.count != 0) { 1440 intel_pt_set_ip(decoder); 1441 intel_pt_log("Omitting PGD ip " x64_fmt "\n", 1442 decoder->ip); 1443 } 1444 decoder->pge = false; 1445 decoder->continuous_period = false; 1446 return 0; 1447 1448 case INTEL_PT_TIP_PGE: 1449 decoder->pge = true; 1450 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1451 decoder->ip); 1452 decoder->state.from_ip = 0; 1453 if (decoder->packet.count == 0) { 1454 decoder->state.to_ip = 0; 1455 } else { 1456 intel_pt_set_ip(decoder); 1457 decoder->state.to_ip = decoder->ip; 1458 } 1459 return 0; 1460 1461 case INTEL_PT_TIP: 1462 decoder->state.from_ip = decoder->ip; 1463 if (decoder->packet.count == 0) { 1464 decoder->state.to_ip = 0; 1465 } else { 1466 intel_pt_set_ip(decoder); 1467 decoder->state.to_ip = decoder->ip; 1468 } 1469 return 0; 1470 1471 case INTEL_PT_PIP: 1472 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1473 break; 1474 1475 case INTEL_PT_MTC: 1476 intel_pt_calc_mtc_timestamp(decoder); 1477 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1478 decoder->state.type |= INTEL_PT_INSTRUCTION; 1479 break; 1480 1481 case INTEL_PT_CYC: 1482 intel_pt_calc_cyc_timestamp(decoder); 1483 break; 1484 1485 case INTEL_PT_MODE_EXEC: 1486 decoder->exec_mode = decoder->packet.payload; 1487 break; 1488 1489 case INTEL_PT_VMCS: 1490 case INTEL_PT_MNT: 1491 case INTEL_PT_PAD: 1492 break; 1493 1494 default: 1495 return intel_pt_bug(decoder); 1496 } 1497 } 1498 } 1499 1500 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 1501 { 1502 bool no_tip = false; 1503 int err; 1504 1505 while (1) { 1506 err = intel_pt_get_next_packet(decoder); 1507 if (err) 1508 return err; 1509 next: 1510 switch (decoder->packet.type) { 1511 case INTEL_PT_TNT: 1512 if (!decoder->packet.count) 1513 break; 1514 decoder->tnt = decoder->packet; 1515 decoder->pkt_state = INTEL_PT_STATE_TNT; 1516 err = intel_pt_walk_tnt(decoder); 1517 if (err == -EAGAIN) 1518 break; 1519 return err; 1520 1521 case INTEL_PT_TIP_PGD: 1522 if (decoder->packet.count != 0) 1523 intel_pt_set_last_ip(decoder); 1524 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 1525 return intel_pt_walk_tip(decoder); 1526 1527 case INTEL_PT_TIP_PGE: { 1528 decoder->pge = true; 1529 if (decoder->packet.count == 0) { 1530 intel_pt_log_at("Skipping zero TIP.PGE", 1531 decoder->pos); 1532 break; 1533 } 1534 intel_pt_set_ip(decoder); 1535 decoder->state.from_ip = 0; 1536 decoder->state.to_ip = decoder->ip; 1537 return 0; 1538 } 1539 1540 case INTEL_PT_OVF: 1541 return intel_pt_overflow(decoder); 1542 1543 case INTEL_PT_TIP: 1544 if (decoder->packet.count != 0) 1545 intel_pt_set_last_ip(decoder); 1546 decoder->pkt_state = INTEL_PT_STATE_TIP; 1547 return intel_pt_walk_tip(decoder); 1548 1549 case INTEL_PT_FUP: 1550 if (decoder->packet.count == 0) { 1551 intel_pt_log_at("Skipping zero FUP", 1552 decoder->pos); 1553 no_tip = false; 1554 break; 1555 } 1556 intel_pt_set_last_ip(decoder); 1557 err = intel_pt_walk_fup(decoder); 1558 if (err != -EAGAIN) { 1559 if (err) 1560 return err; 1561 if (no_tip) 1562 decoder->pkt_state = 1563 INTEL_PT_STATE_FUP_NO_TIP; 1564 else 1565 decoder->pkt_state = INTEL_PT_STATE_FUP; 1566 return 0; 1567 } 1568 if (no_tip) { 1569 no_tip = false; 1570 break; 1571 } 1572 return intel_pt_walk_fup_tip(decoder); 1573 1574 case INTEL_PT_TRACESTOP: 1575 decoder->pge = false; 1576 decoder->continuous_period = false; 1577 intel_pt_clear_tx_flags(decoder); 1578 decoder->have_tma = false; 1579 break; 1580 1581 case INTEL_PT_PSB: 1582 intel_pt_clear_stack(&decoder->stack); 1583 err = intel_pt_walk_psbend(decoder); 1584 if (err == -EAGAIN) 1585 goto next; 1586 if (err) 1587 return err; 1588 break; 1589 1590 case INTEL_PT_PIP: 1591 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1592 break; 1593 1594 case INTEL_PT_MTC: 1595 intel_pt_calc_mtc_timestamp(decoder); 1596 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 1597 break; 1598 /* 1599 * Ensure that there has been an instruction since the 1600 * last MTC. 1601 */ 1602 if (!decoder->mtc_insn) 1603 break; 1604 decoder->mtc_insn = false; 1605 /* Ensure that there is a timestamp */ 1606 if (!decoder->timestamp) 1607 break; 1608 decoder->state.type = INTEL_PT_INSTRUCTION; 1609 decoder->state.from_ip = decoder->ip; 1610 decoder->state.to_ip = 0; 1611 decoder->mtc_insn = false; 1612 return 0; 1613 1614 case INTEL_PT_TSC: 1615 intel_pt_calc_tsc_timestamp(decoder); 1616 break; 1617 1618 case INTEL_PT_TMA: 1619 intel_pt_calc_tma(decoder); 1620 break; 1621 1622 case INTEL_PT_CYC: 1623 intel_pt_calc_cyc_timestamp(decoder); 1624 break; 1625 1626 case INTEL_PT_CBR: 1627 intel_pt_calc_cbr(decoder); 1628 break; 1629 1630 case INTEL_PT_MODE_EXEC: 1631 decoder->exec_mode = decoder->packet.payload; 1632 break; 1633 1634 case INTEL_PT_MODE_TSX: 1635 /* MODE_TSX need not be followed by FUP */ 1636 if (!decoder->pge) { 1637 intel_pt_update_in_tx(decoder); 1638 break; 1639 } 1640 err = intel_pt_mode_tsx(decoder, &no_tip); 1641 if (err) 1642 return err; 1643 goto next; 1644 1645 case INTEL_PT_BAD: /* Does not happen */ 1646 return intel_pt_bug(decoder); 1647 1648 case INTEL_PT_PSBEND: 1649 case INTEL_PT_VMCS: 1650 case INTEL_PT_MNT: 1651 case INTEL_PT_PAD: 1652 break; 1653 1654 default: 1655 return intel_pt_bug(decoder); 1656 } 1657 } 1658 } 1659 1660 /* Walk PSB+ packets to get in sync. */ 1661 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 1662 { 1663 int err; 1664 1665 while (1) { 1666 err = intel_pt_get_next_packet(decoder); 1667 if (err) 1668 return err; 1669 1670 switch (decoder->packet.type) { 1671 case INTEL_PT_TIP_PGD: 1672 decoder->continuous_period = false; 1673 case INTEL_PT_TIP_PGE: 1674 case INTEL_PT_TIP: 1675 intel_pt_log("ERROR: Unexpected packet\n"); 1676 return -ENOENT; 1677 1678 case INTEL_PT_FUP: 1679 decoder->pge = true; 1680 if (decoder->last_ip || decoder->packet.count == 6 || 1681 decoder->packet.count == 0) { 1682 uint64_t current_ip = decoder->ip; 1683 1684 intel_pt_set_ip(decoder); 1685 if (current_ip) 1686 intel_pt_log_to("Setting IP", 1687 decoder->ip); 1688 } 1689 break; 1690 1691 case INTEL_PT_MTC: 1692 intel_pt_calc_mtc_timestamp(decoder); 1693 break; 1694 1695 case INTEL_PT_TSC: 1696 intel_pt_calc_tsc_timestamp(decoder); 1697 break; 1698 1699 case INTEL_PT_TMA: 1700 intel_pt_calc_tma(decoder); 1701 break; 1702 1703 case INTEL_PT_CYC: 1704 intel_pt_calc_cyc_timestamp(decoder); 1705 break; 1706 1707 case INTEL_PT_CBR: 1708 intel_pt_calc_cbr(decoder); 1709 break; 1710 1711 case INTEL_PT_PIP: 1712 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1713 break; 1714 1715 case INTEL_PT_MODE_EXEC: 1716 decoder->exec_mode = decoder->packet.payload; 1717 break; 1718 1719 case INTEL_PT_MODE_TSX: 1720 intel_pt_update_in_tx(decoder); 1721 break; 1722 1723 case INTEL_PT_TRACESTOP: 1724 decoder->pge = false; 1725 decoder->continuous_period = false; 1726 intel_pt_clear_tx_flags(decoder); 1727 case INTEL_PT_TNT: 1728 decoder->have_tma = false; 1729 intel_pt_log("ERROR: Unexpected packet\n"); 1730 if (decoder->ip) 1731 decoder->pkt_state = INTEL_PT_STATE_ERR4; 1732 else 1733 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1734 return -ENOENT; 1735 1736 case INTEL_PT_BAD: /* Does not happen */ 1737 return intel_pt_bug(decoder); 1738 1739 case INTEL_PT_OVF: 1740 return intel_pt_overflow(decoder); 1741 1742 case INTEL_PT_PSBEND: 1743 return 0; 1744 1745 case INTEL_PT_PSB: 1746 case INTEL_PT_VMCS: 1747 case INTEL_PT_MNT: 1748 case INTEL_PT_PAD: 1749 default: 1750 break; 1751 } 1752 } 1753 } 1754 1755 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 1756 { 1757 int err; 1758 1759 while (1) { 1760 err = intel_pt_get_next_packet(decoder); 1761 if (err) 1762 return err; 1763 1764 switch (decoder->packet.type) { 1765 case INTEL_PT_TIP_PGD: 1766 decoder->continuous_period = false; 1767 case INTEL_PT_TIP_PGE: 1768 case INTEL_PT_TIP: 1769 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD; 1770 if (decoder->last_ip || decoder->packet.count == 6 || 1771 decoder->packet.count == 0) 1772 intel_pt_set_ip(decoder); 1773 if (decoder->ip) 1774 return 0; 1775 break; 1776 1777 case INTEL_PT_FUP: 1778 if (decoder->overflow) { 1779 if (decoder->last_ip || 1780 decoder->packet.count == 6 || 1781 decoder->packet.count == 0) 1782 intel_pt_set_ip(decoder); 1783 if (decoder->ip) 1784 return 0; 1785 } 1786 if (decoder->packet.count) 1787 intel_pt_set_last_ip(decoder); 1788 break; 1789 1790 case INTEL_PT_MTC: 1791 intel_pt_calc_mtc_timestamp(decoder); 1792 break; 1793 1794 case INTEL_PT_TSC: 1795 intel_pt_calc_tsc_timestamp(decoder); 1796 break; 1797 1798 case INTEL_PT_TMA: 1799 intel_pt_calc_tma(decoder); 1800 break; 1801 1802 case INTEL_PT_CYC: 1803 intel_pt_calc_cyc_timestamp(decoder); 1804 break; 1805 1806 case INTEL_PT_CBR: 1807 intel_pt_calc_cbr(decoder); 1808 break; 1809 1810 case INTEL_PT_PIP: 1811 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1812 break; 1813 1814 case INTEL_PT_MODE_EXEC: 1815 decoder->exec_mode = decoder->packet.payload; 1816 break; 1817 1818 case INTEL_PT_MODE_TSX: 1819 intel_pt_update_in_tx(decoder); 1820 break; 1821 1822 case INTEL_PT_OVF: 1823 return intel_pt_overflow(decoder); 1824 1825 case INTEL_PT_BAD: /* Does not happen */ 1826 return intel_pt_bug(decoder); 1827 1828 case INTEL_PT_TRACESTOP: 1829 decoder->pge = false; 1830 decoder->continuous_period = false; 1831 intel_pt_clear_tx_flags(decoder); 1832 decoder->have_tma = false; 1833 break; 1834 1835 case INTEL_PT_PSB: 1836 err = intel_pt_walk_psb(decoder); 1837 if (err) 1838 return err; 1839 if (decoder->ip) { 1840 /* Do not have a sample */ 1841 decoder->state.type = 0; 1842 return 0; 1843 } 1844 break; 1845 1846 case INTEL_PT_TNT: 1847 case INTEL_PT_PSBEND: 1848 case INTEL_PT_VMCS: 1849 case INTEL_PT_MNT: 1850 case INTEL_PT_PAD: 1851 default: 1852 break; 1853 } 1854 } 1855 } 1856 1857 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 1858 { 1859 int err; 1860 1861 intel_pt_log("Scanning for full IP\n"); 1862 err = intel_pt_walk_to_ip(decoder); 1863 if (err) 1864 return err; 1865 1866 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1867 decoder->overflow = false; 1868 1869 decoder->state.from_ip = 0; 1870 decoder->state.to_ip = decoder->ip; 1871 intel_pt_log_to("Setting IP", decoder->ip); 1872 1873 return 0; 1874 } 1875 1876 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 1877 { 1878 const unsigned char *end = decoder->buf + decoder->len; 1879 size_t i; 1880 1881 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 1882 if (i > decoder->len) 1883 continue; 1884 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 1885 return i; 1886 } 1887 return 0; 1888 } 1889 1890 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 1891 { 1892 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 1893 const char *psb = INTEL_PT_PSB_STR; 1894 1895 if (rest_psb > decoder->len || 1896 memcmp(decoder->buf, psb + part_psb, rest_psb)) 1897 return 0; 1898 1899 return rest_psb; 1900 } 1901 1902 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 1903 int part_psb) 1904 { 1905 int rest_psb, ret; 1906 1907 decoder->pos += decoder->len; 1908 decoder->len = 0; 1909 1910 ret = intel_pt_get_next_data(decoder); 1911 if (ret) 1912 return ret; 1913 1914 rest_psb = intel_pt_rest_psb(decoder, part_psb); 1915 if (!rest_psb) 1916 return 0; 1917 1918 decoder->pos -= part_psb; 1919 decoder->next_buf = decoder->buf + rest_psb; 1920 decoder->next_len = decoder->len - rest_psb; 1921 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 1922 decoder->buf = decoder->temp_buf; 1923 decoder->len = INTEL_PT_PSB_LEN; 1924 1925 return 0; 1926 } 1927 1928 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 1929 { 1930 unsigned char *next; 1931 int ret; 1932 1933 intel_pt_log("Scanning for PSB\n"); 1934 while (1) { 1935 if (!decoder->len) { 1936 ret = intel_pt_get_next_data(decoder); 1937 if (ret) 1938 return ret; 1939 } 1940 1941 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 1942 INTEL_PT_PSB_LEN); 1943 if (!next) { 1944 int part_psb; 1945 1946 part_psb = intel_pt_part_psb(decoder); 1947 if (part_psb) { 1948 ret = intel_pt_get_split_psb(decoder, part_psb); 1949 if (ret) 1950 return ret; 1951 } else { 1952 decoder->pos += decoder->len; 1953 decoder->len = 0; 1954 } 1955 continue; 1956 } 1957 1958 decoder->pkt_step = next - decoder->buf; 1959 return intel_pt_get_next_packet(decoder); 1960 } 1961 } 1962 1963 static int intel_pt_sync(struct intel_pt_decoder *decoder) 1964 { 1965 int err; 1966 1967 decoder->pge = false; 1968 decoder->continuous_period = false; 1969 decoder->last_ip = 0; 1970 decoder->ip = 0; 1971 intel_pt_clear_stack(&decoder->stack); 1972 1973 err = intel_pt_scan_for_psb(decoder); 1974 if (err) 1975 return err; 1976 1977 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 1978 1979 err = intel_pt_walk_psb(decoder); 1980 if (err) 1981 return err; 1982 1983 if (decoder->ip) { 1984 decoder->state.type = 0; /* Do not have a sample */ 1985 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1986 } else { 1987 return intel_pt_sync_ip(decoder); 1988 } 1989 1990 return 0; 1991 } 1992 1993 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 1994 { 1995 uint64_t est = decoder->timestamp_insn_cnt << 1; 1996 1997 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 1998 goto out; 1999 2000 est *= decoder->max_non_turbo_ratio; 2001 est /= decoder->cbr; 2002 out: 2003 return decoder->timestamp + est; 2004 } 2005 2006 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2007 { 2008 int err; 2009 2010 do { 2011 decoder->state.type = INTEL_PT_BRANCH; 2012 decoder->state.flags = 0; 2013 2014 switch (decoder->pkt_state) { 2015 case INTEL_PT_STATE_NO_PSB: 2016 err = intel_pt_sync(decoder); 2017 break; 2018 case INTEL_PT_STATE_NO_IP: 2019 decoder->last_ip = 0; 2020 /* Fall through */ 2021 case INTEL_PT_STATE_ERR_RESYNC: 2022 err = intel_pt_sync_ip(decoder); 2023 break; 2024 case INTEL_PT_STATE_IN_SYNC: 2025 err = intel_pt_walk_trace(decoder); 2026 break; 2027 case INTEL_PT_STATE_TNT: 2028 err = intel_pt_walk_tnt(decoder); 2029 if (err == -EAGAIN) 2030 err = intel_pt_walk_trace(decoder); 2031 break; 2032 case INTEL_PT_STATE_TIP: 2033 case INTEL_PT_STATE_TIP_PGD: 2034 err = intel_pt_walk_tip(decoder); 2035 break; 2036 case INTEL_PT_STATE_FUP: 2037 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2038 err = intel_pt_walk_fup(decoder); 2039 if (err == -EAGAIN) 2040 err = intel_pt_walk_fup_tip(decoder); 2041 else if (!err) 2042 decoder->pkt_state = INTEL_PT_STATE_FUP; 2043 break; 2044 case INTEL_PT_STATE_FUP_NO_TIP: 2045 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2046 err = intel_pt_walk_fup(decoder); 2047 if (err == -EAGAIN) 2048 err = intel_pt_walk_trace(decoder); 2049 break; 2050 default: 2051 err = intel_pt_bug(decoder); 2052 break; 2053 } 2054 } while (err == -ENOLINK); 2055 2056 decoder->state.err = err ? intel_pt_ext_err(err) : 0; 2057 decoder->state.timestamp = decoder->timestamp; 2058 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2059 decoder->state.cr3 = decoder->cr3; 2060 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2061 2062 if (err) 2063 decoder->state.from_ip = decoder->ip; 2064 2065 return &decoder->state; 2066 } 2067 2068 static bool intel_pt_at_psb(unsigned char *buf, size_t len) 2069 { 2070 if (len < INTEL_PT_PSB_LEN) 2071 return false; 2072 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR, 2073 INTEL_PT_PSB_LEN); 2074 } 2075 2076 /** 2077 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2078 * @buf: pointer to buffer pointer 2079 * @len: size of buffer 2080 * 2081 * Updates the buffer pointer to point to the start of the next PSB packet if 2082 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2083 * @len is adjusted accordingly. 2084 * 2085 * Return: %true if a PSB packet is found, %false otherwise. 2086 */ 2087 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2088 { 2089 unsigned char *next; 2090 2091 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2092 if (next) { 2093 *len -= next - *buf; 2094 *buf = next; 2095 return true; 2096 } 2097 return false; 2098 } 2099 2100 /** 2101 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2102 * packet. 2103 * @buf: pointer to buffer pointer 2104 * @len: size of buffer 2105 * 2106 * Updates the buffer pointer to point to the start of the following PSB packet 2107 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2108 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2109 * 2110 * Return: %true if a PSB packet is found, %false otherwise. 2111 */ 2112 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2113 { 2114 unsigned char *next; 2115 2116 if (!*len) 2117 return false; 2118 2119 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2120 if (next) { 2121 *len -= next - *buf; 2122 *buf = next; 2123 return true; 2124 } 2125 return false; 2126 } 2127 2128 /** 2129 * intel_pt_last_psb - find the last PSB packet in a buffer. 2130 * @buf: buffer 2131 * @len: size of buffer 2132 * 2133 * This function finds the last PSB in a buffer. 2134 * 2135 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2136 */ 2137 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2138 { 2139 const char *n = INTEL_PT_PSB_STR; 2140 unsigned char *p; 2141 size_t k; 2142 2143 if (len < INTEL_PT_PSB_LEN) 2144 return NULL; 2145 2146 k = len - INTEL_PT_PSB_LEN + 1; 2147 while (1) { 2148 p = memrchr(buf, n[0], k); 2149 if (!p) 2150 return NULL; 2151 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2152 return p; 2153 k = p - buf; 2154 if (!k) 2155 return NULL; 2156 } 2157 } 2158 2159 /** 2160 * intel_pt_next_tsc - find and return next TSC. 2161 * @buf: buffer 2162 * @len: size of buffer 2163 * @tsc: TSC value returned 2164 * 2165 * Find a TSC packet in @buf and return the TSC value. This function assumes 2166 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2167 * PSBEND packet is found. 2168 * 2169 * Return: %true if TSC is found, false otherwise. 2170 */ 2171 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc) 2172 { 2173 struct intel_pt_pkt packet; 2174 int ret; 2175 2176 while (len) { 2177 ret = intel_pt_get_packet(buf, len, &packet); 2178 if (ret <= 0) 2179 return false; 2180 if (packet.type == INTEL_PT_TSC) { 2181 *tsc = packet.payload; 2182 return true; 2183 } 2184 if (packet.type == INTEL_PT_PSBEND) 2185 return false; 2186 buf += ret; 2187 len -= ret; 2188 } 2189 return false; 2190 } 2191 2192 /** 2193 * intel_pt_tsc_cmp - compare 7-byte TSCs. 2194 * @tsc1: first TSC to compare 2195 * @tsc2: second TSC to compare 2196 * 2197 * This function compares 7-byte TSC values allowing for the possibility that 2198 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 2199 * around so for that purpose this function assumes the absolute difference is 2200 * less than half the maximum difference. 2201 * 2202 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 2203 * after @tsc2. 2204 */ 2205 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 2206 { 2207 const uint64_t halfway = (1ULL << 55); 2208 2209 if (tsc1 == tsc2) 2210 return 0; 2211 2212 if (tsc1 < tsc2) { 2213 if (tsc2 - tsc1 < halfway) 2214 return -1; 2215 else 2216 return 1; 2217 } else { 2218 if (tsc1 - tsc2 < halfway) 2219 return 1; 2220 else 2221 return -1; 2222 } 2223 } 2224 2225 /** 2226 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 2227 * using TSC. 2228 * @buf_a: first buffer 2229 * @len_a: size of first buffer 2230 * @buf_b: second buffer 2231 * @len_b: size of second buffer 2232 * 2233 * If the trace contains TSC we can look at the last TSC of @buf_a and the 2234 * first TSC of @buf_b in order to determine if the buffers overlap, and then 2235 * walk forward in @buf_b until a later TSC is found. A precondition is that 2236 * @buf_a and @buf_b are positioned at a PSB. 2237 * 2238 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2239 * @buf_b + @len_b if there is no non-overlapped data. 2240 */ 2241 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 2242 size_t len_a, 2243 unsigned char *buf_b, 2244 size_t len_b) 2245 { 2246 uint64_t tsc_a, tsc_b; 2247 unsigned char *p; 2248 size_t len; 2249 2250 p = intel_pt_last_psb(buf_a, len_a); 2251 if (!p) 2252 return buf_b; /* No PSB in buf_a => no overlap */ 2253 2254 len = len_a - (p - buf_a); 2255 if (!intel_pt_next_tsc(p, len, &tsc_a)) { 2256 /* The last PSB+ in buf_a is incomplete, so go back one more */ 2257 len_a -= len; 2258 p = intel_pt_last_psb(buf_a, len_a); 2259 if (!p) 2260 return buf_b; /* No full PSB+ => assume no overlap */ 2261 len = len_a - (p - buf_a); 2262 if (!intel_pt_next_tsc(p, len, &tsc_a)) 2263 return buf_b; /* No TSC in buf_a => assume no overlap */ 2264 } 2265 2266 while (1) { 2267 /* Ignore PSB+ with no TSC */ 2268 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) && 2269 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0) 2270 return buf_b; /* tsc_a < tsc_b => no overlap */ 2271 2272 if (!intel_pt_step_psb(&buf_b, &len_b)) 2273 return buf_b + len_b; /* No PSB in buf_b => no data */ 2274 } 2275 } 2276 2277 /** 2278 * intel_pt_find_overlap - determine start of non-overlapped trace data. 2279 * @buf_a: first buffer 2280 * @len_a: size of first buffer 2281 * @buf_b: second buffer 2282 * @len_b: size of second buffer 2283 * @have_tsc: can use TSC packets to detect overlap 2284 * 2285 * When trace samples or snapshots are recorded there is the possibility that 2286 * the data overlaps. Note that, for the purposes of decoding, data is only 2287 * useful if it begins with a PSB packet. 2288 * 2289 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2290 * @buf_b + @len_b if there is no non-overlapped data. 2291 */ 2292 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 2293 unsigned char *buf_b, size_t len_b, 2294 bool have_tsc) 2295 { 2296 unsigned char *found; 2297 2298 /* Buffer 'b' must start at PSB so throw away everything before that */ 2299 if (!intel_pt_next_psb(&buf_b, &len_b)) 2300 return buf_b + len_b; /* No PSB */ 2301 2302 if (!intel_pt_next_psb(&buf_a, &len_a)) 2303 return buf_b; /* No overlap */ 2304 2305 if (have_tsc) { 2306 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b); 2307 if (found) 2308 return found; 2309 } 2310 2311 /* 2312 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 2313 * we can ignore the first part of buffer 'a'. 2314 */ 2315 while (len_b < len_a) { 2316 if (!intel_pt_step_psb(&buf_a, &len_a)) 2317 return buf_b; /* No overlap */ 2318 } 2319 2320 /* Now len_b >= len_a */ 2321 if (len_b > len_a) { 2322 /* The leftover buffer 'b' must start at a PSB */ 2323 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2324 if (!intel_pt_step_psb(&buf_a, &len_a)) 2325 return buf_b; /* No overlap */ 2326 } 2327 } 2328 2329 while (1) { 2330 /* Potential overlap so check the bytes */ 2331 found = memmem(buf_a, len_a, buf_b, len_a); 2332 if (found) 2333 return buf_b + len_a; 2334 2335 /* Try again at next PSB in buffer 'a' */ 2336 if (!intel_pt_step_psb(&buf_a, &len_a)) 2337 return buf_b; /* No overlap */ 2338 2339 /* The leftover buffer 'b' must start at a PSB */ 2340 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2341 if (!intel_pt_step_psb(&buf_a, &len_a)) 2342 return buf_b; /* No overlap */ 2343 } 2344 } 2345 } 2346