1 /* 2 * intel_pt_decoder.c: Intel Processor Trace support 3 * Copyright (c) 2013-2014, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #ifndef _GNU_SOURCE 17 #define _GNU_SOURCE 18 #endif 19 #include <stdlib.h> 20 #include <stdbool.h> 21 #include <string.h> 22 #include <errno.h> 23 #include <stdint.h> 24 #include <inttypes.h> 25 26 #include "../cache.h" 27 #include "../util.h" 28 29 #include "intel-pt-insn-decoder.h" 30 #include "intel-pt-pkt-decoder.h" 31 #include "intel-pt-decoder.h" 32 #include "intel-pt-log.h" 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define INTEL_PT_RETURN 1 39 40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 41 #define INTEL_PT_MAX_LOOPS 10000 42 43 struct intel_pt_blk { 44 struct intel_pt_blk *prev; 45 uint64_t ip[INTEL_PT_BLK_SIZE]; 46 }; 47 48 struct intel_pt_stack { 49 struct intel_pt_blk *blk; 50 struct intel_pt_blk *spare; 51 int pos; 52 }; 53 54 enum intel_pt_pkt_state { 55 INTEL_PT_STATE_NO_PSB, 56 INTEL_PT_STATE_NO_IP, 57 INTEL_PT_STATE_ERR_RESYNC, 58 INTEL_PT_STATE_IN_SYNC, 59 INTEL_PT_STATE_TNT, 60 INTEL_PT_STATE_TIP, 61 INTEL_PT_STATE_TIP_PGD, 62 INTEL_PT_STATE_FUP, 63 INTEL_PT_STATE_FUP_NO_TIP, 64 }; 65 66 #ifdef INTEL_PT_STRICT 67 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 68 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 69 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 70 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 71 #else 72 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 73 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 74 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 75 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 76 #endif 77 78 struct intel_pt_decoder { 79 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 80 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 81 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 82 uint64_t max_insn_cnt, void *data); 83 bool (*pgd_ip)(uint64_t ip, void *data); 84 void *data; 85 struct intel_pt_state state; 86 const unsigned char *buf; 87 size_t len; 88 bool return_compression; 89 bool mtc_insn; 90 bool pge; 91 bool have_tma; 92 bool have_cyc; 93 bool fixup_last_mtc; 94 uint64_t pos; 95 uint64_t last_ip; 96 uint64_t ip; 97 uint64_t cr3; 98 uint64_t timestamp; 99 uint64_t tsc_timestamp; 100 uint64_t ref_timestamp; 101 uint64_t ret_addr; 102 uint64_t ctc_timestamp; 103 uint64_t ctc_delta; 104 uint64_t cycle_cnt; 105 uint64_t cyc_ref_timestamp; 106 uint32_t last_mtc; 107 uint32_t tsc_ctc_ratio_n; 108 uint32_t tsc_ctc_ratio_d; 109 uint32_t tsc_ctc_mult; 110 uint32_t tsc_slip; 111 uint32_t ctc_rem_mask; 112 int mtc_shift; 113 struct intel_pt_stack stack; 114 enum intel_pt_pkt_state pkt_state; 115 struct intel_pt_pkt packet; 116 struct intel_pt_pkt tnt; 117 int pkt_step; 118 int pkt_len; 119 int last_packet_type; 120 unsigned int cbr; 121 unsigned int max_non_turbo_ratio; 122 double max_non_turbo_ratio_fp; 123 double cbr_cyc_to_tsc; 124 double calc_cyc_to_tsc; 125 bool have_calc_cyc_to_tsc; 126 int exec_mode; 127 unsigned int insn_bytes; 128 uint64_t period; 129 enum intel_pt_period_type period_type; 130 uint64_t tot_insn_cnt; 131 uint64_t period_insn_cnt; 132 uint64_t period_mask; 133 uint64_t period_ticks; 134 uint64_t last_masked_timestamp; 135 bool continuous_period; 136 bool overflow; 137 bool set_fup_tx_flags; 138 unsigned int fup_tx_flags; 139 unsigned int tx_flags; 140 uint64_t timestamp_insn_cnt; 141 uint64_t stuck_ip; 142 int no_progress; 143 int stuck_ip_prd; 144 int stuck_ip_cnt; 145 const unsigned char *next_buf; 146 size_t next_len; 147 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 148 }; 149 150 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 151 { 152 int i; 153 154 for (i = 0; x != 1; i++) 155 x >>= 1; 156 157 return x << i; 158 } 159 160 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 161 { 162 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 163 uint64_t period; 164 165 period = intel_pt_lower_power_of_2(decoder->period); 166 decoder->period_mask = ~(period - 1); 167 decoder->period_ticks = period; 168 } 169 } 170 171 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 172 { 173 if (!d) 174 return 0; 175 return (t / d) * n + ((t % d) * n) / d; 176 } 177 178 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 179 { 180 struct intel_pt_decoder *decoder; 181 182 if (!params->get_trace || !params->walk_insn) 183 return NULL; 184 185 decoder = zalloc(sizeof(struct intel_pt_decoder)); 186 if (!decoder) 187 return NULL; 188 189 decoder->get_trace = params->get_trace; 190 decoder->walk_insn = params->walk_insn; 191 decoder->pgd_ip = params->pgd_ip; 192 decoder->data = params->data; 193 decoder->return_compression = params->return_compression; 194 195 decoder->period = params->period; 196 decoder->period_type = params->period_type; 197 198 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 199 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 200 201 intel_pt_setup_period(decoder); 202 203 decoder->mtc_shift = params->mtc_period; 204 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 205 206 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 207 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 208 209 if (!decoder->tsc_ctc_ratio_n) 210 decoder->tsc_ctc_ratio_d = 0; 211 212 if (decoder->tsc_ctc_ratio_d) { 213 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 214 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 215 decoder->tsc_ctc_ratio_d; 216 217 /* 218 * Allow for timestamps appearing to backwards because a TSC 219 * packet has slipped past a MTC packet, so allow 2 MTC ticks 220 * or ... 221 */ 222 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift, 223 decoder->tsc_ctc_ratio_n, 224 decoder->tsc_ctc_ratio_d); 225 } 226 /* ... or 0x100 paranoia */ 227 if (decoder->tsc_slip < 0x100) 228 decoder->tsc_slip = 0x100; 229 230 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 231 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 232 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 233 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 234 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 235 236 return decoder; 237 } 238 239 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 240 { 241 struct intel_pt_blk *blk = stack->blk; 242 243 stack->blk = blk->prev; 244 if (!stack->spare) 245 stack->spare = blk; 246 else 247 free(blk); 248 } 249 250 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 251 { 252 if (!stack->pos) { 253 if (!stack->blk) 254 return 0; 255 intel_pt_pop_blk(stack); 256 if (!stack->blk) 257 return 0; 258 stack->pos = INTEL_PT_BLK_SIZE; 259 } 260 return stack->blk->ip[--stack->pos]; 261 } 262 263 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 264 { 265 struct intel_pt_blk *blk; 266 267 if (stack->spare) { 268 blk = stack->spare; 269 stack->spare = NULL; 270 } else { 271 blk = malloc(sizeof(struct intel_pt_blk)); 272 if (!blk) 273 return -ENOMEM; 274 } 275 276 blk->prev = stack->blk; 277 stack->blk = blk; 278 stack->pos = 0; 279 return 0; 280 } 281 282 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 283 { 284 int err; 285 286 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 287 err = intel_pt_alloc_blk(stack); 288 if (err) 289 return err; 290 } 291 292 stack->blk->ip[stack->pos++] = ip; 293 return 0; 294 } 295 296 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 297 { 298 while (stack->blk) 299 intel_pt_pop_blk(stack); 300 stack->pos = 0; 301 } 302 303 static void intel_pt_free_stack(struct intel_pt_stack *stack) 304 { 305 intel_pt_clear_stack(stack); 306 zfree(&stack->blk); 307 zfree(&stack->spare); 308 } 309 310 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 311 { 312 intel_pt_free_stack(&decoder->stack); 313 free(decoder); 314 } 315 316 static int intel_pt_ext_err(int code) 317 { 318 switch (code) { 319 case -ENOMEM: 320 return INTEL_PT_ERR_NOMEM; 321 case -ENOSYS: 322 return INTEL_PT_ERR_INTERN; 323 case -EBADMSG: 324 return INTEL_PT_ERR_BADPKT; 325 case -ENODATA: 326 return INTEL_PT_ERR_NODATA; 327 case -EILSEQ: 328 return INTEL_PT_ERR_NOINSN; 329 case -ENOENT: 330 return INTEL_PT_ERR_MISMAT; 331 case -EOVERFLOW: 332 return INTEL_PT_ERR_OVR; 333 case -ENOSPC: 334 return INTEL_PT_ERR_LOST; 335 case -ELOOP: 336 return INTEL_PT_ERR_NELOOP; 337 default: 338 return INTEL_PT_ERR_UNK; 339 } 340 } 341 342 static const char *intel_pt_err_msgs[] = { 343 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 344 [INTEL_PT_ERR_INTERN] = "Internal error", 345 [INTEL_PT_ERR_BADPKT] = "Bad packet", 346 [INTEL_PT_ERR_NODATA] = "No more data", 347 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 348 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 349 [INTEL_PT_ERR_OVR] = "Overflow packet", 350 [INTEL_PT_ERR_LOST] = "Lost trace data", 351 [INTEL_PT_ERR_UNK] = "Unknown error!", 352 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 353 }; 354 355 int intel_pt__strerror(int code, char *buf, size_t buflen) 356 { 357 if (code < 1 || code >= INTEL_PT_ERR_MAX) 358 code = INTEL_PT_ERR_UNK; 359 strlcpy(buf, intel_pt_err_msgs[code], buflen); 360 return 0; 361 } 362 363 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 364 uint64_t last_ip) 365 { 366 uint64_t ip; 367 368 switch (packet->count) { 369 case 1: 370 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 371 packet->payload; 372 break; 373 case 2: 374 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 375 packet->payload; 376 break; 377 case 3: 378 ip = packet->payload; 379 /* Sign-extend 6-byte ip */ 380 if (ip & (uint64_t)0x800000000000ULL) 381 ip |= (uint64_t)0xffff000000000000ULL; 382 break; 383 case 4: 384 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 385 packet->payload; 386 break; 387 case 6: 388 ip = packet->payload; 389 break; 390 default: 391 return 0; 392 } 393 394 return ip; 395 } 396 397 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 398 { 399 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 400 } 401 402 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 403 { 404 intel_pt_set_last_ip(decoder); 405 decoder->ip = decoder->last_ip; 406 } 407 408 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 409 { 410 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 411 decoder->buf); 412 } 413 414 static int intel_pt_bug(struct intel_pt_decoder *decoder) 415 { 416 intel_pt_log("ERROR: Internal error\n"); 417 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 418 return -ENOSYS; 419 } 420 421 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 422 { 423 decoder->tx_flags = 0; 424 } 425 426 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 427 { 428 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 429 } 430 431 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 432 { 433 intel_pt_clear_tx_flags(decoder); 434 decoder->have_tma = false; 435 decoder->pkt_len = 1; 436 decoder->pkt_step = 1; 437 intel_pt_decoder_log_packet(decoder); 438 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 439 intel_pt_log("ERROR: Bad packet\n"); 440 decoder->pkt_state = INTEL_PT_STATE_ERR1; 441 } 442 return -EBADMSG; 443 } 444 445 static int intel_pt_get_data(struct intel_pt_decoder *decoder) 446 { 447 struct intel_pt_buffer buffer = { .buf = 0, }; 448 int ret; 449 450 decoder->pkt_step = 0; 451 452 intel_pt_log("Getting more data\n"); 453 ret = decoder->get_trace(&buffer, decoder->data); 454 if (ret) 455 return ret; 456 decoder->buf = buffer.buf; 457 decoder->len = buffer.len; 458 if (!decoder->len) { 459 intel_pt_log("No more data\n"); 460 return -ENODATA; 461 } 462 if (!buffer.consecutive) { 463 decoder->ip = 0; 464 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 465 decoder->ref_timestamp = buffer.ref_timestamp; 466 decoder->timestamp = 0; 467 decoder->have_tma = false; 468 decoder->state.trace_nr = buffer.trace_nr; 469 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 470 decoder->ref_timestamp); 471 return -ENOLINK; 472 } 473 474 return 0; 475 } 476 477 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder) 478 { 479 if (!decoder->next_buf) 480 return intel_pt_get_data(decoder); 481 482 decoder->buf = decoder->next_buf; 483 decoder->len = decoder->next_len; 484 decoder->next_buf = 0; 485 decoder->next_len = 0; 486 return 0; 487 } 488 489 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 490 { 491 unsigned char *buf = decoder->temp_buf; 492 size_t old_len, len, n; 493 int ret; 494 495 old_len = decoder->len; 496 len = decoder->len; 497 memcpy(buf, decoder->buf, len); 498 499 ret = intel_pt_get_data(decoder); 500 if (ret) { 501 decoder->pos += old_len; 502 return ret < 0 ? ret : -EINVAL; 503 } 504 505 n = INTEL_PT_PKT_MAX_SZ - len; 506 if (n > decoder->len) 507 n = decoder->len; 508 memcpy(buf + len, decoder->buf, n); 509 len += n; 510 511 ret = intel_pt_get_packet(buf, len, &decoder->packet); 512 if (ret < (int)old_len) { 513 decoder->next_buf = decoder->buf; 514 decoder->next_len = decoder->len; 515 decoder->buf = buf; 516 decoder->len = old_len; 517 return intel_pt_bad_packet(decoder); 518 } 519 520 decoder->next_buf = decoder->buf + (ret - old_len); 521 decoder->next_len = decoder->len - (ret - old_len); 522 523 decoder->buf = buf; 524 decoder->len = ret; 525 526 return ret; 527 } 528 529 struct intel_pt_pkt_info { 530 struct intel_pt_decoder *decoder; 531 struct intel_pt_pkt packet; 532 uint64_t pos; 533 int pkt_len; 534 int last_packet_type; 535 void *data; 536 }; 537 538 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 539 540 /* Lookahead packets in current buffer */ 541 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 542 intel_pt_pkt_cb_t cb, void *data) 543 { 544 struct intel_pt_pkt_info pkt_info; 545 const unsigned char *buf = decoder->buf; 546 size_t len = decoder->len; 547 int ret; 548 549 pkt_info.decoder = decoder; 550 pkt_info.pos = decoder->pos; 551 pkt_info.pkt_len = decoder->pkt_step; 552 pkt_info.last_packet_type = decoder->last_packet_type; 553 pkt_info.data = data; 554 555 while (1) { 556 do { 557 pkt_info.pos += pkt_info.pkt_len; 558 buf += pkt_info.pkt_len; 559 len -= pkt_info.pkt_len; 560 561 if (!len) 562 return INTEL_PT_NEED_MORE_BYTES; 563 564 ret = intel_pt_get_packet(buf, len, &pkt_info.packet); 565 if (!ret) 566 return INTEL_PT_NEED_MORE_BYTES; 567 if (ret < 0) 568 return ret; 569 570 pkt_info.pkt_len = ret; 571 } while (pkt_info.packet.type == INTEL_PT_PAD); 572 573 ret = cb(&pkt_info); 574 if (ret) 575 return 0; 576 577 pkt_info.last_packet_type = pkt_info.packet.type; 578 } 579 } 580 581 struct intel_pt_calc_cyc_to_tsc_info { 582 uint64_t cycle_cnt; 583 unsigned int cbr; 584 uint32_t last_mtc; 585 uint64_t ctc_timestamp; 586 uint64_t ctc_delta; 587 uint64_t tsc_timestamp; 588 uint64_t timestamp; 589 bool have_tma; 590 bool fixup_last_mtc; 591 bool from_mtc; 592 double cbr_cyc_to_tsc; 593 }; 594 595 /* 596 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 597 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 598 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 599 * packet by copying the missing bits from the current MTC assuming the least 600 * difference between the two, and that the current MTC comes after last_mtc. 601 */ 602 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 603 uint32_t *last_mtc) 604 { 605 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 606 uint32_t mask = ~(first_missing_bit - 1); 607 608 *last_mtc |= mtc & mask; 609 if (*last_mtc >= mtc) { 610 *last_mtc -= first_missing_bit; 611 *last_mtc &= 0xff; 612 } 613 } 614 615 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 616 { 617 struct intel_pt_decoder *decoder = pkt_info->decoder; 618 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 619 uint64_t timestamp; 620 double cyc_to_tsc; 621 unsigned int cbr; 622 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 623 624 switch (pkt_info->packet.type) { 625 case INTEL_PT_TNT: 626 case INTEL_PT_TIP_PGE: 627 case INTEL_PT_TIP: 628 case INTEL_PT_FUP: 629 case INTEL_PT_PSB: 630 case INTEL_PT_PIP: 631 case INTEL_PT_MODE_EXEC: 632 case INTEL_PT_MODE_TSX: 633 case INTEL_PT_PSBEND: 634 case INTEL_PT_PAD: 635 case INTEL_PT_VMCS: 636 case INTEL_PT_MNT: 637 return 0; 638 639 case INTEL_PT_MTC: 640 if (!data->have_tma) 641 return 0; 642 643 mtc = pkt_info->packet.payload; 644 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 645 data->fixup_last_mtc = false; 646 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 647 &data->last_mtc); 648 } 649 if (mtc > data->last_mtc) 650 mtc_delta = mtc - data->last_mtc; 651 else 652 mtc_delta = mtc + 256 - data->last_mtc; 653 data->ctc_delta += mtc_delta << decoder->mtc_shift; 654 data->last_mtc = mtc; 655 656 if (decoder->tsc_ctc_mult) { 657 timestamp = data->ctc_timestamp + 658 data->ctc_delta * decoder->tsc_ctc_mult; 659 } else { 660 timestamp = data->ctc_timestamp + 661 multdiv(data->ctc_delta, 662 decoder->tsc_ctc_ratio_n, 663 decoder->tsc_ctc_ratio_d); 664 } 665 666 if (timestamp < data->timestamp) 667 return 1; 668 669 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 670 data->timestamp = timestamp; 671 return 0; 672 } 673 674 break; 675 676 case INTEL_PT_TSC: 677 timestamp = pkt_info->packet.payload | 678 (data->timestamp & (0xffULL << 56)); 679 if (data->from_mtc && timestamp < data->timestamp && 680 data->timestamp - timestamp < decoder->tsc_slip) 681 return 1; 682 if (timestamp < data->timestamp) 683 timestamp += (1ULL << 56); 684 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 685 if (data->from_mtc) 686 return 1; 687 data->tsc_timestamp = timestamp; 688 data->timestamp = timestamp; 689 return 0; 690 } 691 break; 692 693 case INTEL_PT_TMA: 694 if (data->from_mtc) 695 return 1; 696 697 if (!decoder->tsc_ctc_ratio_d) 698 return 0; 699 700 ctc = pkt_info->packet.payload; 701 fc = pkt_info->packet.count; 702 ctc_rem = ctc & decoder->ctc_rem_mask; 703 704 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 705 706 data->ctc_timestamp = data->tsc_timestamp - fc; 707 if (decoder->tsc_ctc_mult) { 708 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 709 } else { 710 data->ctc_timestamp -= 711 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 712 decoder->tsc_ctc_ratio_d); 713 } 714 715 data->ctc_delta = 0; 716 data->have_tma = true; 717 data->fixup_last_mtc = true; 718 719 return 0; 720 721 case INTEL_PT_CYC: 722 data->cycle_cnt += pkt_info->packet.payload; 723 return 0; 724 725 case INTEL_PT_CBR: 726 cbr = pkt_info->packet.payload; 727 if (data->cbr && data->cbr != cbr) 728 return 1; 729 data->cbr = cbr; 730 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 731 return 0; 732 733 case INTEL_PT_TIP_PGD: 734 case INTEL_PT_TRACESTOP: 735 case INTEL_PT_OVF: 736 case INTEL_PT_BAD: /* Does not happen */ 737 default: 738 return 1; 739 } 740 741 if (!data->cbr && decoder->cbr) { 742 data->cbr = decoder->cbr; 743 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 744 } 745 746 if (!data->cycle_cnt) 747 return 1; 748 749 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 750 751 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 752 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 753 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 754 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 755 return 1; 756 } 757 758 decoder->calc_cyc_to_tsc = cyc_to_tsc; 759 decoder->have_calc_cyc_to_tsc = true; 760 761 if (data->cbr) { 762 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 763 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 764 } else { 765 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 766 cyc_to_tsc, pkt_info->pos); 767 } 768 769 return 1; 770 } 771 772 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 773 bool from_mtc) 774 { 775 struct intel_pt_calc_cyc_to_tsc_info data = { 776 .cycle_cnt = 0, 777 .cbr = 0, 778 .last_mtc = decoder->last_mtc, 779 .ctc_timestamp = decoder->ctc_timestamp, 780 .ctc_delta = decoder->ctc_delta, 781 .tsc_timestamp = decoder->tsc_timestamp, 782 .timestamp = decoder->timestamp, 783 .have_tma = decoder->have_tma, 784 .fixup_last_mtc = decoder->fixup_last_mtc, 785 .from_mtc = from_mtc, 786 .cbr_cyc_to_tsc = 0, 787 }; 788 789 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 790 } 791 792 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 793 { 794 int ret; 795 796 decoder->last_packet_type = decoder->packet.type; 797 798 do { 799 decoder->pos += decoder->pkt_step; 800 decoder->buf += decoder->pkt_step; 801 decoder->len -= decoder->pkt_step; 802 803 if (!decoder->len) { 804 ret = intel_pt_get_next_data(decoder); 805 if (ret) 806 return ret; 807 } 808 809 ret = intel_pt_get_packet(decoder->buf, decoder->len, 810 &decoder->packet); 811 if (ret == INTEL_PT_NEED_MORE_BYTES && 812 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 813 ret = intel_pt_get_split_packet(decoder); 814 if (ret < 0) 815 return ret; 816 } 817 if (ret <= 0) 818 return intel_pt_bad_packet(decoder); 819 820 decoder->pkt_len = ret; 821 decoder->pkt_step = ret; 822 intel_pt_decoder_log_packet(decoder); 823 } while (decoder->packet.type == INTEL_PT_PAD); 824 825 return 0; 826 } 827 828 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 829 { 830 uint64_t timestamp, masked_timestamp; 831 832 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 833 masked_timestamp = timestamp & decoder->period_mask; 834 if (decoder->continuous_period) { 835 if (masked_timestamp != decoder->last_masked_timestamp) 836 return 1; 837 } else { 838 timestamp += 1; 839 masked_timestamp = timestamp & decoder->period_mask; 840 if (masked_timestamp != decoder->last_masked_timestamp) { 841 decoder->last_masked_timestamp = masked_timestamp; 842 decoder->continuous_period = true; 843 } 844 } 845 return decoder->period_ticks - (timestamp - masked_timestamp); 846 } 847 848 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 849 { 850 switch (decoder->period_type) { 851 case INTEL_PT_PERIOD_INSTRUCTIONS: 852 return decoder->period - decoder->period_insn_cnt; 853 case INTEL_PT_PERIOD_TICKS: 854 return intel_pt_next_period(decoder); 855 case INTEL_PT_PERIOD_NONE: 856 case INTEL_PT_PERIOD_MTC: 857 default: 858 return 0; 859 } 860 } 861 862 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 863 { 864 uint64_t timestamp, masked_timestamp; 865 866 switch (decoder->period_type) { 867 case INTEL_PT_PERIOD_INSTRUCTIONS: 868 decoder->period_insn_cnt = 0; 869 break; 870 case INTEL_PT_PERIOD_TICKS: 871 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 872 masked_timestamp = timestamp & decoder->period_mask; 873 decoder->last_masked_timestamp = masked_timestamp; 874 break; 875 case INTEL_PT_PERIOD_NONE: 876 case INTEL_PT_PERIOD_MTC: 877 default: 878 break; 879 } 880 881 decoder->state.type |= INTEL_PT_INSTRUCTION; 882 } 883 884 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 885 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 886 { 887 uint64_t max_insn_cnt, insn_cnt = 0; 888 int err; 889 890 if (!decoder->mtc_insn) 891 decoder->mtc_insn = true; 892 893 max_insn_cnt = intel_pt_next_sample(decoder); 894 895 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 896 max_insn_cnt, decoder->data); 897 898 decoder->tot_insn_cnt += insn_cnt; 899 decoder->timestamp_insn_cnt += insn_cnt; 900 decoder->period_insn_cnt += insn_cnt; 901 902 if (err) { 903 decoder->no_progress = 0; 904 decoder->pkt_state = INTEL_PT_STATE_ERR2; 905 intel_pt_log_at("ERROR: Failed to get instruction", 906 decoder->ip); 907 if (err == -ENOENT) 908 return -ENOLINK; 909 return -EILSEQ; 910 } 911 912 if (ip && decoder->ip == ip) { 913 err = -EAGAIN; 914 goto out; 915 } 916 917 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 918 intel_pt_sample_insn(decoder); 919 920 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 921 decoder->state.type = INTEL_PT_INSTRUCTION; 922 decoder->state.from_ip = decoder->ip; 923 decoder->state.to_ip = 0; 924 decoder->ip += intel_pt_insn->length; 925 err = INTEL_PT_RETURN; 926 goto out; 927 } 928 929 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 930 /* Zero-length calls are excluded */ 931 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 932 intel_pt_insn->rel) { 933 err = intel_pt_push(&decoder->stack, decoder->ip + 934 intel_pt_insn->length); 935 if (err) 936 goto out; 937 } 938 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 939 decoder->ret_addr = intel_pt_pop(&decoder->stack); 940 } 941 942 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 943 int cnt = decoder->no_progress++; 944 945 decoder->state.from_ip = decoder->ip; 946 decoder->ip += intel_pt_insn->length + 947 intel_pt_insn->rel; 948 decoder->state.to_ip = decoder->ip; 949 err = INTEL_PT_RETURN; 950 951 /* 952 * Check for being stuck in a loop. This can happen if a 953 * decoder error results in the decoder erroneously setting the 954 * ip to an address that is itself in an infinite loop that 955 * consumes no packets. When that happens, there must be an 956 * unconditional branch. 957 */ 958 if (cnt) { 959 if (cnt == 1) { 960 decoder->stuck_ip = decoder->state.to_ip; 961 decoder->stuck_ip_prd = 1; 962 decoder->stuck_ip_cnt = 1; 963 } else if (cnt > INTEL_PT_MAX_LOOPS || 964 decoder->state.to_ip == decoder->stuck_ip) { 965 intel_pt_log_at("ERROR: Never-ending loop", 966 decoder->state.to_ip); 967 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 968 err = -ELOOP; 969 goto out; 970 } else if (!--decoder->stuck_ip_cnt) { 971 decoder->stuck_ip_prd += 1; 972 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 973 decoder->stuck_ip = decoder->state.to_ip; 974 } 975 } 976 goto out_no_progress; 977 } 978 out: 979 decoder->no_progress = 0; 980 out_no_progress: 981 decoder->state.insn_op = intel_pt_insn->op; 982 decoder->state.insn_len = intel_pt_insn->length; 983 memcpy(decoder->state.insn, intel_pt_insn->buf, 984 INTEL_PT_INSN_BUF_SZ); 985 986 if (decoder->tx_flags & INTEL_PT_IN_TX) 987 decoder->state.flags |= INTEL_PT_IN_TX; 988 989 return err; 990 } 991 992 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 993 { 994 struct intel_pt_insn intel_pt_insn; 995 uint64_t ip; 996 int err; 997 998 ip = decoder->last_ip; 999 1000 while (1) { 1001 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1002 if (err == INTEL_PT_RETURN) 1003 return 0; 1004 if (err == -EAGAIN) { 1005 if (decoder->set_fup_tx_flags) { 1006 decoder->set_fup_tx_flags = false; 1007 decoder->tx_flags = decoder->fup_tx_flags; 1008 decoder->state.type = INTEL_PT_TRANSACTION; 1009 decoder->state.from_ip = decoder->ip; 1010 decoder->state.to_ip = 0; 1011 decoder->state.flags = decoder->fup_tx_flags; 1012 return 0; 1013 } 1014 return err; 1015 } 1016 decoder->set_fup_tx_flags = false; 1017 if (err) 1018 return err; 1019 1020 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1021 intel_pt_log_at("ERROR: Unexpected indirect branch", 1022 decoder->ip); 1023 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1024 return -ENOENT; 1025 } 1026 1027 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1028 intel_pt_log_at("ERROR: Unexpected conditional branch", 1029 decoder->ip); 1030 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1031 return -ENOENT; 1032 } 1033 1034 intel_pt_bug(decoder); 1035 } 1036 } 1037 1038 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1039 { 1040 struct intel_pt_insn intel_pt_insn; 1041 int err; 1042 1043 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1044 if (err == INTEL_PT_RETURN && 1045 decoder->pgd_ip && 1046 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1047 (decoder->state.type & INTEL_PT_BRANCH) && 1048 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1049 /* Unconditional branch leaving filter region */ 1050 decoder->no_progress = 0; 1051 decoder->pge = false; 1052 decoder->continuous_period = false; 1053 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1054 decoder->state.to_ip = 0; 1055 return 0; 1056 } 1057 if (err == INTEL_PT_RETURN) 1058 return 0; 1059 if (err) 1060 return err; 1061 1062 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1063 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1064 decoder->pge = false; 1065 decoder->continuous_period = false; 1066 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1067 decoder->state.from_ip = decoder->ip; 1068 decoder->state.to_ip = 0; 1069 if (decoder->packet.count != 0) 1070 decoder->ip = decoder->last_ip; 1071 } else { 1072 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1073 decoder->state.from_ip = decoder->ip; 1074 if (decoder->packet.count == 0) { 1075 decoder->state.to_ip = 0; 1076 } else { 1077 decoder->state.to_ip = decoder->last_ip; 1078 decoder->ip = decoder->last_ip; 1079 } 1080 } 1081 return 0; 1082 } 1083 1084 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1085 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1086 intel_pt_insn.rel; 1087 1088 if (decoder->pgd_ip && 1089 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1090 decoder->pgd_ip(to_ip, decoder->data)) { 1091 /* Conditional branch leaving filter region */ 1092 decoder->pge = false; 1093 decoder->continuous_period = false; 1094 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1095 decoder->ip = to_ip; 1096 decoder->state.from_ip = decoder->ip; 1097 decoder->state.to_ip = 0; 1098 return 0; 1099 } 1100 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1101 decoder->ip); 1102 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1103 return -ENOENT; 1104 } 1105 1106 return intel_pt_bug(decoder); 1107 } 1108 1109 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1110 { 1111 struct intel_pt_insn intel_pt_insn; 1112 int err; 1113 1114 while (1) { 1115 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1116 if (err == INTEL_PT_RETURN) 1117 return 0; 1118 if (err) 1119 return err; 1120 1121 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1122 if (!decoder->return_compression) { 1123 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1124 decoder->ip); 1125 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1126 return -ENOENT; 1127 } 1128 if (!decoder->ret_addr) { 1129 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1130 decoder->ip); 1131 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1132 return -ENOENT; 1133 } 1134 if (!(decoder->tnt.payload & BIT63)) { 1135 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1136 decoder->ip); 1137 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1138 return -ENOENT; 1139 } 1140 decoder->tnt.count -= 1; 1141 if (!decoder->tnt.count) 1142 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1143 decoder->tnt.payload <<= 1; 1144 decoder->state.from_ip = decoder->ip; 1145 decoder->ip = decoder->ret_addr; 1146 decoder->state.to_ip = decoder->ip; 1147 return 0; 1148 } 1149 1150 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1151 /* Handle deferred TIPs */ 1152 err = intel_pt_get_next_packet(decoder); 1153 if (err) 1154 return err; 1155 if (decoder->packet.type != INTEL_PT_TIP || 1156 decoder->packet.count == 0) { 1157 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1158 decoder->ip); 1159 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1160 decoder->pkt_step = 0; 1161 return -ENOENT; 1162 } 1163 intel_pt_set_last_ip(decoder); 1164 decoder->state.from_ip = decoder->ip; 1165 decoder->state.to_ip = decoder->last_ip; 1166 decoder->ip = decoder->last_ip; 1167 return 0; 1168 } 1169 1170 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1171 decoder->tnt.count -= 1; 1172 if (!decoder->tnt.count) 1173 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1174 if (decoder->tnt.payload & BIT63) { 1175 decoder->tnt.payload <<= 1; 1176 decoder->state.from_ip = decoder->ip; 1177 decoder->ip += intel_pt_insn.length + 1178 intel_pt_insn.rel; 1179 decoder->state.to_ip = decoder->ip; 1180 return 0; 1181 } 1182 /* Instruction sample for a non-taken branch */ 1183 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1184 decoder->tnt.payload <<= 1; 1185 decoder->state.type = INTEL_PT_INSTRUCTION; 1186 decoder->state.from_ip = decoder->ip; 1187 decoder->state.to_ip = 0; 1188 decoder->ip += intel_pt_insn.length; 1189 return 0; 1190 } 1191 decoder->ip += intel_pt_insn.length; 1192 if (!decoder->tnt.count) 1193 return -EAGAIN; 1194 decoder->tnt.payload <<= 1; 1195 continue; 1196 } 1197 1198 return intel_pt_bug(decoder); 1199 } 1200 } 1201 1202 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1203 { 1204 unsigned int fup_tx_flags; 1205 int err; 1206 1207 fup_tx_flags = decoder->packet.payload & 1208 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1209 err = intel_pt_get_next_packet(decoder); 1210 if (err) 1211 return err; 1212 if (decoder->packet.type == INTEL_PT_FUP) { 1213 decoder->fup_tx_flags = fup_tx_flags; 1214 decoder->set_fup_tx_flags = true; 1215 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1216 *no_tip = true; 1217 } else { 1218 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1219 decoder->pos); 1220 intel_pt_update_in_tx(decoder); 1221 } 1222 return 0; 1223 } 1224 1225 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1226 { 1227 uint64_t timestamp; 1228 1229 decoder->have_tma = false; 1230 1231 if (decoder->ref_timestamp) { 1232 timestamp = decoder->packet.payload | 1233 (decoder->ref_timestamp & (0xffULL << 56)); 1234 if (timestamp < decoder->ref_timestamp) { 1235 if (decoder->ref_timestamp - timestamp > (1ULL << 55)) 1236 timestamp += (1ULL << 56); 1237 } else { 1238 if (timestamp - decoder->ref_timestamp > (1ULL << 55)) 1239 timestamp -= (1ULL << 56); 1240 } 1241 decoder->tsc_timestamp = timestamp; 1242 decoder->timestamp = timestamp; 1243 decoder->ref_timestamp = 0; 1244 decoder->timestamp_insn_cnt = 0; 1245 } else if (decoder->timestamp) { 1246 timestamp = decoder->packet.payload | 1247 (decoder->timestamp & (0xffULL << 56)); 1248 decoder->tsc_timestamp = timestamp; 1249 if (timestamp < decoder->timestamp && 1250 decoder->timestamp - timestamp < decoder->tsc_slip) { 1251 intel_pt_log_to("Suppressing backwards timestamp", 1252 timestamp); 1253 timestamp = decoder->timestamp; 1254 } 1255 if (timestamp < decoder->timestamp) { 1256 intel_pt_log_to("Wraparound timestamp", timestamp); 1257 timestamp += (1ULL << 56); 1258 decoder->tsc_timestamp = timestamp; 1259 } 1260 decoder->timestamp = timestamp; 1261 decoder->timestamp_insn_cnt = 0; 1262 } 1263 1264 if (decoder->last_packet_type == INTEL_PT_CYC) { 1265 decoder->cyc_ref_timestamp = decoder->timestamp; 1266 decoder->cycle_cnt = 0; 1267 decoder->have_calc_cyc_to_tsc = false; 1268 intel_pt_calc_cyc_to_tsc(decoder, false); 1269 } 1270 1271 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1272 } 1273 1274 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1275 { 1276 intel_pt_log("ERROR: Buffer overflow\n"); 1277 intel_pt_clear_tx_flags(decoder); 1278 decoder->have_tma = false; 1279 decoder->cbr = 0; 1280 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1281 decoder->overflow = true; 1282 return -EOVERFLOW; 1283 } 1284 1285 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1286 { 1287 uint32_t ctc = decoder->packet.payload; 1288 uint32_t fc = decoder->packet.count; 1289 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1290 1291 if (!decoder->tsc_ctc_ratio_d) 1292 return; 1293 1294 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1295 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1296 if (decoder->tsc_ctc_mult) { 1297 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1298 } else { 1299 decoder->ctc_timestamp -= multdiv(ctc_rem, 1300 decoder->tsc_ctc_ratio_n, 1301 decoder->tsc_ctc_ratio_d); 1302 } 1303 decoder->ctc_delta = 0; 1304 decoder->have_tma = true; 1305 decoder->fixup_last_mtc = true; 1306 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1307 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1308 } 1309 1310 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1311 { 1312 uint64_t timestamp; 1313 uint32_t mtc, mtc_delta; 1314 1315 if (!decoder->have_tma) 1316 return; 1317 1318 mtc = decoder->packet.payload; 1319 1320 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1321 decoder->fixup_last_mtc = false; 1322 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1323 &decoder->last_mtc); 1324 } 1325 1326 if (mtc > decoder->last_mtc) 1327 mtc_delta = mtc - decoder->last_mtc; 1328 else 1329 mtc_delta = mtc + 256 - decoder->last_mtc; 1330 1331 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1332 1333 if (decoder->tsc_ctc_mult) { 1334 timestamp = decoder->ctc_timestamp + 1335 decoder->ctc_delta * decoder->tsc_ctc_mult; 1336 } else { 1337 timestamp = decoder->ctc_timestamp + 1338 multdiv(decoder->ctc_delta, 1339 decoder->tsc_ctc_ratio_n, 1340 decoder->tsc_ctc_ratio_d); 1341 } 1342 1343 if (timestamp < decoder->timestamp) 1344 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1345 timestamp, decoder->timestamp); 1346 else 1347 decoder->timestamp = timestamp; 1348 1349 decoder->timestamp_insn_cnt = 0; 1350 decoder->last_mtc = mtc; 1351 1352 if (decoder->last_packet_type == INTEL_PT_CYC) { 1353 decoder->cyc_ref_timestamp = decoder->timestamp; 1354 decoder->cycle_cnt = 0; 1355 decoder->have_calc_cyc_to_tsc = false; 1356 intel_pt_calc_cyc_to_tsc(decoder, true); 1357 } 1358 } 1359 1360 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1361 { 1362 unsigned int cbr = decoder->packet.payload; 1363 1364 if (decoder->cbr == cbr) 1365 return; 1366 1367 decoder->cbr = cbr; 1368 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1369 } 1370 1371 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1372 { 1373 uint64_t timestamp = decoder->cyc_ref_timestamp; 1374 1375 decoder->have_cyc = true; 1376 1377 decoder->cycle_cnt += decoder->packet.payload; 1378 1379 if (!decoder->cyc_ref_timestamp) 1380 return; 1381 1382 if (decoder->have_calc_cyc_to_tsc) 1383 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1384 else if (decoder->cbr) 1385 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1386 else 1387 return; 1388 1389 if (timestamp < decoder->timestamp) 1390 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1391 timestamp, decoder->timestamp); 1392 else 1393 decoder->timestamp = timestamp; 1394 1395 decoder->timestamp_insn_cnt = 0; 1396 } 1397 1398 /* Walk PSB+ packets when already in sync. */ 1399 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1400 { 1401 int err; 1402 1403 while (1) { 1404 err = intel_pt_get_next_packet(decoder); 1405 if (err) 1406 return err; 1407 1408 switch (decoder->packet.type) { 1409 case INTEL_PT_PSBEND: 1410 return 0; 1411 1412 case INTEL_PT_TIP_PGD: 1413 case INTEL_PT_TIP_PGE: 1414 case INTEL_PT_TIP: 1415 case INTEL_PT_TNT: 1416 case INTEL_PT_TRACESTOP: 1417 case INTEL_PT_BAD: 1418 case INTEL_PT_PSB: 1419 decoder->have_tma = false; 1420 intel_pt_log("ERROR: Unexpected packet\n"); 1421 return -EAGAIN; 1422 1423 case INTEL_PT_OVF: 1424 return intel_pt_overflow(decoder); 1425 1426 case INTEL_PT_TSC: 1427 intel_pt_calc_tsc_timestamp(decoder); 1428 break; 1429 1430 case INTEL_PT_TMA: 1431 intel_pt_calc_tma(decoder); 1432 break; 1433 1434 case INTEL_PT_CBR: 1435 intel_pt_calc_cbr(decoder); 1436 break; 1437 1438 case INTEL_PT_MODE_EXEC: 1439 decoder->exec_mode = decoder->packet.payload; 1440 break; 1441 1442 case INTEL_PT_PIP: 1443 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1444 break; 1445 1446 case INTEL_PT_FUP: 1447 decoder->pge = true; 1448 intel_pt_set_last_ip(decoder); 1449 break; 1450 1451 case INTEL_PT_MODE_TSX: 1452 intel_pt_update_in_tx(decoder); 1453 break; 1454 1455 case INTEL_PT_MTC: 1456 intel_pt_calc_mtc_timestamp(decoder); 1457 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1458 decoder->state.type |= INTEL_PT_INSTRUCTION; 1459 break; 1460 1461 case INTEL_PT_CYC: 1462 case INTEL_PT_VMCS: 1463 case INTEL_PT_MNT: 1464 case INTEL_PT_PAD: 1465 default: 1466 break; 1467 } 1468 } 1469 } 1470 1471 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1472 { 1473 int err; 1474 1475 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1476 decoder->tx_flags = 0; 1477 decoder->state.flags &= ~INTEL_PT_IN_TX; 1478 decoder->state.flags |= INTEL_PT_ABORT_TX; 1479 } else { 1480 decoder->state.flags |= INTEL_PT_ASYNC; 1481 } 1482 1483 while (1) { 1484 err = intel_pt_get_next_packet(decoder); 1485 if (err) 1486 return err; 1487 1488 switch (decoder->packet.type) { 1489 case INTEL_PT_TNT: 1490 case INTEL_PT_FUP: 1491 case INTEL_PT_TRACESTOP: 1492 case INTEL_PT_PSB: 1493 case INTEL_PT_TSC: 1494 case INTEL_PT_TMA: 1495 case INTEL_PT_CBR: 1496 case INTEL_PT_MODE_TSX: 1497 case INTEL_PT_BAD: 1498 case INTEL_PT_PSBEND: 1499 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1500 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1501 return -ENOENT; 1502 1503 case INTEL_PT_OVF: 1504 return intel_pt_overflow(decoder); 1505 1506 case INTEL_PT_TIP_PGD: 1507 decoder->state.from_ip = decoder->ip; 1508 decoder->state.to_ip = 0; 1509 if (decoder->packet.count != 0) { 1510 intel_pt_set_ip(decoder); 1511 intel_pt_log("Omitting PGD ip " x64_fmt "\n", 1512 decoder->ip); 1513 } 1514 decoder->pge = false; 1515 decoder->continuous_period = false; 1516 return 0; 1517 1518 case INTEL_PT_TIP_PGE: 1519 decoder->pge = true; 1520 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1521 decoder->ip); 1522 decoder->state.from_ip = 0; 1523 if (decoder->packet.count == 0) { 1524 decoder->state.to_ip = 0; 1525 } else { 1526 intel_pt_set_ip(decoder); 1527 decoder->state.to_ip = decoder->ip; 1528 } 1529 return 0; 1530 1531 case INTEL_PT_TIP: 1532 decoder->state.from_ip = decoder->ip; 1533 if (decoder->packet.count == 0) { 1534 decoder->state.to_ip = 0; 1535 } else { 1536 intel_pt_set_ip(decoder); 1537 decoder->state.to_ip = decoder->ip; 1538 } 1539 return 0; 1540 1541 case INTEL_PT_PIP: 1542 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1543 break; 1544 1545 case INTEL_PT_MTC: 1546 intel_pt_calc_mtc_timestamp(decoder); 1547 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1548 decoder->state.type |= INTEL_PT_INSTRUCTION; 1549 break; 1550 1551 case INTEL_PT_CYC: 1552 intel_pt_calc_cyc_timestamp(decoder); 1553 break; 1554 1555 case INTEL_PT_MODE_EXEC: 1556 decoder->exec_mode = decoder->packet.payload; 1557 break; 1558 1559 case INTEL_PT_VMCS: 1560 case INTEL_PT_MNT: 1561 case INTEL_PT_PAD: 1562 break; 1563 1564 default: 1565 return intel_pt_bug(decoder); 1566 } 1567 } 1568 } 1569 1570 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 1571 { 1572 bool no_tip = false; 1573 int err; 1574 1575 while (1) { 1576 err = intel_pt_get_next_packet(decoder); 1577 if (err) 1578 return err; 1579 next: 1580 switch (decoder->packet.type) { 1581 case INTEL_PT_TNT: 1582 if (!decoder->packet.count) 1583 break; 1584 decoder->tnt = decoder->packet; 1585 decoder->pkt_state = INTEL_PT_STATE_TNT; 1586 err = intel_pt_walk_tnt(decoder); 1587 if (err == -EAGAIN) 1588 break; 1589 return err; 1590 1591 case INTEL_PT_TIP_PGD: 1592 if (decoder->packet.count != 0) 1593 intel_pt_set_last_ip(decoder); 1594 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 1595 return intel_pt_walk_tip(decoder); 1596 1597 case INTEL_PT_TIP_PGE: { 1598 decoder->pge = true; 1599 if (decoder->packet.count == 0) { 1600 intel_pt_log_at("Skipping zero TIP.PGE", 1601 decoder->pos); 1602 break; 1603 } 1604 intel_pt_set_ip(decoder); 1605 decoder->state.from_ip = 0; 1606 decoder->state.to_ip = decoder->ip; 1607 return 0; 1608 } 1609 1610 case INTEL_PT_OVF: 1611 return intel_pt_overflow(decoder); 1612 1613 case INTEL_PT_TIP: 1614 if (decoder->packet.count != 0) 1615 intel_pt_set_last_ip(decoder); 1616 decoder->pkt_state = INTEL_PT_STATE_TIP; 1617 return intel_pt_walk_tip(decoder); 1618 1619 case INTEL_PT_FUP: 1620 if (decoder->packet.count == 0) { 1621 intel_pt_log_at("Skipping zero FUP", 1622 decoder->pos); 1623 no_tip = false; 1624 break; 1625 } 1626 intel_pt_set_last_ip(decoder); 1627 err = intel_pt_walk_fup(decoder); 1628 if (err != -EAGAIN) { 1629 if (err) 1630 return err; 1631 if (no_tip) 1632 decoder->pkt_state = 1633 INTEL_PT_STATE_FUP_NO_TIP; 1634 else 1635 decoder->pkt_state = INTEL_PT_STATE_FUP; 1636 return 0; 1637 } 1638 if (no_tip) { 1639 no_tip = false; 1640 break; 1641 } 1642 return intel_pt_walk_fup_tip(decoder); 1643 1644 case INTEL_PT_TRACESTOP: 1645 decoder->pge = false; 1646 decoder->continuous_period = false; 1647 intel_pt_clear_tx_flags(decoder); 1648 decoder->have_tma = false; 1649 break; 1650 1651 case INTEL_PT_PSB: 1652 intel_pt_clear_stack(&decoder->stack); 1653 err = intel_pt_walk_psbend(decoder); 1654 if (err == -EAGAIN) 1655 goto next; 1656 if (err) 1657 return err; 1658 break; 1659 1660 case INTEL_PT_PIP: 1661 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1662 break; 1663 1664 case INTEL_PT_MTC: 1665 intel_pt_calc_mtc_timestamp(decoder); 1666 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 1667 break; 1668 /* 1669 * Ensure that there has been an instruction since the 1670 * last MTC. 1671 */ 1672 if (!decoder->mtc_insn) 1673 break; 1674 decoder->mtc_insn = false; 1675 /* Ensure that there is a timestamp */ 1676 if (!decoder->timestamp) 1677 break; 1678 decoder->state.type = INTEL_PT_INSTRUCTION; 1679 decoder->state.from_ip = decoder->ip; 1680 decoder->state.to_ip = 0; 1681 decoder->mtc_insn = false; 1682 return 0; 1683 1684 case INTEL_PT_TSC: 1685 intel_pt_calc_tsc_timestamp(decoder); 1686 break; 1687 1688 case INTEL_PT_TMA: 1689 intel_pt_calc_tma(decoder); 1690 break; 1691 1692 case INTEL_PT_CYC: 1693 intel_pt_calc_cyc_timestamp(decoder); 1694 break; 1695 1696 case INTEL_PT_CBR: 1697 intel_pt_calc_cbr(decoder); 1698 break; 1699 1700 case INTEL_PT_MODE_EXEC: 1701 decoder->exec_mode = decoder->packet.payload; 1702 break; 1703 1704 case INTEL_PT_MODE_TSX: 1705 /* MODE_TSX need not be followed by FUP */ 1706 if (!decoder->pge) { 1707 intel_pt_update_in_tx(decoder); 1708 break; 1709 } 1710 err = intel_pt_mode_tsx(decoder, &no_tip); 1711 if (err) 1712 return err; 1713 goto next; 1714 1715 case INTEL_PT_BAD: /* Does not happen */ 1716 return intel_pt_bug(decoder); 1717 1718 case INTEL_PT_PSBEND: 1719 case INTEL_PT_VMCS: 1720 case INTEL_PT_MNT: 1721 case INTEL_PT_PAD: 1722 break; 1723 1724 default: 1725 return intel_pt_bug(decoder); 1726 } 1727 } 1728 } 1729 1730 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 1731 { 1732 return decoder->last_ip || decoder->packet.count == 0 || 1733 decoder->packet.count == 3 || decoder->packet.count == 6; 1734 } 1735 1736 /* Walk PSB+ packets to get in sync. */ 1737 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 1738 { 1739 int err; 1740 1741 while (1) { 1742 err = intel_pt_get_next_packet(decoder); 1743 if (err) 1744 return err; 1745 1746 switch (decoder->packet.type) { 1747 case INTEL_PT_TIP_PGD: 1748 decoder->continuous_period = false; 1749 case INTEL_PT_TIP_PGE: 1750 case INTEL_PT_TIP: 1751 intel_pt_log("ERROR: Unexpected packet\n"); 1752 return -ENOENT; 1753 1754 case INTEL_PT_FUP: 1755 decoder->pge = true; 1756 if (intel_pt_have_ip(decoder)) { 1757 uint64_t current_ip = decoder->ip; 1758 1759 intel_pt_set_ip(decoder); 1760 if (current_ip) 1761 intel_pt_log_to("Setting IP", 1762 decoder->ip); 1763 } 1764 break; 1765 1766 case INTEL_PT_MTC: 1767 intel_pt_calc_mtc_timestamp(decoder); 1768 break; 1769 1770 case INTEL_PT_TSC: 1771 intel_pt_calc_tsc_timestamp(decoder); 1772 break; 1773 1774 case INTEL_PT_TMA: 1775 intel_pt_calc_tma(decoder); 1776 break; 1777 1778 case INTEL_PT_CYC: 1779 intel_pt_calc_cyc_timestamp(decoder); 1780 break; 1781 1782 case INTEL_PT_CBR: 1783 intel_pt_calc_cbr(decoder); 1784 break; 1785 1786 case INTEL_PT_PIP: 1787 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1788 break; 1789 1790 case INTEL_PT_MODE_EXEC: 1791 decoder->exec_mode = decoder->packet.payload; 1792 break; 1793 1794 case INTEL_PT_MODE_TSX: 1795 intel_pt_update_in_tx(decoder); 1796 break; 1797 1798 case INTEL_PT_TRACESTOP: 1799 decoder->pge = false; 1800 decoder->continuous_period = false; 1801 intel_pt_clear_tx_flags(decoder); 1802 case INTEL_PT_TNT: 1803 decoder->have_tma = false; 1804 intel_pt_log("ERROR: Unexpected packet\n"); 1805 if (decoder->ip) 1806 decoder->pkt_state = INTEL_PT_STATE_ERR4; 1807 else 1808 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1809 return -ENOENT; 1810 1811 case INTEL_PT_BAD: /* Does not happen */ 1812 return intel_pt_bug(decoder); 1813 1814 case INTEL_PT_OVF: 1815 return intel_pt_overflow(decoder); 1816 1817 case INTEL_PT_PSBEND: 1818 return 0; 1819 1820 case INTEL_PT_PSB: 1821 case INTEL_PT_VMCS: 1822 case INTEL_PT_MNT: 1823 case INTEL_PT_PAD: 1824 default: 1825 break; 1826 } 1827 } 1828 } 1829 1830 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 1831 { 1832 int err; 1833 1834 while (1) { 1835 err = intel_pt_get_next_packet(decoder); 1836 if (err) 1837 return err; 1838 1839 switch (decoder->packet.type) { 1840 case INTEL_PT_TIP_PGD: 1841 decoder->continuous_period = false; 1842 case INTEL_PT_TIP_PGE: 1843 case INTEL_PT_TIP: 1844 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD; 1845 if (intel_pt_have_ip(decoder)) 1846 intel_pt_set_ip(decoder); 1847 if (decoder->ip) 1848 return 0; 1849 break; 1850 1851 case INTEL_PT_FUP: 1852 if (decoder->overflow) { 1853 if (intel_pt_have_ip(decoder)) 1854 intel_pt_set_ip(decoder); 1855 if (decoder->ip) 1856 return 0; 1857 } 1858 if (decoder->packet.count) 1859 intel_pt_set_last_ip(decoder); 1860 break; 1861 1862 case INTEL_PT_MTC: 1863 intel_pt_calc_mtc_timestamp(decoder); 1864 break; 1865 1866 case INTEL_PT_TSC: 1867 intel_pt_calc_tsc_timestamp(decoder); 1868 break; 1869 1870 case INTEL_PT_TMA: 1871 intel_pt_calc_tma(decoder); 1872 break; 1873 1874 case INTEL_PT_CYC: 1875 intel_pt_calc_cyc_timestamp(decoder); 1876 break; 1877 1878 case INTEL_PT_CBR: 1879 intel_pt_calc_cbr(decoder); 1880 break; 1881 1882 case INTEL_PT_PIP: 1883 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1884 break; 1885 1886 case INTEL_PT_MODE_EXEC: 1887 decoder->exec_mode = decoder->packet.payload; 1888 break; 1889 1890 case INTEL_PT_MODE_TSX: 1891 intel_pt_update_in_tx(decoder); 1892 break; 1893 1894 case INTEL_PT_OVF: 1895 return intel_pt_overflow(decoder); 1896 1897 case INTEL_PT_BAD: /* Does not happen */ 1898 return intel_pt_bug(decoder); 1899 1900 case INTEL_PT_TRACESTOP: 1901 decoder->pge = false; 1902 decoder->continuous_period = false; 1903 intel_pt_clear_tx_flags(decoder); 1904 decoder->have_tma = false; 1905 break; 1906 1907 case INTEL_PT_PSB: 1908 err = intel_pt_walk_psb(decoder); 1909 if (err) 1910 return err; 1911 if (decoder->ip) { 1912 /* Do not have a sample */ 1913 decoder->state.type = 0; 1914 return 0; 1915 } 1916 break; 1917 1918 case INTEL_PT_TNT: 1919 case INTEL_PT_PSBEND: 1920 case INTEL_PT_VMCS: 1921 case INTEL_PT_MNT: 1922 case INTEL_PT_PAD: 1923 default: 1924 break; 1925 } 1926 } 1927 } 1928 1929 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 1930 { 1931 int err; 1932 1933 intel_pt_log("Scanning for full IP\n"); 1934 err = intel_pt_walk_to_ip(decoder); 1935 if (err) 1936 return err; 1937 1938 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1939 decoder->overflow = false; 1940 1941 decoder->state.from_ip = 0; 1942 decoder->state.to_ip = decoder->ip; 1943 intel_pt_log_to("Setting IP", decoder->ip); 1944 1945 return 0; 1946 } 1947 1948 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 1949 { 1950 const unsigned char *end = decoder->buf + decoder->len; 1951 size_t i; 1952 1953 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 1954 if (i > decoder->len) 1955 continue; 1956 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 1957 return i; 1958 } 1959 return 0; 1960 } 1961 1962 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 1963 { 1964 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 1965 const char *psb = INTEL_PT_PSB_STR; 1966 1967 if (rest_psb > decoder->len || 1968 memcmp(decoder->buf, psb + part_psb, rest_psb)) 1969 return 0; 1970 1971 return rest_psb; 1972 } 1973 1974 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 1975 int part_psb) 1976 { 1977 int rest_psb, ret; 1978 1979 decoder->pos += decoder->len; 1980 decoder->len = 0; 1981 1982 ret = intel_pt_get_next_data(decoder); 1983 if (ret) 1984 return ret; 1985 1986 rest_psb = intel_pt_rest_psb(decoder, part_psb); 1987 if (!rest_psb) 1988 return 0; 1989 1990 decoder->pos -= part_psb; 1991 decoder->next_buf = decoder->buf + rest_psb; 1992 decoder->next_len = decoder->len - rest_psb; 1993 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 1994 decoder->buf = decoder->temp_buf; 1995 decoder->len = INTEL_PT_PSB_LEN; 1996 1997 return 0; 1998 } 1999 2000 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 2001 { 2002 unsigned char *next; 2003 int ret; 2004 2005 intel_pt_log("Scanning for PSB\n"); 2006 while (1) { 2007 if (!decoder->len) { 2008 ret = intel_pt_get_next_data(decoder); 2009 if (ret) 2010 return ret; 2011 } 2012 2013 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2014 INTEL_PT_PSB_LEN); 2015 if (!next) { 2016 int part_psb; 2017 2018 part_psb = intel_pt_part_psb(decoder); 2019 if (part_psb) { 2020 ret = intel_pt_get_split_psb(decoder, part_psb); 2021 if (ret) 2022 return ret; 2023 } else { 2024 decoder->pos += decoder->len; 2025 decoder->len = 0; 2026 } 2027 continue; 2028 } 2029 2030 decoder->pkt_step = next - decoder->buf; 2031 return intel_pt_get_next_packet(decoder); 2032 } 2033 } 2034 2035 static int intel_pt_sync(struct intel_pt_decoder *decoder) 2036 { 2037 int err; 2038 2039 decoder->pge = false; 2040 decoder->continuous_period = false; 2041 decoder->last_ip = 0; 2042 decoder->ip = 0; 2043 intel_pt_clear_stack(&decoder->stack); 2044 2045 err = intel_pt_scan_for_psb(decoder); 2046 if (err) 2047 return err; 2048 2049 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 2050 2051 err = intel_pt_walk_psb(decoder); 2052 if (err) 2053 return err; 2054 2055 if (decoder->ip) { 2056 decoder->state.type = 0; /* Do not have a sample */ 2057 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2058 } else { 2059 return intel_pt_sync_ip(decoder); 2060 } 2061 2062 return 0; 2063 } 2064 2065 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2066 { 2067 uint64_t est = decoder->timestamp_insn_cnt << 1; 2068 2069 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2070 goto out; 2071 2072 est *= decoder->max_non_turbo_ratio; 2073 est /= decoder->cbr; 2074 out: 2075 return decoder->timestamp + est; 2076 } 2077 2078 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2079 { 2080 int err; 2081 2082 do { 2083 decoder->state.type = INTEL_PT_BRANCH; 2084 decoder->state.flags = 0; 2085 2086 switch (decoder->pkt_state) { 2087 case INTEL_PT_STATE_NO_PSB: 2088 err = intel_pt_sync(decoder); 2089 break; 2090 case INTEL_PT_STATE_NO_IP: 2091 decoder->last_ip = 0; 2092 /* Fall through */ 2093 case INTEL_PT_STATE_ERR_RESYNC: 2094 err = intel_pt_sync_ip(decoder); 2095 break; 2096 case INTEL_PT_STATE_IN_SYNC: 2097 err = intel_pt_walk_trace(decoder); 2098 break; 2099 case INTEL_PT_STATE_TNT: 2100 err = intel_pt_walk_tnt(decoder); 2101 if (err == -EAGAIN) 2102 err = intel_pt_walk_trace(decoder); 2103 break; 2104 case INTEL_PT_STATE_TIP: 2105 case INTEL_PT_STATE_TIP_PGD: 2106 err = intel_pt_walk_tip(decoder); 2107 break; 2108 case INTEL_PT_STATE_FUP: 2109 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2110 err = intel_pt_walk_fup(decoder); 2111 if (err == -EAGAIN) 2112 err = intel_pt_walk_fup_tip(decoder); 2113 else if (!err) 2114 decoder->pkt_state = INTEL_PT_STATE_FUP; 2115 break; 2116 case INTEL_PT_STATE_FUP_NO_TIP: 2117 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2118 err = intel_pt_walk_fup(decoder); 2119 if (err == -EAGAIN) 2120 err = intel_pt_walk_trace(decoder); 2121 break; 2122 default: 2123 err = intel_pt_bug(decoder); 2124 break; 2125 } 2126 } while (err == -ENOLINK); 2127 2128 decoder->state.err = err ? intel_pt_ext_err(err) : 0; 2129 decoder->state.timestamp = decoder->timestamp; 2130 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2131 decoder->state.cr3 = decoder->cr3; 2132 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2133 2134 if (err) 2135 decoder->state.from_ip = decoder->ip; 2136 2137 return &decoder->state; 2138 } 2139 2140 static bool intel_pt_at_psb(unsigned char *buf, size_t len) 2141 { 2142 if (len < INTEL_PT_PSB_LEN) 2143 return false; 2144 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR, 2145 INTEL_PT_PSB_LEN); 2146 } 2147 2148 /** 2149 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2150 * @buf: pointer to buffer pointer 2151 * @len: size of buffer 2152 * 2153 * Updates the buffer pointer to point to the start of the next PSB packet if 2154 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2155 * @len is adjusted accordingly. 2156 * 2157 * Return: %true if a PSB packet is found, %false otherwise. 2158 */ 2159 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2160 { 2161 unsigned char *next; 2162 2163 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2164 if (next) { 2165 *len -= next - *buf; 2166 *buf = next; 2167 return true; 2168 } 2169 return false; 2170 } 2171 2172 /** 2173 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2174 * packet. 2175 * @buf: pointer to buffer pointer 2176 * @len: size of buffer 2177 * 2178 * Updates the buffer pointer to point to the start of the following PSB packet 2179 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2180 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2181 * 2182 * Return: %true if a PSB packet is found, %false otherwise. 2183 */ 2184 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2185 { 2186 unsigned char *next; 2187 2188 if (!*len) 2189 return false; 2190 2191 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2192 if (next) { 2193 *len -= next - *buf; 2194 *buf = next; 2195 return true; 2196 } 2197 return false; 2198 } 2199 2200 /** 2201 * intel_pt_last_psb - find the last PSB packet in a buffer. 2202 * @buf: buffer 2203 * @len: size of buffer 2204 * 2205 * This function finds the last PSB in a buffer. 2206 * 2207 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2208 */ 2209 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2210 { 2211 const char *n = INTEL_PT_PSB_STR; 2212 unsigned char *p; 2213 size_t k; 2214 2215 if (len < INTEL_PT_PSB_LEN) 2216 return NULL; 2217 2218 k = len - INTEL_PT_PSB_LEN + 1; 2219 while (1) { 2220 p = memrchr(buf, n[0], k); 2221 if (!p) 2222 return NULL; 2223 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2224 return p; 2225 k = p - buf; 2226 if (!k) 2227 return NULL; 2228 } 2229 } 2230 2231 /** 2232 * intel_pt_next_tsc - find and return next TSC. 2233 * @buf: buffer 2234 * @len: size of buffer 2235 * @tsc: TSC value returned 2236 * 2237 * Find a TSC packet in @buf and return the TSC value. This function assumes 2238 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2239 * PSBEND packet is found. 2240 * 2241 * Return: %true if TSC is found, false otherwise. 2242 */ 2243 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc) 2244 { 2245 struct intel_pt_pkt packet; 2246 int ret; 2247 2248 while (len) { 2249 ret = intel_pt_get_packet(buf, len, &packet); 2250 if (ret <= 0) 2251 return false; 2252 if (packet.type == INTEL_PT_TSC) { 2253 *tsc = packet.payload; 2254 return true; 2255 } 2256 if (packet.type == INTEL_PT_PSBEND) 2257 return false; 2258 buf += ret; 2259 len -= ret; 2260 } 2261 return false; 2262 } 2263 2264 /** 2265 * intel_pt_tsc_cmp - compare 7-byte TSCs. 2266 * @tsc1: first TSC to compare 2267 * @tsc2: second TSC to compare 2268 * 2269 * This function compares 7-byte TSC values allowing for the possibility that 2270 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 2271 * around so for that purpose this function assumes the absolute difference is 2272 * less than half the maximum difference. 2273 * 2274 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 2275 * after @tsc2. 2276 */ 2277 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 2278 { 2279 const uint64_t halfway = (1ULL << 55); 2280 2281 if (tsc1 == tsc2) 2282 return 0; 2283 2284 if (tsc1 < tsc2) { 2285 if (tsc2 - tsc1 < halfway) 2286 return -1; 2287 else 2288 return 1; 2289 } else { 2290 if (tsc1 - tsc2 < halfway) 2291 return 1; 2292 else 2293 return -1; 2294 } 2295 } 2296 2297 /** 2298 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 2299 * using TSC. 2300 * @buf_a: first buffer 2301 * @len_a: size of first buffer 2302 * @buf_b: second buffer 2303 * @len_b: size of second buffer 2304 * 2305 * If the trace contains TSC we can look at the last TSC of @buf_a and the 2306 * first TSC of @buf_b in order to determine if the buffers overlap, and then 2307 * walk forward in @buf_b until a later TSC is found. A precondition is that 2308 * @buf_a and @buf_b are positioned at a PSB. 2309 * 2310 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2311 * @buf_b + @len_b if there is no non-overlapped data. 2312 */ 2313 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 2314 size_t len_a, 2315 unsigned char *buf_b, 2316 size_t len_b) 2317 { 2318 uint64_t tsc_a, tsc_b; 2319 unsigned char *p; 2320 size_t len; 2321 2322 p = intel_pt_last_psb(buf_a, len_a); 2323 if (!p) 2324 return buf_b; /* No PSB in buf_a => no overlap */ 2325 2326 len = len_a - (p - buf_a); 2327 if (!intel_pt_next_tsc(p, len, &tsc_a)) { 2328 /* The last PSB+ in buf_a is incomplete, so go back one more */ 2329 len_a -= len; 2330 p = intel_pt_last_psb(buf_a, len_a); 2331 if (!p) 2332 return buf_b; /* No full PSB+ => assume no overlap */ 2333 len = len_a - (p - buf_a); 2334 if (!intel_pt_next_tsc(p, len, &tsc_a)) 2335 return buf_b; /* No TSC in buf_a => assume no overlap */ 2336 } 2337 2338 while (1) { 2339 /* Ignore PSB+ with no TSC */ 2340 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) && 2341 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0) 2342 return buf_b; /* tsc_a < tsc_b => no overlap */ 2343 2344 if (!intel_pt_step_psb(&buf_b, &len_b)) 2345 return buf_b + len_b; /* No PSB in buf_b => no data */ 2346 } 2347 } 2348 2349 /** 2350 * intel_pt_find_overlap - determine start of non-overlapped trace data. 2351 * @buf_a: first buffer 2352 * @len_a: size of first buffer 2353 * @buf_b: second buffer 2354 * @len_b: size of second buffer 2355 * @have_tsc: can use TSC packets to detect overlap 2356 * 2357 * When trace samples or snapshots are recorded there is the possibility that 2358 * the data overlaps. Note that, for the purposes of decoding, data is only 2359 * useful if it begins with a PSB packet. 2360 * 2361 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2362 * @buf_b + @len_b if there is no non-overlapped data. 2363 */ 2364 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 2365 unsigned char *buf_b, size_t len_b, 2366 bool have_tsc) 2367 { 2368 unsigned char *found; 2369 2370 /* Buffer 'b' must start at PSB so throw away everything before that */ 2371 if (!intel_pt_next_psb(&buf_b, &len_b)) 2372 return buf_b + len_b; /* No PSB */ 2373 2374 if (!intel_pt_next_psb(&buf_a, &len_a)) 2375 return buf_b; /* No overlap */ 2376 2377 if (have_tsc) { 2378 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b); 2379 if (found) 2380 return found; 2381 } 2382 2383 /* 2384 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 2385 * we can ignore the first part of buffer 'a'. 2386 */ 2387 while (len_b < len_a) { 2388 if (!intel_pt_step_psb(&buf_a, &len_a)) 2389 return buf_b; /* No overlap */ 2390 } 2391 2392 /* Now len_b >= len_a */ 2393 if (len_b > len_a) { 2394 /* The leftover buffer 'b' must start at a PSB */ 2395 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2396 if (!intel_pt_step_psb(&buf_a, &len_a)) 2397 return buf_b; /* No overlap */ 2398 } 2399 } 2400 2401 while (1) { 2402 /* Potential overlap so check the bytes */ 2403 found = memmem(buf_a, len_a, buf_b, len_a); 2404 if (found) 2405 return buf_b + len_a; 2406 2407 /* Try again at next PSB in buffer 'a' */ 2408 if (!intel_pt_step_psb(&buf_a, &len_a)) 2409 return buf_b; /* No overlap */ 2410 2411 /* The leftover buffer 'b' must start at a PSB */ 2412 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2413 if (!intel_pt_step_psb(&buf_a, &len_a)) 2414 return buf_b; /* No overlap */ 2415 } 2416 } 2417 } 2418