1 /* 2 * intel_pt_decoder.c: Intel Processor Trace support 3 * Copyright (c) 2013-2014, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #ifndef _GNU_SOURCE 17 #define _GNU_SOURCE 18 #endif 19 #include <stdlib.h> 20 #include <stdbool.h> 21 #include <string.h> 22 #include <errno.h> 23 #include <stdint.h> 24 #include <inttypes.h> 25 26 #include "../cache.h" 27 #include "../util.h" 28 29 #include "intel-pt-insn-decoder.h" 30 #include "intel-pt-pkt-decoder.h" 31 #include "intel-pt-decoder.h" 32 #include "intel-pt-log.h" 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define INTEL_PT_RETURN 1 39 40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 41 #define INTEL_PT_MAX_LOOPS 10000 42 43 struct intel_pt_blk { 44 struct intel_pt_blk *prev; 45 uint64_t ip[INTEL_PT_BLK_SIZE]; 46 }; 47 48 struct intel_pt_stack { 49 struct intel_pt_blk *blk; 50 struct intel_pt_blk *spare; 51 int pos; 52 }; 53 54 enum intel_pt_pkt_state { 55 INTEL_PT_STATE_NO_PSB, 56 INTEL_PT_STATE_NO_IP, 57 INTEL_PT_STATE_ERR_RESYNC, 58 INTEL_PT_STATE_IN_SYNC, 59 INTEL_PT_STATE_TNT, 60 INTEL_PT_STATE_TIP, 61 INTEL_PT_STATE_TIP_PGD, 62 INTEL_PT_STATE_FUP, 63 INTEL_PT_STATE_FUP_NO_TIP, 64 }; 65 66 #ifdef INTEL_PT_STRICT 67 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 68 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 69 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 70 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 71 #else 72 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 73 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 74 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 75 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 76 #endif 77 78 struct intel_pt_decoder { 79 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 80 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 81 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 82 uint64_t max_insn_cnt, void *data); 83 bool (*pgd_ip)(uint64_t ip, void *data); 84 void *data; 85 struct intel_pt_state state; 86 const unsigned char *buf; 87 size_t len; 88 bool return_compression; 89 bool mtc_insn; 90 bool pge; 91 bool have_tma; 92 bool have_cyc; 93 bool fixup_last_mtc; 94 uint64_t pos; 95 uint64_t last_ip; 96 uint64_t ip; 97 uint64_t cr3; 98 uint64_t timestamp; 99 uint64_t tsc_timestamp; 100 uint64_t ref_timestamp; 101 uint64_t ret_addr; 102 uint64_t ctc_timestamp; 103 uint64_t ctc_delta; 104 uint64_t cycle_cnt; 105 uint64_t cyc_ref_timestamp; 106 uint32_t last_mtc; 107 uint32_t tsc_ctc_ratio_n; 108 uint32_t tsc_ctc_ratio_d; 109 uint32_t tsc_ctc_mult; 110 uint32_t tsc_slip; 111 uint32_t ctc_rem_mask; 112 int mtc_shift; 113 struct intel_pt_stack stack; 114 enum intel_pt_pkt_state pkt_state; 115 struct intel_pt_pkt packet; 116 struct intel_pt_pkt tnt; 117 int pkt_step; 118 int pkt_len; 119 int last_packet_type; 120 unsigned int cbr; 121 unsigned int max_non_turbo_ratio; 122 double max_non_turbo_ratio_fp; 123 double cbr_cyc_to_tsc; 124 double calc_cyc_to_tsc; 125 bool have_calc_cyc_to_tsc; 126 int exec_mode; 127 unsigned int insn_bytes; 128 uint64_t period; 129 enum intel_pt_period_type period_type; 130 uint64_t tot_insn_cnt; 131 uint64_t period_insn_cnt; 132 uint64_t period_mask; 133 uint64_t period_ticks; 134 uint64_t last_masked_timestamp; 135 bool continuous_period; 136 bool overflow; 137 bool set_fup_tx_flags; 138 unsigned int fup_tx_flags; 139 unsigned int tx_flags; 140 uint64_t timestamp_insn_cnt; 141 uint64_t stuck_ip; 142 int no_progress; 143 int stuck_ip_prd; 144 int stuck_ip_cnt; 145 const unsigned char *next_buf; 146 size_t next_len; 147 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 148 }; 149 150 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 151 { 152 int i; 153 154 for (i = 0; x != 1; i++) 155 x >>= 1; 156 157 return x << i; 158 } 159 160 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 161 { 162 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 163 uint64_t period; 164 165 period = intel_pt_lower_power_of_2(decoder->period); 166 decoder->period_mask = ~(period - 1); 167 decoder->period_ticks = period; 168 } 169 } 170 171 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 172 { 173 if (!d) 174 return 0; 175 return (t / d) * n + ((t % d) * n) / d; 176 } 177 178 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 179 { 180 struct intel_pt_decoder *decoder; 181 182 if (!params->get_trace || !params->walk_insn) 183 return NULL; 184 185 decoder = zalloc(sizeof(struct intel_pt_decoder)); 186 if (!decoder) 187 return NULL; 188 189 decoder->get_trace = params->get_trace; 190 decoder->walk_insn = params->walk_insn; 191 decoder->pgd_ip = params->pgd_ip; 192 decoder->data = params->data; 193 decoder->return_compression = params->return_compression; 194 195 decoder->period = params->period; 196 decoder->period_type = params->period_type; 197 198 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 199 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 200 201 intel_pt_setup_period(decoder); 202 203 decoder->mtc_shift = params->mtc_period; 204 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 205 206 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 207 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 208 209 if (!decoder->tsc_ctc_ratio_n) 210 decoder->tsc_ctc_ratio_d = 0; 211 212 if (decoder->tsc_ctc_ratio_d) { 213 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 214 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 215 decoder->tsc_ctc_ratio_d; 216 217 /* 218 * Allow for timestamps appearing to backwards because a TSC 219 * packet has slipped past a MTC packet, so allow 2 MTC ticks 220 * or ... 221 */ 222 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift, 223 decoder->tsc_ctc_ratio_n, 224 decoder->tsc_ctc_ratio_d); 225 } 226 /* ... or 0x100 paranoia */ 227 if (decoder->tsc_slip < 0x100) 228 decoder->tsc_slip = 0x100; 229 230 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 231 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 232 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 233 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 234 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 235 236 return decoder; 237 } 238 239 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 240 { 241 struct intel_pt_blk *blk = stack->blk; 242 243 stack->blk = blk->prev; 244 if (!stack->spare) 245 stack->spare = blk; 246 else 247 free(blk); 248 } 249 250 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 251 { 252 if (!stack->pos) { 253 if (!stack->blk) 254 return 0; 255 intel_pt_pop_blk(stack); 256 if (!stack->blk) 257 return 0; 258 stack->pos = INTEL_PT_BLK_SIZE; 259 } 260 return stack->blk->ip[--stack->pos]; 261 } 262 263 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 264 { 265 struct intel_pt_blk *blk; 266 267 if (stack->spare) { 268 blk = stack->spare; 269 stack->spare = NULL; 270 } else { 271 blk = malloc(sizeof(struct intel_pt_blk)); 272 if (!blk) 273 return -ENOMEM; 274 } 275 276 blk->prev = stack->blk; 277 stack->blk = blk; 278 stack->pos = 0; 279 return 0; 280 } 281 282 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 283 { 284 int err; 285 286 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 287 err = intel_pt_alloc_blk(stack); 288 if (err) 289 return err; 290 } 291 292 stack->blk->ip[stack->pos++] = ip; 293 return 0; 294 } 295 296 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 297 { 298 while (stack->blk) 299 intel_pt_pop_blk(stack); 300 stack->pos = 0; 301 } 302 303 static void intel_pt_free_stack(struct intel_pt_stack *stack) 304 { 305 intel_pt_clear_stack(stack); 306 zfree(&stack->blk); 307 zfree(&stack->spare); 308 } 309 310 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 311 { 312 intel_pt_free_stack(&decoder->stack); 313 free(decoder); 314 } 315 316 static int intel_pt_ext_err(int code) 317 { 318 switch (code) { 319 case -ENOMEM: 320 return INTEL_PT_ERR_NOMEM; 321 case -ENOSYS: 322 return INTEL_PT_ERR_INTERN; 323 case -EBADMSG: 324 return INTEL_PT_ERR_BADPKT; 325 case -ENODATA: 326 return INTEL_PT_ERR_NODATA; 327 case -EILSEQ: 328 return INTEL_PT_ERR_NOINSN; 329 case -ENOENT: 330 return INTEL_PT_ERR_MISMAT; 331 case -EOVERFLOW: 332 return INTEL_PT_ERR_OVR; 333 case -ENOSPC: 334 return INTEL_PT_ERR_LOST; 335 case -ELOOP: 336 return INTEL_PT_ERR_NELOOP; 337 default: 338 return INTEL_PT_ERR_UNK; 339 } 340 } 341 342 static const char *intel_pt_err_msgs[] = { 343 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 344 [INTEL_PT_ERR_INTERN] = "Internal error", 345 [INTEL_PT_ERR_BADPKT] = "Bad packet", 346 [INTEL_PT_ERR_NODATA] = "No more data", 347 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 348 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 349 [INTEL_PT_ERR_OVR] = "Overflow packet", 350 [INTEL_PT_ERR_LOST] = "Lost trace data", 351 [INTEL_PT_ERR_UNK] = "Unknown error!", 352 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 353 }; 354 355 int intel_pt__strerror(int code, char *buf, size_t buflen) 356 { 357 if (code < 1 || code >= INTEL_PT_ERR_MAX) 358 code = INTEL_PT_ERR_UNK; 359 strlcpy(buf, intel_pt_err_msgs[code], buflen); 360 return 0; 361 } 362 363 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 364 uint64_t last_ip) 365 { 366 uint64_t ip; 367 368 switch (packet->count) { 369 case 1: 370 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 371 packet->payload; 372 break; 373 case 2: 374 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 375 packet->payload; 376 break; 377 case 3: 378 ip = packet->payload; 379 /* Sign-extend 6-byte ip */ 380 if (ip & (uint64_t)0x800000000000ULL) 381 ip |= (uint64_t)0xffff000000000000ULL; 382 break; 383 case 4: 384 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 385 packet->payload; 386 break; 387 case 6: 388 ip = packet->payload; 389 break; 390 default: 391 return 0; 392 } 393 394 return ip; 395 } 396 397 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 398 { 399 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 400 } 401 402 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 403 { 404 intel_pt_set_last_ip(decoder); 405 decoder->ip = decoder->last_ip; 406 } 407 408 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 409 { 410 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 411 decoder->buf); 412 } 413 414 static int intel_pt_bug(struct intel_pt_decoder *decoder) 415 { 416 intel_pt_log("ERROR: Internal error\n"); 417 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 418 return -ENOSYS; 419 } 420 421 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 422 { 423 decoder->tx_flags = 0; 424 } 425 426 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 427 { 428 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 429 } 430 431 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 432 { 433 intel_pt_clear_tx_flags(decoder); 434 decoder->have_tma = false; 435 decoder->pkt_len = 1; 436 decoder->pkt_step = 1; 437 intel_pt_decoder_log_packet(decoder); 438 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 439 intel_pt_log("ERROR: Bad packet\n"); 440 decoder->pkt_state = INTEL_PT_STATE_ERR1; 441 } 442 return -EBADMSG; 443 } 444 445 static int intel_pt_get_data(struct intel_pt_decoder *decoder) 446 { 447 struct intel_pt_buffer buffer = { .buf = 0, }; 448 int ret; 449 450 decoder->pkt_step = 0; 451 452 intel_pt_log("Getting more data\n"); 453 ret = decoder->get_trace(&buffer, decoder->data); 454 if (ret) 455 return ret; 456 decoder->buf = buffer.buf; 457 decoder->len = buffer.len; 458 if (!decoder->len) { 459 intel_pt_log("No more data\n"); 460 return -ENODATA; 461 } 462 if (!buffer.consecutive) { 463 decoder->ip = 0; 464 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 465 decoder->ref_timestamp = buffer.ref_timestamp; 466 decoder->timestamp = 0; 467 decoder->have_tma = false; 468 decoder->state.trace_nr = buffer.trace_nr; 469 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 470 decoder->ref_timestamp); 471 return -ENOLINK; 472 } 473 474 return 0; 475 } 476 477 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder) 478 { 479 if (!decoder->next_buf) 480 return intel_pt_get_data(decoder); 481 482 decoder->buf = decoder->next_buf; 483 decoder->len = decoder->next_len; 484 decoder->next_buf = 0; 485 decoder->next_len = 0; 486 return 0; 487 } 488 489 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 490 { 491 unsigned char *buf = decoder->temp_buf; 492 size_t old_len, len, n; 493 int ret; 494 495 old_len = decoder->len; 496 len = decoder->len; 497 memcpy(buf, decoder->buf, len); 498 499 ret = intel_pt_get_data(decoder); 500 if (ret) { 501 decoder->pos += old_len; 502 return ret < 0 ? ret : -EINVAL; 503 } 504 505 n = INTEL_PT_PKT_MAX_SZ - len; 506 if (n > decoder->len) 507 n = decoder->len; 508 memcpy(buf + len, decoder->buf, n); 509 len += n; 510 511 ret = intel_pt_get_packet(buf, len, &decoder->packet); 512 if (ret < (int)old_len) { 513 decoder->next_buf = decoder->buf; 514 decoder->next_len = decoder->len; 515 decoder->buf = buf; 516 decoder->len = old_len; 517 return intel_pt_bad_packet(decoder); 518 } 519 520 decoder->next_buf = decoder->buf + (ret - old_len); 521 decoder->next_len = decoder->len - (ret - old_len); 522 523 decoder->buf = buf; 524 decoder->len = ret; 525 526 return ret; 527 } 528 529 struct intel_pt_pkt_info { 530 struct intel_pt_decoder *decoder; 531 struct intel_pt_pkt packet; 532 uint64_t pos; 533 int pkt_len; 534 int last_packet_type; 535 void *data; 536 }; 537 538 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 539 540 /* Lookahead packets in current buffer */ 541 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 542 intel_pt_pkt_cb_t cb, void *data) 543 { 544 struct intel_pt_pkt_info pkt_info; 545 const unsigned char *buf = decoder->buf; 546 size_t len = decoder->len; 547 int ret; 548 549 pkt_info.decoder = decoder; 550 pkt_info.pos = decoder->pos; 551 pkt_info.pkt_len = decoder->pkt_step; 552 pkt_info.last_packet_type = decoder->last_packet_type; 553 pkt_info.data = data; 554 555 while (1) { 556 do { 557 pkt_info.pos += pkt_info.pkt_len; 558 buf += pkt_info.pkt_len; 559 len -= pkt_info.pkt_len; 560 561 if (!len) 562 return INTEL_PT_NEED_MORE_BYTES; 563 564 ret = intel_pt_get_packet(buf, len, &pkt_info.packet); 565 if (!ret) 566 return INTEL_PT_NEED_MORE_BYTES; 567 if (ret < 0) 568 return ret; 569 570 pkt_info.pkt_len = ret; 571 } while (pkt_info.packet.type == INTEL_PT_PAD); 572 573 ret = cb(&pkt_info); 574 if (ret) 575 return 0; 576 577 pkt_info.last_packet_type = pkt_info.packet.type; 578 } 579 } 580 581 struct intel_pt_calc_cyc_to_tsc_info { 582 uint64_t cycle_cnt; 583 unsigned int cbr; 584 uint32_t last_mtc; 585 uint64_t ctc_timestamp; 586 uint64_t ctc_delta; 587 uint64_t tsc_timestamp; 588 uint64_t timestamp; 589 bool have_tma; 590 bool fixup_last_mtc; 591 bool from_mtc; 592 double cbr_cyc_to_tsc; 593 }; 594 595 /* 596 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 597 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 598 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 599 * packet by copying the missing bits from the current MTC assuming the least 600 * difference between the two, and that the current MTC comes after last_mtc. 601 */ 602 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 603 uint32_t *last_mtc) 604 { 605 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 606 uint32_t mask = ~(first_missing_bit - 1); 607 608 *last_mtc |= mtc & mask; 609 if (*last_mtc >= mtc) { 610 *last_mtc -= first_missing_bit; 611 *last_mtc &= 0xff; 612 } 613 } 614 615 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 616 { 617 struct intel_pt_decoder *decoder = pkt_info->decoder; 618 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 619 uint64_t timestamp; 620 double cyc_to_tsc; 621 unsigned int cbr; 622 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 623 624 switch (pkt_info->packet.type) { 625 case INTEL_PT_TNT: 626 case INTEL_PT_TIP_PGE: 627 case INTEL_PT_TIP: 628 case INTEL_PT_FUP: 629 case INTEL_PT_PSB: 630 case INTEL_PT_PIP: 631 case INTEL_PT_MODE_EXEC: 632 case INTEL_PT_MODE_TSX: 633 case INTEL_PT_PSBEND: 634 case INTEL_PT_PAD: 635 case INTEL_PT_VMCS: 636 case INTEL_PT_MNT: 637 return 0; 638 639 case INTEL_PT_MTC: 640 if (!data->have_tma) 641 return 0; 642 643 mtc = pkt_info->packet.payload; 644 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 645 data->fixup_last_mtc = false; 646 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 647 &data->last_mtc); 648 } 649 if (mtc > data->last_mtc) 650 mtc_delta = mtc - data->last_mtc; 651 else 652 mtc_delta = mtc + 256 - data->last_mtc; 653 data->ctc_delta += mtc_delta << decoder->mtc_shift; 654 data->last_mtc = mtc; 655 656 if (decoder->tsc_ctc_mult) { 657 timestamp = data->ctc_timestamp + 658 data->ctc_delta * decoder->tsc_ctc_mult; 659 } else { 660 timestamp = data->ctc_timestamp + 661 multdiv(data->ctc_delta, 662 decoder->tsc_ctc_ratio_n, 663 decoder->tsc_ctc_ratio_d); 664 } 665 666 if (timestamp < data->timestamp) 667 return 1; 668 669 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 670 data->timestamp = timestamp; 671 return 0; 672 } 673 674 break; 675 676 case INTEL_PT_TSC: 677 timestamp = pkt_info->packet.payload | 678 (data->timestamp & (0xffULL << 56)); 679 if (data->from_mtc && timestamp < data->timestamp && 680 data->timestamp - timestamp < decoder->tsc_slip) 681 return 1; 682 if (timestamp < data->timestamp) 683 timestamp += (1ULL << 56); 684 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 685 if (data->from_mtc) 686 return 1; 687 data->tsc_timestamp = timestamp; 688 data->timestamp = timestamp; 689 return 0; 690 } 691 break; 692 693 case INTEL_PT_TMA: 694 if (data->from_mtc) 695 return 1; 696 697 if (!decoder->tsc_ctc_ratio_d) 698 return 0; 699 700 ctc = pkt_info->packet.payload; 701 fc = pkt_info->packet.count; 702 ctc_rem = ctc & decoder->ctc_rem_mask; 703 704 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 705 706 data->ctc_timestamp = data->tsc_timestamp - fc; 707 if (decoder->tsc_ctc_mult) { 708 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 709 } else { 710 data->ctc_timestamp -= 711 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 712 decoder->tsc_ctc_ratio_d); 713 } 714 715 data->ctc_delta = 0; 716 data->have_tma = true; 717 data->fixup_last_mtc = true; 718 719 return 0; 720 721 case INTEL_PT_CYC: 722 data->cycle_cnt += pkt_info->packet.payload; 723 return 0; 724 725 case INTEL_PT_CBR: 726 cbr = pkt_info->packet.payload; 727 if (data->cbr && data->cbr != cbr) 728 return 1; 729 data->cbr = cbr; 730 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 731 return 0; 732 733 case INTEL_PT_TIP_PGD: 734 case INTEL_PT_TRACESTOP: 735 case INTEL_PT_OVF: 736 case INTEL_PT_BAD: /* Does not happen */ 737 default: 738 return 1; 739 } 740 741 if (!data->cbr && decoder->cbr) { 742 data->cbr = decoder->cbr; 743 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 744 } 745 746 if (!data->cycle_cnt) 747 return 1; 748 749 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 750 751 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 752 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 753 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 754 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 755 return 1; 756 } 757 758 decoder->calc_cyc_to_tsc = cyc_to_tsc; 759 decoder->have_calc_cyc_to_tsc = true; 760 761 if (data->cbr) { 762 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 763 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 764 } else { 765 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 766 cyc_to_tsc, pkt_info->pos); 767 } 768 769 return 1; 770 } 771 772 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 773 bool from_mtc) 774 { 775 struct intel_pt_calc_cyc_to_tsc_info data = { 776 .cycle_cnt = 0, 777 .cbr = 0, 778 .last_mtc = decoder->last_mtc, 779 .ctc_timestamp = decoder->ctc_timestamp, 780 .ctc_delta = decoder->ctc_delta, 781 .tsc_timestamp = decoder->tsc_timestamp, 782 .timestamp = decoder->timestamp, 783 .have_tma = decoder->have_tma, 784 .fixup_last_mtc = decoder->fixup_last_mtc, 785 .from_mtc = from_mtc, 786 .cbr_cyc_to_tsc = 0, 787 }; 788 789 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 790 } 791 792 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 793 { 794 int ret; 795 796 decoder->last_packet_type = decoder->packet.type; 797 798 do { 799 decoder->pos += decoder->pkt_step; 800 decoder->buf += decoder->pkt_step; 801 decoder->len -= decoder->pkt_step; 802 803 if (!decoder->len) { 804 ret = intel_pt_get_next_data(decoder); 805 if (ret) 806 return ret; 807 } 808 809 ret = intel_pt_get_packet(decoder->buf, decoder->len, 810 &decoder->packet); 811 if (ret == INTEL_PT_NEED_MORE_BYTES && 812 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 813 ret = intel_pt_get_split_packet(decoder); 814 if (ret < 0) 815 return ret; 816 } 817 if (ret <= 0) 818 return intel_pt_bad_packet(decoder); 819 820 decoder->pkt_len = ret; 821 decoder->pkt_step = ret; 822 intel_pt_decoder_log_packet(decoder); 823 } while (decoder->packet.type == INTEL_PT_PAD); 824 825 return 0; 826 } 827 828 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 829 { 830 uint64_t timestamp, masked_timestamp; 831 832 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 833 masked_timestamp = timestamp & decoder->period_mask; 834 if (decoder->continuous_period) { 835 if (masked_timestamp != decoder->last_masked_timestamp) 836 return 1; 837 } else { 838 timestamp += 1; 839 masked_timestamp = timestamp & decoder->period_mask; 840 if (masked_timestamp != decoder->last_masked_timestamp) { 841 decoder->last_masked_timestamp = masked_timestamp; 842 decoder->continuous_period = true; 843 } 844 } 845 return decoder->period_ticks - (timestamp - masked_timestamp); 846 } 847 848 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 849 { 850 switch (decoder->period_type) { 851 case INTEL_PT_PERIOD_INSTRUCTIONS: 852 return decoder->period - decoder->period_insn_cnt; 853 case INTEL_PT_PERIOD_TICKS: 854 return intel_pt_next_period(decoder); 855 case INTEL_PT_PERIOD_NONE: 856 case INTEL_PT_PERIOD_MTC: 857 default: 858 return 0; 859 } 860 } 861 862 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 863 { 864 uint64_t timestamp, masked_timestamp; 865 866 switch (decoder->period_type) { 867 case INTEL_PT_PERIOD_INSTRUCTIONS: 868 decoder->period_insn_cnt = 0; 869 break; 870 case INTEL_PT_PERIOD_TICKS: 871 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 872 masked_timestamp = timestamp & decoder->period_mask; 873 decoder->last_masked_timestamp = masked_timestamp; 874 break; 875 case INTEL_PT_PERIOD_NONE: 876 case INTEL_PT_PERIOD_MTC: 877 default: 878 break; 879 } 880 881 decoder->state.type |= INTEL_PT_INSTRUCTION; 882 } 883 884 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 885 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 886 { 887 uint64_t max_insn_cnt, insn_cnt = 0; 888 int err; 889 890 if (!decoder->mtc_insn) 891 decoder->mtc_insn = true; 892 893 max_insn_cnt = intel_pt_next_sample(decoder); 894 895 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 896 max_insn_cnt, decoder->data); 897 898 decoder->tot_insn_cnt += insn_cnt; 899 decoder->timestamp_insn_cnt += insn_cnt; 900 decoder->period_insn_cnt += insn_cnt; 901 902 if (err) { 903 decoder->no_progress = 0; 904 decoder->pkt_state = INTEL_PT_STATE_ERR2; 905 intel_pt_log_at("ERROR: Failed to get instruction", 906 decoder->ip); 907 if (err == -ENOENT) 908 return -ENOLINK; 909 return -EILSEQ; 910 } 911 912 if (ip && decoder->ip == ip) { 913 err = -EAGAIN; 914 goto out; 915 } 916 917 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 918 intel_pt_sample_insn(decoder); 919 920 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 921 decoder->state.type = INTEL_PT_INSTRUCTION; 922 decoder->state.from_ip = decoder->ip; 923 decoder->state.to_ip = 0; 924 decoder->ip += intel_pt_insn->length; 925 err = INTEL_PT_RETURN; 926 goto out; 927 } 928 929 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 930 /* Zero-length calls are excluded */ 931 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 932 intel_pt_insn->rel) { 933 err = intel_pt_push(&decoder->stack, decoder->ip + 934 intel_pt_insn->length); 935 if (err) 936 goto out; 937 } 938 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 939 decoder->ret_addr = intel_pt_pop(&decoder->stack); 940 } 941 942 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 943 int cnt = decoder->no_progress++; 944 945 decoder->state.from_ip = decoder->ip; 946 decoder->ip += intel_pt_insn->length + 947 intel_pt_insn->rel; 948 decoder->state.to_ip = decoder->ip; 949 err = INTEL_PT_RETURN; 950 951 /* 952 * Check for being stuck in a loop. This can happen if a 953 * decoder error results in the decoder erroneously setting the 954 * ip to an address that is itself in an infinite loop that 955 * consumes no packets. When that happens, there must be an 956 * unconditional branch. 957 */ 958 if (cnt) { 959 if (cnt == 1) { 960 decoder->stuck_ip = decoder->state.to_ip; 961 decoder->stuck_ip_prd = 1; 962 decoder->stuck_ip_cnt = 1; 963 } else if (cnt > INTEL_PT_MAX_LOOPS || 964 decoder->state.to_ip == decoder->stuck_ip) { 965 intel_pt_log_at("ERROR: Never-ending loop", 966 decoder->state.to_ip); 967 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 968 err = -ELOOP; 969 goto out; 970 } else if (!--decoder->stuck_ip_cnt) { 971 decoder->stuck_ip_prd += 1; 972 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 973 decoder->stuck_ip = decoder->state.to_ip; 974 } 975 } 976 goto out_no_progress; 977 } 978 out: 979 decoder->no_progress = 0; 980 out_no_progress: 981 decoder->state.insn_op = intel_pt_insn->op; 982 decoder->state.insn_len = intel_pt_insn->length; 983 984 if (decoder->tx_flags & INTEL_PT_IN_TX) 985 decoder->state.flags |= INTEL_PT_IN_TX; 986 987 return err; 988 } 989 990 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 991 { 992 struct intel_pt_insn intel_pt_insn; 993 uint64_t ip; 994 int err; 995 996 ip = decoder->last_ip; 997 998 while (1) { 999 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1000 if (err == INTEL_PT_RETURN) 1001 return 0; 1002 if (err == -EAGAIN) { 1003 if (decoder->set_fup_tx_flags) { 1004 decoder->set_fup_tx_flags = false; 1005 decoder->tx_flags = decoder->fup_tx_flags; 1006 decoder->state.type = INTEL_PT_TRANSACTION; 1007 decoder->state.from_ip = decoder->ip; 1008 decoder->state.to_ip = 0; 1009 decoder->state.flags = decoder->fup_tx_flags; 1010 return 0; 1011 } 1012 return err; 1013 } 1014 decoder->set_fup_tx_flags = false; 1015 if (err) 1016 return err; 1017 1018 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1019 intel_pt_log_at("ERROR: Unexpected indirect branch", 1020 decoder->ip); 1021 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1022 return -ENOENT; 1023 } 1024 1025 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1026 intel_pt_log_at("ERROR: Unexpected conditional branch", 1027 decoder->ip); 1028 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1029 return -ENOENT; 1030 } 1031 1032 intel_pt_bug(decoder); 1033 } 1034 } 1035 1036 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1037 { 1038 struct intel_pt_insn intel_pt_insn; 1039 int err; 1040 1041 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1042 if (err == INTEL_PT_RETURN && 1043 decoder->pgd_ip && 1044 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1045 (decoder->state.type & INTEL_PT_BRANCH) && 1046 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1047 /* Unconditional branch leaving filter region */ 1048 decoder->no_progress = 0; 1049 decoder->pge = false; 1050 decoder->continuous_period = false; 1051 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1052 decoder->state.to_ip = 0; 1053 return 0; 1054 } 1055 if (err == INTEL_PT_RETURN) 1056 return 0; 1057 if (err) 1058 return err; 1059 1060 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1061 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1062 decoder->pge = false; 1063 decoder->continuous_period = false; 1064 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1065 decoder->state.from_ip = decoder->ip; 1066 decoder->state.to_ip = 0; 1067 if (decoder->packet.count != 0) 1068 decoder->ip = decoder->last_ip; 1069 } else { 1070 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1071 decoder->state.from_ip = decoder->ip; 1072 if (decoder->packet.count == 0) { 1073 decoder->state.to_ip = 0; 1074 } else { 1075 decoder->state.to_ip = decoder->last_ip; 1076 decoder->ip = decoder->last_ip; 1077 } 1078 } 1079 return 0; 1080 } 1081 1082 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1083 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1084 intel_pt_insn.rel; 1085 1086 if (decoder->pgd_ip && 1087 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1088 decoder->pgd_ip(to_ip, decoder->data)) { 1089 /* Conditional branch leaving filter region */ 1090 decoder->pge = false; 1091 decoder->continuous_period = false; 1092 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1093 decoder->ip = to_ip; 1094 decoder->state.from_ip = decoder->ip; 1095 decoder->state.to_ip = 0; 1096 return 0; 1097 } 1098 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1099 decoder->ip); 1100 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1101 return -ENOENT; 1102 } 1103 1104 return intel_pt_bug(decoder); 1105 } 1106 1107 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1108 { 1109 struct intel_pt_insn intel_pt_insn; 1110 int err; 1111 1112 while (1) { 1113 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1114 if (err == INTEL_PT_RETURN) 1115 return 0; 1116 if (err) 1117 return err; 1118 1119 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1120 if (!decoder->return_compression) { 1121 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1122 decoder->ip); 1123 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1124 return -ENOENT; 1125 } 1126 if (!decoder->ret_addr) { 1127 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1128 decoder->ip); 1129 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1130 return -ENOENT; 1131 } 1132 if (!(decoder->tnt.payload & BIT63)) { 1133 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1134 decoder->ip); 1135 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1136 return -ENOENT; 1137 } 1138 decoder->tnt.count -= 1; 1139 if (!decoder->tnt.count) 1140 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1141 decoder->tnt.payload <<= 1; 1142 decoder->state.from_ip = decoder->ip; 1143 decoder->ip = decoder->ret_addr; 1144 decoder->state.to_ip = decoder->ip; 1145 return 0; 1146 } 1147 1148 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1149 /* Handle deferred TIPs */ 1150 err = intel_pt_get_next_packet(decoder); 1151 if (err) 1152 return err; 1153 if (decoder->packet.type != INTEL_PT_TIP || 1154 decoder->packet.count == 0) { 1155 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1156 decoder->ip); 1157 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1158 decoder->pkt_step = 0; 1159 return -ENOENT; 1160 } 1161 intel_pt_set_last_ip(decoder); 1162 decoder->state.from_ip = decoder->ip; 1163 decoder->state.to_ip = decoder->last_ip; 1164 decoder->ip = decoder->last_ip; 1165 return 0; 1166 } 1167 1168 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1169 decoder->tnt.count -= 1; 1170 if (!decoder->tnt.count) 1171 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1172 if (decoder->tnt.payload & BIT63) { 1173 decoder->tnt.payload <<= 1; 1174 decoder->state.from_ip = decoder->ip; 1175 decoder->ip += intel_pt_insn.length + 1176 intel_pt_insn.rel; 1177 decoder->state.to_ip = decoder->ip; 1178 return 0; 1179 } 1180 /* Instruction sample for a non-taken branch */ 1181 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1182 decoder->tnt.payload <<= 1; 1183 decoder->state.type = INTEL_PT_INSTRUCTION; 1184 decoder->state.from_ip = decoder->ip; 1185 decoder->state.to_ip = 0; 1186 decoder->ip += intel_pt_insn.length; 1187 return 0; 1188 } 1189 decoder->ip += intel_pt_insn.length; 1190 if (!decoder->tnt.count) 1191 return -EAGAIN; 1192 decoder->tnt.payload <<= 1; 1193 continue; 1194 } 1195 1196 return intel_pt_bug(decoder); 1197 } 1198 } 1199 1200 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1201 { 1202 unsigned int fup_tx_flags; 1203 int err; 1204 1205 fup_tx_flags = decoder->packet.payload & 1206 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1207 err = intel_pt_get_next_packet(decoder); 1208 if (err) 1209 return err; 1210 if (decoder->packet.type == INTEL_PT_FUP) { 1211 decoder->fup_tx_flags = fup_tx_flags; 1212 decoder->set_fup_tx_flags = true; 1213 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1214 *no_tip = true; 1215 } else { 1216 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1217 decoder->pos); 1218 intel_pt_update_in_tx(decoder); 1219 } 1220 return 0; 1221 } 1222 1223 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1224 { 1225 uint64_t timestamp; 1226 1227 decoder->have_tma = false; 1228 1229 if (decoder->ref_timestamp) { 1230 timestamp = decoder->packet.payload | 1231 (decoder->ref_timestamp & (0xffULL << 56)); 1232 if (timestamp < decoder->ref_timestamp) { 1233 if (decoder->ref_timestamp - timestamp > (1ULL << 55)) 1234 timestamp += (1ULL << 56); 1235 } else { 1236 if (timestamp - decoder->ref_timestamp > (1ULL << 55)) 1237 timestamp -= (1ULL << 56); 1238 } 1239 decoder->tsc_timestamp = timestamp; 1240 decoder->timestamp = timestamp; 1241 decoder->ref_timestamp = 0; 1242 decoder->timestamp_insn_cnt = 0; 1243 } else if (decoder->timestamp) { 1244 timestamp = decoder->packet.payload | 1245 (decoder->timestamp & (0xffULL << 56)); 1246 decoder->tsc_timestamp = timestamp; 1247 if (timestamp < decoder->timestamp && 1248 decoder->timestamp - timestamp < decoder->tsc_slip) { 1249 intel_pt_log_to("Suppressing backwards timestamp", 1250 timestamp); 1251 timestamp = decoder->timestamp; 1252 } 1253 if (timestamp < decoder->timestamp) { 1254 intel_pt_log_to("Wraparound timestamp", timestamp); 1255 timestamp += (1ULL << 56); 1256 decoder->tsc_timestamp = timestamp; 1257 } 1258 decoder->timestamp = timestamp; 1259 decoder->timestamp_insn_cnt = 0; 1260 } 1261 1262 if (decoder->last_packet_type == INTEL_PT_CYC) { 1263 decoder->cyc_ref_timestamp = decoder->timestamp; 1264 decoder->cycle_cnt = 0; 1265 decoder->have_calc_cyc_to_tsc = false; 1266 intel_pt_calc_cyc_to_tsc(decoder, false); 1267 } 1268 1269 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1270 } 1271 1272 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1273 { 1274 intel_pt_log("ERROR: Buffer overflow\n"); 1275 intel_pt_clear_tx_flags(decoder); 1276 decoder->have_tma = false; 1277 decoder->cbr = 0; 1278 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1279 decoder->overflow = true; 1280 return -EOVERFLOW; 1281 } 1282 1283 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1284 { 1285 uint32_t ctc = decoder->packet.payload; 1286 uint32_t fc = decoder->packet.count; 1287 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1288 1289 if (!decoder->tsc_ctc_ratio_d) 1290 return; 1291 1292 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1293 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1294 if (decoder->tsc_ctc_mult) { 1295 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1296 } else { 1297 decoder->ctc_timestamp -= multdiv(ctc_rem, 1298 decoder->tsc_ctc_ratio_n, 1299 decoder->tsc_ctc_ratio_d); 1300 } 1301 decoder->ctc_delta = 0; 1302 decoder->have_tma = true; 1303 decoder->fixup_last_mtc = true; 1304 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1305 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1306 } 1307 1308 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1309 { 1310 uint64_t timestamp; 1311 uint32_t mtc, mtc_delta; 1312 1313 if (!decoder->have_tma) 1314 return; 1315 1316 mtc = decoder->packet.payload; 1317 1318 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1319 decoder->fixup_last_mtc = false; 1320 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1321 &decoder->last_mtc); 1322 } 1323 1324 if (mtc > decoder->last_mtc) 1325 mtc_delta = mtc - decoder->last_mtc; 1326 else 1327 mtc_delta = mtc + 256 - decoder->last_mtc; 1328 1329 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1330 1331 if (decoder->tsc_ctc_mult) { 1332 timestamp = decoder->ctc_timestamp + 1333 decoder->ctc_delta * decoder->tsc_ctc_mult; 1334 } else { 1335 timestamp = decoder->ctc_timestamp + 1336 multdiv(decoder->ctc_delta, 1337 decoder->tsc_ctc_ratio_n, 1338 decoder->tsc_ctc_ratio_d); 1339 } 1340 1341 if (timestamp < decoder->timestamp) 1342 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1343 timestamp, decoder->timestamp); 1344 else 1345 decoder->timestamp = timestamp; 1346 1347 decoder->timestamp_insn_cnt = 0; 1348 decoder->last_mtc = mtc; 1349 1350 if (decoder->last_packet_type == INTEL_PT_CYC) { 1351 decoder->cyc_ref_timestamp = decoder->timestamp; 1352 decoder->cycle_cnt = 0; 1353 decoder->have_calc_cyc_to_tsc = false; 1354 intel_pt_calc_cyc_to_tsc(decoder, true); 1355 } 1356 } 1357 1358 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1359 { 1360 unsigned int cbr = decoder->packet.payload; 1361 1362 if (decoder->cbr == cbr) 1363 return; 1364 1365 decoder->cbr = cbr; 1366 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1367 } 1368 1369 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1370 { 1371 uint64_t timestamp = decoder->cyc_ref_timestamp; 1372 1373 decoder->have_cyc = true; 1374 1375 decoder->cycle_cnt += decoder->packet.payload; 1376 1377 if (!decoder->cyc_ref_timestamp) 1378 return; 1379 1380 if (decoder->have_calc_cyc_to_tsc) 1381 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1382 else if (decoder->cbr) 1383 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1384 else 1385 return; 1386 1387 if (timestamp < decoder->timestamp) 1388 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1389 timestamp, decoder->timestamp); 1390 else 1391 decoder->timestamp = timestamp; 1392 1393 decoder->timestamp_insn_cnt = 0; 1394 } 1395 1396 /* Walk PSB+ packets when already in sync. */ 1397 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1398 { 1399 int err; 1400 1401 while (1) { 1402 err = intel_pt_get_next_packet(decoder); 1403 if (err) 1404 return err; 1405 1406 switch (decoder->packet.type) { 1407 case INTEL_PT_PSBEND: 1408 return 0; 1409 1410 case INTEL_PT_TIP_PGD: 1411 case INTEL_PT_TIP_PGE: 1412 case INTEL_PT_TIP: 1413 case INTEL_PT_TNT: 1414 case INTEL_PT_TRACESTOP: 1415 case INTEL_PT_BAD: 1416 case INTEL_PT_PSB: 1417 decoder->have_tma = false; 1418 intel_pt_log("ERROR: Unexpected packet\n"); 1419 return -EAGAIN; 1420 1421 case INTEL_PT_OVF: 1422 return intel_pt_overflow(decoder); 1423 1424 case INTEL_PT_TSC: 1425 intel_pt_calc_tsc_timestamp(decoder); 1426 break; 1427 1428 case INTEL_PT_TMA: 1429 intel_pt_calc_tma(decoder); 1430 break; 1431 1432 case INTEL_PT_CBR: 1433 intel_pt_calc_cbr(decoder); 1434 break; 1435 1436 case INTEL_PT_MODE_EXEC: 1437 decoder->exec_mode = decoder->packet.payload; 1438 break; 1439 1440 case INTEL_PT_PIP: 1441 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1442 break; 1443 1444 case INTEL_PT_FUP: 1445 decoder->pge = true; 1446 intel_pt_set_last_ip(decoder); 1447 break; 1448 1449 case INTEL_PT_MODE_TSX: 1450 intel_pt_update_in_tx(decoder); 1451 break; 1452 1453 case INTEL_PT_MTC: 1454 intel_pt_calc_mtc_timestamp(decoder); 1455 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1456 decoder->state.type |= INTEL_PT_INSTRUCTION; 1457 break; 1458 1459 case INTEL_PT_CYC: 1460 case INTEL_PT_VMCS: 1461 case INTEL_PT_MNT: 1462 case INTEL_PT_PAD: 1463 default: 1464 break; 1465 } 1466 } 1467 } 1468 1469 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1470 { 1471 int err; 1472 1473 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1474 decoder->tx_flags = 0; 1475 decoder->state.flags &= ~INTEL_PT_IN_TX; 1476 decoder->state.flags |= INTEL_PT_ABORT_TX; 1477 } else { 1478 decoder->state.flags |= INTEL_PT_ASYNC; 1479 } 1480 1481 while (1) { 1482 err = intel_pt_get_next_packet(decoder); 1483 if (err) 1484 return err; 1485 1486 switch (decoder->packet.type) { 1487 case INTEL_PT_TNT: 1488 case INTEL_PT_FUP: 1489 case INTEL_PT_TRACESTOP: 1490 case INTEL_PT_PSB: 1491 case INTEL_PT_TSC: 1492 case INTEL_PT_TMA: 1493 case INTEL_PT_CBR: 1494 case INTEL_PT_MODE_TSX: 1495 case INTEL_PT_BAD: 1496 case INTEL_PT_PSBEND: 1497 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1498 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1499 return -ENOENT; 1500 1501 case INTEL_PT_OVF: 1502 return intel_pt_overflow(decoder); 1503 1504 case INTEL_PT_TIP_PGD: 1505 decoder->state.from_ip = decoder->ip; 1506 decoder->state.to_ip = 0; 1507 if (decoder->packet.count != 0) { 1508 intel_pt_set_ip(decoder); 1509 intel_pt_log("Omitting PGD ip " x64_fmt "\n", 1510 decoder->ip); 1511 } 1512 decoder->pge = false; 1513 decoder->continuous_period = false; 1514 return 0; 1515 1516 case INTEL_PT_TIP_PGE: 1517 decoder->pge = true; 1518 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1519 decoder->ip); 1520 decoder->state.from_ip = 0; 1521 if (decoder->packet.count == 0) { 1522 decoder->state.to_ip = 0; 1523 } else { 1524 intel_pt_set_ip(decoder); 1525 decoder->state.to_ip = decoder->ip; 1526 } 1527 return 0; 1528 1529 case INTEL_PT_TIP: 1530 decoder->state.from_ip = decoder->ip; 1531 if (decoder->packet.count == 0) { 1532 decoder->state.to_ip = 0; 1533 } else { 1534 intel_pt_set_ip(decoder); 1535 decoder->state.to_ip = decoder->ip; 1536 } 1537 return 0; 1538 1539 case INTEL_PT_PIP: 1540 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1541 break; 1542 1543 case INTEL_PT_MTC: 1544 intel_pt_calc_mtc_timestamp(decoder); 1545 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1546 decoder->state.type |= INTEL_PT_INSTRUCTION; 1547 break; 1548 1549 case INTEL_PT_CYC: 1550 intel_pt_calc_cyc_timestamp(decoder); 1551 break; 1552 1553 case INTEL_PT_MODE_EXEC: 1554 decoder->exec_mode = decoder->packet.payload; 1555 break; 1556 1557 case INTEL_PT_VMCS: 1558 case INTEL_PT_MNT: 1559 case INTEL_PT_PAD: 1560 break; 1561 1562 default: 1563 return intel_pt_bug(decoder); 1564 } 1565 } 1566 } 1567 1568 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 1569 { 1570 bool no_tip = false; 1571 int err; 1572 1573 while (1) { 1574 err = intel_pt_get_next_packet(decoder); 1575 if (err) 1576 return err; 1577 next: 1578 switch (decoder->packet.type) { 1579 case INTEL_PT_TNT: 1580 if (!decoder->packet.count) 1581 break; 1582 decoder->tnt = decoder->packet; 1583 decoder->pkt_state = INTEL_PT_STATE_TNT; 1584 err = intel_pt_walk_tnt(decoder); 1585 if (err == -EAGAIN) 1586 break; 1587 return err; 1588 1589 case INTEL_PT_TIP_PGD: 1590 if (decoder->packet.count != 0) 1591 intel_pt_set_last_ip(decoder); 1592 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 1593 return intel_pt_walk_tip(decoder); 1594 1595 case INTEL_PT_TIP_PGE: { 1596 decoder->pge = true; 1597 if (decoder->packet.count == 0) { 1598 intel_pt_log_at("Skipping zero TIP.PGE", 1599 decoder->pos); 1600 break; 1601 } 1602 intel_pt_set_ip(decoder); 1603 decoder->state.from_ip = 0; 1604 decoder->state.to_ip = decoder->ip; 1605 return 0; 1606 } 1607 1608 case INTEL_PT_OVF: 1609 return intel_pt_overflow(decoder); 1610 1611 case INTEL_PT_TIP: 1612 if (decoder->packet.count != 0) 1613 intel_pt_set_last_ip(decoder); 1614 decoder->pkt_state = INTEL_PT_STATE_TIP; 1615 return intel_pt_walk_tip(decoder); 1616 1617 case INTEL_PT_FUP: 1618 if (decoder->packet.count == 0) { 1619 intel_pt_log_at("Skipping zero FUP", 1620 decoder->pos); 1621 no_tip = false; 1622 break; 1623 } 1624 intel_pt_set_last_ip(decoder); 1625 err = intel_pt_walk_fup(decoder); 1626 if (err != -EAGAIN) { 1627 if (err) 1628 return err; 1629 if (no_tip) 1630 decoder->pkt_state = 1631 INTEL_PT_STATE_FUP_NO_TIP; 1632 else 1633 decoder->pkt_state = INTEL_PT_STATE_FUP; 1634 return 0; 1635 } 1636 if (no_tip) { 1637 no_tip = false; 1638 break; 1639 } 1640 return intel_pt_walk_fup_tip(decoder); 1641 1642 case INTEL_PT_TRACESTOP: 1643 decoder->pge = false; 1644 decoder->continuous_period = false; 1645 intel_pt_clear_tx_flags(decoder); 1646 decoder->have_tma = false; 1647 break; 1648 1649 case INTEL_PT_PSB: 1650 intel_pt_clear_stack(&decoder->stack); 1651 err = intel_pt_walk_psbend(decoder); 1652 if (err == -EAGAIN) 1653 goto next; 1654 if (err) 1655 return err; 1656 break; 1657 1658 case INTEL_PT_PIP: 1659 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1660 break; 1661 1662 case INTEL_PT_MTC: 1663 intel_pt_calc_mtc_timestamp(decoder); 1664 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 1665 break; 1666 /* 1667 * Ensure that there has been an instruction since the 1668 * last MTC. 1669 */ 1670 if (!decoder->mtc_insn) 1671 break; 1672 decoder->mtc_insn = false; 1673 /* Ensure that there is a timestamp */ 1674 if (!decoder->timestamp) 1675 break; 1676 decoder->state.type = INTEL_PT_INSTRUCTION; 1677 decoder->state.from_ip = decoder->ip; 1678 decoder->state.to_ip = 0; 1679 decoder->mtc_insn = false; 1680 return 0; 1681 1682 case INTEL_PT_TSC: 1683 intel_pt_calc_tsc_timestamp(decoder); 1684 break; 1685 1686 case INTEL_PT_TMA: 1687 intel_pt_calc_tma(decoder); 1688 break; 1689 1690 case INTEL_PT_CYC: 1691 intel_pt_calc_cyc_timestamp(decoder); 1692 break; 1693 1694 case INTEL_PT_CBR: 1695 intel_pt_calc_cbr(decoder); 1696 break; 1697 1698 case INTEL_PT_MODE_EXEC: 1699 decoder->exec_mode = decoder->packet.payload; 1700 break; 1701 1702 case INTEL_PT_MODE_TSX: 1703 /* MODE_TSX need not be followed by FUP */ 1704 if (!decoder->pge) { 1705 intel_pt_update_in_tx(decoder); 1706 break; 1707 } 1708 err = intel_pt_mode_tsx(decoder, &no_tip); 1709 if (err) 1710 return err; 1711 goto next; 1712 1713 case INTEL_PT_BAD: /* Does not happen */ 1714 return intel_pt_bug(decoder); 1715 1716 case INTEL_PT_PSBEND: 1717 case INTEL_PT_VMCS: 1718 case INTEL_PT_MNT: 1719 case INTEL_PT_PAD: 1720 break; 1721 1722 default: 1723 return intel_pt_bug(decoder); 1724 } 1725 } 1726 } 1727 1728 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 1729 { 1730 return decoder->last_ip || decoder->packet.count == 0 || 1731 decoder->packet.count == 3 || decoder->packet.count == 6; 1732 } 1733 1734 /* Walk PSB+ packets to get in sync. */ 1735 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 1736 { 1737 int err; 1738 1739 while (1) { 1740 err = intel_pt_get_next_packet(decoder); 1741 if (err) 1742 return err; 1743 1744 switch (decoder->packet.type) { 1745 case INTEL_PT_TIP_PGD: 1746 decoder->continuous_period = false; 1747 case INTEL_PT_TIP_PGE: 1748 case INTEL_PT_TIP: 1749 intel_pt_log("ERROR: Unexpected packet\n"); 1750 return -ENOENT; 1751 1752 case INTEL_PT_FUP: 1753 decoder->pge = true; 1754 if (intel_pt_have_ip(decoder)) { 1755 uint64_t current_ip = decoder->ip; 1756 1757 intel_pt_set_ip(decoder); 1758 if (current_ip) 1759 intel_pt_log_to("Setting IP", 1760 decoder->ip); 1761 } 1762 break; 1763 1764 case INTEL_PT_MTC: 1765 intel_pt_calc_mtc_timestamp(decoder); 1766 break; 1767 1768 case INTEL_PT_TSC: 1769 intel_pt_calc_tsc_timestamp(decoder); 1770 break; 1771 1772 case INTEL_PT_TMA: 1773 intel_pt_calc_tma(decoder); 1774 break; 1775 1776 case INTEL_PT_CYC: 1777 intel_pt_calc_cyc_timestamp(decoder); 1778 break; 1779 1780 case INTEL_PT_CBR: 1781 intel_pt_calc_cbr(decoder); 1782 break; 1783 1784 case INTEL_PT_PIP: 1785 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1786 break; 1787 1788 case INTEL_PT_MODE_EXEC: 1789 decoder->exec_mode = decoder->packet.payload; 1790 break; 1791 1792 case INTEL_PT_MODE_TSX: 1793 intel_pt_update_in_tx(decoder); 1794 break; 1795 1796 case INTEL_PT_TRACESTOP: 1797 decoder->pge = false; 1798 decoder->continuous_period = false; 1799 intel_pt_clear_tx_flags(decoder); 1800 case INTEL_PT_TNT: 1801 decoder->have_tma = false; 1802 intel_pt_log("ERROR: Unexpected packet\n"); 1803 if (decoder->ip) 1804 decoder->pkt_state = INTEL_PT_STATE_ERR4; 1805 else 1806 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1807 return -ENOENT; 1808 1809 case INTEL_PT_BAD: /* Does not happen */ 1810 return intel_pt_bug(decoder); 1811 1812 case INTEL_PT_OVF: 1813 return intel_pt_overflow(decoder); 1814 1815 case INTEL_PT_PSBEND: 1816 return 0; 1817 1818 case INTEL_PT_PSB: 1819 case INTEL_PT_VMCS: 1820 case INTEL_PT_MNT: 1821 case INTEL_PT_PAD: 1822 default: 1823 break; 1824 } 1825 } 1826 } 1827 1828 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 1829 { 1830 int err; 1831 1832 while (1) { 1833 err = intel_pt_get_next_packet(decoder); 1834 if (err) 1835 return err; 1836 1837 switch (decoder->packet.type) { 1838 case INTEL_PT_TIP_PGD: 1839 decoder->continuous_period = false; 1840 case INTEL_PT_TIP_PGE: 1841 case INTEL_PT_TIP: 1842 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD; 1843 if (intel_pt_have_ip(decoder)) 1844 intel_pt_set_ip(decoder); 1845 if (decoder->ip) 1846 return 0; 1847 break; 1848 1849 case INTEL_PT_FUP: 1850 if (decoder->overflow) { 1851 if (intel_pt_have_ip(decoder)) 1852 intel_pt_set_ip(decoder); 1853 if (decoder->ip) 1854 return 0; 1855 } 1856 if (decoder->packet.count) 1857 intel_pt_set_last_ip(decoder); 1858 break; 1859 1860 case INTEL_PT_MTC: 1861 intel_pt_calc_mtc_timestamp(decoder); 1862 break; 1863 1864 case INTEL_PT_TSC: 1865 intel_pt_calc_tsc_timestamp(decoder); 1866 break; 1867 1868 case INTEL_PT_TMA: 1869 intel_pt_calc_tma(decoder); 1870 break; 1871 1872 case INTEL_PT_CYC: 1873 intel_pt_calc_cyc_timestamp(decoder); 1874 break; 1875 1876 case INTEL_PT_CBR: 1877 intel_pt_calc_cbr(decoder); 1878 break; 1879 1880 case INTEL_PT_PIP: 1881 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1882 break; 1883 1884 case INTEL_PT_MODE_EXEC: 1885 decoder->exec_mode = decoder->packet.payload; 1886 break; 1887 1888 case INTEL_PT_MODE_TSX: 1889 intel_pt_update_in_tx(decoder); 1890 break; 1891 1892 case INTEL_PT_OVF: 1893 return intel_pt_overflow(decoder); 1894 1895 case INTEL_PT_BAD: /* Does not happen */ 1896 return intel_pt_bug(decoder); 1897 1898 case INTEL_PT_TRACESTOP: 1899 decoder->pge = false; 1900 decoder->continuous_period = false; 1901 intel_pt_clear_tx_flags(decoder); 1902 decoder->have_tma = false; 1903 break; 1904 1905 case INTEL_PT_PSB: 1906 err = intel_pt_walk_psb(decoder); 1907 if (err) 1908 return err; 1909 if (decoder->ip) { 1910 /* Do not have a sample */ 1911 decoder->state.type = 0; 1912 return 0; 1913 } 1914 break; 1915 1916 case INTEL_PT_TNT: 1917 case INTEL_PT_PSBEND: 1918 case INTEL_PT_VMCS: 1919 case INTEL_PT_MNT: 1920 case INTEL_PT_PAD: 1921 default: 1922 break; 1923 } 1924 } 1925 } 1926 1927 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 1928 { 1929 int err; 1930 1931 intel_pt_log("Scanning for full IP\n"); 1932 err = intel_pt_walk_to_ip(decoder); 1933 if (err) 1934 return err; 1935 1936 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1937 decoder->overflow = false; 1938 1939 decoder->state.from_ip = 0; 1940 decoder->state.to_ip = decoder->ip; 1941 intel_pt_log_to("Setting IP", decoder->ip); 1942 1943 return 0; 1944 } 1945 1946 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 1947 { 1948 const unsigned char *end = decoder->buf + decoder->len; 1949 size_t i; 1950 1951 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 1952 if (i > decoder->len) 1953 continue; 1954 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 1955 return i; 1956 } 1957 return 0; 1958 } 1959 1960 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 1961 { 1962 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 1963 const char *psb = INTEL_PT_PSB_STR; 1964 1965 if (rest_psb > decoder->len || 1966 memcmp(decoder->buf, psb + part_psb, rest_psb)) 1967 return 0; 1968 1969 return rest_psb; 1970 } 1971 1972 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 1973 int part_psb) 1974 { 1975 int rest_psb, ret; 1976 1977 decoder->pos += decoder->len; 1978 decoder->len = 0; 1979 1980 ret = intel_pt_get_next_data(decoder); 1981 if (ret) 1982 return ret; 1983 1984 rest_psb = intel_pt_rest_psb(decoder, part_psb); 1985 if (!rest_psb) 1986 return 0; 1987 1988 decoder->pos -= part_psb; 1989 decoder->next_buf = decoder->buf + rest_psb; 1990 decoder->next_len = decoder->len - rest_psb; 1991 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 1992 decoder->buf = decoder->temp_buf; 1993 decoder->len = INTEL_PT_PSB_LEN; 1994 1995 return 0; 1996 } 1997 1998 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 1999 { 2000 unsigned char *next; 2001 int ret; 2002 2003 intel_pt_log("Scanning for PSB\n"); 2004 while (1) { 2005 if (!decoder->len) { 2006 ret = intel_pt_get_next_data(decoder); 2007 if (ret) 2008 return ret; 2009 } 2010 2011 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2012 INTEL_PT_PSB_LEN); 2013 if (!next) { 2014 int part_psb; 2015 2016 part_psb = intel_pt_part_psb(decoder); 2017 if (part_psb) { 2018 ret = intel_pt_get_split_psb(decoder, part_psb); 2019 if (ret) 2020 return ret; 2021 } else { 2022 decoder->pos += decoder->len; 2023 decoder->len = 0; 2024 } 2025 continue; 2026 } 2027 2028 decoder->pkt_step = next - decoder->buf; 2029 return intel_pt_get_next_packet(decoder); 2030 } 2031 } 2032 2033 static int intel_pt_sync(struct intel_pt_decoder *decoder) 2034 { 2035 int err; 2036 2037 decoder->pge = false; 2038 decoder->continuous_period = false; 2039 decoder->last_ip = 0; 2040 decoder->ip = 0; 2041 intel_pt_clear_stack(&decoder->stack); 2042 2043 err = intel_pt_scan_for_psb(decoder); 2044 if (err) 2045 return err; 2046 2047 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 2048 2049 err = intel_pt_walk_psb(decoder); 2050 if (err) 2051 return err; 2052 2053 if (decoder->ip) { 2054 decoder->state.type = 0; /* Do not have a sample */ 2055 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2056 } else { 2057 return intel_pt_sync_ip(decoder); 2058 } 2059 2060 return 0; 2061 } 2062 2063 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2064 { 2065 uint64_t est = decoder->timestamp_insn_cnt << 1; 2066 2067 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2068 goto out; 2069 2070 est *= decoder->max_non_turbo_ratio; 2071 est /= decoder->cbr; 2072 out: 2073 return decoder->timestamp + est; 2074 } 2075 2076 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2077 { 2078 int err; 2079 2080 do { 2081 decoder->state.type = INTEL_PT_BRANCH; 2082 decoder->state.flags = 0; 2083 2084 switch (decoder->pkt_state) { 2085 case INTEL_PT_STATE_NO_PSB: 2086 err = intel_pt_sync(decoder); 2087 break; 2088 case INTEL_PT_STATE_NO_IP: 2089 decoder->last_ip = 0; 2090 /* Fall through */ 2091 case INTEL_PT_STATE_ERR_RESYNC: 2092 err = intel_pt_sync_ip(decoder); 2093 break; 2094 case INTEL_PT_STATE_IN_SYNC: 2095 err = intel_pt_walk_trace(decoder); 2096 break; 2097 case INTEL_PT_STATE_TNT: 2098 err = intel_pt_walk_tnt(decoder); 2099 if (err == -EAGAIN) 2100 err = intel_pt_walk_trace(decoder); 2101 break; 2102 case INTEL_PT_STATE_TIP: 2103 case INTEL_PT_STATE_TIP_PGD: 2104 err = intel_pt_walk_tip(decoder); 2105 break; 2106 case INTEL_PT_STATE_FUP: 2107 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2108 err = intel_pt_walk_fup(decoder); 2109 if (err == -EAGAIN) 2110 err = intel_pt_walk_fup_tip(decoder); 2111 else if (!err) 2112 decoder->pkt_state = INTEL_PT_STATE_FUP; 2113 break; 2114 case INTEL_PT_STATE_FUP_NO_TIP: 2115 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2116 err = intel_pt_walk_fup(decoder); 2117 if (err == -EAGAIN) 2118 err = intel_pt_walk_trace(decoder); 2119 break; 2120 default: 2121 err = intel_pt_bug(decoder); 2122 break; 2123 } 2124 } while (err == -ENOLINK); 2125 2126 decoder->state.err = err ? intel_pt_ext_err(err) : 0; 2127 decoder->state.timestamp = decoder->timestamp; 2128 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2129 decoder->state.cr3 = decoder->cr3; 2130 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2131 2132 if (err) 2133 decoder->state.from_ip = decoder->ip; 2134 2135 return &decoder->state; 2136 } 2137 2138 static bool intel_pt_at_psb(unsigned char *buf, size_t len) 2139 { 2140 if (len < INTEL_PT_PSB_LEN) 2141 return false; 2142 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR, 2143 INTEL_PT_PSB_LEN); 2144 } 2145 2146 /** 2147 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2148 * @buf: pointer to buffer pointer 2149 * @len: size of buffer 2150 * 2151 * Updates the buffer pointer to point to the start of the next PSB packet if 2152 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2153 * @len is adjusted accordingly. 2154 * 2155 * Return: %true if a PSB packet is found, %false otherwise. 2156 */ 2157 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2158 { 2159 unsigned char *next; 2160 2161 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2162 if (next) { 2163 *len -= next - *buf; 2164 *buf = next; 2165 return true; 2166 } 2167 return false; 2168 } 2169 2170 /** 2171 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2172 * packet. 2173 * @buf: pointer to buffer pointer 2174 * @len: size of buffer 2175 * 2176 * Updates the buffer pointer to point to the start of the following PSB packet 2177 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2178 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2179 * 2180 * Return: %true if a PSB packet is found, %false otherwise. 2181 */ 2182 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2183 { 2184 unsigned char *next; 2185 2186 if (!*len) 2187 return false; 2188 2189 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2190 if (next) { 2191 *len -= next - *buf; 2192 *buf = next; 2193 return true; 2194 } 2195 return false; 2196 } 2197 2198 /** 2199 * intel_pt_last_psb - find the last PSB packet in a buffer. 2200 * @buf: buffer 2201 * @len: size of buffer 2202 * 2203 * This function finds the last PSB in a buffer. 2204 * 2205 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2206 */ 2207 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2208 { 2209 const char *n = INTEL_PT_PSB_STR; 2210 unsigned char *p; 2211 size_t k; 2212 2213 if (len < INTEL_PT_PSB_LEN) 2214 return NULL; 2215 2216 k = len - INTEL_PT_PSB_LEN + 1; 2217 while (1) { 2218 p = memrchr(buf, n[0], k); 2219 if (!p) 2220 return NULL; 2221 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2222 return p; 2223 k = p - buf; 2224 if (!k) 2225 return NULL; 2226 } 2227 } 2228 2229 /** 2230 * intel_pt_next_tsc - find and return next TSC. 2231 * @buf: buffer 2232 * @len: size of buffer 2233 * @tsc: TSC value returned 2234 * 2235 * Find a TSC packet in @buf and return the TSC value. This function assumes 2236 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2237 * PSBEND packet is found. 2238 * 2239 * Return: %true if TSC is found, false otherwise. 2240 */ 2241 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc) 2242 { 2243 struct intel_pt_pkt packet; 2244 int ret; 2245 2246 while (len) { 2247 ret = intel_pt_get_packet(buf, len, &packet); 2248 if (ret <= 0) 2249 return false; 2250 if (packet.type == INTEL_PT_TSC) { 2251 *tsc = packet.payload; 2252 return true; 2253 } 2254 if (packet.type == INTEL_PT_PSBEND) 2255 return false; 2256 buf += ret; 2257 len -= ret; 2258 } 2259 return false; 2260 } 2261 2262 /** 2263 * intel_pt_tsc_cmp - compare 7-byte TSCs. 2264 * @tsc1: first TSC to compare 2265 * @tsc2: second TSC to compare 2266 * 2267 * This function compares 7-byte TSC values allowing for the possibility that 2268 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 2269 * around so for that purpose this function assumes the absolute difference is 2270 * less than half the maximum difference. 2271 * 2272 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 2273 * after @tsc2. 2274 */ 2275 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 2276 { 2277 const uint64_t halfway = (1ULL << 55); 2278 2279 if (tsc1 == tsc2) 2280 return 0; 2281 2282 if (tsc1 < tsc2) { 2283 if (tsc2 - tsc1 < halfway) 2284 return -1; 2285 else 2286 return 1; 2287 } else { 2288 if (tsc1 - tsc2 < halfway) 2289 return 1; 2290 else 2291 return -1; 2292 } 2293 } 2294 2295 /** 2296 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 2297 * using TSC. 2298 * @buf_a: first buffer 2299 * @len_a: size of first buffer 2300 * @buf_b: second buffer 2301 * @len_b: size of second buffer 2302 * 2303 * If the trace contains TSC we can look at the last TSC of @buf_a and the 2304 * first TSC of @buf_b in order to determine if the buffers overlap, and then 2305 * walk forward in @buf_b until a later TSC is found. A precondition is that 2306 * @buf_a and @buf_b are positioned at a PSB. 2307 * 2308 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2309 * @buf_b + @len_b if there is no non-overlapped data. 2310 */ 2311 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 2312 size_t len_a, 2313 unsigned char *buf_b, 2314 size_t len_b) 2315 { 2316 uint64_t tsc_a, tsc_b; 2317 unsigned char *p; 2318 size_t len; 2319 2320 p = intel_pt_last_psb(buf_a, len_a); 2321 if (!p) 2322 return buf_b; /* No PSB in buf_a => no overlap */ 2323 2324 len = len_a - (p - buf_a); 2325 if (!intel_pt_next_tsc(p, len, &tsc_a)) { 2326 /* The last PSB+ in buf_a is incomplete, so go back one more */ 2327 len_a -= len; 2328 p = intel_pt_last_psb(buf_a, len_a); 2329 if (!p) 2330 return buf_b; /* No full PSB+ => assume no overlap */ 2331 len = len_a - (p - buf_a); 2332 if (!intel_pt_next_tsc(p, len, &tsc_a)) 2333 return buf_b; /* No TSC in buf_a => assume no overlap */ 2334 } 2335 2336 while (1) { 2337 /* Ignore PSB+ with no TSC */ 2338 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) && 2339 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0) 2340 return buf_b; /* tsc_a < tsc_b => no overlap */ 2341 2342 if (!intel_pt_step_psb(&buf_b, &len_b)) 2343 return buf_b + len_b; /* No PSB in buf_b => no data */ 2344 } 2345 } 2346 2347 /** 2348 * intel_pt_find_overlap - determine start of non-overlapped trace data. 2349 * @buf_a: first buffer 2350 * @len_a: size of first buffer 2351 * @buf_b: second buffer 2352 * @len_b: size of second buffer 2353 * @have_tsc: can use TSC packets to detect overlap 2354 * 2355 * When trace samples or snapshots are recorded there is the possibility that 2356 * the data overlaps. Note that, for the purposes of decoding, data is only 2357 * useful if it begins with a PSB packet. 2358 * 2359 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2360 * @buf_b + @len_b if there is no non-overlapped data. 2361 */ 2362 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 2363 unsigned char *buf_b, size_t len_b, 2364 bool have_tsc) 2365 { 2366 unsigned char *found; 2367 2368 /* Buffer 'b' must start at PSB so throw away everything before that */ 2369 if (!intel_pt_next_psb(&buf_b, &len_b)) 2370 return buf_b + len_b; /* No PSB */ 2371 2372 if (!intel_pt_next_psb(&buf_a, &len_a)) 2373 return buf_b; /* No overlap */ 2374 2375 if (have_tsc) { 2376 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b); 2377 if (found) 2378 return found; 2379 } 2380 2381 /* 2382 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 2383 * we can ignore the first part of buffer 'a'. 2384 */ 2385 while (len_b < len_a) { 2386 if (!intel_pt_step_psb(&buf_a, &len_a)) 2387 return buf_b; /* No overlap */ 2388 } 2389 2390 /* Now len_b >= len_a */ 2391 if (len_b > len_a) { 2392 /* The leftover buffer 'b' must start at a PSB */ 2393 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2394 if (!intel_pt_step_psb(&buf_a, &len_a)) 2395 return buf_b; /* No overlap */ 2396 } 2397 } 2398 2399 while (1) { 2400 /* Potential overlap so check the bytes */ 2401 found = memmem(buf_a, len_a, buf_b, len_a); 2402 if (found) 2403 return buf_b + len_a; 2404 2405 /* Try again at next PSB in buffer 'a' */ 2406 if (!intel_pt_step_psb(&buf_a, &len_a)) 2407 return buf_b; /* No overlap */ 2408 2409 /* The leftover buffer 'b' must start at a PSB */ 2410 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2411 if (!intel_pt_step_psb(&buf_a, &len_a)) 2412 return buf_b; /* No overlap */ 2413 } 2414 } 2415 } 2416