1 /* 2 * intel_pt_decoder.c: Intel Processor Trace support 3 * Copyright (c) 2013-2014, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #ifndef _GNU_SOURCE 17 #define _GNU_SOURCE 18 #endif 19 #include <stdlib.h> 20 #include <stdbool.h> 21 #include <string.h> 22 #include <errno.h> 23 #include <stdint.h> 24 #include <inttypes.h> 25 #include <linux/compiler.h> 26 27 #include "../cache.h" 28 #include "../util.h" 29 30 #include "intel-pt-insn-decoder.h" 31 #include "intel-pt-pkt-decoder.h" 32 #include "intel-pt-decoder.h" 33 #include "intel-pt-log.h" 34 35 #define INTEL_PT_BLK_SIZE 1024 36 37 #define BIT63 (((uint64_t)1 << 63)) 38 39 #define INTEL_PT_RETURN 1 40 41 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 42 #define INTEL_PT_MAX_LOOPS 10000 43 44 struct intel_pt_blk { 45 struct intel_pt_blk *prev; 46 uint64_t ip[INTEL_PT_BLK_SIZE]; 47 }; 48 49 struct intel_pt_stack { 50 struct intel_pt_blk *blk; 51 struct intel_pt_blk *spare; 52 int pos; 53 }; 54 55 enum intel_pt_pkt_state { 56 INTEL_PT_STATE_NO_PSB, 57 INTEL_PT_STATE_NO_IP, 58 INTEL_PT_STATE_ERR_RESYNC, 59 INTEL_PT_STATE_IN_SYNC, 60 INTEL_PT_STATE_TNT, 61 INTEL_PT_STATE_TIP, 62 INTEL_PT_STATE_TIP_PGD, 63 INTEL_PT_STATE_FUP, 64 INTEL_PT_STATE_FUP_NO_TIP, 65 }; 66 67 #ifdef INTEL_PT_STRICT 68 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 69 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 70 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 71 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 72 #else 73 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 74 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 75 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 76 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 77 #endif 78 79 struct intel_pt_decoder { 80 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 81 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 82 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 83 uint64_t max_insn_cnt, void *data); 84 bool (*pgd_ip)(uint64_t ip, void *data); 85 void *data; 86 struct intel_pt_state state; 87 const unsigned char *buf; 88 size_t len; 89 bool return_compression; 90 bool mtc_insn; 91 bool pge; 92 bool have_tma; 93 bool have_cyc; 94 bool fixup_last_mtc; 95 uint64_t pos; 96 uint64_t last_ip; 97 uint64_t ip; 98 uint64_t cr3; 99 uint64_t timestamp; 100 uint64_t tsc_timestamp; 101 uint64_t ref_timestamp; 102 uint64_t ret_addr; 103 uint64_t ctc_timestamp; 104 uint64_t ctc_delta; 105 uint64_t cycle_cnt; 106 uint64_t cyc_ref_timestamp; 107 uint32_t last_mtc; 108 uint32_t tsc_ctc_ratio_n; 109 uint32_t tsc_ctc_ratio_d; 110 uint32_t tsc_ctc_mult; 111 uint32_t tsc_slip; 112 uint32_t ctc_rem_mask; 113 int mtc_shift; 114 struct intel_pt_stack stack; 115 enum intel_pt_pkt_state pkt_state; 116 struct intel_pt_pkt packet; 117 struct intel_pt_pkt tnt; 118 int pkt_step; 119 int pkt_len; 120 int last_packet_type; 121 unsigned int cbr; 122 unsigned int max_non_turbo_ratio; 123 double max_non_turbo_ratio_fp; 124 double cbr_cyc_to_tsc; 125 double calc_cyc_to_tsc; 126 bool have_calc_cyc_to_tsc; 127 int exec_mode; 128 unsigned int insn_bytes; 129 uint64_t period; 130 enum intel_pt_period_type period_type; 131 uint64_t tot_insn_cnt; 132 uint64_t period_insn_cnt; 133 uint64_t period_mask; 134 uint64_t period_ticks; 135 uint64_t last_masked_timestamp; 136 bool continuous_period; 137 bool overflow; 138 bool set_fup_tx_flags; 139 unsigned int fup_tx_flags; 140 unsigned int tx_flags; 141 uint64_t timestamp_insn_cnt; 142 uint64_t stuck_ip; 143 int no_progress; 144 int stuck_ip_prd; 145 int stuck_ip_cnt; 146 const unsigned char *next_buf; 147 size_t next_len; 148 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 149 }; 150 151 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 152 { 153 int i; 154 155 for (i = 0; x != 1; i++) 156 x >>= 1; 157 158 return x << i; 159 } 160 161 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 162 { 163 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 164 uint64_t period; 165 166 period = intel_pt_lower_power_of_2(decoder->period); 167 decoder->period_mask = ~(period - 1); 168 decoder->period_ticks = period; 169 } 170 } 171 172 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 173 { 174 if (!d) 175 return 0; 176 return (t / d) * n + ((t % d) * n) / d; 177 } 178 179 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 180 { 181 struct intel_pt_decoder *decoder; 182 183 if (!params->get_trace || !params->walk_insn) 184 return NULL; 185 186 decoder = zalloc(sizeof(struct intel_pt_decoder)); 187 if (!decoder) 188 return NULL; 189 190 decoder->get_trace = params->get_trace; 191 decoder->walk_insn = params->walk_insn; 192 decoder->pgd_ip = params->pgd_ip; 193 decoder->data = params->data; 194 decoder->return_compression = params->return_compression; 195 196 decoder->period = params->period; 197 decoder->period_type = params->period_type; 198 199 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 200 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 201 202 intel_pt_setup_period(decoder); 203 204 decoder->mtc_shift = params->mtc_period; 205 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 206 207 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 208 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 209 210 if (!decoder->tsc_ctc_ratio_n) 211 decoder->tsc_ctc_ratio_d = 0; 212 213 if (decoder->tsc_ctc_ratio_d) { 214 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 215 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 216 decoder->tsc_ctc_ratio_d; 217 218 /* 219 * Allow for timestamps appearing to backwards because a TSC 220 * packet has slipped past a MTC packet, so allow 2 MTC ticks 221 * or ... 222 */ 223 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift, 224 decoder->tsc_ctc_ratio_n, 225 decoder->tsc_ctc_ratio_d); 226 } 227 /* ... or 0x100 paranoia */ 228 if (decoder->tsc_slip < 0x100) 229 decoder->tsc_slip = 0x100; 230 231 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 232 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 233 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 234 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 235 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 236 237 return decoder; 238 } 239 240 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 241 { 242 struct intel_pt_blk *blk = stack->blk; 243 244 stack->blk = blk->prev; 245 if (!stack->spare) 246 stack->spare = blk; 247 else 248 free(blk); 249 } 250 251 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 252 { 253 if (!stack->pos) { 254 if (!stack->blk) 255 return 0; 256 intel_pt_pop_blk(stack); 257 if (!stack->blk) 258 return 0; 259 stack->pos = INTEL_PT_BLK_SIZE; 260 } 261 return stack->blk->ip[--stack->pos]; 262 } 263 264 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 265 { 266 struct intel_pt_blk *blk; 267 268 if (stack->spare) { 269 blk = stack->spare; 270 stack->spare = NULL; 271 } else { 272 blk = malloc(sizeof(struct intel_pt_blk)); 273 if (!blk) 274 return -ENOMEM; 275 } 276 277 blk->prev = stack->blk; 278 stack->blk = blk; 279 stack->pos = 0; 280 return 0; 281 } 282 283 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 284 { 285 int err; 286 287 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 288 err = intel_pt_alloc_blk(stack); 289 if (err) 290 return err; 291 } 292 293 stack->blk->ip[stack->pos++] = ip; 294 return 0; 295 } 296 297 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 298 { 299 while (stack->blk) 300 intel_pt_pop_blk(stack); 301 stack->pos = 0; 302 } 303 304 static void intel_pt_free_stack(struct intel_pt_stack *stack) 305 { 306 intel_pt_clear_stack(stack); 307 zfree(&stack->blk); 308 zfree(&stack->spare); 309 } 310 311 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 312 { 313 intel_pt_free_stack(&decoder->stack); 314 free(decoder); 315 } 316 317 static int intel_pt_ext_err(int code) 318 { 319 switch (code) { 320 case -ENOMEM: 321 return INTEL_PT_ERR_NOMEM; 322 case -ENOSYS: 323 return INTEL_PT_ERR_INTERN; 324 case -EBADMSG: 325 return INTEL_PT_ERR_BADPKT; 326 case -ENODATA: 327 return INTEL_PT_ERR_NODATA; 328 case -EILSEQ: 329 return INTEL_PT_ERR_NOINSN; 330 case -ENOENT: 331 return INTEL_PT_ERR_MISMAT; 332 case -EOVERFLOW: 333 return INTEL_PT_ERR_OVR; 334 case -ENOSPC: 335 return INTEL_PT_ERR_LOST; 336 case -ELOOP: 337 return INTEL_PT_ERR_NELOOP; 338 default: 339 return INTEL_PT_ERR_UNK; 340 } 341 } 342 343 static const char *intel_pt_err_msgs[] = { 344 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 345 [INTEL_PT_ERR_INTERN] = "Internal error", 346 [INTEL_PT_ERR_BADPKT] = "Bad packet", 347 [INTEL_PT_ERR_NODATA] = "No more data", 348 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 349 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 350 [INTEL_PT_ERR_OVR] = "Overflow packet", 351 [INTEL_PT_ERR_LOST] = "Lost trace data", 352 [INTEL_PT_ERR_UNK] = "Unknown error!", 353 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 354 }; 355 356 int intel_pt__strerror(int code, char *buf, size_t buflen) 357 { 358 if (code < 1 || code >= INTEL_PT_ERR_MAX) 359 code = INTEL_PT_ERR_UNK; 360 strlcpy(buf, intel_pt_err_msgs[code], buflen); 361 return 0; 362 } 363 364 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 365 uint64_t last_ip) 366 { 367 uint64_t ip; 368 369 switch (packet->count) { 370 case 1: 371 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 372 packet->payload; 373 break; 374 case 2: 375 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 376 packet->payload; 377 break; 378 case 3: 379 ip = packet->payload; 380 /* Sign-extend 6-byte ip */ 381 if (ip & (uint64_t)0x800000000000ULL) 382 ip |= (uint64_t)0xffff000000000000ULL; 383 break; 384 case 4: 385 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 386 packet->payload; 387 break; 388 case 6: 389 ip = packet->payload; 390 break; 391 default: 392 return 0; 393 } 394 395 return ip; 396 } 397 398 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 399 { 400 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 401 } 402 403 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 404 { 405 intel_pt_set_last_ip(decoder); 406 decoder->ip = decoder->last_ip; 407 } 408 409 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 410 { 411 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 412 decoder->buf); 413 } 414 415 static int intel_pt_bug(struct intel_pt_decoder *decoder) 416 { 417 intel_pt_log("ERROR: Internal error\n"); 418 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 419 return -ENOSYS; 420 } 421 422 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 423 { 424 decoder->tx_flags = 0; 425 } 426 427 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 428 { 429 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 430 } 431 432 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 433 { 434 intel_pt_clear_tx_flags(decoder); 435 decoder->have_tma = false; 436 decoder->pkt_len = 1; 437 decoder->pkt_step = 1; 438 intel_pt_decoder_log_packet(decoder); 439 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 440 intel_pt_log("ERROR: Bad packet\n"); 441 decoder->pkt_state = INTEL_PT_STATE_ERR1; 442 } 443 return -EBADMSG; 444 } 445 446 static int intel_pt_get_data(struct intel_pt_decoder *decoder) 447 { 448 struct intel_pt_buffer buffer = { .buf = 0, }; 449 int ret; 450 451 decoder->pkt_step = 0; 452 453 intel_pt_log("Getting more data\n"); 454 ret = decoder->get_trace(&buffer, decoder->data); 455 if (ret) 456 return ret; 457 decoder->buf = buffer.buf; 458 decoder->len = buffer.len; 459 if (!decoder->len) { 460 intel_pt_log("No more data\n"); 461 return -ENODATA; 462 } 463 if (!buffer.consecutive) { 464 decoder->ip = 0; 465 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 466 decoder->ref_timestamp = buffer.ref_timestamp; 467 decoder->timestamp = 0; 468 decoder->have_tma = false; 469 decoder->state.trace_nr = buffer.trace_nr; 470 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 471 decoder->ref_timestamp); 472 return -ENOLINK; 473 } 474 475 return 0; 476 } 477 478 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder) 479 { 480 if (!decoder->next_buf) 481 return intel_pt_get_data(decoder); 482 483 decoder->buf = decoder->next_buf; 484 decoder->len = decoder->next_len; 485 decoder->next_buf = 0; 486 decoder->next_len = 0; 487 return 0; 488 } 489 490 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 491 { 492 unsigned char *buf = decoder->temp_buf; 493 size_t old_len, len, n; 494 int ret; 495 496 old_len = decoder->len; 497 len = decoder->len; 498 memcpy(buf, decoder->buf, len); 499 500 ret = intel_pt_get_data(decoder); 501 if (ret) { 502 decoder->pos += old_len; 503 return ret < 0 ? ret : -EINVAL; 504 } 505 506 n = INTEL_PT_PKT_MAX_SZ - len; 507 if (n > decoder->len) 508 n = decoder->len; 509 memcpy(buf + len, decoder->buf, n); 510 len += n; 511 512 ret = intel_pt_get_packet(buf, len, &decoder->packet); 513 if (ret < (int)old_len) { 514 decoder->next_buf = decoder->buf; 515 decoder->next_len = decoder->len; 516 decoder->buf = buf; 517 decoder->len = old_len; 518 return intel_pt_bad_packet(decoder); 519 } 520 521 decoder->next_buf = decoder->buf + (ret - old_len); 522 decoder->next_len = decoder->len - (ret - old_len); 523 524 decoder->buf = buf; 525 decoder->len = ret; 526 527 return ret; 528 } 529 530 struct intel_pt_pkt_info { 531 struct intel_pt_decoder *decoder; 532 struct intel_pt_pkt packet; 533 uint64_t pos; 534 int pkt_len; 535 int last_packet_type; 536 void *data; 537 }; 538 539 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 540 541 /* Lookahead packets in current buffer */ 542 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 543 intel_pt_pkt_cb_t cb, void *data) 544 { 545 struct intel_pt_pkt_info pkt_info; 546 const unsigned char *buf = decoder->buf; 547 size_t len = decoder->len; 548 int ret; 549 550 pkt_info.decoder = decoder; 551 pkt_info.pos = decoder->pos; 552 pkt_info.pkt_len = decoder->pkt_step; 553 pkt_info.last_packet_type = decoder->last_packet_type; 554 pkt_info.data = data; 555 556 while (1) { 557 do { 558 pkt_info.pos += pkt_info.pkt_len; 559 buf += pkt_info.pkt_len; 560 len -= pkt_info.pkt_len; 561 562 if (!len) 563 return INTEL_PT_NEED_MORE_BYTES; 564 565 ret = intel_pt_get_packet(buf, len, &pkt_info.packet); 566 if (!ret) 567 return INTEL_PT_NEED_MORE_BYTES; 568 if (ret < 0) 569 return ret; 570 571 pkt_info.pkt_len = ret; 572 } while (pkt_info.packet.type == INTEL_PT_PAD); 573 574 ret = cb(&pkt_info); 575 if (ret) 576 return 0; 577 578 pkt_info.last_packet_type = pkt_info.packet.type; 579 } 580 } 581 582 struct intel_pt_calc_cyc_to_tsc_info { 583 uint64_t cycle_cnt; 584 unsigned int cbr; 585 uint32_t last_mtc; 586 uint64_t ctc_timestamp; 587 uint64_t ctc_delta; 588 uint64_t tsc_timestamp; 589 uint64_t timestamp; 590 bool have_tma; 591 bool fixup_last_mtc; 592 bool from_mtc; 593 double cbr_cyc_to_tsc; 594 }; 595 596 /* 597 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 598 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 599 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 600 * packet by copying the missing bits from the current MTC assuming the least 601 * difference between the two, and that the current MTC comes after last_mtc. 602 */ 603 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 604 uint32_t *last_mtc) 605 { 606 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 607 uint32_t mask = ~(first_missing_bit - 1); 608 609 *last_mtc |= mtc & mask; 610 if (*last_mtc >= mtc) { 611 *last_mtc -= first_missing_bit; 612 *last_mtc &= 0xff; 613 } 614 } 615 616 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 617 { 618 struct intel_pt_decoder *decoder = pkt_info->decoder; 619 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 620 uint64_t timestamp; 621 double cyc_to_tsc; 622 unsigned int cbr; 623 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 624 625 switch (pkt_info->packet.type) { 626 case INTEL_PT_TNT: 627 case INTEL_PT_TIP_PGE: 628 case INTEL_PT_TIP: 629 case INTEL_PT_FUP: 630 case INTEL_PT_PSB: 631 case INTEL_PT_PIP: 632 case INTEL_PT_MODE_EXEC: 633 case INTEL_PT_MODE_TSX: 634 case INTEL_PT_PSBEND: 635 case INTEL_PT_PAD: 636 case INTEL_PT_VMCS: 637 case INTEL_PT_MNT: 638 return 0; 639 640 case INTEL_PT_MTC: 641 if (!data->have_tma) 642 return 0; 643 644 mtc = pkt_info->packet.payload; 645 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 646 data->fixup_last_mtc = false; 647 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 648 &data->last_mtc); 649 } 650 if (mtc > data->last_mtc) 651 mtc_delta = mtc - data->last_mtc; 652 else 653 mtc_delta = mtc + 256 - data->last_mtc; 654 data->ctc_delta += mtc_delta << decoder->mtc_shift; 655 data->last_mtc = mtc; 656 657 if (decoder->tsc_ctc_mult) { 658 timestamp = data->ctc_timestamp + 659 data->ctc_delta * decoder->tsc_ctc_mult; 660 } else { 661 timestamp = data->ctc_timestamp + 662 multdiv(data->ctc_delta, 663 decoder->tsc_ctc_ratio_n, 664 decoder->tsc_ctc_ratio_d); 665 } 666 667 if (timestamp < data->timestamp) 668 return 1; 669 670 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 671 data->timestamp = timestamp; 672 return 0; 673 } 674 675 break; 676 677 case INTEL_PT_TSC: 678 timestamp = pkt_info->packet.payload | 679 (data->timestamp & (0xffULL << 56)); 680 if (data->from_mtc && timestamp < data->timestamp && 681 data->timestamp - timestamp < decoder->tsc_slip) 682 return 1; 683 if (timestamp < data->timestamp) 684 timestamp += (1ULL << 56); 685 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 686 if (data->from_mtc) 687 return 1; 688 data->tsc_timestamp = timestamp; 689 data->timestamp = timestamp; 690 return 0; 691 } 692 break; 693 694 case INTEL_PT_TMA: 695 if (data->from_mtc) 696 return 1; 697 698 if (!decoder->tsc_ctc_ratio_d) 699 return 0; 700 701 ctc = pkt_info->packet.payload; 702 fc = pkt_info->packet.count; 703 ctc_rem = ctc & decoder->ctc_rem_mask; 704 705 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 706 707 data->ctc_timestamp = data->tsc_timestamp - fc; 708 if (decoder->tsc_ctc_mult) { 709 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 710 } else { 711 data->ctc_timestamp -= 712 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 713 decoder->tsc_ctc_ratio_d); 714 } 715 716 data->ctc_delta = 0; 717 data->have_tma = true; 718 data->fixup_last_mtc = true; 719 720 return 0; 721 722 case INTEL_PT_CYC: 723 data->cycle_cnt += pkt_info->packet.payload; 724 return 0; 725 726 case INTEL_PT_CBR: 727 cbr = pkt_info->packet.payload; 728 if (data->cbr && data->cbr != cbr) 729 return 1; 730 data->cbr = cbr; 731 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 732 return 0; 733 734 case INTEL_PT_TIP_PGD: 735 case INTEL_PT_TRACESTOP: 736 case INTEL_PT_OVF: 737 case INTEL_PT_BAD: /* Does not happen */ 738 default: 739 return 1; 740 } 741 742 if (!data->cbr && decoder->cbr) { 743 data->cbr = decoder->cbr; 744 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 745 } 746 747 if (!data->cycle_cnt) 748 return 1; 749 750 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 751 752 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 753 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 754 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 755 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 756 return 1; 757 } 758 759 decoder->calc_cyc_to_tsc = cyc_to_tsc; 760 decoder->have_calc_cyc_to_tsc = true; 761 762 if (data->cbr) { 763 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 764 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 765 } else { 766 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 767 cyc_to_tsc, pkt_info->pos); 768 } 769 770 return 1; 771 } 772 773 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 774 bool from_mtc) 775 { 776 struct intel_pt_calc_cyc_to_tsc_info data = { 777 .cycle_cnt = 0, 778 .cbr = 0, 779 .last_mtc = decoder->last_mtc, 780 .ctc_timestamp = decoder->ctc_timestamp, 781 .ctc_delta = decoder->ctc_delta, 782 .tsc_timestamp = decoder->tsc_timestamp, 783 .timestamp = decoder->timestamp, 784 .have_tma = decoder->have_tma, 785 .fixup_last_mtc = decoder->fixup_last_mtc, 786 .from_mtc = from_mtc, 787 .cbr_cyc_to_tsc = 0, 788 }; 789 790 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 791 } 792 793 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 794 { 795 int ret; 796 797 decoder->last_packet_type = decoder->packet.type; 798 799 do { 800 decoder->pos += decoder->pkt_step; 801 decoder->buf += decoder->pkt_step; 802 decoder->len -= decoder->pkt_step; 803 804 if (!decoder->len) { 805 ret = intel_pt_get_next_data(decoder); 806 if (ret) 807 return ret; 808 } 809 810 ret = intel_pt_get_packet(decoder->buf, decoder->len, 811 &decoder->packet); 812 if (ret == INTEL_PT_NEED_MORE_BYTES && 813 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 814 ret = intel_pt_get_split_packet(decoder); 815 if (ret < 0) 816 return ret; 817 } 818 if (ret <= 0) 819 return intel_pt_bad_packet(decoder); 820 821 decoder->pkt_len = ret; 822 decoder->pkt_step = ret; 823 intel_pt_decoder_log_packet(decoder); 824 } while (decoder->packet.type == INTEL_PT_PAD); 825 826 return 0; 827 } 828 829 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 830 { 831 uint64_t timestamp, masked_timestamp; 832 833 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 834 masked_timestamp = timestamp & decoder->period_mask; 835 if (decoder->continuous_period) { 836 if (masked_timestamp != decoder->last_masked_timestamp) 837 return 1; 838 } else { 839 timestamp += 1; 840 masked_timestamp = timestamp & decoder->period_mask; 841 if (masked_timestamp != decoder->last_masked_timestamp) { 842 decoder->last_masked_timestamp = masked_timestamp; 843 decoder->continuous_period = true; 844 } 845 } 846 return decoder->period_ticks - (timestamp - masked_timestamp); 847 } 848 849 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 850 { 851 switch (decoder->period_type) { 852 case INTEL_PT_PERIOD_INSTRUCTIONS: 853 return decoder->period - decoder->period_insn_cnt; 854 case INTEL_PT_PERIOD_TICKS: 855 return intel_pt_next_period(decoder); 856 case INTEL_PT_PERIOD_NONE: 857 case INTEL_PT_PERIOD_MTC: 858 default: 859 return 0; 860 } 861 } 862 863 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 864 { 865 uint64_t timestamp, masked_timestamp; 866 867 switch (decoder->period_type) { 868 case INTEL_PT_PERIOD_INSTRUCTIONS: 869 decoder->period_insn_cnt = 0; 870 break; 871 case INTEL_PT_PERIOD_TICKS: 872 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 873 masked_timestamp = timestamp & decoder->period_mask; 874 decoder->last_masked_timestamp = masked_timestamp; 875 break; 876 case INTEL_PT_PERIOD_NONE: 877 case INTEL_PT_PERIOD_MTC: 878 default: 879 break; 880 } 881 882 decoder->state.type |= INTEL_PT_INSTRUCTION; 883 } 884 885 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 886 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 887 { 888 uint64_t max_insn_cnt, insn_cnt = 0; 889 int err; 890 891 if (!decoder->mtc_insn) 892 decoder->mtc_insn = true; 893 894 max_insn_cnt = intel_pt_next_sample(decoder); 895 896 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 897 max_insn_cnt, decoder->data); 898 899 decoder->tot_insn_cnt += insn_cnt; 900 decoder->timestamp_insn_cnt += insn_cnt; 901 decoder->period_insn_cnt += insn_cnt; 902 903 if (err) { 904 decoder->no_progress = 0; 905 decoder->pkt_state = INTEL_PT_STATE_ERR2; 906 intel_pt_log_at("ERROR: Failed to get instruction", 907 decoder->ip); 908 if (err == -ENOENT) 909 return -ENOLINK; 910 return -EILSEQ; 911 } 912 913 if (ip && decoder->ip == ip) { 914 err = -EAGAIN; 915 goto out; 916 } 917 918 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 919 intel_pt_sample_insn(decoder); 920 921 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 922 decoder->state.type = INTEL_PT_INSTRUCTION; 923 decoder->state.from_ip = decoder->ip; 924 decoder->state.to_ip = 0; 925 decoder->ip += intel_pt_insn->length; 926 err = INTEL_PT_RETURN; 927 goto out; 928 } 929 930 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 931 /* Zero-length calls are excluded */ 932 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 933 intel_pt_insn->rel) { 934 err = intel_pt_push(&decoder->stack, decoder->ip + 935 intel_pt_insn->length); 936 if (err) 937 goto out; 938 } 939 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 940 decoder->ret_addr = intel_pt_pop(&decoder->stack); 941 } 942 943 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 944 int cnt = decoder->no_progress++; 945 946 decoder->state.from_ip = decoder->ip; 947 decoder->ip += intel_pt_insn->length + 948 intel_pt_insn->rel; 949 decoder->state.to_ip = decoder->ip; 950 err = INTEL_PT_RETURN; 951 952 /* 953 * Check for being stuck in a loop. This can happen if a 954 * decoder error results in the decoder erroneously setting the 955 * ip to an address that is itself in an infinite loop that 956 * consumes no packets. When that happens, there must be an 957 * unconditional branch. 958 */ 959 if (cnt) { 960 if (cnt == 1) { 961 decoder->stuck_ip = decoder->state.to_ip; 962 decoder->stuck_ip_prd = 1; 963 decoder->stuck_ip_cnt = 1; 964 } else if (cnt > INTEL_PT_MAX_LOOPS || 965 decoder->state.to_ip == decoder->stuck_ip) { 966 intel_pt_log_at("ERROR: Never-ending loop", 967 decoder->state.to_ip); 968 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 969 err = -ELOOP; 970 goto out; 971 } else if (!--decoder->stuck_ip_cnt) { 972 decoder->stuck_ip_prd += 1; 973 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 974 decoder->stuck_ip = decoder->state.to_ip; 975 } 976 } 977 goto out_no_progress; 978 } 979 out: 980 decoder->no_progress = 0; 981 out_no_progress: 982 decoder->state.insn_op = intel_pt_insn->op; 983 decoder->state.insn_len = intel_pt_insn->length; 984 memcpy(decoder->state.insn, intel_pt_insn->buf, 985 INTEL_PT_INSN_BUF_SZ); 986 987 if (decoder->tx_flags & INTEL_PT_IN_TX) 988 decoder->state.flags |= INTEL_PT_IN_TX; 989 990 return err; 991 } 992 993 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 994 { 995 struct intel_pt_insn intel_pt_insn; 996 uint64_t ip; 997 int err; 998 999 ip = decoder->last_ip; 1000 1001 while (1) { 1002 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1003 if (err == INTEL_PT_RETURN) 1004 return 0; 1005 if (err == -EAGAIN) { 1006 if (decoder->set_fup_tx_flags) { 1007 decoder->set_fup_tx_flags = false; 1008 decoder->tx_flags = decoder->fup_tx_flags; 1009 decoder->state.type = INTEL_PT_TRANSACTION; 1010 decoder->state.from_ip = decoder->ip; 1011 decoder->state.to_ip = 0; 1012 decoder->state.flags = decoder->fup_tx_flags; 1013 return 0; 1014 } 1015 return err; 1016 } 1017 decoder->set_fup_tx_flags = false; 1018 if (err) 1019 return err; 1020 1021 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1022 intel_pt_log_at("ERROR: Unexpected indirect branch", 1023 decoder->ip); 1024 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1025 return -ENOENT; 1026 } 1027 1028 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1029 intel_pt_log_at("ERROR: Unexpected conditional branch", 1030 decoder->ip); 1031 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1032 return -ENOENT; 1033 } 1034 1035 intel_pt_bug(decoder); 1036 } 1037 } 1038 1039 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1040 { 1041 struct intel_pt_insn intel_pt_insn; 1042 int err; 1043 1044 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1045 if (err == INTEL_PT_RETURN && 1046 decoder->pgd_ip && 1047 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1048 (decoder->state.type & INTEL_PT_BRANCH) && 1049 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1050 /* Unconditional branch leaving filter region */ 1051 decoder->no_progress = 0; 1052 decoder->pge = false; 1053 decoder->continuous_period = false; 1054 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1055 decoder->state.to_ip = 0; 1056 return 0; 1057 } 1058 if (err == INTEL_PT_RETURN) 1059 return 0; 1060 if (err) 1061 return err; 1062 1063 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1064 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1065 decoder->pge = false; 1066 decoder->continuous_period = false; 1067 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1068 decoder->state.from_ip = decoder->ip; 1069 decoder->state.to_ip = 0; 1070 if (decoder->packet.count != 0) 1071 decoder->ip = decoder->last_ip; 1072 } else { 1073 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1074 decoder->state.from_ip = decoder->ip; 1075 if (decoder->packet.count == 0) { 1076 decoder->state.to_ip = 0; 1077 } else { 1078 decoder->state.to_ip = decoder->last_ip; 1079 decoder->ip = decoder->last_ip; 1080 } 1081 } 1082 return 0; 1083 } 1084 1085 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1086 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1087 intel_pt_insn.rel; 1088 1089 if (decoder->pgd_ip && 1090 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1091 decoder->pgd_ip(to_ip, decoder->data)) { 1092 /* Conditional branch leaving filter region */ 1093 decoder->pge = false; 1094 decoder->continuous_period = false; 1095 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1096 decoder->ip = to_ip; 1097 decoder->state.from_ip = decoder->ip; 1098 decoder->state.to_ip = 0; 1099 return 0; 1100 } 1101 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1102 decoder->ip); 1103 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1104 return -ENOENT; 1105 } 1106 1107 return intel_pt_bug(decoder); 1108 } 1109 1110 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1111 { 1112 struct intel_pt_insn intel_pt_insn; 1113 int err; 1114 1115 while (1) { 1116 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1117 if (err == INTEL_PT_RETURN) 1118 return 0; 1119 if (err) 1120 return err; 1121 1122 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1123 if (!decoder->return_compression) { 1124 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1125 decoder->ip); 1126 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1127 return -ENOENT; 1128 } 1129 if (!decoder->ret_addr) { 1130 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1131 decoder->ip); 1132 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1133 return -ENOENT; 1134 } 1135 if (!(decoder->tnt.payload & BIT63)) { 1136 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1137 decoder->ip); 1138 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1139 return -ENOENT; 1140 } 1141 decoder->tnt.count -= 1; 1142 if (!decoder->tnt.count) 1143 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1144 decoder->tnt.payload <<= 1; 1145 decoder->state.from_ip = decoder->ip; 1146 decoder->ip = decoder->ret_addr; 1147 decoder->state.to_ip = decoder->ip; 1148 return 0; 1149 } 1150 1151 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1152 /* Handle deferred TIPs */ 1153 err = intel_pt_get_next_packet(decoder); 1154 if (err) 1155 return err; 1156 if (decoder->packet.type != INTEL_PT_TIP || 1157 decoder->packet.count == 0) { 1158 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1159 decoder->ip); 1160 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1161 decoder->pkt_step = 0; 1162 return -ENOENT; 1163 } 1164 intel_pt_set_last_ip(decoder); 1165 decoder->state.from_ip = decoder->ip; 1166 decoder->state.to_ip = decoder->last_ip; 1167 decoder->ip = decoder->last_ip; 1168 return 0; 1169 } 1170 1171 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1172 decoder->tnt.count -= 1; 1173 if (!decoder->tnt.count) 1174 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1175 if (decoder->tnt.payload & BIT63) { 1176 decoder->tnt.payload <<= 1; 1177 decoder->state.from_ip = decoder->ip; 1178 decoder->ip += intel_pt_insn.length + 1179 intel_pt_insn.rel; 1180 decoder->state.to_ip = decoder->ip; 1181 return 0; 1182 } 1183 /* Instruction sample for a non-taken branch */ 1184 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1185 decoder->tnt.payload <<= 1; 1186 decoder->state.type = INTEL_PT_INSTRUCTION; 1187 decoder->state.from_ip = decoder->ip; 1188 decoder->state.to_ip = 0; 1189 decoder->ip += intel_pt_insn.length; 1190 return 0; 1191 } 1192 decoder->ip += intel_pt_insn.length; 1193 if (!decoder->tnt.count) 1194 return -EAGAIN; 1195 decoder->tnt.payload <<= 1; 1196 continue; 1197 } 1198 1199 return intel_pt_bug(decoder); 1200 } 1201 } 1202 1203 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1204 { 1205 unsigned int fup_tx_flags; 1206 int err; 1207 1208 fup_tx_flags = decoder->packet.payload & 1209 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1210 err = intel_pt_get_next_packet(decoder); 1211 if (err) 1212 return err; 1213 if (decoder->packet.type == INTEL_PT_FUP) { 1214 decoder->fup_tx_flags = fup_tx_flags; 1215 decoder->set_fup_tx_flags = true; 1216 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1217 *no_tip = true; 1218 } else { 1219 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1220 decoder->pos); 1221 intel_pt_update_in_tx(decoder); 1222 } 1223 return 0; 1224 } 1225 1226 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1227 { 1228 uint64_t timestamp; 1229 1230 decoder->have_tma = false; 1231 1232 if (decoder->ref_timestamp) { 1233 timestamp = decoder->packet.payload | 1234 (decoder->ref_timestamp & (0xffULL << 56)); 1235 if (timestamp < decoder->ref_timestamp) { 1236 if (decoder->ref_timestamp - timestamp > (1ULL << 55)) 1237 timestamp += (1ULL << 56); 1238 } else { 1239 if (timestamp - decoder->ref_timestamp > (1ULL << 55)) 1240 timestamp -= (1ULL << 56); 1241 } 1242 decoder->tsc_timestamp = timestamp; 1243 decoder->timestamp = timestamp; 1244 decoder->ref_timestamp = 0; 1245 decoder->timestamp_insn_cnt = 0; 1246 } else if (decoder->timestamp) { 1247 timestamp = decoder->packet.payload | 1248 (decoder->timestamp & (0xffULL << 56)); 1249 decoder->tsc_timestamp = timestamp; 1250 if (timestamp < decoder->timestamp && 1251 decoder->timestamp - timestamp < decoder->tsc_slip) { 1252 intel_pt_log_to("Suppressing backwards timestamp", 1253 timestamp); 1254 timestamp = decoder->timestamp; 1255 } 1256 if (timestamp < decoder->timestamp) { 1257 intel_pt_log_to("Wraparound timestamp", timestamp); 1258 timestamp += (1ULL << 56); 1259 decoder->tsc_timestamp = timestamp; 1260 } 1261 decoder->timestamp = timestamp; 1262 decoder->timestamp_insn_cnt = 0; 1263 } 1264 1265 if (decoder->last_packet_type == INTEL_PT_CYC) { 1266 decoder->cyc_ref_timestamp = decoder->timestamp; 1267 decoder->cycle_cnt = 0; 1268 decoder->have_calc_cyc_to_tsc = false; 1269 intel_pt_calc_cyc_to_tsc(decoder, false); 1270 } 1271 1272 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1273 } 1274 1275 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1276 { 1277 intel_pt_log("ERROR: Buffer overflow\n"); 1278 intel_pt_clear_tx_flags(decoder); 1279 decoder->have_tma = false; 1280 decoder->cbr = 0; 1281 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1282 decoder->overflow = true; 1283 return -EOVERFLOW; 1284 } 1285 1286 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1287 { 1288 uint32_t ctc = decoder->packet.payload; 1289 uint32_t fc = decoder->packet.count; 1290 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1291 1292 if (!decoder->tsc_ctc_ratio_d) 1293 return; 1294 1295 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1296 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1297 if (decoder->tsc_ctc_mult) { 1298 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1299 } else { 1300 decoder->ctc_timestamp -= multdiv(ctc_rem, 1301 decoder->tsc_ctc_ratio_n, 1302 decoder->tsc_ctc_ratio_d); 1303 } 1304 decoder->ctc_delta = 0; 1305 decoder->have_tma = true; 1306 decoder->fixup_last_mtc = true; 1307 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1308 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1309 } 1310 1311 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1312 { 1313 uint64_t timestamp; 1314 uint32_t mtc, mtc_delta; 1315 1316 if (!decoder->have_tma) 1317 return; 1318 1319 mtc = decoder->packet.payload; 1320 1321 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1322 decoder->fixup_last_mtc = false; 1323 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1324 &decoder->last_mtc); 1325 } 1326 1327 if (mtc > decoder->last_mtc) 1328 mtc_delta = mtc - decoder->last_mtc; 1329 else 1330 mtc_delta = mtc + 256 - decoder->last_mtc; 1331 1332 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1333 1334 if (decoder->tsc_ctc_mult) { 1335 timestamp = decoder->ctc_timestamp + 1336 decoder->ctc_delta * decoder->tsc_ctc_mult; 1337 } else { 1338 timestamp = decoder->ctc_timestamp + 1339 multdiv(decoder->ctc_delta, 1340 decoder->tsc_ctc_ratio_n, 1341 decoder->tsc_ctc_ratio_d); 1342 } 1343 1344 if (timestamp < decoder->timestamp) 1345 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1346 timestamp, decoder->timestamp); 1347 else 1348 decoder->timestamp = timestamp; 1349 1350 decoder->timestamp_insn_cnt = 0; 1351 decoder->last_mtc = mtc; 1352 1353 if (decoder->last_packet_type == INTEL_PT_CYC) { 1354 decoder->cyc_ref_timestamp = decoder->timestamp; 1355 decoder->cycle_cnt = 0; 1356 decoder->have_calc_cyc_to_tsc = false; 1357 intel_pt_calc_cyc_to_tsc(decoder, true); 1358 } 1359 } 1360 1361 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1362 { 1363 unsigned int cbr = decoder->packet.payload; 1364 1365 if (decoder->cbr == cbr) 1366 return; 1367 1368 decoder->cbr = cbr; 1369 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1370 } 1371 1372 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1373 { 1374 uint64_t timestamp = decoder->cyc_ref_timestamp; 1375 1376 decoder->have_cyc = true; 1377 1378 decoder->cycle_cnt += decoder->packet.payload; 1379 1380 if (!decoder->cyc_ref_timestamp) 1381 return; 1382 1383 if (decoder->have_calc_cyc_to_tsc) 1384 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1385 else if (decoder->cbr) 1386 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1387 else 1388 return; 1389 1390 if (timestamp < decoder->timestamp) 1391 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1392 timestamp, decoder->timestamp); 1393 else 1394 decoder->timestamp = timestamp; 1395 1396 decoder->timestamp_insn_cnt = 0; 1397 } 1398 1399 /* Walk PSB+ packets when already in sync. */ 1400 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1401 { 1402 int err; 1403 1404 while (1) { 1405 err = intel_pt_get_next_packet(decoder); 1406 if (err) 1407 return err; 1408 1409 switch (decoder->packet.type) { 1410 case INTEL_PT_PSBEND: 1411 return 0; 1412 1413 case INTEL_PT_TIP_PGD: 1414 case INTEL_PT_TIP_PGE: 1415 case INTEL_PT_TIP: 1416 case INTEL_PT_TNT: 1417 case INTEL_PT_TRACESTOP: 1418 case INTEL_PT_BAD: 1419 case INTEL_PT_PSB: 1420 decoder->have_tma = false; 1421 intel_pt_log("ERROR: Unexpected packet\n"); 1422 return -EAGAIN; 1423 1424 case INTEL_PT_OVF: 1425 return intel_pt_overflow(decoder); 1426 1427 case INTEL_PT_TSC: 1428 intel_pt_calc_tsc_timestamp(decoder); 1429 break; 1430 1431 case INTEL_PT_TMA: 1432 intel_pt_calc_tma(decoder); 1433 break; 1434 1435 case INTEL_PT_CBR: 1436 intel_pt_calc_cbr(decoder); 1437 break; 1438 1439 case INTEL_PT_MODE_EXEC: 1440 decoder->exec_mode = decoder->packet.payload; 1441 break; 1442 1443 case INTEL_PT_PIP: 1444 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1445 break; 1446 1447 case INTEL_PT_FUP: 1448 decoder->pge = true; 1449 intel_pt_set_last_ip(decoder); 1450 break; 1451 1452 case INTEL_PT_MODE_TSX: 1453 intel_pt_update_in_tx(decoder); 1454 break; 1455 1456 case INTEL_PT_MTC: 1457 intel_pt_calc_mtc_timestamp(decoder); 1458 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1459 decoder->state.type |= INTEL_PT_INSTRUCTION; 1460 break; 1461 1462 case INTEL_PT_CYC: 1463 case INTEL_PT_VMCS: 1464 case INTEL_PT_MNT: 1465 case INTEL_PT_PAD: 1466 default: 1467 break; 1468 } 1469 } 1470 } 1471 1472 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1473 { 1474 int err; 1475 1476 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1477 decoder->tx_flags = 0; 1478 decoder->state.flags &= ~INTEL_PT_IN_TX; 1479 decoder->state.flags |= INTEL_PT_ABORT_TX; 1480 } else { 1481 decoder->state.flags |= INTEL_PT_ASYNC; 1482 } 1483 1484 while (1) { 1485 err = intel_pt_get_next_packet(decoder); 1486 if (err) 1487 return err; 1488 1489 switch (decoder->packet.type) { 1490 case INTEL_PT_TNT: 1491 case INTEL_PT_FUP: 1492 case INTEL_PT_TRACESTOP: 1493 case INTEL_PT_PSB: 1494 case INTEL_PT_TSC: 1495 case INTEL_PT_TMA: 1496 case INTEL_PT_CBR: 1497 case INTEL_PT_MODE_TSX: 1498 case INTEL_PT_BAD: 1499 case INTEL_PT_PSBEND: 1500 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1501 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1502 return -ENOENT; 1503 1504 case INTEL_PT_OVF: 1505 return intel_pt_overflow(decoder); 1506 1507 case INTEL_PT_TIP_PGD: 1508 decoder->state.from_ip = decoder->ip; 1509 decoder->state.to_ip = 0; 1510 if (decoder->packet.count != 0) { 1511 intel_pt_set_ip(decoder); 1512 intel_pt_log("Omitting PGD ip " x64_fmt "\n", 1513 decoder->ip); 1514 } 1515 decoder->pge = false; 1516 decoder->continuous_period = false; 1517 return 0; 1518 1519 case INTEL_PT_TIP_PGE: 1520 decoder->pge = true; 1521 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1522 decoder->ip); 1523 decoder->state.from_ip = 0; 1524 if (decoder->packet.count == 0) { 1525 decoder->state.to_ip = 0; 1526 } else { 1527 intel_pt_set_ip(decoder); 1528 decoder->state.to_ip = decoder->ip; 1529 } 1530 return 0; 1531 1532 case INTEL_PT_TIP: 1533 decoder->state.from_ip = decoder->ip; 1534 if (decoder->packet.count == 0) { 1535 decoder->state.to_ip = 0; 1536 } else { 1537 intel_pt_set_ip(decoder); 1538 decoder->state.to_ip = decoder->ip; 1539 } 1540 return 0; 1541 1542 case INTEL_PT_PIP: 1543 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1544 break; 1545 1546 case INTEL_PT_MTC: 1547 intel_pt_calc_mtc_timestamp(decoder); 1548 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1549 decoder->state.type |= INTEL_PT_INSTRUCTION; 1550 break; 1551 1552 case INTEL_PT_CYC: 1553 intel_pt_calc_cyc_timestamp(decoder); 1554 break; 1555 1556 case INTEL_PT_MODE_EXEC: 1557 decoder->exec_mode = decoder->packet.payload; 1558 break; 1559 1560 case INTEL_PT_VMCS: 1561 case INTEL_PT_MNT: 1562 case INTEL_PT_PAD: 1563 break; 1564 1565 default: 1566 return intel_pt_bug(decoder); 1567 } 1568 } 1569 } 1570 1571 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 1572 { 1573 bool no_tip = false; 1574 int err; 1575 1576 while (1) { 1577 err = intel_pt_get_next_packet(decoder); 1578 if (err) 1579 return err; 1580 next: 1581 switch (decoder->packet.type) { 1582 case INTEL_PT_TNT: 1583 if (!decoder->packet.count) 1584 break; 1585 decoder->tnt = decoder->packet; 1586 decoder->pkt_state = INTEL_PT_STATE_TNT; 1587 err = intel_pt_walk_tnt(decoder); 1588 if (err == -EAGAIN) 1589 break; 1590 return err; 1591 1592 case INTEL_PT_TIP_PGD: 1593 if (decoder->packet.count != 0) 1594 intel_pt_set_last_ip(decoder); 1595 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 1596 return intel_pt_walk_tip(decoder); 1597 1598 case INTEL_PT_TIP_PGE: { 1599 decoder->pge = true; 1600 if (decoder->packet.count == 0) { 1601 intel_pt_log_at("Skipping zero TIP.PGE", 1602 decoder->pos); 1603 break; 1604 } 1605 intel_pt_set_ip(decoder); 1606 decoder->state.from_ip = 0; 1607 decoder->state.to_ip = decoder->ip; 1608 return 0; 1609 } 1610 1611 case INTEL_PT_OVF: 1612 return intel_pt_overflow(decoder); 1613 1614 case INTEL_PT_TIP: 1615 if (decoder->packet.count != 0) 1616 intel_pt_set_last_ip(decoder); 1617 decoder->pkt_state = INTEL_PT_STATE_TIP; 1618 return intel_pt_walk_tip(decoder); 1619 1620 case INTEL_PT_FUP: 1621 if (decoder->packet.count == 0) { 1622 intel_pt_log_at("Skipping zero FUP", 1623 decoder->pos); 1624 no_tip = false; 1625 break; 1626 } 1627 intel_pt_set_last_ip(decoder); 1628 err = intel_pt_walk_fup(decoder); 1629 if (err != -EAGAIN) { 1630 if (err) 1631 return err; 1632 if (no_tip) 1633 decoder->pkt_state = 1634 INTEL_PT_STATE_FUP_NO_TIP; 1635 else 1636 decoder->pkt_state = INTEL_PT_STATE_FUP; 1637 return 0; 1638 } 1639 if (no_tip) { 1640 no_tip = false; 1641 break; 1642 } 1643 return intel_pt_walk_fup_tip(decoder); 1644 1645 case INTEL_PT_TRACESTOP: 1646 decoder->pge = false; 1647 decoder->continuous_period = false; 1648 intel_pt_clear_tx_flags(decoder); 1649 decoder->have_tma = false; 1650 break; 1651 1652 case INTEL_PT_PSB: 1653 intel_pt_clear_stack(&decoder->stack); 1654 err = intel_pt_walk_psbend(decoder); 1655 if (err == -EAGAIN) 1656 goto next; 1657 if (err) 1658 return err; 1659 break; 1660 1661 case INTEL_PT_PIP: 1662 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1663 break; 1664 1665 case INTEL_PT_MTC: 1666 intel_pt_calc_mtc_timestamp(decoder); 1667 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 1668 break; 1669 /* 1670 * Ensure that there has been an instruction since the 1671 * last MTC. 1672 */ 1673 if (!decoder->mtc_insn) 1674 break; 1675 decoder->mtc_insn = false; 1676 /* Ensure that there is a timestamp */ 1677 if (!decoder->timestamp) 1678 break; 1679 decoder->state.type = INTEL_PT_INSTRUCTION; 1680 decoder->state.from_ip = decoder->ip; 1681 decoder->state.to_ip = 0; 1682 decoder->mtc_insn = false; 1683 return 0; 1684 1685 case INTEL_PT_TSC: 1686 intel_pt_calc_tsc_timestamp(decoder); 1687 break; 1688 1689 case INTEL_PT_TMA: 1690 intel_pt_calc_tma(decoder); 1691 break; 1692 1693 case INTEL_PT_CYC: 1694 intel_pt_calc_cyc_timestamp(decoder); 1695 break; 1696 1697 case INTEL_PT_CBR: 1698 intel_pt_calc_cbr(decoder); 1699 break; 1700 1701 case INTEL_PT_MODE_EXEC: 1702 decoder->exec_mode = decoder->packet.payload; 1703 break; 1704 1705 case INTEL_PT_MODE_TSX: 1706 /* MODE_TSX need not be followed by FUP */ 1707 if (!decoder->pge) { 1708 intel_pt_update_in_tx(decoder); 1709 break; 1710 } 1711 err = intel_pt_mode_tsx(decoder, &no_tip); 1712 if (err) 1713 return err; 1714 goto next; 1715 1716 case INTEL_PT_BAD: /* Does not happen */ 1717 return intel_pt_bug(decoder); 1718 1719 case INTEL_PT_PSBEND: 1720 case INTEL_PT_VMCS: 1721 case INTEL_PT_MNT: 1722 case INTEL_PT_PAD: 1723 break; 1724 1725 default: 1726 return intel_pt_bug(decoder); 1727 } 1728 } 1729 } 1730 1731 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 1732 { 1733 return decoder->last_ip || decoder->packet.count == 0 || 1734 decoder->packet.count == 3 || decoder->packet.count == 6; 1735 } 1736 1737 /* Walk PSB+ packets to get in sync. */ 1738 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 1739 { 1740 int err; 1741 1742 while (1) { 1743 err = intel_pt_get_next_packet(decoder); 1744 if (err) 1745 return err; 1746 1747 switch (decoder->packet.type) { 1748 case INTEL_PT_TIP_PGD: 1749 decoder->continuous_period = false; 1750 __fallthrough; 1751 case INTEL_PT_TIP_PGE: 1752 case INTEL_PT_TIP: 1753 intel_pt_log("ERROR: Unexpected packet\n"); 1754 return -ENOENT; 1755 1756 case INTEL_PT_FUP: 1757 decoder->pge = true; 1758 if (intel_pt_have_ip(decoder)) { 1759 uint64_t current_ip = decoder->ip; 1760 1761 intel_pt_set_ip(decoder); 1762 if (current_ip) 1763 intel_pt_log_to("Setting IP", 1764 decoder->ip); 1765 } 1766 break; 1767 1768 case INTEL_PT_MTC: 1769 intel_pt_calc_mtc_timestamp(decoder); 1770 break; 1771 1772 case INTEL_PT_TSC: 1773 intel_pt_calc_tsc_timestamp(decoder); 1774 break; 1775 1776 case INTEL_PT_TMA: 1777 intel_pt_calc_tma(decoder); 1778 break; 1779 1780 case INTEL_PT_CYC: 1781 intel_pt_calc_cyc_timestamp(decoder); 1782 break; 1783 1784 case INTEL_PT_CBR: 1785 intel_pt_calc_cbr(decoder); 1786 break; 1787 1788 case INTEL_PT_PIP: 1789 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1790 break; 1791 1792 case INTEL_PT_MODE_EXEC: 1793 decoder->exec_mode = decoder->packet.payload; 1794 break; 1795 1796 case INTEL_PT_MODE_TSX: 1797 intel_pt_update_in_tx(decoder); 1798 break; 1799 1800 case INTEL_PT_TRACESTOP: 1801 decoder->pge = false; 1802 decoder->continuous_period = false; 1803 intel_pt_clear_tx_flags(decoder); 1804 __fallthrough; 1805 1806 case INTEL_PT_TNT: 1807 decoder->have_tma = false; 1808 intel_pt_log("ERROR: Unexpected packet\n"); 1809 if (decoder->ip) 1810 decoder->pkt_state = INTEL_PT_STATE_ERR4; 1811 else 1812 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1813 return -ENOENT; 1814 1815 case INTEL_PT_BAD: /* Does not happen */ 1816 return intel_pt_bug(decoder); 1817 1818 case INTEL_PT_OVF: 1819 return intel_pt_overflow(decoder); 1820 1821 case INTEL_PT_PSBEND: 1822 return 0; 1823 1824 case INTEL_PT_PSB: 1825 case INTEL_PT_VMCS: 1826 case INTEL_PT_MNT: 1827 case INTEL_PT_PAD: 1828 default: 1829 break; 1830 } 1831 } 1832 } 1833 1834 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 1835 { 1836 int err; 1837 1838 while (1) { 1839 err = intel_pt_get_next_packet(decoder); 1840 if (err) 1841 return err; 1842 1843 switch (decoder->packet.type) { 1844 case INTEL_PT_TIP_PGD: 1845 decoder->continuous_period = false; 1846 __fallthrough; 1847 case INTEL_PT_TIP_PGE: 1848 case INTEL_PT_TIP: 1849 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD; 1850 if (intel_pt_have_ip(decoder)) 1851 intel_pt_set_ip(decoder); 1852 if (decoder->ip) 1853 return 0; 1854 break; 1855 1856 case INTEL_PT_FUP: 1857 if (decoder->overflow) { 1858 if (intel_pt_have_ip(decoder)) 1859 intel_pt_set_ip(decoder); 1860 if (decoder->ip) 1861 return 0; 1862 } 1863 if (decoder->packet.count) 1864 intel_pt_set_last_ip(decoder); 1865 break; 1866 1867 case INTEL_PT_MTC: 1868 intel_pt_calc_mtc_timestamp(decoder); 1869 break; 1870 1871 case INTEL_PT_TSC: 1872 intel_pt_calc_tsc_timestamp(decoder); 1873 break; 1874 1875 case INTEL_PT_TMA: 1876 intel_pt_calc_tma(decoder); 1877 break; 1878 1879 case INTEL_PT_CYC: 1880 intel_pt_calc_cyc_timestamp(decoder); 1881 break; 1882 1883 case INTEL_PT_CBR: 1884 intel_pt_calc_cbr(decoder); 1885 break; 1886 1887 case INTEL_PT_PIP: 1888 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1889 break; 1890 1891 case INTEL_PT_MODE_EXEC: 1892 decoder->exec_mode = decoder->packet.payload; 1893 break; 1894 1895 case INTEL_PT_MODE_TSX: 1896 intel_pt_update_in_tx(decoder); 1897 break; 1898 1899 case INTEL_PT_OVF: 1900 return intel_pt_overflow(decoder); 1901 1902 case INTEL_PT_BAD: /* Does not happen */ 1903 return intel_pt_bug(decoder); 1904 1905 case INTEL_PT_TRACESTOP: 1906 decoder->pge = false; 1907 decoder->continuous_period = false; 1908 intel_pt_clear_tx_flags(decoder); 1909 decoder->have_tma = false; 1910 break; 1911 1912 case INTEL_PT_PSB: 1913 err = intel_pt_walk_psb(decoder); 1914 if (err) 1915 return err; 1916 if (decoder->ip) { 1917 /* Do not have a sample */ 1918 decoder->state.type = 0; 1919 return 0; 1920 } 1921 break; 1922 1923 case INTEL_PT_TNT: 1924 case INTEL_PT_PSBEND: 1925 case INTEL_PT_VMCS: 1926 case INTEL_PT_MNT: 1927 case INTEL_PT_PAD: 1928 default: 1929 break; 1930 } 1931 } 1932 } 1933 1934 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 1935 { 1936 int err; 1937 1938 intel_pt_log("Scanning for full IP\n"); 1939 err = intel_pt_walk_to_ip(decoder); 1940 if (err) 1941 return err; 1942 1943 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1944 decoder->overflow = false; 1945 1946 decoder->state.from_ip = 0; 1947 decoder->state.to_ip = decoder->ip; 1948 intel_pt_log_to("Setting IP", decoder->ip); 1949 1950 return 0; 1951 } 1952 1953 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 1954 { 1955 const unsigned char *end = decoder->buf + decoder->len; 1956 size_t i; 1957 1958 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 1959 if (i > decoder->len) 1960 continue; 1961 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 1962 return i; 1963 } 1964 return 0; 1965 } 1966 1967 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 1968 { 1969 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 1970 const char *psb = INTEL_PT_PSB_STR; 1971 1972 if (rest_psb > decoder->len || 1973 memcmp(decoder->buf, psb + part_psb, rest_psb)) 1974 return 0; 1975 1976 return rest_psb; 1977 } 1978 1979 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 1980 int part_psb) 1981 { 1982 int rest_psb, ret; 1983 1984 decoder->pos += decoder->len; 1985 decoder->len = 0; 1986 1987 ret = intel_pt_get_next_data(decoder); 1988 if (ret) 1989 return ret; 1990 1991 rest_psb = intel_pt_rest_psb(decoder, part_psb); 1992 if (!rest_psb) 1993 return 0; 1994 1995 decoder->pos -= part_psb; 1996 decoder->next_buf = decoder->buf + rest_psb; 1997 decoder->next_len = decoder->len - rest_psb; 1998 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 1999 decoder->buf = decoder->temp_buf; 2000 decoder->len = INTEL_PT_PSB_LEN; 2001 2002 return 0; 2003 } 2004 2005 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 2006 { 2007 unsigned char *next; 2008 int ret; 2009 2010 intel_pt_log("Scanning for PSB\n"); 2011 while (1) { 2012 if (!decoder->len) { 2013 ret = intel_pt_get_next_data(decoder); 2014 if (ret) 2015 return ret; 2016 } 2017 2018 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2019 INTEL_PT_PSB_LEN); 2020 if (!next) { 2021 int part_psb; 2022 2023 part_psb = intel_pt_part_psb(decoder); 2024 if (part_psb) { 2025 ret = intel_pt_get_split_psb(decoder, part_psb); 2026 if (ret) 2027 return ret; 2028 } else { 2029 decoder->pos += decoder->len; 2030 decoder->len = 0; 2031 } 2032 continue; 2033 } 2034 2035 decoder->pkt_step = next - decoder->buf; 2036 return intel_pt_get_next_packet(decoder); 2037 } 2038 } 2039 2040 static int intel_pt_sync(struct intel_pt_decoder *decoder) 2041 { 2042 int err; 2043 2044 decoder->pge = false; 2045 decoder->continuous_period = false; 2046 decoder->last_ip = 0; 2047 decoder->ip = 0; 2048 intel_pt_clear_stack(&decoder->stack); 2049 2050 err = intel_pt_scan_for_psb(decoder); 2051 if (err) 2052 return err; 2053 2054 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 2055 2056 err = intel_pt_walk_psb(decoder); 2057 if (err) 2058 return err; 2059 2060 if (decoder->ip) { 2061 decoder->state.type = 0; /* Do not have a sample */ 2062 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2063 } else { 2064 return intel_pt_sync_ip(decoder); 2065 } 2066 2067 return 0; 2068 } 2069 2070 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2071 { 2072 uint64_t est = decoder->timestamp_insn_cnt << 1; 2073 2074 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2075 goto out; 2076 2077 est *= decoder->max_non_turbo_ratio; 2078 est /= decoder->cbr; 2079 out: 2080 return decoder->timestamp + est; 2081 } 2082 2083 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2084 { 2085 int err; 2086 2087 do { 2088 decoder->state.type = INTEL_PT_BRANCH; 2089 decoder->state.flags = 0; 2090 2091 switch (decoder->pkt_state) { 2092 case INTEL_PT_STATE_NO_PSB: 2093 err = intel_pt_sync(decoder); 2094 break; 2095 case INTEL_PT_STATE_NO_IP: 2096 decoder->last_ip = 0; 2097 /* Fall through */ 2098 case INTEL_PT_STATE_ERR_RESYNC: 2099 err = intel_pt_sync_ip(decoder); 2100 break; 2101 case INTEL_PT_STATE_IN_SYNC: 2102 err = intel_pt_walk_trace(decoder); 2103 break; 2104 case INTEL_PT_STATE_TNT: 2105 err = intel_pt_walk_tnt(decoder); 2106 if (err == -EAGAIN) 2107 err = intel_pt_walk_trace(decoder); 2108 break; 2109 case INTEL_PT_STATE_TIP: 2110 case INTEL_PT_STATE_TIP_PGD: 2111 err = intel_pt_walk_tip(decoder); 2112 break; 2113 case INTEL_PT_STATE_FUP: 2114 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2115 err = intel_pt_walk_fup(decoder); 2116 if (err == -EAGAIN) 2117 err = intel_pt_walk_fup_tip(decoder); 2118 else if (!err) 2119 decoder->pkt_state = INTEL_PT_STATE_FUP; 2120 break; 2121 case INTEL_PT_STATE_FUP_NO_TIP: 2122 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2123 err = intel_pt_walk_fup(decoder); 2124 if (err == -EAGAIN) 2125 err = intel_pt_walk_trace(decoder); 2126 break; 2127 default: 2128 err = intel_pt_bug(decoder); 2129 break; 2130 } 2131 } while (err == -ENOLINK); 2132 2133 decoder->state.err = err ? intel_pt_ext_err(err) : 0; 2134 decoder->state.timestamp = decoder->timestamp; 2135 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2136 decoder->state.cr3 = decoder->cr3; 2137 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2138 2139 if (err) 2140 decoder->state.from_ip = decoder->ip; 2141 2142 return &decoder->state; 2143 } 2144 2145 static bool intel_pt_at_psb(unsigned char *buf, size_t len) 2146 { 2147 if (len < INTEL_PT_PSB_LEN) 2148 return false; 2149 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR, 2150 INTEL_PT_PSB_LEN); 2151 } 2152 2153 /** 2154 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2155 * @buf: pointer to buffer pointer 2156 * @len: size of buffer 2157 * 2158 * Updates the buffer pointer to point to the start of the next PSB packet if 2159 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2160 * @len is adjusted accordingly. 2161 * 2162 * Return: %true if a PSB packet is found, %false otherwise. 2163 */ 2164 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2165 { 2166 unsigned char *next; 2167 2168 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2169 if (next) { 2170 *len -= next - *buf; 2171 *buf = next; 2172 return true; 2173 } 2174 return false; 2175 } 2176 2177 /** 2178 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2179 * packet. 2180 * @buf: pointer to buffer pointer 2181 * @len: size of buffer 2182 * 2183 * Updates the buffer pointer to point to the start of the following PSB packet 2184 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2185 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2186 * 2187 * Return: %true if a PSB packet is found, %false otherwise. 2188 */ 2189 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2190 { 2191 unsigned char *next; 2192 2193 if (!*len) 2194 return false; 2195 2196 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2197 if (next) { 2198 *len -= next - *buf; 2199 *buf = next; 2200 return true; 2201 } 2202 return false; 2203 } 2204 2205 /** 2206 * intel_pt_last_psb - find the last PSB packet in a buffer. 2207 * @buf: buffer 2208 * @len: size of buffer 2209 * 2210 * This function finds the last PSB in a buffer. 2211 * 2212 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2213 */ 2214 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2215 { 2216 const char *n = INTEL_PT_PSB_STR; 2217 unsigned char *p; 2218 size_t k; 2219 2220 if (len < INTEL_PT_PSB_LEN) 2221 return NULL; 2222 2223 k = len - INTEL_PT_PSB_LEN + 1; 2224 while (1) { 2225 p = memrchr(buf, n[0], k); 2226 if (!p) 2227 return NULL; 2228 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2229 return p; 2230 k = p - buf; 2231 if (!k) 2232 return NULL; 2233 } 2234 } 2235 2236 /** 2237 * intel_pt_next_tsc - find and return next TSC. 2238 * @buf: buffer 2239 * @len: size of buffer 2240 * @tsc: TSC value returned 2241 * 2242 * Find a TSC packet in @buf and return the TSC value. This function assumes 2243 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2244 * PSBEND packet is found. 2245 * 2246 * Return: %true if TSC is found, false otherwise. 2247 */ 2248 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc) 2249 { 2250 struct intel_pt_pkt packet; 2251 int ret; 2252 2253 while (len) { 2254 ret = intel_pt_get_packet(buf, len, &packet); 2255 if (ret <= 0) 2256 return false; 2257 if (packet.type == INTEL_PT_TSC) { 2258 *tsc = packet.payload; 2259 return true; 2260 } 2261 if (packet.type == INTEL_PT_PSBEND) 2262 return false; 2263 buf += ret; 2264 len -= ret; 2265 } 2266 return false; 2267 } 2268 2269 /** 2270 * intel_pt_tsc_cmp - compare 7-byte TSCs. 2271 * @tsc1: first TSC to compare 2272 * @tsc2: second TSC to compare 2273 * 2274 * This function compares 7-byte TSC values allowing for the possibility that 2275 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 2276 * around so for that purpose this function assumes the absolute difference is 2277 * less than half the maximum difference. 2278 * 2279 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 2280 * after @tsc2. 2281 */ 2282 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 2283 { 2284 const uint64_t halfway = (1ULL << 55); 2285 2286 if (tsc1 == tsc2) 2287 return 0; 2288 2289 if (tsc1 < tsc2) { 2290 if (tsc2 - tsc1 < halfway) 2291 return -1; 2292 else 2293 return 1; 2294 } else { 2295 if (tsc1 - tsc2 < halfway) 2296 return 1; 2297 else 2298 return -1; 2299 } 2300 } 2301 2302 /** 2303 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 2304 * using TSC. 2305 * @buf_a: first buffer 2306 * @len_a: size of first buffer 2307 * @buf_b: second buffer 2308 * @len_b: size of second buffer 2309 * 2310 * If the trace contains TSC we can look at the last TSC of @buf_a and the 2311 * first TSC of @buf_b in order to determine if the buffers overlap, and then 2312 * walk forward in @buf_b until a later TSC is found. A precondition is that 2313 * @buf_a and @buf_b are positioned at a PSB. 2314 * 2315 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2316 * @buf_b + @len_b if there is no non-overlapped data. 2317 */ 2318 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 2319 size_t len_a, 2320 unsigned char *buf_b, 2321 size_t len_b) 2322 { 2323 uint64_t tsc_a, tsc_b; 2324 unsigned char *p; 2325 size_t len; 2326 2327 p = intel_pt_last_psb(buf_a, len_a); 2328 if (!p) 2329 return buf_b; /* No PSB in buf_a => no overlap */ 2330 2331 len = len_a - (p - buf_a); 2332 if (!intel_pt_next_tsc(p, len, &tsc_a)) { 2333 /* The last PSB+ in buf_a is incomplete, so go back one more */ 2334 len_a -= len; 2335 p = intel_pt_last_psb(buf_a, len_a); 2336 if (!p) 2337 return buf_b; /* No full PSB+ => assume no overlap */ 2338 len = len_a - (p - buf_a); 2339 if (!intel_pt_next_tsc(p, len, &tsc_a)) 2340 return buf_b; /* No TSC in buf_a => assume no overlap */ 2341 } 2342 2343 while (1) { 2344 /* Ignore PSB+ with no TSC */ 2345 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) && 2346 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0) 2347 return buf_b; /* tsc_a < tsc_b => no overlap */ 2348 2349 if (!intel_pt_step_psb(&buf_b, &len_b)) 2350 return buf_b + len_b; /* No PSB in buf_b => no data */ 2351 } 2352 } 2353 2354 /** 2355 * intel_pt_find_overlap - determine start of non-overlapped trace data. 2356 * @buf_a: first buffer 2357 * @len_a: size of first buffer 2358 * @buf_b: second buffer 2359 * @len_b: size of second buffer 2360 * @have_tsc: can use TSC packets to detect overlap 2361 * 2362 * When trace samples or snapshots are recorded there is the possibility that 2363 * the data overlaps. Note that, for the purposes of decoding, data is only 2364 * useful if it begins with a PSB packet. 2365 * 2366 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2367 * @buf_b + @len_b if there is no non-overlapped data. 2368 */ 2369 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 2370 unsigned char *buf_b, size_t len_b, 2371 bool have_tsc) 2372 { 2373 unsigned char *found; 2374 2375 /* Buffer 'b' must start at PSB so throw away everything before that */ 2376 if (!intel_pt_next_psb(&buf_b, &len_b)) 2377 return buf_b + len_b; /* No PSB */ 2378 2379 if (!intel_pt_next_psb(&buf_a, &len_a)) 2380 return buf_b; /* No overlap */ 2381 2382 if (have_tsc) { 2383 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b); 2384 if (found) 2385 return found; 2386 } 2387 2388 /* 2389 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 2390 * we can ignore the first part of buffer 'a'. 2391 */ 2392 while (len_b < len_a) { 2393 if (!intel_pt_step_psb(&buf_a, &len_a)) 2394 return buf_b; /* No overlap */ 2395 } 2396 2397 /* Now len_b >= len_a */ 2398 if (len_b > len_a) { 2399 /* The leftover buffer 'b' must start at a PSB */ 2400 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2401 if (!intel_pt_step_psb(&buf_a, &len_a)) 2402 return buf_b; /* No overlap */ 2403 } 2404 } 2405 2406 while (1) { 2407 /* Potential overlap so check the bytes */ 2408 found = memmem(buf_a, len_a, buf_b, len_a); 2409 if (found) 2410 return buf_b + len_a; 2411 2412 /* Try again at next PSB in buffer 'a' */ 2413 if (!intel_pt_step_psb(&buf_a, &len_a)) 2414 return buf_b; /* No overlap */ 2415 2416 /* The leftover buffer 'b' must start at a PSB */ 2417 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2418 if (!intel_pt_step_psb(&buf_a, &len_a)) 2419 return buf_b; /* No overlap */ 2420 } 2421 } 2422 } 2423