1 /* 2 * intel_pt_decoder.c: Intel Processor Trace support 3 * Copyright (c) 2013-2014, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #ifndef _GNU_SOURCE 17 #define _GNU_SOURCE 18 #endif 19 #include <stdlib.h> 20 #include <stdbool.h> 21 #include <string.h> 22 #include <errno.h> 23 #include <stdint.h> 24 #include <inttypes.h> 25 26 #include "../cache.h" 27 #include "../util.h" 28 29 #include "intel-pt-insn-decoder.h" 30 #include "intel-pt-pkt-decoder.h" 31 #include "intel-pt-decoder.h" 32 #include "intel-pt-log.h" 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define INTEL_PT_RETURN 1 39 40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 41 #define INTEL_PT_MAX_LOOPS 10000 42 43 struct intel_pt_blk { 44 struct intel_pt_blk *prev; 45 uint64_t ip[INTEL_PT_BLK_SIZE]; 46 }; 47 48 struct intel_pt_stack { 49 struct intel_pt_blk *blk; 50 struct intel_pt_blk *spare; 51 int pos; 52 }; 53 54 enum intel_pt_pkt_state { 55 INTEL_PT_STATE_NO_PSB, 56 INTEL_PT_STATE_NO_IP, 57 INTEL_PT_STATE_ERR_RESYNC, 58 INTEL_PT_STATE_IN_SYNC, 59 INTEL_PT_STATE_TNT, 60 INTEL_PT_STATE_TIP, 61 INTEL_PT_STATE_TIP_PGD, 62 INTEL_PT_STATE_FUP, 63 INTEL_PT_STATE_FUP_NO_TIP, 64 }; 65 66 #ifdef INTEL_PT_STRICT 67 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 68 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 69 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 70 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 71 #else 72 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 73 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 74 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 75 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 76 #endif 77 78 struct intel_pt_decoder { 79 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 80 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 81 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 82 uint64_t max_insn_cnt, void *data); 83 void *data; 84 struct intel_pt_state state; 85 const unsigned char *buf; 86 size_t len; 87 bool return_compression; 88 bool mtc_insn; 89 bool pge; 90 bool have_tma; 91 bool have_cyc; 92 uint64_t pos; 93 uint64_t last_ip; 94 uint64_t ip; 95 uint64_t cr3; 96 uint64_t timestamp; 97 uint64_t tsc_timestamp; 98 uint64_t ref_timestamp; 99 uint64_t ret_addr; 100 uint64_t ctc_timestamp; 101 uint64_t ctc_delta; 102 uint64_t cycle_cnt; 103 uint64_t cyc_ref_timestamp; 104 uint32_t last_mtc; 105 uint32_t tsc_ctc_ratio_n; 106 uint32_t tsc_ctc_ratio_d; 107 uint32_t tsc_ctc_mult; 108 uint32_t tsc_slip; 109 uint32_t ctc_rem_mask; 110 int mtc_shift; 111 struct intel_pt_stack stack; 112 enum intel_pt_pkt_state pkt_state; 113 struct intel_pt_pkt packet; 114 struct intel_pt_pkt tnt; 115 int pkt_step; 116 int pkt_len; 117 int last_packet_type; 118 unsigned int cbr; 119 unsigned int max_non_turbo_ratio; 120 double max_non_turbo_ratio_fp; 121 double cbr_cyc_to_tsc; 122 double calc_cyc_to_tsc; 123 bool have_calc_cyc_to_tsc; 124 int exec_mode; 125 unsigned int insn_bytes; 126 uint64_t period; 127 enum intel_pt_period_type period_type; 128 uint64_t tot_insn_cnt; 129 uint64_t period_insn_cnt; 130 uint64_t period_mask; 131 uint64_t period_ticks; 132 uint64_t last_masked_timestamp; 133 bool continuous_period; 134 bool overflow; 135 bool set_fup_tx_flags; 136 unsigned int fup_tx_flags; 137 unsigned int tx_flags; 138 uint64_t timestamp_insn_cnt; 139 uint64_t stuck_ip; 140 int no_progress; 141 int stuck_ip_prd; 142 int stuck_ip_cnt; 143 const unsigned char *next_buf; 144 size_t next_len; 145 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 146 }; 147 148 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 149 { 150 int i; 151 152 for (i = 0; x != 1; i++) 153 x >>= 1; 154 155 return x << i; 156 } 157 158 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 159 { 160 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 161 uint64_t period; 162 163 period = intel_pt_lower_power_of_2(decoder->period); 164 decoder->period_mask = ~(period - 1); 165 decoder->period_ticks = period; 166 } 167 } 168 169 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 170 { 171 if (!d) 172 return 0; 173 return (t / d) * n + ((t % d) * n) / d; 174 } 175 176 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 177 { 178 struct intel_pt_decoder *decoder; 179 180 if (!params->get_trace || !params->walk_insn) 181 return NULL; 182 183 decoder = zalloc(sizeof(struct intel_pt_decoder)); 184 if (!decoder) 185 return NULL; 186 187 decoder->get_trace = params->get_trace; 188 decoder->walk_insn = params->walk_insn; 189 decoder->data = params->data; 190 decoder->return_compression = params->return_compression; 191 192 decoder->period = params->period; 193 decoder->period_type = params->period_type; 194 195 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 196 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 197 198 intel_pt_setup_period(decoder); 199 200 decoder->mtc_shift = params->mtc_period; 201 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 202 203 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 204 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 205 206 if (!decoder->tsc_ctc_ratio_n) 207 decoder->tsc_ctc_ratio_d = 0; 208 209 if (decoder->tsc_ctc_ratio_d) { 210 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 211 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 212 decoder->tsc_ctc_ratio_d; 213 214 /* 215 * Allow for timestamps appearing to backwards because a TSC 216 * packet has slipped past a MTC packet, so allow 2 MTC ticks 217 * or ... 218 */ 219 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift, 220 decoder->tsc_ctc_ratio_n, 221 decoder->tsc_ctc_ratio_d); 222 } 223 /* ... or 0x100 paranoia */ 224 if (decoder->tsc_slip < 0x100) 225 decoder->tsc_slip = 0x100; 226 227 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 228 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 229 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 230 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 231 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 232 233 return decoder; 234 } 235 236 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 237 { 238 struct intel_pt_blk *blk = stack->blk; 239 240 stack->blk = blk->prev; 241 if (!stack->spare) 242 stack->spare = blk; 243 else 244 free(blk); 245 } 246 247 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 248 { 249 if (!stack->pos) { 250 if (!stack->blk) 251 return 0; 252 intel_pt_pop_blk(stack); 253 if (!stack->blk) 254 return 0; 255 stack->pos = INTEL_PT_BLK_SIZE; 256 } 257 return stack->blk->ip[--stack->pos]; 258 } 259 260 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 261 { 262 struct intel_pt_blk *blk; 263 264 if (stack->spare) { 265 blk = stack->spare; 266 stack->spare = NULL; 267 } else { 268 blk = malloc(sizeof(struct intel_pt_blk)); 269 if (!blk) 270 return -ENOMEM; 271 } 272 273 blk->prev = stack->blk; 274 stack->blk = blk; 275 stack->pos = 0; 276 return 0; 277 } 278 279 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 280 { 281 int err; 282 283 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 284 err = intel_pt_alloc_blk(stack); 285 if (err) 286 return err; 287 } 288 289 stack->blk->ip[stack->pos++] = ip; 290 return 0; 291 } 292 293 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 294 { 295 while (stack->blk) 296 intel_pt_pop_blk(stack); 297 stack->pos = 0; 298 } 299 300 static void intel_pt_free_stack(struct intel_pt_stack *stack) 301 { 302 intel_pt_clear_stack(stack); 303 zfree(&stack->blk); 304 zfree(&stack->spare); 305 } 306 307 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 308 { 309 intel_pt_free_stack(&decoder->stack); 310 free(decoder); 311 } 312 313 static int intel_pt_ext_err(int code) 314 { 315 switch (code) { 316 case -ENOMEM: 317 return INTEL_PT_ERR_NOMEM; 318 case -ENOSYS: 319 return INTEL_PT_ERR_INTERN; 320 case -EBADMSG: 321 return INTEL_PT_ERR_BADPKT; 322 case -ENODATA: 323 return INTEL_PT_ERR_NODATA; 324 case -EILSEQ: 325 return INTEL_PT_ERR_NOINSN; 326 case -ENOENT: 327 return INTEL_PT_ERR_MISMAT; 328 case -EOVERFLOW: 329 return INTEL_PT_ERR_OVR; 330 case -ENOSPC: 331 return INTEL_PT_ERR_LOST; 332 case -ELOOP: 333 return INTEL_PT_ERR_NELOOP; 334 default: 335 return INTEL_PT_ERR_UNK; 336 } 337 } 338 339 static const char *intel_pt_err_msgs[] = { 340 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 341 [INTEL_PT_ERR_INTERN] = "Internal error", 342 [INTEL_PT_ERR_BADPKT] = "Bad packet", 343 [INTEL_PT_ERR_NODATA] = "No more data", 344 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 345 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 346 [INTEL_PT_ERR_OVR] = "Overflow packet", 347 [INTEL_PT_ERR_LOST] = "Lost trace data", 348 [INTEL_PT_ERR_UNK] = "Unknown error!", 349 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 350 }; 351 352 int intel_pt__strerror(int code, char *buf, size_t buflen) 353 { 354 if (code < 1 || code >= INTEL_PT_ERR_MAX) 355 code = INTEL_PT_ERR_UNK; 356 strlcpy(buf, intel_pt_err_msgs[code], buflen); 357 return 0; 358 } 359 360 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 361 uint64_t last_ip) 362 { 363 uint64_t ip; 364 365 switch (packet->count) { 366 case 1: 367 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 368 packet->payload; 369 break; 370 case 2: 371 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 372 packet->payload; 373 break; 374 case 3: 375 ip = packet->payload; 376 /* Sign-extend 6-byte ip */ 377 if (ip & (uint64_t)0x800000000000ULL) 378 ip |= (uint64_t)0xffff000000000000ULL; 379 break; 380 case 4: 381 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 382 packet->payload; 383 break; 384 case 6: 385 ip = packet->payload; 386 break; 387 default: 388 return 0; 389 } 390 391 return ip; 392 } 393 394 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 395 { 396 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 397 } 398 399 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 400 { 401 intel_pt_set_last_ip(decoder); 402 decoder->ip = decoder->last_ip; 403 } 404 405 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 406 { 407 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 408 decoder->buf); 409 } 410 411 static int intel_pt_bug(struct intel_pt_decoder *decoder) 412 { 413 intel_pt_log("ERROR: Internal error\n"); 414 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 415 return -ENOSYS; 416 } 417 418 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 419 { 420 decoder->tx_flags = 0; 421 } 422 423 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 424 { 425 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 426 } 427 428 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 429 { 430 intel_pt_clear_tx_flags(decoder); 431 decoder->have_tma = false; 432 decoder->pkt_len = 1; 433 decoder->pkt_step = 1; 434 intel_pt_decoder_log_packet(decoder); 435 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 436 intel_pt_log("ERROR: Bad packet\n"); 437 decoder->pkt_state = INTEL_PT_STATE_ERR1; 438 } 439 return -EBADMSG; 440 } 441 442 static int intel_pt_get_data(struct intel_pt_decoder *decoder) 443 { 444 struct intel_pt_buffer buffer = { .buf = 0, }; 445 int ret; 446 447 decoder->pkt_step = 0; 448 449 intel_pt_log("Getting more data\n"); 450 ret = decoder->get_trace(&buffer, decoder->data); 451 if (ret) 452 return ret; 453 decoder->buf = buffer.buf; 454 decoder->len = buffer.len; 455 if (!decoder->len) { 456 intel_pt_log("No more data\n"); 457 return -ENODATA; 458 } 459 if (!buffer.consecutive) { 460 decoder->ip = 0; 461 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 462 decoder->ref_timestamp = buffer.ref_timestamp; 463 decoder->timestamp = 0; 464 decoder->have_tma = false; 465 decoder->state.trace_nr = buffer.trace_nr; 466 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 467 decoder->ref_timestamp); 468 return -ENOLINK; 469 } 470 471 return 0; 472 } 473 474 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder) 475 { 476 if (!decoder->next_buf) 477 return intel_pt_get_data(decoder); 478 479 decoder->buf = decoder->next_buf; 480 decoder->len = decoder->next_len; 481 decoder->next_buf = 0; 482 decoder->next_len = 0; 483 return 0; 484 } 485 486 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 487 { 488 unsigned char *buf = decoder->temp_buf; 489 size_t old_len, len, n; 490 int ret; 491 492 old_len = decoder->len; 493 len = decoder->len; 494 memcpy(buf, decoder->buf, len); 495 496 ret = intel_pt_get_data(decoder); 497 if (ret) { 498 decoder->pos += old_len; 499 return ret < 0 ? ret : -EINVAL; 500 } 501 502 n = INTEL_PT_PKT_MAX_SZ - len; 503 if (n > decoder->len) 504 n = decoder->len; 505 memcpy(buf + len, decoder->buf, n); 506 len += n; 507 508 ret = intel_pt_get_packet(buf, len, &decoder->packet); 509 if (ret < (int)old_len) { 510 decoder->next_buf = decoder->buf; 511 decoder->next_len = decoder->len; 512 decoder->buf = buf; 513 decoder->len = old_len; 514 return intel_pt_bad_packet(decoder); 515 } 516 517 decoder->next_buf = decoder->buf + (ret - old_len); 518 decoder->next_len = decoder->len - (ret - old_len); 519 520 decoder->buf = buf; 521 decoder->len = ret; 522 523 return ret; 524 } 525 526 struct intel_pt_pkt_info { 527 struct intel_pt_decoder *decoder; 528 struct intel_pt_pkt packet; 529 uint64_t pos; 530 int pkt_len; 531 int last_packet_type; 532 void *data; 533 }; 534 535 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 536 537 /* Lookahead packets in current buffer */ 538 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 539 intel_pt_pkt_cb_t cb, void *data) 540 { 541 struct intel_pt_pkt_info pkt_info; 542 const unsigned char *buf = decoder->buf; 543 size_t len = decoder->len; 544 int ret; 545 546 pkt_info.decoder = decoder; 547 pkt_info.pos = decoder->pos; 548 pkt_info.pkt_len = decoder->pkt_step; 549 pkt_info.last_packet_type = decoder->last_packet_type; 550 pkt_info.data = data; 551 552 while (1) { 553 do { 554 pkt_info.pos += pkt_info.pkt_len; 555 buf += pkt_info.pkt_len; 556 len -= pkt_info.pkt_len; 557 558 if (!len) 559 return INTEL_PT_NEED_MORE_BYTES; 560 561 ret = intel_pt_get_packet(buf, len, &pkt_info.packet); 562 if (!ret) 563 return INTEL_PT_NEED_MORE_BYTES; 564 if (ret < 0) 565 return ret; 566 567 pkt_info.pkt_len = ret; 568 } while (pkt_info.packet.type == INTEL_PT_PAD); 569 570 ret = cb(&pkt_info); 571 if (ret) 572 return 0; 573 574 pkt_info.last_packet_type = pkt_info.packet.type; 575 } 576 } 577 578 struct intel_pt_calc_cyc_to_tsc_info { 579 uint64_t cycle_cnt; 580 unsigned int cbr; 581 uint32_t last_mtc; 582 uint64_t ctc_timestamp; 583 uint64_t ctc_delta; 584 uint64_t tsc_timestamp; 585 uint64_t timestamp; 586 bool have_tma; 587 bool from_mtc; 588 double cbr_cyc_to_tsc; 589 }; 590 591 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 592 { 593 struct intel_pt_decoder *decoder = pkt_info->decoder; 594 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 595 uint64_t timestamp; 596 double cyc_to_tsc; 597 unsigned int cbr; 598 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 599 600 switch (pkt_info->packet.type) { 601 case INTEL_PT_TNT: 602 case INTEL_PT_TIP_PGE: 603 case INTEL_PT_TIP: 604 case INTEL_PT_FUP: 605 case INTEL_PT_PSB: 606 case INTEL_PT_PIP: 607 case INTEL_PT_MODE_EXEC: 608 case INTEL_PT_MODE_TSX: 609 case INTEL_PT_PSBEND: 610 case INTEL_PT_PAD: 611 case INTEL_PT_VMCS: 612 case INTEL_PT_MNT: 613 return 0; 614 615 case INTEL_PT_MTC: 616 if (!data->have_tma) 617 return 0; 618 619 mtc = pkt_info->packet.payload; 620 if (mtc > data->last_mtc) 621 mtc_delta = mtc - data->last_mtc; 622 else 623 mtc_delta = mtc + 256 - data->last_mtc; 624 data->ctc_delta += mtc_delta << decoder->mtc_shift; 625 data->last_mtc = mtc; 626 627 if (decoder->tsc_ctc_mult) { 628 timestamp = data->ctc_timestamp + 629 data->ctc_delta * decoder->tsc_ctc_mult; 630 } else { 631 timestamp = data->ctc_timestamp + 632 multdiv(data->ctc_delta, 633 decoder->tsc_ctc_ratio_n, 634 decoder->tsc_ctc_ratio_d); 635 } 636 637 if (timestamp < data->timestamp) 638 return 1; 639 640 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 641 data->timestamp = timestamp; 642 return 0; 643 } 644 645 break; 646 647 case INTEL_PT_TSC: 648 timestamp = pkt_info->packet.payload | 649 (data->timestamp & (0xffULL << 56)); 650 if (data->from_mtc && timestamp < data->timestamp && 651 data->timestamp - timestamp < decoder->tsc_slip) 652 return 1; 653 if (timestamp < data->timestamp) 654 timestamp += (1ULL << 56); 655 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 656 if (data->from_mtc) 657 return 1; 658 data->tsc_timestamp = timestamp; 659 data->timestamp = timestamp; 660 return 0; 661 } 662 break; 663 664 case INTEL_PT_TMA: 665 if (data->from_mtc) 666 return 1; 667 668 if (!decoder->tsc_ctc_ratio_d) 669 return 0; 670 671 ctc = pkt_info->packet.payload; 672 fc = pkt_info->packet.count; 673 ctc_rem = ctc & decoder->ctc_rem_mask; 674 675 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 676 677 data->ctc_timestamp = data->tsc_timestamp - fc; 678 if (decoder->tsc_ctc_mult) { 679 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 680 } else { 681 data->ctc_timestamp -= 682 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 683 decoder->tsc_ctc_ratio_d); 684 } 685 686 data->ctc_delta = 0; 687 data->have_tma = true; 688 689 return 0; 690 691 case INTEL_PT_CYC: 692 data->cycle_cnt += pkt_info->packet.payload; 693 return 0; 694 695 case INTEL_PT_CBR: 696 cbr = pkt_info->packet.payload; 697 if (data->cbr && data->cbr != cbr) 698 return 1; 699 data->cbr = cbr; 700 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 701 return 0; 702 703 case INTEL_PT_TIP_PGD: 704 case INTEL_PT_TRACESTOP: 705 case INTEL_PT_OVF: 706 case INTEL_PT_BAD: /* Does not happen */ 707 default: 708 return 1; 709 } 710 711 if (!data->cbr && decoder->cbr) { 712 data->cbr = decoder->cbr; 713 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 714 } 715 716 if (!data->cycle_cnt) 717 return 1; 718 719 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 720 721 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 722 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 723 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 724 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 725 return 1; 726 } 727 728 decoder->calc_cyc_to_tsc = cyc_to_tsc; 729 decoder->have_calc_cyc_to_tsc = true; 730 731 if (data->cbr) { 732 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 733 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 734 } else { 735 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 736 cyc_to_tsc, pkt_info->pos); 737 } 738 739 return 1; 740 } 741 742 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 743 bool from_mtc) 744 { 745 struct intel_pt_calc_cyc_to_tsc_info data = { 746 .cycle_cnt = 0, 747 .cbr = 0, 748 .last_mtc = decoder->last_mtc, 749 .ctc_timestamp = decoder->ctc_timestamp, 750 .ctc_delta = decoder->ctc_delta, 751 .tsc_timestamp = decoder->tsc_timestamp, 752 .timestamp = decoder->timestamp, 753 .have_tma = decoder->have_tma, 754 .from_mtc = from_mtc, 755 .cbr_cyc_to_tsc = 0, 756 }; 757 758 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 759 } 760 761 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 762 { 763 int ret; 764 765 decoder->last_packet_type = decoder->packet.type; 766 767 do { 768 decoder->pos += decoder->pkt_step; 769 decoder->buf += decoder->pkt_step; 770 decoder->len -= decoder->pkt_step; 771 772 if (!decoder->len) { 773 ret = intel_pt_get_next_data(decoder); 774 if (ret) 775 return ret; 776 } 777 778 ret = intel_pt_get_packet(decoder->buf, decoder->len, 779 &decoder->packet); 780 if (ret == INTEL_PT_NEED_MORE_BYTES && 781 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 782 ret = intel_pt_get_split_packet(decoder); 783 if (ret < 0) 784 return ret; 785 } 786 if (ret <= 0) 787 return intel_pt_bad_packet(decoder); 788 789 decoder->pkt_len = ret; 790 decoder->pkt_step = ret; 791 intel_pt_decoder_log_packet(decoder); 792 } while (decoder->packet.type == INTEL_PT_PAD); 793 794 return 0; 795 } 796 797 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 798 { 799 uint64_t timestamp, masked_timestamp; 800 801 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 802 masked_timestamp = timestamp & decoder->period_mask; 803 if (decoder->continuous_period) { 804 if (masked_timestamp != decoder->last_masked_timestamp) 805 return 1; 806 } else { 807 timestamp += 1; 808 masked_timestamp = timestamp & decoder->period_mask; 809 if (masked_timestamp != decoder->last_masked_timestamp) { 810 decoder->last_masked_timestamp = masked_timestamp; 811 decoder->continuous_period = true; 812 } 813 } 814 return decoder->period_ticks - (timestamp - masked_timestamp); 815 } 816 817 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 818 { 819 switch (decoder->period_type) { 820 case INTEL_PT_PERIOD_INSTRUCTIONS: 821 return decoder->period - decoder->period_insn_cnt; 822 case INTEL_PT_PERIOD_TICKS: 823 return intel_pt_next_period(decoder); 824 case INTEL_PT_PERIOD_NONE: 825 case INTEL_PT_PERIOD_MTC: 826 default: 827 return 0; 828 } 829 } 830 831 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 832 { 833 uint64_t timestamp, masked_timestamp; 834 835 switch (decoder->period_type) { 836 case INTEL_PT_PERIOD_INSTRUCTIONS: 837 decoder->period_insn_cnt = 0; 838 break; 839 case INTEL_PT_PERIOD_TICKS: 840 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 841 masked_timestamp = timestamp & decoder->period_mask; 842 decoder->last_masked_timestamp = masked_timestamp; 843 break; 844 case INTEL_PT_PERIOD_NONE: 845 case INTEL_PT_PERIOD_MTC: 846 default: 847 break; 848 } 849 850 decoder->state.type |= INTEL_PT_INSTRUCTION; 851 } 852 853 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 854 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 855 { 856 uint64_t max_insn_cnt, insn_cnt = 0; 857 int err; 858 859 if (!decoder->mtc_insn) 860 decoder->mtc_insn = true; 861 862 max_insn_cnt = intel_pt_next_sample(decoder); 863 864 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 865 max_insn_cnt, decoder->data); 866 867 decoder->tot_insn_cnt += insn_cnt; 868 decoder->timestamp_insn_cnt += insn_cnt; 869 decoder->period_insn_cnt += insn_cnt; 870 871 if (err) { 872 decoder->no_progress = 0; 873 decoder->pkt_state = INTEL_PT_STATE_ERR2; 874 intel_pt_log_at("ERROR: Failed to get instruction", 875 decoder->ip); 876 if (err == -ENOENT) 877 return -ENOLINK; 878 return -EILSEQ; 879 } 880 881 if (ip && decoder->ip == ip) { 882 err = -EAGAIN; 883 goto out; 884 } 885 886 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 887 intel_pt_sample_insn(decoder); 888 889 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 890 decoder->state.type = INTEL_PT_INSTRUCTION; 891 decoder->state.from_ip = decoder->ip; 892 decoder->state.to_ip = 0; 893 decoder->ip += intel_pt_insn->length; 894 err = INTEL_PT_RETURN; 895 goto out; 896 } 897 898 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 899 /* Zero-length calls are excluded */ 900 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 901 intel_pt_insn->rel) { 902 err = intel_pt_push(&decoder->stack, decoder->ip + 903 intel_pt_insn->length); 904 if (err) 905 goto out; 906 } 907 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 908 decoder->ret_addr = intel_pt_pop(&decoder->stack); 909 } 910 911 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 912 int cnt = decoder->no_progress++; 913 914 decoder->state.from_ip = decoder->ip; 915 decoder->ip += intel_pt_insn->length + 916 intel_pt_insn->rel; 917 decoder->state.to_ip = decoder->ip; 918 err = INTEL_PT_RETURN; 919 920 /* 921 * Check for being stuck in a loop. This can happen if a 922 * decoder error results in the decoder erroneously setting the 923 * ip to an address that is itself in an infinite loop that 924 * consumes no packets. When that happens, there must be an 925 * unconditional branch. 926 */ 927 if (cnt) { 928 if (cnt == 1) { 929 decoder->stuck_ip = decoder->state.to_ip; 930 decoder->stuck_ip_prd = 1; 931 decoder->stuck_ip_cnt = 1; 932 } else if (cnt > INTEL_PT_MAX_LOOPS || 933 decoder->state.to_ip == decoder->stuck_ip) { 934 intel_pt_log_at("ERROR: Never-ending loop", 935 decoder->state.to_ip); 936 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 937 err = -ELOOP; 938 goto out; 939 } else if (!--decoder->stuck_ip_cnt) { 940 decoder->stuck_ip_prd += 1; 941 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 942 decoder->stuck_ip = decoder->state.to_ip; 943 } 944 } 945 goto out_no_progress; 946 } 947 out: 948 decoder->no_progress = 0; 949 out_no_progress: 950 decoder->state.insn_op = intel_pt_insn->op; 951 decoder->state.insn_len = intel_pt_insn->length; 952 953 if (decoder->tx_flags & INTEL_PT_IN_TX) 954 decoder->state.flags |= INTEL_PT_IN_TX; 955 956 return err; 957 } 958 959 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 960 { 961 struct intel_pt_insn intel_pt_insn; 962 uint64_t ip; 963 int err; 964 965 ip = decoder->last_ip; 966 967 while (1) { 968 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 969 if (err == INTEL_PT_RETURN) 970 return 0; 971 if (err == -EAGAIN) { 972 if (decoder->set_fup_tx_flags) { 973 decoder->set_fup_tx_flags = false; 974 decoder->tx_flags = decoder->fup_tx_flags; 975 decoder->state.type = INTEL_PT_TRANSACTION; 976 decoder->state.from_ip = decoder->ip; 977 decoder->state.to_ip = 0; 978 decoder->state.flags = decoder->fup_tx_flags; 979 return 0; 980 } 981 return err; 982 } 983 decoder->set_fup_tx_flags = false; 984 if (err) 985 return err; 986 987 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 988 intel_pt_log_at("ERROR: Unexpected indirect branch", 989 decoder->ip); 990 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 991 return -ENOENT; 992 } 993 994 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 995 intel_pt_log_at("ERROR: Unexpected conditional branch", 996 decoder->ip); 997 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 998 return -ENOENT; 999 } 1000 1001 intel_pt_bug(decoder); 1002 } 1003 } 1004 1005 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1006 { 1007 struct intel_pt_insn intel_pt_insn; 1008 int err; 1009 1010 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1011 if (err == INTEL_PT_RETURN) 1012 return 0; 1013 if (err) 1014 return err; 1015 1016 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1017 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1018 decoder->pge = false; 1019 decoder->continuous_period = false; 1020 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1021 decoder->state.from_ip = decoder->ip; 1022 decoder->state.to_ip = 0; 1023 if (decoder->packet.count != 0) 1024 decoder->ip = decoder->last_ip; 1025 } else { 1026 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1027 decoder->state.from_ip = decoder->ip; 1028 if (decoder->packet.count == 0) { 1029 decoder->state.to_ip = 0; 1030 } else { 1031 decoder->state.to_ip = decoder->last_ip; 1032 decoder->ip = decoder->last_ip; 1033 } 1034 } 1035 return 0; 1036 } 1037 1038 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1039 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1040 decoder->ip); 1041 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1042 return -ENOENT; 1043 } 1044 1045 return intel_pt_bug(decoder); 1046 } 1047 1048 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1049 { 1050 struct intel_pt_insn intel_pt_insn; 1051 int err; 1052 1053 while (1) { 1054 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1055 if (err == INTEL_PT_RETURN) 1056 return 0; 1057 if (err) 1058 return err; 1059 1060 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1061 if (!decoder->return_compression) { 1062 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1063 decoder->ip); 1064 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1065 return -ENOENT; 1066 } 1067 if (!decoder->ret_addr) { 1068 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1069 decoder->ip); 1070 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1071 return -ENOENT; 1072 } 1073 if (!(decoder->tnt.payload & BIT63)) { 1074 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1075 decoder->ip); 1076 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1077 return -ENOENT; 1078 } 1079 decoder->tnt.count -= 1; 1080 if (!decoder->tnt.count) 1081 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1082 decoder->tnt.payload <<= 1; 1083 decoder->state.from_ip = decoder->ip; 1084 decoder->ip = decoder->ret_addr; 1085 decoder->state.to_ip = decoder->ip; 1086 return 0; 1087 } 1088 1089 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1090 /* Handle deferred TIPs */ 1091 err = intel_pt_get_next_packet(decoder); 1092 if (err) 1093 return err; 1094 if (decoder->packet.type != INTEL_PT_TIP || 1095 decoder->packet.count == 0) { 1096 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1097 decoder->ip); 1098 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1099 decoder->pkt_step = 0; 1100 return -ENOENT; 1101 } 1102 intel_pt_set_last_ip(decoder); 1103 decoder->state.from_ip = decoder->ip; 1104 decoder->state.to_ip = decoder->last_ip; 1105 decoder->ip = decoder->last_ip; 1106 return 0; 1107 } 1108 1109 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1110 decoder->tnt.count -= 1; 1111 if (!decoder->tnt.count) 1112 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1113 if (decoder->tnt.payload & BIT63) { 1114 decoder->tnt.payload <<= 1; 1115 decoder->state.from_ip = decoder->ip; 1116 decoder->ip += intel_pt_insn.length + 1117 intel_pt_insn.rel; 1118 decoder->state.to_ip = decoder->ip; 1119 return 0; 1120 } 1121 /* Instruction sample for a non-taken branch */ 1122 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1123 decoder->tnt.payload <<= 1; 1124 decoder->state.type = INTEL_PT_INSTRUCTION; 1125 decoder->state.from_ip = decoder->ip; 1126 decoder->state.to_ip = 0; 1127 decoder->ip += intel_pt_insn.length; 1128 return 0; 1129 } 1130 decoder->ip += intel_pt_insn.length; 1131 if (!decoder->tnt.count) 1132 return -EAGAIN; 1133 decoder->tnt.payload <<= 1; 1134 continue; 1135 } 1136 1137 return intel_pt_bug(decoder); 1138 } 1139 } 1140 1141 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1142 { 1143 unsigned int fup_tx_flags; 1144 int err; 1145 1146 fup_tx_flags = decoder->packet.payload & 1147 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1148 err = intel_pt_get_next_packet(decoder); 1149 if (err) 1150 return err; 1151 if (decoder->packet.type == INTEL_PT_FUP) { 1152 decoder->fup_tx_flags = fup_tx_flags; 1153 decoder->set_fup_tx_flags = true; 1154 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1155 *no_tip = true; 1156 } else { 1157 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1158 decoder->pos); 1159 intel_pt_update_in_tx(decoder); 1160 } 1161 return 0; 1162 } 1163 1164 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1165 { 1166 uint64_t timestamp; 1167 1168 decoder->have_tma = false; 1169 1170 if (decoder->ref_timestamp) { 1171 timestamp = decoder->packet.payload | 1172 (decoder->ref_timestamp & (0xffULL << 56)); 1173 if (timestamp < decoder->ref_timestamp) { 1174 if (decoder->ref_timestamp - timestamp > (1ULL << 55)) 1175 timestamp += (1ULL << 56); 1176 } else { 1177 if (timestamp - decoder->ref_timestamp > (1ULL << 55)) 1178 timestamp -= (1ULL << 56); 1179 } 1180 decoder->tsc_timestamp = timestamp; 1181 decoder->timestamp = timestamp; 1182 decoder->ref_timestamp = 0; 1183 decoder->timestamp_insn_cnt = 0; 1184 } else if (decoder->timestamp) { 1185 timestamp = decoder->packet.payload | 1186 (decoder->timestamp & (0xffULL << 56)); 1187 decoder->tsc_timestamp = timestamp; 1188 if (timestamp < decoder->timestamp && 1189 decoder->timestamp - timestamp < decoder->tsc_slip) { 1190 intel_pt_log_to("Suppressing backwards timestamp", 1191 timestamp); 1192 timestamp = decoder->timestamp; 1193 } 1194 if (timestamp < decoder->timestamp) { 1195 intel_pt_log_to("Wraparound timestamp", timestamp); 1196 timestamp += (1ULL << 56); 1197 decoder->tsc_timestamp = timestamp; 1198 } 1199 decoder->timestamp = timestamp; 1200 decoder->timestamp_insn_cnt = 0; 1201 } 1202 1203 if (decoder->last_packet_type == INTEL_PT_CYC) { 1204 decoder->cyc_ref_timestamp = decoder->timestamp; 1205 decoder->cycle_cnt = 0; 1206 decoder->have_calc_cyc_to_tsc = false; 1207 intel_pt_calc_cyc_to_tsc(decoder, false); 1208 } 1209 1210 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1211 } 1212 1213 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1214 { 1215 intel_pt_log("ERROR: Buffer overflow\n"); 1216 intel_pt_clear_tx_flags(decoder); 1217 decoder->have_tma = false; 1218 decoder->cbr = 0; 1219 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1220 decoder->overflow = true; 1221 return -EOVERFLOW; 1222 } 1223 1224 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1225 { 1226 uint32_t ctc = decoder->packet.payload; 1227 uint32_t fc = decoder->packet.count; 1228 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1229 1230 if (!decoder->tsc_ctc_ratio_d) 1231 return; 1232 1233 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1234 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1235 if (decoder->tsc_ctc_mult) { 1236 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1237 } else { 1238 decoder->ctc_timestamp -= multdiv(ctc_rem, 1239 decoder->tsc_ctc_ratio_n, 1240 decoder->tsc_ctc_ratio_d); 1241 } 1242 decoder->ctc_delta = 0; 1243 decoder->have_tma = true; 1244 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1245 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1246 } 1247 1248 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1249 { 1250 uint64_t timestamp; 1251 uint32_t mtc, mtc_delta; 1252 1253 if (!decoder->have_tma) 1254 return; 1255 1256 mtc = decoder->packet.payload; 1257 1258 if (mtc > decoder->last_mtc) 1259 mtc_delta = mtc - decoder->last_mtc; 1260 else 1261 mtc_delta = mtc + 256 - decoder->last_mtc; 1262 1263 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1264 1265 if (decoder->tsc_ctc_mult) { 1266 timestamp = decoder->ctc_timestamp + 1267 decoder->ctc_delta * decoder->tsc_ctc_mult; 1268 } else { 1269 timestamp = decoder->ctc_timestamp + 1270 multdiv(decoder->ctc_delta, 1271 decoder->tsc_ctc_ratio_n, 1272 decoder->tsc_ctc_ratio_d); 1273 } 1274 1275 if (timestamp < decoder->timestamp) 1276 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1277 timestamp, decoder->timestamp); 1278 else 1279 decoder->timestamp = timestamp; 1280 1281 decoder->timestamp_insn_cnt = 0; 1282 decoder->last_mtc = mtc; 1283 1284 if (decoder->last_packet_type == INTEL_PT_CYC) { 1285 decoder->cyc_ref_timestamp = decoder->timestamp; 1286 decoder->cycle_cnt = 0; 1287 decoder->have_calc_cyc_to_tsc = false; 1288 intel_pt_calc_cyc_to_tsc(decoder, true); 1289 } 1290 } 1291 1292 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1293 { 1294 unsigned int cbr = decoder->packet.payload; 1295 1296 if (decoder->cbr == cbr) 1297 return; 1298 1299 decoder->cbr = cbr; 1300 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1301 } 1302 1303 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1304 { 1305 uint64_t timestamp = decoder->cyc_ref_timestamp; 1306 1307 decoder->have_cyc = true; 1308 1309 decoder->cycle_cnt += decoder->packet.payload; 1310 1311 if (!decoder->cyc_ref_timestamp) 1312 return; 1313 1314 if (decoder->have_calc_cyc_to_tsc) 1315 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1316 else if (decoder->cbr) 1317 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1318 else 1319 return; 1320 1321 if (timestamp < decoder->timestamp) 1322 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1323 timestamp, decoder->timestamp); 1324 else 1325 decoder->timestamp = timestamp; 1326 } 1327 1328 /* Walk PSB+ packets when already in sync. */ 1329 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1330 { 1331 int err; 1332 1333 while (1) { 1334 err = intel_pt_get_next_packet(decoder); 1335 if (err) 1336 return err; 1337 1338 switch (decoder->packet.type) { 1339 case INTEL_PT_PSBEND: 1340 return 0; 1341 1342 case INTEL_PT_TIP_PGD: 1343 case INTEL_PT_TIP_PGE: 1344 case INTEL_PT_TIP: 1345 case INTEL_PT_TNT: 1346 case INTEL_PT_TRACESTOP: 1347 case INTEL_PT_BAD: 1348 case INTEL_PT_PSB: 1349 decoder->have_tma = false; 1350 intel_pt_log("ERROR: Unexpected packet\n"); 1351 return -EAGAIN; 1352 1353 case INTEL_PT_OVF: 1354 return intel_pt_overflow(decoder); 1355 1356 case INTEL_PT_TSC: 1357 intel_pt_calc_tsc_timestamp(decoder); 1358 break; 1359 1360 case INTEL_PT_TMA: 1361 intel_pt_calc_tma(decoder); 1362 break; 1363 1364 case INTEL_PT_CBR: 1365 intel_pt_calc_cbr(decoder); 1366 break; 1367 1368 case INTEL_PT_MODE_EXEC: 1369 decoder->exec_mode = decoder->packet.payload; 1370 break; 1371 1372 case INTEL_PT_PIP: 1373 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1374 break; 1375 1376 case INTEL_PT_FUP: 1377 decoder->pge = true; 1378 intel_pt_set_last_ip(decoder); 1379 break; 1380 1381 case INTEL_PT_MODE_TSX: 1382 intel_pt_update_in_tx(decoder); 1383 break; 1384 1385 case INTEL_PT_MTC: 1386 intel_pt_calc_mtc_timestamp(decoder); 1387 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1388 decoder->state.type |= INTEL_PT_INSTRUCTION; 1389 break; 1390 1391 case INTEL_PT_CYC: 1392 case INTEL_PT_VMCS: 1393 case INTEL_PT_MNT: 1394 case INTEL_PT_PAD: 1395 default: 1396 break; 1397 } 1398 } 1399 } 1400 1401 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1402 { 1403 int err; 1404 1405 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1406 decoder->tx_flags = 0; 1407 decoder->state.flags &= ~INTEL_PT_IN_TX; 1408 decoder->state.flags |= INTEL_PT_ABORT_TX; 1409 } else { 1410 decoder->state.flags |= INTEL_PT_ASYNC; 1411 } 1412 1413 while (1) { 1414 err = intel_pt_get_next_packet(decoder); 1415 if (err) 1416 return err; 1417 1418 switch (decoder->packet.type) { 1419 case INTEL_PT_TNT: 1420 case INTEL_PT_FUP: 1421 case INTEL_PT_TRACESTOP: 1422 case INTEL_PT_PSB: 1423 case INTEL_PT_TSC: 1424 case INTEL_PT_TMA: 1425 case INTEL_PT_CBR: 1426 case INTEL_PT_MODE_TSX: 1427 case INTEL_PT_BAD: 1428 case INTEL_PT_PSBEND: 1429 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1430 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1431 return -ENOENT; 1432 1433 case INTEL_PT_OVF: 1434 return intel_pt_overflow(decoder); 1435 1436 case INTEL_PT_TIP_PGD: 1437 decoder->state.from_ip = decoder->ip; 1438 decoder->state.to_ip = 0; 1439 if (decoder->packet.count != 0) { 1440 intel_pt_set_ip(decoder); 1441 intel_pt_log("Omitting PGD ip " x64_fmt "\n", 1442 decoder->ip); 1443 } 1444 decoder->pge = false; 1445 decoder->continuous_period = false; 1446 return 0; 1447 1448 case INTEL_PT_TIP_PGE: 1449 decoder->pge = true; 1450 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1451 decoder->ip); 1452 decoder->state.from_ip = 0; 1453 if (decoder->packet.count == 0) { 1454 decoder->state.to_ip = 0; 1455 } else { 1456 intel_pt_set_ip(decoder); 1457 decoder->state.to_ip = decoder->ip; 1458 } 1459 return 0; 1460 1461 case INTEL_PT_TIP: 1462 decoder->state.from_ip = decoder->ip; 1463 if (decoder->packet.count == 0) { 1464 decoder->state.to_ip = 0; 1465 } else { 1466 intel_pt_set_ip(decoder); 1467 decoder->state.to_ip = decoder->ip; 1468 } 1469 return 0; 1470 1471 case INTEL_PT_PIP: 1472 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1473 break; 1474 1475 case INTEL_PT_MTC: 1476 intel_pt_calc_mtc_timestamp(decoder); 1477 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1478 decoder->state.type |= INTEL_PT_INSTRUCTION; 1479 break; 1480 1481 case INTEL_PT_CYC: 1482 intel_pt_calc_cyc_timestamp(decoder); 1483 break; 1484 1485 case INTEL_PT_MODE_EXEC: 1486 decoder->exec_mode = decoder->packet.payload; 1487 break; 1488 1489 case INTEL_PT_VMCS: 1490 case INTEL_PT_MNT: 1491 case INTEL_PT_PAD: 1492 break; 1493 1494 default: 1495 return intel_pt_bug(decoder); 1496 } 1497 } 1498 } 1499 1500 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 1501 { 1502 bool no_tip = false; 1503 int err; 1504 1505 while (1) { 1506 err = intel_pt_get_next_packet(decoder); 1507 if (err) 1508 return err; 1509 next: 1510 switch (decoder->packet.type) { 1511 case INTEL_PT_TNT: 1512 if (!decoder->packet.count) 1513 break; 1514 decoder->tnt = decoder->packet; 1515 decoder->pkt_state = INTEL_PT_STATE_TNT; 1516 err = intel_pt_walk_tnt(decoder); 1517 if (err == -EAGAIN) 1518 break; 1519 return err; 1520 1521 case INTEL_PT_TIP_PGD: 1522 if (decoder->packet.count != 0) 1523 intel_pt_set_last_ip(decoder); 1524 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 1525 return intel_pt_walk_tip(decoder); 1526 1527 case INTEL_PT_TIP_PGE: { 1528 decoder->pge = true; 1529 if (decoder->packet.count == 0) { 1530 intel_pt_log_at("Skipping zero TIP.PGE", 1531 decoder->pos); 1532 break; 1533 } 1534 intel_pt_set_ip(decoder); 1535 decoder->state.from_ip = 0; 1536 decoder->state.to_ip = decoder->ip; 1537 return 0; 1538 } 1539 1540 case INTEL_PT_OVF: 1541 return intel_pt_overflow(decoder); 1542 1543 case INTEL_PT_TIP: 1544 if (decoder->packet.count != 0) 1545 intel_pt_set_last_ip(decoder); 1546 decoder->pkt_state = INTEL_PT_STATE_TIP; 1547 return intel_pt_walk_tip(decoder); 1548 1549 case INTEL_PT_FUP: 1550 if (decoder->packet.count == 0) { 1551 intel_pt_log_at("Skipping zero FUP", 1552 decoder->pos); 1553 no_tip = false; 1554 break; 1555 } 1556 intel_pt_set_last_ip(decoder); 1557 err = intel_pt_walk_fup(decoder); 1558 if (err != -EAGAIN) { 1559 if (err) 1560 return err; 1561 if (no_tip) 1562 decoder->pkt_state = 1563 INTEL_PT_STATE_FUP_NO_TIP; 1564 else 1565 decoder->pkt_state = INTEL_PT_STATE_FUP; 1566 return 0; 1567 } 1568 if (no_tip) { 1569 no_tip = false; 1570 break; 1571 } 1572 return intel_pt_walk_fup_tip(decoder); 1573 1574 case INTEL_PT_TRACESTOP: 1575 decoder->pge = false; 1576 decoder->continuous_period = false; 1577 intel_pt_clear_tx_flags(decoder); 1578 decoder->have_tma = false; 1579 break; 1580 1581 case INTEL_PT_PSB: 1582 intel_pt_clear_stack(&decoder->stack); 1583 err = intel_pt_walk_psbend(decoder); 1584 if (err == -EAGAIN) 1585 goto next; 1586 if (err) 1587 return err; 1588 break; 1589 1590 case INTEL_PT_PIP: 1591 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1592 break; 1593 1594 case INTEL_PT_MTC: 1595 intel_pt_calc_mtc_timestamp(decoder); 1596 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 1597 break; 1598 /* 1599 * Ensure that there has been an instruction since the 1600 * last MTC. 1601 */ 1602 if (!decoder->mtc_insn) 1603 break; 1604 decoder->mtc_insn = false; 1605 /* Ensure that there is a timestamp */ 1606 if (!decoder->timestamp) 1607 break; 1608 decoder->state.type = INTEL_PT_INSTRUCTION; 1609 decoder->state.from_ip = decoder->ip; 1610 decoder->state.to_ip = 0; 1611 decoder->mtc_insn = false; 1612 return 0; 1613 1614 case INTEL_PT_TSC: 1615 intel_pt_calc_tsc_timestamp(decoder); 1616 break; 1617 1618 case INTEL_PT_TMA: 1619 intel_pt_calc_tma(decoder); 1620 break; 1621 1622 case INTEL_PT_CYC: 1623 intel_pt_calc_cyc_timestamp(decoder); 1624 break; 1625 1626 case INTEL_PT_CBR: 1627 intel_pt_calc_cbr(decoder); 1628 break; 1629 1630 case INTEL_PT_MODE_EXEC: 1631 decoder->exec_mode = decoder->packet.payload; 1632 break; 1633 1634 case INTEL_PT_MODE_TSX: 1635 /* MODE_TSX need not be followed by FUP */ 1636 if (!decoder->pge) { 1637 intel_pt_update_in_tx(decoder); 1638 break; 1639 } 1640 err = intel_pt_mode_tsx(decoder, &no_tip); 1641 if (err) 1642 return err; 1643 goto next; 1644 1645 case INTEL_PT_BAD: /* Does not happen */ 1646 return intel_pt_bug(decoder); 1647 1648 case INTEL_PT_PSBEND: 1649 case INTEL_PT_VMCS: 1650 case INTEL_PT_MNT: 1651 case INTEL_PT_PAD: 1652 break; 1653 1654 default: 1655 return intel_pt_bug(decoder); 1656 } 1657 } 1658 } 1659 1660 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 1661 { 1662 return decoder->last_ip || decoder->packet.count == 0 || 1663 decoder->packet.count == 3 || decoder->packet.count == 6; 1664 } 1665 1666 /* Walk PSB+ packets to get in sync. */ 1667 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 1668 { 1669 int err; 1670 1671 while (1) { 1672 err = intel_pt_get_next_packet(decoder); 1673 if (err) 1674 return err; 1675 1676 switch (decoder->packet.type) { 1677 case INTEL_PT_TIP_PGD: 1678 decoder->continuous_period = false; 1679 case INTEL_PT_TIP_PGE: 1680 case INTEL_PT_TIP: 1681 intel_pt_log("ERROR: Unexpected packet\n"); 1682 return -ENOENT; 1683 1684 case INTEL_PT_FUP: 1685 decoder->pge = true; 1686 if (intel_pt_have_ip(decoder)) { 1687 uint64_t current_ip = decoder->ip; 1688 1689 intel_pt_set_ip(decoder); 1690 if (current_ip) 1691 intel_pt_log_to("Setting IP", 1692 decoder->ip); 1693 } 1694 break; 1695 1696 case INTEL_PT_MTC: 1697 intel_pt_calc_mtc_timestamp(decoder); 1698 break; 1699 1700 case INTEL_PT_TSC: 1701 intel_pt_calc_tsc_timestamp(decoder); 1702 break; 1703 1704 case INTEL_PT_TMA: 1705 intel_pt_calc_tma(decoder); 1706 break; 1707 1708 case INTEL_PT_CYC: 1709 intel_pt_calc_cyc_timestamp(decoder); 1710 break; 1711 1712 case INTEL_PT_CBR: 1713 intel_pt_calc_cbr(decoder); 1714 break; 1715 1716 case INTEL_PT_PIP: 1717 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1718 break; 1719 1720 case INTEL_PT_MODE_EXEC: 1721 decoder->exec_mode = decoder->packet.payload; 1722 break; 1723 1724 case INTEL_PT_MODE_TSX: 1725 intel_pt_update_in_tx(decoder); 1726 break; 1727 1728 case INTEL_PT_TRACESTOP: 1729 decoder->pge = false; 1730 decoder->continuous_period = false; 1731 intel_pt_clear_tx_flags(decoder); 1732 case INTEL_PT_TNT: 1733 decoder->have_tma = false; 1734 intel_pt_log("ERROR: Unexpected packet\n"); 1735 if (decoder->ip) 1736 decoder->pkt_state = INTEL_PT_STATE_ERR4; 1737 else 1738 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1739 return -ENOENT; 1740 1741 case INTEL_PT_BAD: /* Does not happen */ 1742 return intel_pt_bug(decoder); 1743 1744 case INTEL_PT_OVF: 1745 return intel_pt_overflow(decoder); 1746 1747 case INTEL_PT_PSBEND: 1748 return 0; 1749 1750 case INTEL_PT_PSB: 1751 case INTEL_PT_VMCS: 1752 case INTEL_PT_MNT: 1753 case INTEL_PT_PAD: 1754 default: 1755 break; 1756 } 1757 } 1758 } 1759 1760 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 1761 { 1762 int err; 1763 1764 while (1) { 1765 err = intel_pt_get_next_packet(decoder); 1766 if (err) 1767 return err; 1768 1769 switch (decoder->packet.type) { 1770 case INTEL_PT_TIP_PGD: 1771 decoder->continuous_period = false; 1772 case INTEL_PT_TIP_PGE: 1773 case INTEL_PT_TIP: 1774 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD; 1775 if (intel_pt_have_ip(decoder)) 1776 intel_pt_set_ip(decoder); 1777 if (decoder->ip) 1778 return 0; 1779 break; 1780 1781 case INTEL_PT_FUP: 1782 if (decoder->overflow) { 1783 if (intel_pt_have_ip(decoder)) 1784 intel_pt_set_ip(decoder); 1785 if (decoder->ip) 1786 return 0; 1787 } 1788 if (decoder->packet.count) 1789 intel_pt_set_last_ip(decoder); 1790 break; 1791 1792 case INTEL_PT_MTC: 1793 intel_pt_calc_mtc_timestamp(decoder); 1794 break; 1795 1796 case INTEL_PT_TSC: 1797 intel_pt_calc_tsc_timestamp(decoder); 1798 break; 1799 1800 case INTEL_PT_TMA: 1801 intel_pt_calc_tma(decoder); 1802 break; 1803 1804 case INTEL_PT_CYC: 1805 intel_pt_calc_cyc_timestamp(decoder); 1806 break; 1807 1808 case INTEL_PT_CBR: 1809 intel_pt_calc_cbr(decoder); 1810 break; 1811 1812 case INTEL_PT_PIP: 1813 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1814 break; 1815 1816 case INTEL_PT_MODE_EXEC: 1817 decoder->exec_mode = decoder->packet.payload; 1818 break; 1819 1820 case INTEL_PT_MODE_TSX: 1821 intel_pt_update_in_tx(decoder); 1822 break; 1823 1824 case INTEL_PT_OVF: 1825 return intel_pt_overflow(decoder); 1826 1827 case INTEL_PT_BAD: /* Does not happen */ 1828 return intel_pt_bug(decoder); 1829 1830 case INTEL_PT_TRACESTOP: 1831 decoder->pge = false; 1832 decoder->continuous_period = false; 1833 intel_pt_clear_tx_flags(decoder); 1834 decoder->have_tma = false; 1835 break; 1836 1837 case INTEL_PT_PSB: 1838 err = intel_pt_walk_psb(decoder); 1839 if (err) 1840 return err; 1841 if (decoder->ip) { 1842 /* Do not have a sample */ 1843 decoder->state.type = 0; 1844 return 0; 1845 } 1846 break; 1847 1848 case INTEL_PT_TNT: 1849 case INTEL_PT_PSBEND: 1850 case INTEL_PT_VMCS: 1851 case INTEL_PT_MNT: 1852 case INTEL_PT_PAD: 1853 default: 1854 break; 1855 } 1856 } 1857 } 1858 1859 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 1860 { 1861 int err; 1862 1863 intel_pt_log("Scanning for full IP\n"); 1864 err = intel_pt_walk_to_ip(decoder); 1865 if (err) 1866 return err; 1867 1868 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1869 decoder->overflow = false; 1870 1871 decoder->state.from_ip = 0; 1872 decoder->state.to_ip = decoder->ip; 1873 intel_pt_log_to("Setting IP", decoder->ip); 1874 1875 return 0; 1876 } 1877 1878 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 1879 { 1880 const unsigned char *end = decoder->buf + decoder->len; 1881 size_t i; 1882 1883 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 1884 if (i > decoder->len) 1885 continue; 1886 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 1887 return i; 1888 } 1889 return 0; 1890 } 1891 1892 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 1893 { 1894 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 1895 const char *psb = INTEL_PT_PSB_STR; 1896 1897 if (rest_psb > decoder->len || 1898 memcmp(decoder->buf, psb + part_psb, rest_psb)) 1899 return 0; 1900 1901 return rest_psb; 1902 } 1903 1904 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 1905 int part_psb) 1906 { 1907 int rest_psb, ret; 1908 1909 decoder->pos += decoder->len; 1910 decoder->len = 0; 1911 1912 ret = intel_pt_get_next_data(decoder); 1913 if (ret) 1914 return ret; 1915 1916 rest_psb = intel_pt_rest_psb(decoder, part_psb); 1917 if (!rest_psb) 1918 return 0; 1919 1920 decoder->pos -= part_psb; 1921 decoder->next_buf = decoder->buf + rest_psb; 1922 decoder->next_len = decoder->len - rest_psb; 1923 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 1924 decoder->buf = decoder->temp_buf; 1925 decoder->len = INTEL_PT_PSB_LEN; 1926 1927 return 0; 1928 } 1929 1930 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 1931 { 1932 unsigned char *next; 1933 int ret; 1934 1935 intel_pt_log("Scanning for PSB\n"); 1936 while (1) { 1937 if (!decoder->len) { 1938 ret = intel_pt_get_next_data(decoder); 1939 if (ret) 1940 return ret; 1941 } 1942 1943 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 1944 INTEL_PT_PSB_LEN); 1945 if (!next) { 1946 int part_psb; 1947 1948 part_psb = intel_pt_part_psb(decoder); 1949 if (part_psb) { 1950 ret = intel_pt_get_split_psb(decoder, part_psb); 1951 if (ret) 1952 return ret; 1953 } else { 1954 decoder->pos += decoder->len; 1955 decoder->len = 0; 1956 } 1957 continue; 1958 } 1959 1960 decoder->pkt_step = next - decoder->buf; 1961 return intel_pt_get_next_packet(decoder); 1962 } 1963 } 1964 1965 static int intel_pt_sync(struct intel_pt_decoder *decoder) 1966 { 1967 int err; 1968 1969 decoder->pge = false; 1970 decoder->continuous_period = false; 1971 decoder->last_ip = 0; 1972 decoder->ip = 0; 1973 intel_pt_clear_stack(&decoder->stack); 1974 1975 err = intel_pt_scan_for_psb(decoder); 1976 if (err) 1977 return err; 1978 1979 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 1980 1981 err = intel_pt_walk_psb(decoder); 1982 if (err) 1983 return err; 1984 1985 if (decoder->ip) { 1986 decoder->state.type = 0; /* Do not have a sample */ 1987 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1988 } else { 1989 return intel_pt_sync_ip(decoder); 1990 } 1991 1992 return 0; 1993 } 1994 1995 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 1996 { 1997 uint64_t est = decoder->timestamp_insn_cnt << 1; 1998 1999 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2000 goto out; 2001 2002 est *= decoder->max_non_turbo_ratio; 2003 est /= decoder->cbr; 2004 out: 2005 return decoder->timestamp + est; 2006 } 2007 2008 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2009 { 2010 int err; 2011 2012 do { 2013 decoder->state.type = INTEL_PT_BRANCH; 2014 decoder->state.flags = 0; 2015 2016 switch (decoder->pkt_state) { 2017 case INTEL_PT_STATE_NO_PSB: 2018 err = intel_pt_sync(decoder); 2019 break; 2020 case INTEL_PT_STATE_NO_IP: 2021 decoder->last_ip = 0; 2022 /* Fall through */ 2023 case INTEL_PT_STATE_ERR_RESYNC: 2024 err = intel_pt_sync_ip(decoder); 2025 break; 2026 case INTEL_PT_STATE_IN_SYNC: 2027 err = intel_pt_walk_trace(decoder); 2028 break; 2029 case INTEL_PT_STATE_TNT: 2030 err = intel_pt_walk_tnt(decoder); 2031 if (err == -EAGAIN) 2032 err = intel_pt_walk_trace(decoder); 2033 break; 2034 case INTEL_PT_STATE_TIP: 2035 case INTEL_PT_STATE_TIP_PGD: 2036 err = intel_pt_walk_tip(decoder); 2037 break; 2038 case INTEL_PT_STATE_FUP: 2039 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2040 err = intel_pt_walk_fup(decoder); 2041 if (err == -EAGAIN) 2042 err = intel_pt_walk_fup_tip(decoder); 2043 else if (!err) 2044 decoder->pkt_state = INTEL_PT_STATE_FUP; 2045 break; 2046 case INTEL_PT_STATE_FUP_NO_TIP: 2047 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2048 err = intel_pt_walk_fup(decoder); 2049 if (err == -EAGAIN) 2050 err = intel_pt_walk_trace(decoder); 2051 break; 2052 default: 2053 err = intel_pt_bug(decoder); 2054 break; 2055 } 2056 } while (err == -ENOLINK); 2057 2058 decoder->state.err = err ? intel_pt_ext_err(err) : 0; 2059 decoder->state.timestamp = decoder->timestamp; 2060 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2061 decoder->state.cr3 = decoder->cr3; 2062 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2063 2064 if (err) 2065 decoder->state.from_ip = decoder->ip; 2066 2067 return &decoder->state; 2068 } 2069 2070 static bool intel_pt_at_psb(unsigned char *buf, size_t len) 2071 { 2072 if (len < INTEL_PT_PSB_LEN) 2073 return false; 2074 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR, 2075 INTEL_PT_PSB_LEN); 2076 } 2077 2078 /** 2079 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2080 * @buf: pointer to buffer pointer 2081 * @len: size of buffer 2082 * 2083 * Updates the buffer pointer to point to the start of the next PSB packet if 2084 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2085 * @len is adjusted accordingly. 2086 * 2087 * Return: %true if a PSB packet is found, %false otherwise. 2088 */ 2089 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2090 { 2091 unsigned char *next; 2092 2093 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2094 if (next) { 2095 *len -= next - *buf; 2096 *buf = next; 2097 return true; 2098 } 2099 return false; 2100 } 2101 2102 /** 2103 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2104 * packet. 2105 * @buf: pointer to buffer pointer 2106 * @len: size of buffer 2107 * 2108 * Updates the buffer pointer to point to the start of the following PSB packet 2109 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2110 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2111 * 2112 * Return: %true if a PSB packet is found, %false otherwise. 2113 */ 2114 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2115 { 2116 unsigned char *next; 2117 2118 if (!*len) 2119 return false; 2120 2121 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2122 if (next) { 2123 *len -= next - *buf; 2124 *buf = next; 2125 return true; 2126 } 2127 return false; 2128 } 2129 2130 /** 2131 * intel_pt_last_psb - find the last PSB packet in a buffer. 2132 * @buf: buffer 2133 * @len: size of buffer 2134 * 2135 * This function finds the last PSB in a buffer. 2136 * 2137 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2138 */ 2139 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2140 { 2141 const char *n = INTEL_PT_PSB_STR; 2142 unsigned char *p; 2143 size_t k; 2144 2145 if (len < INTEL_PT_PSB_LEN) 2146 return NULL; 2147 2148 k = len - INTEL_PT_PSB_LEN + 1; 2149 while (1) { 2150 p = memrchr(buf, n[0], k); 2151 if (!p) 2152 return NULL; 2153 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2154 return p; 2155 k = p - buf; 2156 if (!k) 2157 return NULL; 2158 } 2159 } 2160 2161 /** 2162 * intel_pt_next_tsc - find and return next TSC. 2163 * @buf: buffer 2164 * @len: size of buffer 2165 * @tsc: TSC value returned 2166 * 2167 * Find a TSC packet in @buf and return the TSC value. This function assumes 2168 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2169 * PSBEND packet is found. 2170 * 2171 * Return: %true if TSC is found, false otherwise. 2172 */ 2173 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc) 2174 { 2175 struct intel_pt_pkt packet; 2176 int ret; 2177 2178 while (len) { 2179 ret = intel_pt_get_packet(buf, len, &packet); 2180 if (ret <= 0) 2181 return false; 2182 if (packet.type == INTEL_PT_TSC) { 2183 *tsc = packet.payload; 2184 return true; 2185 } 2186 if (packet.type == INTEL_PT_PSBEND) 2187 return false; 2188 buf += ret; 2189 len -= ret; 2190 } 2191 return false; 2192 } 2193 2194 /** 2195 * intel_pt_tsc_cmp - compare 7-byte TSCs. 2196 * @tsc1: first TSC to compare 2197 * @tsc2: second TSC to compare 2198 * 2199 * This function compares 7-byte TSC values allowing for the possibility that 2200 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 2201 * around so for that purpose this function assumes the absolute difference is 2202 * less than half the maximum difference. 2203 * 2204 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 2205 * after @tsc2. 2206 */ 2207 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 2208 { 2209 const uint64_t halfway = (1ULL << 55); 2210 2211 if (tsc1 == tsc2) 2212 return 0; 2213 2214 if (tsc1 < tsc2) { 2215 if (tsc2 - tsc1 < halfway) 2216 return -1; 2217 else 2218 return 1; 2219 } else { 2220 if (tsc1 - tsc2 < halfway) 2221 return 1; 2222 else 2223 return -1; 2224 } 2225 } 2226 2227 /** 2228 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 2229 * using TSC. 2230 * @buf_a: first buffer 2231 * @len_a: size of first buffer 2232 * @buf_b: second buffer 2233 * @len_b: size of second buffer 2234 * 2235 * If the trace contains TSC we can look at the last TSC of @buf_a and the 2236 * first TSC of @buf_b in order to determine if the buffers overlap, and then 2237 * walk forward in @buf_b until a later TSC is found. A precondition is that 2238 * @buf_a and @buf_b are positioned at a PSB. 2239 * 2240 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2241 * @buf_b + @len_b if there is no non-overlapped data. 2242 */ 2243 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 2244 size_t len_a, 2245 unsigned char *buf_b, 2246 size_t len_b) 2247 { 2248 uint64_t tsc_a, tsc_b; 2249 unsigned char *p; 2250 size_t len; 2251 2252 p = intel_pt_last_psb(buf_a, len_a); 2253 if (!p) 2254 return buf_b; /* No PSB in buf_a => no overlap */ 2255 2256 len = len_a - (p - buf_a); 2257 if (!intel_pt_next_tsc(p, len, &tsc_a)) { 2258 /* The last PSB+ in buf_a is incomplete, so go back one more */ 2259 len_a -= len; 2260 p = intel_pt_last_psb(buf_a, len_a); 2261 if (!p) 2262 return buf_b; /* No full PSB+ => assume no overlap */ 2263 len = len_a - (p - buf_a); 2264 if (!intel_pt_next_tsc(p, len, &tsc_a)) 2265 return buf_b; /* No TSC in buf_a => assume no overlap */ 2266 } 2267 2268 while (1) { 2269 /* Ignore PSB+ with no TSC */ 2270 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) && 2271 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0) 2272 return buf_b; /* tsc_a < tsc_b => no overlap */ 2273 2274 if (!intel_pt_step_psb(&buf_b, &len_b)) 2275 return buf_b + len_b; /* No PSB in buf_b => no data */ 2276 } 2277 } 2278 2279 /** 2280 * intel_pt_find_overlap - determine start of non-overlapped trace data. 2281 * @buf_a: first buffer 2282 * @len_a: size of first buffer 2283 * @buf_b: second buffer 2284 * @len_b: size of second buffer 2285 * @have_tsc: can use TSC packets to detect overlap 2286 * 2287 * When trace samples or snapshots are recorded there is the possibility that 2288 * the data overlaps. Note that, for the purposes of decoding, data is only 2289 * useful if it begins with a PSB packet. 2290 * 2291 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2292 * @buf_b + @len_b if there is no non-overlapped data. 2293 */ 2294 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 2295 unsigned char *buf_b, size_t len_b, 2296 bool have_tsc) 2297 { 2298 unsigned char *found; 2299 2300 /* Buffer 'b' must start at PSB so throw away everything before that */ 2301 if (!intel_pt_next_psb(&buf_b, &len_b)) 2302 return buf_b + len_b; /* No PSB */ 2303 2304 if (!intel_pt_next_psb(&buf_a, &len_a)) 2305 return buf_b; /* No overlap */ 2306 2307 if (have_tsc) { 2308 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b); 2309 if (found) 2310 return found; 2311 } 2312 2313 /* 2314 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 2315 * we can ignore the first part of buffer 'a'. 2316 */ 2317 while (len_b < len_a) { 2318 if (!intel_pt_step_psb(&buf_a, &len_a)) 2319 return buf_b; /* No overlap */ 2320 } 2321 2322 /* Now len_b >= len_a */ 2323 if (len_b > len_a) { 2324 /* The leftover buffer 'b' must start at a PSB */ 2325 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2326 if (!intel_pt_step_psb(&buf_a, &len_a)) 2327 return buf_b; /* No overlap */ 2328 } 2329 } 2330 2331 while (1) { 2332 /* Potential overlap so check the bytes */ 2333 found = memmem(buf_a, len_a, buf_b, len_a); 2334 if (found) 2335 return buf_b + len_a; 2336 2337 /* Try again at next PSB in buffer 'a' */ 2338 if (!intel_pt_step_psb(&buf_a, &len_a)) 2339 return buf_b; /* No overlap */ 2340 2341 /* The leftover buffer 'b' must start at a PSB */ 2342 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { 2343 if (!intel_pt_step_psb(&buf_a, &len_a)) 2344 return buf_b; /* No overlap */ 2345 } 2346 } 2347 } 2348