1 /* 2 * intel_pt_decoder.c: Intel Processor Trace support 3 * Copyright (c) 2013-2014, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #ifndef _GNU_SOURCE 17 #define _GNU_SOURCE 18 #endif 19 #include <stdlib.h> 20 #include <stdbool.h> 21 #include <string.h> 22 #include <errno.h> 23 #include <stdint.h> 24 #include <inttypes.h> 25 #include <linux/compiler.h> 26 27 #include "../cache.h" 28 #include "../util.h" 29 30 #include "intel-pt-insn-decoder.h" 31 #include "intel-pt-pkt-decoder.h" 32 #include "intel-pt-decoder.h" 33 #include "intel-pt-log.h" 34 35 #define INTEL_PT_BLK_SIZE 1024 36 37 #define BIT63 (((uint64_t)1 << 63)) 38 39 #define INTEL_PT_RETURN 1 40 41 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 42 #define INTEL_PT_MAX_LOOPS 10000 43 44 struct intel_pt_blk { 45 struct intel_pt_blk *prev; 46 uint64_t ip[INTEL_PT_BLK_SIZE]; 47 }; 48 49 struct intel_pt_stack { 50 struct intel_pt_blk *blk; 51 struct intel_pt_blk *spare; 52 int pos; 53 }; 54 55 enum intel_pt_pkt_state { 56 INTEL_PT_STATE_NO_PSB, 57 INTEL_PT_STATE_NO_IP, 58 INTEL_PT_STATE_ERR_RESYNC, 59 INTEL_PT_STATE_IN_SYNC, 60 INTEL_PT_STATE_TNT, 61 INTEL_PT_STATE_TIP, 62 INTEL_PT_STATE_TIP_PGD, 63 INTEL_PT_STATE_FUP, 64 INTEL_PT_STATE_FUP_NO_TIP, 65 }; 66 67 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 68 { 69 switch (pkt_state) { 70 case INTEL_PT_STATE_NO_PSB: 71 case INTEL_PT_STATE_NO_IP: 72 case INTEL_PT_STATE_ERR_RESYNC: 73 case INTEL_PT_STATE_IN_SYNC: 74 case INTEL_PT_STATE_TNT: 75 return true; 76 case INTEL_PT_STATE_TIP: 77 case INTEL_PT_STATE_TIP_PGD: 78 case INTEL_PT_STATE_FUP: 79 case INTEL_PT_STATE_FUP_NO_TIP: 80 return false; 81 default: 82 return true; 83 }; 84 } 85 86 #ifdef INTEL_PT_STRICT 87 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 88 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 89 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 90 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 91 #else 92 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 93 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 94 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 95 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 96 #endif 97 98 struct intel_pt_decoder { 99 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 100 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 101 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 102 uint64_t max_insn_cnt, void *data); 103 bool (*pgd_ip)(uint64_t ip, void *data); 104 void *data; 105 struct intel_pt_state state; 106 const unsigned char *buf; 107 size_t len; 108 bool return_compression; 109 bool branch_enable; 110 bool mtc_insn; 111 bool pge; 112 bool have_tma; 113 bool have_cyc; 114 bool fixup_last_mtc; 115 bool have_last_ip; 116 enum intel_pt_param_flags flags; 117 uint64_t pos; 118 uint64_t last_ip; 119 uint64_t ip; 120 uint64_t cr3; 121 uint64_t timestamp; 122 uint64_t tsc_timestamp; 123 uint64_t ref_timestamp; 124 uint64_t sample_timestamp; 125 uint64_t ret_addr; 126 uint64_t ctc_timestamp; 127 uint64_t ctc_delta; 128 uint64_t cycle_cnt; 129 uint64_t cyc_ref_timestamp; 130 uint32_t last_mtc; 131 uint32_t tsc_ctc_ratio_n; 132 uint32_t tsc_ctc_ratio_d; 133 uint32_t tsc_ctc_mult; 134 uint32_t tsc_slip; 135 uint32_t ctc_rem_mask; 136 int mtc_shift; 137 struct intel_pt_stack stack; 138 enum intel_pt_pkt_state pkt_state; 139 struct intel_pt_pkt packet; 140 struct intel_pt_pkt tnt; 141 int pkt_step; 142 int pkt_len; 143 int last_packet_type; 144 unsigned int cbr; 145 unsigned int cbr_seen; 146 unsigned int max_non_turbo_ratio; 147 double max_non_turbo_ratio_fp; 148 double cbr_cyc_to_tsc; 149 double calc_cyc_to_tsc; 150 bool have_calc_cyc_to_tsc; 151 int exec_mode; 152 unsigned int insn_bytes; 153 uint64_t period; 154 enum intel_pt_period_type period_type; 155 uint64_t tot_insn_cnt; 156 uint64_t period_insn_cnt; 157 uint64_t period_mask; 158 uint64_t period_ticks; 159 uint64_t last_masked_timestamp; 160 bool continuous_period; 161 bool overflow; 162 bool set_fup_tx_flags; 163 bool set_fup_ptw; 164 bool set_fup_mwait; 165 bool set_fup_pwre; 166 bool set_fup_exstop; 167 unsigned int fup_tx_flags; 168 unsigned int tx_flags; 169 uint64_t fup_ptw_payload; 170 uint64_t fup_mwait_payload; 171 uint64_t fup_pwre_payload; 172 uint64_t cbr_payload; 173 uint64_t timestamp_insn_cnt; 174 uint64_t sample_insn_cnt; 175 uint64_t stuck_ip; 176 int no_progress; 177 int stuck_ip_prd; 178 int stuck_ip_cnt; 179 const unsigned char *next_buf; 180 size_t next_len; 181 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 182 }; 183 184 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 185 { 186 int i; 187 188 for (i = 0; x != 1; i++) 189 x >>= 1; 190 191 return x << i; 192 } 193 194 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 195 { 196 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 197 uint64_t period; 198 199 period = intel_pt_lower_power_of_2(decoder->period); 200 decoder->period_mask = ~(period - 1); 201 decoder->period_ticks = period; 202 } 203 } 204 205 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 206 { 207 if (!d) 208 return 0; 209 return (t / d) * n + ((t % d) * n) / d; 210 } 211 212 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 213 { 214 struct intel_pt_decoder *decoder; 215 216 if (!params->get_trace || !params->walk_insn) 217 return NULL; 218 219 decoder = zalloc(sizeof(struct intel_pt_decoder)); 220 if (!decoder) 221 return NULL; 222 223 decoder->get_trace = params->get_trace; 224 decoder->walk_insn = params->walk_insn; 225 decoder->pgd_ip = params->pgd_ip; 226 decoder->data = params->data; 227 decoder->return_compression = params->return_compression; 228 decoder->branch_enable = params->branch_enable; 229 230 decoder->flags = params->flags; 231 232 decoder->period = params->period; 233 decoder->period_type = params->period_type; 234 235 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 236 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 237 238 intel_pt_setup_period(decoder); 239 240 decoder->mtc_shift = params->mtc_period; 241 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 242 243 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 244 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 245 246 if (!decoder->tsc_ctc_ratio_n) 247 decoder->tsc_ctc_ratio_d = 0; 248 249 if (decoder->tsc_ctc_ratio_d) { 250 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 251 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 252 decoder->tsc_ctc_ratio_d; 253 254 /* 255 * Allow for timestamps appearing to backwards because a TSC 256 * packet has slipped past a MTC packet, so allow 2 MTC ticks 257 * or ... 258 */ 259 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift, 260 decoder->tsc_ctc_ratio_n, 261 decoder->tsc_ctc_ratio_d); 262 } 263 /* ... or 0x100 paranoia */ 264 if (decoder->tsc_slip < 0x100) 265 decoder->tsc_slip = 0x100; 266 267 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 268 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 269 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 270 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 271 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 272 273 return decoder; 274 } 275 276 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 277 { 278 struct intel_pt_blk *blk = stack->blk; 279 280 stack->blk = blk->prev; 281 if (!stack->spare) 282 stack->spare = blk; 283 else 284 free(blk); 285 } 286 287 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 288 { 289 if (!stack->pos) { 290 if (!stack->blk) 291 return 0; 292 intel_pt_pop_blk(stack); 293 if (!stack->blk) 294 return 0; 295 stack->pos = INTEL_PT_BLK_SIZE; 296 } 297 return stack->blk->ip[--stack->pos]; 298 } 299 300 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 301 { 302 struct intel_pt_blk *blk; 303 304 if (stack->spare) { 305 blk = stack->spare; 306 stack->spare = NULL; 307 } else { 308 blk = malloc(sizeof(struct intel_pt_blk)); 309 if (!blk) 310 return -ENOMEM; 311 } 312 313 blk->prev = stack->blk; 314 stack->blk = blk; 315 stack->pos = 0; 316 return 0; 317 } 318 319 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 320 { 321 int err; 322 323 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 324 err = intel_pt_alloc_blk(stack); 325 if (err) 326 return err; 327 } 328 329 stack->blk->ip[stack->pos++] = ip; 330 return 0; 331 } 332 333 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 334 { 335 while (stack->blk) 336 intel_pt_pop_blk(stack); 337 stack->pos = 0; 338 } 339 340 static void intel_pt_free_stack(struct intel_pt_stack *stack) 341 { 342 intel_pt_clear_stack(stack); 343 zfree(&stack->blk); 344 zfree(&stack->spare); 345 } 346 347 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 348 { 349 intel_pt_free_stack(&decoder->stack); 350 free(decoder); 351 } 352 353 static int intel_pt_ext_err(int code) 354 { 355 switch (code) { 356 case -ENOMEM: 357 return INTEL_PT_ERR_NOMEM; 358 case -ENOSYS: 359 return INTEL_PT_ERR_INTERN; 360 case -EBADMSG: 361 return INTEL_PT_ERR_BADPKT; 362 case -ENODATA: 363 return INTEL_PT_ERR_NODATA; 364 case -EILSEQ: 365 return INTEL_PT_ERR_NOINSN; 366 case -ENOENT: 367 return INTEL_PT_ERR_MISMAT; 368 case -EOVERFLOW: 369 return INTEL_PT_ERR_OVR; 370 case -ENOSPC: 371 return INTEL_PT_ERR_LOST; 372 case -ELOOP: 373 return INTEL_PT_ERR_NELOOP; 374 default: 375 return INTEL_PT_ERR_UNK; 376 } 377 } 378 379 static const char *intel_pt_err_msgs[] = { 380 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 381 [INTEL_PT_ERR_INTERN] = "Internal error", 382 [INTEL_PT_ERR_BADPKT] = "Bad packet", 383 [INTEL_PT_ERR_NODATA] = "No more data", 384 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 385 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 386 [INTEL_PT_ERR_OVR] = "Overflow packet", 387 [INTEL_PT_ERR_LOST] = "Lost trace data", 388 [INTEL_PT_ERR_UNK] = "Unknown error!", 389 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 390 }; 391 392 int intel_pt__strerror(int code, char *buf, size_t buflen) 393 { 394 if (code < 1 || code >= INTEL_PT_ERR_MAX) 395 code = INTEL_PT_ERR_UNK; 396 strlcpy(buf, intel_pt_err_msgs[code], buflen); 397 return 0; 398 } 399 400 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 401 uint64_t last_ip) 402 { 403 uint64_t ip; 404 405 switch (packet->count) { 406 case 1: 407 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 408 packet->payload; 409 break; 410 case 2: 411 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 412 packet->payload; 413 break; 414 case 3: 415 ip = packet->payload; 416 /* Sign-extend 6-byte ip */ 417 if (ip & (uint64_t)0x800000000000ULL) 418 ip |= (uint64_t)0xffff000000000000ULL; 419 break; 420 case 4: 421 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 422 packet->payload; 423 break; 424 case 6: 425 ip = packet->payload; 426 break; 427 default: 428 return 0; 429 } 430 431 return ip; 432 } 433 434 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 435 { 436 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 437 decoder->have_last_ip = true; 438 } 439 440 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 441 { 442 intel_pt_set_last_ip(decoder); 443 decoder->ip = decoder->last_ip; 444 } 445 446 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 447 { 448 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 449 decoder->buf); 450 } 451 452 static int intel_pt_bug(struct intel_pt_decoder *decoder) 453 { 454 intel_pt_log("ERROR: Internal error\n"); 455 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 456 return -ENOSYS; 457 } 458 459 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 460 { 461 decoder->tx_flags = 0; 462 } 463 464 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 465 { 466 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 467 } 468 469 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 470 { 471 intel_pt_clear_tx_flags(decoder); 472 decoder->have_tma = false; 473 decoder->pkt_len = 1; 474 decoder->pkt_step = 1; 475 intel_pt_decoder_log_packet(decoder); 476 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 477 intel_pt_log("ERROR: Bad packet\n"); 478 decoder->pkt_state = INTEL_PT_STATE_ERR1; 479 } 480 return -EBADMSG; 481 } 482 483 static int intel_pt_get_data(struct intel_pt_decoder *decoder) 484 { 485 struct intel_pt_buffer buffer = { .buf = 0, }; 486 int ret; 487 488 decoder->pkt_step = 0; 489 490 intel_pt_log("Getting more data\n"); 491 ret = decoder->get_trace(&buffer, decoder->data); 492 if (ret) 493 return ret; 494 decoder->buf = buffer.buf; 495 decoder->len = buffer.len; 496 if (!decoder->len) { 497 intel_pt_log("No more data\n"); 498 return -ENODATA; 499 } 500 if (!buffer.consecutive) { 501 decoder->ip = 0; 502 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 503 decoder->ref_timestamp = buffer.ref_timestamp; 504 decoder->timestamp = 0; 505 decoder->have_tma = false; 506 decoder->state.trace_nr = buffer.trace_nr; 507 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 508 decoder->ref_timestamp); 509 return -ENOLINK; 510 } 511 512 return 0; 513 } 514 515 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder) 516 { 517 if (!decoder->next_buf) 518 return intel_pt_get_data(decoder); 519 520 decoder->buf = decoder->next_buf; 521 decoder->len = decoder->next_len; 522 decoder->next_buf = 0; 523 decoder->next_len = 0; 524 return 0; 525 } 526 527 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 528 { 529 unsigned char *buf = decoder->temp_buf; 530 size_t old_len, len, n; 531 int ret; 532 533 old_len = decoder->len; 534 len = decoder->len; 535 memcpy(buf, decoder->buf, len); 536 537 ret = intel_pt_get_data(decoder); 538 if (ret) { 539 decoder->pos += old_len; 540 return ret < 0 ? ret : -EINVAL; 541 } 542 543 n = INTEL_PT_PKT_MAX_SZ - len; 544 if (n > decoder->len) 545 n = decoder->len; 546 memcpy(buf + len, decoder->buf, n); 547 len += n; 548 549 ret = intel_pt_get_packet(buf, len, &decoder->packet); 550 if (ret < (int)old_len) { 551 decoder->next_buf = decoder->buf; 552 decoder->next_len = decoder->len; 553 decoder->buf = buf; 554 decoder->len = old_len; 555 return intel_pt_bad_packet(decoder); 556 } 557 558 decoder->next_buf = decoder->buf + (ret - old_len); 559 decoder->next_len = decoder->len - (ret - old_len); 560 561 decoder->buf = buf; 562 decoder->len = ret; 563 564 return ret; 565 } 566 567 struct intel_pt_pkt_info { 568 struct intel_pt_decoder *decoder; 569 struct intel_pt_pkt packet; 570 uint64_t pos; 571 int pkt_len; 572 int last_packet_type; 573 void *data; 574 }; 575 576 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 577 578 /* Lookahead packets in current buffer */ 579 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 580 intel_pt_pkt_cb_t cb, void *data) 581 { 582 struct intel_pt_pkt_info pkt_info; 583 const unsigned char *buf = decoder->buf; 584 size_t len = decoder->len; 585 int ret; 586 587 pkt_info.decoder = decoder; 588 pkt_info.pos = decoder->pos; 589 pkt_info.pkt_len = decoder->pkt_step; 590 pkt_info.last_packet_type = decoder->last_packet_type; 591 pkt_info.data = data; 592 593 while (1) { 594 do { 595 pkt_info.pos += pkt_info.pkt_len; 596 buf += pkt_info.pkt_len; 597 len -= pkt_info.pkt_len; 598 599 if (!len) 600 return INTEL_PT_NEED_MORE_BYTES; 601 602 ret = intel_pt_get_packet(buf, len, &pkt_info.packet); 603 if (!ret) 604 return INTEL_PT_NEED_MORE_BYTES; 605 if (ret < 0) 606 return ret; 607 608 pkt_info.pkt_len = ret; 609 } while (pkt_info.packet.type == INTEL_PT_PAD); 610 611 ret = cb(&pkt_info); 612 if (ret) 613 return 0; 614 615 pkt_info.last_packet_type = pkt_info.packet.type; 616 } 617 } 618 619 struct intel_pt_calc_cyc_to_tsc_info { 620 uint64_t cycle_cnt; 621 unsigned int cbr; 622 uint32_t last_mtc; 623 uint64_t ctc_timestamp; 624 uint64_t ctc_delta; 625 uint64_t tsc_timestamp; 626 uint64_t timestamp; 627 bool have_tma; 628 bool fixup_last_mtc; 629 bool from_mtc; 630 double cbr_cyc_to_tsc; 631 }; 632 633 /* 634 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 635 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 636 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 637 * packet by copying the missing bits from the current MTC assuming the least 638 * difference between the two, and that the current MTC comes after last_mtc. 639 */ 640 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 641 uint32_t *last_mtc) 642 { 643 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 644 uint32_t mask = ~(first_missing_bit - 1); 645 646 *last_mtc |= mtc & mask; 647 if (*last_mtc >= mtc) { 648 *last_mtc -= first_missing_bit; 649 *last_mtc &= 0xff; 650 } 651 } 652 653 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 654 { 655 struct intel_pt_decoder *decoder = pkt_info->decoder; 656 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 657 uint64_t timestamp; 658 double cyc_to_tsc; 659 unsigned int cbr; 660 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 661 662 switch (pkt_info->packet.type) { 663 case INTEL_PT_TNT: 664 case INTEL_PT_TIP_PGE: 665 case INTEL_PT_TIP: 666 case INTEL_PT_FUP: 667 case INTEL_PT_PSB: 668 case INTEL_PT_PIP: 669 case INTEL_PT_MODE_EXEC: 670 case INTEL_PT_MODE_TSX: 671 case INTEL_PT_PSBEND: 672 case INTEL_PT_PAD: 673 case INTEL_PT_VMCS: 674 case INTEL_PT_MNT: 675 case INTEL_PT_PTWRITE: 676 case INTEL_PT_PTWRITE_IP: 677 return 0; 678 679 case INTEL_PT_MTC: 680 if (!data->have_tma) 681 return 0; 682 683 mtc = pkt_info->packet.payload; 684 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 685 data->fixup_last_mtc = false; 686 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 687 &data->last_mtc); 688 } 689 if (mtc > data->last_mtc) 690 mtc_delta = mtc - data->last_mtc; 691 else 692 mtc_delta = mtc + 256 - data->last_mtc; 693 data->ctc_delta += mtc_delta << decoder->mtc_shift; 694 data->last_mtc = mtc; 695 696 if (decoder->tsc_ctc_mult) { 697 timestamp = data->ctc_timestamp + 698 data->ctc_delta * decoder->tsc_ctc_mult; 699 } else { 700 timestamp = data->ctc_timestamp + 701 multdiv(data->ctc_delta, 702 decoder->tsc_ctc_ratio_n, 703 decoder->tsc_ctc_ratio_d); 704 } 705 706 if (timestamp < data->timestamp) 707 return 1; 708 709 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 710 data->timestamp = timestamp; 711 return 0; 712 } 713 714 break; 715 716 case INTEL_PT_TSC: 717 /* 718 * For now, do not support using TSC packets - refer 719 * intel_pt_calc_cyc_to_tsc(). 720 */ 721 if (data->from_mtc) 722 return 1; 723 timestamp = pkt_info->packet.payload | 724 (data->timestamp & (0xffULL << 56)); 725 if (data->from_mtc && timestamp < data->timestamp && 726 data->timestamp - timestamp < decoder->tsc_slip) 727 return 1; 728 if (timestamp < data->timestamp) 729 timestamp += (1ULL << 56); 730 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 731 if (data->from_mtc) 732 return 1; 733 data->tsc_timestamp = timestamp; 734 data->timestamp = timestamp; 735 return 0; 736 } 737 break; 738 739 case INTEL_PT_TMA: 740 if (data->from_mtc) 741 return 1; 742 743 if (!decoder->tsc_ctc_ratio_d) 744 return 0; 745 746 ctc = pkt_info->packet.payload; 747 fc = pkt_info->packet.count; 748 ctc_rem = ctc & decoder->ctc_rem_mask; 749 750 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 751 752 data->ctc_timestamp = data->tsc_timestamp - fc; 753 if (decoder->tsc_ctc_mult) { 754 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 755 } else { 756 data->ctc_timestamp -= 757 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 758 decoder->tsc_ctc_ratio_d); 759 } 760 761 data->ctc_delta = 0; 762 data->have_tma = true; 763 data->fixup_last_mtc = true; 764 765 return 0; 766 767 case INTEL_PT_CYC: 768 data->cycle_cnt += pkt_info->packet.payload; 769 return 0; 770 771 case INTEL_PT_CBR: 772 cbr = pkt_info->packet.payload; 773 if (data->cbr && data->cbr != cbr) 774 return 1; 775 data->cbr = cbr; 776 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 777 return 0; 778 779 case INTEL_PT_TIP_PGD: 780 case INTEL_PT_TRACESTOP: 781 case INTEL_PT_EXSTOP: 782 case INTEL_PT_EXSTOP_IP: 783 case INTEL_PT_MWAIT: 784 case INTEL_PT_PWRE: 785 case INTEL_PT_PWRX: 786 case INTEL_PT_OVF: 787 case INTEL_PT_BAD: /* Does not happen */ 788 default: 789 return 1; 790 } 791 792 if (!data->cbr && decoder->cbr) { 793 data->cbr = decoder->cbr; 794 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 795 } 796 797 if (!data->cycle_cnt) 798 return 1; 799 800 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 801 802 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 803 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 804 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 805 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 806 return 1; 807 } 808 809 decoder->calc_cyc_to_tsc = cyc_to_tsc; 810 decoder->have_calc_cyc_to_tsc = true; 811 812 if (data->cbr) { 813 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 814 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 815 } else { 816 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 817 cyc_to_tsc, pkt_info->pos); 818 } 819 820 return 1; 821 } 822 823 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 824 bool from_mtc) 825 { 826 struct intel_pt_calc_cyc_to_tsc_info data = { 827 .cycle_cnt = 0, 828 .cbr = 0, 829 .last_mtc = decoder->last_mtc, 830 .ctc_timestamp = decoder->ctc_timestamp, 831 .ctc_delta = decoder->ctc_delta, 832 .tsc_timestamp = decoder->tsc_timestamp, 833 .timestamp = decoder->timestamp, 834 .have_tma = decoder->have_tma, 835 .fixup_last_mtc = decoder->fixup_last_mtc, 836 .from_mtc = from_mtc, 837 .cbr_cyc_to_tsc = 0, 838 }; 839 840 /* 841 * For now, do not support using TSC packets for at least the reasons: 842 * 1) timing might have stopped 843 * 2) TSC packets within PSB+ can slip against CYC packets 844 */ 845 if (!from_mtc) 846 return; 847 848 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 849 } 850 851 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 852 { 853 int ret; 854 855 decoder->last_packet_type = decoder->packet.type; 856 857 do { 858 decoder->pos += decoder->pkt_step; 859 decoder->buf += decoder->pkt_step; 860 decoder->len -= decoder->pkt_step; 861 862 if (!decoder->len) { 863 ret = intel_pt_get_next_data(decoder); 864 if (ret) 865 return ret; 866 } 867 868 ret = intel_pt_get_packet(decoder->buf, decoder->len, 869 &decoder->packet); 870 if (ret == INTEL_PT_NEED_MORE_BYTES && 871 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 872 ret = intel_pt_get_split_packet(decoder); 873 if (ret < 0) 874 return ret; 875 } 876 if (ret <= 0) 877 return intel_pt_bad_packet(decoder); 878 879 decoder->pkt_len = ret; 880 decoder->pkt_step = ret; 881 intel_pt_decoder_log_packet(decoder); 882 } while (decoder->packet.type == INTEL_PT_PAD); 883 884 return 0; 885 } 886 887 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 888 { 889 uint64_t timestamp, masked_timestamp; 890 891 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 892 masked_timestamp = timestamp & decoder->period_mask; 893 if (decoder->continuous_period) { 894 if (masked_timestamp != decoder->last_masked_timestamp) 895 return 1; 896 } else { 897 timestamp += 1; 898 masked_timestamp = timestamp & decoder->period_mask; 899 if (masked_timestamp != decoder->last_masked_timestamp) { 900 decoder->last_masked_timestamp = masked_timestamp; 901 decoder->continuous_period = true; 902 } 903 } 904 return decoder->period_ticks - (timestamp - masked_timestamp); 905 } 906 907 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 908 { 909 switch (decoder->period_type) { 910 case INTEL_PT_PERIOD_INSTRUCTIONS: 911 return decoder->period - decoder->period_insn_cnt; 912 case INTEL_PT_PERIOD_TICKS: 913 return intel_pt_next_period(decoder); 914 case INTEL_PT_PERIOD_NONE: 915 case INTEL_PT_PERIOD_MTC: 916 default: 917 return 0; 918 } 919 } 920 921 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 922 { 923 uint64_t timestamp, masked_timestamp; 924 925 switch (decoder->period_type) { 926 case INTEL_PT_PERIOD_INSTRUCTIONS: 927 decoder->period_insn_cnt = 0; 928 break; 929 case INTEL_PT_PERIOD_TICKS: 930 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 931 masked_timestamp = timestamp & decoder->period_mask; 932 decoder->last_masked_timestamp = masked_timestamp; 933 break; 934 case INTEL_PT_PERIOD_NONE: 935 case INTEL_PT_PERIOD_MTC: 936 default: 937 break; 938 } 939 940 decoder->state.type |= INTEL_PT_INSTRUCTION; 941 } 942 943 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 944 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 945 { 946 uint64_t max_insn_cnt, insn_cnt = 0; 947 int err; 948 949 if (!decoder->mtc_insn) 950 decoder->mtc_insn = true; 951 952 max_insn_cnt = intel_pt_next_sample(decoder); 953 954 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 955 max_insn_cnt, decoder->data); 956 957 decoder->tot_insn_cnt += insn_cnt; 958 decoder->timestamp_insn_cnt += insn_cnt; 959 decoder->sample_insn_cnt += insn_cnt; 960 decoder->period_insn_cnt += insn_cnt; 961 962 if (err) { 963 decoder->no_progress = 0; 964 decoder->pkt_state = INTEL_PT_STATE_ERR2; 965 intel_pt_log_at("ERROR: Failed to get instruction", 966 decoder->ip); 967 if (err == -ENOENT) 968 return -ENOLINK; 969 return -EILSEQ; 970 } 971 972 if (ip && decoder->ip == ip) { 973 err = -EAGAIN; 974 goto out; 975 } 976 977 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 978 intel_pt_sample_insn(decoder); 979 980 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 981 decoder->state.type = INTEL_PT_INSTRUCTION; 982 decoder->state.from_ip = decoder->ip; 983 decoder->state.to_ip = 0; 984 decoder->ip += intel_pt_insn->length; 985 err = INTEL_PT_RETURN; 986 goto out; 987 } 988 989 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 990 /* Zero-length calls are excluded */ 991 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 992 intel_pt_insn->rel) { 993 err = intel_pt_push(&decoder->stack, decoder->ip + 994 intel_pt_insn->length); 995 if (err) 996 goto out; 997 } 998 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 999 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1000 } 1001 1002 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1003 int cnt = decoder->no_progress++; 1004 1005 decoder->state.from_ip = decoder->ip; 1006 decoder->ip += intel_pt_insn->length + 1007 intel_pt_insn->rel; 1008 decoder->state.to_ip = decoder->ip; 1009 err = INTEL_PT_RETURN; 1010 1011 /* 1012 * Check for being stuck in a loop. This can happen if a 1013 * decoder error results in the decoder erroneously setting the 1014 * ip to an address that is itself in an infinite loop that 1015 * consumes no packets. When that happens, there must be an 1016 * unconditional branch. 1017 */ 1018 if (cnt) { 1019 if (cnt == 1) { 1020 decoder->stuck_ip = decoder->state.to_ip; 1021 decoder->stuck_ip_prd = 1; 1022 decoder->stuck_ip_cnt = 1; 1023 } else if (cnt > INTEL_PT_MAX_LOOPS || 1024 decoder->state.to_ip == decoder->stuck_ip) { 1025 intel_pt_log_at("ERROR: Never-ending loop", 1026 decoder->state.to_ip); 1027 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1028 err = -ELOOP; 1029 goto out; 1030 } else if (!--decoder->stuck_ip_cnt) { 1031 decoder->stuck_ip_prd += 1; 1032 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1033 decoder->stuck_ip = decoder->state.to_ip; 1034 } 1035 } 1036 goto out_no_progress; 1037 } 1038 out: 1039 decoder->no_progress = 0; 1040 out_no_progress: 1041 decoder->state.insn_op = intel_pt_insn->op; 1042 decoder->state.insn_len = intel_pt_insn->length; 1043 memcpy(decoder->state.insn, intel_pt_insn->buf, 1044 INTEL_PT_INSN_BUF_SZ); 1045 1046 if (decoder->tx_flags & INTEL_PT_IN_TX) 1047 decoder->state.flags |= INTEL_PT_IN_TX; 1048 1049 return err; 1050 } 1051 1052 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1053 { 1054 bool ret = false; 1055 1056 if (decoder->set_fup_tx_flags) { 1057 decoder->set_fup_tx_flags = false; 1058 decoder->tx_flags = decoder->fup_tx_flags; 1059 decoder->state.type = INTEL_PT_TRANSACTION; 1060 decoder->state.from_ip = decoder->ip; 1061 decoder->state.to_ip = 0; 1062 decoder->state.flags = decoder->fup_tx_flags; 1063 return true; 1064 } 1065 if (decoder->set_fup_ptw) { 1066 decoder->set_fup_ptw = false; 1067 decoder->state.type = INTEL_PT_PTW; 1068 decoder->state.flags |= INTEL_PT_FUP_IP; 1069 decoder->state.from_ip = decoder->ip; 1070 decoder->state.to_ip = 0; 1071 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1072 return true; 1073 } 1074 if (decoder->set_fup_mwait) { 1075 decoder->set_fup_mwait = false; 1076 decoder->state.type = INTEL_PT_MWAIT_OP; 1077 decoder->state.from_ip = decoder->ip; 1078 decoder->state.to_ip = 0; 1079 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1080 ret = true; 1081 } 1082 if (decoder->set_fup_pwre) { 1083 decoder->set_fup_pwre = false; 1084 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1085 decoder->state.type &= ~INTEL_PT_BRANCH; 1086 decoder->state.from_ip = decoder->ip; 1087 decoder->state.to_ip = 0; 1088 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1089 ret = true; 1090 } 1091 if (decoder->set_fup_exstop) { 1092 decoder->set_fup_exstop = false; 1093 decoder->state.type |= INTEL_PT_EX_STOP; 1094 decoder->state.type &= ~INTEL_PT_BRANCH; 1095 decoder->state.flags |= INTEL_PT_FUP_IP; 1096 decoder->state.from_ip = decoder->ip; 1097 decoder->state.to_ip = 0; 1098 ret = true; 1099 } 1100 return ret; 1101 } 1102 1103 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1104 struct intel_pt_insn *intel_pt_insn, 1105 uint64_t ip, int err) 1106 { 1107 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1108 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1109 ip == decoder->ip + intel_pt_insn->length; 1110 } 1111 1112 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1113 { 1114 struct intel_pt_insn intel_pt_insn; 1115 uint64_t ip; 1116 int err; 1117 1118 ip = decoder->last_ip; 1119 1120 while (1) { 1121 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1122 if (err == INTEL_PT_RETURN) 1123 return 0; 1124 if (err == -EAGAIN || 1125 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1126 if (intel_pt_fup_event(decoder)) 1127 return 0; 1128 return -EAGAIN; 1129 } 1130 decoder->set_fup_tx_flags = false; 1131 if (err) 1132 return err; 1133 1134 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1135 intel_pt_log_at("ERROR: Unexpected indirect branch", 1136 decoder->ip); 1137 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1138 return -ENOENT; 1139 } 1140 1141 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1142 intel_pt_log_at("ERROR: Unexpected conditional branch", 1143 decoder->ip); 1144 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1145 return -ENOENT; 1146 } 1147 1148 intel_pt_bug(decoder); 1149 } 1150 } 1151 1152 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1153 { 1154 struct intel_pt_insn intel_pt_insn; 1155 int err; 1156 1157 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1158 if (err == INTEL_PT_RETURN && 1159 decoder->pgd_ip && 1160 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1161 (decoder->state.type & INTEL_PT_BRANCH) && 1162 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1163 /* Unconditional branch leaving filter region */ 1164 decoder->no_progress = 0; 1165 decoder->pge = false; 1166 decoder->continuous_period = false; 1167 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1168 decoder->state.to_ip = 0; 1169 return 0; 1170 } 1171 if (err == INTEL_PT_RETURN) 1172 return 0; 1173 if (err) 1174 return err; 1175 1176 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1177 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1178 decoder->pge = false; 1179 decoder->continuous_period = false; 1180 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1181 decoder->state.from_ip = decoder->ip; 1182 decoder->state.to_ip = 0; 1183 if (decoder->packet.count != 0) 1184 decoder->ip = decoder->last_ip; 1185 } else { 1186 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1187 decoder->state.from_ip = decoder->ip; 1188 if (decoder->packet.count == 0) { 1189 decoder->state.to_ip = 0; 1190 } else { 1191 decoder->state.to_ip = decoder->last_ip; 1192 decoder->ip = decoder->last_ip; 1193 } 1194 } 1195 return 0; 1196 } 1197 1198 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1199 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1200 intel_pt_insn.rel; 1201 1202 if (decoder->pgd_ip && 1203 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1204 decoder->pgd_ip(to_ip, decoder->data)) { 1205 /* Conditional branch leaving filter region */ 1206 decoder->pge = false; 1207 decoder->continuous_period = false; 1208 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1209 decoder->ip = to_ip; 1210 decoder->state.from_ip = decoder->ip; 1211 decoder->state.to_ip = 0; 1212 return 0; 1213 } 1214 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1215 decoder->ip); 1216 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1217 return -ENOENT; 1218 } 1219 1220 return intel_pt_bug(decoder); 1221 } 1222 1223 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1224 { 1225 struct intel_pt_insn intel_pt_insn; 1226 int err; 1227 1228 while (1) { 1229 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1230 if (err == INTEL_PT_RETURN) 1231 return 0; 1232 if (err) 1233 return err; 1234 1235 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1236 if (!decoder->return_compression) { 1237 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1238 decoder->ip); 1239 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1240 return -ENOENT; 1241 } 1242 if (!decoder->ret_addr) { 1243 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1244 decoder->ip); 1245 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1246 return -ENOENT; 1247 } 1248 if (!(decoder->tnt.payload & BIT63)) { 1249 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1250 decoder->ip); 1251 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1252 return -ENOENT; 1253 } 1254 decoder->tnt.count -= 1; 1255 if (!decoder->tnt.count) 1256 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1257 decoder->tnt.payload <<= 1; 1258 decoder->state.from_ip = decoder->ip; 1259 decoder->ip = decoder->ret_addr; 1260 decoder->state.to_ip = decoder->ip; 1261 return 0; 1262 } 1263 1264 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1265 /* Handle deferred TIPs */ 1266 err = intel_pt_get_next_packet(decoder); 1267 if (err) 1268 return err; 1269 if (decoder->packet.type != INTEL_PT_TIP || 1270 decoder->packet.count == 0) { 1271 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1272 decoder->ip); 1273 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1274 decoder->pkt_step = 0; 1275 return -ENOENT; 1276 } 1277 intel_pt_set_last_ip(decoder); 1278 decoder->state.from_ip = decoder->ip; 1279 decoder->state.to_ip = decoder->last_ip; 1280 decoder->ip = decoder->last_ip; 1281 return 0; 1282 } 1283 1284 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1285 decoder->tnt.count -= 1; 1286 if (!decoder->tnt.count) 1287 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1288 if (decoder->tnt.payload & BIT63) { 1289 decoder->tnt.payload <<= 1; 1290 decoder->state.from_ip = decoder->ip; 1291 decoder->ip += intel_pt_insn.length + 1292 intel_pt_insn.rel; 1293 decoder->state.to_ip = decoder->ip; 1294 return 0; 1295 } 1296 /* Instruction sample for a non-taken branch */ 1297 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1298 decoder->tnt.payload <<= 1; 1299 decoder->state.type = INTEL_PT_INSTRUCTION; 1300 decoder->state.from_ip = decoder->ip; 1301 decoder->state.to_ip = 0; 1302 decoder->ip += intel_pt_insn.length; 1303 return 0; 1304 } 1305 decoder->ip += intel_pt_insn.length; 1306 if (!decoder->tnt.count) 1307 return -EAGAIN; 1308 decoder->tnt.payload <<= 1; 1309 continue; 1310 } 1311 1312 return intel_pt_bug(decoder); 1313 } 1314 } 1315 1316 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1317 { 1318 unsigned int fup_tx_flags; 1319 int err; 1320 1321 fup_tx_flags = decoder->packet.payload & 1322 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1323 err = intel_pt_get_next_packet(decoder); 1324 if (err) 1325 return err; 1326 if (decoder->packet.type == INTEL_PT_FUP) { 1327 decoder->fup_tx_flags = fup_tx_flags; 1328 decoder->set_fup_tx_flags = true; 1329 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1330 *no_tip = true; 1331 } else { 1332 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1333 decoder->pos); 1334 intel_pt_update_in_tx(decoder); 1335 } 1336 return 0; 1337 } 1338 1339 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1340 { 1341 uint64_t timestamp; 1342 1343 decoder->have_tma = false; 1344 1345 if (decoder->ref_timestamp) { 1346 timestamp = decoder->packet.payload | 1347 (decoder->ref_timestamp & (0xffULL << 56)); 1348 if (timestamp < decoder->ref_timestamp) { 1349 if (decoder->ref_timestamp - timestamp > (1ULL << 55)) 1350 timestamp += (1ULL << 56); 1351 } else { 1352 if (timestamp - decoder->ref_timestamp > (1ULL << 55)) 1353 timestamp -= (1ULL << 56); 1354 } 1355 decoder->tsc_timestamp = timestamp; 1356 decoder->timestamp = timestamp; 1357 decoder->ref_timestamp = 0; 1358 decoder->timestamp_insn_cnt = 0; 1359 } else if (decoder->timestamp) { 1360 timestamp = decoder->packet.payload | 1361 (decoder->timestamp & (0xffULL << 56)); 1362 decoder->tsc_timestamp = timestamp; 1363 if (timestamp < decoder->timestamp && 1364 decoder->timestamp - timestamp < decoder->tsc_slip) { 1365 intel_pt_log_to("Suppressing backwards timestamp", 1366 timestamp); 1367 timestamp = decoder->timestamp; 1368 } 1369 if (timestamp < decoder->timestamp) { 1370 intel_pt_log_to("Wraparound timestamp", timestamp); 1371 timestamp += (1ULL << 56); 1372 decoder->tsc_timestamp = timestamp; 1373 } 1374 decoder->timestamp = timestamp; 1375 decoder->timestamp_insn_cnt = 0; 1376 } 1377 1378 if (decoder->last_packet_type == INTEL_PT_CYC) { 1379 decoder->cyc_ref_timestamp = decoder->timestamp; 1380 decoder->cycle_cnt = 0; 1381 decoder->have_calc_cyc_to_tsc = false; 1382 intel_pt_calc_cyc_to_tsc(decoder, false); 1383 } 1384 1385 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1386 } 1387 1388 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1389 { 1390 intel_pt_log("ERROR: Buffer overflow\n"); 1391 intel_pt_clear_tx_flags(decoder); 1392 decoder->cbr = 0; 1393 decoder->timestamp_insn_cnt = 0; 1394 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1395 decoder->overflow = true; 1396 return -EOVERFLOW; 1397 } 1398 1399 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1400 { 1401 uint32_t ctc = decoder->packet.payload; 1402 uint32_t fc = decoder->packet.count; 1403 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1404 1405 if (!decoder->tsc_ctc_ratio_d) 1406 return; 1407 1408 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1409 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1410 if (decoder->tsc_ctc_mult) { 1411 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1412 } else { 1413 decoder->ctc_timestamp -= multdiv(ctc_rem, 1414 decoder->tsc_ctc_ratio_n, 1415 decoder->tsc_ctc_ratio_d); 1416 } 1417 decoder->ctc_delta = 0; 1418 decoder->have_tma = true; 1419 decoder->fixup_last_mtc = true; 1420 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1421 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1422 } 1423 1424 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1425 { 1426 uint64_t timestamp; 1427 uint32_t mtc, mtc_delta; 1428 1429 if (!decoder->have_tma) 1430 return; 1431 1432 mtc = decoder->packet.payload; 1433 1434 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1435 decoder->fixup_last_mtc = false; 1436 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1437 &decoder->last_mtc); 1438 } 1439 1440 if (mtc > decoder->last_mtc) 1441 mtc_delta = mtc - decoder->last_mtc; 1442 else 1443 mtc_delta = mtc + 256 - decoder->last_mtc; 1444 1445 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1446 1447 if (decoder->tsc_ctc_mult) { 1448 timestamp = decoder->ctc_timestamp + 1449 decoder->ctc_delta * decoder->tsc_ctc_mult; 1450 } else { 1451 timestamp = decoder->ctc_timestamp + 1452 multdiv(decoder->ctc_delta, 1453 decoder->tsc_ctc_ratio_n, 1454 decoder->tsc_ctc_ratio_d); 1455 } 1456 1457 if (timestamp < decoder->timestamp) 1458 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1459 timestamp, decoder->timestamp); 1460 else 1461 decoder->timestamp = timestamp; 1462 1463 decoder->timestamp_insn_cnt = 0; 1464 decoder->last_mtc = mtc; 1465 1466 if (decoder->last_packet_type == INTEL_PT_CYC) { 1467 decoder->cyc_ref_timestamp = decoder->timestamp; 1468 decoder->cycle_cnt = 0; 1469 decoder->have_calc_cyc_to_tsc = false; 1470 intel_pt_calc_cyc_to_tsc(decoder, true); 1471 } 1472 } 1473 1474 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1475 { 1476 unsigned int cbr = decoder->packet.payload & 0xff; 1477 1478 decoder->cbr_payload = decoder->packet.payload; 1479 1480 if (decoder->cbr == cbr) 1481 return; 1482 1483 decoder->cbr = cbr; 1484 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1485 } 1486 1487 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1488 { 1489 uint64_t timestamp = decoder->cyc_ref_timestamp; 1490 1491 decoder->have_cyc = true; 1492 1493 decoder->cycle_cnt += decoder->packet.payload; 1494 1495 if (!decoder->cyc_ref_timestamp) 1496 return; 1497 1498 if (decoder->have_calc_cyc_to_tsc) 1499 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1500 else if (decoder->cbr) 1501 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1502 else 1503 return; 1504 1505 if (timestamp < decoder->timestamp) 1506 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1507 timestamp, decoder->timestamp); 1508 else 1509 decoder->timestamp = timestamp; 1510 1511 decoder->timestamp_insn_cnt = 0; 1512 } 1513 1514 /* Walk PSB+ packets when already in sync. */ 1515 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1516 { 1517 int err; 1518 1519 while (1) { 1520 err = intel_pt_get_next_packet(decoder); 1521 if (err) 1522 return err; 1523 1524 switch (decoder->packet.type) { 1525 case INTEL_PT_PSBEND: 1526 return 0; 1527 1528 case INTEL_PT_TIP_PGD: 1529 case INTEL_PT_TIP_PGE: 1530 case INTEL_PT_TIP: 1531 case INTEL_PT_TNT: 1532 case INTEL_PT_TRACESTOP: 1533 case INTEL_PT_BAD: 1534 case INTEL_PT_PSB: 1535 case INTEL_PT_PTWRITE: 1536 case INTEL_PT_PTWRITE_IP: 1537 case INTEL_PT_EXSTOP: 1538 case INTEL_PT_EXSTOP_IP: 1539 case INTEL_PT_MWAIT: 1540 case INTEL_PT_PWRE: 1541 case INTEL_PT_PWRX: 1542 decoder->have_tma = false; 1543 intel_pt_log("ERROR: Unexpected packet\n"); 1544 return -EAGAIN; 1545 1546 case INTEL_PT_OVF: 1547 return intel_pt_overflow(decoder); 1548 1549 case INTEL_PT_TSC: 1550 intel_pt_calc_tsc_timestamp(decoder); 1551 break; 1552 1553 case INTEL_PT_TMA: 1554 intel_pt_calc_tma(decoder); 1555 break; 1556 1557 case INTEL_PT_CBR: 1558 intel_pt_calc_cbr(decoder); 1559 break; 1560 1561 case INTEL_PT_MODE_EXEC: 1562 decoder->exec_mode = decoder->packet.payload; 1563 break; 1564 1565 case INTEL_PT_PIP: 1566 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1567 break; 1568 1569 case INTEL_PT_FUP: 1570 decoder->pge = true; 1571 if (decoder->packet.count) 1572 intel_pt_set_last_ip(decoder); 1573 break; 1574 1575 case INTEL_PT_MODE_TSX: 1576 intel_pt_update_in_tx(decoder); 1577 break; 1578 1579 case INTEL_PT_MTC: 1580 intel_pt_calc_mtc_timestamp(decoder); 1581 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1582 decoder->state.type |= INTEL_PT_INSTRUCTION; 1583 break; 1584 1585 case INTEL_PT_CYC: 1586 case INTEL_PT_VMCS: 1587 case INTEL_PT_MNT: 1588 case INTEL_PT_PAD: 1589 default: 1590 break; 1591 } 1592 } 1593 } 1594 1595 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1596 { 1597 int err; 1598 1599 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1600 decoder->tx_flags = 0; 1601 decoder->state.flags &= ~INTEL_PT_IN_TX; 1602 decoder->state.flags |= INTEL_PT_ABORT_TX; 1603 } else { 1604 decoder->state.flags |= INTEL_PT_ASYNC; 1605 } 1606 1607 while (1) { 1608 err = intel_pt_get_next_packet(decoder); 1609 if (err) 1610 return err; 1611 1612 switch (decoder->packet.type) { 1613 case INTEL_PT_TNT: 1614 case INTEL_PT_FUP: 1615 case INTEL_PT_TRACESTOP: 1616 case INTEL_PT_PSB: 1617 case INTEL_PT_TSC: 1618 case INTEL_PT_TMA: 1619 case INTEL_PT_MODE_TSX: 1620 case INTEL_PT_BAD: 1621 case INTEL_PT_PSBEND: 1622 case INTEL_PT_PTWRITE: 1623 case INTEL_PT_PTWRITE_IP: 1624 case INTEL_PT_EXSTOP: 1625 case INTEL_PT_EXSTOP_IP: 1626 case INTEL_PT_MWAIT: 1627 case INTEL_PT_PWRE: 1628 case INTEL_PT_PWRX: 1629 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1630 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1631 decoder->pkt_step = 0; 1632 return -ENOENT; 1633 1634 case INTEL_PT_CBR: 1635 intel_pt_calc_cbr(decoder); 1636 break; 1637 1638 case INTEL_PT_OVF: 1639 return intel_pt_overflow(decoder); 1640 1641 case INTEL_PT_TIP_PGD: 1642 decoder->state.from_ip = decoder->ip; 1643 decoder->state.to_ip = 0; 1644 if (decoder->packet.count != 0) { 1645 intel_pt_set_ip(decoder); 1646 intel_pt_log("Omitting PGD ip " x64_fmt "\n", 1647 decoder->ip); 1648 } 1649 decoder->pge = false; 1650 decoder->continuous_period = false; 1651 return 0; 1652 1653 case INTEL_PT_TIP_PGE: 1654 decoder->pge = true; 1655 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1656 decoder->ip); 1657 decoder->state.from_ip = 0; 1658 if (decoder->packet.count == 0) { 1659 decoder->state.to_ip = 0; 1660 } else { 1661 intel_pt_set_ip(decoder); 1662 decoder->state.to_ip = decoder->ip; 1663 } 1664 return 0; 1665 1666 case INTEL_PT_TIP: 1667 decoder->state.from_ip = decoder->ip; 1668 if (decoder->packet.count == 0) { 1669 decoder->state.to_ip = 0; 1670 } else { 1671 intel_pt_set_ip(decoder); 1672 decoder->state.to_ip = decoder->ip; 1673 } 1674 return 0; 1675 1676 case INTEL_PT_PIP: 1677 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1678 break; 1679 1680 case INTEL_PT_MTC: 1681 intel_pt_calc_mtc_timestamp(decoder); 1682 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1683 decoder->state.type |= INTEL_PT_INSTRUCTION; 1684 break; 1685 1686 case INTEL_PT_CYC: 1687 intel_pt_calc_cyc_timestamp(decoder); 1688 break; 1689 1690 case INTEL_PT_MODE_EXEC: 1691 decoder->exec_mode = decoder->packet.payload; 1692 break; 1693 1694 case INTEL_PT_VMCS: 1695 case INTEL_PT_MNT: 1696 case INTEL_PT_PAD: 1697 break; 1698 1699 default: 1700 return intel_pt_bug(decoder); 1701 } 1702 } 1703 } 1704 1705 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 1706 { 1707 bool no_tip = false; 1708 int err; 1709 1710 while (1) { 1711 err = intel_pt_get_next_packet(decoder); 1712 if (err) 1713 return err; 1714 next: 1715 switch (decoder->packet.type) { 1716 case INTEL_PT_TNT: 1717 if (!decoder->packet.count) 1718 break; 1719 decoder->tnt = decoder->packet; 1720 decoder->pkt_state = INTEL_PT_STATE_TNT; 1721 err = intel_pt_walk_tnt(decoder); 1722 if (err == -EAGAIN) 1723 break; 1724 return err; 1725 1726 case INTEL_PT_TIP_PGD: 1727 if (decoder->packet.count != 0) 1728 intel_pt_set_last_ip(decoder); 1729 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 1730 return intel_pt_walk_tip(decoder); 1731 1732 case INTEL_PT_TIP_PGE: { 1733 decoder->pge = true; 1734 if (decoder->packet.count == 0) { 1735 intel_pt_log_at("Skipping zero TIP.PGE", 1736 decoder->pos); 1737 break; 1738 } 1739 intel_pt_set_ip(decoder); 1740 decoder->state.from_ip = 0; 1741 decoder->state.to_ip = decoder->ip; 1742 return 0; 1743 } 1744 1745 case INTEL_PT_OVF: 1746 return intel_pt_overflow(decoder); 1747 1748 case INTEL_PT_TIP: 1749 if (decoder->packet.count != 0) 1750 intel_pt_set_last_ip(decoder); 1751 decoder->pkt_state = INTEL_PT_STATE_TIP; 1752 return intel_pt_walk_tip(decoder); 1753 1754 case INTEL_PT_FUP: 1755 if (decoder->packet.count == 0) { 1756 intel_pt_log_at("Skipping zero FUP", 1757 decoder->pos); 1758 no_tip = false; 1759 break; 1760 } 1761 intel_pt_set_last_ip(decoder); 1762 if (!decoder->branch_enable) { 1763 decoder->ip = decoder->last_ip; 1764 if (intel_pt_fup_event(decoder)) 1765 return 0; 1766 no_tip = false; 1767 break; 1768 } 1769 if (decoder->set_fup_mwait) 1770 no_tip = true; 1771 err = intel_pt_walk_fup(decoder); 1772 if (err != -EAGAIN) { 1773 if (err) 1774 return err; 1775 if (no_tip) 1776 decoder->pkt_state = 1777 INTEL_PT_STATE_FUP_NO_TIP; 1778 else 1779 decoder->pkt_state = INTEL_PT_STATE_FUP; 1780 return 0; 1781 } 1782 if (no_tip) { 1783 no_tip = false; 1784 break; 1785 } 1786 return intel_pt_walk_fup_tip(decoder); 1787 1788 case INTEL_PT_TRACESTOP: 1789 decoder->pge = false; 1790 decoder->continuous_period = false; 1791 intel_pt_clear_tx_flags(decoder); 1792 decoder->have_tma = false; 1793 break; 1794 1795 case INTEL_PT_PSB: 1796 decoder->last_ip = 0; 1797 decoder->have_last_ip = true; 1798 intel_pt_clear_stack(&decoder->stack); 1799 err = intel_pt_walk_psbend(decoder); 1800 if (err == -EAGAIN) 1801 goto next; 1802 if (err) 1803 return err; 1804 break; 1805 1806 case INTEL_PT_PIP: 1807 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 1808 break; 1809 1810 case INTEL_PT_MTC: 1811 intel_pt_calc_mtc_timestamp(decoder); 1812 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 1813 break; 1814 /* 1815 * Ensure that there has been an instruction since the 1816 * last MTC. 1817 */ 1818 if (!decoder->mtc_insn) 1819 break; 1820 decoder->mtc_insn = false; 1821 /* Ensure that there is a timestamp */ 1822 if (!decoder->timestamp) 1823 break; 1824 decoder->state.type = INTEL_PT_INSTRUCTION; 1825 decoder->state.from_ip = decoder->ip; 1826 decoder->state.to_ip = 0; 1827 decoder->mtc_insn = false; 1828 return 0; 1829 1830 case INTEL_PT_TSC: 1831 intel_pt_calc_tsc_timestamp(decoder); 1832 break; 1833 1834 case INTEL_PT_TMA: 1835 intel_pt_calc_tma(decoder); 1836 break; 1837 1838 case INTEL_PT_CYC: 1839 intel_pt_calc_cyc_timestamp(decoder); 1840 break; 1841 1842 case INTEL_PT_CBR: 1843 intel_pt_calc_cbr(decoder); 1844 if (!decoder->branch_enable && 1845 decoder->cbr != decoder->cbr_seen) { 1846 decoder->cbr_seen = decoder->cbr; 1847 decoder->state.type = INTEL_PT_CBR_CHG; 1848 decoder->state.from_ip = decoder->ip; 1849 decoder->state.to_ip = 0; 1850 decoder->state.cbr_payload = 1851 decoder->packet.payload; 1852 return 0; 1853 } 1854 break; 1855 1856 case INTEL_PT_MODE_EXEC: 1857 decoder->exec_mode = decoder->packet.payload; 1858 break; 1859 1860 case INTEL_PT_MODE_TSX: 1861 /* MODE_TSX need not be followed by FUP */ 1862 if (!decoder->pge) { 1863 intel_pt_update_in_tx(decoder); 1864 break; 1865 } 1866 err = intel_pt_mode_tsx(decoder, &no_tip); 1867 if (err) 1868 return err; 1869 goto next; 1870 1871 case INTEL_PT_BAD: /* Does not happen */ 1872 return intel_pt_bug(decoder); 1873 1874 case INTEL_PT_PSBEND: 1875 case INTEL_PT_VMCS: 1876 case INTEL_PT_MNT: 1877 case INTEL_PT_PAD: 1878 break; 1879 1880 case INTEL_PT_PTWRITE_IP: 1881 decoder->fup_ptw_payload = decoder->packet.payload; 1882 err = intel_pt_get_next_packet(decoder); 1883 if (err) 1884 return err; 1885 if (decoder->packet.type == INTEL_PT_FUP) { 1886 decoder->set_fup_ptw = true; 1887 no_tip = true; 1888 } else { 1889 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 1890 decoder->pos); 1891 } 1892 goto next; 1893 1894 case INTEL_PT_PTWRITE: 1895 decoder->state.type = INTEL_PT_PTW; 1896 decoder->state.from_ip = decoder->ip; 1897 decoder->state.to_ip = 0; 1898 decoder->state.ptw_payload = decoder->packet.payload; 1899 return 0; 1900 1901 case INTEL_PT_MWAIT: 1902 decoder->fup_mwait_payload = decoder->packet.payload; 1903 decoder->set_fup_mwait = true; 1904 break; 1905 1906 case INTEL_PT_PWRE: 1907 if (decoder->set_fup_mwait) { 1908 decoder->fup_pwre_payload = 1909 decoder->packet.payload; 1910 decoder->set_fup_pwre = true; 1911 break; 1912 } 1913 decoder->state.type = INTEL_PT_PWR_ENTRY; 1914 decoder->state.from_ip = decoder->ip; 1915 decoder->state.to_ip = 0; 1916 decoder->state.pwrx_payload = decoder->packet.payload; 1917 return 0; 1918 1919 case INTEL_PT_EXSTOP_IP: 1920 err = intel_pt_get_next_packet(decoder); 1921 if (err) 1922 return err; 1923 if (decoder->packet.type == INTEL_PT_FUP) { 1924 decoder->set_fup_exstop = true; 1925 no_tip = true; 1926 } else { 1927 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 1928 decoder->pos); 1929 } 1930 goto next; 1931 1932 case INTEL_PT_EXSTOP: 1933 decoder->state.type = INTEL_PT_EX_STOP; 1934 decoder->state.from_ip = decoder->ip; 1935 decoder->state.to_ip = 0; 1936 return 0; 1937 1938 case INTEL_PT_PWRX: 1939 decoder->state.type = INTEL_PT_PWR_EXIT; 1940 decoder->state.from_ip = decoder->ip; 1941 decoder->state.to_ip = 0; 1942 decoder->state.pwrx_payload = decoder->packet.payload; 1943 return 0; 1944 1945 default: 1946 return intel_pt_bug(decoder); 1947 } 1948 } 1949 } 1950 1951 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 1952 { 1953 return decoder->packet.count && 1954 (decoder->have_last_ip || decoder->packet.count == 3 || 1955 decoder->packet.count == 6); 1956 } 1957 1958 /* Walk PSB+ packets to get in sync. */ 1959 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 1960 { 1961 int err; 1962 1963 while (1) { 1964 err = intel_pt_get_next_packet(decoder); 1965 if (err) 1966 return err; 1967 1968 switch (decoder->packet.type) { 1969 case INTEL_PT_TIP_PGD: 1970 decoder->continuous_period = false; 1971 __fallthrough; 1972 case INTEL_PT_TIP_PGE: 1973 case INTEL_PT_TIP: 1974 case INTEL_PT_PTWRITE: 1975 case INTEL_PT_PTWRITE_IP: 1976 case INTEL_PT_EXSTOP: 1977 case INTEL_PT_EXSTOP_IP: 1978 case INTEL_PT_MWAIT: 1979 case INTEL_PT_PWRE: 1980 case INTEL_PT_PWRX: 1981 intel_pt_log("ERROR: Unexpected packet\n"); 1982 return -ENOENT; 1983 1984 case INTEL_PT_FUP: 1985 decoder->pge = true; 1986 if (intel_pt_have_ip(decoder)) { 1987 uint64_t current_ip = decoder->ip; 1988 1989 intel_pt_set_ip(decoder); 1990 if (current_ip) 1991 intel_pt_log_to("Setting IP", 1992 decoder->ip); 1993 } 1994 break; 1995 1996 case INTEL_PT_MTC: 1997 intel_pt_calc_mtc_timestamp(decoder); 1998 break; 1999 2000 case INTEL_PT_TSC: 2001 intel_pt_calc_tsc_timestamp(decoder); 2002 break; 2003 2004 case INTEL_PT_TMA: 2005 intel_pt_calc_tma(decoder); 2006 break; 2007 2008 case INTEL_PT_CYC: 2009 intel_pt_calc_cyc_timestamp(decoder); 2010 break; 2011 2012 case INTEL_PT_CBR: 2013 intel_pt_calc_cbr(decoder); 2014 break; 2015 2016 case INTEL_PT_PIP: 2017 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2018 break; 2019 2020 case INTEL_PT_MODE_EXEC: 2021 decoder->exec_mode = decoder->packet.payload; 2022 break; 2023 2024 case INTEL_PT_MODE_TSX: 2025 intel_pt_update_in_tx(decoder); 2026 break; 2027 2028 case INTEL_PT_TRACESTOP: 2029 decoder->pge = false; 2030 decoder->continuous_period = false; 2031 intel_pt_clear_tx_flags(decoder); 2032 __fallthrough; 2033 2034 case INTEL_PT_TNT: 2035 decoder->have_tma = false; 2036 intel_pt_log("ERROR: Unexpected packet\n"); 2037 if (decoder->ip) 2038 decoder->pkt_state = INTEL_PT_STATE_ERR4; 2039 else 2040 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2041 return -ENOENT; 2042 2043 case INTEL_PT_BAD: /* Does not happen */ 2044 return intel_pt_bug(decoder); 2045 2046 case INTEL_PT_OVF: 2047 return intel_pt_overflow(decoder); 2048 2049 case INTEL_PT_PSBEND: 2050 return 0; 2051 2052 case INTEL_PT_PSB: 2053 case INTEL_PT_VMCS: 2054 case INTEL_PT_MNT: 2055 case INTEL_PT_PAD: 2056 default: 2057 break; 2058 } 2059 } 2060 } 2061 2062 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 2063 { 2064 int err; 2065 2066 while (1) { 2067 err = intel_pt_get_next_packet(decoder); 2068 if (err) 2069 return err; 2070 2071 switch (decoder->packet.type) { 2072 case INTEL_PT_TIP_PGD: 2073 decoder->continuous_period = false; 2074 __fallthrough; 2075 case INTEL_PT_TIP_PGE: 2076 case INTEL_PT_TIP: 2077 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD; 2078 if (intel_pt_have_ip(decoder)) 2079 intel_pt_set_ip(decoder); 2080 if (decoder->ip) 2081 return 0; 2082 break; 2083 2084 case INTEL_PT_FUP: 2085 if (intel_pt_have_ip(decoder)) 2086 intel_pt_set_ip(decoder); 2087 if (decoder->ip) 2088 return 0; 2089 break; 2090 2091 case INTEL_PT_MTC: 2092 intel_pt_calc_mtc_timestamp(decoder); 2093 break; 2094 2095 case INTEL_PT_TSC: 2096 intel_pt_calc_tsc_timestamp(decoder); 2097 break; 2098 2099 case INTEL_PT_TMA: 2100 intel_pt_calc_tma(decoder); 2101 break; 2102 2103 case INTEL_PT_CYC: 2104 intel_pt_calc_cyc_timestamp(decoder); 2105 break; 2106 2107 case INTEL_PT_CBR: 2108 intel_pt_calc_cbr(decoder); 2109 break; 2110 2111 case INTEL_PT_PIP: 2112 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); 2113 break; 2114 2115 case INTEL_PT_MODE_EXEC: 2116 decoder->exec_mode = decoder->packet.payload; 2117 break; 2118 2119 case INTEL_PT_MODE_TSX: 2120 intel_pt_update_in_tx(decoder); 2121 break; 2122 2123 case INTEL_PT_OVF: 2124 return intel_pt_overflow(decoder); 2125 2126 case INTEL_PT_BAD: /* Does not happen */ 2127 return intel_pt_bug(decoder); 2128 2129 case INTEL_PT_TRACESTOP: 2130 decoder->pge = false; 2131 decoder->continuous_period = false; 2132 intel_pt_clear_tx_flags(decoder); 2133 decoder->have_tma = false; 2134 break; 2135 2136 case INTEL_PT_PSB: 2137 decoder->last_ip = 0; 2138 decoder->have_last_ip = true; 2139 intel_pt_clear_stack(&decoder->stack); 2140 err = intel_pt_walk_psb(decoder); 2141 if (err) 2142 return err; 2143 if (decoder->ip) { 2144 /* Do not have a sample */ 2145 decoder->state.type = 0; 2146 return 0; 2147 } 2148 break; 2149 2150 case INTEL_PT_TNT: 2151 case INTEL_PT_PSBEND: 2152 case INTEL_PT_VMCS: 2153 case INTEL_PT_MNT: 2154 case INTEL_PT_PAD: 2155 case INTEL_PT_PTWRITE: 2156 case INTEL_PT_PTWRITE_IP: 2157 case INTEL_PT_EXSTOP: 2158 case INTEL_PT_EXSTOP_IP: 2159 case INTEL_PT_MWAIT: 2160 case INTEL_PT_PWRE: 2161 case INTEL_PT_PWRX: 2162 default: 2163 break; 2164 } 2165 } 2166 } 2167 2168 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 2169 { 2170 int err; 2171 2172 decoder->set_fup_tx_flags = false; 2173 decoder->set_fup_ptw = false; 2174 decoder->set_fup_mwait = false; 2175 decoder->set_fup_pwre = false; 2176 decoder->set_fup_exstop = false; 2177 2178 if (!decoder->branch_enable) { 2179 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2180 decoder->overflow = false; 2181 decoder->state.type = 0; /* Do not have a sample */ 2182 return 0; 2183 } 2184 2185 intel_pt_log("Scanning for full IP\n"); 2186 err = intel_pt_walk_to_ip(decoder); 2187 if (err) 2188 return err; 2189 2190 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2191 decoder->overflow = false; 2192 2193 decoder->state.from_ip = 0; 2194 decoder->state.to_ip = decoder->ip; 2195 intel_pt_log_to("Setting IP", decoder->ip); 2196 2197 return 0; 2198 } 2199 2200 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 2201 { 2202 const unsigned char *end = decoder->buf + decoder->len; 2203 size_t i; 2204 2205 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 2206 if (i > decoder->len) 2207 continue; 2208 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 2209 return i; 2210 } 2211 return 0; 2212 } 2213 2214 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 2215 { 2216 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 2217 const char *psb = INTEL_PT_PSB_STR; 2218 2219 if (rest_psb > decoder->len || 2220 memcmp(decoder->buf, psb + part_psb, rest_psb)) 2221 return 0; 2222 2223 return rest_psb; 2224 } 2225 2226 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 2227 int part_psb) 2228 { 2229 int rest_psb, ret; 2230 2231 decoder->pos += decoder->len; 2232 decoder->len = 0; 2233 2234 ret = intel_pt_get_next_data(decoder); 2235 if (ret) 2236 return ret; 2237 2238 rest_psb = intel_pt_rest_psb(decoder, part_psb); 2239 if (!rest_psb) 2240 return 0; 2241 2242 decoder->pos -= part_psb; 2243 decoder->next_buf = decoder->buf + rest_psb; 2244 decoder->next_len = decoder->len - rest_psb; 2245 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2246 decoder->buf = decoder->temp_buf; 2247 decoder->len = INTEL_PT_PSB_LEN; 2248 2249 return 0; 2250 } 2251 2252 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 2253 { 2254 unsigned char *next; 2255 int ret; 2256 2257 intel_pt_log("Scanning for PSB\n"); 2258 while (1) { 2259 if (!decoder->len) { 2260 ret = intel_pt_get_next_data(decoder); 2261 if (ret) 2262 return ret; 2263 } 2264 2265 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2266 INTEL_PT_PSB_LEN); 2267 if (!next) { 2268 int part_psb; 2269 2270 part_psb = intel_pt_part_psb(decoder); 2271 if (part_psb) { 2272 ret = intel_pt_get_split_psb(decoder, part_psb); 2273 if (ret) 2274 return ret; 2275 } else { 2276 decoder->pos += decoder->len; 2277 decoder->len = 0; 2278 } 2279 continue; 2280 } 2281 2282 decoder->pkt_step = next - decoder->buf; 2283 return intel_pt_get_next_packet(decoder); 2284 } 2285 } 2286 2287 static int intel_pt_sync(struct intel_pt_decoder *decoder) 2288 { 2289 int err; 2290 2291 decoder->pge = false; 2292 decoder->continuous_period = false; 2293 decoder->have_last_ip = false; 2294 decoder->last_ip = 0; 2295 decoder->ip = 0; 2296 intel_pt_clear_stack(&decoder->stack); 2297 2298 err = intel_pt_scan_for_psb(decoder); 2299 if (err) 2300 return err; 2301 2302 decoder->have_last_ip = true; 2303 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 2304 2305 err = intel_pt_walk_psb(decoder); 2306 if (err) 2307 return err; 2308 2309 if (decoder->ip) { 2310 decoder->state.type = 0; /* Do not have a sample */ 2311 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2312 } else { 2313 return intel_pt_sync_ip(decoder); 2314 } 2315 2316 return 0; 2317 } 2318 2319 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2320 { 2321 uint64_t est = decoder->sample_insn_cnt << 1; 2322 2323 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2324 goto out; 2325 2326 est *= decoder->max_non_turbo_ratio; 2327 est /= decoder->cbr; 2328 out: 2329 return decoder->sample_timestamp + est; 2330 } 2331 2332 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2333 { 2334 int err; 2335 2336 do { 2337 decoder->state.type = INTEL_PT_BRANCH; 2338 decoder->state.flags = 0; 2339 2340 switch (decoder->pkt_state) { 2341 case INTEL_PT_STATE_NO_PSB: 2342 err = intel_pt_sync(decoder); 2343 break; 2344 case INTEL_PT_STATE_NO_IP: 2345 decoder->have_last_ip = false; 2346 decoder->last_ip = 0; 2347 decoder->ip = 0; 2348 __fallthrough; 2349 case INTEL_PT_STATE_ERR_RESYNC: 2350 err = intel_pt_sync_ip(decoder); 2351 break; 2352 case INTEL_PT_STATE_IN_SYNC: 2353 err = intel_pt_walk_trace(decoder); 2354 break; 2355 case INTEL_PT_STATE_TNT: 2356 err = intel_pt_walk_tnt(decoder); 2357 if (err == -EAGAIN) 2358 err = intel_pt_walk_trace(decoder); 2359 break; 2360 case INTEL_PT_STATE_TIP: 2361 case INTEL_PT_STATE_TIP_PGD: 2362 err = intel_pt_walk_tip(decoder); 2363 break; 2364 case INTEL_PT_STATE_FUP: 2365 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2366 err = intel_pt_walk_fup(decoder); 2367 if (err == -EAGAIN) 2368 err = intel_pt_walk_fup_tip(decoder); 2369 else if (!err) 2370 decoder->pkt_state = INTEL_PT_STATE_FUP; 2371 break; 2372 case INTEL_PT_STATE_FUP_NO_TIP: 2373 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2374 err = intel_pt_walk_fup(decoder); 2375 if (err == -EAGAIN) 2376 err = intel_pt_walk_trace(decoder); 2377 break; 2378 default: 2379 err = intel_pt_bug(decoder); 2380 break; 2381 } 2382 } while (err == -ENOLINK); 2383 2384 if (err) { 2385 decoder->state.err = intel_pt_ext_err(err); 2386 decoder->state.from_ip = decoder->ip; 2387 decoder->sample_timestamp = decoder->timestamp; 2388 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 2389 } else { 2390 decoder->state.err = 0; 2391 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) { 2392 decoder->cbr_seen = decoder->cbr; 2393 decoder->state.type |= INTEL_PT_CBR_CHG; 2394 decoder->state.cbr_payload = decoder->cbr_payload; 2395 } 2396 if (intel_pt_sample_time(decoder->pkt_state)) { 2397 decoder->sample_timestamp = decoder->timestamp; 2398 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 2399 } 2400 } 2401 2402 decoder->state.timestamp = decoder->sample_timestamp; 2403 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 2404 decoder->state.cr3 = decoder->cr3; 2405 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 2406 2407 return &decoder->state; 2408 } 2409 2410 /** 2411 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 2412 * @buf: pointer to buffer pointer 2413 * @len: size of buffer 2414 * 2415 * Updates the buffer pointer to point to the start of the next PSB packet if 2416 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 2417 * @len is adjusted accordingly. 2418 * 2419 * Return: %true if a PSB packet is found, %false otherwise. 2420 */ 2421 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 2422 { 2423 unsigned char *next; 2424 2425 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2426 if (next) { 2427 *len -= next - *buf; 2428 *buf = next; 2429 return true; 2430 } 2431 return false; 2432 } 2433 2434 /** 2435 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 2436 * packet. 2437 * @buf: pointer to buffer pointer 2438 * @len: size of buffer 2439 * 2440 * Updates the buffer pointer to point to the start of the following PSB packet 2441 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 2442 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 2443 * 2444 * Return: %true if a PSB packet is found, %false otherwise. 2445 */ 2446 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 2447 { 2448 unsigned char *next; 2449 2450 if (!*len) 2451 return false; 2452 2453 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2454 if (next) { 2455 *len -= next - *buf; 2456 *buf = next; 2457 return true; 2458 } 2459 return false; 2460 } 2461 2462 /** 2463 * intel_pt_last_psb - find the last PSB packet in a buffer. 2464 * @buf: buffer 2465 * @len: size of buffer 2466 * 2467 * This function finds the last PSB in a buffer. 2468 * 2469 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 2470 */ 2471 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 2472 { 2473 const char *n = INTEL_PT_PSB_STR; 2474 unsigned char *p; 2475 size_t k; 2476 2477 if (len < INTEL_PT_PSB_LEN) 2478 return NULL; 2479 2480 k = len - INTEL_PT_PSB_LEN + 1; 2481 while (1) { 2482 p = memrchr(buf, n[0], k); 2483 if (!p) 2484 return NULL; 2485 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 2486 return p; 2487 k = p - buf; 2488 if (!k) 2489 return NULL; 2490 } 2491 } 2492 2493 /** 2494 * intel_pt_next_tsc - find and return next TSC. 2495 * @buf: buffer 2496 * @len: size of buffer 2497 * @tsc: TSC value returned 2498 * @rem: returns remaining size when TSC is found 2499 * 2500 * Find a TSC packet in @buf and return the TSC value. This function assumes 2501 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 2502 * PSBEND packet is found. 2503 * 2504 * Return: %true if TSC is found, false otherwise. 2505 */ 2506 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 2507 size_t *rem) 2508 { 2509 struct intel_pt_pkt packet; 2510 int ret; 2511 2512 while (len) { 2513 ret = intel_pt_get_packet(buf, len, &packet); 2514 if (ret <= 0) 2515 return false; 2516 if (packet.type == INTEL_PT_TSC) { 2517 *tsc = packet.payload; 2518 *rem = len; 2519 return true; 2520 } 2521 if (packet.type == INTEL_PT_PSBEND) 2522 return false; 2523 buf += ret; 2524 len -= ret; 2525 } 2526 return false; 2527 } 2528 2529 /** 2530 * intel_pt_tsc_cmp - compare 7-byte TSCs. 2531 * @tsc1: first TSC to compare 2532 * @tsc2: second TSC to compare 2533 * 2534 * This function compares 7-byte TSC values allowing for the possibility that 2535 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 2536 * around so for that purpose this function assumes the absolute difference is 2537 * less than half the maximum difference. 2538 * 2539 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 2540 * after @tsc2. 2541 */ 2542 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 2543 { 2544 const uint64_t halfway = (1ULL << 55); 2545 2546 if (tsc1 == tsc2) 2547 return 0; 2548 2549 if (tsc1 < tsc2) { 2550 if (tsc2 - tsc1 < halfway) 2551 return -1; 2552 else 2553 return 1; 2554 } else { 2555 if (tsc1 - tsc2 < halfway) 2556 return 1; 2557 else 2558 return -1; 2559 } 2560 } 2561 2562 /** 2563 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 2564 * using TSC. 2565 * @buf_a: first buffer 2566 * @len_a: size of first buffer 2567 * @buf_b: second buffer 2568 * @len_b: size of second buffer 2569 * @consecutive: returns true if there is data in buf_b that is consecutive 2570 * to buf_a 2571 * 2572 * If the trace contains TSC we can look at the last TSC of @buf_a and the 2573 * first TSC of @buf_b in order to determine if the buffers overlap, and then 2574 * walk forward in @buf_b until a later TSC is found. A precondition is that 2575 * @buf_a and @buf_b are positioned at a PSB. 2576 * 2577 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2578 * @buf_b + @len_b if there is no non-overlapped data. 2579 */ 2580 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 2581 size_t len_a, 2582 unsigned char *buf_b, 2583 size_t len_b, bool *consecutive) 2584 { 2585 uint64_t tsc_a, tsc_b; 2586 unsigned char *p; 2587 size_t len, rem_a, rem_b; 2588 2589 p = intel_pt_last_psb(buf_a, len_a); 2590 if (!p) 2591 return buf_b; /* No PSB in buf_a => no overlap */ 2592 2593 len = len_a - (p - buf_a); 2594 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 2595 /* The last PSB+ in buf_a is incomplete, so go back one more */ 2596 len_a -= len; 2597 p = intel_pt_last_psb(buf_a, len_a); 2598 if (!p) 2599 return buf_b; /* No full PSB+ => assume no overlap */ 2600 len = len_a - (p - buf_a); 2601 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 2602 return buf_b; /* No TSC in buf_a => assume no overlap */ 2603 } 2604 2605 while (1) { 2606 /* Ignore PSB+ with no TSC */ 2607 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 2608 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 2609 2610 /* Same TSC, so buffers are consecutive */ 2611 if (!cmp && rem_b >= rem_a) { 2612 *consecutive = true; 2613 return buf_b + len_b - (rem_b - rem_a); 2614 } 2615 if (cmp < 0) 2616 return buf_b; /* tsc_a < tsc_b => no overlap */ 2617 } 2618 2619 if (!intel_pt_step_psb(&buf_b, &len_b)) 2620 return buf_b + len_b; /* No PSB in buf_b => no data */ 2621 } 2622 } 2623 2624 /** 2625 * intel_pt_find_overlap - determine start of non-overlapped trace data. 2626 * @buf_a: first buffer 2627 * @len_a: size of first buffer 2628 * @buf_b: second buffer 2629 * @len_b: size of second buffer 2630 * @have_tsc: can use TSC packets to detect overlap 2631 * @consecutive: returns true if there is data in buf_b that is consecutive 2632 * to buf_a 2633 * 2634 * When trace samples or snapshots are recorded there is the possibility that 2635 * the data overlaps. Note that, for the purposes of decoding, data is only 2636 * useful if it begins with a PSB packet. 2637 * 2638 * Return: A pointer into @buf_b from where non-overlapped data starts, or 2639 * @buf_b + @len_b if there is no non-overlapped data. 2640 */ 2641 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 2642 unsigned char *buf_b, size_t len_b, 2643 bool have_tsc, bool *consecutive) 2644 { 2645 unsigned char *found; 2646 2647 /* Buffer 'b' must start at PSB so throw away everything before that */ 2648 if (!intel_pt_next_psb(&buf_b, &len_b)) 2649 return buf_b + len_b; /* No PSB */ 2650 2651 if (!intel_pt_next_psb(&buf_a, &len_a)) 2652 return buf_b; /* No overlap */ 2653 2654 if (have_tsc) { 2655 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 2656 consecutive); 2657 if (found) 2658 return found; 2659 } 2660 2661 /* 2662 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 2663 * we can ignore the first part of buffer 'a'. 2664 */ 2665 while (len_b < len_a) { 2666 if (!intel_pt_step_psb(&buf_a, &len_a)) 2667 return buf_b; /* No overlap */ 2668 } 2669 2670 /* Now len_b >= len_a */ 2671 while (1) { 2672 /* Potential overlap so check the bytes */ 2673 found = memmem(buf_a, len_a, buf_b, len_a); 2674 if (found) { 2675 *consecutive = true; 2676 return buf_b + len_a; 2677 } 2678 2679 /* Try again at next PSB in buffer 'a' */ 2680 if (!intel_pt_step_psb(&buf_a, &len_a)) 2681 return buf_b; /* No overlap */ 2682 } 2683 } 2684