1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt.c: Intel Processor Trace support 4 * Copyright (c) 2013-2015, Intel Corporation. 5 */ 6 7 #include <errno.h> 8 #include <stdbool.h> 9 #include <linux/kernel.h> 10 #include <linux/types.h> 11 #include <linux/bitops.h> 12 #include <linux/log2.h> 13 #include <linux/zalloc.h> 14 #include <cpuid.h> 15 16 #include "../../perf.h" 17 #include "../../util/session.h" 18 #include "../../util/event.h" 19 #include "../../util/evlist.h" 20 #include "../../util/evsel.h" 21 #include "../../util/cpumap.h" 22 #include <subcmd/parse-options.h> 23 #include "../../util/parse-events.h" 24 #include "../../util/pmu.h" 25 #include "../../util/debug.h" 26 #include "../../util/auxtrace.h" 27 #include "../../util/tsc.h" 28 #include "../../util/intel-pt.h" 29 30 #define KiB(x) ((x) * 1024) 31 #define MiB(x) ((x) * 1024 * 1024) 32 #define KiB_MASK(x) (KiB(x) - 1) 33 #define MiB_MASK(x) (MiB(x) - 1) 34 35 #define INTEL_PT_PSB_PERIOD_NEAR 256 36 37 struct intel_pt_snapshot_ref { 38 void *ref_buf; 39 size_t ref_offset; 40 bool wrapped; 41 }; 42 43 struct intel_pt_recording { 44 struct auxtrace_record itr; 45 struct perf_pmu *intel_pt_pmu; 46 int have_sched_switch; 47 struct perf_evlist *evlist; 48 bool snapshot_mode; 49 bool snapshot_init_done; 50 size_t snapshot_size; 51 size_t snapshot_ref_buf_size; 52 int snapshot_ref_cnt; 53 struct intel_pt_snapshot_ref *snapshot_refs; 54 size_t priv_size; 55 }; 56 57 static int intel_pt_parse_terms_with_default(struct list_head *formats, 58 const char *str, 59 u64 *config) 60 { 61 struct list_head *terms; 62 struct perf_event_attr attr = { .size = 0, }; 63 int err; 64 65 terms = malloc(sizeof(struct list_head)); 66 if (!terms) 67 return -ENOMEM; 68 69 INIT_LIST_HEAD(terms); 70 71 err = parse_events_terms(terms, str); 72 if (err) 73 goto out_free; 74 75 attr.config = *config; 76 err = perf_pmu__config_terms(formats, &attr, terms, true, NULL); 77 if (err) 78 goto out_free; 79 80 *config = attr.config; 81 out_free: 82 parse_events_terms__delete(terms); 83 return err; 84 } 85 86 static int intel_pt_parse_terms(struct list_head *formats, const char *str, 87 u64 *config) 88 { 89 *config = 0; 90 return intel_pt_parse_terms_with_default(formats, str, config); 91 } 92 93 static u64 intel_pt_masked_bits(u64 mask, u64 bits) 94 { 95 const u64 top_bit = 1ULL << 63; 96 u64 res = 0; 97 int i; 98 99 for (i = 0; i < 64; i++) { 100 if (mask & top_bit) { 101 res <<= 1; 102 if (bits & top_bit) 103 res |= 1; 104 } 105 mask <<= 1; 106 bits <<= 1; 107 } 108 109 return res; 110 } 111 112 static int intel_pt_read_config(struct perf_pmu *intel_pt_pmu, const char *str, 113 struct perf_evlist *evlist, u64 *res) 114 { 115 struct perf_evsel *evsel; 116 u64 mask; 117 118 *res = 0; 119 120 mask = perf_pmu__format_bits(&intel_pt_pmu->format, str); 121 if (!mask) 122 return -EINVAL; 123 124 evlist__for_each_entry(evlist, evsel) { 125 if (evsel->attr.type == intel_pt_pmu->type) { 126 *res = intel_pt_masked_bits(mask, evsel->attr.config); 127 return 0; 128 } 129 } 130 131 return -EINVAL; 132 } 133 134 static size_t intel_pt_psb_period(struct perf_pmu *intel_pt_pmu, 135 struct perf_evlist *evlist) 136 { 137 u64 val; 138 int err, topa_multiple_entries; 139 size_t psb_period; 140 141 if (perf_pmu__scan_file(intel_pt_pmu, "caps/topa_multiple_entries", 142 "%d", &topa_multiple_entries) != 1) 143 topa_multiple_entries = 0; 144 145 /* 146 * Use caps/topa_multiple_entries to indicate early hardware that had 147 * extra frequent PSBs. 148 */ 149 if (!topa_multiple_entries) { 150 psb_period = 256; 151 goto out; 152 } 153 154 err = intel_pt_read_config(intel_pt_pmu, "psb_period", evlist, &val); 155 if (err) 156 val = 0; 157 158 psb_period = 1 << (val + 11); 159 out: 160 pr_debug2("%s psb_period %zu\n", intel_pt_pmu->name, psb_period); 161 return psb_period; 162 } 163 164 static int intel_pt_pick_bit(int bits, int target) 165 { 166 int pos, pick = -1; 167 168 for (pos = 0; bits; bits >>= 1, pos++) { 169 if (bits & 1) { 170 if (pos <= target || pick < 0) 171 pick = pos; 172 if (pos >= target) 173 break; 174 } 175 } 176 177 return pick; 178 } 179 180 static u64 intel_pt_default_config(struct perf_pmu *intel_pt_pmu) 181 { 182 char buf[256]; 183 int mtc, mtc_periods = 0, mtc_period; 184 int psb_cyc, psb_periods, psb_period; 185 int pos = 0; 186 u64 config; 187 char c; 188 189 pos += scnprintf(buf + pos, sizeof(buf) - pos, "tsc"); 190 191 if (perf_pmu__scan_file(intel_pt_pmu, "caps/mtc", "%d", 192 &mtc) != 1) 193 mtc = 1; 194 195 if (mtc) { 196 if (perf_pmu__scan_file(intel_pt_pmu, "caps/mtc_periods", "%x", 197 &mtc_periods) != 1) 198 mtc_periods = 0; 199 if (mtc_periods) { 200 mtc_period = intel_pt_pick_bit(mtc_periods, 3); 201 pos += scnprintf(buf + pos, sizeof(buf) - pos, 202 ",mtc,mtc_period=%d", mtc_period); 203 } 204 } 205 206 if (perf_pmu__scan_file(intel_pt_pmu, "caps/psb_cyc", "%d", 207 &psb_cyc) != 1) 208 psb_cyc = 1; 209 210 if (psb_cyc && mtc_periods) { 211 if (perf_pmu__scan_file(intel_pt_pmu, "caps/psb_periods", "%x", 212 &psb_periods) != 1) 213 psb_periods = 0; 214 if (psb_periods) { 215 psb_period = intel_pt_pick_bit(psb_periods, 3); 216 pos += scnprintf(buf + pos, sizeof(buf) - pos, 217 ",psb_period=%d", psb_period); 218 } 219 } 220 221 if (perf_pmu__scan_file(intel_pt_pmu, "format/pt", "%c", &c) == 1 && 222 perf_pmu__scan_file(intel_pt_pmu, "format/branch", "%c", &c) == 1) 223 pos += scnprintf(buf + pos, sizeof(buf) - pos, ",pt,branch"); 224 225 pr_debug2("%s default config: %s\n", intel_pt_pmu->name, buf); 226 227 intel_pt_parse_terms(&intel_pt_pmu->format, buf, &config); 228 229 return config; 230 } 231 232 static int intel_pt_parse_snapshot_options(struct auxtrace_record *itr, 233 struct record_opts *opts, 234 const char *str) 235 { 236 struct intel_pt_recording *ptr = 237 container_of(itr, struct intel_pt_recording, itr); 238 unsigned long long snapshot_size = 0; 239 char *endptr; 240 241 if (str) { 242 snapshot_size = strtoull(str, &endptr, 0); 243 if (*endptr || snapshot_size > SIZE_MAX) 244 return -1; 245 } 246 247 opts->auxtrace_snapshot_mode = true; 248 opts->auxtrace_snapshot_size = snapshot_size; 249 250 ptr->snapshot_size = snapshot_size; 251 252 return 0; 253 } 254 255 struct perf_event_attr * 256 intel_pt_pmu_default_config(struct perf_pmu *intel_pt_pmu) 257 { 258 struct perf_event_attr *attr; 259 260 attr = zalloc(sizeof(struct perf_event_attr)); 261 if (!attr) 262 return NULL; 263 264 attr->config = intel_pt_default_config(intel_pt_pmu); 265 266 intel_pt_pmu->selectable = true; 267 268 return attr; 269 } 270 271 static const char *intel_pt_find_filter(struct perf_evlist *evlist, 272 struct perf_pmu *intel_pt_pmu) 273 { 274 struct perf_evsel *evsel; 275 276 evlist__for_each_entry(evlist, evsel) { 277 if (evsel->attr.type == intel_pt_pmu->type) 278 return evsel->filter; 279 } 280 281 return NULL; 282 } 283 284 static size_t intel_pt_filter_bytes(const char *filter) 285 { 286 size_t len = filter ? strlen(filter) : 0; 287 288 return len ? roundup(len + 1, 8) : 0; 289 } 290 291 static size_t 292 intel_pt_info_priv_size(struct auxtrace_record *itr, struct perf_evlist *evlist) 293 { 294 struct intel_pt_recording *ptr = 295 container_of(itr, struct intel_pt_recording, itr); 296 const char *filter = intel_pt_find_filter(evlist, ptr->intel_pt_pmu); 297 298 ptr->priv_size = (INTEL_PT_AUXTRACE_PRIV_MAX * sizeof(u64)) + 299 intel_pt_filter_bytes(filter); 300 301 return ptr->priv_size; 302 } 303 304 static void intel_pt_tsc_ctc_ratio(u32 *n, u32 *d) 305 { 306 unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; 307 308 __get_cpuid(0x15, &eax, &ebx, &ecx, &edx); 309 *n = ebx; 310 *d = eax; 311 } 312 313 static int intel_pt_info_fill(struct auxtrace_record *itr, 314 struct perf_session *session, 315 struct auxtrace_info_event *auxtrace_info, 316 size_t priv_size) 317 { 318 struct intel_pt_recording *ptr = 319 container_of(itr, struct intel_pt_recording, itr); 320 struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; 321 struct perf_event_mmap_page *pc; 322 struct perf_tsc_conversion tc = { .time_mult = 0, }; 323 bool cap_user_time_zero = false, per_cpu_mmaps; 324 u64 tsc_bit, mtc_bit, mtc_freq_bits, cyc_bit, noretcomp_bit; 325 u32 tsc_ctc_ratio_n, tsc_ctc_ratio_d; 326 unsigned long max_non_turbo_ratio; 327 size_t filter_str_len; 328 const char *filter; 329 u64 *info; 330 int err; 331 332 if (priv_size != ptr->priv_size) 333 return -EINVAL; 334 335 intel_pt_parse_terms(&intel_pt_pmu->format, "tsc", &tsc_bit); 336 intel_pt_parse_terms(&intel_pt_pmu->format, "noretcomp", 337 &noretcomp_bit); 338 intel_pt_parse_terms(&intel_pt_pmu->format, "mtc", &mtc_bit); 339 mtc_freq_bits = perf_pmu__format_bits(&intel_pt_pmu->format, 340 "mtc_period"); 341 intel_pt_parse_terms(&intel_pt_pmu->format, "cyc", &cyc_bit); 342 343 intel_pt_tsc_ctc_ratio(&tsc_ctc_ratio_n, &tsc_ctc_ratio_d); 344 345 if (perf_pmu__scan_file(intel_pt_pmu, "max_nonturbo_ratio", 346 "%lu", &max_non_turbo_ratio) != 1) 347 max_non_turbo_ratio = 0; 348 349 filter = intel_pt_find_filter(session->evlist, ptr->intel_pt_pmu); 350 filter_str_len = filter ? strlen(filter) : 0; 351 352 if (!session->evlist->nr_mmaps) 353 return -EINVAL; 354 355 pc = session->evlist->mmap[0].base; 356 if (pc) { 357 err = perf_read_tsc_conversion(pc, &tc); 358 if (err) { 359 if (err != -EOPNOTSUPP) 360 return err; 361 } else { 362 cap_user_time_zero = tc.time_mult != 0; 363 } 364 if (!cap_user_time_zero) 365 ui__warning("Intel Processor Trace: TSC not available\n"); 366 } 367 368 per_cpu_mmaps = !cpu_map__empty(session->evlist->cpus); 369 370 auxtrace_info->type = PERF_AUXTRACE_INTEL_PT; 371 auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type; 372 auxtrace_info->priv[INTEL_PT_TIME_SHIFT] = tc.time_shift; 373 auxtrace_info->priv[INTEL_PT_TIME_MULT] = tc.time_mult; 374 auxtrace_info->priv[INTEL_PT_TIME_ZERO] = tc.time_zero; 375 auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO] = cap_user_time_zero; 376 auxtrace_info->priv[INTEL_PT_TSC_BIT] = tsc_bit; 377 auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT] = noretcomp_bit; 378 auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH] = ptr->have_sched_switch; 379 auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE] = ptr->snapshot_mode; 380 auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS] = per_cpu_mmaps; 381 auxtrace_info->priv[INTEL_PT_MTC_BIT] = mtc_bit; 382 auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS] = mtc_freq_bits; 383 auxtrace_info->priv[INTEL_PT_TSC_CTC_N] = tsc_ctc_ratio_n; 384 auxtrace_info->priv[INTEL_PT_TSC_CTC_D] = tsc_ctc_ratio_d; 385 auxtrace_info->priv[INTEL_PT_CYC_BIT] = cyc_bit; 386 auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO] = max_non_turbo_ratio; 387 auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] = filter_str_len; 388 389 info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1; 390 391 if (filter_str_len) { 392 size_t len = intel_pt_filter_bytes(filter); 393 394 strncpy((char *)info, filter, len); 395 info += len >> 3; 396 } 397 398 return 0; 399 } 400 401 static int intel_pt_track_switches(struct perf_evlist *evlist) 402 { 403 const char *sched_switch = "sched:sched_switch"; 404 struct perf_evsel *evsel; 405 int err; 406 407 if (!perf_evlist__can_select_event(evlist, sched_switch)) 408 return -EPERM; 409 410 err = parse_events(evlist, sched_switch, NULL); 411 if (err) { 412 pr_debug2("%s: failed to parse %s, error %d\n", 413 __func__, sched_switch, err); 414 return err; 415 } 416 417 evsel = perf_evlist__last(evlist); 418 419 perf_evsel__set_sample_bit(evsel, CPU); 420 perf_evsel__set_sample_bit(evsel, TIME); 421 422 evsel->system_wide = true; 423 evsel->no_aux_samples = true; 424 evsel->immediate = true; 425 426 return 0; 427 } 428 429 static void intel_pt_valid_str(char *str, size_t len, u64 valid) 430 { 431 unsigned int val, last = 0, state = 1; 432 int p = 0; 433 434 str[0] = '\0'; 435 436 for (val = 0; val <= 64; val++, valid >>= 1) { 437 if (valid & 1) { 438 last = val; 439 switch (state) { 440 case 0: 441 p += scnprintf(str + p, len - p, ","); 442 /* Fall through */ 443 case 1: 444 p += scnprintf(str + p, len - p, "%u", val); 445 state = 2; 446 break; 447 case 2: 448 state = 3; 449 break; 450 case 3: 451 state = 4; 452 break; 453 default: 454 break; 455 } 456 } else { 457 switch (state) { 458 case 3: 459 p += scnprintf(str + p, len - p, ",%u", last); 460 state = 0; 461 break; 462 case 4: 463 p += scnprintf(str + p, len - p, "-%u", last); 464 state = 0; 465 break; 466 default: 467 break; 468 } 469 if (state != 1) 470 state = 0; 471 } 472 } 473 } 474 475 static int intel_pt_val_config_term(struct perf_pmu *intel_pt_pmu, 476 const char *caps, const char *name, 477 const char *supported, u64 config) 478 { 479 char valid_str[256]; 480 unsigned int shift; 481 unsigned long long valid; 482 u64 bits; 483 int ok; 484 485 if (perf_pmu__scan_file(intel_pt_pmu, caps, "%llx", &valid) != 1) 486 valid = 0; 487 488 if (supported && 489 perf_pmu__scan_file(intel_pt_pmu, supported, "%d", &ok) == 1 && !ok) 490 valid = 0; 491 492 valid |= 1; 493 494 bits = perf_pmu__format_bits(&intel_pt_pmu->format, name); 495 496 config &= bits; 497 498 for (shift = 0; bits && !(bits & 1); shift++) 499 bits >>= 1; 500 501 config >>= shift; 502 503 if (config > 63) 504 goto out_err; 505 506 if (valid & (1 << config)) 507 return 0; 508 out_err: 509 intel_pt_valid_str(valid_str, sizeof(valid_str), valid); 510 pr_err("Invalid %s for %s. Valid values are: %s\n", 511 name, INTEL_PT_PMU_NAME, valid_str); 512 return -EINVAL; 513 } 514 515 static int intel_pt_validate_config(struct perf_pmu *intel_pt_pmu, 516 struct perf_evsel *evsel) 517 { 518 int err; 519 char c; 520 521 if (!evsel) 522 return 0; 523 524 /* 525 * If supported, force pass-through config term (pt=1) even if user 526 * sets pt=0, which avoids senseless kernel errors. 527 */ 528 if (perf_pmu__scan_file(intel_pt_pmu, "format/pt", "%c", &c) == 1 && 529 !(evsel->attr.config & 1)) { 530 pr_warning("pt=0 doesn't make sense, forcing pt=1\n"); 531 evsel->attr.config |= 1; 532 } 533 534 err = intel_pt_val_config_term(intel_pt_pmu, "caps/cycle_thresholds", 535 "cyc_thresh", "caps/psb_cyc", 536 evsel->attr.config); 537 if (err) 538 return err; 539 540 err = intel_pt_val_config_term(intel_pt_pmu, "caps/mtc_periods", 541 "mtc_period", "caps/mtc", 542 evsel->attr.config); 543 if (err) 544 return err; 545 546 return intel_pt_val_config_term(intel_pt_pmu, "caps/psb_periods", 547 "psb_period", "caps/psb_cyc", 548 evsel->attr.config); 549 } 550 551 static int intel_pt_recording_options(struct auxtrace_record *itr, 552 struct perf_evlist *evlist, 553 struct record_opts *opts) 554 { 555 struct intel_pt_recording *ptr = 556 container_of(itr, struct intel_pt_recording, itr); 557 struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; 558 bool have_timing_info, need_immediate = false; 559 struct perf_evsel *evsel, *intel_pt_evsel = NULL; 560 const struct cpu_map *cpus = evlist->cpus; 561 bool privileged = geteuid() == 0 || perf_event_paranoid() < 0; 562 u64 tsc_bit; 563 int err; 564 565 ptr->evlist = evlist; 566 ptr->snapshot_mode = opts->auxtrace_snapshot_mode; 567 568 evlist__for_each_entry(evlist, evsel) { 569 if (evsel->attr.type == intel_pt_pmu->type) { 570 if (intel_pt_evsel) { 571 pr_err("There may be only one " INTEL_PT_PMU_NAME " event\n"); 572 return -EINVAL; 573 } 574 evsel->attr.freq = 0; 575 evsel->attr.sample_period = 1; 576 intel_pt_evsel = evsel; 577 opts->full_auxtrace = true; 578 } 579 } 580 581 if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) { 582 pr_err("Snapshot mode (-S option) requires " INTEL_PT_PMU_NAME " PMU event (-e " INTEL_PT_PMU_NAME ")\n"); 583 return -EINVAL; 584 } 585 586 if (opts->use_clockid) { 587 pr_err("Cannot use clockid (-k option) with " INTEL_PT_PMU_NAME "\n"); 588 return -EINVAL; 589 } 590 591 if (!opts->full_auxtrace) 592 return 0; 593 594 err = intel_pt_validate_config(intel_pt_pmu, intel_pt_evsel); 595 if (err) 596 return err; 597 598 /* Set default sizes for snapshot mode */ 599 if (opts->auxtrace_snapshot_mode) { 600 size_t psb_period = intel_pt_psb_period(intel_pt_pmu, evlist); 601 602 if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) { 603 if (privileged) { 604 opts->auxtrace_mmap_pages = MiB(4) / page_size; 605 } else { 606 opts->auxtrace_mmap_pages = KiB(128) / page_size; 607 if (opts->mmap_pages == UINT_MAX) 608 opts->mmap_pages = KiB(256) / page_size; 609 } 610 } else if (!opts->auxtrace_mmap_pages && !privileged && 611 opts->mmap_pages == UINT_MAX) { 612 opts->mmap_pages = KiB(256) / page_size; 613 } 614 if (!opts->auxtrace_snapshot_size) 615 opts->auxtrace_snapshot_size = 616 opts->auxtrace_mmap_pages * (size_t)page_size; 617 if (!opts->auxtrace_mmap_pages) { 618 size_t sz = opts->auxtrace_snapshot_size; 619 620 sz = round_up(sz, page_size) / page_size; 621 opts->auxtrace_mmap_pages = roundup_pow_of_two(sz); 622 } 623 if (opts->auxtrace_snapshot_size > 624 opts->auxtrace_mmap_pages * (size_t)page_size) { 625 pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n", 626 opts->auxtrace_snapshot_size, 627 opts->auxtrace_mmap_pages * (size_t)page_size); 628 return -EINVAL; 629 } 630 if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) { 631 pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n"); 632 return -EINVAL; 633 } 634 pr_debug2("Intel PT snapshot size: %zu\n", 635 opts->auxtrace_snapshot_size); 636 if (psb_period && 637 opts->auxtrace_snapshot_size <= psb_period + 638 INTEL_PT_PSB_PERIOD_NEAR) 639 ui__warning("Intel PT snapshot size (%zu) may be too small for PSB period (%zu)\n", 640 opts->auxtrace_snapshot_size, psb_period); 641 } 642 643 /* Set default sizes for full trace mode */ 644 if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) { 645 if (privileged) { 646 opts->auxtrace_mmap_pages = MiB(4) / page_size; 647 } else { 648 opts->auxtrace_mmap_pages = KiB(128) / page_size; 649 if (opts->mmap_pages == UINT_MAX) 650 opts->mmap_pages = KiB(256) / page_size; 651 } 652 } 653 654 /* Validate auxtrace_mmap_pages */ 655 if (opts->auxtrace_mmap_pages) { 656 size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size; 657 size_t min_sz; 658 659 if (opts->auxtrace_snapshot_mode) 660 min_sz = KiB(4); 661 else 662 min_sz = KiB(8); 663 664 if (sz < min_sz || !is_power_of_2(sz)) { 665 pr_err("Invalid mmap size for Intel Processor Trace: must be at least %zuKiB and a power of 2\n", 666 min_sz / 1024); 667 return -EINVAL; 668 } 669 } 670 671 intel_pt_parse_terms(&intel_pt_pmu->format, "tsc", &tsc_bit); 672 673 if (opts->full_auxtrace && (intel_pt_evsel->attr.config & tsc_bit)) 674 have_timing_info = true; 675 else 676 have_timing_info = false; 677 678 /* 679 * Per-cpu recording needs sched_switch events to distinguish different 680 * threads. 681 */ 682 if (have_timing_info && !cpu_map__empty(cpus)) { 683 if (perf_can_record_switch_events()) { 684 bool cpu_wide = !target__none(&opts->target) && 685 !target__has_task(&opts->target); 686 687 if (!cpu_wide && perf_can_record_cpu_wide()) { 688 struct perf_evsel *switch_evsel; 689 690 err = parse_events(evlist, "dummy:u", NULL); 691 if (err) 692 return err; 693 694 switch_evsel = perf_evlist__last(evlist); 695 696 switch_evsel->attr.freq = 0; 697 switch_evsel->attr.sample_period = 1; 698 switch_evsel->attr.context_switch = 1; 699 700 switch_evsel->system_wide = true; 701 switch_evsel->no_aux_samples = true; 702 switch_evsel->immediate = true; 703 704 perf_evsel__set_sample_bit(switch_evsel, TID); 705 perf_evsel__set_sample_bit(switch_evsel, TIME); 706 perf_evsel__set_sample_bit(switch_evsel, CPU); 707 perf_evsel__reset_sample_bit(switch_evsel, BRANCH_STACK); 708 709 opts->record_switch_events = false; 710 ptr->have_sched_switch = 3; 711 } else { 712 opts->record_switch_events = true; 713 need_immediate = true; 714 if (cpu_wide) 715 ptr->have_sched_switch = 3; 716 else 717 ptr->have_sched_switch = 2; 718 } 719 } else { 720 err = intel_pt_track_switches(evlist); 721 if (err == -EPERM) 722 pr_debug2("Unable to select sched:sched_switch\n"); 723 else if (err) 724 return err; 725 else 726 ptr->have_sched_switch = 1; 727 } 728 } 729 730 if (intel_pt_evsel) { 731 /* 732 * To obtain the auxtrace buffer file descriptor, the auxtrace 733 * event must come first. 734 */ 735 perf_evlist__to_front(evlist, intel_pt_evsel); 736 /* 737 * In the case of per-cpu mmaps, we need the CPU on the 738 * AUX event. 739 */ 740 if (!cpu_map__empty(cpus)) 741 perf_evsel__set_sample_bit(intel_pt_evsel, CPU); 742 } 743 744 /* Add dummy event to keep tracking */ 745 if (opts->full_auxtrace) { 746 struct perf_evsel *tracking_evsel; 747 748 err = parse_events(evlist, "dummy:u", NULL); 749 if (err) 750 return err; 751 752 tracking_evsel = perf_evlist__last(evlist); 753 754 perf_evlist__set_tracking_event(evlist, tracking_evsel); 755 756 tracking_evsel->attr.freq = 0; 757 tracking_evsel->attr.sample_period = 1; 758 759 tracking_evsel->no_aux_samples = true; 760 if (need_immediate) 761 tracking_evsel->immediate = true; 762 763 /* In per-cpu case, always need the time of mmap events etc */ 764 if (!cpu_map__empty(cpus)) { 765 perf_evsel__set_sample_bit(tracking_evsel, TIME); 766 /* And the CPU for switch events */ 767 perf_evsel__set_sample_bit(tracking_evsel, CPU); 768 } 769 perf_evsel__reset_sample_bit(tracking_evsel, BRANCH_STACK); 770 } 771 772 /* 773 * Warn the user when we do not have enough information to decode i.e. 774 * per-cpu with no sched_switch (except workload-only). 775 */ 776 if (!ptr->have_sched_switch && !cpu_map__empty(cpus) && 777 !target__none(&opts->target)) 778 ui__warning("Intel Processor Trace decoding will not be possible except for kernel tracing!\n"); 779 780 return 0; 781 } 782 783 static int intel_pt_snapshot_start(struct auxtrace_record *itr) 784 { 785 struct intel_pt_recording *ptr = 786 container_of(itr, struct intel_pt_recording, itr); 787 struct perf_evsel *evsel; 788 789 evlist__for_each_entry(ptr->evlist, evsel) { 790 if (evsel->attr.type == ptr->intel_pt_pmu->type) 791 return perf_evsel__disable(evsel); 792 } 793 return -EINVAL; 794 } 795 796 static int intel_pt_snapshot_finish(struct auxtrace_record *itr) 797 { 798 struct intel_pt_recording *ptr = 799 container_of(itr, struct intel_pt_recording, itr); 800 struct perf_evsel *evsel; 801 802 evlist__for_each_entry(ptr->evlist, evsel) { 803 if (evsel->attr.type == ptr->intel_pt_pmu->type) 804 return perf_evsel__enable(evsel); 805 } 806 return -EINVAL; 807 } 808 809 static int intel_pt_alloc_snapshot_refs(struct intel_pt_recording *ptr, int idx) 810 { 811 const size_t sz = sizeof(struct intel_pt_snapshot_ref); 812 int cnt = ptr->snapshot_ref_cnt, new_cnt = cnt * 2; 813 struct intel_pt_snapshot_ref *refs; 814 815 if (!new_cnt) 816 new_cnt = 16; 817 818 while (new_cnt <= idx) 819 new_cnt *= 2; 820 821 refs = calloc(new_cnt, sz); 822 if (!refs) 823 return -ENOMEM; 824 825 memcpy(refs, ptr->snapshot_refs, cnt * sz); 826 827 ptr->snapshot_refs = refs; 828 ptr->snapshot_ref_cnt = new_cnt; 829 830 return 0; 831 } 832 833 static void intel_pt_free_snapshot_refs(struct intel_pt_recording *ptr) 834 { 835 int i; 836 837 for (i = 0; i < ptr->snapshot_ref_cnt; i++) 838 zfree(&ptr->snapshot_refs[i].ref_buf); 839 zfree(&ptr->snapshot_refs); 840 } 841 842 static void intel_pt_recording_free(struct auxtrace_record *itr) 843 { 844 struct intel_pt_recording *ptr = 845 container_of(itr, struct intel_pt_recording, itr); 846 847 intel_pt_free_snapshot_refs(ptr); 848 free(ptr); 849 } 850 851 static int intel_pt_alloc_snapshot_ref(struct intel_pt_recording *ptr, int idx, 852 size_t snapshot_buf_size) 853 { 854 size_t ref_buf_size = ptr->snapshot_ref_buf_size; 855 void *ref_buf; 856 857 ref_buf = zalloc(ref_buf_size); 858 if (!ref_buf) 859 return -ENOMEM; 860 861 ptr->snapshot_refs[idx].ref_buf = ref_buf; 862 ptr->snapshot_refs[idx].ref_offset = snapshot_buf_size - ref_buf_size; 863 864 return 0; 865 } 866 867 static size_t intel_pt_snapshot_ref_buf_size(struct intel_pt_recording *ptr, 868 size_t snapshot_buf_size) 869 { 870 const size_t max_size = 256 * 1024; 871 size_t buf_size = 0, psb_period; 872 873 if (ptr->snapshot_size <= 64 * 1024) 874 return 0; 875 876 psb_period = intel_pt_psb_period(ptr->intel_pt_pmu, ptr->evlist); 877 if (psb_period) 878 buf_size = psb_period * 2; 879 880 if (!buf_size || buf_size > max_size) 881 buf_size = max_size; 882 883 if (buf_size >= snapshot_buf_size) 884 return 0; 885 886 if (buf_size >= ptr->snapshot_size / 2) 887 return 0; 888 889 return buf_size; 890 } 891 892 static int intel_pt_snapshot_init(struct intel_pt_recording *ptr, 893 size_t snapshot_buf_size) 894 { 895 if (ptr->snapshot_init_done) 896 return 0; 897 898 ptr->snapshot_init_done = true; 899 900 ptr->snapshot_ref_buf_size = intel_pt_snapshot_ref_buf_size(ptr, 901 snapshot_buf_size); 902 903 return 0; 904 } 905 906 /** 907 * intel_pt_compare_buffers - compare bytes in a buffer to a circular buffer. 908 * @buf1: first buffer 909 * @compare_size: number of bytes to compare 910 * @buf2: second buffer (a circular buffer) 911 * @offs2: offset in second buffer 912 * @buf2_size: size of second buffer 913 * 914 * The comparison allows for the possibility that the bytes to compare in the 915 * circular buffer are not contiguous. It is assumed that @compare_size <= 916 * @buf2_size. This function returns %false if the bytes are identical, %true 917 * otherwise. 918 */ 919 static bool intel_pt_compare_buffers(void *buf1, size_t compare_size, 920 void *buf2, size_t offs2, size_t buf2_size) 921 { 922 size_t end2 = offs2 + compare_size, part_size; 923 924 if (end2 <= buf2_size) 925 return memcmp(buf1, buf2 + offs2, compare_size); 926 927 part_size = end2 - buf2_size; 928 if (memcmp(buf1, buf2 + offs2, part_size)) 929 return true; 930 931 compare_size -= part_size; 932 933 return memcmp(buf1 + part_size, buf2, compare_size); 934 } 935 936 static bool intel_pt_compare_ref(void *ref_buf, size_t ref_offset, 937 size_t ref_size, size_t buf_size, 938 void *data, size_t head) 939 { 940 size_t ref_end = ref_offset + ref_size; 941 942 if (ref_end > buf_size) { 943 if (head > ref_offset || head < ref_end - buf_size) 944 return true; 945 } else if (head > ref_offset && head < ref_end) { 946 return true; 947 } 948 949 return intel_pt_compare_buffers(ref_buf, ref_size, data, ref_offset, 950 buf_size); 951 } 952 953 static void intel_pt_copy_ref(void *ref_buf, size_t ref_size, size_t buf_size, 954 void *data, size_t head) 955 { 956 if (head >= ref_size) { 957 memcpy(ref_buf, data + head - ref_size, ref_size); 958 } else { 959 memcpy(ref_buf, data, head); 960 ref_size -= head; 961 memcpy(ref_buf + head, data + buf_size - ref_size, ref_size); 962 } 963 } 964 965 static bool intel_pt_wrapped(struct intel_pt_recording *ptr, int idx, 966 struct auxtrace_mmap *mm, unsigned char *data, 967 u64 head) 968 { 969 struct intel_pt_snapshot_ref *ref = &ptr->snapshot_refs[idx]; 970 bool wrapped; 971 972 wrapped = intel_pt_compare_ref(ref->ref_buf, ref->ref_offset, 973 ptr->snapshot_ref_buf_size, mm->len, 974 data, head); 975 976 intel_pt_copy_ref(ref->ref_buf, ptr->snapshot_ref_buf_size, mm->len, 977 data, head); 978 979 return wrapped; 980 } 981 982 static bool intel_pt_first_wrap(u64 *data, size_t buf_size) 983 { 984 int i, a, b; 985 986 b = buf_size >> 3; 987 a = b - 512; 988 if (a < 0) 989 a = 0; 990 991 for (i = a; i < b; i++) { 992 if (data[i]) 993 return true; 994 } 995 996 return false; 997 } 998 999 static int intel_pt_find_snapshot(struct auxtrace_record *itr, int idx, 1000 struct auxtrace_mmap *mm, unsigned char *data, 1001 u64 *head, u64 *old) 1002 { 1003 struct intel_pt_recording *ptr = 1004 container_of(itr, struct intel_pt_recording, itr); 1005 bool wrapped; 1006 int err; 1007 1008 pr_debug3("%s: mmap index %d old head %zu new head %zu\n", 1009 __func__, idx, (size_t)*old, (size_t)*head); 1010 1011 err = intel_pt_snapshot_init(ptr, mm->len); 1012 if (err) 1013 goto out_err; 1014 1015 if (idx >= ptr->snapshot_ref_cnt) { 1016 err = intel_pt_alloc_snapshot_refs(ptr, idx); 1017 if (err) 1018 goto out_err; 1019 } 1020 1021 if (ptr->snapshot_ref_buf_size) { 1022 if (!ptr->snapshot_refs[idx].ref_buf) { 1023 err = intel_pt_alloc_snapshot_ref(ptr, idx, mm->len); 1024 if (err) 1025 goto out_err; 1026 } 1027 wrapped = intel_pt_wrapped(ptr, idx, mm, data, *head); 1028 } else { 1029 wrapped = ptr->snapshot_refs[idx].wrapped; 1030 if (!wrapped && intel_pt_first_wrap((u64 *)data, mm->len)) { 1031 ptr->snapshot_refs[idx].wrapped = true; 1032 wrapped = true; 1033 } 1034 } 1035 1036 /* 1037 * In full trace mode 'head' continually increases. However in snapshot 1038 * mode 'head' is an offset within the buffer. Here 'old' and 'head' 1039 * are adjusted to match the full trace case which expects that 'old' is 1040 * always less than 'head'. 1041 */ 1042 if (wrapped) { 1043 *old = *head; 1044 *head += mm->len; 1045 } else { 1046 if (mm->mask) 1047 *old &= mm->mask; 1048 else 1049 *old %= mm->len; 1050 if (*old > *head) 1051 *head += mm->len; 1052 } 1053 1054 pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n", 1055 __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head); 1056 1057 return 0; 1058 1059 out_err: 1060 pr_err("%s: failed, error %d\n", __func__, err); 1061 return err; 1062 } 1063 1064 static u64 intel_pt_reference(struct auxtrace_record *itr __maybe_unused) 1065 { 1066 return rdtsc(); 1067 } 1068 1069 static int intel_pt_read_finish(struct auxtrace_record *itr, int idx) 1070 { 1071 struct intel_pt_recording *ptr = 1072 container_of(itr, struct intel_pt_recording, itr); 1073 struct perf_evsel *evsel; 1074 1075 evlist__for_each_entry(ptr->evlist, evsel) { 1076 if (evsel->attr.type == ptr->intel_pt_pmu->type) 1077 return perf_evlist__enable_event_idx(ptr->evlist, evsel, 1078 idx); 1079 } 1080 return -EINVAL; 1081 } 1082 1083 struct auxtrace_record *intel_pt_recording_init(int *err) 1084 { 1085 struct perf_pmu *intel_pt_pmu = perf_pmu__find(INTEL_PT_PMU_NAME); 1086 struct intel_pt_recording *ptr; 1087 1088 if (!intel_pt_pmu) 1089 return NULL; 1090 1091 if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) { 1092 *err = -errno; 1093 return NULL; 1094 } 1095 1096 ptr = zalloc(sizeof(struct intel_pt_recording)); 1097 if (!ptr) { 1098 *err = -ENOMEM; 1099 return NULL; 1100 } 1101 1102 ptr->intel_pt_pmu = intel_pt_pmu; 1103 ptr->itr.recording_options = intel_pt_recording_options; 1104 ptr->itr.info_priv_size = intel_pt_info_priv_size; 1105 ptr->itr.info_fill = intel_pt_info_fill; 1106 ptr->itr.free = intel_pt_recording_free; 1107 ptr->itr.snapshot_start = intel_pt_snapshot_start; 1108 ptr->itr.snapshot_finish = intel_pt_snapshot_finish; 1109 ptr->itr.find_snapshot = intel_pt_find_snapshot; 1110 ptr->itr.parse_snapshot_options = intel_pt_parse_snapshot_options; 1111 ptr->itr.reference = intel_pt_reference; 1112 ptr->itr.read_finish = intel_pt_read_finish; 1113 return &ptr->itr; 1114 } 1115