1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched/mm.h> 24 #include <linux/sched/clock.h> 25 #include <linux/uaccess.h> 26 #include <linux/slab.h> 27 #include <linux/cpu.h> 28 #include <linux/bitops.h> 29 #include <linux/device.h> 30 #include <linux/nospec.h> 31 #include <linux/static_call.h> 32 33 #include <asm/apic.h> 34 #include <asm/stacktrace.h> 35 #include <asm/nmi.h> 36 #include <asm/smp.h> 37 #include <asm/alternative.h> 38 #include <asm/mmu_context.h> 39 #include <asm/tlbflush.h> 40 #include <asm/timer.h> 41 #include <asm/desc.h> 42 #include <asm/ldt.h> 43 #include <asm/unwind.h> 44 45 #include "perf_event.h" 46 47 struct x86_pmu x86_pmu __read_mostly; 48 static struct pmu pmu; 49 50 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 51 .enabled = 1, 52 .pmu = &pmu, 53 }; 54 55 DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key); 56 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); 57 DEFINE_STATIC_KEY_FALSE(perf_is_hybrid); 58 59 /* 60 * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined 61 * from just a typename, as opposed to an actual function. 62 */ 63 DEFINE_STATIC_CALL_NULL(x86_pmu_handle_irq, *x86_pmu.handle_irq); 64 DEFINE_STATIC_CALL_NULL(x86_pmu_disable_all, *x86_pmu.disable_all); 65 DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all, *x86_pmu.enable_all); 66 DEFINE_STATIC_CALL_NULL(x86_pmu_enable, *x86_pmu.enable); 67 DEFINE_STATIC_CALL_NULL(x86_pmu_disable, *x86_pmu.disable); 68 69 DEFINE_STATIC_CALL_NULL(x86_pmu_add, *x86_pmu.add); 70 DEFINE_STATIC_CALL_NULL(x86_pmu_del, *x86_pmu.del); 71 DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read); 72 73 DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events, *x86_pmu.schedule_events); 74 DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints); 75 DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints); 76 77 DEFINE_STATIC_CALL_NULL(x86_pmu_start_scheduling, *x86_pmu.start_scheduling); 78 DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling); 79 DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling); 80 81 DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task); 82 DEFINE_STATIC_CALL_NULL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx); 83 84 DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); 85 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases); 86 87 /* 88 * This one is magic, it will get called even when PMU init fails (because 89 * there is no PMU), in which case it should simply return NULL. 90 */ 91 DEFINE_STATIC_CALL_RET0(x86_pmu_guest_get_msrs, *x86_pmu.guest_get_msrs); 92 93 u64 __read_mostly hw_cache_event_ids 94 [PERF_COUNT_HW_CACHE_MAX] 95 [PERF_COUNT_HW_CACHE_OP_MAX] 96 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 97 u64 __read_mostly hw_cache_extra_regs 98 [PERF_COUNT_HW_CACHE_MAX] 99 [PERF_COUNT_HW_CACHE_OP_MAX] 100 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 101 102 /* 103 * Propagate event elapsed time into the generic event. 104 * Can only be executed on the CPU where the event is active. 105 * Returns the delta events processed. 106 */ 107 u64 x86_perf_event_update(struct perf_event *event) 108 { 109 struct hw_perf_event *hwc = &event->hw; 110 int shift = 64 - x86_pmu.cntval_bits; 111 u64 prev_raw_count, new_raw_count; 112 u64 delta; 113 114 if (unlikely(!hwc->event_base)) 115 return 0; 116 117 if (unlikely(is_topdown_count(event)) && x86_pmu.update_topdown_event) 118 return x86_pmu.update_topdown_event(event); 119 120 /* 121 * Careful: an NMI might modify the previous event value. 122 * 123 * Our tactic to handle this is to first atomically read and 124 * exchange a new raw count - then add that new-prev delta 125 * count to the generic event atomically: 126 */ 127 again: 128 prev_raw_count = local64_read(&hwc->prev_count); 129 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 130 131 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 132 new_raw_count) != prev_raw_count) 133 goto again; 134 135 /* 136 * Now we have the new raw value and have updated the prev 137 * timestamp already. We can now calculate the elapsed delta 138 * (event-)time and add that to the generic event. 139 * 140 * Careful, not all hw sign-extends above the physical width 141 * of the count. 142 */ 143 delta = (new_raw_count << shift) - (prev_raw_count << shift); 144 delta >>= shift; 145 146 local64_add(delta, &event->count); 147 local64_sub(delta, &hwc->period_left); 148 149 return new_raw_count; 150 } 151 152 /* 153 * Find and validate any extra registers to set up. 154 */ 155 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 156 { 157 struct extra_reg *extra_regs = hybrid(event->pmu, extra_regs); 158 struct hw_perf_event_extra *reg; 159 struct extra_reg *er; 160 161 reg = &event->hw.extra_reg; 162 163 if (!extra_regs) 164 return 0; 165 166 for (er = extra_regs; er->msr; er++) { 167 if (er->event != (config & er->config_mask)) 168 continue; 169 if (event->attr.config1 & ~er->valid_mask) 170 return -EINVAL; 171 /* Check if the extra msrs can be safely accessed*/ 172 if (!er->extra_msr_access) 173 return -ENXIO; 174 175 reg->idx = er->idx; 176 reg->config = event->attr.config1; 177 reg->reg = er->msr; 178 break; 179 } 180 return 0; 181 } 182 183 static atomic_t active_events; 184 static atomic_t pmc_refcount; 185 static DEFINE_MUTEX(pmc_reserve_mutex); 186 187 #ifdef CONFIG_X86_LOCAL_APIC 188 189 static inline int get_possible_num_counters(void) 190 { 191 int i, num_counters = x86_pmu.num_counters; 192 193 if (!is_hybrid()) 194 return num_counters; 195 196 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) 197 num_counters = max_t(int, num_counters, x86_pmu.hybrid_pmu[i].num_counters); 198 199 return num_counters; 200 } 201 202 static bool reserve_pmc_hardware(void) 203 { 204 int i, num_counters = get_possible_num_counters(); 205 206 for (i = 0; i < num_counters; i++) { 207 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 208 goto perfctr_fail; 209 } 210 211 for (i = 0; i < num_counters; i++) { 212 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 213 goto eventsel_fail; 214 } 215 216 return true; 217 218 eventsel_fail: 219 for (i--; i >= 0; i--) 220 release_evntsel_nmi(x86_pmu_config_addr(i)); 221 222 i = num_counters; 223 224 perfctr_fail: 225 for (i--; i >= 0; i--) 226 release_perfctr_nmi(x86_pmu_event_addr(i)); 227 228 return false; 229 } 230 231 static void release_pmc_hardware(void) 232 { 233 int i, num_counters = get_possible_num_counters(); 234 235 for (i = 0; i < num_counters; i++) { 236 release_perfctr_nmi(x86_pmu_event_addr(i)); 237 release_evntsel_nmi(x86_pmu_config_addr(i)); 238 } 239 } 240 241 #else 242 243 static bool reserve_pmc_hardware(void) { return true; } 244 static void release_pmc_hardware(void) {} 245 246 #endif 247 248 bool check_hw_exists(struct pmu *pmu, int num_counters, int num_counters_fixed) 249 { 250 u64 val, val_fail = -1, val_new= ~0; 251 int i, reg, reg_fail = -1, ret = 0; 252 int bios_fail = 0; 253 int reg_safe = -1; 254 255 /* 256 * Check to see if the BIOS enabled any of the counters, if so 257 * complain and bail. 258 */ 259 for (i = 0; i < num_counters; i++) { 260 reg = x86_pmu_config_addr(i); 261 ret = rdmsrl_safe(reg, &val); 262 if (ret) 263 goto msr_fail; 264 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 265 bios_fail = 1; 266 val_fail = val; 267 reg_fail = reg; 268 } else { 269 reg_safe = i; 270 } 271 } 272 273 if (num_counters_fixed) { 274 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 275 ret = rdmsrl_safe(reg, &val); 276 if (ret) 277 goto msr_fail; 278 for (i = 0; i < num_counters_fixed; i++) { 279 if (fixed_counter_disabled(i, pmu)) 280 continue; 281 if (val & (0x03ULL << i*4)) { 282 bios_fail = 1; 283 val_fail = val; 284 reg_fail = reg; 285 } 286 } 287 } 288 289 /* 290 * If all the counters are enabled, the below test will always 291 * fail. The tools will also become useless in this scenario. 292 * Just fail and disable the hardware counters. 293 */ 294 295 if (reg_safe == -1) { 296 reg = reg_safe; 297 goto msr_fail; 298 } 299 300 /* 301 * Read the current value, change it and read it back to see if it 302 * matches, this is needed to detect certain hardware emulators 303 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 304 */ 305 reg = x86_pmu_event_addr(reg_safe); 306 if (rdmsrl_safe(reg, &val)) 307 goto msr_fail; 308 val ^= 0xffffUL; 309 ret = wrmsrl_safe(reg, val); 310 ret |= rdmsrl_safe(reg, &val_new); 311 if (ret || val != val_new) 312 goto msr_fail; 313 314 /* 315 * We still allow the PMU driver to operate: 316 */ 317 if (bios_fail) { 318 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 319 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 320 reg_fail, val_fail); 321 } 322 323 return true; 324 325 msr_fail: 326 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 327 pr_cont("PMU not available due to virtualization, using software events only.\n"); 328 } else { 329 pr_cont("Broken PMU hardware detected, using software events only.\n"); 330 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 331 reg, val_new); 332 } 333 334 return false; 335 } 336 337 static void hw_perf_event_destroy(struct perf_event *event) 338 { 339 x86_release_hardware(); 340 atomic_dec(&active_events); 341 } 342 343 void hw_perf_lbr_event_destroy(struct perf_event *event) 344 { 345 hw_perf_event_destroy(event); 346 347 /* undo the lbr/bts event accounting */ 348 x86_del_exclusive(x86_lbr_exclusive_lbr); 349 } 350 351 static inline int x86_pmu_initialized(void) 352 { 353 return x86_pmu.handle_irq != NULL; 354 } 355 356 static inline int 357 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 358 { 359 struct perf_event_attr *attr = &event->attr; 360 unsigned int cache_type, cache_op, cache_result; 361 u64 config, val; 362 363 config = attr->config; 364 365 cache_type = (config >> 0) & 0xff; 366 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 367 return -EINVAL; 368 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX); 369 370 cache_op = (config >> 8) & 0xff; 371 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 372 return -EINVAL; 373 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX); 374 375 cache_result = (config >> 16) & 0xff; 376 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 377 return -EINVAL; 378 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX); 379 380 val = hybrid_var(event->pmu, hw_cache_event_ids)[cache_type][cache_op][cache_result]; 381 if (val == 0) 382 return -ENOENT; 383 384 if (val == -1) 385 return -EINVAL; 386 387 hwc->config |= val; 388 attr->config1 = hybrid_var(event->pmu, hw_cache_extra_regs)[cache_type][cache_op][cache_result]; 389 return x86_pmu_extra_regs(val, event); 390 } 391 392 int x86_reserve_hardware(void) 393 { 394 int err = 0; 395 396 if (!atomic_inc_not_zero(&pmc_refcount)) { 397 mutex_lock(&pmc_reserve_mutex); 398 if (atomic_read(&pmc_refcount) == 0) { 399 if (!reserve_pmc_hardware()) { 400 err = -EBUSY; 401 } else { 402 reserve_ds_buffers(); 403 reserve_lbr_buffers(); 404 } 405 } 406 if (!err) 407 atomic_inc(&pmc_refcount); 408 mutex_unlock(&pmc_reserve_mutex); 409 } 410 411 return err; 412 } 413 414 void x86_release_hardware(void) 415 { 416 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 417 release_pmc_hardware(); 418 release_ds_buffers(); 419 release_lbr_buffers(); 420 mutex_unlock(&pmc_reserve_mutex); 421 } 422 } 423 424 /* 425 * Check if we can create event of a certain type (that no conflicting events 426 * are present). 427 */ 428 int x86_add_exclusive(unsigned int what) 429 { 430 int i; 431 432 /* 433 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 434 * LBR and BTS are still mutually exclusive. 435 */ 436 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 437 goto out; 438 439 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 440 mutex_lock(&pmc_reserve_mutex); 441 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 442 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 443 goto fail_unlock; 444 } 445 atomic_inc(&x86_pmu.lbr_exclusive[what]); 446 mutex_unlock(&pmc_reserve_mutex); 447 } 448 449 out: 450 atomic_inc(&active_events); 451 return 0; 452 453 fail_unlock: 454 mutex_unlock(&pmc_reserve_mutex); 455 return -EBUSY; 456 } 457 458 void x86_del_exclusive(unsigned int what) 459 { 460 atomic_dec(&active_events); 461 462 /* 463 * See the comment in x86_add_exclusive(). 464 */ 465 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 466 return; 467 468 atomic_dec(&x86_pmu.lbr_exclusive[what]); 469 } 470 471 int x86_setup_perfctr(struct perf_event *event) 472 { 473 struct perf_event_attr *attr = &event->attr; 474 struct hw_perf_event *hwc = &event->hw; 475 u64 config; 476 477 if (!is_sampling_event(event)) { 478 hwc->sample_period = x86_pmu.max_period; 479 hwc->last_period = hwc->sample_period; 480 local64_set(&hwc->period_left, hwc->sample_period); 481 } 482 483 if (attr->type == event->pmu->type) 484 return x86_pmu_extra_regs(event->attr.config, event); 485 486 if (attr->type == PERF_TYPE_HW_CACHE) 487 return set_ext_hw_attr(hwc, event); 488 489 if (attr->config >= x86_pmu.max_events) 490 return -EINVAL; 491 492 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); 493 494 /* 495 * The generic map: 496 */ 497 config = x86_pmu.event_map(attr->config); 498 499 if (config == 0) 500 return -ENOENT; 501 502 if (config == -1LL) 503 return -EINVAL; 504 505 hwc->config |= config; 506 507 return 0; 508 } 509 510 /* 511 * check that branch_sample_type is compatible with 512 * settings needed for precise_ip > 1 which implies 513 * using the LBR to capture ALL taken branches at the 514 * priv levels of the measurement 515 */ 516 static inline int precise_br_compat(struct perf_event *event) 517 { 518 u64 m = event->attr.branch_sample_type; 519 u64 b = 0; 520 521 /* must capture all branches */ 522 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 523 return 0; 524 525 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 526 527 if (!event->attr.exclude_user) 528 b |= PERF_SAMPLE_BRANCH_USER; 529 530 if (!event->attr.exclude_kernel) 531 b |= PERF_SAMPLE_BRANCH_KERNEL; 532 533 /* 534 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 535 */ 536 537 return m == b; 538 } 539 540 int x86_pmu_max_precise(void) 541 { 542 int precise = 0; 543 544 /* Support for constant skid */ 545 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 546 precise++; 547 548 /* Support for IP fixup */ 549 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 550 precise++; 551 552 if (x86_pmu.pebs_prec_dist) 553 precise++; 554 } 555 return precise; 556 } 557 558 int x86_pmu_hw_config(struct perf_event *event) 559 { 560 if (event->attr.precise_ip) { 561 int precise = x86_pmu_max_precise(); 562 563 if (event->attr.precise_ip > precise) 564 return -EOPNOTSUPP; 565 566 /* There's no sense in having PEBS for non sampling events: */ 567 if (!is_sampling_event(event)) 568 return -EINVAL; 569 } 570 /* 571 * check that PEBS LBR correction does not conflict with 572 * whatever the user is asking with attr->branch_sample_type 573 */ 574 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 575 u64 *br_type = &event->attr.branch_sample_type; 576 577 if (has_branch_stack(event)) { 578 if (!precise_br_compat(event)) 579 return -EOPNOTSUPP; 580 581 /* branch_sample_type is compatible */ 582 583 } else { 584 /* 585 * user did not specify branch_sample_type 586 * 587 * For PEBS fixups, we capture all 588 * the branches at the priv level of the 589 * event. 590 */ 591 *br_type = PERF_SAMPLE_BRANCH_ANY; 592 593 if (!event->attr.exclude_user) 594 *br_type |= PERF_SAMPLE_BRANCH_USER; 595 596 if (!event->attr.exclude_kernel) 597 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 598 } 599 } 600 601 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 602 event->attach_state |= PERF_ATTACH_TASK_DATA; 603 604 /* 605 * Generate PMC IRQs: 606 * (keep 'enabled' bit clear for now) 607 */ 608 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 609 610 /* 611 * Count user and OS events unless requested not to 612 */ 613 if (!event->attr.exclude_user) 614 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 615 if (!event->attr.exclude_kernel) 616 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 617 618 if (event->attr.type == event->pmu->type) 619 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 620 621 if (event->attr.sample_period && x86_pmu.limit_period) { 622 if (x86_pmu.limit_period(event, event->attr.sample_period) > 623 event->attr.sample_period) 624 return -EINVAL; 625 } 626 627 /* sample_regs_user never support XMM registers */ 628 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) 629 return -EINVAL; 630 /* 631 * Besides the general purpose registers, XMM registers may 632 * be collected in PEBS on some platforms, e.g. Icelake 633 */ 634 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) { 635 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) 636 return -EINVAL; 637 638 if (!event->attr.precise_ip) 639 return -EINVAL; 640 } 641 642 return x86_setup_perfctr(event); 643 } 644 645 /* 646 * Setup the hardware configuration for a given attr_type 647 */ 648 static int __x86_pmu_event_init(struct perf_event *event) 649 { 650 int err; 651 652 if (!x86_pmu_initialized()) 653 return -ENODEV; 654 655 err = x86_reserve_hardware(); 656 if (err) 657 return err; 658 659 atomic_inc(&active_events); 660 event->destroy = hw_perf_event_destroy; 661 662 event->hw.idx = -1; 663 event->hw.last_cpu = -1; 664 event->hw.last_tag = ~0ULL; 665 666 /* mark unused */ 667 event->hw.extra_reg.idx = EXTRA_REG_NONE; 668 event->hw.branch_reg.idx = EXTRA_REG_NONE; 669 670 return x86_pmu.hw_config(event); 671 } 672 673 void x86_pmu_disable_all(void) 674 { 675 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 676 int idx; 677 678 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 679 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 680 u64 val; 681 682 if (!test_bit(idx, cpuc->active_mask)) 683 continue; 684 rdmsrl(x86_pmu_config_addr(idx), val); 685 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 686 continue; 687 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 688 wrmsrl(x86_pmu_config_addr(idx), val); 689 if (is_counter_pair(hwc)) 690 wrmsrl(x86_pmu_config_addr(idx + 1), 0); 691 } 692 } 693 694 struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr) 695 { 696 return static_call(x86_pmu_guest_get_msrs)(nr); 697 } 698 EXPORT_SYMBOL_GPL(perf_guest_get_msrs); 699 700 /* 701 * There may be PMI landing after enabled=0. The PMI hitting could be before or 702 * after disable_all. 703 * 704 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 705 * It will not be re-enabled in the NMI handler again, because enabled=0. After 706 * handling the NMI, disable_all will be called, which will not change the 707 * state either. If PMI hits after disable_all, the PMU is already disabled 708 * before entering NMI handler. The NMI handler will not change the state 709 * either. 710 * 711 * So either situation is harmless. 712 */ 713 static void x86_pmu_disable(struct pmu *pmu) 714 { 715 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 716 717 if (!x86_pmu_initialized()) 718 return; 719 720 if (!cpuc->enabled) 721 return; 722 723 cpuc->n_added = 0; 724 cpuc->enabled = 0; 725 barrier(); 726 727 static_call(x86_pmu_disable_all)(); 728 } 729 730 void x86_pmu_enable_all(int added) 731 { 732 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 733 int idx; 734 735 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 736 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 737 738 if (!test_bit(idx, cpuc->active_mask)) 739 continue; 740 741 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 742 } 743 } 744 745 static inline int is_x86_event(struct perf_event *event) 746 { 747 int i; 748 749 if (!is_hybrid()) 750 return event->pmu == &pmu; 751 752 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { 753 if (event->pmu == &x86_pmu.hybrid_pmu[i].pmu) 754 return true; 755 } 756 757 return false; 758 } 759 760 struct pmu *x86_get_pmu(unsigned int cpu) 761 { 762 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 763 764 /* 765 * All CPUs of the hybrid type have been offline. 766 * The x86_get_pmu() should not be invoked. 767 */ 768 if (WARN_ON_ONCE(!cpuc->pmu)) 769 return &pmu; 770 771 return cpuc->pmu; 772 } 773 /* 774 * Event scheduler state: 775 * 776 * Assign events iterating over all events and counters, beginning 777 * with events with least weights first. Keep the current iterator 778 * state in struct sched_state. 779 */ 780 struct sched_state { 781 int weight; 782 int event; /* event index */ 783 int counter; /* counter index */ 784 int unassigned; /* number of events to be assigned left */ 785 int nr_gp; /* number of GP counters used */ 786 u64 used; 787 }; 788 789 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 790 #define SCHED_STATES_MAX 2 791 792 struct perf_sched { 793 int max_weight; 794 int max_events; 795 int max_gp; 796 int saved_states; 797 struct event_constraint **constraints; 798 struct sched_state state; 799 struct sched_state saved[SCHED_STATES_MAX]; 800 }; 801 802 /* 803 * Initialize iterator that runs through all events and counters. 804 */ 805 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 806 int num, int wmin, int wmax, int gpmax) 807 { 808 int idx; 809 810 memset(sched, 0, sizeof(*sched)); 811 sched->max_events = num; 812 sched->max_weight = wmax; 813 sched->max_gp = gpmax; 814 sched->constraints = constraints; 815 816 for (idx = 0; idx < num; idx++) { 817 if (constraints[idx]->weight == wmin) 818 break; 819 } 820 821 sched->state.event = idx; /* start with min weight */ 822 sched->state.weight = wmin; 823 sched->state.unassigned = num; 824 } 825 826 static void perf_sched_save_state(struct perf_sched *sched) 827 { 828 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 829 return; 830 831 sched->saved[sched->saved_states] = sched->state; 832 sched->saved_states++; 833 } 834 835 static bool perf_sched_restore_state(struct perf_sched *sched) 836 { 837 if (!sched->saved_states) 838 return false; 839 840 sched->saved_states--; 841 sched->state = sched->saved[sched->saved_states]; 842 843 /* this assignment didn't work out */ 844 /* XXX broken vs EVENT_PAIR */ 845 sched->state.used &= ~BIT_ULL(sched->state.counter); 846 847 /* try the next one */ 848 sched->state.counter++; 849 850 return true; 851 } 852 853 /* 854 * Select a counter for the current event to schedule. Return true on 855 * success. 856 */ 857 static bool __perf_sched_find_counter(struct perf_sched *sched) 858 { 859 struct event_constraint *c; 860 int idx; 861 862 if (!sched->state.unassigned) 863 return false; 864 865 if (sched->state.event >= sched->max_events) 866 return false; 867 868 c = sched->constraints[sched->state.event]; 869 /* Prefer fixed purpose counters */ 870 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 871 idx = INTEL_PMC_IDX_FIXED; 872 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 873 u64 mask = BIT_ULL(idx); 874 875 if (sched->state.used & mask) 876 continue; 877 878 sched->state.used |= mask; 879 goto done; 880 } 881 } 882 883 /* Grab the first unused counter starting with idx */ 884 idx = sched->state.counter; 885 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 886 u64 mask = BIT_ULL(idx); 887 888 if (c->flags & PERF_X86_EVENT_PAIR) 889 mask |= mask << 1; 890 891 if (sched->state.used & mask) 892 continue; 893 894 if (sched->state.nr_gp++ >= sched->max_gp) 895 return false; 896 897 sched->state.used |= mask; 898 goto done; 899 } 900 901 return false; 902 903 done: 904 sched->state.counter = idx; 905 906 if (c->overlap) 907 perf_sched_save_state(sched); 908 909 return true; 910 } 911 912 static bool perf_sched_find_counter(struct perf_sched *sched) 913 { 914 while (!__perf_sched_find_counter(sched)) { 915 if (!perf_sched_restore_state(sched)) 916 return false; 917 } 918 919 return true; 920 } 921 922 /* 923 * Go through all unassigned events and find the next one to schedule. 924 * Take events with the least weight first. Return true on success. 925 */ 926 static bool perf_sched_next_event(struct perf_sched *sched) 927 { 928 struct event_constraint *c; 929 930 if (!sched->state.unassigned || !--sched->state.unassigned) 931 return false; 932 933 do { 934 /* next event */ 935 sched->state.event++; 936 if (sched->state.event >= sched->max_events) { 937 /* next weight */ 938 sched->state.event = 0; 939 sched->state.weight++; 940 if (sched->state.weight > sched->max_weight) 941 return false; 942 } 943 c = sched->constraints[sched->state.event]; 944 } while (c->weight != sched->state.weight); 945 946 sched->state.counter = 0; /* start with first counter */ 947 948 return true; 949 } 950 951 /* 952 * Assign a counter for each event. 953 */ 954 int perf_assign_events(struct event_constraint **constraints, int n, 955 int wmin, int wmax, int gpmax, int *assign) 956 { 957 struct perf_sched sched; 958 959 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 960 961 do { 962 if (!perf_sched_find_counter(&sched)) 963 break; /* failed */ 964 if (assign) 965 assign[sched.state.event] = sched.state.counter; 966 } while (perf_sched_next_event(&sched)); 967 968 return sched.state.unassigned; 969 } 970 EXPORT_SYMBOL_GPL(perf_assign_events); 971 972 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 973 { 974 int num_counters = hybrid(cpuc->pmu, num_counters); 975 struct event_constraint *c; 976 struct perf_event *e; 977 int n0, i, wmin, wmax, unsched = 0; 978 struct hw_perf_event *hwc; 979 u64 used_mask = 0; 980 981 /* 982 * Compute the number of events already present; see x86_pmu_add(), 983 * validate_group() and x86_pmu_commit_txn(). For the former two 984 * cpuc->n_events hasn't been updated yet, while for the latter 985 * cpuc->n_txn contains the number of events added in the current 986 * transaction. 987 */ 988 n0 = cpuc->n_events; 989 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 990 n0 -= cpuc->n_txn; 991 992 static_call_cond(x86_pmu_start_scheduling)(cpuc); 993 994 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 995 c = cpuc->event_constraint[i]; 996 997 /* 998 * Previously scheduled events should have a cached constraint, 999 * while new events should not have one. 1000 */ 1001 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0)); 1002 1003 /* 1004 * Request constraints for new events; or for those events that 1005 * have a dynamic constraint -- for those the constraint can 1006 * change due to external factors (sibling state, allow_tfa). 1007 */ 1008 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) { 1009 c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]); 1010 cpuc->event_constraint[i] = c; 1011 } 1012 1013 wmin = min(wmin, c->weight); 1014 wmax = max(wmax, c->weight); 1015 } 1016 1017 /* 1018 * fastpath, try to reuse previous register 1019 */ 1020 for (i = 0; i < n; i++) { 1021 u64 mask; 1022 1023 hwc = &cpuc->event_list[i]->hw; 1024 c = cpuc->event_constraint[i]; 1025 1026 /* never assigned */ 1027 if (hwc->idx == -1) 1028 break; 1029 1030 /* constraint still honored */ 1031 if (!test_bit(hwc->idx, c->idxmsk)) 1032 break; 1033 1034 mask = BIT_ULL(hwc->idx); 1035 if (is_counter_pair(hwc)) 1036 mask |= mask << 1; 1037 1038 /* not already used */ 1039 if (used_mask & mask) 1040 break; 1041 1042 used_mask |= mask; 1043 1044 if (assign) 1045 assign[i] = hwc->idx; 1046 } 1047 1048 /* slow path */ 1049 if (i != n) { 1050 int gpmax = num_counters; 1051 1052 /* 1053 * Do not allow scheduling of more than half the available 1054 * generic counters. 1055 * 1056 * This helps avoid counter starvation of sibling thread by 1057 * ensuring at most half the counters cannot be in exclusive 1058 * mode. There is no designated counters for the limits. Any 1059 * N/2 counters can be used. This helps with events with 1060 * specific counter constraints. 1061 */ 1062 if (is_ht_workaround_enabled() && !cpuc->is_fake && 1063 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 1064 gpmax /= 2; 1065 1066 /* 1067 * Reduce the amount of available counters to allow fitting 1068 * the extra Merge events needed by large increment events. 1069 */ 1070 if (x86_pmu.flags & PMU_FL_PAIR) { 1071 gpmax = num_counters - cpuc->n_pair; 1072 WARN_ON(gpmax <= 0); 1073 } 1074 1075 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 1076 wmax, gpmax, assign); 1077 } 1078 1079 /* 1080 * In case of success (unsched = 0), mark events as committed, 1081 * so we do not put_constraint() in case new events are added 1082 * and fail to be scheduled 1083 * 1084 * We invoke the lower level commit callback to lock the resource 1085 * 1086 * We do not need to do all of this in case we are called to 1087 * validate an event group (assign == NULL) 1088 */ 1089 if (!unsched && assign) { 1090 for (i = 0; i < n; i++) { 1091 e = cpuc->event_list[i]; 1092 static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]); 1093 } 1094 } else { 1095 for (i = n0; i < n; i++) { 1096 e = cpuc->event_list[i]; 1097 1098 /* 1099 * release events that failed scheduling 1100 */ 1101 static_call_cond(x86_pmu_put_event_constraints)(cpuc, e); 1102 1103 cpuc->event_constraint[i] = NULL; 1104 } 1105 } 1106 1107 static_call_cond(x86_pmu_stop_scheduling)(cpuc); 1108 1109 return unsched ? -EINVAL : 0; 1110 } 1111 1112 static int add_nr_metric_event(struct cpu_hw_events *cpuc, 1113 struct perf_event *event) 1114 { 1115 if (is_metric_event(event)) { 1116 if (cpuc->n_metric == INTEL_TD_METRIC_NUM) 1117 return -EINVAL; 1118 cpuc->n_metric++; 1119 cpuc->n_txn_metric++; 1120 } 1121 1122 return 0; 1123 } 1124 1125 static void del_nr_metric_event(struct cpu_hw_events *cpuc, 1126 struct perf_event *event) 1127 { 1128 if (is_metric_event(event)) 1129 cpuc->n_metric--; 1130 } 1131 1132 static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event, 1133 int max_count, int n) 1134 { 1135 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap); 1136 1137 if (intel_cap.perf_metrics && add_nr_metric_event(cpuc, event)) 1138 return -EINVAL; 1139 1140 if (n >= max_count + cpuc->n_metric) 1141 return -EINVAL; 1142 1143 cpuc->event_list[n] = event; 1144 if (is_counter_pair(&event->hw)) { 1145 cpuc->n_pair++; 1146 cpuc->n_txn_pair++; 1147 } 1148 1149 return 0; 1150 } 1151 1152 /* 1153 * dogrp: true if must collect siblings events (group) 1154 * returns total number of events and error code 1155 */ 1156 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 1157 { 1158 int num_counters = hybrid(cpuc->pmu, num_counters); 1159 int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed); 1160 struct perf_event *event; 1161 int n, max_count; 1162 1163 max_count = num_counters + num_counters_fixed; 1164 1165 /* current number of events already accepted */ 1166 n = cpuc->n_events; 1167 if (!cpuc->n_events) 1168 cpuc->pebs_output = 0; 1169 1170 if (!cpuc->is_fake && leader->attr.precise_ip) { 1171 /* 1172 * For PEBS->PT, if !aux_event, the group leader (PT) went 1173 * away, the group was broken down and this singleton event 1174 * can't schedule any more. 1175 */ 1176 if (is_pebs_pt(leader) && !leader->aux_event) 1177 return -EINVAL; 1178 1179 /* 1180 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS 1181 */ 1182 if (cpuc->pebs_output && 1183 cpuc->pebs_output != is_pebs_pt(leader) + 1) 1184 return -EINVAL; 1185 1186 cpuc->pebs_output = is_pebs_pt(leader) + 1; 1187 } 1188 1189 if (is_x86_event(leader)) { 1190 if (collect_event(cpuc, leader, max_count, n)) 1191 return -EINVAL; 1192 n++; 1193 } 1194 1195 if (!dogrp) 1196 return n; 1197 1198 for_each_sibling_event(event, leader) { 1199 if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF) 1200 continue; 1201 1202 if (collect_event(cpuc, event, max_count, n)) 1203 return -EINVAL; 1204 1205 n++; 1206 } 1207 return n; 1208 } 1209 1210 static inline void x86_assign_hw_event(struct perf_event *event, 1211 struct cpu_hw_events *cpuc, int i) 1212 { 1213 struct hw_perf_event *hwc = &event->hw; 1214 int idx; 1215 1216 idx = hwc->idx = cpuc->assign[i]; 1217 hwc->last_cpu = smp_processor_id(); 1218 hwc->last_tag = ++cpuc->tags[i]; 1219 1220 switch (hwc->idx) { 1221 case INTEL_PMC_IDX_FIXED_BTS: 1222 case INTEL_PMC_IDX_FIXED_VLBR: 1223 hwc->config_base = 0; 1224 hwc->event_base = 0; 1225 break; 1226 1227 case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END: 1228 /* All the metric events are mapped onto the fixed counter 3. */ 1229 idx = INTEL_PMC_IDX_FIXED_SLOTS; 1230 fallthrough; 1231 case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1: 1232 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1233 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + 1234 (idx - INTEL_PMC_IDX_FIXED); 1235 hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) | 1236 INTEL_PMC_FIXED_RDPMC_BASE; 1237 break; 1238 1239 default: 1240 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1241 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1242 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1243 break; 1244 } 1245 } 1246 1247 /** 1248 * x86_perf_rdpmc_index - Return PMC counter used for event 1249 * @event: the perf_event to which the PMC counter was assigned 1250 * 1251 * The counter assigned to this performance event may change if interrupts 1252 * are enabled. This counter should thus never be used while interrupts are 1253 * enabled. Before this function is used to obtain the assigned counter the 1254 * event should be checked for validity using, for example, 1255 * perf_event_read_local(), within the same interrupt disabled section in 1256 * which this counter is planned to be used. 1257 * 1258 * Return: The index of the performance monitoring counter assigned to 1259 * @perf_event. 1260 */ 1261 int x86_perf_rdpmc_index(struct perf_event *event) 1262 { 1263 lockdep_assert_irqs_disabled(); 1264 1265 return event->hw.event_base_rdpmc; 1266 } 1267 1268 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1269 struct cpu_hw_events *cpuc, 1270 int i) 1271 { 1272 return hwc->idx == cpuc->assign[i] && 1273 hwc->last_cpu == smp_processor_id() && 1274 hwc->last_tag == cpuc->tags[i]; 1275 } 1276 1277 static void x86_pmu_start(struct perf_event *event, int flags); 1278 1279 static void x86_pmu_enable(struct pmu *pmu) 1280 { 1281 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1282 struct perf_event *event; 1283 struct hw_perf_event *hwc; 1284 int i, added = cpuc->n_added; 1285 1286 if (!x86_pmu_initialized()) 1287 return; 1288 1289 if (cpuc->enabled) 1290 return; 1291 1292 if (cpuc->n_added) { 1293 int n_running = cpuc->n_events - cpuc->n_added; 1294 /* 1295 * apply assignment obtained either from 1296 * hw_perf_group_sched_in() or x86_pmu_enable() 1297 * 1298 * step1: save events moving to new counters 1299 */ 1300 for (i = 0; i < n_running; i++) { 1301 event = cpuc->event_list[i]; 1302 hwc = &event->hw; 1303 1304 /* 1305 * we can avoid reprogramming counter if: 1306 * - assigned same counter as last time 1307 * - running on same CPU as last time 1308 * - no other event has used the counter since 1309 */ 1310 if (hwc->idx == -1 || 1311 match_prev_assignment(hwc, cpuc, i)) 1312 continue; 1313 1314 /* 1315 * Ensure we don't accidentally enable a stopped 1316 * counter simply because we rescheduled. 1317 */ 1318 if (hwc->state & PERF_HES_STOPPED) 1319 hwc->state |= PERF_HES_ARCH; 1320 1321 x86_pmu_stop(event, PERF_EF_UPDATE); 1322 } 1323 1324 /* 1325 * step2: reprogram moved events into new counters 1326 */ 1327 for (i = 0; i < cpuc->n_events; i++) { 1328 event = cpuc->event_list[i]; 1329 hwc = &event->hw; 1330 1331 if (!match_prev_assignment(hwc, cpuc, i)) 1332 x86_assign_hw_event(event, cpuc, i); 1333 else if (i < n_running) 1334 continue; 1335 1336 if (hwc->state & PERF_HES_ARCH) 1337 continue; 1338 1339 x86_pmu_start(event, PERF_EF_RELOAD); 1340 } 1341 cpuc->n_added = 0; 1342 perf_events_lapic_init(); 1343 } 1344 1345 cpuc->enabled = 1; 1346 barrier(); 1347 1348 static_call(x86_pmu_enable_all)(added); 1349 } 1350 1351 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1352 1353 /* 1354 * Set the next IRQ period, based on the hwc->period_left value. 1355 * To be called with the event disabled in hw: 1356 */ 1357 int x86_perf_event_set_period(struct perf_event *event) 1358 { 1359 struct hw_perf_event *hwc = &event->hw; 1360 s64 left = local64_read(&hwc->period_left); 1361 s64 period = hwc->sample_period; 1362 int ret = 0, idx = hwc->idx; 1363 1364 if (unlikely(!hwc->event_base)) 1365 return 0; 1366 1367 if (unlikely(is_topdown_count(event)) && 1368 x86_pmu.set_topdown_event_period) 1369 return x86_pmu.set_topdown_event_period(event); 1370 1371 /* 1372 * If we are way outside a reasonable range then just skip forward: 1373 */ 1374 if (unlikely(left <= -period)) { 1375 left = period; 1376 local64_set(&hwc->period_left, left); 1377 hwc->last_period = period; 1378 ret = 1; 1379 } 1380 1381 if (unlikely(left <= 0)) { 1382 left += period; 1383 local64_set(&hwc->period_left, left); 1384 hwc->last_period = period; 1385 ret = 1; 1386 } 1387 /* 1388 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1389 */ 1390 if (unlikely(left < 2)) 1391 left = 2; 1392 1393 if (left > x86_pmu.max_period) 1394 left = x86_pmu.max_period; 1395 1396 if (x86_pmu.limit_period) 1397 left = x86_pmu.limit_period(event, left); 1398 1399 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1400 1401 /* 1402 * The hw event starts counting from this event offset, 1403 * mark it to be able to extra future deltas: 1404 */ 1405 local64_set(&hwc->prev_count, (u64)-left); 1406 1407 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1408 1409 /* 1410 * Sign extend the Merge event counter's upper 16 bits since 1411 * we currently declare a 48-bit counter width 1412 */ 1413 if (is_counter_pair(hwc)) 1414 wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff); 1415 1416 /* 1417 * Due to erratum on certan cpu we need 1418 * a second write to be sure the register 1419 * is updated properly 1420 */ 1421 if (x86_pmu.perfctr_second_write) { 1422 wrmsrl(hwc->event_base, 1423 (u64)(-left) & x86_pmu.cntval_mask); 1424 } 1425 1426 perf_event_update_userpage(event); 1427 1428 return ret; 1429 } 1430 1431 void x86_pmu_enable_event(struct perf_event *event) 1432 { 1433 if (__this_cpu_read(cpu_hw_events.enabled)) 1434 __x86_pmu_enable_event(&event->hw, 1435 ARCH_PERFMON_EVENTSEL_ENABLE); 1436 } 1437 1438 /* 1439 * Add a single event to the PMU. 1440 * 1441 * The event is added to the group of enabled events 1442 * but only if it can be scheduled with existing events. 1443 */ 1444 static int x86_pmu_add(struct perf_event *event, int flags) 1445 { 1446 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1447 struct hw_perf_event *hwc; 1448 int assign[X86_PMC_IDX_MAX]; 1449 int n, n0, ret; 1450 1451 hwc = &event->hw; 1452 1453 n0 = cpuc->n_events; 1454 ret = n = collect_events(cpuc, event, false); 1455 if (ret < 0) 1456 goto out; 1457 1458 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1459 if (!(flags & PERF_EF_START)) 1460 hwc->state |= PERF_HES_ARCH; 1461 1462 /* 1463 * If group events scheduling transaction was started, 1464 * skip the schedulability test here, it will be performed 1465 * at commit time (->commit_txn) as a whole. 1466 * 1467 * If commit fails, we'll call ->del() on all events 1468 * for which ->add() was called. 1469 */ 1470 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1471 goto done_collect; 1472 1473 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign); 1474 if (ret) 1475 goto out; 1476 /* 1477 * copy new assignment, now we know it is possible 1478 * will be used by hw_perf_enable() 1479 */ 1480 memcpy(cpuc->assign, assign, n*sizeof(int)); 1481 1482 done_collect: 1483 /* 1484 * Commit the collect_events() state. See x86_pmu_del() and 1485 * x86_pmu_*_txn(). 1486 */ 1487 cpuc->n_events = n; 1488 cpuc->n_added += n - n0; 1489 cpuc->n_txn += n - n0; 1490 1491 /* 1492 * This is before x86_pmu_enable() will call x86_pmu_start(), 1493 * so we enable LBRs before an event needs them etc.. 1494 */ 1495 static_call_cond(x86_pmu_add)(event); 1496 1497 ret = 0; 1498 out: 1499 return ret; 1500 } 1501 1502 static void x86_pmu_start(struct perf_event *event, int flags) 1503 { 1504 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1505 int idx = event->hw.idx; 1506 1507 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1508 return; 1509 1510 if (WARN_ON_ONCE(idx == -1)) 1511 return; 1512 1513 if (flags & PERF_EF_RELOAD) { 1514 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1515 x86_perf_event_set_period(event); 1516 } 1517 1518 event->hw.state = 0; 1519 1520 cpuc->events[idx] = event; 1521 __set_bit(idx, cpuc->active_mask); 1522 static_call(x86_pmu_enable)(event); 1523 perf_event_update_userpage(event); 1524 } 1525 1526 void perf_event_print_debug(void) 1527 { 1528 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1529 u64 pebs, debugctl; 1530 int cpu = smp_processor_id(); 1531 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1532 int num_counters = hybrid(cpuc->pmu, num_counters); 1533 int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed); 1534 struct event_constraint *pebs_constraints = hybrid(cpuc->pmu, pebs_constraints); 1535 unsigned long flags; 1536 int idx; 1537 1538 if (!num_counters) 1539 return; 1540 1541 local_irq_save(flags); 1542 1543 if (x86_pmu.version >= 2) { 1544 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1545 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1546 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1547 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1548 1549 pr_info("\n"); 1550 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1551 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1552 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1553 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1554 if (pebs_constraints) { 1555 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1556 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1557 } 1558 if (x86_pmu.lbr_nr) { 1559 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1560 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1561 } 1562 } 1563 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1564 1565 for (idx = 0; idx < num_counters; idx++) { 1566 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1567 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1568 1569 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1570 1571 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1572 cpu, idx, pmc_ctrl); 1573 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1574 cpu, idx, pmc_count); 1575 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1576 cpu, idx, prev_left); 1577 } 1578 for (idx = 0; idx < num_counters_fixed; idx++) { 1579 if (fixed_counter_disabled(idx, cpuc->pmu)) 1580 continue; 1581 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1582 1583 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1584 cpu, idx, pmc_count); 1585 } 1586 local_irq_restore(flags); 1587 } 1588 1589 void x86_pmu_stop(struct perf_event *event, int flags) 1590 { 1591 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1592 struct hw_perf_event *hwc = &event->hw; 1593 1594 if (test_bit(hwc->idx, cpuc->active_mask)) { 1595 static_call(x86_pmu_disable)(event); 1596 __clear_bit(hwc->idx, cpuc->active_mask); 1597 cpuc->events[hwc->idx] = NULL; 1598 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1599 hwc->state |= PERF_HES_STOPPED; 1600 } 1601 1602 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1603 /* 1604 * Drain the remaining delta count out of a event 1605 * that we are disabling: 1606 */ 1607 x86_perf_event_update(event); 1608 hwc->state |= PERF_HES_UPTODATE; 1609 } 1610 } 1611 1612 static void x86_pmu_del(struct perf_event *event, int flags) 1613 { 1614 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1615 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap); 1616 int i; 1617 1618 /* 1619 * If we're called during a txn, we only need to undo x86_pmu.add. 1620 * The events never got scheduled and ->cancel_txn will truncate 1621 * the event_list. 1622 * 1623 * XXX assumes any ->del() called during a TXN will only be on 1624 * an event added during that same TXN. 1625 */ 1626 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1627 goto do_del; 1628 1629 /* 1630 * Not a TXN, therefore cleanup properly. 1631 */ 1632 x86_pmu_stop(event, PERF_EF_UPDATE); 1633 1634 for (i = 0; i < cpuc->n_events; i++) { 1635 if (event == cpuc->event_list[i]) 1636 break; 1637 } 1638 1639 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1640 return; 1641 1642 /* If we have a newly added event; make sure to decrease n_added. */ 1643 if (i >= cpuc->n_events - cpuc->n_added) 1644 --cpuc->n_added; 1645 1646 static_call_cond(x86_pmu_put_event_constraints)(cpuc, event); 1647 1648 /* Delete the array entry. */ 1649 while (++i < cpuc->n_events) { 1650 cpuc->event_list[i-1] = cpuc->event_list[i]; 1651 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1652 } 1653 cpuc->event_constraint[i-1] = NULL; 1654 --cpuc->n_events; 1655 if (intel_cap.perf_metrics) 1656 del_nr_metric_event(cpuc, event); 1657 1658 perf_event_update_userpage(event); 1659 1660 do_del: 1661 1662 /* 1663 * This is after x86_pmu_stop(); so we disable LBRs after any 1664 * event can need them etc.. 1665 */ 1666 static_call_cond(x86_pmu_del)(event); 1667 } 1668 1669 int x86_pmu_handle_irq(struct pt_regs *regs) 1670 { 1671 struct perf_sample_data data; 1672 struct cpu_hw_events *cpuc; 1673 struct perf_event *event; 1674 int idx, handled = 0; 1675 u64 val; 1676 1677 cpuc = this_cpu_ptr(&cpu_hw_events); 1678 1679 /* 1680 * Some chipsets need to unmask the LVTPC in a particular spot 1681 * inside the nmi handler. As a result, the unmasking was pushed 1682 * into all the nmi handlers. 1683 * 1684 * This generic handler doesn't seem to have any issues where the 1685 * unmasking occurs so it was left at the top. 1686 */ 1687 apic_write(APIC_LVTPC, APIC_DM_NMI); 1688 1689 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1690 if (!test_bit(idx, cpuc->active_mask)) 1691 continue; 1692 1693 event = cpuc->events[idx]; 1694 1695 val = x86_perf_event_update(event); 1696 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1697 continue; 1698 1699 /* 1700 * event overflow 1701 */ 1702 handled++; 1703 perf_sample_data_init(&data, 0, event->hw.last_period); 1704 1705 if (!x86_perf_event_set_period(event)) 1706 continue; 1707 1708 if (perf_event_overflow(event, &data, regs)) 1709 x86_pmu_stop(event, 0); 1710 } 1711 1712 if (handled) 1713 inc_irq_stat(apic_perf_irqs); 1714 1715 return handled; 1716 } 1717 1718 void perf_events_lapic_init(void) 1719 { 1720 if (!x86_pmu.apic || !x86_pmu_initialized()) 1721 return; 1722 1723 /* 1724 * Always use NMI for PMU 1725 */ 1726 apic_write(APIC_LVTPC, APIC_DM_NMI); 1727 } 1728 1729 static int 1730 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1731 { 1732 u64 start_clock; 1733 u64 finish_clock; 1734 int ret; 1735 1736 /* 1737 * All PMUs/events that share this PMI handler should make sure to 1738 * increment active_events for their events. 1739 */ 1740 if (!atomic_read(&active_events)) 1741 return NMI_DONE; 1742 1743 start_clock = sched_clock(); 1744 ret = static_call(x86_pmu_handle_irq)(regs); 1745 finish_clock = sched_clock(); 1746 1747 perf_sample_event_took(finish_clock - start_clock); 1748 1749 return ret; 1750 } 1751 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1752 1753 struct event_constraint emptyconstraint; 1754 struct event_constraint unconstrained; 1755 1756 static int x86_pmu_prepare_cpu(unsigned int cpu) 1757 { 1758 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1759 int i; 1760 1761 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1762 cpuc->kfree_on_online[i] = NULL; 1763 if (x86_pmu.cpu_prepare) 1764 return x86_pmu.cpu_prepare(cpu); 1765 return 0; 1766 } 1767 1768 static int x86_pmu_dead_cpu(unsigned int cpu) 1769 { 1770 if (x86_pmu.cpu_dead) 1771 x86_pmu.cpu_dead(cpu); 1772 return 0; 1773 } 1774 1775 static int x86_pmu_online_cpu(unsigned int cpu) 1776 { 1777 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1778 int i; 1779 1780 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1781 kfree(cpuc->kfree_on_online[i]); 1782 cpuc->kfree_on_online[i] = NULL; 1783 } 1784 return 0; 1785 } 1786 1787 static int x86_pmu_starting_cpu(unsigned int cpu) 1788 { 1789 if (x86_pmu.cpu_starting) 1790 x86_pmu.cpu_starting(cpu); 1791 return 0; 1792 } 1793 1794 static int x86_pmu_dying_cpu(unsigned int cpu) 1795 { 1796 if (x86_pmu.cpu_dying) 1797 x86_pmu.cpu_dying(cpu); 1798 return 0; 1799 } 1800 1801 static void __init pmu_check_apic(void) 1802 { 1803 if (boot_cpu_has(X86_FEATURE_APIC)) 1804 return; 1805 1806 x86_pmu.apic = 0; 1807 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1808 pr_info("no hardware sampling interrupt available.\n"); 1809 1810 /* 1811 * If we have a PMU initialized but no APIC 1812 * interrupts, we cannot sample hardware 1813 * events (user-space has to fall back and 1814 * sample via a hrtimer based software event): 1815 */ 1816 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1817 1818 } 1819 1820 static struct attribute_group x86_pmu_format_group __ro_after_init = { 1821 .name = "format", 1822 .attrs = NULL, 1823 }; 1824 1825 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1826 { 1827 struct perf_pmu_events_attr *pmu_attr = 1828 container_of(attr, struct perf_pmu_events_attr, attr); 1829 u64 config = 0; 1830 1831 if (pmu_attr->id < x86_pmu.max_events) 1832 config = x86_pmu.event_map(pmu_attr->id); 1833 1834 /* string trumps id */ 1835 if (pmu_attr->event_str) 1836 return sprintf(page, "%s", pmu_attr->event_str); 1837 1838 return x86_pmu.events_sysfs_show(page, config); 1839 } 1840 EXPORT_SYMBOL_GPL(events_sysfs_show); 1841 1842 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1843 char *page) 1844 { 1845 struct perf_pmu_events_ht_attr *pmu_attr = 1846 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1847 1848 /* 1849 * Report conditional events depending on Hyper-Threading. 1850 * 1851 * This is overly conservative as usually the HT special 1852 * handling is not needed if the other CPU thread is idle. 1853 * 1854 * Note this does not (and cannot) handle the case when thread 1855 * siblings are invisible, for example with virtualization 1856 * if they are owned by some other guest. The user tool 1857 * has to re-read when a thread sibling gets onlined later. 1858 */ 1859 return sprintf(page, "%s", 1860 topology_max_smt_threads() > 1 ? 1861 pmu_attr->event_str_ht : 1862 pmu_attr->event_str_noht); 1863 } 1864 1865 ssize_t events_hybrid_sysfs_show(struct device *dev, 1866 struct device_attribute *attr, 1867 char *page) 1868 { 1869 struct perf_pmu_events_hybrid_attr *pmu_attr = 1870 container_of(attr, struct perf_pmu_events_hybrid_attr, attr); 1871 struct x86_hybrid_pmu *pmu; 1872 const char *str, *next_str; 1873 int i; 1874 1875 if (hweight64(pmu_attr->pmu_type) == 1) 1876 return sprintf(page, "%s", pmu_attr->event_str); 1877 1878 /* 1879 * Hybrid PMUs may support the same event name, but with different 1880 * event encoding, e.g., the mem-loads event on an Atom PMU has 1881 * different event encoding from a Core PMU. 1882 * 1883 * The event_str includes all event encodings. Each event encoding 1884 * is divided by ";". The order of the event encodings must follow 1885 * the order of the hybrid PMU index. 1886 */ 1887 pmu = container_of(dev_get_drvdata(dev), struct x86_hybrid_pmu, pmu); 1888 1889 str = pmu_attr->event_str; 1890 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { 1891 if (!(x86_pmu.hybrid_pmu[i].cpu_type & pmu_attr->pmu_type)) 1892 continue; 1893 if (x86_pmu.hybrid_pmu[i].cpu_type & pmu->cpu_type) { 1894 next_str = strchr(str, ';'); 1895 if (next_str) 1896 return snprintf(page, next_str - str + 1, "%s", str); 1897 else 1898 return sprintf(page, "%s", str); 1899 } 1900 str = strchr(str, ';'); 1901 str++; 1902 } 1903 1904 return 0; 1905 } 1906 EXPORT_SYMBOL_GPL(events_hybrid_sysfs_show); 1907 1908 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1909 EVENT_ATTR(instructions, INSTRUCTIONS ); 1910 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1911 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1912 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1913 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1914 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1915 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1916 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1917 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1918 1919 static struct attribute *empty_attrs; 1920 1921 static struct attribute *events_attr[] = { 1922 EVENT_PTR(CPU_CYCLES), 1923 EVENT_PTR(INSTRUCTIONS), 1924 EVENT_PTR(CACHE_REFERENCES), 1925 EVENT_PTR(CACHE_MISSES), 1926 EVENT_PTR(BRANCH_INSTRUCTIONS), 1927 EVENT_PTR(BRANCH_MISSES), 1928 EVENT_PTR(BUS_CYCLES), 1929 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1930 EVENT_PTR(STALLED_CYCLES_BACKEND), 1931 EVENT_PTR(REF_CPU_CYCLES), 1932 NULL, 1933 }; 1934 1935 /* 1936 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1937 * out of events_attr attributes. 1938 */ 1939 static umode_t 1940 is_visible(struct kobject *kobj, struct attribute *attr, int idx) 1941 { 1942 struct perf_pmu_events_attr *pmu_attr; 1943 1944 if (idx >= x86_pmu.max_events) 1945 return 0; 1946 1947 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); 1948 /* str trumps id */ 1949 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0; 1950 } 1951 1952 static struct attribute_group x86_pmu_events_group __ro_after_init = { 1953 .name = "events", 1954 .attrs = events_attr, 1955 .is_visible = is_visible, 1956 }; 1957 1958 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1959 { 1960 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1961 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1962 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1963 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1964 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1965 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1966 ssize_t ret; 1967 1968 /* 1969 * We have whole page size to spend and just little data 1970 * to write, so we can safely use sprintf. 1971 */ 1972 ret = sprintf(page, "event=0x%02llx", event); 1973 1974 if (umask) 1975 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1976 1977 if (edge) 1978 ret += sprintf(page + ret, ",edge"); 1979 1980 if (pc) 1981 ret += sprintf(page + ret, ",pc"); 1982 1983 if (any) 1984 ret += sprintf(page + ret, ",any"); 1985 1986 if (inv) 1987 ret += sprintf(page + ret, ",inv"); 1988 1989 if (cmask) 1990 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1991 1992 ret += sprintf(page + ret, "\n"); 1993 1994 return ret; 1995 } 1996 1997 static struct attribute_group x86_pmu_attr_group; 1998 static struct attribute_group x86_pmu_caps_group; 1999 2000 static void x86_pmu_static_call_update(void) 2001 { 2002 static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq); 2003 static_call_update(x86_pmu_disable_all, x86_pmu.disable_all); 2004 static_call_update(x86_pmu_enable_all, x86_pmu.enable_all); 2005 static_call_update(x86_pmu_enable, x86_pmu.enable); 2006 static_call_update(x86_pmu_disable, x86_pmu.disable); 2007 2008 static_call_update(x86_pmu_add, x86_pmu.add); 2009 static_call_update(x86_pmu_del, x86_pmu.del); 2010 static_call_update(x86_pmu_read, x86_pmu.read); 2011 2012 static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events); 2013 static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints); 2014 static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints); 2015 2016 static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling); 2017 static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling); 2018 static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling); 2019 2020 static_call_update(x86_pmu_sched_task, x86_pmu.sched_task); 2021 static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx); 2022 2023 static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs); 2024 static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases); 2025 2026 static_call_update(x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs); 2027 } 2028 2029 static void _x86_pmu_read(struct perf_event *event) 2030 { 2031 x86_perf_event_update(event); 2032 } 2033 2034 void x86_pmu_show_pmu_cap(int num_counters, int num_counters_fixed, 2035 u64 intel_ctrl) 2036 { 2037 pr_info("... version: %d\n", x86_pmu.version); 2038 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 2039 pr_info("... generic registers: %d\n", num_counters); 2040 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 2041 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 2042 pr_info("... fixed-purpose events: %lu\n", 2043 hweight64((((1ULL << num_counters_fixed) - 1) 2044 << INTEL_PMC_IDX_FIXED) & intel_ctrl)); 2045 pr_info("... event mask: %016Lx\n", intel_ctrl); 2046 } 2047 2048 /* 2049 * The generic code is not hybrid friendly. The hybrid_pmu->pmu 2050 * of the first registered PMU is unconditionally assigned to 2051 * each possible cpuctx->ctx.pmu. 2052 * Update the correct hybrid PMU to the cpuctx->ctx.pmu. 2053 */ 2054 void x86_pmu_update_cpu_context(struct pmu *pmu, int cpu) 2055 { 2056 struct perf_cpu_context *cpuctx; 2057 2058 if (!pmu->pmu_cpu_context) 2059 return; 2060 2061 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 2062 cpuctx->ctx.pmu = pmu; 2063 } 2064 2065 static int __init init_hw_perf_events(void) 2066 { 2067 struct x86_pmu_quirk *quirk; 2068 int err; 2069 2070 pr_info("Performance Events: "); 2071 2072 switch (boot_cpu_data.x86_vendor) { 2073 case X86_VENDOR_INTEL: 2074 err = intel_pmu_init(); 2075 break; 2076 case X86_VENDOR_AMD: 2077 err = amd_pmu_init(); 2078 break; 2079 case X86_VENDOR_HYGON: 2080 err = amd_pmu_init(); 2081 x86_pmu.name = "HYGON"; 2082 break; 2083 case X86_VENDOR_ZHAOXIN: 2084 case X86_VENDOR_CENTAUR: 2085 err = zhaoxin_pmu_init(); 2086 break; 2087 default: 2088 err = -ENOTSUPP; 2089 } 2090 if (err != 0) { 2091 pr_cont("no PMU driver, software events only.\n"); 2092 return 0; 2093 } 2094 2095 pmu_check_apic(); 2096 2097 /* sanity check that the hardware exists or is emulated */ 2098 if (!check_hw_exists(&pmu, x86_pmu.num_counters, x86_pmu.num_counters_fixed)) 2099 return 0; 2100 2101 pr_cont("%s PMU driver.\n", x86_pmu.name); 2102 2103 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 2104 2105 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 2106 quirk->func(); 2107 2108 if (!x86_pmu.intel_ctrl) 2109 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 2110 2111 perf_events_lapic_init(); 2112 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 2113 2114 unconstrained = (struct event_constraint) 2115 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 2116 0, x86_pmu.num_counters, 0, 0); 2117 2118 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 2119 2120 if (!x86_pmu.events_sysfs_show) 2121 x86_pmu_events_group.attrs = &empty_attrs; 2122 2123 pmu.attr_update = x86_pmu.attr_update; 2124 2125 if (!is_hybrid()) { 2126 x86_pmu_show_pmu_cap(x86_pmu.num_counters, 2127 x86_pmu.num_counters_fixed, 2128 x86_pmu.intel_ctrl); 2129 } 2130 2131 if (!x86_pmu.read) 2132 x86_pmu.read = _x86_pmu_read; 2133 2134 if (!x86_pmu.guest_get_msrs) 2135 x86_pmu.guest_get_msrs = (void *)&__static_call_return0; 2136 2137 x86_pmu_static_call_update(); 2138 2139 /* 2140 * Install callbacks. Core will call them for each online 2141 * cpu. 2142 */ 2143 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 2144 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 2145 if (err) 2146 return err; 2147 2148 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 2149 "perf/x86:starting", x86_pmu_starting_cpu, 2150 x86_pmu_dying_cpu); 2151 if (err) 2152 goto out; 2153 2154 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 2155 x86_pmu_online_cpu, NULL); 2156 if (err) 2157 goto out1; 2158 2159 if (!is_hybrid()) { 2160 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 2161 if (err) 2162 goto out2; 2163 } else { 2164 u8 cpu_type = get_this_hybrid_cpu_type(); 2165 struct x86_hybrid_pmu *hybrid_pmu; 2166 int i, j; 2167 2168 if (!cpu_type && x86_pmu.get_hybrid_cpu_type) 2169 cpu_type = x86_pmu.get_hybrid_cpu_type(); 2170 2171 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { 2172 hybrid_pmu = &x86_pmu.hybrid_pmu[i]; 2173 2174 hybrid_pmu->pmu = pmu; 2175 hybrid_pmu->pmu.type = -1; 2176 hybrid_pmu->pmu.attr_update = x86_pmu.attr_update; 2177 hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_HETEROGENEOUS_CPUS; 2178 hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE; 2179 2180 err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name, 2181 (hybrid_pmu->cpu_type == hybrid_big) ? PERF_TYPE_RAW : -1); 2182 if (err) 2183 break; 2184 2185 if (cpu_type == hybrid_pmu->cpu_type) 2186 x86_pmu_update_cpu_context(&hybrid_pmu->pmu, raw_smp_processor_id()); 2187 } 2188 2189 if (i < x86_pmu.num_hybrid_pmus) { 2190 for (j = 0; j < i; j++) 2191 perf_pmu_unregister(&x86_pmu.hybrid_pmu[j].pmu); 2192 pr_warn("Failed to register hybrid PMUs\n"); 2193 kfree(x86_pmu.hybrid_pmu); 2194 x86_pmu.hybrid_pmu = NULL; 2195 x86_pmu.num_hybrid_pmus = 0; 2196 goto out2; 2197 } 2198 } 2199 2200 return 0; 2201 2202 out2: 2203 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 2204 out1: 2205 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 2206 out: 2207 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 2208 return err; 2209 } 2210 early_initcall(init_hw_perf_events); 2211 2212 static void x86_pmu_read(struct perf_event *event) 2213 { 2214 static_call(x86_pmu_read)(event); 2215 } 2216 2217 /* 2218 * Start group events scheduling transaction 2219 * Set the flag to make pmu::enable() not perform the 2220 * schedulability test, it will be performed at commit time 2221 * 2222 * We only support PERF_PMU_TXN_ADD transactions. Save the 2223 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 2224 * transactions. 2225 */ 2226 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 2227 { 2228 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2229 2230 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 2231 2232 cpuc->txn_flags = txn_flags; 2233 if (txn_flags & ~PERF_PMU_TXN_ADD) 2234 return; 2235 2236 perf_pmu_disable(pmu); 2237 __this_cpu_write(cpu_hw_events.n_txn, 0); 2238 __this_cpu_write(cpu_hw_events.n_txn_pair, 0); 2239 __this_cpu_write(cpu_hw_events.n_txn_metric, 0); 2240 } 2241 2242 /* 2243 * Stop group events scheduling transaction 2244 * Clear the flag and pmu::enable() will perform the 2245 * schedulability test. 2246 */ 2247 static void x86_pmu_cancel_txn(struct pmu *pmu) 2248 { 2249 unsigned int txn_flags; 2250 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2251 2252 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 2253 2254 txn_flags = cpuc->txn_flags; 2255 cpuc->txn_flags = 0; 2256 if (txn_flags & ~PERF_PMU_TXN_ADD) 2257 return; 2258 2259 /* 2260 * Truncate collected array by the number of events added in this 2261 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 2262 */ 2263 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 2264 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 2265 __this_cpu_sub(cpu_hw_events.n_pair, __this_cpu_read(cpu_hw_events.n_txn_pair)); 2266 __this_cpu_sub(cpu_hw_events.n_metric, __this_cpu_read(cpu_hw_events.n_txn_metric)); 2267 perf_pmu_enable(pmu); 2268 } 2269 2270 /* 2271 * Commit group events scheduling transaction 2272 * Perform the group schedulability test as a whole 2273 * Return 0 if success 2274 * 2275 * Does not cancel the transaction on failure; expects the caller to do this. 2276 */ 2277 static int x86_pmu_commit_txn(struct pmu *pmu) 2278 { 2279 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2280 int assign[X86_PMC_IDX_MAX]; 2281 int n, ret; 2282 2283 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 2284 2285 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 2286 cpuc->txn_flags = 0; 2287 return 0; 2288 } 2289 2290 n = cpuc->n_events; 2291 2292 if (!x86_pmu_initialized()) 2293 return -EAGAIN; 2294 2295 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign); 2296 if (ret) 2297 return ret; 2298 2299 /* 2300 * copy new assignment, now we know it is possible 2301 * will be used by hw_perf_enable() 2302 */ 2303 memcpy(cpuc->assign, assign, n*sizeof(int)); 2304 2305 cpuc->txn_flags = 0; 2306 perf_pmu_enable(pmu); 2307 return 0; 2308 } 2309 /* 2310 * a fake_cpuc is used to validate event groups. Due to 2311 * the extra reg logic, we need to also allocate a fake 2312 * per_core and per_cpu structure. Otherwise, group events 2313 * using extra reg may conflict without the kernel being 2314 * able to catch this when the last event gets added to 2315 * the group. 2316 */ 2317 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 2318 { 2319 intel_cpuc_finish(cpuc); 2320 kfree(cpuc); 2321 } 2322 2323 static struct cpu_hw_events *allocate_fake_cpuc(struct pmu *event_pmu) 2324 { 2325 struct cpu_hw_events *cpuc; 2326 int cpu; 2327 2328 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 2329 if (!cpuc) 2330 return ERR_PTR(-ENOMEM); 2331 cpuc->is_fake = 1; 2332 2333 if (is_hybrid()) { 2334 struct x86_hybrid_pmu *h_pmu; 2335 2336 h_pmu = hybrid_pmu(event_pmu); 2337 if (cpumask_empty(&h_pmu->supported_cpus)) 2338 goto error; 2339 cpu = cpumask_first(&h_pmu->supported_cpus); 2340 } else 2341 cpu = raw_smp_processor_id(); 2342 cpuc->pmu = event_pmu; 2343 2344 if (intel_cpuc_prepare(cpuc, cpu)) 2345 goto error; 2346 2347 return cpuc; 2348 error: 2349 free_fake_cpuc(cpuc); 2350 return ERR_PTR(-ENOMEM); 2351 } 2352 2353 /* 2354 * validate that we can schedule this event 2355 */ 2356 static int validate_event(struct perf_event *event) 2357 { 2358 struct cpu_hw_events *fake_cpuc; 2359 struct event_constraint *c; 2360 int ret = 0; 2361 2362 fake_cpuc = allocate_fake_cpuc(event->pmu); 2363 if (IS_ERR(fake_cpuc)) 2364 return PTR_ERR(fake_cpuc); 2365 2366 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event); 2367 2368 if (!c || !c->weight) 2369 ret = -EINVAL; 2370 2371 if (x86_pmu.put_event_constraints) 2372 x86_pmu.put_event_constraints(fake_cpuc, event); 2373 2374 free_fake_cpuc(fake_cpuc); 2375 2376 return ret; 2377 } 2378 2379 /* 2380 * validate a single event group 2381 * 2382 * validation include: 2383 * - check events are compatible which each other 2384 * - events do not compete for the same counter 2385 * - number of events <= number of counters 2386 * 2387 * validation ensures the group can be loaded onto the 2388 * PMU if it was the only group available. 2389 */ 2390 static int validate_group(struct perf_event *event) 2391 { 2392 struct perf_event *leader = event->group_leader; 2393 struct cpu_hw_events *fake_cpuc; 2394 int ret = -EINVAL, n; 2395 2396 /* 2397 * Reject events from different hybrid PMUs. 2398 */ 2399 if (is_hybrid()) { 2400 struct perf_event *sibling; 2401 struct pmu *pmu = NULL; 2402 2403 if (is_x86_event(leader)) 2404 pmu = leader->pmu; 2405 2406 for_each_sibling_event(sibling, leader) { 2407 if (!is_x86_event(sibling)) 2408 continue; 2409 if (!pmu) 2410 pmu = sibling->pmu; 2411 else if (pmu != sibling->pmu) 2412 return ret; 2413 } 2414 } 2415 2416 fake_cpuc = allocate_fake_cpuc(event->pmu); 2417 if (IS_ERR(fake_cpuc)) 2418 return PTR_ERR(fake_cpuc); 2419 /* 2420 * the event is not yet connected with its 2421 * siblings therefore we must first collect 2422 * existing siblings, then add the new event 2423 * before we can simulate the scheduling 2424 */ 2425 n = collect_events(fake_cpuc, leader, true); 2426 if (n < 0) 2427 goto out; 2428 2429 fake_cpuc->n_events = n; 2430 n = collect_events(fake_cpuc, event, false); 2431 if (n < 0) 2432 goto out; 2433 2434 fake_cpuc->n_events = 0; 2435 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2436 2437 out: 2438 free_fake_cpuc(fake_cpuc); 2439 return ret; 2440 } 2441 2442 static int x86_pmu_event_init(struct perf_event *event) 2443 { 2444 struct x86_hybrid_pmu *pmu = NULL; 2445 int err; 2446 2447 if ((event->attr.type != event->pmu->type) && 2448 (event->attr.type != PERF_TYPE_HARDWARE) && 2449 (event->attr.type != PERF_TYPE_HW_CACHE)) 2450 return -ENOENT; 2451 2452 if (is_hybrid() && (event->cpu != -1)) { 2453 pmu = hybrid_pmu(event->pmu); 2454 if (!cpumask_test_cpu(event->cpu, &pmu->supported_cpus)) 2455 return -ENOENT; 2456 } 2457 2458 err = __x86_pmu_event_init(event); 2459 if (!err) { 2460 if (event->group_leader != event) 2461 err = validate_group(event); 2462 else 2463 err = validate_event(event); 2464 } 2465 if (err) { 2466 if (event->destroy) 2467 event->destroy(event); 2468 } 2469 2470 if (READ_ONCE(x86_pmu.attr_rdpmc) && 2471 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) 2472 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2473 2474 return err; 2475 } 2476 2477 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 2478 { 2479 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2480 return; 2481 2482 /* 2483 * This function relies on not being called concurrently in two 2484 * tasks in the same mm. Otherwise one task could observe 2485 * perf_rdpmc_allowed > 1 and return all the way back to 2486 * userspace with CR4.PCE clear while another task is still 2487 * doing on_each_cpu_mask() to propagate CR4.PCE. 2488 * 2489 * For now, this can't happen because all callers hold mmap_lock 2490 * for write. If this changes, we'll need a different solution. 2491 */ 2492 mmap_assert_write_locked(mm); 2493 2494 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 2495 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1); 2496 } 2497 2498 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 2499 { 2500 2501 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2502 return; 2503 2504 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 2505 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1); 2506 } 2507 2508 static int x86_pmu_event_idx(struct perf_event *event) 2509 { 2510 struct hw_perf_event *hwc = &event->hw; 2511 2512 if (!(hwc->flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2513 return 0; 2514 2515 if (is_metric_idx(hwc->idx)) 2516 return INTEL_PMC_FIXED_RDPMC_METRICS + 1; 2517 else 2518 return hwc->event_base_rdpmc + 1; 2519 } 2520 2521 static ssize_t get_attr_rdpmc(struct device *cdev, 2522 struct device_attribute *attr, 2523 char *buf) 2524 { 2525 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2526 } 2527 2528 static ssize_t set_attr_rdpmc(struct device *cdev, 2529 struct device_attribute *attr, 2530 const char *buf, size_t count) 2531 { 2532 unsigned long val; 2533 ssize_t ret; 2534 2535 ret = kstrtoul(buf, 0, &val); 2536 if (ret) 2537 return ret; 2538 2539 if (val > 2) 2540 return -EINVAL; 2541 2542 if (x86_pmu.attr_rdpmc_broken) 2543 return -ENOTSUPP; 2544 2545 if (val != x86_pmu.attr_rdpmc) { 2546 /* 2547 * Changing into or out of never available or always available, 2548 * aka perf-event-bypassing mode. This path is extremely slow, 2549 * but only root can trigger it, so it's okay. 2550 */ 2551 if (val == 0) 2552 static_branch_inc(&rdpmc_never_available_key); 2553 else if (x86_pmu.attr_rdpmc == 0) 2554 static_branch_dec(&rdpmc_never_available_key); 2555 2556 if (val == 2) 2557 static_branch_inc(&rdpmc_always_available_key); 2558 else if (x86_pmu.attr_rdpmc == 2) 2559 static_branch_dec(&rdpmc_always_available_key); 2560 2561 on_each_cpu(cr4_update_pce, NULL, 1); 2562 x86_pmu.attr_rdpmc = val; 2563 } 2564 2565 return count; 2566 } 2567 2568 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2569 2570 static struct attribute *x86_pmu_attrs[] = { 2571 &dev_attr_rdpmc.attr, 2572 NULL, 2573 }; 2574 2575 static struct attribute_group x86_pmu_attr_group __ro_after_init = { 2576 .attrs = x86_pmu_attrs, 2577 }; 2578 2579 static ssize_t max_precise_show(struct device *cdev, 2580 struct device_attribute *attr, 2581 char *buf) 2582 { 2583 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise()); 2584 } 2585 2586 static DEVICE_ATTR_RO(max_precise); 2587 2588 static struct attribute *x86_pmu_caps_attrs[] = { 2589 &dev_attr_max_precise.attr, 2590 NULL 2591 }; 2592 2593 static struct attribute_group x86_pmu_caps_group __ro_after_init = { 2594 .name = "caps", 2595 .attrs = x86_pmu_caps_attrs, 2596 }; 2597 2598 static const struct attribute_group *x86_pmu_attr_groups[] = { 2599 &x86_pmu_attr_group, 2600 &x86_pmu_format_group, 2601 &x86_pmu_events_group, 2602 &x86_pmu_caps_group, 2603 NULL, 2604 }; 2605 2606 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2607 { 2608 static_call_cond(x86_pmu_sched_task)(ctx, sched_in); 2609 } 2610 2611 static void x86_pmu_swap_task_ctx(struct perf_event_context *prev, 2612 struct perf_event_context *next) 2613 { 2614 static_call_cond(x86_pmu_swap_task_ctx)(prev, next); 2615 } 2616 2617 void perf_check_microcode(void) 2618 { 2619 if (x86_pmu.check_microcode) 2620 x86_pmu.check_microcode(); 2621 } 2622 2623 static int x86_pmu_check_period(struct perf_event *event, u64 value) 2624 { 2625 if (x86_pmu.check_period && x86_pmu.check_period(event, value)) 2626 return -EINVAL; 2627 2628 if (value && x86_pmu.limit_period) { 2629 if (x86_pmu.limit_period(event, value) > value) 2630 return -EINVAL; 2631 } 2632 2633 return 0; 2634 } 2635 2636 static int x86_pmu_aux_output_match(struct perf_event *event) 2637 { 2638 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT)) 2639 return 0; 2640 2641 if (x86_pmu.aux_output_match) 2642 return x86_pmu.aux_output_match(event); 2643 2644 return 0; 2645 } 2646 2647 static int x86_pmu_filter_match(struct perf_event *event) 2648 { 2649 if (x86_pmu.filter_match) 2650 return x86_pmu.filter_match(event); 2651 2652 return 1; 2653 } 2654 2655 static struct pmu pmu = { 2656 .pmu_enable = x86_pmu_enable, 2657 .pmu_disable = x86_pmu_disable, 2658 2659 .attr_groups = x86_pmu_attr_groups, 2660 2661 .event_init = x86_pmu_event_init, 2662 2663 .event_mapped = x86_pmu_event_mapped, 2664 .event_unmapped = x86_pmu_event_unmapped, 2665 2666 .add = x86_pmu_add, 2667 .del = x86_pmu_del, 2668 .start = x86_pmu_start, 2669 .stop = x86_pmu_stop, 2670 .read = x86_pmu_read, 2671 2672 .start_txn = x86_pmu_start_txn, 2673 .cancel_txn = x86_pmu_cancel_txn, 2674 .commit_txn = x86_pmu_commit_txn, 2675 2676 .event_idx = x86_pmu_event_idx, 2677 .sched_task = x86_pmu_sched_task, 2678 .swap_task_ctx = x86_pmu_swap_task_ctx, 2679 .check_period = x86_pmu_check_period, 2680 2681 .aux_output_match = x86_pmu_aux_output_match, 2682 2683 .filter_match = x86_pmu_filter_match, 2684 }; 2685 2686 void arch_perf_update_userpage(struct perf_event *event, 2687 struct perf_event_mmap_page *userpg, u64 now) 2688 { 2689 struct cyc2ns_data data; 2690 u64 offset; 2691 2692 userpg->cap_user_time = 0; 2693 userpg->cap_user_time_zero = 0; 2694 userpg->cap_user_rdpmc = 2695 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2696 userpg->pmc_width = x86_pmu.cntval_bits; 2697 2698 if (!using_native_sched_clock() || !sched_clock_stable()) 2699 return; 2700 2701 cyc2ns_read_begin(&data); 2702 2703 offset = data.cyc2ns_offset + __sched_clock_offset; 2704 2705 /* 2706 * Internal timekeeping for enabled/running/stopped times 2707 * is always in the local_clock domain. 2708 */ 2709 userpg->cap_user_time = 1; 2710 userpg->time_mult = data.cyc2ns_mul; 2711 userpg->time_shift = data.cyc2ns_shift; 2712 userpg->time_offset = offset - now; 2713 2714 /* 2715 * cap_user_time_zero doesn't make sense when we're using a different 2716 * time base for the records. 2717 */ 2718 if (!event->attr.use_clockid) { 2719 userpg->cap_user_time_zero = 1; 2720 userpg->time_zero = offset; 2721 } 2722 2723 cyc2ns_read_end(); 2724 } 2725 2726 /* 2727 * Determine whether the regs were taken from an irq/exception handler rather 2728 * than from perf_arch_fetch_caller_regs(). 2729 */ 2730 static bool perf_hw_regs(struct pt_regs *regs) 2731 { 2732 return regs->flags & X86_EFLAGS_FIXED; 2733 } 2734 2735 void 2736 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2737 { 2738 struct unwind_state state; 2739 unsigned long addr; 2740 2741 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2742 /* TODO: We don't support guest os callchain now */ 2743 return; 2744 } 2745 2746 if (perf_callchain_store(entry, regs->ip)) 2747 return; 2748 2749 if (perf_hw_regs(regs)) 2750 unwind_start(&state, current, regs, NULL); 2751 else 2752 unwind_start(&state, current, NULL, (void *)regs->sp); 2753 2754 for (; !unwind_done(&state); unwind_next_frame(&state)) { 2755 addr = unwind_get_return_address(&state); 2756 if (!addr || perf_callchain_store(entry, addr)) 2757 return; 2758 } 2759 } 2760 2761 static inline int 2762 valid_user_frame(const void __user *fp, unsigned long size) 2763 { 2764 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2765 } 2766 2767 static unsigned long get_segment_base(unsigned int segment) 2768 { 2769 struct desc_struct *desc; 2770 unsigned int idx = segment >> 3; 2771 2772 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2773 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2774 struct ldt_struct *ldt; 2775 2776 /* IRQs are off, so this synchronizes with smp_store_release */ 2777 ldt = READ_ONCE(current->active_mm->context.ldt); 2778 if (!ldt || idx >= ldt->nr_entries) 2779 return 0; 2780 2781 desc = &ldt->entries[idx]; 2782 #else 2783 return 0; 2784 #endif 2785 } else { 2786 if (idx >= GDT_ENTRIES) 2787 return 0; 2788 2789 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2790 } 2791 2792 return get_desc_base(desc); 2793 } 2794 2795 #ifdef CONFIG_IA32_EMULATION 2796 2797 #include <linux/compat.h> 2798 2799 static inline int 2800 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2801 { 2802 /* 32-bit process in 64-bit kernel. */ 2803 unsigned long ss_base, cs_base; 2804 struct stack_frame_ia32 frame; 2805 const struct stack_frame_ia32 __user *fp; 2806 2807 if (user_64bit_mode(regs)) 2808 return 0; 2809 2810 cs_base = get_segment_base(regs->cs); 2811 ss_base = get_segment_base(regs->ss); 2812 2813 fp = compat_ptr(ss_base + regs->bp); 2814 pagefault_disable(); 2815 while (entry->nr < entry->max_stack) { 2816 if (!valid_user_frame(fp, sizeof(frame))) 2817 break; 2818 2819 if (__get_user(frame.next_frame, &fp->next_frame)) 2820 break; 2821 if (__get_user(frame.return_address, &fp->return_address)) 2822 break; 2823 2824 perf_callchain_store(entry, cs_base + frame.return_address); 2825 fp = compat_ptr(ss_base + frame.next_frame); 2826 } 2827 pagefault_enable(); 2828 return 1; 2829 } 2830 #else 2831 static inline int 2832 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2833 { 2834 return 0; 2835 } 2836 #endif 2837 2838 void 2839 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2840 { 2841 struct stack_frame frame; 2842 const struct stack_frame __user *fp; 2843 2844 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2845 /* TODO: We don't support guest os callchain now */ 2846 return; 2847 } 2848 2849 /* 2850 * We don't know what to do with VM86 stacks.. ignore them for now. 2851 */ 2852 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2853 return; 2854 2855 fp = (void __user *)regs->bp; 2856 2857 perf_callchain_store(entry, regs->ip); 2858 2859 if (!nmi_uaccess_okay()) 2860 return; 2861 2862 if (perf_callchain_user32(regs, entry)) 2863 return; 2864 2865 pagefault_disable(); 2866 while (entry->nr < entry->max_stack) { 2867 if (!valid_user_frame(fp, sizeof(frame))) 2868 break; 2869 2870 if (__get_user(frame.next_frame, &fp->next_frame)) 2871 break; 2872 if (__get_user(frame.return_address, &fp->return_address)) 2873 break; 2874 2875 perf_callchain_store(entry, frame.return_address); 2876 fp = (void __user *)frame.next_frame; 2877 } 2878 pagefault_enable(); 2879 } 2880 2881 /* 2882 * Deal with code segment offsets for the various execution modes: 2883 * 2884 * VM86 - the good olde 16 bit days, where the linear address is 2885 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2886 * 2887 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2888 * to figure out what the 32bit base address is. 2889 * 2890 * X32 - has TIF_X32 set, but is running in x86_64 2891 * 2892 * X86_64 - CS,DS,SS,ES are all zero based. 2893 */ 2894 static unsigned long code_segment_base(struct pt_regs *regs) 2895 { 2896 /* 2897 * For IA32 we look at the GDT/LDT segment base to convert the 2898 * effective IP to a linear address. 2899 */ 2900 2901 #ifdef CONFIG_X86_32 2902 /* 2903 * If we are in VM86 mode, add the segment offset to convert to a 2904 * linear address. 2905 */ 2906 if (regs->flags & X86_VM_MASK) 2907 return 0x10 * regs->cs; 2908 2909 if (user_mode(regs) && regs->cs != __USER_CS) 2910 return get_segment_base(regs->cs); 2911 #else 2912 if (user_mode(regs) && !user_64bit_mode(regs) && 2913 regs->cs != __USER32_CS) 2914 return get_segment_base(regs->cs); 2915 #endif 2916 return 0; 2917 } 2918 2919 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2920 { 2921 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2922 return perf_guest_cbs->get_guest_ip(); 2923 2924 return regs->ip + code_segment_base(regs); 2925 } 2926 2927 unsigned long perf_misc_flags(struct pt_regs *regs) 2928 { 2929 int misc = 0; 2930 2931 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2932 if (perf_guest_cbs->is_user_mode()) 2933 misc |= PERF_RECORD_MISC_GUEST_USER; 2934 else 2935 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2936 } else { 2937 if (user_mode(regs)) 2938 misc |= PERF_RECORD_MISC_USER; 2939 else 2940 misc |= PERF_RECORD_MISC_KERNEL; 2941 } 2942 2943 if (regs->flags & PERF_EFLAGS_EXACT) 2944 misc |= PERF_RECORD_MISC_EXACT_IP; 2945 2946 return misc; 2947 } 2948 2949 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2950 { 2951 cap->version = x86_pmu.version; 2952 /* 2953 * KVM doesn't support the hybrid PMU yet. 2954 * Return the common value in global x86_pmu, 2955 * which available for all cores. 2956 */ 2957 cap->num_counters_gp = x86_pmu.num_counters; 2958 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2959 cap->bit_width_gp = x86_pmu.cntval_bits; 2960 cap->bit_width_fixed = x86_pmu.cntval_bits; 2961 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2962 cap->events_mask_len = x86_pmu.events_mask_len; 2963 } 2964 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2965