1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched/mm.h> 24 #include <linux/sched/clock.h> 25 #include <linux/uaccess.h> 26 #include <linux/slab.h> 27 #include <linux/cpu.h> 28 #include <linux/bitops.h> 29 #include <linux/device.h> 30 #include <linux/nospec.h> 31 #include <linux/static_call.h> 32 33 #include <asm/apic.h> 34 #include <asm/stacktrace.h> 35 #include <asm/nmi.h> 36 #include <asm/smp.h> 37 #include <asm/alternative.h> 38 #include <asm/mmu_context.h> 39 #include <asm/tlbflush.h> 40 #include <asm/timer.h> 41 #include <asm/desc.h> 42 #include <asm/ldt.h> 43 #include <asm/unwind.h> 44 45 #include "perf_event.h" 46 47 struct x86_pmu x86_pmu __read_mostly; 48 49 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 50 .enabled = 1, 51 }; 52 53 DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key); 54 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); 55 56 /* 57 * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined 58 * from just a typename, as opposed to an actual function. 59 */ 60 DEFINE_STATIC_CALL_NULL(x86_pmu_handle_irq, *x86_pmu.handle_irq); 61 DEFINE_STATIC_CALL_NULL(x86_pmu_disable_all, *x86_pmu.disable_all); 62 DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all, *x86_pmu.enable_all); 63 DEFINE_STATIC_CALL_NULL(x86_pmu_enable, *x86_pmu.enable); 64 DEFINE_STATIC_CALL_NULL(x86_pmu_disable, *x86_pmu.disable); 65 66 DEFINE_STATIC_CALL_NULL(x86_pmu_add, *x86_pmu.add); 67 DEFINE_STATIC_CALL_NULL(x86_pmu_del, *x86_pmu.del); 68 DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read); 69 70 DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events, *x86_pmu.schedule_events); 71 DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints); 72 DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints); 73 74 DEFINE_STATIC_CALL_NULL(x86_pmu_start_scheduling, *x86_pmu.start_scheduling); 75 DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling); 76 DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling); 77 78 DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task); 79 DEFINE_STATIC_CALL_NULL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx); 80 81 DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); 82 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases); 83 84 DEFINE_STATIC_CALL_NULL(x86_pmu_guest_get_msrs, *x86_pmu.guest_get_msrs); 85 86 u64 __read_mostly hw_cache_event_ids 87 [PERF_COUNT_HW_CACHE_MAX] 88 [PERF_COUNT_HW_CACHE_OP_MAX] 89 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 90 u64 __read_mostly hw_cache_extra_regs 91 [PERF_COUNT_HW_CACHE_MAX] 92 [PERF_COUNT_HW_CACHE_OP_MAX] 93 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 94 95 /* 96 * Propagate event elapsed time into the generic event. 97 * Can only be executed on the CPU where the event is active. 98 * Returns the delta events processed. 99 */ 100 u64 x86_perf_event_update(struct perf_event *event) 101 { 102 struct hw_perf_event *hwc = &event->hw; 103 int shift = 64 - x86_pmu.cntval_bits; 104 u64 prev_raw_count, new_raw_count; 105 u64 delta; 106 107 if (unlikely(!hwc->event_base)) 108 return 0; 109 110 if (unlikely(is_topdown_count(event)) && x86_pmu.update_topdown_event) 111 return x86_pmu.update_topdown_event(event); 112 113 /* 114 * Careful: an NMI might modify the previous event value. 115 * 116 * Our tactic to handle this is to first atomically read and 117 * exchange a new raw count - then add that new-prev delta 118 * count to the generic event atomically: 119 */ 120 again: 121 prev_raw_count = local64_read(&hwc->prev_count); 122 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 123 124 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 125 new_raw_count) != prev_raw_count) 126 goto again; 127 128 /* 129 * Now we have the new raw value and have updated the prev 130 * timestamp already. We can now calculate the elapsed delta 131 * (event-)time and add that to the generic event. 132 * 133 * Careful, not all hw sign-extends above the physical width 134 * of the count. 135 */ 136 delta = (new_raw_count << shift) - (prev_raw_count << shift); 137 delta >>= shift; 138 139 local64_add(delta, &event->count); 140 local64_sub(delta, &hwc->period_left); 141 142 return new_raw_count; 143 } 144 145 /* 146 * Find and validate any extra registers to set up. 147 */ 148 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 149 { 150 struct hw_perf_event_extra *reg; 151 struct extra_reg *er; 152 153 reg = &event->hw.extra_reg; 154 155 if (!x86_pmu.extra_regs) 156 return 0; 157 158 for (er = x86_pmu.extra_regs; er->msr; er++) { 159 if (er->event != (config & er->config_mask)) 160 continue; 161 if (event->attr.config1 & ~er->valid_mask) 162 return -EINVAL; 163 /* Check if the extra msrs can be safely accessed*/ 164 if (!er->extra_msr_access) 165 return -ENXIO; 166 167 reg->idx = er->idx; 168 reg->config = event->attr.config1; 169 reg->reg = er->msr; 170 break; 171 } 172 return 0; 173 } 174 175 static atomic_t active_events; 176 static atomic_t pmc_refcount; 177 static DEFINE_MUTEX(pmc_reserve_mutex); 178 179 #ifdef CONFIG_X86_LOCAL_APIC 180 181 static bool reserve_pmc_hardware(void) 182 { 183 int i; 184 185 for (i = 0; i < x86_pmu.num_counters; i++) { 186 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 187 goto perfctr_fail; 188 } 189 190 for (i = 0; i < x86_pmu.num_counters; i++) { 191 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 192 goto eventsel_fail; 193 } 194 195 return true; 196 197 eventsel_fail: 198 for (i--; i >= 0; i--) 199 release_evntsel_nmi(x86_pmu_config_addr(i)); 200 201 i = x86_pmu.num_counters; 202 203 perfctr_fail: 204 for (i--; i >= 0; i--) 205 release_perfctr_nmi(x86_pmu_event_addr(i)); 206 207 return false; 208 } 209 210 static void release_pmc_hardware(void) 211 { 212 int i; 213 214 for (i = 0; i < x86_pmu.num_counters; i++) { 215 release_perfctr_nmi(x86_pmu_event_addr(i)); 216 release_evntsel_nmi(x86_pmu_config_addr(i)); 217 } 218 } 219 220 #else 221 222 static bool reserve_pmc_hardware(void) { return true; } 223 static void release_pmc_hardware(void) {} 224 225 #endif 226 227 static bool check_hw_exists(void) 228 { 229 u64 val, val_fail = -1, val_new= ~0; 230 int i, reg, reg_fail = -1, ret = 0; 231 int bios_fail = 0; 232 int reg_safe = -1; 233 234 /* 235 * Check to see if the BIOS enabled any of the counters, if so 236 * complain and bail. 237 */ 238 for (i = 0; i < x86_pmu.num_counters; i++) { 239 reg = x86_pmu_config_addr(i); 240 ret = rdmsrl_safe(reg, &val); 241 if (ret) 242 goto msr_fail; 243 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 244 bios_fail = 1; 245 val_fail = val; 246 reg_fail = reg; 247 } else { 248 reg_safe = i; 249 } 250 } 251 252 if (x86_pmu.num_counters_fixed) { 253 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 254 ret = rdmsrl_safe(reg, &val); 255 if (ret) 256 goto msr_fail; 257 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 258 if (fixed_counter_disabled(i)) 259 continue; 260 if (val & (0x03 << i*4)) { 261 bios_fail = 1; 262 val_fail = val; 263 reg_fail = reg; 264 } 265 } 266 } 267 268 /* 269 * If all the counters are enabled, the below test will always 270 * fail. The tools will also become useless in this scenario. 271 * Just fail and disable the hardware counters. 272 */ 273 274 if (reg_safe == -1) { 275 reg = reg_safe; 276 goto msr_fail; 277 } 278 279 /* 280 * Read the current value, change it and read it back to see if it 281 * matches, this is needed to detect certain hardware emulators 282 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 283 */ 284 reg = x86_pmu_event_addr(reg_safe); 285 if (rdmsrl_safe(reg, &val)) 286 goto msr_fail; 287 val ^= 0xffffUL; 288 ret = wrmsrl_safe(reg, val); 289 ret |= rdmsrl_safe(reg, &val_new); 290 if (ret || val != val_new) 291 goto msr_fail; 292 293 /* 294 * We still allow the PMU driver to operate: 295 */ 296 if (bios_fail) { 297 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 298 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 299 reg_fail, val_fail); 300 } 301 302 return true; 303 304 msr_fail: 305 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 306 pr_cont("PMU not available due to virtualization, using software events only.\n"); 307 } else { 308 pr_cont("Broken PMU hardware detected, using software events only.\n"); 309 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 310 reg, val_new); 311 } 312 313 return false; 314 } 315 316 static void hw_perf_event_destroy(struct perf_event *event) 317 { 318 x86_release_hardware(); 319 atomic_dec(&active_events); 320 } 321 322 void hw_perf_lbr_event_destroy(struct perf_event *event) 323 { 324 hw_perf_event_destroy(event); 325 326 /* undo the lbr/bts event accounting */ 327 x86_del_exclusive(x86_lbr_exclusive_lbr); 328 } 329 330 static inline int x86_pmu_initialized(void) 331 { 332 return x86_pmu.handle_irq != NULL; 333 } 334 335 static inline int 336 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 337 { 338 struct perf_event_attr *attr = &event->attr; 339 unsigned int cache_type, cache_op, cache_result; 340 u64 config, val; 341 342 config = attr->config; 343 344 cache_type = (config >> 0) & 0xff; 345 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 346 return -EINVAL; 347 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX); 348 349 cache_op = (config >> 8) & 0xff; 350 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 351 return -EINVAL; 352 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX); 353 354 cache_result = (config >> 16) & 0xff; 355 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 356 return -EINVAL; 357 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX); 358 359 val = hw_cache_event_ids[cache_type][cache_op][cache_result]; 360 361 if (val == 0) 362 return -ENOENT; 363 364 if (val == -1) 365 return -EINVAL; 366 367 hwc->config |= val; 368 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; 369 return x86_pmu_extra_regs(val, event); 370 } 371 372 int x86_reserve_hardware(void) 373 { 374 int err = 0; 375 376 if (!atomic_inc_not_zero(&pmc_refcount)) { 377 mutex_lock(&pmc_reserve_mutex); 378 if (atomic_read(&pmc_refcount) == 0) { 379 if (!reserve_pmc_hardware()) 380 err = -EBUSY; 381 else 382 reserve_ds_buffers(); 383 } 384 if (!err) 385 atomic_inc(&pmc_refcount); 386 mutex_unlock(&pmc_reserve_mutex); 387 } 388 389 return err; 390 } 391 392 void x86_release_hardware(void) 393 { 394 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 395 release_pmc_hardware(); 396 release_ds_buffers(); 397 release_lbr_buffers(); 398 mutex_unlock(&pmc_reserve_mutex); 399 } 400 } 401 402 /* 403 * Check if we can create event of a certain type (that no conflicting events 404 * are present). 405 */ 406 int x86_add_exclusive(unsigned int what) 407 { 408 int i; 409 410 /* 411 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 412 * LBR and BTS are still mutually exclusive. 413 */ 414 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 415 goto out; 416 417 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 418 mutex_lock(&pmc_reserve_mutex); 419 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 420 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 421 goto fail_unlock; 422 } 423 atomic_inc(&x86_pmu.lbr_exclusive[what]); 424 mutex_unlock(&pmc_reserve_mutex); 425 } 426 427 out: 428 atomic_inc(&active_events); 429 return 0; 430 431 fail_unlock: 432 mutex_unlock(&pmc_reserve_mutex); 433 return -EBUSY; 434 } 435 436 void x86_del_exclusive(unsigned int what) 437 { 438 atomic_dec(&active_events); 439 440 /* 441 * See the comment in x86_add_exclusive(). 442 */ 443 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 444 return; 445 446 atomic_dec(&x86_pmu.lbr_exclusive[what]); 447 } 448 449 int x86_setup_perfctr(struct perf_event *event) 450 { 451 struct perf_event_attr *attr = &event->attr; 452 struct hw_perf_event *hwc = &event->hw; 453 u64 config; 454 455 if (!is_sampling_event(event)) { 456 hwc->sample_period = x86_pmu.max_period; 457 hwc->last_period = hwc->sample_period; 458 local64_set(&hwc->period_left, hwc->sample_period); 459 } 460 461 if (attr->type == PERF_TYPE_RAW) 462 return x86_pmu_extra_regs(event->attr.config, event); 463 464 if (attr->type == PERF_TYPE_HW_CACHE) 465 return set_ext_hw_attr(hwc, event); 466 467 if (attr->config >= x86_pmu.max_events) 468 return -EINVAL; 469 470 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); 471 472 /* 473 * The generic map: 474 */ 475 config = x86_pmu.event_map(attr->config); 476 477 if (config == 0) 478 return -ENOENT; 479 480 if (config == -1LL) 481 return -EINVAL; 482 483 hwc->config |= config; 484 485 return 0; 486 } 487 488 /* 489 * check that branch_sample_type is compatible with 490 * settings needed for precise_ip > 1 which implies 491 * using the LBR to capture ALL taken branches at the 492 * priv levels of the measurement 493 */ 494 static inline int precise_br_compat(struct perf_event *event) 495 { 496 u64 m = event->attr.branch_sample_type; 497 u64 b = 0; 498 499 /* must capture all branches */ 500 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 501 return 0; 502 503 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 504 505 if (!event->attr.exclude_user) 506 b |= PERF_SAMPLE_BRANCH_USER; 507 508 if (!event->attr.exclude_kernel) 509 b |= PERF_SAMPLE_BRANCH_KERNEL; 510 511 /* 512 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 513 */ 514 515 return m == b; 516 } 517 518 int x86_pmu_max_precise(void) 519 { 520 int precise = 0; 521 522 /* Support for constant skid */ 523 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 524 precise++; 525 526 /* Support for IP fixup */ 527 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 528 precise++; 529 530 if (x86_pmu.pebs_prec_dist) 531 precise++; 532 } 533 return precise; 534 } 535 536 int x86_pmu_hw_config(struct perf_event *event) 537 { 538 if (event->attr.precise_ip) { 539 int precise = x86_pmu_max_precise(); 540 541 if (event->attr.precise_ip > precise) 542 return -EOPNOTSUPP; 543 544 /* There's no sense in having PEBS for non sampling events: */ 545 if (!is_sampling_event(event)) 546 return -EINVAL; 547 } 548 /* 549 * check that PEBS LBR correction does not conflict with 550 * whatever the user is asking with attr->branch_sample_type 551 */ 552 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 553 u64 *br_type = &event->attr.branch_sample_type; 554 555 if (has_branch_stack(event)) { 556 if (!precise_br_compat(event)) 557 return -EOPNOTSUPP; 558 559 /* branch_sample_type is compatible */ 560 561 } else { 562 /* 563 * user did not specify branch_sample_type 564 * 565 * For PEBS fixups, we capture all 566 * the branches at the priv level of the 567 * event. 568 */ 569 *br_type = PERF_SAMPLE_BRANCH_ANY; 570 571 if (!event->attr.exclude_user) 572 *br_type |= PERF_SAMPLE_BRANCH_USER; 573 574 if (!event->attr.exclude_kernel) 575 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 576 } 577 } 578 579 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 580 event->attach_state |= PERF_ATTACH_TASK_DATA; 581 582 /* 583 * Generate PMC IRQs: 584 * (keep 'enabled' bit clear for now) 585 */ 586 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 587 588 /* 589 * Count user and OS events unless requested not to 590 */ 591 if (!event->attr.exclude_user) 592 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 593 if (!event->attr.exclude_kernel) 594 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 595 596 if (event->attr.type == PERF_TYPE_RAW) 597 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 598 599 if (event->attr.sample_period && x86_pmu.limit_period) { 600 if (x86_pmu.limit_period(event, event->attr.sample_period) > 601 event->attr.sample_period) 602 return -EINVAL; 603 } 604 605 /* sample_regs_user never support XMM registers */ 606 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) 607 return -EINVAL; 608 /* 609 * Besides the general purpose registers, XMM registers may 610 * be collected in PEBS on some platforms, e.g. Icelake 611 */ 612 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) { 613 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) 614 return -EINVAL; 615 616 if (!event->attr.precise_ip) 617 return -EINVAL; 618 } 619 620 return x86_setup_perfctr(event); 621 } 622 623 /* 624 * Setup the hardware configuration for a given attr_type 625 */ 626 static int __x86_pmu_event_init(struct perf_event *event) 627 { 628 int err; 629 630 if (!x86_pmu_initialized()) 631 return -ENODEV; 632 633 err = x86_reserve_hardware(); 634 if (err) 635 return err; 636 637 atomic_inc(&active_events); 638 event->destroy = hw_perf_event_destroy; 639 640 event->hw.idx = -1; 641 event->hw.last_cpu = -1; 642 event->hw.last_tag = ~0ULL; 643 644 /* mark unused */ 645 event->hw.extra_reg.idx = EXTRA_REG_NONE; 646 event->hw.branch_reg.idx = EXTRA_REG_NONE; 647 648 return x86_pmu.hw_config(event); 649 } 650 651 void x86_pmu_disable_all(void) 652 { 653 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 654 int idx; 655 656 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 657 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 658 u64 val; 659 660 if (!test_bit(idx, cpuc->active_mask)) 661 continue; 662 rdmsrl(x86_pmu_config_addr(idx), val); 663 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 664 continue; 665 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 666 wrmsrl(x86_pmu_config_addr(idx), val); 667 if (is_counter_pair(hwc)) 668 wrmsrl(x86_pmu_config_addr(idx + 1), 0); 669 } 670 } 671 672 struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr) 673 { 674 return static_call(x86_pmu_guest_get_msrs)(nr); 675 } 676 EXPORT_SYMBOL_GPL(perf_guest_get_msrs); 677 678 /* 679 * There may be PMI landing after enabled=0. The PMI hitting could be before or 680 * after disable_all. 681 * 682 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 683 * It will not be re-enabled in the NMI handler again, because enabled=0. After 684 * handling the NMI, disable_all will be called, which will not change the 685 * state either. If PMI hits after disable_all, the PMU is already disabled 686 * before entering NMI handler. The NMI handler will not change the state 687 * either. 688 * 689 * So either situation is harmless. 690 */ 691 static void x86_pmu_disable(struct pmu *pmu) 692 { 693 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 694 695 if (!x86_pmu_initialized()) 696 return; 697 698 if (!cpuc->enabled) 699 return; 700 701 cpuc->n_added = 0; 702 cpuc->enabled = 0; 703 barrier(); 704 705 static_call(x86_pmu_disable_all)(); 706 } 707 708 void x86_pmu_enable_all(int added) 709 { 710 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 711 int idx; 712 713 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 714 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 715 716 if (!test_bit(idx, cpuc->active_mask)) 717 continue; 718 719 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 720 } 721 } 722 723 static struct pmu pmu; 724 725 static inline int is_x86_event(struct perf_event *event) 726 { 727 return event->pmu == &pmu; 728 } 729 730 struct pmu *x86_get_pmu(void) 731 { 732 return &pmu; 733 } 734 /* 735 * Event scheduler state: 736 * 737 * Assign events iterating over all events and counters, beginning 738 * with events with least weights first. Keep the current iterator 739 * state in struct sched_state. 740 */ 741 struct sched_state { 742 int weight; 743 int event; /* event index */ 744 int counter; /* counter index */ 745 int unassigned; /* number of events to be assigned left */ 746 int nr_gp; /* number of GP counters used */ 747 u64 used; 748 }; 749 750 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 751 #define SCHED_STATES_MAX 2 752 753 struct perf_sched { 754 int max_weight; 755 int max_events; 756 int max_gp; 757 int saved_states; 758 struct event_constraint **constraints; 759 struct sched_state state; 760 struct sched_state saved[SCHED_STATES_MAX]; 761 }; 762 763 /* 764 * Initialize interator that runs through all events and counters. 765 */ 766 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 767 int num, int wmin, int wmax, int gpmax) 768 { 769 int idx; 770 771 memset(sched, 0, sizeof(*sched)); 772 sched->max_events = num; 773 sched->max_weight = wmax; 774 sched->max_gp = gpmax; 775 sched->constraints = constraints; 776 777 for (idx = 0; idx < num; idx++) { 778 if (constraints[idx]->weight == wmin) 779 break; 780 } 781 782 sched->state.event = idx; /* start with min weight */ 783 sched->state.weight = wmin; 784 sched->state.unassigned = num; 785 } 786 787 static void perf_sched_save_state(struct perf_sched *sched) 788 { 789 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 790 return; 791 792 sched->saved[sched->saved_states] = sched->state; 793 sched->saved_states++; 794 } 795 796 static bool perf_sched_restore_state(struct perf_sched *sched) 797 { 798 if (!sched->saved_states) 799 return false; 800 801 sched->saved_states--; 802 sched->state = sched->saved[sched->saved_states]; 803 804 /* this assignment didn't work out */ 805 /* XXX broken vs EVENT_PAIR */ 806 sched->state.used &= ~BIT_ULL(sched->state.counter); 807 808 /* try the next one */ 809 sched->state.counter++; 810 811 return true; 812 } 813 814 /* 815 * Select a counter for the current event to schedule. Return true on 816 * success. 817 */ 818 static bool __perf_sched_find_counter(struct perf_sched *sched) 819 { 820 struct event_constraint *c; 821 int idx; 822 823 if (!sched->state.unassigned) 824 return false; 825 826 if (sched->state.event >= sched->max_events) 827 return false; 828 829 c = sched->constraints[sched->state.event]; 830 /* Prefer fixed purpose counters */ 831 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 832 idx = INTEL_PMC_IDX_FIXED; 833 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 834 u64 mask = BIT_ULL(idx); 835 836 if (sched->state.used & mask) 837 continue; 838 839 sched->state.used |= mask; 840 goto done; 841 } 842 } 843 844 /* Grab the first unused counter starting with idx */ 845 idx = sched->state.counter; 846 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 847 u64 mask = BIT_ULL(idx); 848 849 if (c->flags & PERF_X86_EVENT_PAIR) 850 mask |= mask << 1; 851 852 if (sched->state.used & mask) 853 continue; 854 855 if (sched->state.nr_gp++ >= sched->max_gp) 856 return false; 857 858 sched->state.used |= mask; 859 goto done; 860 } 861 862 return false; 863 864 done: 865 sched->state.counter = idx; 866 867 if (c->overlap) 868 perf_sched_save_state(sched); 869 870 return true; 871 } 872 873 static bool perf_sched_find_counter(struct perf_sched *sched) 874 { 875 while (!__perf_sched_find_counter(sched)) { 876 if (!perf_sched_restore_state(sched)) 877 return false; 878 } 879 880 return true; 881 } 882 883 /* 884 * Go through all unassigned events and find the next one to schedule. 885 * Take events with the least weight first. Return true on success. 886 */ 887 static bool perf_sched_next_event(struct perf_sched *sched) 888 { 889 struct event_constraint *c; 890 891 if (!sched->state.unassigned || !--sched->state.unassigned) 892 return false; 893 894 do { 895 /* next event */ 896 sched->state.event++; 897 if (sched->state.event >= sched->max_events) { 898 /* next weight */ 899 sched->state.event = 0; 900 sched->state.weight++; 901 if (sched->state.weight > sched->max_weight) 902 return false; 903 } 904 c = sched->constraints[sched->state.event]; 905 } while (c->weight != sched->state.weight); 906 907 sched->state.counter = 0; /* start with first counter */ 908 909 return true; 910 } 911 912 /* 913 * Assign a counter for each event. 914 */ 915 int perf_assign_events(struct event_constraint **constraints, int n, 916 int wmin, int wmax, int gpmax, int *assign) 917 { 918 struct perf_sched sched; 919 920 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 921 922 do { 923 if (!perf_sched_find_counter(&sched)) 924 break; /* failed */ 925 if (assign) 926 assign[sched.state.event] = sched.state.counter; 927 } while (perf_sched_next_event(&sched)); 928 929 return sched.state.unassigned; 930 } 931 EXPORT_SYMBOL_GPL(perf_assign_events); 932 933 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 934 { 935 struct event_constraint *c; 936 struct perf_event *e; 937 int n0, i, wmin, wmax, unsched = 0; 938 struct hw_perf_event *hwc; 939 u64 used_mask = 0; 940 941 /* 942 * Compute the number of events already present; see x86_pmu_add(), 943 * validate_group() and x86_pmu_commit_txn(). For the former two 944 * cpuc->n_events hasn't been updated yet, while for the latter 945 * cpuc->n_txn contains the number of events added in the current 946 * transaction. 947 */ 948 n0 = cpuc->n_events; 949 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 950 n0 -= cpuc->n_txn; 951 952 static_call_cond(x86_pmu_start_scheduling)(cpuc); 953 954 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 955 c = cpuc->event_constraint[i]; 956 957 /* 958 * Previously scheduled events should have a cached constraint, 959 * while new events should not have one. 960 */ 961 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0)); 962 963 /* 964 * Request constraints for new events; or for those events that 965 * have a dynamic constraint -- for those the constraint can 966 * change due to external factors (sibling state, allow_tfa). 967 */ 968 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) { 969 c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]); 970 cpuc->event_constraint[i] = c; 971 } 972 973 wmin = min(wmin, c->weight); 974 wmax = max(wmax, c->weight); 975 } 976 977 /* 978 * fastpath, try to reuse previous register 979 */ 980 for (i = 0; i < n; i++) { 981 u64 mask; 982 983 hwc = &cpuc->event_list[i]->hw; 984 c = cpuc->event_constraint[i]; 985 986 /* never assigned */ 987 if (hwc->idx == -1) 988 break; 989 990 /* constraint still honored */ 991 if (!test_bit(hwc->idx, c->idxmsk)) 992 break; 993 994 mask = BIT_ULL(hwc->idx); 995 if (is_counter_pair(hwc)) 996 mask |= mask << 1; 997 998 /* not already used */ 999 if (used_mask & mask) 1000 break; 1001 1002 used_mask |= mask; 1003 1004 if (assign) 1005 assign[i] = hwc->idx; 1006 } 1007 1008 /* slow path */ 1009 if (i != n) { 1010 int gpmax = x86_pmu.num_counters; 1011 1012 /* 1013 * Do not allow scheduling of more than half the available 1014 * generic counters. 1015 * 1016 * This helps avoid counter starvation of sibling thread by 1017 * ensuring at most half the counters cannot be in exclusive 1018 * mode. There is no designated counters for the limits. Any 1019 * N/2 counters can be used. This helps with events with 1020 * specific counter constraints. 1021 */ 1022 if (is_ht_workaround_enabled() && !cpuc->is_fake && 1023 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 1024 gpmax /= 2; 1025 1026 /* 1027 * Reduce the amount of available counters to allow fitting 1028 * the extra Merge events needed by large increment events. 1029 */ 1030 if (x86_pmu.flags & PMU_FL_PAIR) { 1031 gpmax = x86_pmu.num_counters - cpuc->n_pair; 1032 WARN_ON(gpmax <= 0); 1033 } 1034 1035 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 1036 wmax, gpmax, assign); 1037 } 1038 1039 /* 1040 * In case of success (unsched = 0), mark events as committed, 1041 * so we do not put_constraint() in case new events are added 1042 * and fail to be scheduled 1043 * 1044 * We invoke the lower level commit callback to lock the resource 1045 * 1046 * We do not need to do all of this in case we are called to 1047 * validate an event group (assign == NULL) 1048 */ 1049 if (!unsched && assign) { 1050 for (i = 0; i < n; i++) { 1051 e = cpuc->event_list[i]; 1052 static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]); 1053 } 1054 } else { 1055 for (i = n0; i < n; i++) { 1056 e = cpuc->event_list[i]; 1057 1058 /* 1059 * release events that failed scheduling 1060 */ 1061 static_call_cond(x86_pmu_put_event_constraints)(cpuc, e); 1062 1063 cpuc->event_constraint[i] = NULL; 1064 } 1065 } 1066 1067 static_call_cond(x86_pmu_stop_scheduling)(cpuc); 1068 1069 return unsched ? -EINVAL : 0; 1070 } 1071 1072 static int add_nr_metric_event(struct cpu_hw_events *cpuc, 1073 struct perf_event *event) 1074 { 1075 if (is_metric_event(event)) { 1076 if (cpuc->n_metric == INTEL_TD_METRIC_NUM) 1077 return -EINVAL; 1078 cpuc->n_metric++; 1079 cpuc->n_txn_metric++; 1080 } 1081 1082 return 0; 1083 } 1084 1085 static void del_nr_metric_event(struct cpu_hw_events *cpuc, 1086 struct perf_event *event) 1087 { 1088 if (is_metric_event(event)) 1089 cpuc->n_metric--; 1090 } 1091 1092 static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event, 1093 int max_count, int n) 1094 { 1095 1096 if (x86_pmu.intel_cap.perf_metrics && add_nr_metric_event(cpuc, event)) 1097 return -EINVAL; 1098 1099 if (n >= max_count + cpuc->n_metric) 1100 return -EINVAL; 1101 1102 cpuc->event_list[n] = event; 1103 if (is_counter_pair(&event->hw)) { 1104 cpuc->n_pair++; 1105 cpuc->n_txn_pair++; 1106 } 1107 1108 return 0; 1109 } 1110 1111 /* 1112 * dogrp: true if must collect siblings events (group) 1113 * returns total number of events and error code 1114 */ 1115 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 1116 { 1117 struct perf_event *event; 1118 int n, max_count; 1119 1120 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; 1121 1122 /* current number of events already accepted */ 1123 n = cpuc->n_events; 1124 if (!cpuc->n_events) 1125 cpuc->pebs_output = 0; 1126 1127 if (!cpuc->is_fake && leader->attr.precise_ip) { 1128 /* 1129 * For PEBS->PT, if !aux_event, the group leader (PT) went 1130 * away, the group was broken down and this singleton event 1131 * can't schedule any more. 1132 */ 1133 if (is_pebs_pt(leader) && !leader->aux_event) 1134 return -EINVAL; 1135 1136 /* 1137 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS 1138 */ 1139 if (cpuc->pebs_output && 1140 cpuc->pebs_output != is_pebs_pt(leader) + 1) 1141 return -EINVAL; 1142 1143 cpuc->pebs_output = is_pebs_pt(leader) + 1; 1144 } 1145 1146 if (is_x86_event(leader)) { 1147 if (collect_event(cpuc, leader, max_count, n)) 1148 return -EINVAL; 1149 n++; 1150 } 1151 1152 if (!dogrp) 1153 return n; 1154 1155 for_each_sibling_event(event, leader) { 1156 if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF) 1157 continue; 1158 1159 if (collect_event(cpuc, event, max_count, n)) 1160 return -EINVAL; 1161 1162 n++; 1163 } 1164 return n; 1165 } 1166 1167 static inline void x86_assign_hw_event(struct perf_event *event, 1168 struct cpu_hw_events *cpuc, int i) 1169 { 1170 struct hw_perf_event *hwc = &event->hw; 1171 int idx; 1172 1173 idx = hwc->idx = cpuc->assign[i]; 1174 hwc->last_cpu = smp_processor_id(); 1175 hwc->last_tag = ++cpuc->tags[i]; 1176 1177 switch (hwc->idx) { 1178 case INTEL_PMC_IDX_FIXED_BTS: 1179 case INTEL_PMC_IDX_FIXED_VLBR: 1180 hwc->config_base = 0; 1181 hwc->event_base = 0; 1182 break; 1183 1184 case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END: 1185 /* All the metric events are mapped onto the fixed counter 3. */ 1186 idx = INTEL_PMC_IDX_FIXED_SLOTS; 1187 fallthrough; 1188 case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1: 1189 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1190 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + 1191 (idx - INTEL_PMC_IDX_FIXED); 1192 hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) | 1193 INTEL_PMC_FIXED_RDPMC_BASE; 1194 break; 1195 1196 default: 1197 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1198 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1199 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1200 break; 1201 } 1202 } 1203 1204 /** 1205 * x86_perf_rdpmc_index - Return PMC counter used for event 1206 * @event: the perf_event to which the PMC counter was assigned 1207 * 1208 * The counter assigned to this performance event may change if interrupts 1209 * are enabled. This counter should thus never be used while interrupts are 1210 * enabled. Before this function is used to obtain the assigned counter the 1211 * event should be checked for validity using, for example, 1212 * perf_event_read_local(), within the same interrupt disabled section in 1213 * which this counter is planned to be used. 1214 * 1215 * Return: The index of the performance monitoring counter assigned to 1216 * @perf_event. 1217 */ 1218 int x86_perf_rdpmc_index(struct perf_event *event) 1219 { 1220 lockdep_assert_irqs_disabled(); 1221 1222 return event->hw.event_base_rdpmc; 1223 } 1224 1225 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1226 struct cpu_hw_events *cpuc, 1227 int i) 1228 { 1229 return hwc->idx == cpuc->assign[i] && 1230 hwc->last_cpu == smp_processor_id() && 1231 hwc->last_tag == cpuc->tags[i]; 1232 } 1233 1234 static void x86_pmu_start(struct perf_event *event, int flags); 1235 1236 static void x86_pmu_enable(struct pmu *pmu) 1237 { 1238 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1239 struct perf_event *event; 1240 struct hw_perf_event *hwc; 1241 int i, added = cpuc->n_added; 1242 1243 if (!x86_pmu_initialized()) 1244 return; 1245 1246 if (cpuc->enabled) 1247 return; 1248 1249 if (cpuc->n_added) { 1250 int n_running = cpuc->n_events - cpuc->n_added; 1251 /* 1252 * apply assignment obtained either from 1253 * hw_perf_group_sched_in() or x86_pmu_enable() 1254 * 1255 * step1: save events moving to new counters 1256 */ 1257 for (i = 0; i < n_running; i++) { 1258 event = cpuc->event_list[i]; 1259 hwc = &event->hw; 1260 1261 /* 1262 * we can avoid reprogramming counter if: 1263 * - assigned same counter as last time 1264 * - running on same CPU as last time 1265 * - no other event has used the counter since 1266 */ 1267 if (hwc->idx == -1 || 1268 match_prev_assignment(hwc, cpuc, i)) 1269 continue; 1270 1271 /* 1272 * Ensure we don't accidentally enable a stopped 1273 * counter simply because we rescheduled. 1274 */ 1275 if (hwc->state & PERF_HES_STOPPED) 1276 hwc->state |= PERF_HES_ARCH; 1277 1278 x86_pmu_stop(event, PERF_EF_UPDATE); 1279 } 1280 1281 /* 1282 * step2: reprogram moved events into new counters 1283 */ 1284 for (i = 0; i < cpuc->n_events; i++) { 1285 event = cpuc->event_list[i]; 1286 hwc = &event->hw; 1287 1288 if (!match_prev_assignment(hwc, cpuc, i)) 1289 x86_assign_hw_event(event, cpuc, i); 1290 else if (i < n_running) 1291 continue; 1292 1293 if (hwc->state & PERF_HES_ARCH) 1294 continue; 1295 1296 x86_pmu_start(event, PERF_EF_RELOAD); 1297 } 1298 cpuc->n_added = 0; 1299 perf_events_lapic_init(); 1300 } 1301 1302 cpuc->enabled = 1; 1303 barrier(); 1304 1305 static_call(x86_pmu_enable_all)(added); 1306 } 1307 1308 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1309 1310 /* 1311 * Set the next IRQ period, based on the hwc->period_left value. 1312 * To be called with the event disabled in hw: 1313 */ 1314 int x86_perf_event_set_period(struct perf_event *event) 1315 { 1316 struct hw_perf_event *hwc = &event->hw; 1317 s64 left = local64_read(&hwc->period_left); 1318 s64 period = hwc->sample_period; 1319 int ret = 0, idx = hwc->idx; 1320 1321 if (unlikely(!hwc->event_base)) 1322 return 0; 1323 1324 if (unlikely(is_topdown_count(event)) && 1325 x86_pmu.set_topdown_event_period) 1326 return x86_pmu.set_topdown_event_period(event); 1327 1328 /* 1329 * If we are way outside a reasonable range then just skip forward: 1330 */ 1331 if (unlikely(left <= -period)) { 1332 left = period; 1333 local64_set(&hwc->period_left, left); 1334 hwc->last_period = period; 1335 ret = 1; 1336 } 1337 1338 if (unlikely(left <= 0)) { 1339 left += period; 1340 local64_set(&hwc->period_left, left); 1341 hwc->last_period = period; 1342 ret = 1; 1343 } 1344 /* 1345 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1346 */ 1347 if (unlikely(left < 2)) 1348 left = 2; 1349 1350 if (left > x86_pmu.max_period) 1351 left = x86_pmu.max_period; 1352 1353 if (x86_pmu.limit_period) 1354 left = x86_pmu.limit_period(event, left); 1355 1356 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1357 1358 /* 1359 * The hw event starts counting from this event offset, 1360 * mark it to be able to extra future deltas: 1361 */ 1362 local64_set(&hwc->prev_count, (u64)-left); 1363 1364 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1365 1366 /* 1367 * Sign extend the Merge event counter's upper 16 bits since 1368 * we currently declare a 48-bit counter width 1369 */ 1370 if (is_counter_pair(hwc)) 1371 wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff); 1372 1373 /* 1374 * Due to erratum on certan cpu we need 1375 * a second write to be sure the register 1376 * is updated properly 1377 */ 1378 if (x86_pmu.perfctr_second_write) { 1379 wrmsrl(hwc->event_base, 1380 (u64)(-left) & x86_pmu.cntval_mask); 1381 } 1382 1383 perf_event_update_userpage(event); 1384 1385 return ret; 1386 } 1387 1388 void x86_pmu_enable_event(struct perf_event *event) 1389 { 1390 if (__this_cpu_read(cpu_hw_events.enabled)) 1391 __x86_pmu_enable_event(&event->hw, 1392 ARCH_PERFMON_EVENTSEL_ENABLE); 1393 } 1394 1395 /* 1396 * Add a single event to the PMU. 1397 * 1398 * The event is added to the group of enabled events 1399 * but only if it can be scheduled with existing events. 1400 */ 1401 static int x86_pmu_add(struct perf_event *event, int flags) 1402 { 1403 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1404 struct hw_perf_event *hwc; 1405 int assign[X86_PMC_IDX_MAX]; 1406 int n, n0, ret; 1407 1408 hwc = &event->hw; 1409 1410 n0 = cpuc->n_events; 1411 ret = n = collect_events(cpuc, event, false); 1412 if (ret < 0) 1413 goto out; 1414 1415 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1416 if (!(flags & PERF_EF_START)) 1417 hwc->state |= PERF_HES_ARCH; 1418 1419 /* 1420 * If group events scheduling transaction was started, 1421 * skip the schedulability test here, it will be performed 1422 * at commit time (->commit_txn) as a whole. 1423 * 1424 * If commit fails, we'll call ->del() on all events 1425 * for which ->add() was called. 1426 */ 1427 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1428 goto done_collect; 1429 1430 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign); 1431 if (ret) 1432 goto out; 1433 /* 1434 * copy new assignment, now we know it is possible 1435 * will be used by hw_perf_enable() 1436 */ 1437 memcpy(cpuc->assign, assign, n*sizeof(int)); 1438 1439 done_collect: 1440 /* 1441 * Commit the collect_events() state. See x86_pmu_del() and 1442 * x86_pmu_*_txn(). 1443 */ 1444 cpuc->n_events = n; 1445 cpuc->n_added += n - n0; 1446 cpuc->n_txn += n - n0; 1447 1448 /* 1449 * This is before x86_pmu_enable() will call x86_pmu_start(), 1450 * so we enable LBRs before an event needs them etc.. 1451 */ 1452 static_call_cond(x86_pmu_add)(event); 1453 1454 ret = 0; 1455 out: 1456 return ret; 1457 } 1458 1459 static void x86_pmu_start(struct perf_event *event, int flags) 1460 { 1461 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1462 int idx = event->hw.idx; 1463 1464 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1465 return; 1466 1467 if (WARN_ON_ONCE(idx == -1)) 1468 return; 1469 1470 if (flags & PERF_EF_RELOAD) { 1471 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1472 x86_perf_event_set_period(event); 1473 } 1474 1475 event->hw.state = 0; 1476 1477 cpuc->events[idx] = event; 1478 __set_bit(idx, cpuc->active_mask); 1479 __set_bit(idx, cpuc->running); 1480 static_call(x86_pmu_enable)(event); 1481 perf_event_update_userpage(event); 1482 } 1483 1484 void perf_event_print_debug(void) 1485 { 1486 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1487 u64 pebs, debugctl; 1488 struct cpu_hw_events *cpuc; 1489 unsigned long flags; 1490 int cpu, idx; 1491 1492 if (!x86_pmu.num_counters) 1493 return; 1494 1495 local_irq_save(flags); 1496 1497 cpu = smp_processor_id(); 1498 cpuc = &per_cpu(cpu_hw_events, cpu); 1499 1500 if (x86_pmu.version >= 2) { 1501 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1502 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1503 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1504 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1505 1506 pr_info("\n"); 1507 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1508 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1509 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1510 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1511 if (x86_pmu.pebs_constraints) { 1512 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1513 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1514 } 1515 if (x86_pmu.lbr_nr) { 1516 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1517 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1518 } 1519 } 1520 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1521 1522 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1523 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1524 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1525 1526 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1527 1528 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1529 cpu, idx, pmc_ctrl); 1530 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1531 cpu, idx, pmc_count); 1532 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1533 cpu, idx, prev_left); 1534 } 1535 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { 1536 if (fixed_counter_disabled(idx)) 1537 continue; 1538 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1539 1540 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1541 cpu, idx, pmc_count); 1542 } 1543 local_irq_restore(flags); 1544 } 1545 1546 void x86_pmu_stop(struct perf_event *event, int flags) 1547 { 1548 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1549 struct hw_perf_event *hwc = &event->hw; 1550 1551 if (test_bit(hwc->idx, cpuc->active_mask)) { 1552 static_call(x86_pmu_disable)(event); 1553 __clear_bit(hwc->idx, cpuc->active_mask); 1554 cpuc->events[hwc->idx] = NULL; 1555 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1556 hwc->state |= PERF_HES_STOPPED; 1557 } 1558 1559 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1560 /* 1561 * Drain the remaining delta count out of a event 1562 * that we are disabling: 1563 */ 1564 x86_perf_event_update(event); 1565 hwc->state |= PERF_HES_UPTODATE; 1566 } 1567 } 1568 1569 static void x86_pmu_del(struct perf_event *event, int flags) 1570 { 1571 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1572 int i; 1573 1574 /* 1575 * If we're called during a txn, we only need to undo x86_pmu.add. 1576 * The events never got scheduled and ->cancel_txn will truncate 1577 * the event_list. 1578 * 1579 * XXX assumes any ->del() called during a TXN will only be on 1580 * an event added during that same TXN. 1581 */ 1582 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1583 goto do_del; 1584 1585 /* 1586 * Not a TXN, therefore cleanup properly. 1587 */ 1588 x86_pmu_stop(event, PERF_EF_UPDATE); 1589 1590 for (i = 0; i < cpuc->n_events; i++) { 1591 if (event == cpuc->event_list[i]) 1592 break; 1593 } 1594 1595 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1596 return; 1597 1598 /* If we have a newly added event; make sure to decrease n_added. */ 1599 if (i >= cpuc->n_events - cpuc->n_added) 1600 --cpuc->n_added; 1601 1602 static_call_cond(x86_pmu_put_event_constraints)(cpuc, event); 1603 1604 /* Delete the array entry. */ 1605 while (++i < cpuc->n_events) { 1606 cpuc->event_list[i-1] = cpuc->event_list[i]; 1607 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1608 } 1609 cpuc->event_constraint[i-1] = NULL; 1610 --cpuc->n_events; 1611 if (x86_pmu.intel_cap.perf_metrics) 1612 del_nr_metric_event(cpuc, event); 1613 1614 perf_event_update_userpage(event); 1615 1616 do_del: 1617 1618 /* 1619 * This is after x86_pmu_stop(); so we disable LBRs after any 1620 * event can need them etc.. 1621 */ 1622 static_call_cond(x86_pmu_del)(event); 1623 } 1624 1625 int x86_pmu_handle_irq(struct pt_regs *regs) 1626 { 1627 struct perf_sample_data data; 1628 struct cpu_hw_events *cpuc; 1629 struct perf_event *event; 1630 int idx, handled = 0; 1631 u64 val; 1632 1633 cpuc = this_cpu_ptr(&cpu_hw_events); 1634 1635 /* 1636 * Some chipsets need to unmask the LVTPC in a particular spot 1637 * inside the nmi handler. As a result, the unmasking was pushed 1638 * into all the nmi handlers. 1639 * 1640 * This generic handler doesn't seem to have any issues where the 1641 * unmasking occurs so it was left at the top. 1642 */ 1643 apic_write(APIC_LVTPC, APIC_DM_NMI); 1644 1645 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1646 if (!test_bit(idx, cpuc->active_mask)) 1647 continue; 1648 1649 event = cpuc->events[idx]; 1650 1651 val = x86_perf_event_update(event); 1652 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1653 continue; 1654 1655 /* 1656 * event overflow 1657 */ 1658 handled++; 1659 perf_sample_data_init(&data, 0, event->hw.last_period); 1660 1661 if (!x86_perf_event_set_period(event)) 1662 continue; 1663 1664 if (perf_event_overflow(event, &data, regs)) 1665 x86_pmu_stop(event, 0); 1666 } 1667 1668 if (handled) 1669 inc_irq_stat(apic_perf_irqs); 1670 1671 return handled; 1672 } 1673 1674 void perf_events_lapic_init(void) 1675 { 1676 if (!x86_pmu.apic || !x86_pmu_initialized()) 1677 return; 1678 1679 /* 1680 * Always use NMI for PMU 1681 */ 1682 apic_write(APIC_LVTPC, APIC_DM_NMI); 1683 } 1684 1685 static int 1686 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1687 { 1688 u64 start_clock; 1689 u64 finish_clock; 1690 int ret; 1691 1692 /* 1693 * All PMUs/events that share this PMI handler should make sure to 1694 * increment active_events for their events. 1695 */ 1696 if (!atomic_read(&active_events)) 1697 return NMI_DONE; 1698 1699 start_clock = sched_clock(); 1700 ret = static_call(x86_pmu_handle_irq)(regs); 1701 finish_clock = sched_clock(); 1702 1703 perf_sample_event_took(finish_clock - start_clock); 1704 1705 return ret; 1706 } 1707 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1708 1709 struct event_constraint emptyconstraint; 1710 struct event_constraint unconstrained; 1711 1712 static int x86_pmu_prepare_cpu(unsigned int cpu) 1713 { 1714 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1715 int i; 1716 1717 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1718 cpuc->kfree_on_online[i] = NULL; 1719 if (x86_pmu.cpu_prepare) 1720 return x86_pmu.cpu_prepare(cpu); 1721 return 0; 1722 } 1723 1724 static int x86_pmu_dead_cpu(unsigned int cpu) 1725 { 1726 if (x86_pmu.cpu_dead) 1727 x86_pmu.cpu_dead(cpu); 1728 return 0; 1729 } 1730 1731 static int x86_pmu_online_cpu(unsigned int cpu) 1732 { 1733 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1734 int i; 1735 1736 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1737 kfree(cpuc->kfree_on_online[i]); 1738 cpuc->kfree_on_online[i] = NULL; 1739 } 1740 return 0; 1741 } 1742 1743 static int x86_pmu_starting_cpu(unsigned int cpu) 1744 { 1745 if (x86_pmu.cpu_starting) 1746 x86_pmu.cpu_starting(cpu); 1747 return 0; 1748 } 1749 1750 static int x86_pmu_dying_cpu(unsigned int cpu) 1751 { 1752 if (x86_pmu.cpu_dying) 1753 x86_pmu.cpu_dying(cpu); 1754 return 0; 1755 } 1756 1757 static void __init pmu_check_apic(void) 1758 { 1759 if (boot_cpu_has(X86_FEATURE_APIC)) 1760 return; 1761 1762 x86_pmu.apic = 0; 1763 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1764 pr_info("no hardware sampling interrupt available.\n"); 1765 1766 /* 1767 * If we have a PMU initialized but no APIC 1768 * interrupts, we cannot sample hardware 1769 * events (user-space has to fall back and 1770 * sample via a hrtimer based software event): 1771 */ 1772 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1773 1774 } 1775 1776 static struct attribute_group x86_pmu_format_group __ro_after_init = { 1777 .name = "format", 1778 .attrs = NULL, 1779 }; 1780 1781 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1782 { 1783 struct perf_pmu_events_attr *pmu_attr = 1784 container_of(attr, struct perf_pmu_events_attr, attr); 1785 u64 config = 0; 1786 1787 if (pmu_attr->id < x86_pmu.max_events) 1788 config = x86_pmu.event_map(pmu_attr->id); 1789 1790 /* string trumps id */ 1791 if (pmu_attr->event_str) 1792 return sprintf(page, "%s", pmu_attr->event_str); 1793 1794 return x86_pmu.events_sysfs_show(page, config); 1795 } 1796 EXPORT_SYMBOL_GPL(events_sysfs_show); 1797 1798 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1799 char *page) 1800 { 1801 struct perf_pmu_events_ht_attr *pmu_attr = 1802 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1803 1804 /* 1805 * Report conditional events depending on Hyper-Threading. 1806 * 1807 * This is overly conservative as usually the HT special 1808 * handling is not needed if the other CPU thread is idle. 1809 * 1810 * Note this does not (and cannot) handle the case when thread 1811 * siblings are invisible, for example with virtualization 1812 * if they are owned by some other guest. The user tool 1813 * has to re-read when a thread sibling gets onlined later. 1814 */ 1815 return sprintf(page, "%s", 1816 topology_max_smt_threads() > 1 ? 1817 pmu_attr->event_str_ht : 1818 pmu_attr->event_str_noht); 1819 } 1820 1821 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1822 EVENT_ATTR(instructions, INSTRUCTIONS ); 1823 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1824 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1825 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1826 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1827 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1828 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1829 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1830 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1831 1832 static struct attribute *empty_attrs; 1833 1834 static struct attribute *events_attr[] = { 1835 EVENT_PTR(CPU_CYCLES), 1836 EVENT_PTR(INSTRUCTIONS), 1837 EVENT_PTR(CACHE_REFERENCES), 1838 EVENT_PTR(CACHE_MISSES), 1839 EVENT_PTR(BRANCH_INSTRUCTIONS), 1840 EVENT_PTR(BRANCH_MISSES), 1841 EVENT_PTR(BUS_CYCLES), 1842 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1843 EVENT_PTR(STALLED_CYCLES_BACKEND), 1844 EVENT_PTR(REF_CPU_CYCLES), 1845 NULL, 1846 }; 1847 1848 /* 1849 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1850 * out of events_attr attributes. 1851 */ 1852 static umode_t 1853 is_visible(struct kobject *kobj, struct attribute *attr, int idx) 1854 { 1855 struct perf_pmu_events_attr *pmu_attr; 1856 1857 if (idx >= x86_pmu.max_events) 1858 return 0; 1859 1860 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); 1861 /* str trumps id */ 1862 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0; 1863 } 1864 1865 static struct attribute_group x86_pmu_events_group __ro_after_init = { 1866 .name = "events", 1867 .attrs = events_attr, 1868 .is_visible = is_visible, 1869 }; 1870 1871 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1872 { 1873 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1874 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1875 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1876 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1877 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1878 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1879 ssize_t ret; 1880 1881 /* 1882 * We have whole page size to spend and just little data 1883 * to write, so we can safely use sprintf. 1884 */ 1885 ret = sprintf(page, "event=0x%02llx", event); 1886 1887 if (umask) 1888 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1889 1890 if (edge) 1891 ret += sprintf(page + ret, ",edge"); 1892 1893 if (pc) 1894 ret += sprintf(page + ret, ",pc"); 1895 1896 if (any) 1897 ret += sprintf(page + ret, ",any"); 1898 1899 if (inv) 1900 ret += sprintf(page + ret, ",inv"); 1901 1902 if (cmask) 1903 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1904 1905 ret += sprintf(page + ret, "\n"); 1906 1907 return ret; 1908 } 1909 1910 static struct attribute_group x86_pmu_attr_group; 1911 static struct attribute_group x86_pmu_caps_group; 1912 1913 static void x86_pmu_static_call_update(void) 1914 { 1915 static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq); 1916 static_call_update(x86_pmu_disable_all, x86_pmu.disable_all); 1917 static_call_update(x86_pmu_enable_all, x86_pmu.enable_all); 1918 static_call_update(x86_pmu_enable, x86_pmu.enable); 1919 static_call_update(x86_pmu_disable, x86_pmu.disable); 1920 1921 static_call_update(x86_pmu_add, x86_pmu.add); 1922 static_call_update(x86_pmu_del, x86_pmu.del); 1923 static_call_update(x86_pmu_read, x86_pmu.read); 1924 1925 static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events); 1926 static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints); 1927 static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints); 1928 1929 static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling); 1930 static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling); 1931 static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling); 1932 1933 static_call_update(x86_pmu_sched_task, x86_pmu.sched_task); 1934 static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx); 1935 1936 static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs); 1937 static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases); 1938 1939 static_call_update(x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs); 1940 } 1941 1942 static void _x86_pmu_read(struct perf_event *event) 1943 { 1944 x86_perf_event_update(event); 1945 } 1946 1947 static inline struct perf_guest_switch_msr * 1948 perf_guest_get_msrs_nop(int *nr) 1949 { 1950 *nr = 0; 1951 return NULL; 1952 } 1953 1954 static int __init init_hw_perf_events(void) 1955 { 1956 struct x86_pmu_quirk *quirk; 1957 int err; 1958 1959 pr_info("Performance Events: "); 1960 1961 switch (boot_cpu_data.x86_vendor) { 1962 case X86_VENDOR_INTEL: 1963 err = intel_pmu_init(); 1964 break; 1965 case X86_VENDOR_AMD: 1966 err = amd_pmu_init(); 1967 break; 1968 case X86_VENDOR_HYGON: 1969 err = amd_pmu_init(); 1970 x86_pmu.name = "HYGON"; 1971 break; 1972 case X86_VENDOR_ZHAOXIN: 1973 case X86_VENDOR_CENTAUR: 1974 err = zhaoxin_pmu_init(); 1975 break; 1976 default: 1977 err = -ENOTSUPP; 1978 } 1979 if (err != 0) { 1980 pr_cont("no PMU driver, software events only.\n"); 1981 return 0; 1982 } 1983 1984 pmu_check_apic(); 1985 1986 /* sanity check that the hardware exists or is emulated */ 1987 if (!check_hw_exists()) 1988 return 0; 1989 1990 pr_cont("%s PMU driver.\n", x86_pmu.name); 1991 1992 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1993 1994 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1995 quirk->func(); 1996 1997 if (!x86_pmu.intel_ctrl) 1998 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 1999 2000 perf_events_lapic_init(); 2001 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 2002 2003 unconstrained = (struct event_constraint) 2004 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 2005 0, x86_pmu.num_counters, 0, 0); 2006 2007 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 2008 2009 if (!x86_pmu.events_sysfs_show) 2010 x86_pmu_events_group.attrs = &empty_attrs; 2011 2012 pmu.attr_update = x86_pmu.attr_update; 2013 2014 pr_info("... version: %d\n", x86_pmu.version); 2015 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 2016 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 2017 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 2018 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 2019 pr_info("... fixed-purpose events: %lu\n", 2020 hweight64((((1ULL << x86_pmu.num_counters_fixed) - 1) 2021 << INTEL_PMC_IDX_FIXED) & x86_pmu.intel_ctrl)); 2022 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); 2023 2024 if (!x86_pmu.read) 2025 x86_pmu.read = _x86_pmu_read; 2026 2027 if (!x86_pmu.guest_get_msrs) 2028 x86_pmu.guest_get_msrs = perf_guest_get_msrs_nop; 2029 2030 x86_pmu_static_call_update(); 2031 2032 /* 2033 * Install callbacks. Core will call them for each online 2034 * cpu. 2035 */ 2036 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 2037 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 2038 if (err) 2039 return err; 2040 2041 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 2042 "perf/x86:starting", x86_pmu_starting_cpu, 2043 x86_pmu_dying_cpu); 2044 if (err) 2045 goto out; 2046 2047 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 2048 x86_pmu_online_cpu, NULL); 2049 if (err) 2050 goto out1; 2051 2052 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 2053 if (err) 2054 goto out2; 2055 2056 return 0; 2057 2058 out2: 2059 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 2060 out1: 2061 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 2062 out: 2063 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 2064 return err; 2065 } 2066 early_initcall(init_hw_perf_events); 2067 2068 static void x86_pmu_read(struct perf_event *event) 2069 { 2070 static_call(x86_pmu_read)(event); 2071 } 2072 2073 /* 2074 * Start group events scheduling transaction 2075 * Set the flag to make pmu::enable() not perform the 2076 * schedulability test, it will be performed at commit time 2077 * 2078 * We only support PERF_PMU_TXN_ADD transactions. Save the 2079 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 2080 * transactions. 2081 */ 2082 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 2083 { 2084 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2085 2086 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 2087 2088 cpuc->txn_flags = txn_flags; 2089 if (txn_flags & ~PERF_PMU_TXN_ADD) 2090 return; 2091 2092 perf_pmu_disable(pmu); 2093 __this_cpu_write(cpu_hw_events.n_txn, 0); 2094 __this_cpu_write(cpu_hw_events.n_txn_pair, 0); 2095 __this_cpu_write(cpu_hw_events.n_txn_metric, 0); 2096 } 2097 2098 /* 2099 * Stop group events scheduling transaction 2100 * Clear the flag and pmu::enable() will perform the 2101 * schedulability test. 2102 */ 2103 static void x86_pmu_cancel_txn(struct pmu *pmu) 2104 { 2105 unsigned int txn_flags; 2106 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2107 2108 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 2109 2110 txn_flags = cpuc->txn_flags; 2111 cpuc->txn_flags = 0; 2112 if (txn_flags & ~PERF_PMU_TXN_ADD) 2113 return; 2114 2115 /* 2116 * Truncate collected array by the number of events added in this 2117 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 2118 */ 2119 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 2120 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 2121 __this_cpu_sub(cpu_hw_events.n_pair, __this_cpu_read(cpu_hw_events.n_txn_pair)); 2122 __this_cpu_sub(cpu_hw_events.n_metric, __this_cpu_read(cpu_hw_events.n_txn_metric)); 2123 perf_pmu_enable(pmu); 2124 } 2125 2126 /* 2127 * Commit group events scheduling transaction 2128 * Perform the group schedulability test as a whole 2129 * Return 0 if success 2130 * 2131 * Does not cancel the transaction on failure; expects the caller to do this. 2132 */ 2133 static int x86_pmu_commit_txn(struct pmu *pmu) 2134 { 2135 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2136 int assign[X86_PMC_IDX_MAX]; 2137 int n, ret; 2138 2139 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 2140 2141 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 2142 cpuc->txn_flags = 0; 2143 return 0; 2144 } 2145 2146 n = cpuc->n_events; 2147 2148 if (!x86_pmu_initialized()) 2149 return -EAGAIN; 2150 2151 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign); 2152 if (ret) 2153 return ret; 2154 2155 /* 2156 * copy new assignment, now we know it is possible 2157 * will be used by hw_perf_enable() 2158 */ 2159 memcpy(cpuc->assign, assign, n*sizeof(int)); 2160 2161 cpuc->txn_flags = 0; 2162 perf_pmu_enable(pmu); 2163 return 0; 2164 } 2165 /* 2166 * a fake_cpuc is used to validate event groups. Due to 2167 * the extra reg logic, we need to also allocate a fake 2168 * per_core and per_cpu structure. Otherwise, group events 2169 * using extra reg may conflict without the kernel being 2170 * able to catch this when the last event gets added to 2171 * the group. 2172 */ 2173 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 2174 { 2175 intel_cpuc_finish(cpuc); 2176 kfree(cpuc); 2177 } 2178 2179 static struct cpu_hw_events *allocate_fake_cpuc(void) 2180 { 2181 struct cpu_hw_events *cpuc; 2182 int cpu = raw_smp_processor_id(); 2183 2184 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 2185 if (!cpuc) 2186 return ERR_PTR(-ENOMEM); 2187 cpuc->is_fake = 1; 2188 2189 if (intel_cpuc_prepare(cpuc, cpu)) 2190 goto error; 2191 2192 return cpuc; 2193 error: 2194 free_fake_cpuc(cpuc); 2195 return ERR_PTR(-ENOMEM); 2196 } 2197 2198 /* 2199 * validate that we can schedule this event 2200 */ 2201 static int validate_event(struct perf_event *event) 2202 { 2203 struct cpu_hw_events *fake_cpuc; 2204 struct event_constraint *c; 2205 int ret = 0; 2206 2207 fake_cpuc = allocate_fake_cpuc(); 2208 if (IS_ERR(fake_cpuc)) 2209 return PTR_ERR(fake_cpuc); 2210 2211 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event); 2212 2213 if (!c || !c->weight) 2214 ret = -EINVAL; 2215 2216 if (x86_pmu.put_event_constraints) 2217 x86_pmu.put_event_constraints(fake_cpuc, event); 2218 2219 free_fake_cpuc(fake_cpuc); 2220 2221 return ret; 2222 } 2223 2224 /* 2225 * validate a single event group 2226 * 2227 * validation include: 2228 * - check events are compatible which each other 2229 * - events do not compete for the same counter 2230 * - number of events <= number of counters 2231 * 2232 * validation ensures the group can be loaded onto the 2233 * PMU if it was the only group available. 2234 */ 2235 static int validate_group(struct perf_event *event) 2236 { 2237 struct perf_event *leader = event->group_leader; 2238 struct cpu_hw_events *fake_cpuc; 2239 int ret = -EINVAL, n; 2240 2241 fake_cpuc = allocate_fake_cpuc(); 2242 if (IS_ERR(fake_cpuc)) 2243 return PTR_ERR(fake_cpuc); 2244 /* 2245 * the event is not yet connected with its 2246 * siblings therefore we must first collect 2247 * existing siblings, then add the new event 2248 * before we can simulate the scheduling 2249 */ 2250 n = collect_events(fake_cpuc, leader, true); 2251 if (n < 0) 2252 goto out; 2253 2254 fake_cpuc->n_events = n; 2255 n = collect_events(fake_cpuc, event, false); 2256 if (n < 0) 2257 goto out; 2258 2259 fake_cpuc->n_events = 0; 2260 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2261 2262 out: 2263 free_fake_cpuc(fake_cpuc); 2264 return ret; 2265 } 2266 2267 static int x86_pmu_event_init(struct perf_event *event) 2268 { 2269 struct pmu *tmp; 2270 int err; 2271 2272 switch (event->attr.type) { 2273 case PERF_TYPE_RAW: 2274 case PERF_TYPE_HARDWARE: 2275 case PERF_TYPE_HW_CACHE: 2276 break; 2277 2278 default: 2279 return -ENOENT; 2280 } 2281 2282 err = __x86_pmu_event_init(event); 2283 if (!err) { 2284 /* 2285 * we temporarily connect event to its pmu 2286 * such that validate_group() can classify 2287 * it as an x86 event using is_x86_event() 2288 */ 2289 tmp = event->pmu; 2290 event->pmu = &pmu; 2291 2292 if (event->group_leader != event) 2293 err = validate_group(event); 2294 else 2295 err = validate_event(event); 2296 2297 event->pmu = tmp; 2298 } 2299 if (err) { 2300 if (event->destroy) 2301 event->destroy(event); 2302 } 2303 2304 if (READ_ONCE(x86_pmu.attr_rdpmc) && 2305 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) 2306 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2307 2308 return err; 2309 } 2310 2311 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 2312 { 2313 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2314 return; 2315 2316 /* 2317 * This function relies on not being called concurrently in two 2318 * tasks in the same mm. Otherwise one task could observe 2319 * perf_rdpmc_allowed > 1 and return all the way back to 2320 * userspace with CR4.PCE clear while another task is still 2321 * doing on_each_cpu_mask() to propagate CR4.PCE. 2322 * 2323 * For now, this can't happen because all callers hold mmap_lock 2324 * for write. If this changes, we'll need a different solution. 2325 */ 2326 mmap_assert_write_locked(mm); 2327 2328 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 2329 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1); 2330 } 2331 2332 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 2333 { 2334 2335 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2336 return; 2337 2338 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 2339 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1); 2340 } 2341 2342 static int x86_pmu_event_idx(struct perf_event *event) 2343 { 2344 struct hw_perf_event *hwc = &event->hw; 2345 2346 if (!(hwc->flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2347 return 0; 2348 2349 if (is_metric_idx(hwc->idx)) 2350 return INTEL_PMC_FIXED_RDPMC_METRICS + 1; 2351 else 2352 return hwc->event_base_rdpmc + 1; 2353 } 2354 2355 static ssize_t get_attr_rdpmc(struct device *cdev, 2356 struct device_attribute *attr, 2357 char *buf) 2358 { 2359 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2360 } 2361 2362 static ssize_t set_attr_rdpmc(struct device *cdev, 2363 struct device_attribute *attr, 2364 const char *buf, size_t count) 2365 { 2366 unsigned long val; 2367 ssize_t ret; 2368 2369 ret = kstrtoul(buf, 0, &val); 2370 if (ret) 2371 return ret; 2372 2373 if (val > 2) 2374 return -EINVAL; 2375 2376 if (x86_pmu.attr_rdpmc_broken) 2377 return -ENOTSUPP; 2378 2379 if (val != x86_pmu.attr_rdpmc) { 2380 /* 2381 * Changing into or out of never available or always available, 2382 * aka perf-event-bypassing mode. This path is extremely slow, 2383 * but only root can trigger it, so it's okay. 2384 */ 2385 if (val == 0) 2386 static_branch_inc(&rdpmc_never_available_key); 2387 else if (x86_pmu.attr_rdpmc == 0) 2388 static_branch_dec(&rdpmc_never_available_key); 2389 2390 if (val == 2) 2391 static_branch_inc(&rdpmc_always_available_key); 2392 else if (x86_pmu.attr_rdpmc == 2) 2393 static_branch_dec(&rdpmc_always_available_key); 2394 2395 on_each_cpu(cr4_update_pce, NULL, 1); 2396 x86_pmu.attr_rdpmc = val; 2397 } 2398 2399 return count; 2400 } 2401 2402 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2403 2404 static struct attribute *x86_pmu_attrs[] = { 2405 &dev_attr_rdpmc.attr, 2406 NULL, 2407 }; 2408 2409 static struct attribute_group x86_pmu_attr_group __ro_after_init = { 2410 .attrs = x86_pmu_attrs, 2411 }; 2412 2413 static ssize_t max_precise_show(struct device *cdev, 2414 struct device_attribute *attr, 2415 char *buf) 2416 { 2417 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise()); 2418 } 2419 2420 static DEVICE_ATTR_RO(max_precise); 2421 2422 static struct attribute *x86_pmu_caps_attrs[] = { 2423 &dev_attr_max_precise.attr, 2424 NULL 2425 }; 2426 2427 static struct attribute_group x86_pmu_caps_group __ro_after_init = { 2428 .name = "caps", 2429 .attrs = x86_pmu_caps_attrs, 2430 }; 2431 2432 static const struct attribute_group *x86_pmu_attr_groups[] = { 2433 &x86_pmu_attr_group, 2434 &x86_pmu_format_group, 2435 &x86_pmu_events_group, 2436 &x86_pmu_caps_group, 2437 NULL, 2438 }; 2439 2440 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2441 { 2442 static_call_cond(x86_pmu_sched_task)(ctx, sched_in); 2443 } 2444 2445 static void x86_pmu_swap_task_ctx(struct perf_event_context *prev, 2446 struct perf_event_context *next) 2447 { 2448 static_call_cond(x86_pmu_swap_task_ctx)(prev, next); 2449 } 2450 2451 void perf_check_microcode(void) 2452 { 2453 if (x86_pmu.check_microcode) 2454 x86_pmu.check_microcode(); 2455 } 2456 2457 static int x86_pmu_check_period(struct perf_event *event, u64 value) 2458 { 2459 if (x86_pmu.check_period && x86_pmu.check_period(event, value)) 2460 return -EINVAL; 2461 2462 if (value && x86_pmu.limit_period) { 2463 if (x86_pmu.limit_period(event, value) > value) 2464 return -EINVAL; 2465 } 2466 2467 return 0; 2468 } 2469 2470 static int x86_pmu_aux_output_match(struct perf_event *event) 2471 { 2472 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT)) 2473 return 0; 2474 2475 if (x86_pmu.aux_output_match) 2476 return x86_pmu.aux_output_match(event); 2477 2478 return 0; 2479 } 2480 2481 static struct pmu pmu = { 2482 .pmu_enable = x86_pmu_enable, 2483 .pmu_disable = x86_pmu_disable, 2484 2485 .attr_groups = x86_pmu_attr_groups, 2486 2487 .event_init = x86_pmu_event_init, 2488 2489 .event_mapped = x86_pmu_event_mapped, 2490 .event_unmapped = x86_pmu_event_unmapped, 2491 2492 .add = x86_pmu_add, 2493 .del = x86_pmu_del, 2494 .start = x86_pmu_start, 2495 .stop = x86_pmu_stop, 2496 .read = x86_pmu_read, 2497 2498 .start_txn = x86_pmu_start_txn, 2499 .cancel_txn = x86_pmu_cancel_txn, 2500 .commit_txn = x86_pmu_commit_txn, 2501 2502 .event_idx = x86_pmu_event_idx, 2503 .sched_task = x86_pmu_sched_task, 2504 .swap_task_ctx = x86_pmu_swap_task_ctx, 2505 .check_period = x86_pmu_check_period, 2506 2507 .aux_output_match = x86_pmu_aux_output_match, 2508 }; 2509 2510 void arch_perf_update_userpage(struct perf_event *event, 2511 struct perf_event_mmap_page *userpg, u64 now) 2512 { 2513 struct cyc2ns_data data; 2514 u64 offset; 2515 2516 userpg->cap_user_time = 0; 2517 userpg->cap_user_time_zero = 0; 2518 userpg->cap_user_rdpmc = 2519 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2520 userpg->pmc_width = x86_pmu.cntval_bits; 2521 2522 if (!using_native_sched_clock() || !sched_clock_stable()) 2523 return; 2524 2525 cyc2ns_read_begin(&data); 2526 2527 offset = data.cyc2ns_offset + __sched_clock_offset; 2528 2529 /* 2530 * Internal timekeeping for enabled/running/stopped times 2531 * is always in the local_clock domain. 2532 */ 2533 userpg->cap_user_time = 1; 2534 userpg->time_mult = data.cyc2ns_mul; 2535 userpg->time_shift = data.cyc2ns_shift; 2536 userpg->time_offset = offset - now; 2537 2538 /* 2539 * cap_user_time_zero doesn't make sense when we're using a different 2540 * time base for the records. 2541 */ 2542 if (!event->attr.use_clockid) { 2543 userpg->cap_user_time_zero = 1; 2544 userpg->time_zero = offset; 2545 } 2546 2547 cyc2ns_read_end(); 2548 } 2549 2550 /* 2551 * Determine whether the regs were taken from an irq/exception handler rather 2552 * than from perf_arch_fetch_caller_regs(). 2553 */ 2554 static bool perf_hw_regs(struct pt_regs *regs) 2555 { 2556 return regs->flags & X86_EFLAGS_FIXED; 2557 } 2558 2559 void 2560 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2561 { 2562 struct unwind_state state; 2563 unsigned long addr; 2564 2565 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2566 /* TODO: We don't support guest os callchain now */ 2567 return; 2568 } 2569 2570 if (perf_callchain_store(entry, regs->ip)) 2571 return; 2572 2573 if (perf_hw_regs(regs)) 2574 unwind_start(&state, current, regs, NULL); 2575 else 2576 unwind_start(&state, current, NULL, (void *)regs->sp); 2577 2578 for (; !unwind_done(&state); unwind_next_frame(&state)) { 2579 addr = unwind_get_return_address(&state); 2580 if (!addr || perf_callchain_store(entry, addr)) 2581 return; 2582 } 2583 } 2584 2585 static inline int 2586 valid_user_frame(const void __user *fp, unsigned long size) 2587 { 2588 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2589 } 2590 2591 static unsigned long get_segment_base(unsigned int segment) 2592 { 2593 struct desc_struct *desc; 2594 unsigned int idx = segment >> 3; 2595 2596 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2597 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2598 struct ldt_struct *ldt; 2599 2600 /* IRQs are off, so this synchronizes with smp_store_release */ 2601 ldt = READ_ONCE(current->active_mm->context.ldt); 2602 if (!ldt || idx >= ldt->nr_entries) 2603 return 0; 2604 2605 desc = &ldt->entries[idx]; 2606 #else 2607 return 0; 2608 #endif 2609 } else { 2610 if (idx >= GDT_ENTRIES) 2611 return 0; 2612 2613 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2614 } 2615 2616 return get_desc_base(desc); 2617 } 2618 2619 #ifdef CONFIG_IA32_EMULATION 2620 2621 #include <linux/compat.h> 2622 2623 static inline int 2624 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2625 { 2626 /* 32-bit process in 64-bit kernel. */ 2627 unsigned long ss_base, cs_base; 2628 struct stack_frame_ia32 frame; 2629 const struct stack_frame_ia32 __user *fp; 2630 2631 if (user_64bit_mode(regs)) 2632 return 0; 2633 2634 cs_base = get_segment_base(regs->cs); 2635 ss_base = get_segment_base(regs->ss); 2636 2637 fp = compat_ptr(ss_base + regs->bp); 2638 pagefault_disable(); 2639 while (entry->nr < entry->max_stack) { 2640 if (!valid_user_frame(fp, sizeof(frame))) 2641 break; 2642 2643 if (__get_user(frame.next_frame, &fp->next_frame)) 2644 break; 2645 if (__get_user(frame.return_address, &fp->return_address)) 2646 break; 2647 2648 perf_callchain_store(entry, cs_base + frame.return_address); 2649 fp = compat_ptr(ss_base + frame.next_frame); 2650 } 2651 pagefault_enable(); 2652 return 1; 2653 } 2654 #else 2655 static inline int 2656 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2657 { 2658 return 0; 2659 } 2660 #endif 2661 2662 void 2663 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2664 { 2665 struct stack_frame frame; 2666 const struct stack_frame __user *fp; 2667 2668 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2669 /* TODO: We don't support guest os callchain now */ 2670 return; 2671 } 2672 2673 /* 2674 * We don't know what to do with VM86 stacks.. ignore them for now. 2675 */ 2676 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2677 return; 2678 2679 fp = (void __user *)regs->bp; 2680 2681 perf_callchain_store(entry, regs->ip); 2682 2683 if (!nmi_uaccess_okay()) 2684 return; 2685 2686 if (perf_callchain_user32(regs, entry)) 2687 return; 2688 2689 pagefault_disable(); 2690 while (entry->nr < entry->max_stack) { 2691 if (!valid_user_frame(fp, sizeof(frame))) 2692 break; 2693 2694 if (__get_user(frame.next_frame, &fp->next_frame)) 2695 break; 2696 if (__get_user(frame.return_address, &fp->return_address)) 2697 break; 2698 2699 perf_callchain_store(entry, frame.return_address); 2700 fp = (void __user *)frame.next_frame; 2701 } 2702 pagefault_enable(); 2703 } 2704 2705 /* 2706 * Deal with code segment offsets for the various execution modes: 2707 * 2708 * VM86 - the good olde 16 bit days, where the linear address is 2709 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2710 * 2711 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2712 * to figure out what the 32bit base address is. 2713 * 2714 * X32 - has TIF_X32 set, but is running in x86_64 2715 * 2716 * X86_64 - CS,DS,SS,ES are all zero based. 2717 */ 2718 static unsigned long code_segment_base(struct pt_regs *regs) 2719 { 2720 /* 2721 * For IA32 we look at the GDT/LDT segment base to convert the 2722 * effective IP to a linear address. 2723 */ 2724 2725 #ifdef CONFIG_X86_32 2726 /* 2727 * If we are in VM86 mode, add the segment offset to convert to a 2728 * linear address. 2729 */ 2730 if (regs->flags & X86_VM_MASK) 2731 return 0x10 * regs->cs; 2732 2733 if (user_mode(regs) && regs->cs != __USER_CS) 2734 return get_segment_base(regs->cs); 2735 #else 2736 if (user_mode(regs) && !user_64bit_mode(regs) && 2737 regs->cs != __USER32_CS) 2738 return get_segment_base(regs->cs); 2739 #endif 2740 return 0; 2741 } 2742 2743 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2744 { 2745 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2746 return perf_guest_cbs->get_guest_ip(); 2747 2748 return regs->ip + code_segment_base(regs); 2749 } 2750 2751 unsigned long perf_misc_flags(struct pt_regs *regs) 2752 { 2753 int misc = 0; 2754 2755 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2756 if (perf_guest_cbs->is_user_mode()) 2757 misc |= PERF_RECORD_MISC_GUEST_USER; 2758 else 2759 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2760 } else { 2761 if (user_mode(regs)) 2762 misc |= PERF_RECORD_MISC_USER; 2763 else 2764 misc |= PERF_RECORD_MISC_KERNEL; 2765 } 2766 2767 if (regs->flags & PERF_EFLAGS_EXACT) 2768 misc |= PERF_RECORD_MISC_EXACT_IP; 2769 2770 return misc; 2771 } 2772 2773 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2774 { 2775 cap->version = x86_pmu.version; 2776 cap->num_counters_gp = x86_pmu.num_counters; 2777 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2778 cap->bit_width_gp = x86_pmu.cntval_bits; 2779 cap->bit_width_fixed = x86_pmu.cntval_bits; 2780 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2781 cap->events_mask_len = x86_pmu.events_mask_len; 2782 } 2783 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2784