1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched/mm.h> 24 #include <linux/sched/clock.h> 25 #include <linux/uaccess.h> 26 #include <linux/slab.h> 27 #include <linux/cpu.h> 28 #include <linux/bitops.h> 29 #include <linux/device.h> 30 #include <linux/nospec.h> 31 32 #include <asm/apic.h> 33 #include <asm/stacktrace.h> 34 #include <asm/nmi.h> 35 #include <asm/smp.h> 36 #include <asm/alternative.h> 37 #include <asm/mmu_context.h> 38 #include <asm/tlbflush.h> 39 #include <asm/timer.h> 40 #include <asm/desc.h> 41 #include <asm/ldt.h> 42 #include <asm/unwind.h> 43 44 #include "perf_event.h" 45 46 struct x86_pmu x86_pmu __read_mostly; 47 48 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 49 .enabled = 1, 50 }; 51 52 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); 53 54 u64 __read_mostly hw_cache_event_ids 55 [PERF_COUNT_HW_CACHE_MAX] 56 [PERF_COUNT_HW_CACHE_OP_MAX] 57 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 58 u64 __read_mostly hw_cache_extra_regs 59 [PERF_COUNT_HW_CACHE_MAX] 60 [PERF_COUNT_HW_CACHE_OP_MAX] 61 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 62 63 /* 64 * Propagate event elapsed time into the generic event. 65 * Can only be executed on the CPU where the event is active. 66 * Returns the delta events processed. 67 */ 68 u64 x86_perf_event_update(struct perf_event *event) 69 { 70 struct hw_perf_event *hwc = &event->hw; 71 int shift = 64 - x86_pmu.cntval_bits; 72 u64 prev_raw_count, new_raw_count; 73 int idx = hwc->idx; 74 u64 delta; 75 76 if (idx == INTEL_PMC_IDX_FIXED_BTS) 77 return 0; 78 79 /* 80 * Careful: an NMI might modify the previous event value. 81 * 82 * Our tactic to handle this is to first atomically read and 83 * exchange a new raw count - then add that new-prev delta 84 * count to the generic event atomically: 85 */ 86 again: 87 prev_raw_count = local64_read(&hwc->prev_count); 88 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 89 90 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 91 new_raw_count) != prev_raw_count) 92 goto again; 93 94 /* 95 * Now we have the new raw value and have updated the prev 96 * timestamp already. We can now calculate the elapsed delta 97 * (event-)time and add that to the generic event. 98 * 99 * Careful, not all hw sign-extends above the physical width 100 * of the count. 101 */ 102 delta = (new_raw_count << shift) - (prev_raw_count << shift); 103 delta >>= shift; 104 105 local64_add(delta, &event->count); 106 local64_sub(delta, &hwc->period_left); 107 108 return new_raw_count; 109 } 110 111 /* 112 * Find and validate any extra registers to set up. 113 */ 114 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 115 { 116 struct hw_perf_event_extra *reg; 117 struct extra_reg *er; 118 119 reg = &event->hw.extra_reg; 120 121 if (!x86_pmu.extra_regs) 122 return 0; 123 124 for (er = x86_pmu.extra_regs; er->msr; er++) { 125 if (er->event != (config & er->config_mask)) 126 continue; 127 if (event->attr.config1 & ~er->valid_mask) 128 return -EINVAL; 129 /* Check if the extra msrs can be safely accessed*/ 130 if (!er->extra_msr_access) 131 return -ENXIO; 132 133 reg->idx = er->idx; 134 reg->config = event->attr.config1; 135 reg->reg = er->msr; 136 break; 137 } 138 return 0; 139 } 140 141 static atomic_t active_events; 142 static atomic_t pmc_refcount; 143 static DEFINE_MUTEX(pmc_reserve_mutex); 144 145 #ifdef CONFIG_X86_LOCAL_APIC 146 147 static bool reserve_pmc_hardware(void) 148 { 149 int i; 150 151 for (i = 0; i < x86_pmu.num_counters; i++) { 152 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 153 goto perfctr_fail; 154 } 155 156 for (i = 0; i < x86_pmu.num_counters; i++) { 157 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 158 goto eventsel_fail; 159 } 160 161 return true; 162 163 eventsel_fail: 164 for (i--; i >= 0; i--) 165 release_evntsel_nmi(x86_pmu_config_addr(i)); 166 167 i = x86_pmu.num_counters; 168 169 perfctr_fail: 170 for (i--; i >= 0; i--) 171 release_perfctr_nmi(x86_pmu_event_addr(i)); 172 173 return false; 174 } 175 176 static void release_pmc_hardware(void) 177 { 178 int i; 179 180 for (i = 0; i < x86_pmu.num_counters; i++) { 181 release_perfctr_nmi(x86_pmu_event_addr(i)); 182 release_evntsel_nmi(x86_pmu_config_addr(i)); 183 } 184 } 185 186 #else 187 188 static bool reserve_pmc_hardware(void) { return true; } 189 static void release_pmc_hardware(void) {} 190 191 #endif 192 193 static bool check_hw_exists(void) 194 { 195 u64 val, val_fail = -1, val_new= ~0; 196 int i, reg, reg_fail = -1, ret = 0; 197 int bios_fail = 0; 198 int reg_safe = -1; 199 200 /* 201 * Check to see if the BIOS enabled any of the counters, if so 202 * complain and bail. 203 */ 204 for (i = 0; i < x86_pmu.num_counters; i++) { 205 reg = x86_pmu_config_addr(i); 206 ret = rdmsrl_safe(reg, &val); 207 if (ret) 208 goto msr_fail; 209 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 210 bios_fail = 1; 211 val_fail = val; 212 reg_fail = reg; 213 } else { 214 reg_safe = i; 215 } 216 } 217 218 if (x86_pmu.num_counters_fixed) { 219 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 220 ret = rdmsrl_safe(reg, &val); 221 if (ret) 222 goto msr_fail; 223 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 224 if (val & (0x03 << i*4)) { 225 bios_fail = 1; 226 val_fail = val; 227 reg_fail = reg; 228 } 229 } 230 } 231 232 /* 233 * If all the counters are enabled, the below test will always 234 * fail. The tools will also become useless in this scenario. 235 * Just fail and disable the hardware counters. 236 */ 237 238 if (reg_safe == -1) { 239 reg = reg_safe; 240 goto msr_fail; 241 } 242 243 /* 244 * Read the current value, change it and read it back to see if it 245 * matches, this is needed to detect certain hardware emulators 246 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 247 */ 248 reg = x86_pmu_event_addr(reg_safe); 249 if (rdmsrl_safe(reg, &val)) 250 goto msr_fail; 251 val ^= 0xffffUL; 252 ret = wrmsrl_safe(reg, val); 253 ret |= rdmsrl_safe(reg, &val_new); 254 if (ret || val != val_new) 255 goto msr_fail; 256 257 /* 258 * We still allow the PMU driver to operate: 259 */ 260 if (bios_fail) { 261 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 262 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 263 reg_fail, val_fail); 264 } 265 266 return true; 267 268 msr_fail: 269 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 270 pr_cont("PMU not available due to virtualization, using software events only.\n"); 271 } else { 272 pr_cont("Broken PMU hardware detected, using software events only.\n"); 273 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 274 reg, val_new); 275 } 276 277 return false; 278 } 279 280 static void hw_perf_event_destroy(struct perf_event *event) 281 { 282 x86_release_hardware(); 283 atomic_dec(&active_events); 284 } 285 286 void hw_perf_lbr_event_destroy(struct perf_event *event) 287 { 288 hw_perf_event_destroy(event); 289 290 /* undo the lbr/bts event accounting */ 291 x86_del_exclusive(x86_lbr_exclusive_lbr); 292 } 293 294 static inline int x86_pmu_initialized(void) 295 { 296 return x86_pmu.handle_irq != NULL; 297 } 298 299 static inline int 300 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 301 { 302 struct perf_event_attr *attr = &event->attr; 303 unsigned int cache_type, cache_op, cache_result; 304 u64 config, val; 305 306 config = attr->config; 307 308 cache_type = (config >> 0) & 0xff; 309 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 310 return -EINVAL; 311 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX); 312 313 cache_op = (config >> 8) & 0xff; 314 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 315 return -EINVAL; 316 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX); 317 318 cache_result = (config >> 16) & 0xff; 319 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 320 return -EINVAL; 321 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX); 322 323 val = hw_cache_event_ids[cache_type][cache_op][cache_result]; 324 325 if (val == 0) 326 return -ENOENT; 327 328 if (val == -1) 329 return -EINVAL; 330 331 hwc->config |= val; 332 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; 333 return x86_pmu_extra_regs(val, event); 334 } 335 336 int x86_reserve_hardware(void) 337 { 338 int err = 0; 339 340 if (!atomic_inc_not_zero(&pmc_refcount)) { 341 mutex_lock(&pmc_reserve_mutex); 342 if (atomic_read(&pmc_refcount) == 0) { 343 if (!reserve_pmc_hardware()) 344 err = -EBUSY; 345 else 346 reserve_ds_buffers(); 347 } 348 if (!err) 349 atomic_inc(&pmc_refcount); 350 mutex_unlock(&pmc_reserve_mutex); 351 } 352 353 return err; 354 } 355 356 void x86_release_hardware(void) 357 { 358 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 359 release_pmc_hardware(); 360 release_ds_buffers(); 361 mutex_unlock(&pmc_reserve_mutex); 362 } 363 } 364 365 /* 366 * Check if we can create event of a certain type (that no conflicting events 367 * are present). 368 */ 369 int x86_add_exclusive(unsigned int what) 370 { 371 int i; 372 373 /* 374 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 375 * LBR and BTS are still mutually exclusive. 376 */ 377 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 378 return 0; 379 380 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 381 mutex_lock(&pmc_reserve_mutex); 382 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 383 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 384 goto fail_unlock; 385 } 386 atomic_inc(&x86_pmu.lbr_exclusive[what]); 387 mutex_unlock(&pmc_reserve_mutex); 388 } 389 390 atomic_inc(&active_events); 391 return 0; 392 393 fail_unlock: 394 mutex_unlock(&pmc_reserve_mutex); 395 return -EBUSY; 396 } 397 398 void x86_del_exclusive(unsigned int what) 399 { 400 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 401 return; 402 403 atomic_dec(&x86_pmu.lbr_exclusive[what]); 404 atomic_dec(&active_events); 405 } 406 407 int x86_setup_perfctr(struct perf_event *event) 408 { 409 struct perf_event_attr *attr = &event->attr; 410 struct hw_perf_event *hwc = &event->hw; 411 u64 config; 412 413 if (!is_sampling_event(event)) { 414 hwc->sample_period = x86_pmu.max_period; 415 hwc->last_period = hwc->sample_period; 416 local64_set(&hwc->period_left, hwc->sample_period); 417 } 418 419 if (attr->type == PERF_TYPE_RAW) 420 return x86_pmu_extra_regs(event->attr.config, event); 421 422 if (attr->type == PERF_TYPE_HW_CACHE) 423 return set_ext_hw_attr(hwc, event); 424 425 if (attr->config >= x86_pmu.max_events) 426 return -EINVAL; 427 428 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); 429 430 /* 431 * The generic map: 432 */ 433 config = x86_pmu.event_map(attr->config); 434 435 if (config == 0) 436 return -ENOENT; 437 438 if (config == -1LL) 439 return -EINVAL; 440 441 hwc->config |= config; 442 443 return 0; 444 } 445 446 /* 447 * check that branch_sample_type is compatible with 448 * settings needed for precise_ip > 1 which implies 449 * using the LBR to capture ALL taken branches at the 450 * priv levels of the measurement 451 */ 452 static inline int precise_br_compat(struct perf_event *event) 453 { 454 u64 m = event->attr.branch_sample_type; 455 u64 b = 0; 456 457 /* must capture all branches */ 458 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 459 return 0; 460 461 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 462 463 if (!event->attr.exclude_user) 464 b |= PERF_SAMPLE_BRANCH_USER; 465 466 if (!event->attr.exclude_kernel) 467 b |= PERF_SAMPLE_BRANCH_KERNEL; 468 469 /* 470 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 471 */ 472 473 return m == b; 474 } 475 476 int x86_pmu_max_precise(void) 477 { 478 int precise = 0; 479 480 /* Support for constant skid */ 481 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 482 precise++; 483 484 /* Support for IP fixup */ 485 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 486 precise++; 487 488 if (x86_pmu.pebs_prec_dist) 489 precise++; 490 } 491 return precise; 492 } 493 494 int x86_pmu_hw_config(struct perf_event *event) 495 { 496 if (event->attr.precise_ip) { 497 int precise = x86_pmu_max_precise(); 498 499 if (event->attr.precise_ip > precise) 500 return -EOPNOTSUPP; 501 502 /* There's no sense in having PEBS for non sampling events: */ 503 if (!is_sampling_event(event)) 504 return -EINVAL; 505 } 506 /* 507 * check that PEBS LBR correction does not conflict with 508 * whatever the user is asking with attr->branch_sample_type 509 */ 510 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 511 u64 *br_type = &event->attr.branch_sample_type; 512 513 if (has_branch_stack(event)) { 514 if (!precise_br_compat(event)) 515 return -EOPNOTSUPP; 516 517 /* branch_sample_type is compatible */ 518 519 } else { 520 /* 521 * user did not specify branch_sample_type 522 * 523 * For PEBS fixups, we capture all 524 * the branches at the priv level of the 525 * event. 526 */ 527 *br_type = PERF_SAMPLE_BRANCH_ANY; 528 529 if (!event->attr.exclude_user) 530 *br_type |= PERF_SAMPLE_BRANCH_USER; 531 532 if (!event->attr.exclude_kernel) 533 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 534 } 535 } 536 537 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 538 event->attach_state |= PERF_ATTACH_TASK_DATA; 539 540 /* 541 * Generate PMC IRQs: 542 * (keep 'enabled' bit clear for now) 543 */ 544 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 545 546 /* 547 * Count user and OS events unless requested not to 548 */ 549 if (!event->attr.exclude_user) 550 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 551 if (!event->attr.exclude_kernel) 552 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 553 554 if (event->attr.type == PERF_TYPE_RAW) 555 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 556 557 if (event->attr.sample_period && x86_pmu.limit_period) { 558 if (x86_pmu.limit_period(event, event->attr.sample_period) > 559 event->attr.sample_period) 560 return -EINVAL; 561 } 562 563 /* sample_regs_user never support XMM registers */ 564 if (unlikely(event->attr.sample_regs_user & PEBS_XMM_REGS)) 565 return -EINVAL; 566 /* 567 * Besides the general purpose registers, XMM registers may 568 * be collected in PEBS on some platforms, e.g. Icelake 569 */ 570 if (unlikely(event->attr.sample_regs_intr & PEBS_XMM_REGS)) { 571 if (x86_pmu.pebs_no_xmm_regs) 572 return -EINVAL; 573 574 if (!event->attr.precise_ip) 575 return -EINVAL; 576 } 577 578 return x86_setup_perfctr(event); 579 } 580 581 /* 582 * Setup the hardware configuration for a given attr_type 583 */ 584 static int __x86_pmu_event_init(struct perf_event *event) 585 { 586 int err; 587 588 if (!x86_pmu_initialized()) 589 return -ENODEV; 590 591 err = x86_reserve_hardware(); 592 if (err) 593 return err; 594 595 atomic_inc(&active_events); 596 event->destroy = hw_perf_event_destroy; 597 598 event->hw.idx = -1; 599 event->hw.last_cpu = -1; 600 event->hw.last_tag = ~0ULL; 601 602 /* mark unused */ 603 event->hw.extra_reg.idx = EXTRA_REG_NONE; 604 event->hw.branch_reg.idx = EXTRA_REG_NONE; 605 606 return x86_pmu.hw_config(event); 607 } 608 609 void x86_pmu_disable_all(void) 610 { 611 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 612 int idx; 613 614 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 615 u64 val; 616 617 if (!test_bit(idx, cpuc->active_mask)) 618 continue; 619 rdmsrl(x86_pmu_config_addr(idx), val); 620 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 621 continue; 622 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 623 wrmsrl(x86_pmu_config_addr(idx), val); 624 } 625 } 626 627 /* 628 * There may be PMI landing after enabled=0. The PMI hitting could be before or 629 * after disable_all. 630 * 631 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 632 * It will not be re-enabled in the NMI handler again, because enabled=0. After 633 * handling the NMI, disable_all will be called, which will not change the 634 * state either. If PMI hits after disable_all, the PMU is already disabled 635 * before entering NMI handler. The NMI handler will not change the state 636 * either. 637 * 638 * So either situation is harmless. 639 */ 640 static void x86_pmu_disable(struct pmu *pmu) 641 { 642 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 643 644 if (!x86_pmu_initialized()) 645 return; 646 647 if (!cpuc->enabled) 648 return; 649 650 cpuc->n_added = 0; 651 cpuc->enabled = 0; 652 barrier(); 653 654 x86_pmu.disable_all(); 655 } 656 657 void x86_pmu_enable_all(int added) 658 { 659 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 660 int idx; 661 662 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 663 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 664 665 if (!test_bit(idx, cpuc->active_mask)) 666 continue; 667 668 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 669 } 670 } 671 672 static struct pmu pmu; 673 674 static inline int is_x86_event(struct perf_event *event) 675 { 676 return event->pmu == &pmu; 677 } 678 679 struct pmu *x86_get_pmu(void) 680 { 681 return &pmu; 682 } 683 /* 684 * Event scheduler state: 685 * 686 * Assign events iterating over all events and counters, beginning 687 * with events with least weights first. Keep the current iterator 688 * state in struct sched_state. 689 */ 690 struct sched_state { 691 int weight; 692 int event; /* event index */ 693 int counter; /* counter index */ 694 int unassigned; /* number of events to be assigned left */ 695 int nr_gp; /* number of GP counters used */ 696 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 697 }; 698 699 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 700 #define SCHED_STATES_MAX 2 701 702 struct perf_sched { 703 int max_weight; 704 int max_events; 705 int max_gp; 706 int saved_states; 707 struct event_constraint **constraints; 708 struct sched_state state; 709 struct sched_state saved[SCHED_STATES_MAX]; 710 }; 711 712 /* 713 * Initialize interator that runs through all events and counters. 714 */ 715 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 716 int num, int wmin, int wmax, int gpmax) 717 { 718 int idx; 719 720 memset(sched, 0, sizeof(*sched)); 721 sched->max_events = num; 722 sched->max_weight = wmax; 723 sched->max_gp = gpmax; 724 sched->constraints = constraints; 725 726 for (idx = 0; idx < num; idx++) { 727 if (constraints[idx]->weight == wmin) 728 break; 729 } 730 731 sched->state.event = idx; /* start with min weight */ 732 sched->state.weight = wmin; 733 sched->state.unassigned = num; 734 } 735 736 static void perf_sched_save_state(struct perf_sched *sched) 737 { 738 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 739 return; 740 741 sched->saved[sched->saved_states] = sched->state; 742 sched->saved_states++; 743 } 744 745 static bool perf_sched_restore_state(struct perf_sched *sched) 746 { 747 if (!sched->saved_states) 748 return false; 749 750 sched->saved_states--; 751 sched->state = sched->saved[sched->saved_states]; 752 753 /* continue with next counter: */ 754 clear_bit(sched->state.counter++, sched->state.used); 755 756 return true; 757 } 758 759 /* 760 * Select a counter for the current event to schedule. Return true on 761 * success. 762 */ 763 static bool __perf_sched_find_counter(struct perf_sched *sched) 764 { 765 struct event_constraint *c; 766 int idx; 767 768 if (!sched->state.unassigned) 769 return false; 770 771 if (sched->state.event >= sched->max_events) 772 return false; 773 774 c = sched->constraints[sched->state.event]; 775 /* Prefer fixed purpose counters */ 776 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 777 idx = INTEL_PMC_IDX_FIXED; 778 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 779 if (!__test_and_set_bit(idx, sched->state.used)) 780 goto done; 781 } 782 } 783 784 /* Grab the first unused counter starting with idx */ 785 idx = sched->state.counter; 786 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 787 if (!__test_and_set_bit(idx, sched->state.used)) { 788 if (sched->state.nr_gp++ >= sched->max_gp) 789 return false; 790 791 goto done; 792 } 793 } 794 795 return false; 796 797 done: 798 sched->state.counter = idx; 799 800 if (c->overlap) 801 perf_sched_save_state(sched); 802 803 return true; 804 } 805 806 static bool perf_sched_find_counter(struct perf_sched *sched) 807 { 808 while (!__perf_sched_find_counter(sched)) { 809 if (!perf_sched_restore_state(sched)) 810 return false; 811 } 812 813 return true; 814 } 815 816 /* 817 * Go through all unassigned events and find the next one to schedule. 818 * Take events with the least weight first. Return true on success. 819 */ 820 static bool perf_sched_next_event(struct perf_sched *sched) 821 { 822 struct event_constraint *c; 823 824 if (!sched->state.unassigned || !--sched->state.unassigned) 825 return false; 826 827 do { 828 /* next event */ 829 sched->state.event++; 830 if (sched->state.event >= sched->max_events) { 831 /* next weight */ 832 sched->state.event = 0; 833 sched->state.weight++; 834 if (sched->state.weight > sched->max_weight) 835 return false; 836 } 837 c = sched->constraints[sched->state.event]; 838 } while (c->weight != sched->state.weight); 839 840 sched->state.counter = 0; /* start with first counter */ 841 842 return true; 843 } 844 845 /* 846 * Assign a counter for each event. 847 */ 848 int perf_assign_events(struct event_constraint **constraints, int n, 849 int wmin, int wmax, int gpmax, int *assign) 850 { 851 struct perf_sched sched; 852 853 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 854 855 do { 856 if (!perf_sched_find_counter(&sched)) 857 break; /* failed */ 858 if (assign) 859 assign[sched.state.event] = sched.state.counter; 860 } while (perf_sched_next_event(&sched)); 861 862 return sched.state.unassigned; 863 } 864 EXPORT_SYMBOL_GPL(perf_assign_events); 865 866 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 867 { 868 struct event_constraint *c; 869 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 870 struct perf_event *e; 871 int n0, i, wmin, wmax, unsched = 0; 872 struct hw_perf_event *hwc; 873 874 bitmap_zero(used_mask, X86_PMC_IDX_MAX); 875 876 /* 877 * Compute the number of events already present; see x86_pmu_add(), 878 * validate_group() and x86_pmu_commit_txn(). For the former two 879 * cpuc->n_events hasn't been updated yet, while for the latter 880 * cpuc->n_txn contains the number of events added in the current 881 * transaction. 882 */ 883 n0 = cpuc->n_events; 884 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 885 n0 -= cpuc->n_txn; 886 887 if (x86_pmu.start_scheduling) 888 x86_pmu.start_scheduling(cpuc); 889 890 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 891 c = cpuc->event_constraint[i]; 892 893 /* 894 * Previously scheduled events should have a cached constraint, 895 * while new events should not have one. 896 */ 897 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0)); 898 899 /* 900 * Request constraints for new events; or for those events that 901 * have a dynamic constraint -- for those the constraint can 902 * change due to external factors (sibling state, allow_tfa). 903 */ 904 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) { 905 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]); 906 cpuc->event_constraint[i] = c; 907 } 908 909 wmin = min(wmin, c->weight); 910 wmax = max(wmax, c->weight); 911 } 912 913 /* 914 * fastpath, try to reuse previous register 915 */ 916 for (i = 0; i < n; i++) { 917 hwc = &cpuc->event_list[i]->hw; 918 c = cpuc->event_constraint[i]; 919 920 /* never assigned */ 921 if (hwc->idx == -1) 922 break; 923 924 /* constraint still honored */ 925 if (!test_bit(hwc->idx, c->idxmsk)) 926 break; 927 928 /* not already used */ 929 if (test_bit(hwc->idx, used_mask)) 930 break; 931 932 __set_bit(hwc->idx, used_mask); 933 if (assign) 934 assign[i] = hwc->idx; 935 } 936 937 /* slow path */ 938 if (i != n) { 939 int gpmax = x86_pmu.num_counters; 940 941 /* 942 * Do not allow scheduling of more than half the available 943 * generic counters. 944 * 945 * This helps avoid counter starvation of sibling thread by 946 * ensuring at most half the counters cannot be in exclusive 947 * mode. There is no designated counters for the limits. Any 948 * N/2 counters can be used. This helps with events with 949 * specific counter constraints. 950 */ 951 if (is_ht_workaround_enabled() && !cpuc->is_fake && 952 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 953 gpmax /= 2; 954 955 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 956 wmax, gpmax, assign); 957 } 958 959 /* 960 * In case of success (unsched = 0), mark events as committed, 961 * so we do not put_constraint() in case new events are added 962 * and fail to be scheduled 963 * 964 * We invoke the lower level commit callback to lock the resource 965 * 966 * We do not need to do all of this in case we are called to 967 * validate an event group (assign == NULL) 968 */ 969 if (!unsched && assign) { 970 for (i = 0; i < n; i++) { 971 e = cpuc->event_list[i]; 972 if (x86_pmu.commit_scheduling) 973 x86_pmu.commit_scheduling(cpuc, i, assign[i]); 974 } 975 } else { 976 for (i = n0; i < n; i++) { 977 e = cpuc->event_list[i]; 978 979 /* 980 * release events that failed scheduling 981 */ 982 if (x86_pmu.put_event_constraints) 983 x86_pmu.put_event_constraints(cpuc, e); 984 985 cpuc->event_constraint[i] = NULL; 986 } 987 } 988 989 if (x86_pmu.stop_scheduling) 990 x86_pmu.stop_scheduling(cpuc); 991 992 return unsched ? -EINVAL : 0; 993 } 994 995 /* 996 * dogrp: true if must collect siblings events (group) 997 * returns total number of events and error code 998 */ 999 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 1000 { 1001 struct perf_event *event; 1002 int n, max_count; 1003 1004 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; 1005 1006 /* current number of events already accepted */ 1007 n = cpuc->n_events; 1008 1009 if (is_x86_event(leader)) { 1010 if (n >= max_count) 1011 return -EINVAL; 1012 cpuc->event_list[n] = leader; 1013 n++; 1014 } 1015 if (!dogrp) 1016 return n; 1017 1018 for_each_sibling_event(event, leader) { 1019 if (!is_x86_event(event) || 1020 event->state <= PERF_EVENT_STATE_OFF) 1021 continue; 1022 1023 if (n >= max_count) 1024 return -EINVAL; 1025 1026 cpuc->event_list[n] = event; 1027 n++; 1028 } 1029 return n; 1030 } 1031 1032 static inline void x86_assign_hw_event(struct perf_event *event, 1033 struct cpu_hw_events *cpuc, int i) 1034 { 1035 struct hw_perf_event *hwc = &event->hw; 1036 1037 hwc->idx = cpuc->assign[i]; 1038 hwc->last_cpu = smp_processor_id(); 1039 hwc->last_tag = ++cpuc->tags[i]; 1040 1041 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) { 1042 hwc->config_base = 0; 1043 hwc->event_base = 0; 1044 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) { 1045 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1046 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED); 1047 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30; 1048 } else { 1049 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1050 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1051 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1052 } 1053 } 1054 1055 /** 1056 * x86_perf_rdpmc_index - Return PMC counter used for event 1057 * @event: the perf_event to which the PMC counter was assigned 1058 * 1059 * The counter assigned to this performance event may change if interrupts 1060 * are enabled. This counter should thus never be used while interrupts are 1061 * enabled. Before this function is used to obtain the assigned counter the 1062 * event should be checked for validity using, for example, 1063 * perf_event_read_local(), within the same interrupt disabled section in 1064 * which this counter is planned to be used. 1065 * 1066 * Return: The index of the performance monitoring counter assigned to 1067 * @perf_event. 1068 */ 1069 int x86_perf_rdpmc_index(struct perf_event *event) 1070 { 1071 lockdep_assert_irqs_disabled(); 1072 1073 return event->hw.event_base_rdpmc; 1074 } 1075 1076 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1077 struct cpu_hw_events *cpuc, 1078 int i) 1079 { 1080 return hwc->idx == cpuc->assign[i] && 1081 hwc->last_cpu == smp_processor_id() && 1082 hwc->last_tag == cpuc->tags[i]; 1083 } 1084 1085 static void x86_pmu_start(struct perf_event *event, int flags); 1086 1087 static void x86_pmu_enable(struct pmu *pmu) 1088 { 1089 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1090 struct perf_event *event; 1091 struct hw_perf_event *hwc; 1092 int i, added = cpuc->n_added; 1093 1094 if (!x86_pmu_initialized()) 1095 return; 1096 1097 if (cpuc->enabled) 1098 return; 1099 1100 if (cpuc->n_added) { 1101 int n_running = cpuc->n_events - cpuc->n_added; 1102 /* 1103 * apply assignment obtained either from 1104 * hw_perf_group_sched_in() or x86_pmu_enable() 1105 * 1106 * step1: save events moving to new counters 1107 */ 1108 for (i = 0; i < n_running; i++) { 1109 event = cpuc->event_list[i]; 1110 hwc = &event->hw; 1111 1112 /* 1113 * we can avoid reprogramming counter if: 1114 * - assigned same counter as last time 1115 * - running on same CPU as last time 1116 * - no other event has used the counter since 1117 */ 1118 if (hwc->idx == -1 || 1119 match_prev_assignment(hwc, cpuc, i)) 1120 continue; 1121 1122 /* 1123 * Ensure we don't accidentally enable a stopped 1124 * counter simply because we rescheduled. 1125 */ 1126 if (hwc->state & PERF_HES_STOPPED) 1127 hwc->state |= PERF_HES_ARCH; 1128 1129 x86_pmu_stop(event, PERF_EF_UPDATE); 1130 } 1131 1132 /* 1133 * step2: reprogram moved events into new counters 1134 */ 1135 for (i = 0; i < cpuc->n_events; i++) { 1136 event = cpuc->event_list[i]; 1137 hwc = &event->hw; 1138 1139 if (!match_prev_assignment(hwc, cpuc, i)) 1140 x86_assign_hw_event(event, cpuc, i); 1141 else if (i < n_running) 1142 continue; 1143 1144 if (hwc->state & PERF_HES_ARCH) 1145 continue; 1146 1147 x86_pmu_start(event, PERF_EF_RELOAD); 1148 } 1149 cpuc->n_added = 0; 1150 perf_events_lapic_init(); 1151 } 1152 1153 cpuc->enabled = 1; 1154 barrier(); 1155 1156 x86_pmu.enable_all(added); 1157 } 1158 1159 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1160 1161 /* 1162 * Set the next IRQ period, based on the hwc->period_left value. 1163 * To be called with the event disabled in hw: 1164 */ 1165 int x86_perf_event_set_period(struct perf_event *event) 1166 { 1167 struct hw_perf_event *hwc = &event->hw; 1168 s64 left = local64_read(&hwc->period_left); 1169 s64 period = hwc->sample_period; 1170 int ret = 0, idx = hwc->idx; 1171 1172 if (idx == INTEL_PMC_IDX_FIXED_BTS) 1173 return 0; 1174 1175 /* 1176 * If we are way outside a reasonable range then just skip forward: 1177 */ 1178 if (unlikely(left <= -period)) { 1179 left = period; 1180 local64_set(&hwc->period_left, left); 1181 hwc->last_period = period; 1182 ret = 1; 1183 } 1184 1185 if (unlikely(left <= 0)) { 1186 left += period; 1187 local64_set(&hwc->period_left, left); 1188 hwc->last_period = period; 1189 ret = 1; 1190 } 1191 /* 1192 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1193 */ 1194 if (unlikely(left < 2)) 1195 left = 2; 1196 1197 if (left > x86_pmu.max_period) 1198 left = x86_pmu.max_period; 1199 1200 if (x86_pmu.limit_period) 1201 left = x86_pmu.limit_period(event, left); 1202 1203 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1204 1205 /* 1206 * The hw event starts counting from this event offset, 1207 * mark it to be able to extra future deltas: 1208 */ 1209 local64_set(&hwc->prev_count, (u64)-left); 1210 1211 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1212 1213 /* 1214 * Due to erratum on certan cpu we need 1215 * a second write to be sure the register 1216 * is updated properly 1217 */ 1218 if (x86_pmu.perfctr_second_write) { 1219 wrmsrl(hwc->event_base, 1220 (u64)(-left) & x86_pmu.cntval_mask); 1221 } 1222 1223 perf_event_update_userpage(event); 1224 1225 return ret; 1226 } 1227 1228 void x86_pmu_enable_event(struct perf_event *event) 1229 { 1230 if (__this_cpu_read(cpu_hw_events.enabled)) 1231 __x86_pmu_enable_event(&event->hw, 1232 ARCH_PERFMON_EVENTSEL_ENABLE); 1233 } 1234 1235 /* 1236 * Add a single event to the PMU. 1237 * 1238 * The event is added to the group of enabled events 1239 * but only if it can be scehduled with existing events. 1240 */ 1241 static int x86_pmu_add(struct perf_event *event, int flags) 1242 { 1243 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1244 struct hw_perf_event *hwc; 1245 int assign[X86_PMC_IDX_MAX]; 1246 int n, n0, ret; 1247 1248 hwc = &event->hw; 1249 1250 n0 = cpuc->n_events; 1251 ret = n = collect_events(cpuc, event, false); 1252 if (ret < 0) 1253 goto out; 1254 1255 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1256 if (!(flags & PERF_EF_START)) 1257 hwc->state |= PERF_HES_ARCH; 1258 1259 /* 1260 * If group events scheduling transaction was started, 1261 * skip the schedulability test here, it will be performed 1262 * at commit time (->commit_txn) as a whole. 1263 * 1264 * If commit fails, we'll call ->del() on all events 1265 * for which ->add() was called. 1266 */ 1267 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1268 goto done_collect; 1269 1270 ret = x86_pmu.schedule_events(cpuc, n, assign); 1271 if (ret) 1272 goto out; 1273 /* 1274 * copy new assignment, now we know it is possible 1275 * will be used by hw_perf_enable() 1276 */ 1277 memcpy(cpuc->assign, assign, n*sizeof(int)); 1278 1279 done_collect: 1280 /* 1281 * Commit the collect_events() state. See x86_pmu_del() and 1282 * x86_pmu_*_txn(). 1283 */ 1284 cpuc->n_events = n; 1285 cpuc->n_added += n - n0; 1286 cpuc->n_txn += n - n0; 1287 1288 if (x86_pmu.add) { 1289 /* 1290 * This is before x86_pmu_enable() will call x86_pmu_start(), 1291 * so we enable LBRs before an event needs them etc.. 1292 */ 1293 x86_pmu.add(event); 1294 } 1295 1296 ret = 0; 1297 out: 1298 return ret; 1299 } 1300 1301 static void x86_pmu_start(struct perf_event *event, int flags) 1302 { 1303 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1304 int idx = event->hw.idx; 1305 1306 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1307 return; 1308 1309 if (WARN_ON_ONCE(idx == -1)) 1310 return; 1311 1312 if (flags & PERF_EF_RELOAD) { 1313 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1314 x86_perf_event_set_period(event); 1315 } 1316 1317 event->hw.state = 0; 1318 1319 cpuc->events[idx] = event; 1320 __set_bit(idx, cpuc->active_mask); 1321 __set_bit(idx, cpuc->running); 1322 x86_pmu.enable(event); 1323 perf_event_update_userpage(event); 1324 } 1325 1326 void perf_event_print_debug(void) 1327 { 1328 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1329 u64 pebs, debugctl; 1330 struct cpu_hw_events *cpuc; 1331 unsigned long flags; 1332 int cpu, idx; 1333 1334 if (!x86_pmu.num_counters) 1335 return; 1336 1337 local_irq_save(flags); 1338 1339 cpu = smp_processor_id(); 1340 cpuc = &per_cpu(cpu_hw_events, cpu); 1341 1342 if (x86_pmu.version >= 2) { 1343 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1344 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1345 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1346 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1347 1348 pr_info("\n"); 1349 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1350 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1351 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1352 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1353 if (x86_pmu.pebs_constraints) { 1354 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1355 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1356 } 1357 if (x86_pmu.lbr_nr) { 1358 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1359 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1360 } 1361 } 1362 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1363 1364 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1365 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1366 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1367 1368 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1369 1370 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1371 cpu, idx, pmc_ctrl); 1372 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1373 cpu, idx, pmc_count); 1374 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1375 cpu, idx, prev_left); 1376 } 1377 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { 1378 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1379 1380 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1381 cpu, idx, pmc_count); 1382 } 1383 local_irq_restore(flags); 1384 } 1385 1386 void x86_pmu_stop(struct perf_event *event, int flags) 1387 { 1388 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1389 struct hw_perf_event *hwc = &event->hw; 1390 1391 if (test_bit(hwc->idx, cpuc->active_mask)) { 1392 x86_pmu.disable(event); 1393 __clear_bit(hwc->idx, cpuc->active_mask); 1394 cpuc->events[hwc->idx] = NULL; 1395 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1396 hwc->state |= PERF_HES_STOPPED; 1397 } 1398 1399 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1400 /* 1401 * Drain the remaining delta count out of a event 1402 * that we are disabling: 1403 */ 1404 x86_perf_event_update(event); 1405 hwc->state |= PERF_HES_UPTODATE; 1406 } 1407 } 1408 1409 static void x86_pmu_del(struct perf_event *event, int flags) 1410 { 1411 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1412 int i; 1413 1414 /* 1415 * If we're called during a txn, we only need to undo x86_pmu.add. 1416 * The events never got scheduled and ->cancel_txn will truncate 1417 * the event_list. 1418 * 1419 * XXX assumes any ->del() called during a TXN will only be on 1420 * an event added during that same TXN. 1421 */ 1422 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1423 goto do_del; 1424 1425 /* 1426 * Not a TXN, therefore cleanup properly. 1427 */ 1428 x86_pmu_stop(event, PERF_EF_UPDATE); 1429 1430 for (i = 0; i < cpuc->n_events; i++) { 1431 if (event == cpuc->event_list[i]) 1432 break; 1433 } 1434 1435 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1436 return; 1437 1438 /* If we have a newly added event; make sure to decrease n_added. */ 1439 if (i >= cpuc->n_events - cpuc->n_added) 1440 --cpuc->n_added; 1441 1442 if (x86_pmu.put_event_constraints) 1443 x86_pmu.put_event_constraints(cpuc, event); 1444 1445 /* Delete the array entry. */ 1446 while (++i < cpuc->n_events) { 1447 cpuc->event_list[i-1] = cpuc->event_list[i]; 1448 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1449 } 1450 cpuc->event_constraint[i-1] = NULL; 1451 --cpuc->n_events; 1452 1453 perf_event_update_userpage(event); 1454 1455 do_del: 1456 if (x86_pmu.del) { 1457 /* 1458 * This is after x86_pmu_stop(); so we disable LBRs after any 1459 * event can need them etc.. 1460 */ 1461 x86_pmu.del(event); 1462 } 1463 } 1464 1465 int x86_pmu_handle_irq(struct pt_regs *regs) 1466 { 1467 struct perf_sample_data data; 1468 struct cpu_hw_events *cpuc; 1469 struct perf_event *event; 1470 int idx, handled = 0; 1471 u64 val; 1472 1473 cpuc = this_cpu_ptr(&cpu_hw_events); 1474 1475 /* 1476 * Some chipsets need to unmask the LVTPC in a particular spot 1477 * inside the nmi handler. As a result, the unmasking was pushed 1478 * into all the nmi handlers. 1479 * 1480 * This generic handler doesn't seem to have any issues where the 1481 * unmasking occurs so it was left at the top. 1482 */ 1483 apic_write(APIC_LVTPC, APIC_DM_NMI); 1484 1485 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1486 if (!test_bit(idx, cpuc->active_mask)) 1487 continue; 1488 1489 event = cpuc->events[idx]; 1490 1491 val = x86_perf_event_update(event); 1492 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1493 continue; 1494 1495 /* 1496 * event overflow 1497 */ 1498 handled++; 1499 perf_sample_data_init(&data, 0, event->hw.last_period); 1500 1501 if (!x86_perf_event_set_period(event)) 1502 continue; 1503 1504 if (perf_event_overflow(event, &data, regs)) 1505 x86_pmu_stop(event, 0); 1506 } 1507 1508 if (handled) 1509 inc_irq_stat(apic_perf_irqs); 1510 1511 return handled; 1512 } 1513 1514 void perf_events_lapic_init(void) 1515 { 1516 if (!x86_pmu.apic || !x86_pmu_initialized()) 1517 return; 1518 1519 /* 1520 * Always use NMI for PMU 1521 */ 1522 apic_write(APIC_LVTPC, APIC_DM_NMI); 1523 } 1524 1525 static int 1526 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1527 { 1528 u64 start_clock; 1529 u64 finish_clock; 1530 int ret; 1531 1532 /* 1533 * All PMUs/events that share this PMI handler should make sure to 1534 * increment active_events for their events. 1535 */ 1536 if (!atomic_read(&active_events)) 1537 return NMI_DONE; 1538 1539 start_clock = sched_clock(); 1540 ret = x86_pmu.handle_irq(regs); 1541 finish_clock = sched_clock(); 1542 1543 perf_sample_event_took(finish_clock - start_clock); 1544 1545 return ret; 1546 } 1547 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1548 1549 struct event_constraint emptyconstraint; 1550 struct event_constraint unconstrained; 1551 1552 static int x86_pmu_prepare_cpu(unsigned int cpu) 1553 { 1554 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1555 int i; 1556 1557 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1558 cpuc->kfree_on_online[i] = NULL; 1559 if (x86_pmu.cpu_prepare) 1560 return x86_pmu.cpu_prepare(cpu); 1561 return 0; 1562 } 1563 1564 static int x86_pmu_dead_cpu(unsigned int cpu) 1565 { 1566 if (x86_pmu.cpu_dead) 1567 x86_pmu.cpu_dead(cpu); 1568 return 0; 1569 } 1570 1571 static int x86_pmu_online_cpu(unsigned int cpu) 1572 { 1573 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1574 int i; 1575 1576 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1577 kfree(cpuc->kfree_on_online[i]); 1578 cpuc->kfree_on_online[i] = NULL; 1579 } 1580 return 0; 1581 } 1582 1583 static int x86_pmu_starting_cpu(unsigned int cpu) 1584 { 1585 if (x86_pmu.cpu_starting) 1586 x86_pmu.cpu_starting(cpu); 1587 return 0; 1588 } 1589 1590 static int x86_pmu_dying_cpu(unsigned int cpu) 1591 { 1592 if (x86_pmu.cpu_dying) 1593 x86_pmu.cpu_dying(cpu); 1594 return 0; 1595 } 1596 1597 static void __init pmu_check_apic(void) 1598 { 1599 if (boot_cpu_has(X86_FEATURE_APIC)) 1600 return; 1601 1602 x86_pmu.apic = 0; 1603 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1604 pr_info("no hardware sampling interrupt available.\n"); 1605 1606 /* 1607 * If we have a PMU initialized but no APIC 1608 * interrupts, we cannot sample hardware 1609 * events (user-space has to fall back and 1610 * sample via a hrtimer based software event): 1611 */ 1612 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1613 1614 } 1615 1616 static struct attribute_group x86_pmu_format_group __ro_after_init = { 1617 .name = "format", 1618 .attrs = NULL, 1619 }; 1620 1621 /* 1622 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1623 * out of events_attr attributes. 1624 */ 1625 static void __init filter_events(struct attribute **attrs) 1626 { 1627 struct device_attribute *d; 1628 struct perf_pmu_events_attr *pmu_attr; 1629 int offset = 0; 1630 int i, j; 1631 1632 for (i = 0; attrs[i]; i++) { 1633 d = (struct device_attribute *)attrs[i]; 1634 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr); 1635 /* str trumps id */ 1636 if (pmu_attr->event_str) 1637 continue; 1638 if (x86_pmu.event_map(i + offset)) 1639 continue; 1640 1641 for (j = i; attrs[j]; j++) 1642 attrs[j] = attrs[j + 1]; 1643 1644 /* Check the shifted attr. */ 1645 i--; 1646 1647 /* 1648 * event_map() is index based, the attrs array is organized 1649 * by increasing event index. If we shift the events, then 1650 * we need to compensate for the event_map(), otherwise 1651 * we are looking up the wrong event in the map 1652 */ 1653 offset++; 1654 } 1655 } 1656 1657 /* Merge two pointer arrays */ 1658 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b) 1659 { 1660 struct attribute **new; 1661 int j, i; 1662 1663 for (j = 0; a && a[j]; j++) 1664 ; 1665 for (i = 0; b && b[i]; i++) 1666 j++; 1667 j++; 1668 1669 new = kmalloc_array(j, sizeof(struct attribute *), GFP_KERNEL); 1670 if (!new) 1671 return NULL; 1672 1673 j = 0; 1674 for (i = 0; a && a[i]; i++) 1675 new[j++] = a[i]; 1676 for (i = 0; b && b[i]; i++) 1677 new[j++] = b[i]; 1678 new[j] = NULL; 1679 1680 return new; 1681 } 1682 1683 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1684 { 1685 struct perf_pmu_events_attr *pmu_attr = \ 1686 container_of(attr, struct perf_pmu_events_attr, attr); 1687 u64 config = x86_pmu.event_map(pmu_attr->id); 1688 1689 /* string trumps id */ 1690 if (pmu_attr->event_str) 1691 return sprintf(page, "%s", pmu_attr->event_str); 1692 1693 return x86_pmu.events_sysfs_show(page, config); 1694 } 1695 EXPORT_SYMBOL_GPL(events_sysfs_show); 1696 1697 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1698 char *page) 1699 { 1700 struct perf_pmu_events_ht_attr *pmu_attr = 1701 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1702 1703 /* 1704 * Report conditional events depending on Hyper-Threading. 1705 * 1706 * This is overly conservative as usually the HT special 1707 * handling is not needed if the other CPU thread is idle. 1708 * 1709 * Note this does not (and cannot) handle the case when thread 1710 * siblings are invisible, for example with virtualization 1711 * if they are owned by some other guest. The user tool 1712 * has to re-read when a thread sibling gets onlined later. 1713 */ 1714 return sprintf(page, "%s", 1715 topology_max_smt_threads() > 1 ? 1716 pmu_attr->event_str_ht : 1717 pmu_attr->event_str_noht); 1718 } 1719 1720 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1721 EVENT_ATTR(instructions, INSTRUCTIONS ); 1722 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1723 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1724 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1725 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1726 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1727 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1728 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1729 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1730 1731 static struct attribute *empty_attrs; 1732 1733 static struct attribute *events_attr[] = { 1734 EVENT_PTR(CPU_CYCLES), 1735 EVENT_PTR(INSTRUCTIONS), 1736 EVENT_PTR(CACHE_REFERENCES), 1737 EVENT_PTR(CACHE_MISSES), 1738 EVENT_PTR(BRANCH_INSTRUCTIONS), 1739 EVENT_PTR(BRANCH_MISSES), 1740 EVENT_PTR(BUS_CYCLES), 1741 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1742 EVENT_PTR(STALLED_CYCLES_BACKEND), 1743 EVENT_PTR(REF_CPU_CYCLES), 1744 NULL, 1745 }; 1746 1747 static struct attribute_group x86_pmu_events_group __ro_after_init = { 1748 .name = "events", 1749 .attrs = events_attr, 1750 }; 1751 1752 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1753 { 1754 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1755 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1756 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1757 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1758 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1759 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1760 ssize_t ret; 1761 1762 /* 1763 * We have whole page size to spend and just little data 1764 * to write, so we can safely use sprintf. 1765 */ 1766 ret = sprintf(page, "event=0x%02llx", event); 1767 1768 if (umask) 1769 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1770 1771 if (edge) 1772 ret += sprintf(page + ret, ",edge"); 1773 1774 if (pc) 1775 ret += sprintf(page + ret, ",pc"); 1776 1777 if (any) 1778 ret += sprintf(page + ret, ",any"); 1779 1780 if (inv) 1781 ret += sprintf(page + ret, ",inv"); 1782 1783 if (cmask) 1784 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1785 1786 ret += sprintf(page + ret, "\n"); 1787 1788 return ret; 1789 } 1790 1791 static struct attribute_group x86_pmu_attr_group; 1792 static struct attribute_group x86_pmu_caps_group; 1793 1794 static int __init init_hw_perf_events(void) 1795 { 1796 struct x86_pmu_quirk *quirk; 1797 int err; 1798 1799 pr_info("Performance Events: "); 1800 1801 switch (boot_cpu_data.x86_vendor) { 1802 case X86_VENDOR_INTEL: 1803 err = intel_pmu_init(); 1804 break; 1805 case X86_VENDOR_AMD: 1806 err = amd_pmu_init(); 1807 break; 1808 case X86_VENDOR_HYGON: 1809 err = amd_pmu_init(); 1810 x86_pmu.name = "HYGON"; 1811 break; 1812 default: 1813 err = -ENOTSUPP; 1814 } 1815 if (err != 0) { 1816 pr_cont("no PMU driver, software events only.\n"); 1817 return 0; 1818 } 1819 1820 pmu_check_apic(); 1821 1822 /* sanity check that the hardware exists or is emulated */ 1823 if (!check_hw_exists()) 1824 return 0; 1825 1826 pr_cont("%s PMU driver.\n", x86_pmu.name); 1827 1828 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1829 1830 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1831 quirk->func(); 1832 1833 if (!x86_pmu.intel_ctrl) 1834 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 1835 1836 perf_events_lapic_init(); 1837 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 1838 1839 unconstrained = (struct event_constraint) 1840 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1841 0, x86_pmu.num_counters, 0, 0); 1842 1843 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1844 1845 if (x86_pmu.caps_attrs) { 1846 struct attribute **tmp; 1847 1848 tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs); 1849 if (!WARN_ON(!tmp)) 1850 x86_pmu_caps_group.attrs = tmp; 1851 } 1852 1853 if (x86_pmu.event_attrs) 1854 x86_pmu_events_group.attrs = x86_pmu.event_attrs; 1855 1856 if (!x86_pmu.events_sysfs_show) 1857 x86_pmu_events_group.attrs = &empty_attrs; 1858 else 1859 filter_events(x86_pmu_events_group.attrs); 1860 1861 if (x86_pmu.cpu_events) { 1862 struct attribute **tmp; 1863 1864 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events); 1865 if (!WARN_ON(!tmp)) 1866 x86_pmu_events_group.attrs = tmp; 1867 } 1868 1869 if (x86_pmu.attrs) { 1870 struct attribute **tmp; 1871 1872 tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs); 1873 if (!WARN_ON(!tmp)) 1874 x86_pmu_attr_group.attrs = tmp; 1875 } 1876 1877 pr_info("... version: %d\n", x86_pmu.version); 1878 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 1879 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 1880 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 1881 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 1882 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed); 1883 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); 1884 1885 /* 1886 * Install callbacks. Core will call them for each online 1887 * cpu. 1888 */ 1889 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 1890 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 1891 if (err) 1892 return err; 1893 1894 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 1895 "perf/x86:starting", x86_pmu_starting_cpu, 1896 x86_pmu_dying_cpu); 1897 if (err) 1898 goto out; 1899 1900 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 1901 x86_pmu_online_cpu, NULL); 1902 if (err) 1903 goto out1; 1904 1905 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 1906 if (err) 1907 goto out2; 1908 1909 return 0; 1910 1911 out2: 1912 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 1913 out1: 1914 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 1915 out: 1916 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 1917 return err; 1918 } 1919 early_initcall(init_hw_perf_events); 1920 1921 static inline void x86_pmu_read(struct perf_event *event) 1922 { 1923 if (x86_pmu.read) 1924 return x86_pmu.read(event); 1925 x86_perf_event_update(event); 1926 } 1927 1928 /* 1929 * Start group events scheduling transaction 1930 * Set the flag to make pmu::enable() not perform the 1931 * schedulability test, it will be performed at commit time 1932 * 1933 * We only support PERF_PMU_TXN_ADD transactions. Save the 1934 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 1935 * transactions. 1936 */ 1937 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 1938 { 1939 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1940 1941 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 1942 1943 cpuc->txn_flags = txn_flags; 1944 if (txn_flags & ~PERF_PMU_TXN_ADD) 1945 return; 1946 1947 perf_pmu_disable(pmu); 1948 __this_cpu_write(cpu_hw_events.n_txn, 0); 1949 } 1950 1951 /* 1952 * Stop group events scheduling transaction 1953 * Clear the flag and pmu::enable() will perform the 1954 * schedulability test. 1955 */ 1956 static void x86_pmu_cancel_txn(struct pmu *pmu) 1957 { 1958 unsigned int txn_flags; 1959 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1960 1961 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1962 1963 txn_flags = cpuc->txn_flags; 1964 cpuc->txn_flags = 0; 1965 if (txn_flags & ~PERF_PMU_TXN_ADD) 1966 return; 1967 1968 /* 1969 * Truncate collected array by the number of events added in this 1970 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 1971 */ 1972 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 1973 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 1974 perf_pmu_enable(pmu); 1975 } 1976 1977 /* 1978 * Commit group events scheduling transaction 1979 * Perform the group schedulability test as a whole 1980 * Return 0 if success 1981 * 1982 * Does not cancel the transaction on failure; expects the caller to do this. 1983 */ 1984 static int x86_pmu_commit_txn(struct pmu *pmu) 1985 { 1986 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1987 int assign[X86_PMC_IDX_MAX]; 1988 int n, ret; 1989 1990 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1991 1992 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 1993 cpuc->txn_flags = 0; 1994 return 0; 1995 } 1996 1997 n = cpuc->n_events; 1998 1999 if (!x86_pmu_initialized()) 2000 return -EAGAIN; 2001 2002 ret = x86_pmu.schedule_events(cpuc, n, assign); 2003 if (ret) 2004 return ret; 2005 2006 /* 2007 * copy new assignment, now we know it is possible 2008 * will be used by hw_perf_enable() 2009 */ 2010 memcpy(cpuc->assign, assign, n*sizeof(int)); 2011 2012 cpuc->txn_flags = 0; 2013 perf_pmu_enable(pmu); 2014 return 0; 2015 } 2016 /* 2017 * a fake_cpuc is used to validate event groups. Due to 2018 * the extra reg logic, we need to also allocate a fake 2019 * per_core and per_cpu structure. Otherwise, group events 2020 * using extra reg may conflict without the kernel being 2021 * able to catch this when the last event gets added to 2022 * the group. 2023 */ 2024 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 2025 { 2026 intel_cpuc_finish(cpuc); 2027 kfree(cpuc); 2028 } 2029 2030 static struct cpu_hw_events *allocate_fake_cpuc(void) 2031 { 2032 struct cpu_hw_events *cpuc; 2033 int cpu = raw_smp_processor_id(); 2034 2035 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 2036 if (!cpuc) 2037 return ERR_PTR(-ENOMEM); 2038 cpuc->is_fake = 1; 2039 2040 if (intel_cpuc_prepare(cpuc, cpu)) 2041 goto error; 2042 2043 return cpuc; 2044 error: 2045 free_fake_cpuc(cpuc); 2046 return ERR_PTR(-ENOMEM); 2047 } 2048 2049 /* 2050 * validate that we can schedule this event 2051 */ 2052 static int validate_event(struct perf_event *event) 2053 { 2054 struct cpu_hw_events *fake_cpuc; 2055 struct event_constraint *c; 2056 int ret = 0; 2057 2058 fake_cpuc = allocate_fake_cpuc(); 2059 if (IS_ERR(fake_cpuc)) 2060 return PTR_ERR(fake_cpuc); 2061 2062 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event); 2063 2064 if (!c || !c->weight) 2065 ret = -EINVAL; 2066 2067 if (x86_pmu.put_event_constraints) 2068 x86_pmu.put_event_constraints(fake_cpuc, event); 2069 2070 free_fake_cpuc(fake_cpuc); 2071 2072 return ret; 2073 } 2074 2075 /* 2076 * validate a single event group 2077 * 2078 * validation include: 2079 * - check events are compatible which each other 2080 * - events do not compete for the same counter 2081 * - number of events <= number of counters 2082 * 2083 * validation ensures the group can be loaded onto the 2084 * PMU if it was the only group available. 2085 */ 2086 static int validate_group(struct perf_event *event) 2087 { 2088 struct perf_event *leader = event->group_leader; 2089 struct cpu_hw_events *fake_cpuc; 2090 int ret = -EINVAL, n; 2091 2092 fake_cpuc = allocate_fake_cpuc(); 2093 if (IS_ERR(fake_cpuc)) 2094 return PTR_ERR(fake_cpuc); 2095 /* 2096 * the event is not yet connected with its 2097 * siblings therefore we must first collect 2098 * existing siblings, then add the new event 2099 * before we can simulate the scheduling 2100 */ 2101 n = collect_events(fake_cpuc, leader, true); 2102 if (n < 0) 2103 goto out; 2104 2105 fake_cpuc->n_events = n; 2106 n = collect_events(fake_cpuc, event, false); 2107 if (n < 0) 2108 goto out; 2109 2110 fake_cpuc->n_events = 0; 2111 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2112 2113 out: 2114 free_fake_cpuc(fake_cpuc); 2115 return ret; 2116 } 2117 2118 static int x86_pmu_event_init(struct perf_event *event) 2119 { 2120 struct pmu *tmp; 2121 int err; 2122 2123 switch (event->attr.type) { 2124 case PERF_TYPE_RAW: 2125 case PERF_TYPE_HARDWARE: 2126 case PERF_TYPE_HW_CACHE: 2127 break; 2128 2129 default: 2130 return -ENOENT; 2131 } 2132 2133 err = __x86_pmu_event_init(event); 2134 if (!err) { 2135 /* 2136 * we temporarily connect event to its pmu 2137 * such that validate_group() can classify 2138 * it as an x86 event using is_x86_event() 2139 */ 2140 tmp = event->pmu; 2141 event->pmu = &pmu; 2142 2143 if (event->group_leader != event) 2144 err = validate_group(event); 2145 else 2146 err = validate_event(event); 2147 2148 event->pmu = tmp; 2149 } 2150 if (err) { 2151 if (event->destroy) 2152 event->destroy(event); 2153 } 2154 2155 if (READ_ONCE(x86_pmu.attr_rdpmc) && 2156 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) 2157 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2158 2159 return err; 2160 } 2161 2162 static void refresh_pce(void *ignored) 2163 { 2164 load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm)); 2165 } 2166 2167 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 2168 { 2169 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2170 return; 2171 2172 /* 2173 * This function relies on not being called concurrently in two 2174 * tasks in the same mm. Otherwise one task could observe 2175 * perf_rdpmc_allowed > 1 and return all the way back to 2176 * userspace with CR4.PCE clear while another task is still 2177 * doing on_each_cpu_mask() to propagate CR4.PCE. 2178 * 2179 * For now, this can't happen because all callers hold mmap_sem 2180 * for write. If this changes, we'll need a different solution. 2181 */ 2182 lockdep_assert_held_exclusive(&mm->mmap_sem); 2183 2184 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 2185 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2186 } 2187 2188 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 2189 { 2190 2191 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2192 return; 2193 2194 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 2195 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2196 } 2197 2198 static int x86_pmu_event_idx(struct perf_event *event) 2199 { 2200 int idx = event->hw.idx; 2201 2202 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2203 return 0; 2204 2205 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) { 2206 idx -= INTEL_PMC_IDX_FIXED; 2207 idx |= 1 << 30; 2208 } 2209 2210 return idx + 1; 2211 } 2212 2213 static ssize_t get_attr_rdpmc(struct device *cdev, 2214 struct device_attribute *attr, 2215 char *buf) 2216 { 2217 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2218 } 2219 2220 static ssize_t set_attr_rdpmc(struct device *cdev, 2221 struct device_attribute *attr, 2222 const char *buf, size_t count) 2223 { 2224 unsigned long val; 2225 ssize_t ret; 2226 2227 ret = kstrtoul(buf, 0, &val); 2228 if (ret) 2229 return ret; 2230 2231 if (val > 2) 2232 return -EINVAL; 2233 2234 if (x86_pmu.attr_rdpmc_broken) 2235 return -ENOTSUPP; 2236 2237 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) { 2238 /* 2239 * Changing into or out of always available, aka 2240 * perf-event-bypassing mode. This path is extremely slow, 2241 * but only root can trigger it, so it's okay. 2242 */ 2243 if (val == 2) 2244 static_branch_inc(&rdpmc_always_available_key); 2245 else 2246 static_branch_dec(&rdpmc_always_available_key); 2247 on_each_cpu(refresh_pce, NULL, 1); 2248 } 2249 2250 x86_pmu.attr_rdpmc = val; 2251 2252 return count; 2253 } 2254 2255 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2256 2257 static struct attribute *x86_pmu_attrs[] = { 2258 &dev_attr_rdpmc.attr, 2259 NULL, 2260 }; 2261 2262 static struct attribute_group x86_pmu_attr_group __ro_after_init = { 2263 .attrs = x86_pmu_attrs, 2264 }; 2265 2266 static ssize_t max_precise_show(struct device *cdev, 2267 struct device_attribute *attr, 2268 char *buf) 2269 { 2270 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise()); 2271 } 2272 2273 static DEVICE_ATTR_RO(max_precise); 2274 2275 static struct attribute *x86_pmu_caps_attrs[] = { 2276 &dev_attr_max_precise.attr, 2277 NULL 2278 }; 2279 2280 static struct attribute_group x86_pmu_caps_group __ro_after_init = { 2281 .name = "caps", 2282 .attrs = x86_pmu_caps_attrs, 2283 }; 2284 2285 static const struct attribute_group *x86_pmu_attr_groups[] = { 2286 &x86_pmu_attr_group, 2287 &x86_pmu_format_group, 2288 &x86_pmu_events_group, 2289 &x86_pmu_caps_group, 2290 NULL, 2291 }; 2292 2293 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2294 { 2295 if (x86_pmu.sched_task) 2296 x86_pmu.sched_task(ctx, sched_in); 2297 } 2298 2299 void perf_check_microcode(void) 2300 { 2301 if (x86_pmu.check_microcode) 2302 x86_pmu.check_microcode(); 2303 } 2304 2305 static int x86_pmu_check_period(struct perf_event *event, u64 value) 2306 { 2307 if (x86_pmu.check_period && x86_pmu.check_period(event, value)) 2308 return -EINVAL; 2309 2310 if (value && x86_pmu.limit_period) { 2311 if (x86_pmu.limit_period(event, value) > value) 2312 return -EINVAL; 2313 } 2314 2315 return 0; 2316 } 2317 2318 static struct pmu pmu = { 2319 .pmu_enable = x86_pmu_enable, 2320 .pmu_disable = x86_pmu_disable, 2321 2322 .attr_groups = x86_pmu_attr_groups, 2323 2324 .event_init = x86_pmu_event_init, 2325 2326 .event_mapped = x86_pmu_event_mapped, 2327 .event_unmapped = x86_pmu_event_unmapped, 2328 2329 .add = x86_pmu_add, 2330 .del = x86_pmu_del, 2331 .start = x86_pmu_start, 2332 .stop = x86_pmu_stop, 2333 .read = x86_pmu_read, 2334 2335 .start_txn = x86_pmu_start_txn, 2336 .cancel_txn = x86_pmu_cancel_txn, 2337 .commit_txn = x86_pmu_commit_txn, 2338 2339 .event_idx = x86_pmu_event_idx, 2340 .sched_task = x86_pmu_sched_task, 2341 .task_ctx_size = sizeof(struct x86_perf_task_context), 2342 .check_period = x86_pmu_check_period, 2343 }; 2344 2345 void arch_perf_update_userpage(struct perf_event *event, 2346 struct perf_event_mmap_page *userpg, u64 now) 2347 { 2348 struct cyc2ns_data data; 2349 u64 offset; 2350 2351 userpg->cap_user_time = 0; 2352 userpg->cap_user_time_zero = 0; 2353 userpg->cap_user_rdpmc = 2354 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2355 userpg->pmc_width = x86_pmu.cntval_bits; 2356 2357 if (!using_native_sched_clock() || !sched_clock_stable()) 2358 return; 2359 2360 cyc2ns_read_begin(&data); 2361 2362 offset = data.cyc2ns_offset + __sched_clock_offset; 2363 2364 /* 2365 * Internal timekeeping for enabled/running/stopped times 2366 * is always in the local_clock domain. 2367 */ 2368 userpg->cap_user_time = 1; 2369 userpg->time_mult = data.cyc2ns_mul; 2370 userpg->time_shift = data.cyc2ns_shift; 2371 userpg->time_offset = offset - now; 2372 2373 /* 2374 * cap_user_time_zero doesn't make sense when we're using a different 2375 * time base for the records. 2376 */ 2377 if (!event->attr.use_clockid) { 2378 userpg->cap_user_time_zero = 1; 2379 userpg->time_zero = offset; 2380 } 2381 2382 cyc2ns_read_end(); 2383 } 2384 2385 /* 2386 * Determine whether the regs were taken from an irq/exception handler rather 2387 * than from perf_arch_fetch_caller_regs(). 2388 */ 2389 static bool perf_hw_regs(struct pt_regs *regs) 2390 { 2391 return regs->flags & X86_EFLAGS_FIXED; 2392 } 2393 2394 void 2395 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2396 { 2397 struct unwind_state state; 2398 unsigned long addr; 2399 2400 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2401 /* TODO: We don't support guest os callchain now */ 2402 return; 2403 } 2404 2405 if (perf_hw_regs(regs)) { 2406 if (perf_callchain_store(entry, regs->ip)) 2407 return; 2408 unwind_start(&state, current, regs, NULL); 2409 } else { 2410 unwind_start(&state, current, NULL, (void *)regs->sp); 2411 } 2412 2413 for (; !unwind_done(&state); unwind_next_frame(&state)) { 2414 addr = unwind_get_return_address(&state); 2415 if (!addr || perf_callchain_store(entry, addr)) 2416 return; 2417 } 2418 } 2419 2420 static inline int 2421 valid_user_frame(const void __user *fp, unsigned long size) 2422 { 2423 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2424 } 2425 2426 static unsigned long get_segment_base(unsigned int segment) 2427 { 2428 struct desc_struct *desc; 2429 unsigned int idx = segment >> 3; 2430 2431 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2432 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2433 struct ldt_struct *ldt; 2434 2435 /* IRQs are off, so this synchronizes with smp_store_release */ 2436 ldt = READ_ONCE(current->active_mm->context.ldt); 2437 if (!ldt || idx >= ldt->nr_entries) 2438 return 0; 2439 2440 desc = &ldt->entries[idx]; 2441 #else 2442 return 0; 2443 #endif 2444 } else { 2445 if (idx >= GDT_ENTRIES) 2446 return 0; 2447 2448 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2449 } 2450 2451 return get_desc_base(desc); 2452 } 2453 2454 #ifdef CONFIG_IA32_EMULATION 2455 2456 #include <linux/compat.h> 2457 2458 static inline int 2459 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2460 { 2461 /* 32-bit process in 64-bit kernel. */ 2462 unsigned long ss_base, cs_base; 2463 struct stack_frame_ia32 frame; 2464 const void __user *fp; 2465 2466 if (!test_thread_flag(TIF_IA32)) 2467 return 0; 2468 2469 cs_base = get_segment_base(regs->cs); 2470 ss_base = get_segment_base(regs->ss); 2471 2472 fp = compat_ptr(ss_base + regs->bp); 2473 pagefault_disable(); 2474 while (entry->nr < entry->max_stack) { 2475 unsigned long bytes; 2476 frame.next_frame = 0; 2477 frame.return_address = 0; 2478 2479 if (!valid_user_frame(fp, sizeof(frame))) 2480 break; 2481 2482 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4); 2483 if (bytes != 0) 2484 break; 2485 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4); 2486 if (bytes != 0) 2487 break; 2488 2489 perf_callchain_store(entry, cs_base + frame.return_address); 2490 fp = compat_ptr(ss_base + frame.next_frame); 2491 } 2492 pagefault_enable(); 2493 return 1; 2494 } 2495 #else 2496 static inline int 2497 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2498 { 2499 return 0; 2500 } 2501 #endif 2502 2503 void 2504 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2505 { 2506 struct stack_frame frame; 2507 const unsigned long __user *fp; 2508 2509 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2510 /* TODO: We don't support guest os callchain now */ 2511 return; 2512 } 2513 2514 /* 2515 * We don't know what to do with VM86 stacks.. ignore them for now. 2516 */ 2517 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2518 return; 2519 2520 fp = (unsigned long __user *)regs->bp; 2521 2522 perf_callchain_store(entry, regs->ip); 2523 2524 if (!nmi_uaccess_okay()) 2525 return; 2526 2527 if (perf_callchain_user32(regs, entry)) 2528 return; 2529 2530 pagefault_disable(); 2531 while (entry->nr < entry->max_stack) { 2532 unsigned long bytes; 2533 2534 frame.next_frame = NULL; 2535 frame.return_address = 0; 2536 2537 if (!valid_user_frame(fp, sizeof(frame))) 2538 break; 2539 2540 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); 2541 if (bytes != 0) 2542 break; 2543 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); 2544 if (bytes != 0) 2545 break; 2546 2547 perf_callchain_store(entry, frame.return_address); 2548 fp = (void __user *)frame.next_frame; 2549 } 2550 pagefault_enable(); 2551 } 2552 2553 /* 2554 * Deal with code segment offsets for the various execution modes: 2555 * 2556 * VM86 - the good olde 16 bit days, where the linear address is 2557 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2558 * 2559 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2560 * to figure out what the 32bit base address is. 2561 * 2562 * X32 - has TIF_X32 set, but is running in x86_64 2563 * 2564 * X86_64 - CS,DS,SS,ES are all zero based. 2565 */ 2566 static unsigned long code_segment_base(struct pt_regs *regs) 2567 { 2568 /* 2569 * For IA32 we look at the GDT/LDT segment base to convert the 2570 * effective IP to a linear address. 2571 */ 2572 2573 #ifdef CONFIG_X86_32 2574 /* 2575 * If we are in VM86 mode, add the segment offset to convert to a 2576 * linear address. 2577 */ 2578 if (regs->flags & X86_VM_MASK) 2579 return 0x10 * regs->cs; 2580 2581 if (user_mode(regs) && regs->cs != __USER_CS) 2582 return get_segment_base(regs->cs); 2583 #else 2584 if (user_mode(regs) && !user_64bit_mode(regs) && 2585 regs->cs != __USER32_CS) 2586 return get_segment_base(regs->cs); 2587 #endif 2588 return 0; 2589 } 2590 2591 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2592 { 2593 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2594 return perf_guest_cbs->get_guest_ip(); 2595 2596 return regs->ip + code_segment_base(regs); 2597 } 2598 2599 unsigned long perf_misc_flags(struct pt_regs *regs) 2600 { 2601 int misc = 0; 2602 2603 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2604 if (perf_guest_cbs->is_user_mode()) 2605 misc |= PERF_RECORD_MISC_GUEST_USER; 2606 else 2607 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2608 } else { 2609 if (user_mode(regs)) 2610 misc |= PERF_RECORD_MISC_USER; 2611 else 2612 misc |= PERF_RECORD_MISC_KERNEL; 2613 } 2614 2615 if (regs->flags & PERF_EFLAGS_EXACT) 2616 misc |= PERF_RECORD_MISC_EXACT_IP; 2617 2618 return misc; 2619 } 2620 2621 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2622 { 2623 cap->version = x86_pmu.version; 2624 cap->num_counters_gp = x86_pmu.num_counters; 2625 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2626 cap->bit_width_gp = x86_pmu.cntval_bits; 2627 cap->bit_width_fixed = x86_pmu.cntval_bits; 2628 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2629 cap->events_mask_len = x86_pmu.events_mask_len; 2630 } 2631 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2632